diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index 001e3a84d8..e938fcc5b3 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -14,7 +14,7 @@ inputs: cuda-components: description: "A list of the CTK components to install as a comma-separated list. e.g. 'cuda_nvcc,cuda_nvrtc,cuda_cudart'" required: false - default: "cuda_nvcc,cuda_cudart,cuda_crt,libnvvm,cuda_nvrtc,cuda_profiler_api,cuda_cccl,libnvjitlink,libcufile,libnvfatbin" + default: "cuda_nvcc,cuda_cudart,cuda_crt,libnvvm,cuda_nvrtc,cuda_profiler_api,cuda_cccl,cuda_cupti,libnvjitlink,libcufile,libnvfatbin" cuda-path: description: "where the CTK components will be installed to, relative to $PWD" required: false diff --git a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/descriptor_catalog.py b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/descriptor_catalog.py index e189bb127a..89fa07445d 100644 --- a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/descriptor_catalog.py +++ b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/descriptor_catalog.py @@ -266,6 +266,29 @@ class DescriptorSpec: linux_sonames=("libcufile.so.0",), site_packages_linux=("nvidia/cu13/lib", "nvidia/cufile/lib"), ), + DescriptorSpec( + name="cupti", + packaged_with="ctk", + linux_sonames=("libcupti.so.12", "libcupti.so.13"), + windows_dlls=( + "cupti64_2025.4.1.dll", + "cupti64_2025.3.1.dll", + "cupti64_2025.2.1.dll", + "cupti64_2025.1.1.dll", + "cupti64_2024.3.2.dll", + "cupti64_2024.2.1.dll", + "cupti64_2024.1.1.dll", + "cupti64_2023.3.1.dll", + "cupti64_2023.2.2.dll", + "cupti64_2023.1.1.dll", + "cupti64_2022.4.1.dll", + ), + site_packages_linux=("nvidia/cu13/lib", "nvidia/cuda_cupti/lib"), + site_packages_windows=("nvidia/cu13/bin/x86_64", "nvidia/cuda_cupti/bin"), + anchor_rel_dirs_linux=("extras/CUPTI/lib64", "lib"), + anchor_rel_dirs_windows=("extras/CUPTI/lib64", "bin"), + ctk_root_canary_anchor_libnames=("cudart",), + ), # ----------------------------------------------------------------------- # Third-party / separately packaged libraries # ----------------------------------------------------------------------- diff --git a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/search_platform.py b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/search_platform.py index 817ac0b65f..95e0f4dd1e 100644 --- a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/search_platform.py +++ b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/search_platform.py @@ -141,10 +141,20 @@ def find_in_lib_dir( error_messages: list[str], attachments: list[str], ) -> str | None: + # Most libraries have both unversioned and versioned files/symlinks (exact match first) so_name = os.path.join(lib_dir, lib_searched_for) if os.path.isfile(so_name): return so_name - error_messages.append(f"No such file: {so_name}") + # Some libraries only exist as versioned files (e.g., libcupti.so.13 in conda), + # so the glob fallback is needed + file_wild = lib_searched_for + "*" + # Only one match is expected, but to ensure deterministic behavior in unexpected + # situations, and to be internally consistent, we sort in reverse order with the + # intent to return the newest version first. + for so_name in sorted(glob.glob(os.path.join(lib_dir, file_wild)), reverse=True): + if os.path.isfile(so_name): + return so_name + error_messages.append(f"No such file: {file_wild}") attachments.append(f' listdir("{lib_dir}"):') if not os.path.isdir(lib_dir): attachments.append(" DIRECTORY DOES NOT EXIST") diff --git a/cuda_pathfinder/pyproject.toml b/cuda_pathfinder/pyproject.toml index 21299d3366..fdd01b763b 100644 --- a/cuda_pathfinder/pyproject.toml +++ b/cuda_pathfinder/pyproject.toml @@ -19,7 +19,7 @@ test = [ ] # Internal organization of test dependencies. cu12 = [ - "cuda-toolkit[nvcc,cublas,nvrtc,cudart,cufft,curand,cusolver,cusparse,npp,nvfatbin,nvjitlink,nvjpeg,cccl]==12.*", + "cuda-toolkit[nvcc,cublas,nvrtc,cudart,cufft,curand,cusolver,cusparse,npp,nvfatbin,nvjitlink,nvjpeg,cccl,cupti]==12.*", "cuda-toolkit[cufile]==12.*; sys_platform != 'win32'", "cutensor-cu12", "nvidia-cublasmp-cu12; sys_platform != 'win32'", @@ -31,7 +31,7 @@ cu12 = [ "nvidia-nvshmem-cu12; sys_platform != 'win32'", ] cu13 = [ - "cuda-toolkit[nvcc,cublas,nvrtc,cudart,cufft,curand,cusolver,cusparse,npp,nvfatbin,nvjitlink,nvjpeg,cccl,nvvm]==13.*", + "cuda-toolkit[nvcc,cublas,nvrtc,cudart,cufft,curand,cusolver,cusparse,npp,nvfatbin,nvjitlink,nvjpeg,cccl,cupti,nvvm]==13.*", "cuda-toolkit[cufile]==13.*; sys_platform != 'win32'", "cutensor-cu13", "nvidia-cublasmp-cu13; sys_platform != 'win32'", diff --git a/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib_using_mocker.py b/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib_using_mocker.py new file mode 100644 index 0000000000..3510d1933e --- /dev/null +++ b/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib_using_mocker.py @@ -0,0 +1,173 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from cuda.pathfinder._dynamic_libs import load_nvidia_dynamic_lib as load_mod +from cuda.pathfinder._dynamic_libs import search_steps as steps_mod +from cuda.pathfinder._dynamic_libs.load_dl_common import DynamicLibNotFoundError, LoadedDL +from cuda.pathfinder._dynamic_libs.load_nvidia_dynamic_lib import ( + _load_lib_no_cache, + _resolve_system_loaded_abs_path_in_subprocess, +) +from cuda.pathfinder._dynamic_libs.search_steps import EARLY_FIND_STEPS +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS + +_MODULE = "cuda.pathfinder._dynamic_libs.load_nvidia_dynamic_lib" +_STEPS_MODULE = "cuda.pathfinder._dynamic_libs.search_steps" + + +@pytest.fixture(autouse=True) +def _clear_canary_subprocess_probe_cache(): + _resolve_system_loaded_abs_path_in_subprocess.cache_clear() + yield + _resolve_system_loaded_abs_path_in_subprocess.cache_clear() + + +def _make_loaded_dl(path, found_via): + return LoadedDL(path, False, 0xDEAD, found_via) + + +def _create_cupti_in_ctk(ctk_root): + """Create a fake cupti lib in extras/CUPTI/lib64.""" + if IS_WINDOWS: + cupti_dir = ctk_root / "extras" / "CUPTI" / "lib64" + cupti_dir.mkdir(parents=True, exist_ok=True) + cupti_lib = cupti_dir / "cupti64_2025.4.1.dll" + else: + cupti_dir = ctk_root / "extras" / "CUPTI" / "lib64" + cupti_dir.mkdir(parents=True, exist_ok=True) + cupti_lib = cupti_dir / "libcupti.so.13" + # Create symlink like real CTK installations + cupti_symlink = cupti_dir / "libcupti.so" + cupti_symlink.symlink_to("libcupti.so.13") + cupti_lib.write_bytes(b"fake") + return cupti_lib + + +# --------------------------------------------------------------------------- +# Conda tests +# Note: Site-packages and CTK are covered by real CI tests. +# Mock tests focus on Conda (not covered by real CI) and error paths. +# --------------------------------------------------------------------------- + + +def test_cupti_found_in_conda(tmp_path, mocker, monkeypatch): + """Test finding cupti in conda environment.""" + if IS_WINDOWS: + pytest.skip("Windows support for cupti not yet implemented") + + # Create conda structure + conda_prefix = tmp_path / "conda_env" + conda_lib_dir = conda_prefix / "lib" + conda_lib_dir.mkdir(parents=True) + cupti_lib = conda_lib_dir / "libcupti.so.13" + cupti_lib.write_bytes(b"fake") + + # Mock conda discovery + monkeypatch.setenv("CONDA_PREFIX", str(conda_prefix)) + + # Disable site-packages search + def _run_find_steps_without_site_packages(ctx, steps): + if steps is EARLY_FIND_STEPS: + # Skip site-packages, only run conda + from cuda.pathfinder._dynamic_libs.search_steps import find_in_conda + + result = find_in_conda(ctx) + return result + return steps_mod.run_find_steps(ctx, steps) + + mocker.patch(f"{_MODULE}.run_find_steps", side_effect=_run_find_steps_without_site_packages) + mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch(f"{_MODULE}.load_dependencies") + mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) + mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=None) + mocker.patch(f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=None) + mocker.patch.object( + load_mod.LOADER, + "load_with_abs_path", + side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), + ) + + result = _load_lib_no_cache("cupti") + assert result.found_via == "conda" + assert result.abs_path == str(cupti_lib) + + +# --------------------------------------------------------------------------- +# Error path tests +# --------------------------------------------------------------------------- + + +def test_cupti_not_found_raises_error(mocker): + """Test that DynamicLibNotFoundError is raised when cupti is not found.""" + if IS_WINDOWS: + pytest.skip("Windows support for cupti not yet implemented") + + # Mock all search paths to return None + def _run_find_steps_disabled(ctx, steps): + return None + + mocker.patch(f"{_MODULE}.run_find_steps", side_effect=_run_find_steps_disabled) + mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch(f"{_MODULE}.load_dependencies") + mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) + mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=None) + mocker.patch( + f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", + return_value=None, + ) + + with pytest.raises(DynamicLibNotFoundError): + _load_lib_no_cache("cupti") + + +# --------------------------------------------------------------------------- +# Search order tests (Conda-specific, since Conda is not covered by real CI) +# --------------------------------------------------------------------------- + + +def test_cupti_search_order_conda_before_cuda_home(tmp_path, mocker, monkeypatch): + """Test that conda is searched before CUDA_HOME (CTK). + + This test is important because Conda is not covered by real CI tests, + so we need to verify the search order between Conda and CTK. + """ + if IS_WINDOWS: + pytest.skip("Windows support for cupti not yet implemented") + + # Create both conda and CUDA_HOME structures + conda_prefix = tmp_path / "conda_env" + conda_lib_dir = conda_prefix / "lib" + conda_lib_dir.mkdir(parents=True) + conda_cupti_lib = conda_lib_dir / "libcupti.so.13" + conda_cupti_lib.write_bytes(b"fake") + + ctk_root = tmp_path / "cuda-13.1" + _create_cupti_in_ctk(ctk_root) + + # Mock discovery - disable site-packages, enable conda + def _run_find_steps_without_site_packages(ctx, steps): + if steps is EARLY_FIND_STEPS: + # Skip site-packages, only run conda + from cuda.pathfinder._dynamic_libs.search_steps import find_in_conda + + result = find_in_conda(ctx) + return result + return steps_mod.run_find_steps(ctx, steps) + + mocker.patch(f"{_MODULE}.run_find_steps", side_effect=_run_find_steps_without_site_packages) + monkeypatch.setenv("CONDA_PREFIX", str(conda_prefix)) + mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch(f"{_MODULE}.load_dependencies") + mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) + mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=str(ctk_root)) + mocker.patch.object( + load_mod.LOADER, + "load_with_abs_path", + side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), + ) + + result = _load_lib_no_cache("cupti") + assert result.found_via == "conda" + assert result.abs_path == str(conda_cupti_lib)