From ee855a6f392da9ed82b7d54ad2e8b57b1d7e5480 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 25 Feb 2026 22:00:54 -0800 Subject: [PATCH 1/9] Add Linux support for loading libcupti.so.12 and libcupti.so.13 This commit adds support for finding and loading CUPTI libraries on Linux through cuda.pathfinder. It implements support for all enumerated installation methods: - Site-packages: nvidia/cuda_cupti/lib (CUDA 12) and nvidia/cu13/lib (CUDA 13) - Conda: $CONDA_PREFIX/lib (colocated with other CUDA libraries) - CTK via CUDA_HOME: $CUDA_HOME/extras/CUPTI/lib64 - CTK via canary probe: system CTK root discovery (similar to nvvm) Changes: - Add 'cupti' to supported library names and SONAMEs - Add site-packages paths for CUDA 12 and 13 - Add cupti to CTK root canary discoverable libraries - Update find_nvidia_dynamic_lib to handle extras/CUPTI/lib64 path - Add logic to distinguish CTK (extras/CUPTI/lib64) vs conda (lib) paths - Update _find_so_using_lib_dir to support versioned libraries via glob - Add comprehensive mock tests covering all installation methods Fixes #1572 (Linux support) Made-with: Cursor --- .../_dynamic_libs/find_nvidia_dynamic_lib.py | 18 +- .../_dynamic_libs/supported_nvidia_libs.py | 8 +- ...st_load_nvidia_dynamic_lib_using_mocker.py | 366 ++++++++++++++++++ 3 files changed, 388 insertions(+), 4 deletions(-) create mode 100644 cuda_pathfinder/tests/test_load_nvidia_dynamic_lib_using_mocker.py diff --git a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/find_nvidia_dynamic_lib.py b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/find_nvidia_dynamic_lib.py index b93523f36a..70bd800897 100644 --- a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/find_nvidia_dynamic_lib.py +++ b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/find_nvidia_dynamic_lib.py @@ -80,7 +80,7 @@ def _find_dll_using_nvidia_bin_dirs( def _find_lib_dir_using_anchor_point(libname: str, anchor_point: str, linux_lib_dir: str) -> str | None: # Resolve paths for the four cases: - # Windows/Linux x nvvm yes/no + # Windows/Linux x nvvm/cupti yes/no if IS_WINDOWS: if libname == "nvvm": # noqa: SIM108 rel_paths = [ @@ -93,8 +93,15 @@ def _find_lib_dir_using_anchor_point(libname: str, anchor_point: str, linux_lib_ "bin", # CTK 12 ] else: - if libname == "nvvm": # noqa: SIM108 + if libname == "nvvm": rel_paths = ["nvvm/lib64"] + elif libname == "cupti": + # cupti is in extras/CUPTI/lib64 for CTK, but in lib for conda + # Check if this looks like a CTK root (has extras/CUPTI) or conda (uses linux_lib_dir) + if os.path.isdir(os.path.join(anchor_point, "extras", "CUPTI")): + rel_paths = ["extras/CUPTI/lib64"] + else: + rel_paths = [linux_lib_dir] else: rel_paths = [linux_lib_dir] @@ -128,7 +135,12 @@ def _find_so_using_lib_dir( so_name = os.path.join(lib_dir, so_basename) if os.path.isfile(so_name): return so_name - error_messages.append(f"No such file: {so_name}") + # Look for a versioned library using glob (e.g., libcupti.so.12, libcupti.so.13) + file_wild = so_basename + "*" + for so_name in sorted(glob.glob(os.path.join(lib_dir, file_wild))): + if os.path.isfile(so_name): + return so_name + error_messages.append(f"No such file: {file_wild}") attachments.append(f' listdir("{lib_dir}"):') if not os.path.isdir(lib_dir): attachments.append(" DIRECTORY DOES NOT EXIST") diff --git a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py index d4225233c2..5d55d82ec4 100644 --- a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py +++ b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py @@ -51,6 +51,7 @@ SUPPORTED_LIBNAMES_LINUX_ONLY = ( "cufile", # "cufile_rdma", # Requires libmlx5.so + "cupti", ) SUPPORTED_LIBNAMES_LINUX = SUPPORTED_LIBNAMES_COMMON + SUPPORTED_LIBNAMES_LINUX_ONLY @@ -201,6 +202,10 @@ "libnvrtc.so.13", ), "nvvm": ("libnvvm.so.4",), + "cupti": ( + "libcupti.so.12", + "libcupti.so.13", + ), } SUPPORTED_LINUX_SONAMES_OTHER = { "cublasmp": ("libcublasmp.so.0",), @@ -363,7 +368,7 @@ # CTK root in an isolated child process. # - discoverable libs: libs that are allowed to use the CTK-root canary fallback. _CTK_ROOT_CANARY_ANCHOR_LIBNAMES = ("cudart",) -_CTK_ROOT_CANARY_DISCOVERABLE_LIBNAMES = ("nvvm",) +_CTK_ROOT_CANARY_DISCOVERABLE_LIBNAMES = ("nvvm", "cupti") # Based on output of toolshed/make_site_packages_libdirs_linux.py SITE_PACKAGES_LIBDIRS_LINUX_CTK = { @@ -395,6 +400,7 @@ "nvjpeg": ("nvidia/cu13/lib", "nvidia/nvjpeg/lib"), "nvrtc": ("nvidia/cu13/lib", "nvidia/cuda_nvrtc/lib"), "nvvm": ("nvidia/cu13/lib", "nvidia/cuda_nvcc/nvvm/lib64"), + "cupti": ("nvidia/cu13/lib", "nvidia/cuda_cupti/lib"), } SITE_PACKAGES_LIBDIRS_LINUX_OTHER = { "cublasmp": ("nvidia/cublasmp/cu13/lib", "nvidia/cublasmp/cu12/lib"), diff --git a/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib_using_mocker.py b/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib_using_mocker.py new file mode 100644 index 0000000000..1c8b56f044 --- /dev/null +++ b/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib_using_mocker.py @@ -0,0 +1,366 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import site + +import pytest + +from cuda.pathfinder._dynamic_libs.find_nvidia_dynamic_lib import _FindNvidiaDynamicLib +from cuda.pathfinder._dynamic_libs.load_dl_common import LoadedDL +from cuda.pathfinder._dynamic_libs.load_nvidia_dynamic_lib import ( + _load_lib_no_cache, + _resolve_system_loaded_abs_path_in_subprocess, +) +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS + +_MODULE = "cuda.pathfinder._dynamic_libs.load_nvidia_dynamic_lib" +_FIND_MODULE = "cuda.pathfinder._dynamic_libs.find_nvidia_dynamic_lib" + + +@pytest.fixture(autouse=True) +def _clear_canary_subprocess_probe_cache(): + _resolve_system_loaded_abs_path_in_subprocess.cache_clear() + yield + _resolve_system_loaded_abs_path_in_subprocess.cache_clear() + + +def _make_loaded_dl(path, found_via): + return LoadedDL(path, False, 0xDEAD, found_via) + + +def _create_cudart_in_ctk(ctk_root): + """Create a fake cudart lib in the platform-appropriate CTK subdirectory.""" + if IS_WINDOWS: + cudart_dir = ctk_root / "bin" / "x64" + cudart_dir.mkdir(parents=True, exist_ok=True) + cudart_lib = cudart_dir / "cudart64_13.dll" + else: + cudart_dir = ctk_root / "lib64" + cudart_dir.mkdir(parents=True, exist_ok=True) + cudart_lib = cudart_dir / "libcudart.so.13" + cudart_lib.write_bytes(b"fake") + return cudart_lib + + +def _create_cupti_in_ctk(ctk_root): + """Create a fake cupti lib in extras/CUPTI/lib64.""" + if IS_WINDOWS: + cupti_dir = ctk_root / "extras" / "CUPTI" / "lib64" + cupti_dir.mkdir(parents=True, exist_ok=True) + cupti_lib = cupti_dir / "cupti64_2025.4.1.dll" + else: + cupti_dir = ctk_root / "extras" / "CUPTI" / "lib64" + cupti_dir.mkdir(parents=True, exist_ok=True) + cupti_lib = cupti_dir / "libcupti.so.13" + # Create symlink like real CTK installations + cupti_symlink = cupti_dir / "libcupti.so" + cupti_symlink.symlink_to("libcupti.so.13") + cupti_lib.write_bytes(b"fake") + return cupti_lib + + +def _fake_canary_path(ctk_root): + """Return a fake canary lib path that would resolve to the given CTK root.""" + if IS_WINDOWS: + return str(ctk_root / "bin" / "x64" / "cudart64_13.dll") + return str(ctk_root / "lib64" / "libcudart.so.13") + + +# --------------------------------------------------------------------------- +# Site-packages tests +# --------------------------------------------------------------------------- + + +def test_cupti_found_in_site_packages_cuda12(tmp_path, mocker, monkeypatch): + """Test finding cupti in site-packages for CUDA 12.""" + if IS_WINDOWS: + pytest.skip("Windows support for cupti not yet implemented") + + # Create site-packages structure for CUDA 12 + site_packages = tmp_path / "site-packages" + cupti_dir = site_packages / "nvidia" / "cuda_cupti" / "lib" + cupti_dir.mkdir(parents=True) + cupti_lib = cupti_dir / "libcupti.so.12" + cupti_lib.write_bytes(b"fake") + + # Mock site-packages discovery + monkeypatch.setattr(site, "getsitepackages", lambda: [str(site_packages)]) + mocker.patch.object(_FindNvidiaDynamicLib, "try_with_conda_prefix", return_value=None) + mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch(f"{_MODULE}.load_dependencies") + mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) + mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=None) + mocker.patch(f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=None) + mocker.patch( + f"{_MODULE}.load_with_abs_path", + side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + ) + + result = _load_lib_no_cache("cupti") + assert result.found_via == "site-packages" + assert result.abs_path == str(cupti_lib) + + +def test_cupti_found_in_site_packages_cuda13(tmp_path, mocker, monkeypatch): + """Test finding cupti in site-packages for CUDA 13.""" + if IS_WINDOWS: + pytest.skip("Windows support for cupti not yet implemented") + + # Create site-packages structure for CUDA 13 + site_packages = tmp_path / "site-packages" + cupti_dir = site_packages / "nvidia" / "cu13" / "lib" + cupti_dir.mkdir(parents=True) + cupti_lib = cupti_dir / "libcupti.so.13" + cupti_lib.write_bytes(b"fake") + + # Mock site-packages discovery + monkeypatch.setattr(site, "getsitepackages", lambda: [str(site_packages)]) + mocker.patch.object(_FindNvidiaDynamicLib, "try_with_conda_prefix", return_value=None) + mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch(f"{_MODULE}.load_dependencies") + mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) + mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=None) + mocker.patch(f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=None) + mocker.patch( + f"{_MODULE}.load_with_abs_path", + side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + ) + + result = _load_lib_no_cache("cupti") + assert result.found_via == "site-packages" + assert result.abs_path == str(cupti_lib) + + +# --------------------------------------------------------------------------- +# Conda tests +# --------------------------------------------------------------------------- + + +def test_cupti_found_in_conda(tmp_path, mocker, monkeypatch): + """Test finding cupti in conda environment.""" + if IS_WINDOWS: + pytest.skip("Windows support for cupti not yet implemented") + + # Create conda structure + conda_prefix = tmp_path / "conda_env" + conda_lib_dir = conda_prefix / "lib" + conda_lib_dir.mkdir(parents=True) + cupti_lib = conda_lib_dir / "libcupti.so.13" + cupti_lib.write_bytes(b"fake") + + # Mock conda discovery + monkeypatch.setenv("CONDA_PREFIX", str(conda_prefix)) + mocker.patch.object(_FindNvidiaDynamicLib, "try_site_packages", return_value=None) + mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch(f"{_MODULE}.load_dependencies") + mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) + mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=None) + mocker.patch(f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=None) + mocker.patch( + f"{_MODULE}.load_with_abs_path", + side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + ) + + result = _load_lib_no_cache("cupti") + assert result.found_via == "conda" + assert result.abs_path == str(cupti_lib) + + +# --------------------------------------------------------------------------- +# CTK via CUDA_HOME tests +# --------------------------------------------------------------------------- + + +def test_cupti_found_via_cuda_home(tmp_path, mocker): + """Test finding cupti via CUDA_HOME pointing to CTK installation.""" + if IS_WINDOWS: + pytest.skip("Windows support for cupti not yet implemented") + + ctk_root = tmp_path / "cuda-13.1" + cupti_lib = _create_cupti_in_ctk(ctk_root) + + # Mock CUDA_HOME discovery + mocker.patch.object(_FindNvidiaDynamicLib, "try_site_packages", return_value=None) + mocker.patch.object(_FindNvidiaDynamicLib, "try_with_conda_prefix", return_value=None) + mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch(f"{_MODULE}.load_dependencies") + mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=str(ctk_root)) + mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) + mocker.patch( + f"{_MODULE}.load_with_abs_path", + side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + ) + + result = _load_lib_no_cache("cupti") + assert result.found_via == "CUDA_HOME" + # The finder resolves symlinks, so it finds libcupti.so (the symlink) not libcupti.so.13 + assert result.abs_path == str(cupti_lib.parent / "libcupti.so") + + +# --------------------------------------------------------------------------- +# CTK via canary probe tests +# --------------------------------------------------------------------------- + + +def test_cupti_found_via_canary_probe(tmp_path, mocker): + """Test finding cupti via CTK root canary probe.""" + if IS_WINDOWS: + pytest.skip("Windows support for cupti not yet implemented") + + ctk_root = tmp_path / "cuda-13.1" + _create_cudart_in_ctk(ctk_root) + cupti_lib = _create_cupti_in_ctk(ctk_root) + + # Mock canary probe discovery + mocker.patch.object(_FindNvidiaDynamicLib, "try_site_packages", return_value=None) + mocker.patch.object(_FindNvidiaDynamicLib, "try_with_conda_prefix", return_value=None) + mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch(f"{_MODULE}.load_dependencies") + mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) + mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=None) + mocker.patch( + f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", + return_value=_fake_canary_path(ctk_root), + ) + mocker.patch( + f"{_MODULE}.load_with_abs_path", + side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + ) + + result = _load_lib_no_cache("cupti") + assert result.found_via == "system-ctk-root" + # The finder resolves symlinks, so it finds libcupti.so (the symlink) not libcupti.so.13 + assert result.abs_path == str(cupti_lib.parent / "libcupti.so") + + +def test_cupti_not_found_raises_error(mocker): + """Test that DynamicLibNotFoundError is raised when cupti is not found.""" + if IS_WINDOWS: + pytest.skip("Windows support for cupti not yet implemented") + + from cuda.pathfinder._dynamic_libs.load_dl_common import DynamicLibNotFoundError + + # Mock all search paths to return None + mocker.patch.object(_FindNvidiaDynamicLib, "try_site_packages", return_value=None) + mocker.patch.object(_FindNvidiaDynamicLib, "try_with_conda_prefix", return_value=None) + mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch(f"{_MODULE}.load_dependencies") + mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) + mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=None) + mocker.patch( + f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", + return_value=None, + ) + + with pytest.raises(DynamicLibNotFoundError): + _load_lib_no_cache("cupti") + + +# --------------------------------------------------------------------------- +# Search order tests +# --------------------------------------------------------------------------- + + +def test_cupti_search_order_site_packages_before_conda(tmp_path, mocker, monkeypatch): + """Test that site-packages is searched before conda.""" + if IS_WINDOWS: + pytest.skip("Windows support for cupti not yet implemented") + + # Create both site-packages and conda structures + site_packages = tmp_path / "site-packages" + site_cupti_dir = site_packages / "nvidia" / "cu13" / "lib" + site_cupti_dir.mkdir(parents=True) + site_cupti_lib = site_cupti_dir / "libcupti.so.13" + site_cupti_lib.write_bytes(b"fake") + + conda_prefix = tmp_path / "conda_env" + conda_lib_dir = conda_prefix / "lib" + conda_lib_dir.mkdir(parents=True) + conda_cupti_lib = conda_lib_dir / "libcupti.so.13" + conda_cupti_lib.write_bytes(b"fake") + + # Mock discovery + monkeypatch.setattr(site, "getsitepackages", lambda: [str(site_packages)]) + monkeypatch.setenv("CONDA_PREFIX", str(conda_prefix)) + mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch(f"{_MODULE}.load_dependencies") + mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) + mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=None) + mocker.patch(f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=None) + mocker.patch( + f"{_MODULE}.load_with_abs_path", + side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + ) + + result = _load_lib_no_cache("cupti") + assert result.found_via == "site-packages" + assert result.abs_path == str(site_cupti_lib) + + +def test_cupti_search_order_conda_before_cuda_home(tmp_path, mocker, monkeypatch): + """Test that conda is searched before CUDA_HOME.""" + if IS_WINDOWS: + pytest.skip("Windows support for cupti not yet implemented") + + # Create both conda and CUDA_HOME structures + conda_prefix = tmp_path / "conda_env" + conda_lib_dir = conda_prefix / "lib" + conda_lib_dir.mkdir(parents=True) + conda_cupti_lib = conda_lib_dir / "libcupti.so.13" + conda_cupti_lib.write_bytes(b"fake") + + ctk_root = tmp_path / "cuda-13.1" + _create_cupti_in_ctk(ctk_root) + + # Mock discovery + mocker.patch.object(_FindNvidiaDynamicLib, "try_site_packages", return_value=None) + monkeypatch.setenv("CONDA_PREFIX", str(conda_prefix)) + mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch(f"{_MODULE}.load_dependencies") + mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) + mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=str(ctk_root)) + mocker.patch( + f"{_MODULE}.load_with_abs_path", + side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + ) + + result = _load_lib_no_cache("cupti") + assert result.found_via == "conda" + assert result.abs_path == str(conda_cupti_lib) + + +def test_cupti_search_order_cuda_home_before_canary(tmp_path, mocker): + """Test that CUDA_HOME is searched before canary probe.""" + if IS_WINDOWS: + pytest.skip("Windows support for cupti not yet implemented") + + # Create two CTK roots: one for CUDA_HOME, one for canary + cuda_home_root = tmp_path / "cuda-home" + cuda_home_cupti = _create_cupti_in_ctk(cuda_home_root) + + canary_root = tmp_path / "cuda-system" + _create_cudart_in_ctk(canary_root) + _create_cupti_in_ctk(canary_root) + + # Mock discovery + mocker.patch.object(_FindNvidiaDynamicLib, "try_site_packages", return_value=None) + mocker.patch.object(_FindNvidiaDynamicLib, "try_with_conda_prefix", return_value=None) + mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch(f"{_MODULE}.load_dependencies") + mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) + mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=str(cuda_home_root)) + canary_mock = mocker.patch( + f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", + return_value=_fake_canary_path(canary_root), + ) + mocker.patch( + f"{_MODULE}.load_with_abs_path", + side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + ) + + result = _load_lib_no_cache("cupti") + assert result.found_via == "CUDA_HOME" + # The finder resolves symlinks, so it finds libcupti.so (the symlink) not libcupti.so.13 + assert result.abs_path == str(cuda_home_cupti.parent / "libcupti.so") + # Canary should not have been called + canary_mock.assert_not_called() From 7bee9c53a40269c03067f355b8f8e90937fafc70 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 4 Mar 2026 21:29:27 -0800 Subject: [PATCH 2/9] Update cupti tests to use new SearchContext-based API Migrated test_load_nvidia_dynamic_lib_using_mocker.py from the old _FindNvidiaDynamicLib API to the new descriptor-based SearchContext API. Changes: - Replace _FindNvidiaDynamicLib imports with search_steps and load_nvidia_dynamic_lib modules - Update mocks to use run_find_steps, LOADER, and SearchContext - Use LIB_DESCRIPTORS to get cupti descriptor - Update all test functions to work with the new search step architecture Made-with: Cursor --- ...st_load_nvidia_dynamic_lib_using_mocker.py | 182 +++++++++++------- 1 file changed, 110 insertions(+), 72 deletions(-) diff --git a/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib_using_mocker.py b/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib_using_mocker.py index 1c8b56f044..06bbf6a3fc 100644 --- a/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib_using_mocker.py +++ b/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib_using_mocker.py @@ -5,16 +5,18 @@ import pytest -from cuda.pathfinder._dynamic_libs.find_nvidia_dynamic_lib import _FindNvidiaDynamicLib -from cuda.pathfinder._dynamic_libs.load_dl_common import LoadedDL +from cuda.pathfinder._dynamic_libs import load_nvidia_dynamic_lib as load_mod +from cuda.pathfinder._dynamic_libs import search_steps as steps_mod +from cuda.pathfinder._dynamic_libs.load_dl_common import DynamicLibNotFoundError, LoadedDL from cuda.pathfinder._dynamic_libs.load_nvidia_dynamic_lib import ( _load_lib_no_cache, _resolve_system_loaded_abs_path_in_subprocess, ) +from cuda.pathfinder._dynamic_libs.search_steps import EARLY_FIND_STEPS from cuda.pathfinder._utils.platform_aware import IS_WINDOWS _MODULE = "cuda.pathfinder._dynamic_libs.load_nvidia_dynamic_lib" -_FIND_MODULE = "cuda.pathfinder._dynamic_libs.find_nvidia_dynamic_lib" +_STEPS_MODULE = "cuda.pathfinder._dynamic_libs.search_steps" @pytest.fixture(autouse=True) @@ -85,15 +87,15 @@ def test_cupti_found_in_site_packages_cuda12(tmp_path, mocker, monkeypatch): # Mock site-packages discovery monkeypatch.setattr(site, "getsitepackages", lambda: [str(site_packages)]) - mocker.patch.object(_FindNvidiaDynamicLib, "try_with_conda_prefix", return_value=None) - mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) mocker.patch(f"{_MODULE}.load_dependencies") - mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) - mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=None) + mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) + mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=None) mocker.patch(f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=None) - mocker.patch( - f"{_MODULE}.load_with_abs_path", - side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + mocker.patch.object( + load_mod.LOADER, + "load_with_abs_path", + side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), ) result = _load_lib_no_cache("cupti") @@ -115,15 +117,15 @@ def test_cupti_found_in_site_packages_cuda13(tmp_path, mocker, monkeypatch): # Mock site-packages discovery monkeypatch.setattr(site, "getsitepackages", lambda: [str(site_packages)]) - mocker.patch.object(_FindNvidiaDynamicLib, "try_with_conda_prefix", return_value=None) - mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) mocker.patch(f"{_MODULE}.load_dependencies") - mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) - mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=None) + mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) + mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=None) mocker.patch(f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=None) - mocker.patch( - f"{_MODULE}.load_with_abs_path", - side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + mocker.patch.object( + load_mod.LOADER, + "load_with_abs_path", + side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), ) result = _load_lib_no_cache("cupti") @@ -150,15 +152,27 @@ def test_cupti_found_in_conda(tmp_path, mocker, monkeypatch): # Mock conda discovery monkeypatch.setenv("CONDA_PREFIX", str(conda_prefix)) - mocker.patch.object(_FindNvidiaDynamicLib, "try_site_packages", return_value=None) - mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + + # Disable site-packages search + def _run_find_steps_without_site_packages(ctx, steps): + if steps is EARLY_FIND_STEPS: + # Skip site-packages, only run conda + from cuda.pathfinder._dynamic_libs.search_steps import find_in_conda + + result = find_in_conda(ctx) + return result + return steps_mod.run_find_steps(ctx, steps) + + mocker.patch(f"{_MODULE}.run_find_steps", side_effect=_run_find_steps_without_site_packages) + mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) mocker.patch(f"{_MODULE}.load_dependencies") - mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) - mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=None) + mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) + mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=None) mocker.patch(f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=None) - mocker.patch( - f"{_MODULE}.load_with_abs_path", - side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + mocker.patch.object( + load_mod.LOADER, + "load_with_abs_path", + side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), ) result = _load_lib_no_cache("cupti") @@ -179,16 +193,21 @@ def test_cupti_found_via_cuda_home(tmp_path, mocker): ctk_root = tmp_path / "cuda-13.1" cupti_lib = _create_cupti_in_ctk(ctk_root) - # Mock CUDA_HOME discovery - mocker.patch.object(_FindNvidiaDynamicLib, "try_site_packages", return_value=None) - mocker.patch.object(_FindNvidiaDynamicLib, "try_with_conda_prefix", return_value=None) - mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + # Mock CUDA_HOME discovery - disable early find steps + def _run_find_steps_without_early(ctx, steps): + if steps is EARLY_FIND_STEPS: + return None + return steps_mod.run_find_steps(ctx, steps) + + mocker.patch(f"{_MODULE}.run_find_steps", side_effect=_run_find_steps_without_early) + mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) mocker.patch(f"{_MODULE}.load_dependencies") - mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=str(ctk_root)) - mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) - mocker.patch( - f"{_MODULE}.load_with_abs_path", - side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=str(ctk_root)) + mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) + mocker.patch.object( + load_mod.LOADER, + "load_with_abs_path", + side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), ) result = _load_lib_no_cache("cupti") @@ -211,20 +230,23 @@ def test_cupti_found_via_canary_probe(tmp_path, mocker): _create_cudart_in_ctk(ctk_root) cupti_lib = _create_cupti_in_ctk(ctk_root) - # Mock canary probe discovery - mocker.patch.object(_FindNvidiaDynamicLib, "try_site_packages", return_value=None) - mocker.patch.object(_FindNvidiaDynamicLib, "try_with_conda_prefix", return_value=None) - mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + # Mock canary probe discovery - disable all early/late find steps + def _run_find_steps_disabled(ctx, steps): + return None + + mocker.patch(f"{_MODULE}.run_find_steps", side_effect=_run_find_steps_disabled) + mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) mocker.patch(f"{_MODULE}.load_dependencies") - mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) - mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=None) + mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) + mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=None) mocker.patch( f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=_fake_canary_path(ctk_root), ) - mocker.patch( - f"{_MODULE}.load_with_abs_path", - side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + mocker.patch.object( + load_mod.LOADER, + "load_with_abs_path", + side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), ) result = _load_lib_no_cache("cupti") @@ -238,15 +260,15 @@ def test_cupti_not_found_raises_error(mocker): if IS_WINDOWS: pytest.skip("Windows support for cupti not yet implemented") - from cuda.pathfinder._dynamic_libs.load_dl_common import DynamicLibNotFoundError - # Mock all search paths to return None - mocker.patch.object(_FindNvidiaDynamicLib, "try_site_packages", return_value=None) - mocker.patch.object(_FindNvidiaDynamicLib, "try_with_conda_prefix", return_value=None) - mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + def _run_find_steps_disabled(ctx, steps): + return None + + mocker.patch(f"{_MODULE}.run_find_steps", side_effect=_run_find_steps_disabled) + mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) mocker.patch(f"{_MODULE}.load_dependencies") - mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) - mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=None) + mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) + mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=None) mocker.patch( f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=None, @@ -282,14 +304,15 @@ def test_cupti_search_order_site_packages_before_conda(tmp_path, mocker, monkeyp # Mock discovery monkeypatch.setattr(site, "getsitepackages", lambda: [str(site_packages)]) monkeypatch.setenv("CONDA_PREFIX", str(conda_prefix)) - mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) mocker.patch(f"{_MODULE}.load_dependencies") - mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) - mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=None) + mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) + mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=None) mocker.patch(f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=None) - mocker.patch( - f"{_MODULE}.load_with_abs_path", - side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + mocker.patch.object( + load_mod.LOADER, + "load_with_abs_path", + side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), ) result = _load_lib_no_cache("cupti") @@ -312,16 +335,26 @@ def test_cupti_search_order_conda_before_cuda_home(tmp_path, mocker, monkeypatch ctk_root = tmp_path / "cuda-13.1" _create_cupti_in_ctk(ctk_root) - # Mock discovery - mocker.patch.object(_FindNvidiaDynamicLib, "try_site_packages", return_value=None) + # Mock discovery - disable site-packages, enable conda + def _run_find_steps_without_site_packages(ctx, steps): + if steps is EARLY_FIND_STEPS: + # Skip site-packages, only run conda + from cuda.pathfinder._dynamic_libs.search_steps import find_in_conda + + result = find_in_conda(ctx) + return result + return steps_mod.run_find_steps(ctx, steps) + + mocker.patch(f"{_MODULE}.run_find_steps", side_effect=_run_find_steps_without_site_packages) monkeypatch.setenv("CONDA_PREFIX", str(conda_prefix)) - mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) mocker.patch(f"{_MODULE}.load_dependencies") - mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) - mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=str(ctk_root)) - mocker.patch( - f"{_MODULE}.load_with_abs_path", - side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) + mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=str(ctk_root)) + mocker.patch.object( + load_mod.LOADER, + "load_with_abs_path", + side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), ) result = _load_lib_no_cache("cupti") @@ -342,20 +375,25 @@ def test_cupti_search_order_cuda_home_before_canary(tmp_path, mocker): _create_cudart_in_ctk(canary_root) _create_cupti_in_ctk(canary_root) - # Mock discovery - mocker.patch.object(_FindNvidiaDynamicLib, "try_site_packages", return_value=None) - mocker.patch.object(_FindNvidiaDynamicLib, "try_with_conda_prefix", return_value=None) - mocker.patch(f"{_MODULE}.check_if_already_loaded_from_elsewhere", return_value=None) + # Mock discovery - disable early find steps + def _run_find_steps_without_early(ctx, steps): + if steps is load_mod.EARLY_FIND_STEPS: + return None + return steps_mod.run_find_steps(ctx, steps) + + mocker.patch(f"{_MODULE}.run_find_steps", side_effect=_run_find_steps_without_early) + mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) mocker.patch(f"{_MODULE}.load_dependencies") - mocker.patch(f"{_MODULE}.load_with_system_search", return_value=None) - mocker.patch(f"{_FIND_MODULE}.get_cuda_home_or_path", return_value=str(cuda_home_root)) + mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) + mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=str(cuda_home_root)) canary_mock = mocker.patch( f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=_fake_canary_path(canary_root), ) - mocker.patch( - f"{_MODULE}.load_with_abs_path", - side_effect=lambda _libname, path, via: _make_loaded_dl(path, via), + mocker.patch.object( + load_mod.LOADER, + "load_with_abs_path", + side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), ) result = _load_lib_no_cache("cupti") From b819eee826ee32e0a10b15ecdd32416a07104eaa Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 4 Mar 2026 21:36:07 -0800 Subject: [PATCH 3/9] Remove unused CTK canary variables from supported_nvidia_libs.py These variables (_CTK_ROOT_CANARY_ANCHOR_LIBNAMES and _CTK_ROOT_CANARY_DISCOVERABLE_LIBNAMES) were added in the cupti PR but are not used in the new descriptor-based architecture. The new code uses desc.ctk_root_canary_anchor_libnames directly from descriptors. Made-with: Cursor --- .../_dynamic_libs/supported_nvidia_libs.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py index 8123d53b76..db06411c6d 100644 --- a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py +++ b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py @@ -25,6 +25,7 @@ SUPPORTED_LIBNAMES_WINDOWS_ONLY = tuple( desc.name for desc in _CTK_DESCRIPTORS if desc.windows_dlls and not desc.linux_sonames ) + SUPPORTED_LIBNAMES_LINUX = SUPPORTED_LIBNAMES_COMMON + SUPPORTED_LIBNAMES_LINUX_ONLY SUPPORTED_LIBNAMES_WINDOWS = SUPPORTED_LIBNAMES_COMMON + SUPPORTED_LIBNAMES_WINDOWS_ONLY SUPPORTED_LIBNAMES_ALL = SUPPORTED_LIBNAMES_COMMON + SUPPORTED_LIBNAMES_LINUX_ONLY + SUPPORTED_LIBNAMES_WINDOWS_ONLY @@ -50,19 +51,6 @@ desc.name for desc in DESCRIPTOR_CATALOG if desc.requires_rtld_deepbind and desc.linux_sonames ) -# CTK root canary probe config: -# - anchor libs: expected on the standard system loader path and used to derive -# CTK root in an isolated child process. -# - discoverable libs: libs that are allowed to use the CTK-root canary fallback. -# Note: These are derived from the descriptor catalog for backward compatibility. -# The new code uses desc.ctk_root_canary_anchor_libnames directly from descriptors. -_CTK_ROOT_CANARY_ANCHOR_LIBNAMES = tuple( - desc.ctk_root_canary_anchor_libnames[0] for desc in DESCRIPTOR_CATALOG if desc.ctk_root_canary_anchor_libnames -) -_CTK_ROOT_CANARY_DISCOVERABLE_LIBNAMES = tuple( - desc.name for desc in DESCRIPTOR_CATALOG if desc.ctk_root_canary_anchor_libnames -) - # Based on output of toolshed/make_site_packages_libdirs_linux.py SITE_PACKAGES_LIBDIRS_LINUX_CTK = { desc.name: desc.site_packages_linux for desc in _CTK_DESCRIPTORS if desc.site_packages_linux From 9ba64b8888fea3e945da192c3368398feb861728 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 5 Mar 2026 08:31:20 -0800 Subject: [PATCH 4/9] Improve comment for change in LinuxSearchPlatform.find_in_lib_dir() --- .../cuda/pathfinder/_dynamic_libs/search_platform.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/search_platform.py b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/search_platform.py index b1bfe8597a..864f0f8a02 100644 --- a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/search_platform.py +++ b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/search_platform.py @@ -141,10 +141,12 @@ def find_in_lib_dir( error_messages: list[str], attachments: list[str], ) -> str | None: + # Most libraries have both unversioned and versioned files/symlinks (exact match first) so_name = os.path.join(lib_dir, lib_searched_for) if os.path.isfile(so_name): return so_name - # Look for a versioned library using glob (e.g., libcupti.so.12, libcupti.so.13) + # Some libraries only exist as versioned files (e.g., libcupti.so.13 in conda), + # so the glob fallback is needed file_wild = lib_searched_for + "*" for so_name in sorted(glob.glob(os.path.join(lib_dir, file_wild))): if os.path.isfile(so_name): From caddf5e27a3a90ebd89639b0b24f2db5b171a4b1 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 5 Mar 2026 09:24:58 -0800 Subject: [PATCH 5/9] Add cputi to cu12, cu13 groups in cuda_pathfinder/pyproject.toml --- cuda_pathfinder/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cuda_pathfinder/pyproject.toml b/cuda_pathfinder/pyproject.toml index 21299d3366..fdd01b763b 100644 --- a/cuda_pathfinder/pyproject.toml +++ b/cuda_pathfinder/pyproject.toml @@ -19,7 +19,7 @@ test = [ ] # Internal organization of test dependencies. cu12 = [ - "cuda-toolkit[nvcc,cublas,nvrtc,cudart,cufft,curand,cusolver,cusparse,npp,nvfatbin,nvjitlink,nvjpeg,cccl]==12.*", + "cuda-toolkit[nvcc,cublas,nvrtc,cudart,cufft,curand,cusolver,cusparse,npp,nvfatbin,nvjitlink,nvjpeg,cccl,cupti]==12.*", "cuda-toolkit[cufile]==12.*; sys_platform != 'win32'", "cutensor-cu12", "nvidia-cublasmp-cu12; sys_platform != 'win32'", @@ -31,7 +31,7 @@ cu12 = [ "nvidia-nvshmem-cu12; sys_platform != 'win32'", ] cu13 = [ - "cuda-toolkit[nvcc,cublas,nvrtc,cudart,cufft,curand,cusolver,cusparse,npp,nvfatbin,nvjitlink,nvjpeg,cccl,nvvm]==13.*", + "cuda-toolkit[nvcc,cublas,nvrtc,cudart,cufft,curand,cusolver,cusparse,npp,nvfatbin,nvjitlink,nvjpeg,cccl,cupti,nvvm]==13.*", "cuda-toolkit[cufile]==13.*; sys_platform != 'win32'", "cutensor-cu13", "nvidia-cublasmp-cu13; sys_platform != 'win32'", From 3d465a35511912d3f341339f0d2d60e59b52e4e7 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 5 Mar 2026 09:30:31 -0800 Subject: [PATCH 6/9] Add cuda_cupti to cuda-components in .github/actions/fetch_ctk/action.yml --- .github/actions/fetch_ctk/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index 001e3a84d8..e938fcc5b3 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -14,7 +14,7 @@ inputs: cuda-components: description: "A list of the CTK components to install as a comma-separated list. e.g. 'cuda_nvcc,cuda_nvrtc,cuda_cudart'" required: false - default: "cuda_nvcc,cuda_cudart,cuda_crt,libnvvm,cuda_nvrtc,cuda_profiler_api,cuda_cccl,libnvjitlink,libcufile,libnvfatbin" + default: "cuda_nvcc,cuda_cudart,cuda_crt,libnvvm,cuda_nvrtc,cuda_profiler_api,cuda_cccl,cuda_cupti,libnvjitlink,libcufile,libnvfatbin" cuda-path: description: "where the CTK components will be installed to, relative to $PWD" required: false From 386ac7c7f867701b218958261c8f1d81a25282e7 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 5 Mar 2026 11:51:08 -0800 Subject: [PATCH 7/9] Add windows_dlls, site_packages_windows, anchor_rel_dirs_windows for cupti in /descriptor_catalog.py --- .../_dynamic_libs/descriptor_catalog.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/descriptor_catalog.py b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/descriptor_catalog.py index c43950c0de..89fa07445d 100644 --- a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/descriptor_catalog.py +++ b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/descriptor_catalog.py @@ -270,8 +270,23 @@ class DescriptorSpec: name="cupti", packaged_with="ctk", linux_sonames=("libcupti.so.12", "libcupti.so.13"), + windows_dlls=( + "cupti64_2025.4.1.dll", + "cupti64_2025.3.1.dll", + "cupti64_2025.2.1.dll", + "cupti64_2025.1.1.dll", + "cupti64_2024.3.2.dll", + "cupti64_2024.2.1.dll", + "cupti64_2024.1.1.dll", + "cupti64_2023.3.1.dll", + "cupti64_2023.2.2.dll", + "cupti64_2023.1.1.dll", + "cupti64_2022.4.1.dll", + ), site_packages_linux=("nvidia/cu13/lib", "nvidia/cuda_cupti/lib"), + site_packages_windows=("nvidia/cu13/bin/x86_64", "nvidia/cuda_cupti/bin"), anchor_rel_dirs_linux=("extras/CUPTI/lib64", "lib"), + anchor_rel_dirs_windows=("extras/CUPTI/lib64", "bin"), ctk_root_canary_anchor_libnames=("cudart",), ), # ----------------------------------------------------------------------- From 40f60225db8f5ba0fbe065901e364c6c453d636f Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 5 Mar 2026 12:15:41 -0800 Subject: [PATCH 8/9] test: Refactor cupti mock tests to focus on Conda and error paths Remove tests covered by real CI: - Site-packages tests (CUDA 12 and 13) - covered by real CI - CTK tests (CUDA_HOME and canary probe) - covered by real CI - Search order tests involving site-packages/CTK - covered by real CI Keep tests not covered by real CI: - Conda discovery test - Conda not covered by real CI - Error path test (not found) - error path not covered - Conda vs CTK search order test - Conda not covered by real CI Also remove unused imports and helper functions. Made-with: Cursor --- ...st_load_nvidia_dynamic_lib_using_mocker.py | 249 +----------------- 1 file changed, 9 insertions(+), 240 deletions(-) diff --git a/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib_using_mocker.py b/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib_using_mocker.py index 06bbf6a3fc..3510d1933e 100644 --- a/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib_using_mocker.py +++ b/cuda_pathfinder/tests/test_load_nvidia_dynamic_lib_using_mocker.py @@ -1,8 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import site - import pytest from cuda.pathfinder._dynamic_libs import load_nvidia_dynamic_lib as load_mod @@ -30,20 +28,6 @@ def _make_loaded_dl(path, found_via): return LoadedDL(path, False, 0xDEAD, found_via) -def _create_cudart_in_ctk(ctk_root): - """Create a fake cudart lib in the platform-appropriate CTK subdirectory.""" - if IS_WINDOWS: - cudart_dir = ctk_root / "bin" / "x64" - cudart_dir.mkdir(parents=True, exist_ok=True) - cudart_lib = cudart_dir / "cudart64_13.dll" - else: - cudart_dir = ctk_root / "lib64" - cudart_dir.mkdir(parents=True, exist_ok=True) - cudart_lib = cudart_dir / "libcudart.so.13" - cudart_lib.write_bytes(b"fake") - return cudart_lib - - def _create_cupti_in_ctk(ctk_root): """Create a fake cupti lib in extras/CUPTI/lib64.""" if IS_WINDOWS: @@ -61,80 +45,10 @@ def _create_cupti_in_ctk(ctk_root): return cupti_lib -def _fake_canary_path(ctk_root): - """Return a fake canary lib path that would resolve to the given CTK root.""" - if IS_WINDOWS: - return str(ctk_root / "bin" / "x64" / "cudart64_13.dll") - return str(ctk_root / "lib64" / "libcudart.so.13") - - -# --------------------------------------------------------------------------- -# Site-packages tests -# --------------------------------------------------------------------------- - - -def test_cupti_found_in_site_packages_cuda12(tmp_path, mocker, monkeypatch): - """Test finding cupti in site-packages for CUDA 12.""" - if IS_WINDOWS: - pytest.skip("Windows support for cupti not yet implemented") - - # Create site-packages structure for CUDA 12 - site_packages = tmp_path / "site-packages" - cupti_dir = site_packages / "nvidia" / "cuda_cupti" / "lib" - cupti_dir.mkdir(parents=True) - cupti_lib = cupti_dir / "libcupti.so.12" - cupti_lib.write_bytes(b"fake") - - # Mock site-packages discovery - monkeypatch.setattr(site, "getsitepackages", lambda: [str(site_packages)]) - mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) - mocker.patch(f"{_MODULE}.load_dependencies") - mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) - mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=None) - mocker.patch(f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=None) - mocker.patch.object( - load_mod.LOADER, - "load_with_abs_path", - side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), - ) - - result = _load_lib_no_cache("cupti") - assert result.found_via == "site-packages" - assert result.abs_path == str(cupti_lib) - - -def test_cupti_found_in_site_packages_cuda13(tmp_path, mocker, monkeypatch): - """Test finding cupti in site-packages for CUDA 13.""" - if IS_WINDOWS: - pytest.skip("Windows support for cupti not yet implemented") - - # Create site-packages structure for CUDA 13 - site_packages = tmp_path / "site-packages" - cupti_dir = site_packages / "nvidia" / "cu13" / "lib" - cupti_dir.mkdir(parents=True) - cupti_lib = cupti_dir / "libcupti.so.13" - cupti_lib.write_bytes(b"fake") - - # Mock site-packages discovery - monkeypatch.setattr(site, "getsitepackages", lambda: [str(site_packages)]) - mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) - mocker.patch(f"{_MODULE}.load_dependencies") - mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) - mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=None) - mocker.patch(f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=None) - mocker.patch.object( - load_mod.LOADER, - "load_with_abs_path", - side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), - ) - - result = _load_lib_no_cache("cupti") - assert result.found_via == "site-packages" - assert result.abs_path == str(cupti_lib) - - # --------------------------------------------------------------------------- # Conda tests +# Note: Site-packages and CTK are covered by real CI tests. +# Mock tests focus on Conda (not covered by real CI) and error paths. # --------------------------------------------------------------------------- @@ -181,80 +95,10 @@ def _run_find_steps_without_site_packages(ctx, steps): # --------------------------------------------------------------------------- -# CTK via CUDA_HOME tests +# Error path tests # --------------------------------------------------------------------------- -def test_cupti_found_via_cuda_home(tmp_path, mocker): - """Test finding cupti via CUDA_HOME pointing to CTK installation.""" - if IS_WINDOWS: - pytest.skip("Windows support for cupti not yet implemented") - - ctk_root = tmp_path / "cuda-13.1" - cupti_lib = _create_cupti_in_ctk(ctk_root) - - # Mock CUDA_HOME discovery - disable early find steps - def _run_find_steps_without_early(ctx, steps): - if steps is EARLY_FIND_STEPS: - return None - return steps_mod.run_find_steps(ctx, steps) - - mocker.patch(f"{_MODULE}.run_find_steps", side_effect=_run_find_steps_without_early) - mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) - mocker.patch(f"{_MODULE}.load_dependencies") - mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=str(ctk_root)) - mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) - mocker.patch.object( - load_mod.LOADER, - "load_with_abs_path", - side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), - ) - - result = _load_lib_no_cache("cupti") - assert result.found_via == "CUDA_HOME" - # The finder resolves symlinks, so it finds libcupti.so (the symlink) not libcupti.so.13 - assert result.abs_path == str(cupti_lib.parent / "libcupti.so") - - -# --------------------------------------------------------------------------- -# CTK via canary probe tests -# --------------------------------------------------------------------------- - - -def test_cupti_found_via_canary_probe(tmp_path, mocker): - """Test finding cupti via CTK root canary probe.""" - if IS_WINDOWS: - pytest.skip("Windows support for cupti not yet implemented") - - ctk_root = tmp_path / "cuda-13.1" - _create_cudart_in_ctk(ctk_root) - cupti_lib = _create_cupti_in_ctk(ctk_root) - - # Mock canary probe discovery - disable all early/late find steps - def _run_find_steps_disabled(ctx, steps): - return None - - mocker.patch(f"{_MODULE}.run_find_steps", side_effect=_run_find_steps_disabled) - mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) - mocker.patch(f"{_MODULE}.load_dependencies") - mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) - mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=None) - mocker.patch( - f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", - return_value=_fake_canary_path(ctk_root), - ) - mocker.patch.object( - load_mod.LOADER, - "load_with_abs_path", - side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), - ) - - result = _load_lib_no_cache("cupti") - assert result.found_via == "system-ctk-root" - # The finder resolves symlinks, so it finds libcupti.so (the symlink) not libcupti.so.13 - assert result.abs_path == str(cupti_lib.parent / "libcupti.so") - - def test_cupti_not_found_raises_error(mocker): """Test that DynamicLibNotFoundError is raised when cupti is not found.""" if IS_WINDOWS: @@ -279,49 +123,16 @@ def _run_find_steps_disabled(ctx, steps): # --------------------------------------------------------------------------- -# Search order tests +# Search order tests (Conda-specific, since Conda is not covered by real CI) # --------------------------------------------------------------------------- -def test_cupti_search_order_site_packages_before_conda(tmp_path, mocker, monkeypatch): - """Test that site-packages is searched before conda.""" - if IS_WINDOWS: - pytest.skip("Windows support for cupti not yet implemented") - - # Create both site-packages and conda structures - site_packages = tmp_path / "site-packages" - site_cupti_dir = site_packages / "nvidia" / "cu13" / "lib" - site_cupti_dir.mkdir(parents=True) - site_cupti_lib = site_cupti_dir / "libcupti.so.13" - site_cupti_lib.write_bytes(b"fake") - - conda_prefix = tmp_path / "conda_env" - conda_lib_dir = conda_prefix / "lib" - conda_lib_dir.mkdir(parents=True) - conda_cupti_lib = conda_lib_dir / "libcupti.so.13" - conda_cupti_lib.write_bytes(b"fake") - - # Mock discovery - monkeypatch.setattr(site, "getsitepackages", lambda: [str(site_packages)]) - monkeypatch.setenv("CONDA_PREFIX", str(conda_prefix)) - mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) - mocker.patch(f"{_MODULE}.load_dependencies") - mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) - mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=None) - mocker.patch(f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", return_value=None) - mocker.patch.object( - load_mod.LOADER, - "load_with_abs_path", - side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), - ) - - result = _load_lib_no_cache("cupti") - assert result.found_via == "site-packages" - assert result.abs_path == str(site_cupti_lib) - - def test_cupti_search_order_conda_before_cuda_home(tmp_path, mocker, monkeypatch): - """Test that conda is searched before CUDA_HOME.""" + """Test that conda is searched before CUDA_HOME (CTK). + + This test is important because Conda is not covered by real CI tests, + so we need to verify the search order between Conda and CTK. + """ if IS_WINDOWS: pytest.skip("Windows support for cupti not yet implemented") @@ -360,45 +171,3 @@ def _run_find_steps_without_site_packages(ctx, steps): result = _load_lib_no_cache("cupti") assert result.found_via == "conda" assert result.abs_path == str(conda_cupti_lib) - - -def test_cupti_search_order_cuda_home_before_canary(tmp_path, mocker): - """Test that CUDA_HOME is searched before canary probe.""" - if IS_WINDOWS: - pytest.skip("Windows support for cupti not yet implemented") - - # Create two CTK roots: one for CUDA_HOME, one for canary - cuda_home_root = tmp_path / "cuda-home" - cuda_home_cupti = _create_cupti_in_ctk(cuda_home_root) - - canary_root = tmp_path / "cuda-system" - _create_cudart_in_ctk(canary_root) - _create_cupti_in_ctk(canary_root) - - # Mock discovery - disable early find steps - def _run_find_steps_without_early(ctx, steps): - if steps is load_mod.EARLY_FIND_STEPS: - return None - return steps_mod.run_find_steps(ctx, steps) - - mocker.patch(f"{_MODULE}.run_find_steps", side_effect=_run_find_steps_without_early) - mocker.patch.object(load_mod.LOADER, "check_if_already_loaded_from_elsewhere", return_value=None) - mocker.patch(f"{_MODULE}.load_dependencies") - mocker.patch.object(load_mod.LOADER, "load_with_system_search", return_value=None) - mocker.patch(f"{_STEPS_MODULE}.get_cuda_home_or_path", return_value=str(cuda_home_root)) - canary_mock = mocker.patch( - f"{_MODULE}._resolve_system_loaded_abs_path_in_subprocess", - return_value=_fake_canary_path(canary_root), - ) - mocker.patch.object( - load_mod.LOADER, - "load_with_abs_path", - side_effect=lambda _desc, path, via: _make_loaded_dl(path, via), - ) - - result = _load_lib_no_cache("cupti") - assert result.found_via == "CUDA_HOME" - # The finder resolves symlinks, so it finds libcupti.so (the symlink) not libcupti.so.13 - assert result.abs_path == str(cuda_home_cupti.parent / "libcupti.so") - # Canary should not have been called - canary_mock.assert_not_called() From ef4714797b198924ad889585a053c1e96443b02b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 6 Mar 2026 09:08:50 -0800 Subject: [PATCH 9/9] Add reverse=True to glob sorting --- .../cuda/pathfinder/_dynamic_libs/search_platform.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/search_platform.py b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/search_platform.py index 864f0f8a02..95e0f4dd1e 100644 --- a/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/search_platform.py +++ b/cuda_pathfinder/cuda/pathfinder/_dynamic_libs/search_platform.py @@ -148,7 +148,10 @@ def find_in_lib_dir( # Some libraries only exist as versioned files (e.g., libcupti.so.13 in conda), # so the glob fallback is needed file_wild = lib_searched_for + "*" - for so_name in sorted(glob.glob(os.path.join(lib_dir, file_wild))): + # Only one match is expected, but to ensure deterministic behavior in unexpected + # situations, and to be internally consistent, we sort in reverse order with the + # intent to return the newest version first. + for so_name in sorted(glob.glob(os.path.join(lib_dir, file_wild)), reverse=True): if os.path.isfile(so_name): return so_name error_messages.append(f"No such file: {file_wild}")