diff --git a/.gitignore b/.gitignore index b8eaa5ca208..6938baaaff4 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ __pycache__/ !*_impl.cpp !cuda_bindings/cuda/bindings/_lib/param_packer.cpp !cuda_bindings/cuda/bindings/_bindings/loader.cpp +!cuda_core/cuda/core/_cpp/**/*.cpp cache_driver cache_runtime cache_nvrtc diff --git a/ci/tools/merge_cuda_core_wheels.py b/ci/tools/merge_cuda_core_wheels.py index 23a8a21289f..6e7f51e73b6 100644 --- a/ci/tools/merge_cuda_core_wheels.py +++ b/ci/tools/merge_cuda_core_wheels.py @@ -133,9 +133,11 @@ def merge_wheels(wheels: list[Path], output_dir: Path, show_wheel_contents: bool # Copy version-specific directories from each wheel into versioned subdirectories base_dir = Path("cuda") / "core" + versioned_dirs = set() for i, wheel_dir in enumerate(extracted_wheels): cuda_version = wheels[i].name.split(".cu")[1].split(".")[0] versioned_dir = base_wheel / base_dir / f"cu{cuda_version}" + versioned_dirs.add(versioned_dir.name) # Copy entire directory tree from source wheel to versioned directory print(f" Copying {wheel_dir / base_dir} to {versioned_dir}", file=sys.stderr) @@ -145,25 +147,17 @@ def merge_wheels(wheels: list[Path], output_dir: Path, show_wheel_contents: bool os.truncate(versioned_dir / "__init__.py", 0) print("\n=== Removing files from cuda/core/ directory ===", file=sys.stderr) - items_to_keep = ( - "__init__.py", - "_version.py", - "_include", - "_cpp", # Headers for Cython development - "cu12", - "cu13", - ) - # _resource_handles is shared (not CUDA-version-specific) and must stay - # at top level. It's imported early in __init__.py before versioned code. - items_to_keep_prefix = ("_resource_handles",) + # Only what cuda/core/__init__.py uses before it rewrites __path__ to the + # versioned subpackage stays at top level: it imports _version, then + # redirects every later import into the versioned tree. Anything else + # left at top level is a dead copy that nothing imports. + items_to_keep = {"__init__.py", "_version.py", *versioned_dirs} all_items = os.scandir(base_wheel / base_dir) removed_count = 0 for f in all_items: f_abspath = f.path if f.name in items_to_keep: continue - if any(f.name.startswith(prefix) for prefix in items_to_keep_prefix): - continue if f.is_dir(): print(f" Removing directory: {f.name}", file=sys.stderr) shutil.rmtree(f_abspath) @@ -172,6 +166,11 @@ def merge_wheels(wheels: list[Path], output_dir: Path, show_wheel_contents: bool os.remove(f_abspath) removed_count += 1 print(f"Removed {removed_count} items from cuda/core/ directory", file=sys.stderr) + remaining = {entry.name for entry in os.scandir(base_wheel / base_dir)} + if remaining != items_to_keep: + raise RuntimeError( + f"unexpected top level under cuda/core/: {sorted(remaining)} (expected {sorted(items_to_keep)})" + ) # Repack the merged wheel output_dir.mkdir(parents=True, exist_ok=True) diff --git a/ci/tools/tests/test_compute_ci_plan.py b/ci/tools/tests/test_compute_ci_plan.py index 79a83394dfa..40d1ebd9431 100644 --- a/ci/tools/tests/test_compute_ci_plan.py +++ b/ci/tools/tests/test_compute_ci_plan.py @@ -113,7 +113,7 @@ def test_ignored_paths_select_no_work(self) -> None: "cuda_core/tests/fixtures/pixi.toml", "benchmarks/cuda_bindings/pixi.toml", "benchmarks/cuda_bindings/AGENTS.md", - "cuda_core/cuda/core/_cpp/DESIGN.md", + "cuda_core/cuda/core/_cpp/rt/DESIGN.md", "cuda_bindings/README.md", "cuda_core/README.md", "new-area/pixi.toml", diff --git a/cuda_core/AGENTS.md b/cuda_core/AGENTS.md index 9d80ab74aaa..cec6daf78a7 100644 --- a/cuda_core/AGENTS.md +++ b/cuda_core/AGENTS.md @@ -19,8 +19,9 @@ This file describes `cuda_core`, the high-level Pythonic CUDA subpackage in the - system-level APIs: `cuda/core/system/` - compile/link path: `_program.pyx`, `_linker.pyx`, `_module.pyx` - execution path: `_launcher.pyx`, `_launch_config.pyx`, `_stream.pyx` -- **C++ helpers**: module-specific C++ implementations live under - `cuda/core/_cpp/`. +- **C++ helpers**: module-specific C++ lives under `cuda/core/_cpp/`, either as + one `_cpp/.cpp` or as a directory `_cpp//` whose sources all + compile into the `_` extension (`_cpp/rt/` for `_rt`). - **Build backend**: `build_hooks.py` handles Cython extension setup and build dependency wiring. @@ -90,7 +91,7 @@ and agents should flag violations. objects that are not meant to be shared (e.g., the thread-local `Device`) do not need such guards (see #2321). Reference-count integrity is guaranteed; cache value-identity/idempotency is not. -- **Entry points assume the GIL is held**: the helpers in `_cpp/resource_handles.*` +- **Entry points assume the GIL is held**: the helpers in `_cpp/rt/` are called from Cython with the GIL held and do not re-acquire it. Driver and destructor callbacks run at arbitrary times, so they take the GIL (`with gil`) and probe for interpreter shutdown before touching Python objects. @@ -101,6 +102,66 @@ and agents should flag violations. (kernel arguments, memcpy/memset operands, `dst_owner`/`src_owner`, and host-callback closures) inherit this contract. +## Failure handling + +The user-facing contract lives in `docs/source/error_handling.rst`; the rules +below are for contributors. Reviewers and agents should flag violations. + +- **Raise by default**: any failure on a path where an exception can propagate + raises. Driver statuses go through `HANDLE_RETURN` (Cython) or are returned as + `CUresult` from the C++ handle layer and then `HANDLE_RETURN`ed; never + replace a `CUresult` with a generic `RuntimeError`, and drain + `get_last_error()` immediately after a handle constructor returns empty so a + stale status cannot be misattributed later. +- **Guarantees**: a call that creates a resource must create nothing when it + raises (undo the creation if a later step fails). Every call except + `Device.set_current` must leave the calling thread's current context as it + found it. Do not hand-roll `cuCtxPush/Pop/SetCurrent` sequences in Cython; use + the handle layer's scoped-context helpers (`invoke_in_context`, + `invoke_in_context_or_undo`, `cleanup_in_context`, `context_get_device`, + `graph_node_set_params`) so the failure handling exists in one place. +- **Publish before you raise**: when a driver mutation has succeeded and a later + step can still fail, commit whatever keeps that mutation memory-safe (for + example the graph attachment that retains a node's new owners) before raising + the later error. Rolling back the retention of a live mutation creates a + dangling reference. When ownership cannot be established, retain the + resources anyway (leak) rather than release them; a leak is always preferred + to a use-after-free. +- **Non-propagating paths never raise and never discard a status**: shared_ptr + deleters, `__dealloc__` and CUDA callbacks report through one channel, `report_cuda_error()` / `report_message()` in C++ (the + `pw_*` wrappers) or `warnings.warn(..., CUDAWarning)` in Cython and Python, + which emits `cuda.core.CUDAWarning`. No `print(file=sys.stderr)` and no + `fprintf` outside that helper. `CUDA_ERROR_DEINITIALIZED` is filtered by the + helper because it means the driver is shutting down. +- **Rollback failure**: the original exception propagates; the failed rollback + is attached to it with `note_or_report_cuda_error()` (a PEP 678 note on + Python 3.11+, reported out-of-band on 3.10), or chained with + `raise ... from` when a second exception must be raised. Catching everything + (bare `except:` or `except BaseException:`) is acceptable only for + rollback-then-`raise` blocks, where the rollback must also run for + `KeyboardInterrupt`. +- **Finalization**: once `py_is_finalizing()` is true, do no Python work from + destructors or callbacks and accept the leak (see + `_cpp/rt/py.hpp` and `_cpp/rt/GRAPH_ATTACHMENTS.md`). +- **Aborting**: `std::abort` (or any process termination) is reserved for an + internal invariant violation where continuing could corrupt memory or produce + silently wrong results *and* no leak-based fallback exists. A failed CUDA + call, including a failed context restoration, never qualifies: raise or + report instead. There is currently no such path; if one is ever needed it + must go through a single helper that writes a diagnostic (call, CUDA error, + invariant, "please report") and a Python traceback of all threads to stderr + before aborting (as `faulthandler` does, via the GIL-free + `_Py_DumpTracebackThreads`; no Python-object work), must never trigger + during interpreter finalization or for driver-shutdown errors, and must be + called out in the docs and release notes. An *implicit* abort (an exception + escaping a `noexcept` function or a deleter, including `std::bad_alloc` from + an allocation inside `noexcept` code) is a bug (#1489, #2417), not a policy + choice: `noexcept` helpers must not allocate, or must catch what they call. +- **Testing**: inject restoration failures with + `cuda.core._rt._set_context_restore_fault_for_testing`; assert + reports with `pytest.warns(CUDAWarning)` or `warnings.catch_warnings`, never + by matching stderr text. + ## API design guidelines These are some API design guidelines we try to follow when adding new APIs to diff --git a/cuda_core/build_hooks.py b/cuda_core/build_hooks.py index fc112b5a74e..47bd4e7c66e 100644 --- a/cuda_core/build_hooks.py +++ b/cuda_core/build_hooks.py @@ -183,6 +183,45 @@ def _relativize_extension_sources(extensions) -> None: ] +def _extension_sources(mod_name): + """The module's .pyx plus its C++, if any: every .cpp under + cuda/core/_cpp//, or the single legacy file cuda/core/_cpp/.cpp. + Example: _tensor_map.pyx compiles _cpp/tensor_map.cpp.""" + sources = [f"cuda/core/{mod_name}.pyx"] + cpp_stem = Path("cuda", "core", "_cpp", mod_name.lstrip("_")) + if cpp_stem.is_dir(): + cpp_sources = sorted(str(path) for path in cpp_stem.rglob("*.cpp")) + if not cpp_sources: + raise RuntimeError(f"{cpp_stem}/ exists but contains no .cpp files") + sources.extend(cpp_sources) + elif cpp_stem.with_suffix(".cpp").is_file(): + sources.append(str(cpp_stem.with_suffix(".cpp"))) + return sources + + +def _extension_depends(): + """Headers whose edits must rebuild an extension: every header under a + directory-form module's cuda/core/_cpp// (a single-file module has + none). + + The same list serves every extension. A module that cimports a + directory-form module compiles against the header its .pxd names, and + cythonize copies each `depends` entry into its build directory before + compiling, so the copied header finds its sibling includes beside it + (quoted includes resolve next to the copy, not in the source tree). + Listing the whole directory keeps the rule free of include parsing; the + cost is that every extension rebuilds when any of these headers changes, + exactly as editing the one monolithic header did before the split.""" + cpp = Path("cuda", "core", "_cpp") + return sorted( + str(path) + for module_dir in cpp.iterdir() + if module_dir.is_dir() + for path in module_dir.rglob("*") + if path.suffix in (".h", ".hpp") + ) + + def _build_cuda_core(debug=False): # Customizing the build hooks is needed because we must defer cythonization until cuda-bindings, # now a required build-time dependency that's dynamically installed via the other hook below, @@ -227,18 +266,6 @@ def module_names(): continue yield mod - def get_sources(mod_name): - """Get source files for a module, including any .cpp files.""" - sources = [f"cuda/core/{mod_name}.pyx"] - - # Add module-specific .cpp file from _cpp/ directory if it exists - # Example: _resource_handles.pyx finds _cpp/resource_handles.cpp. - cpp_file = f"cuda/core/_cpp/{mod_name.lstrip('_')}.cpp" - if os.path.exists(cpp_file): - sources.append(cpp_file) - - return sources - all_include_dirs = [os.path.join(cuda_path, "include")] extra_compile_args = [] extra_link_args = [] @@ -261,10 +288,12 @@ def get_sources(mod_name): # related to free-threading builds. extra_compile_args += ["-DCYTHON_TRACE_NOGIL=1", "-DCYTHON_USE_SYS_MONITORING=0"] + depends = _extension_depends() ext_modules = tuple( Extension( f"cuda.core.{mod.replace(os.path.sep, '.')}", - sources=get_sources(mod), + sources=_extension_sources(mod), + depends=depends, include_dirs=[ "cuda/core/_include", "cuda/core/_cpp", @@ -295,6 +324,10 @@ def get_sources(mod_name): # CUDA_PYTHON_COVERAGE deliberately generates in-tree so the sources can # be packaged; every other build gets its own per-configuration cache, # anchored alongside the stamp so both resolve the same from any cwd. + # Cython also copies each extension's extern headers and `depends` under + # this directory and compiles against the copies. Copies are refreshed by + # mtime and never deleted, so remove build/ after renaming or deleting a + # header under _cpp/. build_dir="." if COMPILE_FOR_COVERAGE else str(_BUILD_DIR / "cython" / f"cu{cuda_major}"), nthreads=nthreads, compiler_directives=compiler_directives, diff --git a/cuda_core/cuda/core/__init__.py b/cuda_core/cuda/core/__init__.py index 7864ae794ca..149fe327f12 100644 --- a/cuda_core/cuda/core/__init__.py +++ b/cuda_core/cuda/core/__init__.py @@ -102,8 +102,10 @@ class _PatchedProperty(metaclass=_PatchedPropMeta): from cuda.core._stream import __all__ as _stream_all from cuda.core._tensor_map import * from cuda.core._tensor_map import __all__ as _tensor_map_all +from cuda.core._utils.cuda_utils import CUDAWarning __all__ = [ + "CUDAWarning", *_context_all, *_device_all, *_device_resources_all, diff --git a/cuda_core/cuda/core/_context.pxd b/cuda_core/cuda/core/_context.pxd index 078c0c33415..3476fffb322 100644 --- a/cuda_core/cuda/core/_context.pxd +++ b/cuda_core/cuda/core/_context.pxd @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -from cuda.core._resource_handles cimport ContextHandle, GreenCtxHandle +from cuda.core._rt cimport ContextHandle, GreenCtxHandle cdef class Context: """Cython declaration for Context class. diff --git a/cuda_core/cuda/core/_context.pyx b/cuda_core/cuda/core/_context.pyx index 6855d0cc255..7f5fba9f80f 100644 --- a/cuda_core/cuda/core/_context.pyx +++ b/cuda_core/cuda/core/_context.pyx @@ -13,7 +13,7 @@ import cython from cuda.bindings cimport cydriver from cuda.core._device_resources cimport DeviceResources, SMResource, WorkqueueResource from cuda.core._device_resources import SMResource, WorkqueueResource -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( ContextHandle, GreenCtxHandle, as_cu, diff --git a/cuda_core/cuda/core/_cpp/resource_handles.cpp b/cuda_core/cuda/core/_cpp/resource_handles.cpp deleted file mode 100644 index 46f7b019379..00000000000 --- a/cuda_core/cuda/core/_cpp/resource_handles.cpp +++ /dev/null @@ -1,3098 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// SPDX-License-Identifier: Apache-2.0 - -#include - -#include "resource_handles.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifndef _WIN32 -#include -#endif - -namespace cuda_core { - -// ============================================================================ -// CUDA driver function pointers -// -// These are populated by _resource_handles.pyx at module import time using -// function pointers extracted from cuda.bindings.cydriver.__pyx_capi__. -// ============================================================================ - -decltype(&cuGetErrorName) p_cuGetErrorName = nullptr; -decltype(&cuGetErrorString) p_cuGetErrorString = nullptr; - -decltype(&cuDevicePrimaryCtxRetain) p_cuDevicePrimaryCtxRetain = nullptr; -decltype(&cuDevicePrimaryCtxRelease) p_cuDevicePrimaryCtxRelease = nullptr; -decltype(&cuCtxGetCurrent) p_cuCtxGetCurrent = nullptr; -decltype(&cuCtxSetCurrent) p_cuCtxSetCurrent = nullptr; -decltype(&cuCtxSynchronize) p_cuCtxSynchronize = nullptr; -decltype(&cuCtxGetStreamPriorityRange) p_cuCtxGetStreamPriorityRange = nullptr; -decltype(&cuGreenCtxCreate) p_cuGreenCtxCreate = nullptr; -decltype(&cuGreenCtxDestroy) p_cuGreenCtxDestroy = nullptr; -decltype(&cuCtxFromGreenCtx) p_cuCtxFromGreenCtx = nullptr; -decltype(&cuDevResourceGenerateDesc) p_cuDevResourceGenerateDesc = nullptr; - -decltype(&cuGreenCtxStreamCreate) p_cuGreenCtxStreamCreate = nullptr; - -decltype(&cuStreamCreateWithPriority) p_cuStreamCreateWithPriority = nullptr; -decltype(&cuStreamDestroy) p_cuStreamDestroy = nullptr; -decltype(&cuStreamGetCtx) p_cuStreamGetCtx = nullptr; - -decltype(&cuEventCreate) p_cuEventCreate = nullptr; -decltype(&cuEventDestroy) p_cuEventDestroy = nullptr; -decltype(&cuIpcOpenEventHandle) p_cuIpcOpenEventHandle = nullptr; - -decltype(&cuDeviceGetCount) p_cuDeviceGetCount = nullptr; - -decltype(&cuMemPoolSetAccess) p_cuMemPoolSetAccess = nullptr; -decltype(&cuMemPoolDestroy) p_cuMemPoolDestroy = nullptr; -decltype(&cuMemPoolCreate) p_cuMemPoolCreate = nullptr; -decltype(&cuDeviceGetMemPool) p_cuDeviceGetMemPool = nullptr; -decltype(&cuMemPoolImportFromShareableHandle) p_cuMemPoolImportFromShareableHandle = nullptr; - -decltype(&cuMemAllocFromPoolAsync) p_cuMemAllocFromPoolAsync = nullptr; -decltype(&cuMemAllocAsync) p_cuMemAllocAsync = nullptr; -decltype(&cuMemAlloc) p_cuMemAlloc = nullptr; -decltype(&cuMemAllocHost) p_cuMemAllocHost = nullptr; - -decltype(&cuMemFreeAsync) p_cuMemFreeAsync = nullptr; -decltype(&cuMemFree) p_cuMemFree = nullptr; -decltype(&cuMemFreeHost) p_cuMemFreeHost = nullptr; - -decltype(&cuMemPoolImportPointer) p_cuMemPoolImportPointer = nullptr; - -decltype(&cuLibraryLoadFromFile) p_cuLibraryLoadFromFile = nullptr; -decltype(&cuLibraryLoadData) p_cuLibraryLoadData = nullptr; -decltype(&cuLibraryUnload) p_cuLibraryUnload = nullptr; -decltype(&cuLibraryGetKernel) p_cuLibraryGetKernel = nullptr; - -// Graph -decltype(&cuGraphDestroy) p_cuGraphDestroy = nullptr; -decltype(&cuGraphInstantiateWithParams) p_cuGraphInstantiateWithParams = nullptr; -decltype(&cuGraphExecUpdate) p_cuGraphExecUpdate = nullptr; -decltype(&cuGraphExecDestroy) p_cuGraphExecDestroy = nullptr; -decltype(&cuUserObjectCreate) p_cuUserObjectCreate = nullptr; -decltype(&cuUserObjectRelease) p_cuUserObjectRelease = nullptr; -decltype(&cuGraphRetainUserObject) p_cuGraphRetainUserObject = nullptr; -decltype(&cuGraphReleaseUserObject) p_cuGraphReleaseUserObject = nullptr; -decltype(&cuGraphNodeFindInClone) p_cuGraphNodeFindInClone = nullptr; -decltype(&cuGraphChildGraphNodeGetGraph) p_cuGraphChildGraphNodeGetGraph = nullptr; - -// Linker -decltype(&cuLinkDestroy) p_cuLinkDestroy = nullptr; - -// GL interop pointers -decltype(&cuGraphicsUnmapResources) p_cuGraphicsUnmapResources = nullptr; -decltype(&cuGraphicsUnregisterResource) p_cuGraphicsUnregisterResource = nullptr; - -decltype(&cuArray3DCreate) p_cuArray3DCreate = nullptr; -decltype(&cuArrayDestroy) p_cuArrayDestroy = nullptr; -decltype(&cuMipmappedArrayCreate) p_cuMipmappedArrayCreate = nullptr; -decltype(&cuMipmappedArrayDestroy) p_cuMipmappedArrayDestroy = nullptr; -decltype(&cuMipmappedArrayGetLevel) p_cuMipmappedArrayGetLevel = nullptr; -decltype(&cuTexObjectCreate) p_cuTexObjectCreate = nullptr; -decltype(&cuTexObjectDestroy) p_cuTexObjectDestroy = nullptr; -decltype(&cuSurfObjectCreate) p_cuSurfObjectCreate = nullptr; -decltype(&cuSurfObjectDestroy) p_cuSurfObjectDestroy = nullptr; - -// SM resource split (13.1+ — may be null on older drivers/bindings) -#if CUDA_VERSION >= 13010 -decltype(&cuDevSmResourceSplit) p_cuDevSmResourceSplit = nullptr; -#else -void* p_cuDevSmResourceSplit = nullptr; -#endif - -// cuMemcpyWithAttributesAsync (13.2+ — may be null on older drivers/bindings) -#if CUDA_VERSION >= 13020 -decltype(&cuMemcpyWithAttributesAsync) p_cuMemcpyWithAttributesAsync = nullptr; -#else -void* p_cuMemcpyWithAttributesAsync = nullptr; -#endif - -// NVRTC function pointers -decltype(&nvrtcDestroyProgram) p_nvrtcDestroyProgram = nullptr; - -// NVVM function pointers (may be null if NVVM is not available) -NvvmDestroyProgramFn p_nvvmDestroyProgram = nullptr; - -// nvJitLink function pointers (may be null if nvJitLink is not available) -NvJitLinkDestroyFn p_nvJitLinkDestroy = nullptr; - -// ============================================================================ -// GIL and scoped-context management helpers -// ============================================================================ - -namespace { - -// Conditionally release the GIL while calling into the CUDA driver. -class GILReleaseGuard { -public: - GILReleaseGuard() noexcept { - if (!Py_IsInitialized() || py_is_finalizing()) { - return; - } - if (PyGILState_Check()) { - tstate_ = PyEval_SaveThread(); - } - } - - ~GILReleaseGuard() { - if (tstate_) { - PyEval_RestoreThread(tstate_); - } - } - - GILReleaseGuard(const GILReleaseGuard&) = delete; - GILReleaseGuard& operator=(const GILReleaseGuard&) = delete; - -private: - PyThreadState* tstate_ = nullptr; -}; - -// Helper to acquire the GIL when we might not hold it. -// Use in C++ destructors that need to manipulate Python objects. -class GILAcquireGuard { -public: - GILAcquireGuard() : acquired_(false) { - // Don't try to acquire GIL if Python is finalizing - if (!Py_IsInitialized() || py_is_finalizing()) { - return; - } - gstate_ = PyGILState_Ensure(); - acquired_ = true; - } - - ~GILAcquireGuard() { - if (acquired_) { - PyGILState_Release(gstate_); - } - } - - bool acquired() const { return acquired_; } - - // Non-copyable, non-movable - GILAcquireGuard(const GILAcquireGuard&) = delete; - GILAcquireGuard& operator=(const GILAcquireGuard&) = delete; - -private: - PyGILState_STATE gstate_; - bool acquired_; -}; - -void warn_on_cuda_error(const char* operation, CUresult status, const char* detail = nullptr) noexcept; - -// Make a context current and record the state needed to restore it. -// An empty handle is a no-op: the operation runs in the caller's current -// context, and nothing is restored on exit. -CUresult enter_context(const ContextHandle& h_context, CUcontext* previous, int* changed) noexcept { - *previous = nullptr; - *changed = 0; - CUcontext target = as_cu(h_context); - if (!target) { - return CUDA_SUCCESS; - } - - GILReleaseGuard gil; - CUresult status = p_cuCtxGetCurrent(previous); - if (status != CUDA_SUCCESS || *previous == target) { - return status; - } - status = p_cuCtxSetCurrent(target); - *changed = status == CUDA_SUCCESS; - return status; -} - -// Restore the previous context and preserve an earlier operation error. -CUresult exit_context(CUcontext previous, int changed, CUresult operation_status) noexcept { - CUresult restore_status = CUDA_SUCCESS; - if (changed) { - GILReleaseGuard gil; - restore_status = p_cuCtxSetCurrent(previous); - } - if (operation_status != CUDA_SUCCESS && restore_status != CUDA_SUCCESS) { - warn_on_cuda_error("cuCtxSetCurrent (restoring the caller's context)", restore_status); - } - return operation_status != CUDA_SUCCESS ? operation_status : restore_status; -} - -// Require a callable to be invocable without throwing. -#define ASSERT_NOTHROW_INVOCABLE(...) \ - static_assert(std::is_nothrow_invocable_v<__VA_ARGS__>, "operation must be noexcept") - -// Store a stream and any state needed to preserve deallocation ordering. -struct DeallocationStream { - StreamHandle h_stream; - std::thread::id ptds_tid{}; -}; - -// Return whether a stream handle needs a current context to resolve it. -bool is_default_stream(CUstream stream) noexcept { - return stream == nullptr || stream == CU_STREAM_LEGACY || stream == CU_STREAM_PER_THREAD; -} - -// Return the context a deallocation-stream token must run under. Real streams -// resolve their own context; default-stream tokens use the context bound at -// allocation time. Warn when PTDS deallocation crosses host threads. -ContextHandle deallocation_context(const DeallocationStream& stream) noexcept { - if (!is_default_stream(as_cu(stream.h_stream))) { - return {}; - } - if (stream.ptds_tid != std::thread::id{} - && stream.ptds_tid != std::this_thread::get_id()) { - std::fprintf( - stderr, - "Warning: Buffer deallocation for a per-thread default stream " - "is running on a different host thread than the one that recorded " - "the deallocation stream; ordering relative to the allocating " - "thread's PTDS is not preserved\n"); - } - return get_stream_context(stream.h_stream); -} - -// Run an operation with the requested context current. -template -CUresult invoke_in_context(const ContextHandle& h_context, Fn&& operation, Args&&... args) noexcept { - ASSERT_NOTHROW_INVOCABLE(Fn&&, Args&&...); - if (!h_context) { - return CUDA_ERROR_INVALID_CONTEXT; - } - CUcontext previous = nullptr; - int changed = 0; - CUresult status = enter_context(h_context, &previous, &changed); - if (status == CUDA_SUCCESS) { - status = std::invoke(std::forward(operation), std::forward(args)...); - } - return exit_context(previous, changed, status); -} - -// Run a creation operation and undo it if context restoration fails. -// Context-independent undo always runs. Context-sensitive undo runs only -// after verifying that the target context remains current; otherwise the -// resource leaks rather than risking cleanup in the wrong context. -template -CUresult invoke_in_context_or_undo(const ContextHandle& h_context, Fn&& operation, - Undo&& undo, bool undo_requires_target_context) noexcept { - ASSERT_NOTHROW_INVOCABLE(Fn&&); - ASSERT_NOTHROW_INVOCABLE(Undo&&); - if (!h_context) { - return CUDA_ERROR_INVALID_CONTEXT; - } - CUcontext previous = nullptr; - int changed = 0; - CUresult status = enter_context(h_context, &previous, &changed); - if (status != CUDA_SUCCESS) { - return status; - } - status = std::invoke(std::forward(operation)); - CUresult composite = exit_context(previous, changed, status); - if (status == CUDA_SUCCESS && composite != CUDA_SUCCESS) { - bool undo_ok = true; - if (undo_requires_target_context) { - CUcontext current = nullptr; - undo_ok = p_cuCtxGetCurrent(¤t) == CUDA_SUCCESS - && current == as_cu(h_context); - } - if (undo_ok) { - std::invoke(std::forward(undo)); - } else { - warn_on_cuda_error( - "cuCtxSetCurrent (restoring the caller's context)", composite, - "failed; cleanup of the new resource skipped because its context " - "is no longer current (resource leaked)"); - } - } - return composite; -} - -// Write a warning that includes the CUDA error name and description. -void warn_on_cuda_error(const char* operation, CUresult status, const char* detail) noexcept { - const char* error_name = nullptr; - const char* error_description = nullptr; - CUresult name_status = p_cuGetErrorName(status, &error_name); - CUresult description_status = p_cuGetErrorString(status, &error_description); - - if (name_status == CUDA_SUCCESS && description_status == CUDA_SUCCESS) { - if (detail) { - std::fprintf(stderr, "Warning: %s %s: %s: %s\n", - operation, detail, error_name, error_description); - } else { - std::fprintf(stderr, "Warning: %s failed: %s: %s\n", - operation, error_name, error_description); - } - } else { - if (detail) { - std::fprintf(stderr, "Warning: %s %s (CUDA error %d)\n", - operation, detail, static_cast(status)); - } else { - std::fprintf(stderr, "Warning: %s failed (CUDA error %d)\n", - operation, static_cast(status)); - } - } -} - -// Run cleanup with the requested context current. Warn and skip the operation -// if activation fails, and independently warn on operation or restoration -// failure. Return the operation or activation status; restoration never -// changes the return value. -template -CUresult cleanup_in_context(const ContextHandle& h_context, const char* name, - Fn&& operation, Args&&... args) noexcept { - ASSERT_NOTHROW_INVOCABLE(Fn&&, Args&&...); - CUcontext previous = nullptr; - int changed = 0; - CUresult status = enter_context(h_context, &previous, &changed); - if (status != CUDA_SUCCESS) { - warn_on_cuda_error(name, status, - "skipped (context activation failed; resource leaked)"); - } else { - status = std::invoke(std::forward(operation), std::forward(args)...); - if (status != CUDA_SUCCESS) { - warn_on_cuda_error(name, status); - } - } - CUresult restore = exit_context(previous, changed, CUDA_SUCCESS); - if (restore != CUDA_SUCCESS) { - warn_on_cuda_error(name, restore, "failed while restoring the caller's context"); - } - return status; -} - -#undef ASSERT_NOTHROW_INVOCABLE - -// Decorate a CUDA operation to warn whenever it returns an error. -template -class WarnOnFailure { -public: - explicit WarnOnFailure(const char* operation) noexcept : operation_(operation) {} - - template - CUresult operator()(Args&&... args) const noexcept { - CUresult status = Function(std::forward(args)...); - if (status != CUDA_SUCCESS) { - warn_on_cuda_error(operation_, status); - } - return status; - } - -private: - const char* operation_; -}; - -// Warning-decorated CUDA operations used by non-throwing cleanup paths. -const WarnOnFailure pw_cuStreamDestroy{"cuStreamDestroy"}; -const WarnOnFailure pw_cuEventDestroy{"cuEventDestroy"}; -const WarnOnFailure pw_cuMemFree{"cuMemFree"}; -const WarnOnFailure pw_cuMemFreeAsync{"cuMemFreeAsync"}; -const WarnOnFailure pw_cuArrayDestroy{"cuArrayDestroy"}; -const WarnOnFailure pw_cuMipmappedArrayDestroy{"cuMipmappedArrayDestroy"}; -const WarnOnFailure pw_cuTexObjectDestroy{"cuTexObjectDestroy"}; -const WarnOnFailure pw_cuSurfObjectDestroy{"cuSurfObjectDestroy"}; - -} // namespace - -// Synchronize the provided context. -CUresult context_synchronize(const ContextHandle& h_context) noexcept { - GILReleaseGuard gil; - return invoke_in_context(h_context, []() noexcept { - return p_cuCtxSynchronize(); - }); -} - -// Query the stream priority range for the provided context. -CUresult context_get_stream_priority_range(const ContextHandle& h_context, - int* least_priority, - int* greatest_priority) noexcept { - GILReleaseGuard gil; - return invoke_in_context(h_context, [&]() noexcept { - return p_cuCtxGetStreamPriorityRange(least_priority, greatest_priority); - }); -} - -// ============================================================================ -// CUDA user-object deferred cleanup -// -// CUDA invokes a user-object destructor on an internal thread where CUDA -// calls are forbidden. Payload cleanup can release resource handles whose -// deleters call CUDA, so the callback only transfers a preallocated intrusive -// node to this process-lifetime queue. One coalesced pending call drains all -// queued payloads from Python's main thread. -// ============================================================================ - -// Intrusive base for payloads transferred out of CUDA's callback. -struct DeferredCleanupItem { - DeferredCleanupItem* next = nullptr; - virtual ~DeferredCleanupItem() noexcept = default; -}; - -namespace { - -// Process-lifetime MPSC queue that drains payloads from Python's main thread. -class DeferredCleanupQueue { -public: - // Transfer one preallocated cleanup item from a producer to the queue. - void enqueue(DeferredCleanupItem* item) noexcept { - DeferredCleanupItem* head = head_.load(std::memory_order_relaxed); - do { - item->next = head; - } while (!head_.compare_exchange_weak( - head, item, std::memory_order_release, std::memory_order_relaxed)); - schedule(); - } - - // Permanently disable pending-call scheduling during interpreter shutdown. - void stop() noexcept { - accepting_.store(false, std::memory_order_release); - } - - // Reattempt scheduling after Py_AddPendingCall() found CPython's bounded - // pending-call queue full and left payloads queued for a later safe entry. - void retry_schedule() noexcept { - schedule(); - } - -private: - // Adapt queue draining to CPython's int (*)(void*) callback ABI. - static int pending_call(void* arg) noexcept { - static_cast(arg)->drain(); - return 0; - } - - // Coalesce all queued work behind at most one CPython pending call. - void schedule() noexcept { - if (!accepting_.load(std::memory_order_acquire)) { - return; - } - if (!Py_IsInitialized() || py_is_finalizing()) { - stop(); - return; - } - if (!head_.load(std::memory_order_acquire)) { - return; - } - bool expected = false; - if (!scheduled_.compare_exchange_strong( - expected, true, std::memory_order_acq_rel, - std::memory_order_relaxed)) { - return; - } - if (Py_AddPendingCall(&DeferredCleanupQueue::pending_call, this) != 0) { - // Keep every payload queued. A later enqueue or safe cuda-core - // entry can retry without blocking CUDA's callback thread. - scheduled_.store(false, std::memory_order_release); - } - } - - // Detach and destroy all queued payloads from Python's main thread. - void drain() noexcept { - if (!Py_IsInitialized() || py_is_finalizing()) { - stop(); - scheduled_.store(false, std::memory_order_release); - return; // Intentionally leak intact payloads during shutdown. - } - - while (DeferredCleanupItem* list = - head_.exchange(nullptr, std::memory_order_acquire)) { - while (list) { - DeferredCleanupItem* next = list->next; - delete list; - list = next; - } - } - - scheduled_.store(false, std::memory_order_release); - if (head_.load(std::memory_order_acquire)) { - schedule(); - } - } - - // Head of the intrusive multi-producer, single-consumer payload stack. - std::atomic head_{nullptr}; - // True while one cuda-core drain callback is pending or executing. - std::atomic scheduled_{false}; - // False once shutdown begins, causing later payloads to be leaked safely. - std::atomic accepting_{true}; -}; - -// Published once at module initialization and intentionally never freed. -std::atomic deferred_cleanup_queue{nullptr}; - -void ensure_deferred_cleanup_ready() { - DeferredCleanupQueue* queue = - deferred_cleanup_queue.load(std::memory_order_acquire); - if (!queue) { - throw std::runtime_error("deferred cleanup is not initialized"); - } - queue->retry_schedule(); -} - -// CUDA's CUhostFn ABI is void (*)(void*); recover and enqueue the cleanup item. -void enqueue_cleanup(void* item) noexcept { - auto* cleanup = static_cast(item); - if (DeferredCleanupQueue* queue = - deferred_cleanup_queue.load(std::memory_order_acquire)) { - queue->enqueue(cleanup); - } -} - -} // namespace - -// Module initialization calls this once with the GIL held, which serializes -// the check, allocation, and publication below. -void initialize_deferred_cleanup() { - if (deferred_cleanup_queue.load(std::memory_order_acquire)) { - return; - } - auto* queue = new DeferredCleanupQueue(); - deferred_cleanup_queue.store(queue, std::memory_order_release); -} - -void retry_deferred_cleanup() noexcept { - if (!Py_IsInitialized() || py_is_finalizing()) { - return; - } - if (DeferredCleanupQueue* queue = - deferred_cleanup_queue.load(std::memory_order_acquire)) { - queue->retry_schedule(); - } -} - -// ============================================================================ -// Handle reverse-lookup registry -// -// Maps raw CUDA handles (CUevent, CUkernel, etc.) back to their owning -// shared_ptr so that _ref constructors can recover full metadata. -// Uses weak_ptr to avoid preventing destruction. -// ============================================================================ - -template> -class HandleRegistry { -public: - using MapType = std::unordered_map, Hash>; - - void register_handle(const Key& key, const Handle& h) { - std::lock_guard lock(mutex_); - map_[key] = h; - } - - void unregister_handle(const Key& key) noexcept { - std::lock_guard lock(mutex_); - map_.erase(key); - } - - void register_handles(const std::vector& handles) { - std::lock_guard lock(mutex_); - for (const Handle& h : handles) { - if (h) { - map_[*h] = h; - } - } - } - - Handle lookup(const Key& key) { - std::lock_guard lock(mutex_); - auto it = map_.find(key); - if (it != map_.end()) { - if (auto h = it->second.lock()) { - return h; - } - map_.erase(it); - } - return {}; - } - - template - Handle get_or_create(const Key& key, Factory&& create) { - std::lock_guard lock(mutex_); - auto it = map_.find(key); - if (it != map_.end()) { - if (Handle h = it->second.lock()) { - return h; - } - map_.erase(it); - } - - Handle h = create(); - if (h) { - map_[key] = h; - } - return h; - } - - MapType drain() noexcept { - std::lock_guard lock(mutex_); - MapType extracted; - extracted.swap(map_); - return extracted; - } - -private: - std::mutex mutex_; - MapType map_; -}; - -// ============================================================================ -// Thread-local error handling -// ============================================================================ - -// Thread-local status of the most recent CUDA API call in this module. -static thread_local CUresult err = CUDA_SUCCESS; - -// Return and clear the calling thread's most recent CUDA error. -CUresult get_last_error() noexcept { - CUresult e = err; - err = CUDA_SUCCESS; - return e; -} - -// Return the calling thread's most recent CUDA error without clearing it. -CUresult peek_last_error() noexcept { - return err; -} - -void clear_last_error() noexcept { - err = CUDA_SUCCESS; -} - -// ============================================================================ -// Context Handles -// ============================================================================ - -namespace { -struct ContextBox { - CUcontext resource; - GreenCtxHandle h_green_ctx; -}; - -struct GreenCtxBox { - CUgreenCtx resource; -}; - -static const ContextBox* get_box(const ContextHandle& h) noexcept { - const CUcontext* p = h.get(); - return reinterpret_cast( - reinterpret_cast(p) - offsetof(ContextBox, resource) - ); -} - -// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) -static HandleRegistry context_registry; - -// Create a context handle reference, with optional green context as source. -ContextHandle create_context_handle_ref(CUcontext ctx, GreenCtxHandle h_green_ctx) { - if (!ctx) { - return {}; - } - if (auto h = context_registry.lookup(ctx)) { - return h; - } - auto box = std::shared_ptr( - new ContextBox{ctx, std::move(h_green_ctx)}, - [](const ContextBox* b) { - context_registry.unregister_handle(b->resource); - delete b; - } - ); - ContextHandle h(box, &box->resource); - context_registry.register_handle(ctx, h); - return h; -} -} // namespace - -ContextHandle create_context_handle_ref(CUcontext ctx) { - return create_context_handle_ref(ctx, {}); -} - -ContextHandle create_context_handle_from_green_ctx(const GreenCtxHandle& h_green_ctx) { - GILReleaseGuard gil; - if (!h_green_ctx) { - return {}; - } - if (!p_cuCtxFromGreenCtx) { - err = CUDA_ERROR_NOT_SUPPORTED; - return {}; - } - - CUcontext ctx = nullptr; - if (CUDA_SUCCESS != (err = p_cuCtxFromGreenCtx(&ctx, as_cu(h_green_ctx)))) { - return {}; - } - - return create_context_handle_ref(ctx, h_green_ctx); -} - -GreenCtxHandle get_context_green_ctx(const ContextHandle& h) noexcept { - if (!h) { - return {}; - } - return get_box(h)->h_green_ctx; -} - -GreenCtxHandle create_green_ctx_handle(CUdevResource* resources, unsigned int nbResources, - CUdevice dev, unsigned int flags) { - GILReleaseGuard gil; - if (!p_cuDevResourceGenerateDesc || !p_cuGreenCtxCreate || !p_cuGreenCtxDestroy) { - err = CUDA_ERROR_NOT_SUPPORTED; - return {}; - } - - CUdevResourceDesc desc = nullptr; - if (CUDA_SUCCESS != (err = p_cuDevResourceGenerateDesc(&desc, resources, nbResources))) { - return {}; - } - - CUgreenCtx green_ctx = nullptr; - if (CUDA_SUCCESS != (err = p_cuGreenCtxCreate(&green_ctx, desc, dev, flags))) { - return {}; - } - - auto box = std::shared_ptr( - new GreenCtxBox{green_ctx}, - [](const GreenCtxBox* b) { - GILReleaseGuard gil; - p_cuGreenCtxDestroy(b->resource); - delete b; - } - ); - return GreenCtxHandle(box, &box->resource); -} - -GreenCtxHandle create_green_ctx_handle_ref(CUgreenCtx green_ctx) { - if (!green_ctx) { - return {}; - } - auto box = std::make_shared(GreenCtxBox{green_ctx}); - return GreenCtxHandle(box, &box->resource); -} - -// Thread-local cache of primary contexts indexed by device ID -static thread_local std::vector primary_context_cache; - -ContextHandle get_primary_context(int device_id) { - // Check thread-local cache - if (static_cast(device_id) < primary_context_cache.size()) { - if (auto cached = primary_context_cache[device_id]) { - return cached; - } - } - - // Cache miss - acquire primary context from driver - GILReleaseGuard gil; - CUcontext ctx; - if (CUDA_SUCCESS != (err = p_cuDevicePrimaryCtxRetain(&ctx, device_id))) { - return {}; - } - - auto box = std::shared_ptr( - new ContextBox{ctx, {}}, - [device_id](const ContextBox* b) { - context_registry.unregister_handle(b->resource); - GILReleaseGuard gil; - p_cuDevicePrimaryCtxRelease(device_id); - delete b; - } - ); - auto h = ContextHandle(box, &box->resource); - context_registry.register_handle(ctx, h); - - // Update cache - if (static_cast(device_id) >= primary_context_cache.size()) { - primary_context_cache.resize(device_id + 1); - } - primary_context_cache[device_id] = h; - return h; -} - -ContextHandle get_current_context() { - GILReleaseGuard gil; - CUcontext ctx = nullptr; - if (CUDA_SUCCESS != (err = p_cuCtxGetCurrent(&ctx))) { - return {}; - } - if (!ctx) { - return {}; // No current context (not an error) - } - return create_context_handle_ref(ctx); -} - -// ============================================================================ -// Stream Handles -// ============================================================================ - -namespace { -struct StreamBox { - CUstream resource; - ContextHandle h_context; -}; - -static const StreamBox* get_box(const StreamHandle& h) noexcept { - const CUstream* p = h.get(); - return reinterpret_cast( - reinterpret_cast(p) - offsetof(StreamBox, resource) - ); -} - -// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) -static HandleRegistry stream_registry; -} // namespace - -StreamHandle create_stream_handle(const ContextHandle& h_ctx, unsigned int flags, int priority) { - GILReleaseGuard gil; - CUstream stream = nullptr; - GreenCtxHandle h_green = get_context_green_ctx(h_ctx); - if (h_green) { - err = p_cuGreenCtxStreamCreate - ? p_cuGreenCtxStreamCreate(&stream, as_cu(h_green), flags, priority) - : CUDA_ERROR_NOT_SUPPORTED; - } else { - err = invoke_in_context_or_undo( - h_ctx, - [&]() noexcept { return p_cuStreamCreateWithPriority(&stream, flags, priority); }, - [&]() noexcept { pw_cuStreamDestroy(stream); }, - /*undo_requires_target_context=*/false); - } - if (err != CUDA_SUCCESS) { - return {}; - } - - auto box = std::shared_ptr( - new StreamBox{stream, h_ctx}, - [](const StreamBox* b) { - stream_registry.unregister_handle(b->resource); - GILReleaseGuard gil; - pw_cuStreamDestroy(b->resource); - delete b; - } - ); - StreamHandle h(box, &box->resource); - stream_registry.register_handle(stream, h); - return h; -} - -StreamHandle create_stream_handle_ref(CUstream stream) { - if (auto h = stream_registry.lookup(stream)) { - return h; - } - auto box = std::shared_ptr( - new StreamBox{stream, {}}, - [](const StreamBox* b) { - stream_registry.unregister_handle(b->resource); - delete b; - } - ); - StreamHandle h(box, &box->resource); - stream_registry.register_handle(stream, h); - return h; -} - -StreamHandle create_stream_handle_with_owner(CUstream stream, PyObject* owner) { - if (auto h = stream_registry.lookup(stream)) { - // Reuse handles that already carry structural context metadata, e.g. - // cuda-core-owned streams. - if (get_box(h)->h_context) { - return h; - } - } - if (!owner) { - return create_stream_handle_ref(stream); - } - // GIL required when owner is provided - GILAcquireGuard gil; - if (!gil.acquired()) { - // Python finalizing - fall back to ref version (no owner tracking) - return create_stream_handle_ref(stream); - } - Py_INCREF(owner); - // Owner-backed handles are NOT registered in the stream registry to avoid - // corruption when multiple owners wrap the same CUstream (each stacks its - // own Py_INCREF/Py_DECREF independently). - auto box = std::shared_ptr( - new StreamBox{stream, {}}, - [owner](const StreamBox* b) { - GILAcquireGuard gil; - if (gil.acquired()) { - Py_DECREF(owner); - } - delete b; - } - ); - return StreamHandle(box, &box->resource); -} - -void py_object_user_object_destroy(void* py_object) noexcept { - if (!py_object) { - return; - } - GILAcquireGuard gil; - if (!gil.acquired()) { - return; - } - Py_DECREF(reinterpret_cast(py_object)); -} - -// Return the context retained by a stream handle. -ContextHandle get_stream_context(const StreamHandle& h) noexcept { - return h ? get_box(h)->h_context : ContextHandle{}; -} - -StreamHandle get_legacy_stream() { - static StreamHandle handle = create_stream_handle_ref(CU_STREAM_LEGACY); - return handle; -} - -StreamHandle get_per_thread_stream() { - static StreamHandle handle = create_stream_handle_ref(CU_STREAM_PER_THREAD); - return handle; -} - -StreamHandle create_context_bound_legacy_stream(const ContextHandle& h_context) { - if (!h_context) { - return {}; - } - // Default deleter: this handle never owns CU_STREAM_LEGACY, so nothing - // needs to run when the last reference is released. - auto box = std::make_shared(StreamBox{CU_STREAM_LEGACY, h_context}); - return StreamHandle(box, &box->resource); -} - -// ============================================================================ -// Deallocation streams -// -// A DeallocationStream is a StreamHandle used for ordering frees. It differs -// from an ordinary StreamHandle only for default-stream tokens, for which it -// stores the (de)allocation context. Ordinarily, the LEGACY and PER_THREAD -// default streams resolve to whichever context is active at the time they are -// used, but for storing deallocation recipes we need to pin the context. With -// the PER_THREAD token, it is not possible to restore the original stream when -// deallocation runs on a different thread. Therefore, in that case the -// allocating host thread id is also stored so that cross-thread frees can be -// detected and warnings can be issued. -// ============================================================================ - -// Real streams are copied unchanged. Default-stream tokens without an embedded -// context are bound to the current context. Returns false (and sets err) when a -// default-stream token cannot be bound because no context is current. -static bool make_deallocation_stream( - const StreamHandle& h, DeallocationStream& out) noexcept { - out = {}; - if (!h) { - return true; - } - - const CUstream stream = as_cu(h); - if (!is_default_stream(stream)) { - out = DeallocationStream{h, {}}; - return true; - } - - StreamHandle h_bound = h; - if (!get_stream_context(h)) { - ContextHandle h_ctx = get_current_context(); - if (!h_ctx) { - if (err == CUDA_SUCCESS) { - err = CUDA_ERROR_INVALID_CONTEXT; - } - return false; - } - // Do not register in stream_registry: the token value alone is not - // a unique stream identity (context is part of the meaning). - auto box = std::shared_ptr( - new StreamBox{stream, h_ctx}); - h_bound = StreamHandle(box, &box->resource); - } - - std::thread::id ptds_tid{}; - if (stream == CU_STREAM_PER_THREAD) { - ptds_tid = std::this_thread::get_id(); - } - out = DeallocationStream{std::move(h_bound), ptds_tid}; - return true; -} - -// ============================================================================ -// Event Handles -// ============================================================================ - -namespace { -struct EventBox { - CUevent resource; - bool timing_enabled; - bool is_blocking_sync; - bool ipc_enabled; - int device_id; - ContextHandle h_context; -}; -} // namespace - -static const EventBox* get_box(const EventHandle& h) { - const CUevent* p = h.get(); - return reinterpret_cast( - reinterpret_cast(p) - offsetof(EventBox, resource) - ); -} - -bool get_event_timing_enabled(const EventHandle& h) noexcept { - return h ? get_box(h)->timing_enabled : false; -} - -bool get_event_is_blocking_sync(const EventHandle& h) noexcept { - return h ? get_box(h)->is_blocking_sync : false; -} - -bool get_event_ipc_enabled(const EventHandle& h) noexcept { - return h ? get_box(h)->ipc_enabled : false; -} - -int get_event_device_id(const EventHandle& h) noexcept { - return h ? get_box(h)->device_id : -1; -} - -// Return the context retained by an event handle. -ContextHandle get_event_context(const EventHandle& h) noexcept { - return h ? get_box(h)->h_context : ContextHandle{}; -} - -// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) -static HandleRegistry event_registry; - -EventHandle create_event_handle(const ContextHandle& h_ctx, unsigned int flags, - bool timing_enabled, bool is_blocking_sync, - bool ipc_enabled, int device_id) { - GILReleaseGuard gil; - CUevent event = nullptr; - err = invoke_in_context_or_undo( - h_ctx, - [&]() noexcept { return p_cuEventCreate(&event, flags); }, - [&]() noexcept { pw_cuEventDestroy(event); }, - /*undo_requires_target_context=*/false); - if (err != CUDA_SUCCESS) { - return {}; - } - - auto box = std::shared_ptr( - new EventBox{event, timing_enabled, is_blocking_sync, ipc_enabled, device_id, h_ctx}, - [](const EventBox* b) { - event_registry.unregister_handle(b->resource); - GILReleaseGuard gil; - pw_cuEventDestroy(b->resource); - delete b; - } - ); - EventHandle h(box, &box->resource); - event_registry.register_handle(event, h); - return h; -} - -EventHandle create_event_handle_for_stream(CUstream stream, unsigned int flags) { - // Resolve the stream's owning context (for default-stream tokens this is - // the current context, per cuStreamGetCtx) and create the event there, so - // it can be recorded on `stream` no matter which context is current. - CUcontext ctx = nullptr; - { - GILReleaseGuard gil; - err = p_cuStreamGetCtx(stream, &ctx); - } - if (err != CUDA_SUCCESS) { - return {}; - } - if (!ctx) { - err = CUDA_ERROR_INVALID_CONTEXT; - return {}; - } - return create_event_handle(create_context_handle_ref(ctx), flags, false, false, false, -1); -} - -EventHandle create_event_handle_ref(CUevent event) { - if (auto h = event_registry.lookup(event)) { - return h; - } - auto box = std::make_shared(EventBox{event, false, false, false, -1, {}}); - return EventHandle(box, &box->resource); -} - -EventHandle create_event_handle_ipc(const CUipcEventHandle& ipc_handle, - bool is_blocking_sync) { - GILReleaseGuard gil; - CUevent event; - if (CUDA_SUCCESS != (err = p_cuIpcOpenEventHandle(&event, ipc_handle))) { - return {}; - } - - auto box = std::shared_ptr( - new EventBox{event, false, is_blocking_sync, true, -1, {}}, - [](const EventBox* b) { - event_registry.unregister_handle(b->resource); - GILReleaseGuard gil; - pw_cuEventDestroy(b->resource); - delete b; - } - ); - EventHandle h(box, &box->resource); - event_registry.register_handle(event, h); - return h; -} - -// ============================================================================ -// Memory Pool Handles -// ============================================================================ - -namespace { -struct MemoryPoolBox { - CUmemoryPool resource; -}; -} // namespace - -// Helper to clear peer access before destroying a memory pool. -// Works around nvbug 5698116: recycled pool handles inherit peer access state. -// Must be noexcept since it's called from a shared_ptr deleter. -static void clear_mempool_peer_access(CUmemoryPool pool) noexcept { - try { - int device_count = 0; - if (p_cuDeviceGetCount(&device_count) != CUDA_SUCCESS || device_count <= 0) { - return; - } - - std::vector clear_access(device_count); - for (int i = 0; i < device_count; ++i) { - clear_access[i].location.type = CU_MEM_LOCATION_TYPE_DEVICE; - clear_access[i].location.id = i; - clear_access[i].flags = CU_MEM_ACCESS_FLAGS_PROT_NONE; - } - p_cuMemPoolSetAccess(pool, clear_access.data(), device_count); // Best effort - } catch (...) { - // Swallow exceptions - this is best-effort cleanup in destructor context - } -} - -static MemoryPoolHandle wrap_mempool_owned(CUmemoryPool pool) { - auto box = std::shared_ptr( - new MemoryPoolBox{pool}, - [](const MemoryPoolBox* b) { - GILReleaseGuard gil; - clear_mempool_peer_access(b->resource); - p_cuMemPoolDestroy(b->resource); - delete b; - } - ); - return MemoryPoolHandle(box, &box->resource); -} - -MemoryPoolHandle create_mempool_handle(const CUmemPoolProps& props) { - GILReleaseGuard gil; - CUmemoryPool pool; - if (CUDA_SUCCESS != (err = p_cuMemPoolCreate(&pool, &props))) { - return {}; - } - return wrap_mempool_owned(pool); -} - -MemoryPoolHandle create_mempool_handle_ref(CUmemoryPool pool) { - auto box = std::make_shared(MemoryPoolBox{pool}); - return MemoryPoolHandle(box, &box->resource); -} - -MemoryPoolHandle get_device_mempool(int device_id) { - GILReleaseGuard gil; - CUmemoryPool pool; - if (CUDA_SUCCESS != (err = p_cuDeviceGetMemPool(&pool, device_id))) { - return {}; - } - return create_mempool_handle_ref(pool); -} - -MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type) { - GILReleaseGuard gil; - CUmemoryPool pool; - auto handle_ptr = reinterpret_cast(static_cast(fd)); - if (CUDA_SUCCESS != (err = p_cuMemPoolImportFromShareableHandle(&pool, handle_ptr, handle_type, 0))) { - return {}; - } - return wrap_mempool_owned(pool); -} - -// ============================================================================ -// Device Pointer Handles -// ============================================================================ - -namespace { -struct DevicePtrBox { - CUdeviceptr resource; - // Mutable so set_deallocation_stream() can update free ordering through a - // const DevicePtrHandle. Built with make_deallocation_stream so default- - // stream tokens carry a bound context. - mutable DeallocationStream deallocation; -}; -} // namespace - -// Recovers the owning DevicePtrBox from the aliased CUdeviceptr pointer. -// This works because DevicePtrHandle is a shared_ptr alias pointing to -// &box->resource, so we can compute the containing struct using offsetof. -// The const_cast is safe because we only use this to access the mutable -// deallocation member or in the deleter (where the box is being destroyed). -static DevicePtrBox* get_box(const DevicePtrHandle& h) { - const CUdeviceptr* p = h.get(); - return reinterpret_cast( - reinterpret_cast(const_cast(p)) - offsetof(DevicePtrBox, resource) - ); -} - -// Return the stream that orders a device pointer's deallocation. -StreamHandle deallocation_stream(const DevicePtrHandle& h) noexcept { - return get_box(h)->deallocation.h_stream; -} - -// Replace the stream that orders a device pointer's deallocation. -CUresult set_deallocation_stream(const DevicePtrHandle& h, const StreamHandle& h_stream) noexcept { - if (!h) { - return CUDA_ERROR_INVALID_VALUE; - } - DeallocationStream ds; - if (!make_deallocation_stream(h_stream, ds)) { - return err != CUDA_SUCCESS ? err : CUDA_ERROR_INVALID_CONTEXT; - } - get_box(h)->deallocation = std::move(ds); - return CUDA_SUCCESS; -} - -DevicePtrHandle deviceptr_alloc_from_pool(size_t size, const MemoryPoolHandle& h_pool, const StreamHandle& h_stream) { - GILReleaseGuard gil; - CUdeviceptr ptr; - if (CUDA_SUCCESS != (err = p_cuMemAllocFromPoolAsync(&ptr, size, *h_pool, as_cu(h_stream)))) { - return {}; - } - - DeallocationStream ds; - if (!make_deallocation_stream(h_stream, ds)) { - pw_cuMemFreeAsync(ptr, as_cu(h_stream)); - return {}; - } - - auto box = std::shared_ptr( - new DevicePtrBox{ptr, std::move(ds)}, - [h_pool](DevicePtrBox* b) { - GILReleaseGuard gil; - const DeallocationStream& stream = b->deallocation; - cleanup_in_context( - deallocation_context(stream), "cuMemFreeAsync", - [&]() noexcept { - return p_cuMemFreeAsync( - b->resource, as_cu(stream.h_stream)); - }); - delete b; - } - ); - return DevicePtrHandle(box, &box->resource); -} - -DevicePtrHandle deviceptr_alloc_async(size_t size, const StreamHandle& h_stream) { - GILReleaseGuard gil; - CUdeviceptr ptr; - if (CUDA_SUCCESS != (err = p_cuMemAllocAsync(&ptr, size, as_cu(h_stream)))) { - return {}; - } - - DeallocationStream ds; - if (!make_deallocation_stream(h_stream, ds)) { - pw_cuMemFreeAsync(ptr, as_cu(h_stream)); - return {}; - } - - auto box = std::shared_ptr( - new DevicePtrBox{ptr, std::move(ds)}, - [](DevicePtrBox* b) { - GILReleaseGuard gil; - const DeallocationStream& stream = b->deallocation; - cleanup_in_context( - deallocation_context(stream), "cuMemFreeAsync", - [&]() noexcept { - return p_cuMemFreeAsync( - b->resource, as_cu(stream.h_stream)); - }); - delete b; - } - ); - return DevicePtrHandle(box, &box->resource); -} - -// Allocate device memory synchronously with the provided context current. -CUresult deviceptr_alloc_raw(CUdeviceptr* ptr, size_t size, - const ContextHandle& h_context) noexcept { - GILReleaseGuard gil; - return invoke_in_context_or_undo( - h_context, - [&]() noexcept { return p_cuMemAlloc(ptr, size); }, - [&]() noexcept { pw_cuMemFree(*ptr); }, - /*undo_requires_target_context=*/false); -} - -DevicePtrHandle deviceptr_alloc_host(size_t size) { - GILReleaseGuard gil; - void* ptr; - if (CUDA_SUCCESS != (err = p_cuMemAllocHost(&ptr, size))) { - return {}; - } - - auto box = std::shared_ptr( - new DevicePtrBox{reinterpret_cast(ptr), DeallocationStream{}}, - [](DevicePtrBox* b) { - GILReleaseGuard gil; - p_cuMemFreeHost(reinterpret_cast(b->resource)); - delete b; - } - ); - return DevicePtrHandle(box, &box->resource); -} - -DevicePtrHandle deviceptr_create_ref(CUdeviceptr ptr) { - auto box = std::make_shared(DevicePtrBox{ptr, DeallocationStream{}}); - return DevicePtrHandle(box, &box->resource); -} - -DevicePtrHandle deviceptr_create_with_owner(CUdeviceptr ptr, PyObject* owner) { - if (!owner) { - return deviceptr_create_ref(ptr); - } - // GIL required when owner is provided - GILAcquireGuard gil; - if (!gil.acquired()) { - // Python finalizing - fall back to ref version (no owner tracking) - return deviceptr_create_ref(ptr); - } - Py_INCREF(owner); - auto box = std::shared_ptr( - new DevicePtrBox{ptr, DeallocationStream{}}, - [owner](DevicePtrBox* b) { - GILAcquireGuard gil; - if (gil.acquired()) { - Py_DECREF(owner); - } - delete b; - } - ); - return DevicePtrHandle(box, &box->resource); -} - -DevicePtrHandle deviceptr_create_mapped_graphics( - CUdeviceptr ptr, - const GraphicsResourceHandle& h_resource, - const StreamHandle& h_stream -) { - DeallocationStream ds; - if (!make_deallocation_stream(h_stream, ds)) { - return {}; - } - auto box = std::shared_ptr( - new DevicePtrBox{ptr, std::move(ds)}, - [h_resource](DevicePtrBox* b) { - GILReleaseGuard gil; - CUgraphicsResource resource = as_cu(h_resource); - const DeallocationStream& stream = b->deallocation; - cleanup_in_context( - deallocation_context(stream), "cuGraphicsUnmapResources", - [&]() noexcept { - return p_cuGraphicsUnmapResources( - 1, &resource, as_cu(stream.h_stream)); - }); - delete b; - } - ); - return DevicePtrHandle(box, &box->resource); -} - -// ============================================================================ -// MemoryResource-owned Device Pointer Handles -// ============================================================================ - -static MRDeallocCallback mr_dealloc_cb = nullptr; - -void register_mr_dealloc_callback(MRDeallocCallback cb) { - mr_dealloc_cb = cb; -} - -DevicePtrHandle deviceptr_create_with_mr(CUdeviceptr ptr, size_t size, PyObject* mr) { - if (!mr) { - return deviceptr_create_ref(ptr); - } - // GIL required when mr is provided - GILAcquireGuard gil; - if (!gil.acquired()) { - return deviceptr_create_ref(ptr); - } - Py_INCREF(mr); - auto box = std::shared_ptr( - new DevicePtrBox{ptr, DeallocationStream{}}, - [mr, size](DevicePtrBox* b) { - GILAcquireGuard gil; - if (gil.acquired()) { - if (mr_dealloc_cb) { - const DeallocationStream& stream = b->deallocation; - cleanup_in_context( - deallocation_context(stream), "MemoryResource.deallocate", - [&]() noexcept { - mr_dealloc_cb(mr, b->resource, size, stream.h_stream); - return CUDA_SUCCESS; - }); - } - Py_DECREF(mr); - } - delete b; - } - ); - return DevicePtrHandle(box, &box->resource); -} - -// ============================================================================ -// IPC Pointer Cache -// ============================================================================ -// This cache handles duplicate IPC imports, which behave differently depending -// on the memory type: -// -// 1. Memory pool allocations (DeviceMemoryResource): -// Multiple imports of the same allocation succeed and return duplicate -// pointers. However, the driver has a reference counting bug (nvbug 5570902) -// where the first cuMemFreeAsync incorrectly unmaps the memory even when -// imported multiple times. A driver fix is expected. -// -// 2. Pinned memory allocations (PinnedMemoryResource): -// Duplicate imports result in CUDA_ERROR_ALREADY_MAPPED. -// -// The cache solves both issues by checking the cache before calling -// cuMemPoolImportPointer and returning the existing handle for duplicate -// imports. This provides a consistent user experience where the same IPC -// descriptor can be imported multiple times regardless of memory type. -// -// The cache key is the export_data bytes (CUmemPoolPtrExportData), not the -// returned pointer, because we must check before calling the driver API. - - -// TODO: When driver fix for nvbug 5570902 is available, consider whether -// the cache is still needed for memory pool allocations (it will still be -// needed for pinned memory). -static bool use_ipc_ptr_cache() { - return true; -} - -namespace { -// Wrapper for CUmemPoolPtrExportData to use as map key -struct ExportDataKey { - CUmemPoolPtrExportData data; - - bool operator==(const ExportDataKey& other) const { - return std::memcmp(&data, &other.data, sizeof(data)) == 0; - } -}; - -struct ExportDataKeyHash { - std::size_t operator()(const ExportDataKey& key) const { - // Simple hash of the bytes - std::size_t h = 0; - const auto* bytes = reinterpret_cast(&key.data); - for (std::size_t i = 0; i < sizeof(key.data); ++i) { - h = h * 31 + bytes[i]; - } - return h; - } -}; - -} - -static HandleRegistry ipc_ptr_cache; -static std::mutex ipc_import_mutex; - -DevicePtrHandle deviceptr_import_ipc(const MemoryPoolHandle& h_pool, const void* export_data, const StreamHandle& h_stream) { - auto data = const_cast( - reinterpret_cast(export_data)); - - if (use_ipc_ptr_cache()) { - ExportDataKey key; - std::memcpy(&key.data, data, sizeof(key.data)); - - std::lock_guard lock(ipc_import_mutex); - - if (auto h = ipc_ptr_cache.lookup(key)) { - return h; - } - - GILReleaseGuard gil; - CUdeviceptr ptr; - if (CUDA_SUCCESS != (err = p_cuMemPoolImportPointer(&ptr, *h_pool, data))) { - return {}; - } - - DeallocationStream ds; - if (!make_deallocation_stream(h_stream, ds)) { - pw_cuMemFreeAsync(ptr, as_cu(h_stream)); - return {}; - } - - auto box = std::shared_ptr( - new DevicePtrBox{ptr, std::move(ds)}, - [h_pool, key](DevicePtrBox* b) { - ipc_ptr_cache.unregister_handle(key); - GILReleaseGuard gil; - const DeallocationStream& stream = b->deallocation; - cleanup_in_context( - deallocation_context(stream), "cuMemFreeAsync", - [&]() noexcept { - return p_cuMemFreeAsync( - b->resource, as_cu(stream.h_stream)); - }); - delete b; - } - ); - DevicePtrHandle h(box, &box->resource); - ipc_ptr_cache.register_handle(key, h); - return h; - - } else { - GILReleaseGuard gil; - CUdeviceptr ptr; - if (CUDA_SUCCESS != (err = p_cuMemPoolImportPointer(&ptr, *h_pool, data))) { - return {}; - } - - DeallocationStream ds; - if (!make_deallocation_stream(h_stream, ds)) { - pw_cuMemFreeAsync(ptr, as_cu(h_stream)); - return {}; - } - - auto box = std::shared_ptr( - new DevicePtrBox{ptr, std::move(ds)}, - [h_pool](DevicePtrBox* b) { - GILReleaseGuard gil; - const DeallocationStream& stream = b->deallocation; - cleanup_in_context( - deallocation_context(stream), "cuMemFreeAsync", - [&]() noexcept { - return p_cuMemFreeAsync( - b->resource, as_cu(stream.h_stream)); - }); - delete b; - } - ); - return DevicePtrHandle(box, &box->resource); - } -} - -// ============================================================================ -// Library Handles -// ============================================================================ - -namespace { -struct LibraryBox { - CUlibrary resource; -}; -} // namespace - -LibraryHandle create_library_handle_from_file(const char* path) { - GILReleaseGuard gil; - CUlibrary library; - if (CUDA_SUCCESS != (err = p_cuLibraryLoadFromFile(&library, path, nullptr, nullptr, 0, nullptr, nullptr, 0))) { - return {}; - } - - auto box = std::shared_ptr( - new LibraryBox{library}, - [](const LibraryBox* b) { - GILReleaseGuard gil; - // TODO: re-enable once LibraryBox tracks its owning context - // p_cuLibraryUnload(b->resource); - delete b; - } - ); - return LibraryHandle(box, &box->resource); -} - -LibraryHandle create_library_handle_from_data(const void* data) { - GILReleaseGuard gil; - CUlibrary library; - if (CUDA_SUCCESS != (err = p_cuLibraryLoadData(&library, data, nullptr, nullptr, 0, nullptr, nullptr, 0))) { - return {}; - } - - auto box = std::shared_ptr( - new LibraryBox{library}, - [](const LibraryBox* b) { - GILReleaseGuard gil; - // TODO: re-enable once LibraryBox tracks its owning context - // p_cuLibraryUnload(b->resource); - delete b; - } - ); - return LibraryHandle(box, &box->resource); -} - -LibraryHandle create_library_handle_ref(CUlibrary library) { - auto box = std::make_shared(LibraryBox{library}); - return LibraryHandle(box, &box->resource); -} - -// ============================================================================ -// Kernel Handles -// ============================================================================ - -namespace { -struct KernelBox { - CUkernel resource; - LibraryHandle h_library; -}; -} // namespace - -static const KernelBox* get_box(const KernelHandle& h) { - const CUkernel* p = h.get(); - return reinterpret_cast( - reinterpret_cast(p) - offsetof(KernelBox, resource) - ); -} - -// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) -static HandleRegistry kernel_registry; - -KernelHandle create_kernel_handle(const LibraryHandle& h_library, const char* name) { - GILReleaseGuard gil; - CUkernel kernel; - if (CUDA_SUCCESS != (err = p_cuLibraryGetKernel(&kernel, *h_library, name))) { - return {}; - } - - auto box = std::make_shared(KernelBox{kernel, h_library}); - KernelHandle h(box, &box->resource); - kernel_registry.register_handle(kernel, h); - return h; -} - -KernelHandle create_kernel_handle_ref(CUkernel kernel) { - if (auto h = kernel_registry.lookup(kernel)) { - return h; - } - auto box = std::make_shared(KernelBox{kernel, {}}); - return KernelHandle(box, &box->resource); -} - -LibraryHandle get_kernel_library(const KernelHandle& h) noexcept { - if (!h) return {}; - return get_box(h)->h_library; -} - -// ============================================================================ -// Graph Handles -// ============================================================================ - -namespace { - -struct NodeAttachment; -using GraphAttachmentMap = std::map; - -struct GraphHierarchy; - -// Standard-layout alias target for GraphHandle. -struct GraphBoxBase { - CUgraph resource = nullptr; -}; - -// Canonical state for one CUgraph. Its GraphHandle aliases resource, whose -// address remains stable for the lifetime of the hierarchy. -struct GraphBox : GraphBoxBase { - GraphHierarchy* hierarchy = nullptr; // Non-owning back-reference. - GraphBox* parent = nullptr; // Null for the root graph. - CUgraphNode owner_node = nullptr; // Node in parent that owns this graph. - GraphAttachmentMap attachments; // Non-owning attachment index. - HandleRegistry node_handles; - - GraphBox( - CUgraph resource_, - GraphHierarchy* hierarchy_, - GraphBox* parent_ = nullptr, - CUgraphNode owner_node_ = nullptr) noexcept - : GraphBoxBase{resource_}, - hierarchy(hierarchy_), - parent(parent_), - owner_node(owner_node_) {} -}; - -// Shared owner of stable GraphBox storage. Every GraphHandle aliases the same -// control block, so any graph handle keeps the entire hierarchy alive. -struct GraphHierarchy { - std::list graphs; // Parent boxes precede their descendants. - std::list graveyard; // Retired child graph tombstones. - - GraphBox* root() noexcept { - return graphs.empty() ? nullptr : &graphs.front(); - } -}; - -// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) -using GraphRegistry = HandleRegistry; -static GraphRegistry graph_registry; - -// Immutable resource owners for one version of a graph node's parameters. -// Inheriting DeferredCleanupItem lets CUDA's user-object destructor enqueue -// the payload without destroying owners on the callback thread. -struct NodeAttachment : DeferredCleanupItem { - CUuserObject object = nullptr; - std::array owners; - - NodeAttachment(OpaqueHandle owner0, OpaqueHandle owner1) - : owners{std::move(owner0), std::move(owner1)} {} -}; - -// shared_ptr deleters for the payloads that need one. Typed handles convert to -// OpaqueHandle by assignment and reuse their own control block, so they need no -// deleter here. The Python deleter follows the owner-release pattern used by -// the stream/deviceptr handles above. -void py_deleter(const void* p) noexcept { - GILAcquireGuard gil; - if (gil.acquired()) { - Py_DECREF(const_cast(static_cast(p))); - } -} - -void free_deleter(const void* p) noexcept { - std::free(const_cast(p)); -} - -GraphBox* get_box(const GraphHandle& h) noexcept { - auto* value = reinterpret_cast(h.get()); - return const_cast( - static_cast(value)); -} - -// Rekey a staged attachment map from source nodes to their cloned nodes. -// The caller must release the GIL before calling this function. -CUresult rekey_attachments( - GraphAttachmentMap& attachments, CUgraph cloned_graph) { - if (!cloned_graph) { - return CUDA_ERROR_INVALID_VALUE; - } - if (!p_cuGraphNodeFindInClone) { - return CUDA_ERROR_NOT_SUPPORTED; - } - - GraphAttachmentMap remapped; - while (!attachments.empty()) { - auto attachment = attachments.extract(attachments.begin()); - CUgraphNode cloned_node = nullptr; - CUresult status = p_cuGraphNodeFindInClone( - &cloned_node, attachment.key(), cloned_graph); - if (status != CUDA_SUCCESS) { - return status; - } - attachment.key() = cloned_node; - if (!remapped.insert(std::move(attachment)).inserted) { - return CUDA_ERROR_INVALID_VALUE; - } - } - attachments.swap(remapped); - return CUDA_SUCCESS; -} - -struct StagedGraphMetadata { - const GraphBox* source; - GraphBox* clone; - GraphAttachmentMap* attachments; -}; -using StagedGraphMetadataList = std::vector; - -// Copy a source hierarchy into detached metadata before CUDA mutation. -void stage_graph_metadata( - const GraphBox& source, - GraphBox& clone, - GraphAttachmentMap& attachments, - std::list& subgraphs, - StagedGraphMetadataList& staged) { - attachments = source.attachments; - staged.push_back({&source, &clone, &attachments}); - - for (const GraphBox& source_child : source.hierarchy->graphs) { - if (source_child.parent != &source || !source_child.resource) { - continue; - } - GraphBox& cloned_child = subgraphs.emplace_back( - nullptr, - clone.hierarchy, - &clone, - nullptr); - stage_graph_metadata( - source_child, - cloned_child, - cloned_child.attachments, - subgraphs, - staged); - } -} - -// Bind staged metadata to a CUDA-cloned hierarchy. The root clone resource -// must be populated before entry. The caller must release the GIL. -CUresult rekey_graph_metadata( - StagedGraphMetadataList& staged) { - if (!p_cuGraphNodeFindInClone || !p_cuGraphChildGraphNodeGetGraph) { - return CUDA_ERROR_NOT_SUPPORTED; - } - - CUresult status; - for (size_t i = 0; i < staged.size(); ++i) { - const GraphBox& source = *staged[i].source; - GraphBox& clone = *staged[i].clone; - if (i != 0) { - CUgraphNode cloned_owner = nullptr; - status = p_cuGraphNodeFindInClone( - &cloned_owner, - source.owner_node, - clone.parent->resource); - if (status == CUDA_SUCCESS) { - status = p_cuGraphChildGraphNodeGetGraph( - cloned_owner, &clone.resource); - } - if (status != CUDA_SUCCESS) { - return status; - } - clone.owner_node = cloned_owner; - } - - status = rekey_attachments( - *staged[i].attachments, clone.resource); - if (status != CUDA_SUCCESS) { - return status; - } - } - return CUDA_SUCCESS; -} - -} // namespace - -OpaqueHandle make_opaque_py(PyObject* obj) { - Py_INCREF(obj); - return OpaqueHandle(static_cast(obj), py_deleter); -} - -OpaqueHandle make_opaque_malloc(void* buf) { - return OpaqueHandle(static_cast(buf), free_deleter); -} - -// State held by PreparedAttachment between preparation and commit. It keeps the -// graph alive, tracks the graph-retained replacement, and holds a preallocated -// map entry so commit cannot allocate. Destroying PreparedAttachment rolls back -// the staged user-object retain unless graph_commit_attachment publishes it. -struct PreparedAttachmentState { - GraphHandle h_graph; - NodeAttachment* replacement = nullptr; - GraphAttachmentMap::node_type replacement_entry; - - explicit PreparedAttachmentState(GraphHandle h_graph_) - : h_graph(std::move(h_graph_)) {} -}; - -void rollback_prepared_attachment( - PreparedAttachmentState* state) noexcept { - if (!state) { - return; - } - if (state->replacement) { - GraphBox* box = get_box(state->h_graph); - if (box->resource) { - GILReleaseGuard gil; - p_cuGraphReleaseUserObject( - box->resource, state->replacement->object, 1); - } - } - delete state; -} - -// Detached metadata for a replacement embedded graph hierarchy. Preparation -// copies every attachment map and allocates every GraphBox before CUDA destroys -// the old embedded graph. Commit only rekeys and publishes it. -struct PreparedChildGraphUpdateState { - GraphHandle h_parent; - GraphHandle h_source; - GraphBox* old_root = nullptr; - CUgraphNode owner_node = nullptr; - std::list replacement; - StagedGraphMetadataList staged; - std::vector handles; - - PreparedChildGraphUpdateState( - GraphHandle h_parent_, - GraphHandle h_source_, - GraphBox* old_root_, - CUgraphNode owner_node_) - : h_parent(std::move(h_parent_)), - h_source(std::move(h_source_)), - old_root(old_root_), - owner_node(owner_node_) {} -}; - -GraphHandle create_graph_handle(CUgraph graph) { - if (!graph) { - return {}; - } - - auto hierarchy = std::shared_ptr( - new GraphHierarchy{}, - [](GraphHierarchy* hierarchy) { - for (const GraphBox& box : hierarchy->graphs) { - if (box.resource) { - graph_registry.unregister_handle(box.resource); - } - } - GraphBox* root = hierarchy->root(); - if (root && root->resource) { - GILReleaseGuard gil; - p_cuGraphDestroy(root->resource); - } - retry_deferred_cleanup(); - delete hierarchy; - } - ); - GraphBox& root = hierarchy->graphs.emplace_back( - graph, hierarchy.get()); - - GraphHandle h_graph(hierarchy, &root.resource); - graph_registry.register_handle(graph, h_graph); - return h_graph; -} - -GraphHandle create_child_graph_handle( - CUgraph child_graph, const GraphHandle& h_parent, - CUgraphNode owner_node) { - if (!child_graph || !h_parent || !owner_node) { - return {}; - } - if (GraphHandle h_graph = graph_registry.lookup(child_graph)) { - return h_graph; - } - - GraphBox* parent = get_box(h_parent); - GraphHierarchy* hierarchy = parent->hierarchy; - GraphBox& child = hierarchy->graphs.emplace_back( - child_graph, hierarchy, parent, owner_node); - - GraphHandle h_child(h_parent, &child.resource); - graph_registry.register_handle(child_graph, h_child); - return h_child; -} - -CUresult graph_prepare_child_graph_update( - const GraphHandle& h_parent, - const GraphHandle& h_old_child, - CUgraphNode owner_node, - const GraphHandle& h_source, - PreparedChildGraphUpdate* out_prepared) { - if (!h_parent || !h_old_child || !owner_node || - !h_source || !out_prepared) { - return CUDA_ERROR_INVALID_VALUE; - } - out_prepared->reset(); - - GraphBox* parent = get_box(h_parent); - GraphBox* old_root = get_box(h_old_child); - GraphBox* source = get_box(h_source); - // A source from the destination hierarchy can include the old embedded - // subtree whose raw node keys CUDA destroys during replacement. - if (!parent->resource || !old_root->resource || !source->resource || - old_root->parent != parent || - old_root->owner_node != owner_node || - source->hierarchy == parent->hierarchy) { - return CUDA_ERROR_INVALID_VALUE; - } - - PreparedChildGraphUpdate prepared = - std::make_shared( - h_parent, h_source, old_root, owner_node); - - GraphBox& replacement_root = - prepared->replacement.emplace_back( - nullptr, parent->hierarchy, parent, owner_node); - stage_graph_metadata( - *source, - replacement_root, - replacement_root.attachments, - prepared->replacement, - prepared->staged); - - const size_t graph_count = prepared->staged.size(); - prepared->handles.reserve(graph_count); - for (const StagedGraphMetadata& graph : prepared->staged) { - prepared->handles.emplace_back( - h_parent, &graph.clone->resource); - } - - *out_prepared = std::move(prepared); - return CUDA_SUCCESS; -} - -void publish_child_graph_update( - PreparedChildGraphUpdateState& state, - GraphHandle* out_child) { - GraphBox* parent = get_box(state.h_parent); - parent->hierarchy->graphs.splice( - parent->hierarchy->graphs.end(), state.replacement); - *out_child = state.handles.front(); - graph_registry.register_handles(state.handles); -} - -CUresult graph_commit_child_graph_update( - PreparedChildGraphUpdate& prepared, - GraphHandle* out_child) { - if (!prepared || !out_child) { - return CUDA_ERROR_INVALID_VALUE; - } - out_child->reset(); - - PreparedChildGraphUpdateState& state = *prepared; - GraphBox* parent = get_box(state.h_parent); - if (!parent->resource || !state.old_root->resource) { - prepared.reset(); - return CUDA_ERROR_INVALID_VALUE; - } - - CUresult status = CUDA_ERROR_NOT_SUPPORTED; - CUgraph cloned_root = nullptr; - if (p_cuGraphChildGraphNodeGetGraph) { - GILReleaseGuard gil; - status = p_cuGraphChildGraphNodeGetGraph( - state.owner_node, &cloned_root); - if (status == CUDA_SUCCESS) { - state.staged.front().clone->resource = cloned_root; - status = rekey_graph_metadata(state.staged); - } - } - - // CUDA has already destroyed the old embedded graph. No replacement - // metadata is visible yet, so this selects only the old generation. - invalidate_child_graph_state( - state.h_parent, state.owner_node); - - if (status != CUDA_SUCCESS) { - prepared.reset(); - throw std::runtime_error( - "failed to update graph metadata after child graph replacement"); - } - - publish_child_graph_update(state, out_child); - prepared.reset(); - return status; -} - -CUresult graph_get_attachment( - const GraphHandle& h_graph, CUgraphNode node, - OpaqueHandle* owner0, OpaqueHandle* owner1) { - if (!h_graph || !node || (!owner0 && !owner1)) { - return CUDA_ERROR_INVALID_VALUE; - } - if (owner0) { - owner0->reset(); - } - if (owner1) { - owner1->reset(); - } - - GraphBox* box = get_box(h_graph); - if (!box->resource) { - return CUDA_ERROR_INVALID_VALUE; - } - auto it = box->attachments.find(node); - if (it != box->attachments.end()) { - if (owner0) { - *owner0 = it->second->owners[0]; - } - if (owner1) { - *owner1 = it->second->owners[1]; - } - } - return CUDA_SUCCESS; -} - -CUresult graph_prepare_attachment( - const GraphHandle& h_graph, - OpaqueHandle owner0, - OpaqueHandle owner1, - PreparedAttachment* out_prepared) { - if (!out_prepared) { - return CUDA_ERROR_INVALID_VALUE; - } - out_prepared->reset(); - if (!h_graph) { - return CUDA_ERROR_INVALID_VALUE; - } - - GraphBox* box = get_box(h_graph); - if (!box->resource) { - return CUDA_ERROR_INVALID_VALUE; - } - if (!p_cuGraphReleaseUserObject) { - return CUDA_ERROR_NOT_SUPPORTED; - } - - PreparedAttachment prepared( - new PreparedAttachmentState(h_graph), - PreparedAttachmentDeleter{rollback_prepared_attachment}); - if (owner0 || owner1) { - if (!p_cuUserObjectCreate || !p_cuUserObjectRelease || - !p_cuGraphRetainUserObject) { - return CUDA_ERROR_NOT_SUPPORTED; - } - - ensure_deferred_cleanup_ready(); - prepared->replacement = new NodeAttachment( - std::move(owner0), std::move(owner1)); - GraphAttachmentMap staged; - try { - staged.emplace(nullptr, prepared->replacement); - prepared->replacement_entry = - staged.extract(staged.begin()); - } catch (...) { - delete prepared->replacement; - prepared->replacement = nullptr; - throw; - } - auto* cleanup_item = - static_cast( - prepared->replacement); - - CUuserObject object = nullptr; - CUresult status; - { - GILReleaseGuard gil; - status = p_cuUserObjectCreate( - &object, cleanup_item, - reinterpret_cast(enqueue_cleanup), - 1, CU_USER_OBJECT_NO_DESTRUCTOR_SYNC); - if (status != CUDA_SUCCESS) { - prepared->replacement_entry.mapped() = nullptr; - delete prepared->replacement; - prepared->replacement = nullptr; - return status; - } - prepared->replacement->object = object; - status = p_cuGraphRetainUserObject( - box->resource, object, 1, CU_GRAPH_USER_OBJECT_MOVE); - if (status != CUDA_SUCCESS) { - prepared->replacement_entry.mapped() = nullptr; - prepared->replacement = nullptr; - p_cuUserObjectRelease(object, 1); - return status; - } - } - } - - *out_prepared = std::move(prepared); - return CUDA_SUCCESS; -} - -CUresult graph_commit_attachment( - PreparedAttachment& prepared, - CUgraphNode node) { - if (!prepared) { - return CUDA_ERROR_INVALID_VALUE; - } - - GraphHandle h_graph = prepared->h_graph; - GraphBox* box = get_box(h_graph); - if (!box->resource || (!node && !prepared->replacement)) { - delete prepared.release(); - return CUDA_ERROR_INVALID_VALUE; - } - if (!node) { - delete prepared.release(); - return CUDA_SUCCESS; - } - - // Publish the replacement or removal before releasing the previous graph - // reference; that release can make the previous payload eligible for - // destruction. - NodeAttachment* previous = nullptr; - auto it = box->attachments.find(node); - if (it == box->attachments.end()) { - if (prepared->replacement) { - prepared->replacement_entry.key() = node; - auto result = box->attachments.insert( - std::move(prepared->replacement_entry)); - if (!result.inserted) { - prepared->replacement_entry = - std::move(result.node); - delete prepared.release(); - return CUDA_ERROR_INVALID_VALUE; - } - } - } else { - previous = it->second; - if (prepared->replacement) { - it->second = prepared->replacement; - } else { - box->attachments.erase(it); - } - } - - delete prepared.release(); - if (!previous) { - return CUDA_SUCCESS; - } - GILReleaseGuard gil; - return p_cuGraphReleaseUserObject( - box->resource, previous->object, 1); -} - -CUresult graph_clone_attachments( - const GraphHandle& h_clone, - const GraphHandle& h_source) { - if (!h_clone || !h_source) { - return CUDA_ERROR_INVALID_VALUE; - } - - GraphBox* clone = get_box(h_clone); - GraphBox* source = get_box(h_source); - if (!clone->resource || !source->resource || - !clone->attachments.empty()) { - return CUDA_ERROR_INVALID_VALUE; - } - - // Build and rekey the clone metadata off-hierarchy so a CUDA mapping error - // cannot partially publish it. - GraphAttachmentMap attachments; - std::list subgraphs; - StagedGraphMetadataList staged; - stage_graph_metadata( - *source, *clone, attachments, subgraphs, staged); - - std::vector handles; - handles.reserve(subgraphs.size()); - for (GraphBox& graph : subgraphs) { - handles.emplace_back(h_clone, &graph.resource); - } - - CUresult status; - { - GILReleaseGuard gil; - status = rekey_graph_metadata(staged); - } - if (status != CUDA_SUCCESS) { - return status; - } - - clone->attachments.swap(attachments); - if (subgraphs.empty()) { - return CUDA_SUCCESS; - } - - clone->hierarchy->graphs.splice( - clone->hierarchy->graphs.end(), subgraphs); - graph_registry.register_handles(handles); - return CUDA_SUCCESS; -} - -// ============================================================================ -// Graph Exec Handles -// ============================================================================ - -namespace { - -// Append-only owners introduced by individual executable-node updates. CUDA -// owns this payload through a user object propagated into the CUgraphExec. -struct ExecAttachments : DeferredCleanupItem { - CUuserObject object = nullptr; - std::vector owners; -}; - -struct GraphExecBox { - CUgraphExec resource = nullptr; - ExecAttachments* attachments = nullptr; // Non-owning. - - ~GraphExecBox() noexcept { - if (resource) { - GILReleaseGuard gil; - p_cuGraphExecDestroy(resource); - } - // The accumulator fields may be dangling after exec destruction. - retry_deferred_cleanup(); - } -}; - -GraphExecBox* get_exec_box(const GraphExecHandle& h) noexcept { - return const_cast( - reinterpret_cast(h.get())); -} - -GraphExecHandle make_graph_exec_handle( - CUgraphExec graph_exec, ExecAttachments* attachments) { - struct RawGraphExecGuard { - CUgraphExec resource; - - ~RawGraphExecGuard() noexcept { - if (resource) { - GILReleaseGuard gil; - p_cuGraphExecDestroy(resource); - } - retry_deferred_cleanup(); - } - } guard{graph_exec}; - - auto box = std::make_shared(); - box->resource = graph_exec; - box->attachments = attachments; - guard.resource = nullptr; - return GraphExecHandle(box, &box->resource); -} - -// Holds a fresh accumulator retained on the source graph across a CUDA call -// that propagates user objects into an exec. Releasing drops the source's -// reference: after successful propagation the exec keeps the accumulator -// alive, and otherwise this drops its last reference. -struct ExecAttachmentStaging { - GraphHandle h_source; - ExecAttachments* accumulator = nullptr; - - ~ExecAttachmentStaging() noexcept { - release(); - } - - CUresult release() noexcept { - if (!h_source || !accumulator) { - return CUDA_SUCCESS; - } - const CUuserObject object = accumulator->object; - const GraphHandle source = std::move(h_source); - accumulator = nullptr; - GILReleaseGuard gil; - return p_cuGraphReleaseUserObject(*source, object, 1); - } -}; - -// Create an accumulator and retain it on h_source, so that a following -// instantiation or whole-graph update propagates a reference into the exec. -CUresult stage_exec_attachments( - const GraphHandle& h_source, ExecAttachmentStaging* out_staging) { - if (!p_cuUserObjectCreate || !p_cuUserObjectRelease || - !p_cuGraphRetainUserObject || !p_cuGraphReleaseUserObject) { - return CUDA_ERROR_NOT_SUPPORTED; - } - - ensure_deferred_cleanup_ready(); - auto* accumulator = new ExecAttachments; - - CUuserObject object = nullptr; - CUresult status; - { - GILReleaseGuard gil; - status = p_cuUserObjectCreate( - &object, - static_cast(accumulator), - reinterpret_cast(enqueue_cleanup), - 1, - CU_USER_OBJECT_NO_DESTRUCTOR_SYNC); - if (status != CUDA_SUCCESS) { - delete accumulator; - return status; - } - accumulator->object = object; - status = p_cuGraphRetainUserObject( - *h_source, object, 1, CU_GRAPH_USER_OBJECT_MOVE); - if (status != CUDA_SUCCESS) { - // Dropping the last reference retires the accumulator. - p_cuUserObjectRelease(object, 1); - return status; - } - } - - out_staging->h_source = h_source; - out_staging->accumulator = accumulator; - return CUDA_SUCCESS; -} - -} // namespace - -// State held by PreparedExecAttachment between preparation and commit. It keeps -// the exec alive and remembers the accumulator size before the append, so that -// rollback can drop owners staged for a mutation that CUDA rejected. -struct PreparedExecAttachmentState { - GraphExecHandle h_exec; - ExecAttachments* attachments = nullptr; - size_t original_size = 0; - - PreparedExecAttachmentState( - GraphExecHandle h_exec_, - ExecAttachments* attachments_, - size_t original_size_) - : h_exec(std::move(h_exec_)), - attachments(attachments_), - original_size(original_size_) {} -}; - -void rollback_prepared_exec_attachment( - PreparedExecAttachmentState* state) noexcept { - if (!state) { - return; - } - if (state->attachments) { - while (state->attachments->owners.size() > state->original_size) { - state->attachments->owners.pop_back(); - } - } - delete state; -} - -GraphExecHandle create_graph_exec_handle( - const GraphHandle& h_source, - CUDA_GRAPH_INSTANTIATE_PARAMS* params) { - if (!h_source || !*h_source || !params) { - err = CUDA_ERROR_INVALID_VALUE; - return {}; - } - if (!p_cuGraphInstantiateWithParams) { - err = CUDA_ERROR_NOT_SUPPORTED; - return {}; - } - - ExecAttachmentStaging staging; - if (CUDA_SUCCESS != (err = stage_exec_attachments(h_source, &staging))) { - return {}; - } - - CUgraphExec graph_exec = nullptr; - { - GILReleaseGuard gil; - err = p_cuGraphInstantiateWithParams(&graph_exec, *h_source, params); - } - if (err != CUDA_SUCCESS) { - return {}; - } - // CUDA can report a specific failure while returning success. The exec is - // then unusable, so it stays unadopted for the caller to diagnose from - // params->result_out. - if (params->result_out != CUDA_GRAPH_INSTANTIATE_SUCCESS) { - return {}; - } - if (!graph_exec) { - err = CUDA_ERROR_INVALID_VALUE; - return {}; - } - - GraphExecHandle h_exec = make_graph_exec_handle( - graph_exec, staging.accumulator); - if (CUDA_SUCCESS != (err = staging.release())) { - return {}; - } - return h_exec; -} - -CUresult graph_exec_update( - const GraphExecHandle& h_exec, - const GraphHandle& h_source, - CUgraphExecUpdateResultInfo* result_info) { - if (!h_exec || !h_source || !*h_source || !result_info) { - return CUDA_ERROR_INVALID_VALUE; - } - if (!p_cuGraphExecUpdate) { - return CUDA_ERROR_NOT_SUPPORTED; - } - - GraphExecBox* box = get_exec_box(h_exec); - if (!box->resource) { - return CUDA_ERROR_INVALID_VALUE; - } - - ExecAttachmentStaging staging; - CUresult status = stage_exec_attachments(h_source, &staging); - if (status != CUDA_SUCCESS) { - return status; - } - - { - GILReleaseGuard gil; - status = p_cuGraphExecUpdate(box->resource, *h_source, result_info); - } - if (status != CUDA_SUCCESS) { - return status; - } - - // CUDA may already have retired the old accumulator. Publish the new one - // before releasing the source graph's temporary reference. - box->attachments = staging.accumulator; - return staging.release(); -} - -CUresult graph_prepare_exec_attachment( - const GraphExecHandle& h_exec, - OpaqueHandle owner0, - OpaqueHandle owner1, - PreparedExecAttachment* out_prepared) { - if (!out_prepared) { - return CUDA_ERROR_INVALID_VALUE; - } - out_prepared->reset(); - if (!h_exec) { - return CUDA_ERROR_INVALID_VALUE; - } - - GraphExecBox* box = get_exec_box(h_exec); - if (!box->resource || !box->attachments) { - return CUDA_ERROR_INVALID_VALUE; - } - - ExecAttachments* attachments = box->attachments; - const size_t original_size = attachments->owners.size(); - const size_t additions = - static_cast(static_cast(owner0)) + - static_cast(static_cast(owner1)); - // Reserve before staging so that rollback and commit cannot allocate. - attachments->owners.reserve(original_size + additions); - PreparedExecAttachment prepared( - new PreparedExecAttachmentState(h_exec, attachments, original_size), - PreparedExecAttachmentDeleter{rollback_prepared_exec_attachment}); - if (owner0) { - attachments->owners.emplace_back(std::move(owner0)); - } - if (owner1) { - attachments->owners.emplace_back(std::move(owner1)); - } - *out_prepared = std::move(prepared); - return CUDA_SUCCESS; -} - -void graph_commit_exec_attachment( - PreparedExecAttachment& prepared) noexcept { - delete prepared.release(); -} - -namespace { -struct GraphNodeBox { - mutable CUgraphNode resource; - GraphHandle h_graph; -}; -} // namespace - -static const GraphNodeBox* get_box(const GraphNodeHandle& h) { - const CUgraphNode* p = h.get(); - return reinterpret_cast( - reinterpret_cast(p) - offsetof(GraphNodeBox, resource) - ); -} - -// graphs is ordered parent-before-child. Nulling a selected box marks its -// later descendants, whose parent pointers remain valid after list splicing. -// This permits one allocation-free sweep of the hierarchy. -void invalidate_child_graph_state( - const GraphHandle& h_parent, - CUgraphNode owner_node) noexcept { - if (!h_parent || !owner_node) { - return; - } - - GraphBox* parent = get_box(h_parent); - if (!parent->resource) { - return; - } - GraphHierarchy& hierarchy = *parent->hierarchy; - for (auto it = hierarchy.graphs.begin(); - it != hierarchy.graphs.end();) { - auto graph = it++; - bool is_owned_root = graph->parent == parent && - graph->owner_node == owner_node; - bool is_descendant = graph->parent && - !graph->parent->resource; - if (!is_owned_root && !is_descendant) { - continue; - } - - // Empty node_handles and invalidate each one. - for (auto& entry : graph->node_handles.drain()) { - if (GraphNodeHandle h_node = entry.second.lock()) { - get_box(h_node)->resource = nullptr; - } - } - graph_registry.unregister_handle(graph->resource); - graph->resource = nullptr; - graph->attachments.clear(); - hierarchy.graveyard.splice( - hierarchy.graveyard.end(), hierarchy.graphs, graph); - } -} - -GraphNodeHandle create_graph_node_handle(CUgraphNode node, const GraphHandle& h_graph) { - if (!node) { - auto box = std::make_shared( - GraphNodeBox{nullptr, h_graph}); - return GraphNodeHandle(box, &box->resource); - } - - GraphBox* graph = get_box(h_graph); - return graph->node_handles.get_or_create( - node, - [node, &h_graph] { - auto box = std::make_shared( - GraphNodeBox{node, h_graph}); - return GraphNodeHandle(box, &box->resource); - }); -} - -GraphHandle graph_node_get_graph(const GraphNodeHandle& h) noexcept { - return h ? get_box(h)->h_graph : GraphHandle{}; -} - -void invalidate_graph_node(const GraphNodeHandle& h) noexcept { - if (!h) { - return; - } - - const GraphNodeBox* node_box = get_box(h); - CUgraphNode node = node_box->resource; - if (!node) { - return; - } - GraphBox* graph = get_box(node_box->h_graph); - graph->node_handles.unregister_handle(node); - node_box->resource = nullptr; -} - -// ============================================================================ -// Graphics Resource Handles -// ============================================================================ - -namespace { -struct GraphicsResourceBox { - CUgraphicsResource resource; -}; -} // namespace - -GraphicsResourceHandle create_graphics_resource_handle(CUgraphicsResource resource) { - auto box = std::shared_ptr( - new GraphicsResourceBox{resource}, - [](const GraphicsResourceBox* b) { - GILReleaseGuard gil; - p_cuGraphicsUnregisterResource(b->resource); - delete b; - } - ); - return GraphicsResourceHandle(box, &box->resource); -} - -// ============================================================================ -// NVRTC Program Handles -// ============================================================================ - -namespace { -struct NvrtcProgramBox { - nvrtcProgram resource; -}; -} // namespace - -NvrtcProgramHandle create_nvrtc_program_handle(nvrtcProgram prog) { - auto box = std::shared_ptr( - new NvrtcProgramBox{prog}, - [](NvrtcProgramBox* b) { - // Note: nvrtcDestroyProgram takes nvrtcProgram* and nulls it, - // but we're deleting the box anyway so nulling is harmless. - // Errors are ignored (standard destructor practice). - p_nvrtcDestroyProgram(&b->resource); - delete b; - } - ); - return NvrtcProgramHandle(box, &box->resource); -} - -NvrtcProgramHandle create_nvrtc_program_handle_ref(nvrtcProgram prog) { - auto box = std::make_shared(NvrtcProgramBox{prog}); - return NvrtcProgramHandle(box, &box->resource); -} - -// ============================================================================ -// NVVM Program Handles -// ============================================================================ - -namespace { -struct NvvmProgramBox { - NvvmProgramValue resource; -}; -} // namespace - -NvvmProgramHandle create_nvvm_program_handle(nvvmProgram prog) { - auto box = std::shared_ptr( - new NvvmProgramBox{{prog}}, - [](NvvmProgramBox* b) { - // Note: nvvmDestroyProgram takes nvvmProgram* and nulls it, - // but we're deleting the box anyway so nulling is harmless. - // If NVVM is not available, the function pointer is null. - if (p_nvvmDestroyProgram) { - p_nvvmDestroyProgram(&b->resource.raw); - } - delete b; - } - ); - return NvvmProgramHandle(box, &box->resource); -} - -NvvmProgramHandle create_nvvm_program_handle_ref(nvvmProgram prog) { - auto box = std::make_shared(NvvmProgramBox{{prog}}); - return NvvmProgramHandle(box, &box->resource); -} - -// ============================================================================ -// nvJitLink Handles -// ============================================================================ - -namespace { -struct NvJitLinkBox { - NvJitLinkValue resource; -}; -} // namespace - -NvJitLinkHandle create_nvjitlink_handle(nvJitLink_t handle) { - auto box = std::shared_ptr( - new NvJitLinkBox{{handle}}, - [](NvJitLinkBox* b) { - // Note: nvJitLinkDestroy takes nvJitLinkHandle* and nulls it, - // but we're deleting the box anyway so nulling is harmless. - // If nvJitLink is not available, the function pointer is null. - if (p_nvJitLinkDestroy) { - p_nvJitLinkDestroy(&b->resource.raw); - } - delete b; - } - ); - return NvJitLinkHandle(box, &box->resource); -} - -NvJitLinkHandle create_nvjitlink_handle_ref(nvJitLink_t handle) { - auto box = std::make_shared(NvJitLinkBox{{handle}}); - return NvJitLinkHandle(box, &box->resource); -} - -// ============================================================================ -// cuLink Handles -// ============================================================================ - -namespace { -struct CuLinkBox { - CUlinkState resource; -}; -} // namespace - -CuLinkHandle create_culink_handle(CUlinkState state) { - auto box = std::shared_ptr( - new CuLinkBox{state}, - [](CuLinkBox* b) { - // cuLinkDestroy takes CUlinkState by value (not pointer). - // Errors are ignored (standard destructor practice). - if (p_cuLinkDestroy) { - p_cuLinkDestroy(b->resource); - } - delete b; - } - ); - return CuLinkHandle(box, &box->resource); -} - -CuLinkHandle create_culink_handle_ref(CUlinkState state) { - auto box = std::make_shared(CuLinkBox{state}); - return CuLinkHandle(box, &box->resource); -} - -// ============================================================================ -// File Descriptor Handles -// ============================================================================ - -FileDescriptorHandle create_fd_handle(int fd) { -#ifdef _WIN32 - throw std::runtime_error("create_fd_handle is not supported on Windows"); -#else - return FileDescriptorHandle( - new int(fd), - [](const int* p) { ::close(*p); delete p; } - ); -#endif -} - -FileDescriptorHandle create_fd_handle_ref(int fd) { -#ifdef _WIN32 - throw std::runtime_error("create_fd_handle_ref is not supported on Windows"); -#else - return std::make_shared(fd); -#endif -} - -// ============================================================================ -// Array / mipmapped-array / texture / surface handles (PR #467) -// ============================================================================ - -namespace { -struct ArrayBox { - CUarray resource; - // Non-null only for a mipmap-level view: keeps the parent mipmap (the real - // owner of the level's storage) alive for as long as the level is held. - MipmappedArrayHandle h_parent; - ContextHandle h_context; -}; - -struct MipmappedArrayBox { - CUmipmappedArray resource; - ContextHandle h_context; -}; - -// Texture and surface objects are per-context pool indices. Destroying one -// with the wrong context current can silently succeed without freeing it or -// can free an unrelated object, so destruction must enter the creating -// context. Handle-based resources resolve their own context and must not. -struct TexObjectBox { - // Tagged so TexObjectHandle is a distinct C++ type from DevicePtrHandle / - // SurfObjectHandle (all wrap `unsigned long long`). - TexObjectValue resource; - // Type-erased backing dependency (OpaqueArrayHandle / MipmappedArrayHandle / - // DevicePtrHandle). The texture's resource is a union; we only need to keep - // whichever backing it was built from alive, never to dereference it. - std::shared_ptr h_backing; - ContextHandle h_context; -}; - -struct SurfObjectBox { - SurfObjectValue resource; - OpaqueArrayHandle h_array; // surfaces are always array-backed - ContextHandle h_context; -}; - -// Recover an array's owning box from its aliased resource pointer. -const ArrayBox* get_box(const OpaqueArrayHandle& h) noexcept { - const CUarray* p = h.get(); - return reinterpret_cast( - reinterpret_cast(p) - offsetof(ArrayBox, resource)); -} - -// Recover a mipmapped array's owning box from its aliased resource pointer. -const MipmappedArrayBox* get_box(const MipmappedArrayHandle& h) noexcept { - const CUmipmappedArray* p = h.get(); - return reinterpret_cast( - reinterpret_cast(p) - - offsetof(MipmappedArrayBox, resource)); -} - -// Wrap an array with shared owning-destruction behavior. -static OpaqueArrayHandle wrap_array_owned(CUarray arr, ContextHandle h_context) { - auto box = std::shared_ptr( - new ArrayBox{arr, {}, std::move(h_context)}, - [](const ArrayBox* b) { - GILReleaseGuard gil; - pw_cuArrayDestroy(b->resource); - delete b; - } - ); - return OpaqueArrayHandle(box, &box->resource); -} - -} // namespace - -OpaqueArrayHandle create_array_handle(const ContextHandle& h_context, const CUDA_ARRAY3D_DESCRIPTOR& desc) { - GILReleaseGuard gil; - CUarray arr = nullptr; - err = invoke_in_context_or_undo( - h_context, - [&]() noexcept { return p_cuArray3DCreate(&arr, &desc); }, - [&]() noexcept { pw_cuArrayDestroy(arr); }, - /*undo_requires_target_context=*/false); - if (err != CUDA_SUCCESS) { - return {}; - } - return wrap_array_owned(arr, h_context); -} - -OpaqueArrayHandle create_array_handle_ref(CUarray arr) { - if (!arr) { - return {}; - } - auto box = std::make_shared(ArrayBox{arr, {}, {}}); - return OpaqueArrayHandle(box, &box->resource); -} - -OpaqueArrayHandle create_array_handle_owning(CUarray arr) { - if (!arr) { - return {}; - } - return wrap_array_owned(arr, {}); -} - -// Return the context retained by an array handle. -ContextHandle get_array_context(const OpaqueArrayHandle& h) noexcept { - return h ? get_box(h)->h_context : ContextHandle{}; -} - -OpaqueArrayHandle create_array_level_handle(const MipmappedArrayHandle& h_mip, unsigned int level) { - GILReleaseGuard gil; - CUarray arr; - ContextHandle h_context = h_mip ? get_box(h_mip)->h_context : ContextHandle{}; - if (CUDA_SUCCESS != (err = p_cuMipmappedArrayGetLevel(&arr, as_cu(h_mip), level))) { - return {}; - } - // Non-owning level view: storage belongs to the mipmap. Embed the mipmap - // handle so the parent outlives this level; the deleter does not destroy. - auto box = std::shared_ptr( - new ArrayBox{arr, h_mip, h_context}, - [](const ArrayBox* b) { delete b; } - ); - return OpaqueArrayHandle(box, &box->resource); -} - -MipmappedArrayHandle create_mipmapped_array_handle(const ContextHandle& h_context, - const CUDA_ARRAY3D_DESCRIPTOR& desc, - unsigned int num_levels) { - GILReleaseGuard gil; - CUmipmappedArray mip = nullptr; - err = invoke_in_context_or_undo( - h_context, - [&]() noexcept { return p_cuMipmappedArrayCreate(&mip, &desc, num_levels); }, - [&]() noexcept { pw_cuMipmappedArrayDestroy(mip); }, - /*undo_requires_target_context=*/false); - if (err != CUDA_SUCCESS) { - return {}; - } - auto box = std::shared_ptr( - new MipmappedArrayBox{mip, h_context}, - [](const MipmappedArrayBox* b) { - GILReleaseGuard gil; - pw_cuMipmappedArrayDestroy(b->resource); - delete b; - } - ); - return MipmappedArrayHandle(box, &box->resource); -} - -// Return the context retained by a mipmapped array handle. -ContextHandle get_mipmapped_array_context(const MipmappedArrayHandle& h) noexcept { - return h ? get_box(h)->h_context : ContextHandle{}; -} - -namespace { -TexObjectHandle make_tex_object_handle(const CUDA_RESOURCE_DESC& res, - const CUDA_TEXTURE_DESC& tex, - std::shared_ptr h_backing, - const ContextHandle& h_context) { - GILReleaseGuard gil; - CUtexObject obj = 0; - err = invoke_in_context_or_undo( - h_context, - [&]() noexcept { return p_cuTexObjectCreate(&obj, &res, &tex, nullptr); }, - [&]() noexcept { pw_cuTexObjectDestroy(obj); }, - /*undo_requires_target_context=*/true); - if (err != CUDA_SUCCESS) { - return {}; - } - auto box = std::shared_ptr( - new TexObjectBox{TexObjectValue{obj}, std::move(h_backing), h_context}, - [](const TexObjectBox* b) { - GILReleaseGuard gil; - cleanup_in_context(b->h_context, "cuTexObjectDestroy", [&]() noexcept { - return p_cuTexObjectDestroy(b->resource.raw); - }); - delete b; - } - ); - return TexObjectHandle(box, &box->resource); -} -} // namespace - -TexObjectHandle create_tex_object_handle_array(const ContextHandle& h_context, - const CUDA_RESOURCE_DESC& res, - const CUDA_TEXTURE_DESC& tex, - const OpaqueArrayHandle& h_backing) { - return make_tex_object_handle(res, tex, h_backing, h_context); -} - -TexObjectHandle create_tex_object_handle_mipmap(const ContextHandle& h_context, - const CUDA_RESOURCE_DESC& res, - const CUDA_TEXTURE_DESC& tex, - const MipmappedArrayHandle& h_backing) { - return make_tex_object_handle(res, tex, h_backing, h_context); -} - -TexObjectHandle create_tex_object_handle_linear(const ContextHandle& h_context, - const CUDA_RESOURCE_DESC& res, - const CUDA_TEXTURE_DESC& tex, - const DevicePtrHandle& h_backing) { - return make_tex_object_handle(res, tex, h_backing, h_context); -} - -SurfObjectHandle create_surf_object_handle(const ContextHandle& h_context, - const CUDA_RESOURCE_DESC& res, - const OpaqueArrayHandle& h_backing) { - GILReleaseGuard gil; - CUsurfObject obj = 0; - err = invoke_in_context_or_undo( - h_context, - [&]() noexcept { return p_cuSurfObjectCreate(&obj, &res); }, - [&]() noexcept { pw_cuSurfObjectDestroy(obj); }, - /*undo_requires_target_context=*/true); - if (err != CUDA_SUCCESS) { - return {}; - } - auto box = std::shared_ptr( - new SurfObjectBox{SurfObjectValue{obj}, h_backing, h_context}, - [](const SurfObjectBox* b) { - GILReleaseGuard gil; - cleanup_in_context(b->h_context, "cuSurfObjectDestroy", [&]() noexcept { - return p_cuSurfObjectDestroy(b->resource.raw); - }); - delete b; - } - ); - return SurfObjectHandle(box, &box->resource); -} - -// ============================================================================ -// SM resource split wrapper -// ============================================================================ - -CUresult sm_resource_split(CUdevResource* result, unsigned int nbGroups, - const CUdevResource* input, CUdevResource* remainder, - unsigned int flags, void* groupParams) { -#if CUDA_VERSION >= 13010 - if (!p_cuDevSmResourceSplit) { - return CUDA_ERROR_NOT_SUPPORTED; - } - return p_cuDevSmResourceSplit( - result, nbGroups, input, remainder, flags, - static_cast(groupParams)); -#else - return CUDA_ERROR_NOT_SUPPORTED; -#endif -} - -bool has_sm_resource_split() noexcept { - return p_cuDevSmResourceSplit != nullptr; -} - -// ============================================================================ -// cuMemcpyWithAttributesAsync wrapper -// ============================================================================ - -CUresult memcpy_with_attributes_async(CUdeviceptr dst, CUdeviceptr src, size_t size, - void* attr, CUstream hStream) { -#if CUDA_VERSION >= 13020 - if (!p_cuMemcpyWithAttributesAsync) { - return CUDA_ERROR_NOT_SUPPORTED; - } - return p_cuMemcpyWithAttributesAsync( - dst, src, size, static_cast(attr), hStream); -#else - return CUDA_ERROR_NOT_SUPPORTED; -#endif -} - -bool has_memcpy_with_attributes_async() noexcept { - return p_cuMemcpyWithAttributesAsync != nullptr; -} - -} // namespace cuda_core diff --git a/cuda_core/cuda/core/_cpp/DESIGN.md b/cuda_core/cuda/core/_cpp/rt/DESIGN.md similarity index 69% rename from cuda_core/cuda/core/_cpp/DESIGN.md rename to cuda_core/cuda/core/_cpp/rt/DESIGN.md index 6615f21c4ba..46290d625de 100644 --- a/cuda_core/cuda/core/_cpp/DESIGN.md +++ b/cuda_core/cuda/core/_cpp/rt/DESIGN.md @@ -106,37 +106,47 @@ return as_py(h_stream) # cuda.bindings.driver.CUstream ``` cuda/core/ -├── _resource_handles.pyx # Cython module (compiles resource_handles.cpp) -├── _resource_handles.pxd # Cython declarations for consumer modules -└── _cpp/ - ├── resource_handles.hpp # C++ API declarations - └── resource_handles.cpp # C++ implementation +├── _rt.pyx # Cython module (compiles everything under _cpp/rt/) +├── _rt.pxd # Cython declarations for consumer modules +└── _cpp/rt/ + ├── rt.hpp # Module umbrella, named only by _rt.pyx + ├── handles.hpp # Consumer umbrella, named only by _rt.pxd + ├── types.hpp # Handle aliases, tagged values, inline accessors + ├── api.hpp # Prototypes of the handle factories and accessors + ├── driver_api.hpp/.cpp # Driver function-pointer table, version-gated shims + ├── error.hpp/.cpp # Thread-local error state, non-propagating reporting + ├── py.hpp # The one header that includes + ├── context_scope.hpp # Scoped-context helpers (namespace detail) + ├── internal.hpp # Registry, cleanup wrappers, deferred-cleanup item (namespace detail) + ├── py_report.cpp, py_deferred_cleanup.cpp # Python-coupled bodies + └── context.cpp, stream.cpp, event.cpp, memory.cpp, program.cpp, + graph.cpp, graph_exec.cpp, texture.cpp # One resource family per file ``` ### Build Implications -The `_cpp/` subdirectory contains C++ source files that are compiled into the -`_resource_handles` extension module. Other Cython modules in cuda.core do **not** -link against this code directly—they `cimport` functions from -`_resource_handles.pxd`, and calls go through `_resource_handles.so` at runtime. +Every `.cpp` under `_cpp/rt/` is compiled into the one `_rt` extension module. +Other Cython modules in cuda.core do **not** link against this code +directly—they `cimport` functions from `_rt.pxd`, and calls go through +`_rt.so` at runtime. ## Cross-Module Function Sharing **Problem**: Cython extension modules compile independently. If multiple modules -(`_memory.pyx`, `_ipc.pyx`, etc.) each linked `resource_handles.cpp`, they would +(`_memory.pyx`, `_ipc.pyx`, etc.) each linked the C++ under `_cpp/rt/`, they would each have their own copies of: - Static driver function pointers - Thread-local error state - Other static data, including global caches -**Solution**: Only `_resource_handles.so` links the C++ code. The `.pyx` file +**Solution**: Only `_rt.so` links the C++ code. The `.pyx` file uses `cdef extern from` to declare C++ functions with Cython-accessible names: ```cython -# In _resource_handles.pyx -cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": - StreamHandle create_stream_handle "cuda_core::create_stream_handle" ( +# In _rt.pyx +cdef extern from "_cpp/rt/rt.hpp" namespace "cuda_core::rt": + StreamHandle create_stream_handle "cuda_core::rt::create_stream_handle" ( ContextHandle h_ctx, unsigned int flags, int priority) nogil # ... other functions ``` @@ -144,18 +154,18 @@ cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": The `.pxd` file declares these same functions so other modules can `cimport` them: ```cython -# In _resource_handles.pxd +# In _rt.pxd cdef StreamHandle create_stream_handle( ContextHandle h_ctx, unsigned int flags, int priority) noexcept nogil ``` The `cdef extern from` declaration in the `.pyx` satisfies the `.pxd` declaration directly—no wrapper functions are needed. When consumer modules `cimport` these -functions, Cython generates calls through `_resource_handles.so` at runtime. +functions, Cython generates calls through `_rt.so` at runtime. This ensures all static and thread-local state lives in a single shared library, avoiding the duplicate state problem. -## CUDA Driver API Capsule (`_CUDA_DRIVER_API_V1`) +## CUDA driver function pointers via cuda-bindings' `__pyx_capi__` **Problem**: cuda.core cannot directly call CUDA driver functions because: @@ -165,13 +175,13 @@ avoiding the duplicate state problem. **Solution**: The C++ code declares extern function pointer variables: ```cpp -// resource_handles.hpp +// driver_api.hpp extern decltype(&cuStreamCreateWithPriority) p_cuStreamCreateWithPriority; extern decltype(&cuMemPoolCreate) p_cuMemPoolCreate; // ... etc ``` -At module import time, `_resource_handles.pyx` populates these pointers by +At module import time, `_rt.pyx` populates these pointers by extracting them from `cuda.bindings.cydriver.__pyx_capi__`: ```cython @@ -275,10 +285,56 @@ Related functions: - `peek_last_error()`: Returns the error without clearing it - `clear_last_error()`: Clears the error state +The C++ layer never raises Python exceptions: it runs `nogil` and `noexcept`, +and is called from deleters, CUDA callbacks and GIL-released code where raising +is impossible. Status is turned into `CUDAError` in one place, `HANDLE_RETURN` +in the Cython layer. Which status convention a function uses is decided by its +return value. Factories return the handle, so their status goes to thread-local +`err` and is read with `get_last_error()`. Functions that do not produce a +handle (`context_synchronize`, `context_get_device`, `graph_node_set_params`, +the `graph_*_attachment` family, `deviceptr_alloc_raw`) return the `CUresult` +directly and deliver results through out-parameters, mirroring the driver API; +their callers `HANDLE_RETURN` the value. The two conventions never mix. + +### Context-scoped operations + +Operations that must run in a specific context use `invoke_in_context` / +`invoke_in_context_or_undo` (propagating paths) and `cleanup_in_context` +(deleters). They switch the current context, run the operation, and restore the +caller's context. When restoration fails after the operation succeeded, the +creation is undone and the restoration status is returned. When both fail, the +operation status is returned. Either way the helper records a thread-local +detail keyed to the returned status (`take_last_error_detail(status)`) that +`_check_driver_error` attaches to the raised `CUDAError` as a PEP 678 note +(appended to the message on Python 3.10), so the user learns that the caller's +context was not restored, which context is current and, for a double failure, +why restoration failed. Keying the detail to its status keeps it from attaching +to an unrelated error if the caller never raises that status; `enter_context` +clears any stale detail. Tests inject restoration failures with +`set_context_restore_fault_for_testing()`. + +### Reporting from non-propagating paths + +Deleters and CUDA callbacks cannot raise. They report through +`report_cuda_error()` / `report_message()` (the `pw_*` wrappers decorate +destroy calls with it), which emit a `cuda.core.CUDAWarning` through +the Python warnings machinery when the interpreter is usable, deliver an +escalated warning as an unraisable exception, and fall back to stderr when the +GIL cannot be taken (for example during finalization). `CUDA_ERROR_DEINITIALIZED` +is never reported because it means the driver is shutting down. No status is +discarded silently anywhere in this layer, and nothing in this layer terminates +the process; see `docs/source/error_handling.rst` and the "Failure handling" +section of `AGENTS.md` for the policy. + +A rollback that fails inside a Cython `except` block is not a non-propagating +path: `note_or_report_cuda_error()` attaches it as a note to the exception being +handled (`PyErr_GetHandledException`, Python 3.11+) and falls back to a report +only when there is no such exception or notes are unavailable. + ## Usage from Cython ```cython -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( StreamHandle, create_stream_handle, as_cu, @@ -307,7 +363,7 @@ The resource handle design: 2. **Encodes lifetimes structurally** via embedded handle dependencies. 3. **Uses Cython's `cimport` mechanism** to share C++ code across modules without duplicate static/thread-local state. -4. **Uses a capsule** to resolve CUDA driver symbols dynamically through cuda-bindings. +4. **Resolves CUDA driver symbols** dynamically through cuda-bindings' `__pyx_capi__` capsules. 5. **Provides overloaded accessors** (`as_cu`, `as_intptr`, `as_py`) since handles cannot have attributes without unnecessary Python object wrappers. diff --git a/cuda_core/cuda/core/_cpp/GRAPH_ATTACHMENTS.md b/cuda_core/cuda/core/_cpp/rt/GRAPH_ATTACHMENTS.md similarity index 100% rename from cuda_core/cuda/core/_cpp/GRAPH_ATTACHMENTS.md rename to cuda_core/cuda/core/_cpp/rt/GRAPH_ATTACHMENTS.md diff --git a/cuda_core/cuda/core/_cpp/REGISTRY_DESIGN.md b/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md similarity index 97% rename from cuda_core/cuda/core/_cpp/REGISTRY_DESIGN.md rename to cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md index 42311eddb48..55d3fb043a9 100644 --- a/cuda_core/cuda/core/_cpp/REGISTRY_DESIGN.md +++ b/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md @@ -19,7 +19,7 @@ expires. ## Level 1: Driver Handle -> Resource Handle (C++) -`HandleRegistry` in `resource_handles.cpp` maps a raw CUDA handle +`HandleRegistry` in `_cpp/rt/internal.hpp` maps a raw CUDA handle (e.g., `CUevent`, `CUkernel`, `CUgraph`) to a `weak_ptr` for its owning resource handle. When a `_ref` constructor receives a raw handle, it checks the registry first. If found, it returns the existing diff --git a/cuda_core/cuda/core/_cpp/resource_handles.hpp b/cuda_core/cuda/core/_cpp/rt/api.hpp similarity index 51% rename from cuda_core/cuda/core/_cpp/resource_handles.hpp rename to cuda_core/cuda/core/_cpp/rt/api.hpp index 419710ea0d9..c52e0851cd1 100644 --- a/cuda_core/cuda/core/_cpp/resource_handles.hpp +++ b/cuda_core/cuda/core/_cpp/rt/api.hpp @@ -4,223 +4,12 @@ #pragma once -#include +#include "types.hpp" #include #include -#include -#include - -// Forward declaration for NVVM - avoids nvvm.h dependency -// Use void* to match cuda.bindings.cynvvm's typedef -using nvvmProgram = void*; - -// Forward declaration for nvJitLink - avoids nvJitLink.h dependency -// Use void* to match cuda.bindings.cynvjitlink's typedef -using nvJitLink_t = void*; - -namespace cuda_core { - -// ============================================================================ -// TaggedHandle - make void*-based handle types distinct for overloading -// -// Both nvvmProgram and nvJitLink_t are void*, so shared_ptr -// would be the same C++ type for both. TaggedHandle wraps the raw -// value with a unique tag type, making each shared_ptr type distinct. -// ============================================================================ - -template -struct TaggedHandle { - T raw; -}; - -using NvvmProgramValue = TaggedHandle; -using NvJitLinkValue = TaggedHandle; - -// CUtexObject, CUsurfObject and CUdeviceptr are all `unsigned long long`, so -// shared_ptr et al. would be the *same* C++ type as -// DevicePtrHandle (and each other), collapsing the as_cu/as_intptr/as_py -// overload sets. Tag them to keep each handle type distinct, exactly as the -// NVVM / nvJitLink handles above do. -using TexObjectValue = TaggedHandle; -using SurfObjectValue = TaggedHandle; - -// ============================================================================ -// Thread-local error handling -// ============================================================================ - -// Get and clear the last CUDA error (like cudaGetLastError) -CUresult get_last_error() noexcept; - -// Get the last CUDA error without clearing it (like cudaPeekAtLastError) -CUresult peek_last_error() noexcept; - -// Explicitly clear the last error -void clear_last_error() noexcept; - -// ============================================================================ -// CUDA driver function pointers -// -// These are populated by _resource_handles.pyx at module import time using -// function pointers extracted from cuda.bindings.cydriver.__pyx_capi__. -// ============================================================================ - -extern decltype(&cuGetErrorName) p_cuGetErrorName; -extern decltype(&cuGetErrorString) p_cuGetErrorString; - -extern decltype(&cuDevicePrimaryCtxRetain) p_cuDevicePrimaryCtxRetain; -extern decltype(&cuDevicePrimaryCtxRelease) p_cuDevicePrimaryCtxRelease; -extern decltype(&cuCtxGetCurrent) p_cuCtxGetCurrent; -extern decltype(&cuCtxSetCurrent) p_cuCtxSetCurrent; -extern decltype(&cuCtxSynchronize) p_cuCtxSynchronize; -extern decltype(&cuCtxGetStreamPriorityRange) p_cuCtxGetStreamPriorityRange; -extern decltype(&cuGreenCtxCreate) p_cuGreenCtxCreate; -extern decltype(&cuGreenCtxDestroy) p_cuGreenCtxDestroy; -extern decltype(&cuCtxFromGreenCtx) p_cuCtxFromGreenCtx; -extern decltype(&cuDevResourceGenerateDesc) p_cuDevResourceGenerateDesc; - -extern decltype(&cuGreenCtxStreamCreate) p_cuGreenCtxStreamCreate; - -extern decltype(&cuStreamCreateWithPriority) p_cuStreamCreateWithPriority; -extern decltype(&cuStreamDestroy) p_cuStreamDestroy; -extern decltype(&cuStreamGetCtx) p_cuStreamGetCtx; - -extern decltype(&cuEventCreate) p_cuEventCreate; -extern decltype(&cuEventDestroy) p_cuEventDestroy; -extern decltype(&cuIpcOpenEventHandle) p_cuIpcOpenEventHandle; - -extern decltype(&cuDeviceGetCount) p_cuDeviceGetCount; - -extern decltype(&cuMemPoolSetAccess) p_cuMemPoolSetAccess; -extern decltype(&cuMemPoolDestroy) p_cuMemPoolDestroy; -extern decltype(&cuMemPoolCreate) p_cuMemPoolCreate; -extern decltype(&cuDeviceGetMemPool) p_cuDeviceGetMemPool; -extern decltype(&cuMemPoolImportFromShareableHandle) p_cuMemPoolImportFromShareableHandle; - -extern decltype(&cuMemAllocFromPoolAsync) p_cuMemAllocFromPoolAsync; -extern decltype(&cuMemAllocAsync) p_cuMemAllocAsync; -extern decltype(&cuMemAlloc) p_cuMemAlloc; -extern decltype(&cuMemAllocHost) p_cuMemAllocHost; - -extern decltype(&cuMemFreeAsync) p_cuMemFreeAsync; -extern decltype(&cuMemFree) p_cuMemFree; -extern decltype(&cuMemFreeHost) p_cuMemFreeHost; - -extern decltype(&cuMemPoolImportPointer) p_cuMemPoolImportPointer; - -// Library -extern decltype(&cuLibraryLoadFromFile) p_cuLibraryLoadFromFile; -extern decltype(&cuLibraryLoadData) p_cuLibraryLoadData; -extern decltype(&cuLibraryUnload) p_cuLibraryUnload; -extern decltype(&cuLibraryGetKernel) p_cuLibraryGetKernel; - -// Graph -extern decltype(&cuGraphDestroy) p_cuGraphDestroy; -extern decltype(&cuGraphInstantiateWithParams) p_cuGraphInstantiateWithParams; -extern decltype(&cuGraphExecUpdate) p_cuGraphExecUpdate; -extern decltype(&cuGraphExecDestroy) p_cuGraphExecDestroy; -extern decltype(&cuUserObjectCreate) p_cuUserObjectCreate; -extern decltype(&cuUserObjectRelease) p_cuUserObjectRelease; -extern decltype(&cuGraphRetainUserObject) p_cuGraphRetainUserObject; -extern decltype(&cuGraphReleaseUserObject) p_cuGraphReleaseUserObject; -extern decltype(&cuGraphNodeFindInClone) p_cuGraphNodeFindInClone; -extern decltype(&cuGraphChildGraphNodeGetGraph) p_cuGraphChildGraphNodeGetGraph; - -// Linker -extern decltype(&cuLinkDestroy) p_cuLinkDestroy; - -// Graphics interop -extern decltype(&cuGraphicsUnmapResources) p_cuGraphicsUnmapResources; -extern decltype(&cuGraphicsUnregisterResource) p_cuGraphicsUnregisterResource; - -// Texture / surface / array (PR #467) -extern decltype(&cuArray3DCreate) p_cuArray3DCreate; -extern decltype(&cuArrayDestroy) p_cuArrayDestroy; -extern decltype(&cuMipmappedArrayCreate) p_cuMipmappedArrayCreate; -extern decltype(&cuMipmappedArrayDestroy) p_cuMipmappedArrayDestroy; -extern decltype(&cuMipmappedArrayGetLevel) p_cuMipmappedArrayGetLevel; -extern decltype(&cuTexObjectCreate) p_cuTexObjectCreate; -extern decltype(&cuTexObjectDestroy) p_cuTexObjectDestroy; -extern decltype(&cuSurfObjectCreate) p_cuSurfObjectCreate; -extern decltype(&cuSurfObjectDestroy) p_cuSurfObjectDestroy; - -// SM resource split (13.1+ — may be null on older drivers/bindings) -#if CUDA_VERSION >= 13010 -extern decltype(&cuDevSmResourceSplit) p_cuDevSmResourceSplit; -#else -// cuDevSmResourceSplit doesn't exist in CUDA < 13.1 headers, so use a -// void* placeholder. The pointer is always null when built against 12.x. -extern void* p_cuDevSmResourceSplit; -#endif - -// cuMemcpyWithAttributesAsync (13.2+ — may be null on older drivers/bindings) -#if CUDA_VERSION >= 13020 -extern decltype(&cuMemcpyWithAttributesAsync) p_cuMemcpyWithAttributesAsync; -#else -// cuMemcpyWithAttributesAsync doesn't exist in CUDA < 13.2 headers, so use a -// void* placeholder. The pointer is always null when built against older CUDA. -extern void* p_cuMemcpyWithAttributesAsync; -#endif - -// ============================================================================ -// NVRTC function pointers -// -// These are populated by _resource_handles.pyx at module import time using -// function pointers extracted from cuda.bindings.cynvrtc.__pyx_capi__. -// ============================================================================ - -extern decltype(&nvrtcDestroyProgram) p_nvrtcDestroyProgram; - -// ============================================================================ -// NVVM function pointers -// -// These are populated by _resource_handles.pyx at module import time using -// function pointers extracted from cuda.bindings.cynvvm.__pyx_capi__. -// Note: May be null if NVVM is not available at runtime. -// ============================================================================ - -// Function pointer type for nvvmDestroyProgram (avoids nvvm.h dependency) -// Signature: nvvmResult nvvmDestroyProgram(nvvmProgram *prog) -using NvvmDestroyProgramFn = int (*)(nvvmProgram*); -extern NvvmDestroyProgramFn p_nvvmDestroyProgram; - -// ============================================================================ -// nvJitLink function pointers -// -// These are populated by _resource_handles.pyx at module import time using -// function pointers extracted from cuda.bindings.cynvjitlink.__pyx_capi__. -// Note: May be null if nvJitLink is not available at runtime. -// ============================================================================ - -// Function pointer type for nvJitLinkDestroy (avoids nvJitLink.h dependency) -// Signature: nvJitLinkResult nvJitLinkDestroy(nvJitLinkHandle *handle) -using NvJitLinkDestroyFn = int (*)(nvJitLink_t*); -extern NvJitLinkDestroyFn p_nvJitLinkDestroy; - -// ============================================================================ -// Handle type aliases - expose only the raw CUDA resource -// ============================================================================ - -using ContextHandle = std::shared_ptr; -using GreenCtxHandle = std::shared_ptr; -using StreamHandle = std::shared_ptr; -using EventHandle = std::shared_ptr; -using MemoryPoolHandle = std::shared_ptr; -using LibraryHandle = std::shared_ptr; -using KernelHandle = std::shared_ptr; -using GraphHandle = std::shared_ptr; -using GraphExecHandle = std::shared_ptr; -using GraphNodeHandle = std::shared_ptr; -using GraphicsResourceHandle = std::shared_ptr; -using NvrtcProgramHandle = std::shared_ptr; -using NvvmProgramHandle = std::shared_ptr; -using NvJitLinkHandle = std::shared_ptr; -using CuLinkHandle = std::shared_ptr; -using FileDescriptorHandle = std::shared_ptr; -using OpaqueArrayHandle = std::shared_ptr; -using MipmappedArrayHandle = std::shared_ptr; -using TexObjectHandle = std::shared_ptr; -using SurfObjectHandle = std::shared_ptr; +#include +namespace cuda_core::rt { // ============================================================================ // Context handle functions @@ -263,6 +52,22 @@ CUresult context_get_stream_priority_range( int* least_priority, int* greatest_priority) noexcept; +// Query the device of the provided context. +// Returns CUDA_ERROR_INVALID_CONTEXT for an empty handle. +CUresult context_get_device(const ContextHandle& h_context, CUdevice* device) noexcept; + +// Call cuGraphNodeSetParams with h_context current (empty handle: the caller's +// context). Returns the update status; *restore_status receives a failure to +// restore the caller's context after a successful update, which the caller +// raises only after publishing the metadata that depends on the update. +// Returns CUDA_ERROR_NOT_SUPPORTED when the driver lacks cuGraphNodeSetParams. +// Implemented in graph.cpp +CUresult graph_node_set_params( + CUgraphNode node, + CUgraphNodeParams* params, + const ContextHandle& h_context, + CUresult* restore_status) noexcept; + // ============================================================================ // Stream handle functions // ============================================================================ @@ -279,18 +84,11 @@ StreamHandle create_stream_handle(const ContextHandle& h_ctx, unsigned int flags // Caller is responsible for keeping the stream's context alive. StreamHandle create_stream_handle_ref(CUstream stream); -// Create a non-owning stream handle that prevents a Python owner from being GC'd. -// The owner's refcount is incremented; decremented when handle is released. -// The owner is responsible for keeping the stream's context alive. -StreamHandle create_stream_handle_with_owner(CUstream stream, PyObject* owner); - -// Destroy a Python-backed CUDA user object by decref'ing it when safe. -// If Python is finalized or finalizing, the object is intentionally leaked. -void py_object_user_object_destroy(void* py_object) noexcept; - // Initialize the process-lifetime CUDA user-object cleanup queue. Called once // from module initialization while Python is fully initialized. +// Implemented in py_deferred_cleanup.cpp void initialize_deferred_cleanup(); +// Implemented in py_deferred_cleanup.cpp void retry_deferred_cleanup() noexcept; // Return the context dependency associated with a stream handle, if any. @@ -383,8 +181,6 @@ MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType han // Device pointer handle functions // ============================================================================ -using DevicePtrHandle = std::shared_ptr; - // Allocate device memory from a pool asynchronously via cuMemAllocFromPoolAsync. // The pointer structurally depends on the provided pool handle (captured in deleter). // When the last reference is released, cuMemFreeAsync is called on the stored stream. @@ -415,12 +211,6 @@ DevicePtrHandle deviceptr_alloc_host(size_t size); // The pointer will NOT be freed when the handle is released. DevicePtrHandle deviceptr_create_ref(CUdeviceptr ptr); -// Create a non-owning device pointer handle that prevents a Python owner from being GC'd. -// The owner's refcount is incremented; decremented when handle is released. -// The pointer will NOT be freed when the handle is released. -// If owner is nullptr, equivalent to deviceptr_create_ref. -DevicePtrHandle deviceptr_create_with_owner(CUdeviceptr ptr, PyObject* owner); - // Create a device pointer handle for a mapped graphics resource. // The pointer structurally depends on the provided graphics resource handle. // When the last reference is released, cuGraphicsUnmapResources is called on @@ -431,23 +221,6 @@ DevicePtrHandle deviceptr_create_mapped_graphics( const GraphicsResourceHandle& h_resource, const StreamHandle& h_stream); -// Callback type for MemoryResource deallocation. -// Called from the shared_ptr deleter when a handle created via -// deviceptr_create_with_mr is destroyed. The implementation is responsible -// for converting raw C types to Python objects and calling -// mr.deallocate(ptr, size, stream). -using MRDeallocCallback = void (*)(PyObject* mr, CUdeviceptr ptr, - size_t size, const StreamHandle& stream); - -// Register the MR deallocation callback. -void register_mr_dealloc_callback(MRDeallocCallback cb); - -// Create a device pointer handle whose destructor calls mr.deallocate() -// via the registered callback. The mr's refcount is incremented and -// decremented when the handle is released. -// If mr is nullptr, equivalent to deviceptr_create_ref. -DevicePtrHandle deviceptr_create_with_mr(CUdeviceptr ptr, size_t size, PyObject* mr); - // Import a device pointer from IPC via cuMemPoolImportPointer. // When the last reference is released, cuMemFreeAsync is called on the stored stream. // Note: Does not yet implement reference counting for nvbug 5570902. @@ -534,53 +307,9 @@ GraphHandle create_child_graph_handle( // after CUDA copies or destroys graph state. // ============================================================================ -// Type-erased shared owner of an attached resource. Typed handles such as -// EventHandle and KernelHandle convert to OpaqueHandle by assignment, reusing -// their existing control block; the helpers below build OpaqueHandles for the -// two cases that need a custom deleter. -using OpaqueHandle = std::shared_ptr; - -// Build an OpaqueHandle from a Python object: increments its refcount now and -// decrements it (under the GIL) on release. The caller must hold the GIL. -OpaqueHandle make_opaque_py(PyObject* obj); - // Build an OpaqueHandle from a malloc'd buffer: std::free on release. OpaqueHandle make_opaque_malloc(void* buf); -struct PreparedAttachmentState; -using PreparedAttachmentRollback = - void (*)(PreparedAttachmentState*) noexcept; -struct PreparedAttachmentDeleter { - PreparedAttachmentRollback rollback = nullptr; - - void operator()(PreparedAttachmentState* state) const noexcept { - rollback(state); - } -}; -using PreparedAttachment = - std::unique_ptr; - -struct PreparedChildGraphUpdateState; -// Opaque unpublished hierarchy transaction; releasing it discards staged -// metadata unless graph_commit_child_graph_update publishes the replacement. -using PreparedChildGraphUpdate = - std::shared_ptr; - -struct PreparedExecAttachmentState; -using PreparedExecAttachmentRollback = - void (*)(PreparedExecAttachmentState*) noexcept; -struct PreparedExecAttachmentDeleter { - PreparedExecAttachmentRollback rollback = nullptr; - - void operator()(PreparedExecAttachmentState* state) const noexcept { - rollback(state); - } -}; -// Opaque append transaction. Releasing it rolls back newly appended owners -// unless graph_commit_exec_attachment has kept them. -using PreparedExecAttachment = - std::unique_ptr; - // Copy requested owners from node's current attachment. Pass nullptr to ignore // either owner; a missing attachment produces empty handles. CUresult graph_get_attachment( @@ -821,359 +550,4 @@ SurfObjectHandle create_surf_object_handle(const ContextHandle& h_context, const CUDA_RESOURCE_DESC& res, const OpaqueArrayHandle& h_backing); -// ============================================================================ -// Overloaded helper functions to extract raw resources from handles -// ============================================================================ - -// as_cu() - extract the raw CUDA handle -inline CUcontext as_cu(const ContextHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUgreenCtx as_cu(const GreenCtxHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUstream as_cu(const StreamHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUevent as_cu(const EventHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUmemoryPool as_cu(const MemoryPoolHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUdeviceptr as_cu(const DevicePtrHandle& h) noexcept { - return h ? *h : 0; -} - -inline CUlibrary as_cu(const LibraryHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUmodule as_cu(const CUmodule& h) noexcept { - return h; -} - -inline CUkernel as_cu(const KernelHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUgraph as_cu(const GraphHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUgraphExec as_cu(const GraphExecHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUgraphNode as_cu(const GraphNodeHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUgraphicsResource as_cu(const GraphicsResourceHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline nvrtcProgram as_cu(const NvrtcProgramHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline nvvmProgram as_cu(const NvvmProgramHandle& h) noexcept { - return h ? h->raw : nullptr; -} - -inline nvJitLink_t as_cu(const NvJitLinkHandle& h) noexcept { - return h ? h->raw : nullptr; -} - -inline CUlinkState as_cu(const CuLinkHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUarray as_cu(const OpaqueArrayHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUmipmappedArray as_cu(const MipmappedArrayHandle& h) noexcept { - return h ? *h : nullptr; -} - -// CUtexObject / CUsurfObject are integer-valued (like CUdeviceptr); null is 0. -// The raw value lives in the tagged wrapper's `raw` field. -inline CUtexObject as_cu(const TexObjectHandle& h) noexcept { - return h ? h->raw : 0; -} - -inline CUsurfObject as_cu(const SurfObjectHandle& h) noexcept { - return h ? h->raw : 0; -} - -// as_intptr() - extract handle as intptr_t for Python interop -// Using signed intptr_t per C standard convention and issue #1342 -inline std::intptr_t as_intptr(const ContextHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const GreenCtxHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const StreamHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const EventHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const MemoryPoolHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const DevicePtrHandle& h) noexcept { - return static_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const LibraryHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const CUmodule& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const KernelHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const GraphHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const GraphExecHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const GraphNodeHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const GraphicsResourceHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const NvrtcProgramHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const NvvmProgramHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const NvJitLinkHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const CuLinkHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const FileDescriptorHandle& h) noexcept { - return h ? static_cast(*h) : -1; -} - -inline std::intptr_t as_intptr(const OpaqueArrayHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const MipmappedArrayHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const TexObjectHandle& h) noexcept { - return static_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const SurfObjectHandle& h) noexcept { - return static_cast(as_cu(h)); -} - -// as_py() - convert handle to Python wrapper object (returns new reference) -#if PY_VERSION_HEX < 0x030D0000 -extern "C" int _Py_IsFinalizing(void); -#endif - -// Best-effort probe for interpreter shutdown. -// -// In CPython this is not a hard guarantee: finalization can begin after this -// returns false but before a later PyGILState_Ensure() or other Python C-API -// call. -// -// If that race is lost on a non-finalizer thread, CPython's behavior is -// version-dependent: on older supported versions (3.10-3.13) it may abruptly -// terminate the current thread (historically via PyThread_exit_thread(), -// without normal C++ unwinding), while on newer versions (3.14+) it may hang -// the thread until process exit. -// -// We still use this check because the policy in this layer is to avoid Python -// work once shutdown is underway and accept an intentional leak or skipped -// Python conversion in that edge case rather than add more complex deferral -// machinery. -inline bool py_is_finalizing() noexcept { -#if PY_VERSION_HEX >= 0x030D0000 - return Py_IsFinalizing(); -#else - return _Py_IsFinalizing() != 0; -#endif -} - -namespace detail { -// n.b. class lookup is not cached to avoid deadlock hazard, see DESIGN.md -inline PyObject* make_py(const char* module_name, const char* class_name, std::intptr_t value) noexcept { - if (py_is_finalizing()) { - Py_RETURN_NONE; - } - PyObject* mod = PyImport_ImportModule(module_name); - if (!mod) return nullptr; - PyObject* cls = PyObject_GetAttrString(mod, class_name); - Py_DECREF(mod); - if (!cls) return nullptr; - PyObject* result = PyObject_CallFunction(cls, "L", value); - Py_DECREF(cls); - return result; -} -} // namespace detail - -inline PyObject* as_py(const ContextHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUcontext", as_intptr(h)); -} - -inline PyObject* as_py(const GreenCtxHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUgreenCtx", as_intptr(h)); -} - -inline PyObject* as_py(const StreamHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUstream", as_intptr(h)); -} - -inline PyObject* as_py(const EventHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUevent", as_intptr(h)); -} - -inline PyObject* as_py(const MemoryPoolHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUmemoryPool", as_intptr(h)); -} - -inline PyObject* as_py(const DevicePtrHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUdeviceptr", as_intptr(h)); -} - -inline PyObject* as_py(const LibraryHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUlibrary", as_intptr(h)); -} - -inline PyObject* as_py(const CUmodule& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUmodule", as_intptr(h)); -} - -inline PyObject* as_py(const KernelHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUkernel", as_intptr(h)); -} - -inline PyObject* as_py(const GraphHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUgraph", as_intptr(h)); -} - -inline PyObject* as_py(const GraphExecHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUgraphExec", as_intptr(h)); -} - -inline PyObject* as_py(const GraphNodeHandle& h) noexcept { - if (!as_intptr(h)) { - Py_RETURN_NONE; - } - return detail::make_py("cuda.bindings.driver", "CUgraphNode", as_intptr(h)); -} - -inline PyObject* as_py(const NvrtcProgramHandle& h) noexcept { - return detail::make_py("cuda.bindings.nvrtc", "nvrtcProgram", as_intptr(h)); -} - -inline PyObject* as_py(const NvvmProgramHandle& h) noexcept { - // NVVM bindings use raw integers, not wrapper classes - return PyLong_FromSsize_t(as_intptr(h)); -} - -inline PyObject* as_py(const NvJitLinkHandle& h) noexcept { - // nvJitLink bindings use raw integers, not wrapper classes - return PyLong_FromSsize_t(as_intptr(h)); -} - -inline PyObject* as_py(const CuLinkHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUlinkState", as_intptr(h)); -} - -inline PyObject* as_py(const GraphicsResourceHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUgraphicsResource", as_intptr(h)); -} - -inline PyObject* as_py(const FileDescriptorHandle& h) noexcept { - return PyLong_FromSsize_t(as_intptr(h)); -} - -inline PyObject* as_py(const OpaqueArrayHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUarray", as_intptr(h)); -} - -inline PyObject* as_py(const MipmappedArrayHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUmipmappedArray", as_intptr(h)); -} - -inline PyObject* as_py(const TexObjectHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUtexObject", as_intptr(h)); -} - -inline PyObject* as_py(const SurfObjectHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUsurfObject", as_intptr(h)); -} - -// ============================================================================ -// SM resource split wrapper (13.1+) -// -// Calls through p_cuDevSmResourceSplit if available, otherwise returns -// CUDA_ERROR_NOT_SUPPORTED. This avoids a direct Cython cimport of the -// cydriver cdef function, which would fail at module init on cuda-bindings -// < 13.1 (see https://github.com/NVIDIA/cuda-python/issues/2063). -// ============================================================================ - -// groupParams is void* so the Cython declaration doesn't reference -// CU_DEV_SM_RESOURCE_GROUP_PARAMS (absent from cuda-bindings 13.0 .pxd). -CUresult sm_resource_split(CUdevResource* result, unsigned int nbGroups, - const CUdevResource* input, CUdevResource* remainder, - unsigned int flags, void* groupParams); - -// Returns true if the cuDevSmResourceSplit function pointer is available. -bool has_sm_resource_split() noexcept; - -// ============================================================================ -// cuMemcpyWithAttributesAsync wrapper (13.2+) -// -// Calls through p_cuMemcpyWithAttributesAsync if available, otherwise returns -// CUDA_ERROR_NOT_SUPPORTED. This avoids a direct Cython cimport of the -// cydriver cdef function, which would fail at module init on cuda-bindings -// < 13.2 (see https://github.com/NVIDIA/cuda-python/issues/2063). -// ============================================================================ - -// attr is void* so the Cython declaration doesn't reference CUmemcpyAttributes -// (absent from cuda-bindings built against CUDA < 12.8). The C++ side casts it. -CUresult memcpy_with_attributes_async(CUdeviceptr dst, CUdeviceptr src, size_t size, - void* attr, CUstream hStream); - -// Returns true if the cuMemcpyWithAttributesAsync function pointer is available. -bool has_memcpy_with_attributes_async() noexcept; - -} // namespace cuda_core +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/context.cpp b/cuda_core/cuda/core/_cpp/rt/context.cpp new file mode 100644 index 00000000000..cd60db109bb --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/context.cpp @@ -0,0 +1,267 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "context_scope.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +namespace { +// Thread-local fault injected into the next context restoration (tests only). +thread_local CUresult context_restore_fault = CUDA_SUCCESS; +} // namespace + +void set_context_restore_fault_for_testing(CUresult status) noexcept { + context_restore_fault = status; +} + +namespace detail { +// Make a context current and record the state needed to restore it. +// An empty handle is a no-op: the operation runs in the caller's current +// context, and nothing is restored on exit. invoke_in_context and +// invoke_in_context_or_undo reject empty handles before getting here; only +// graph_node_set_params relies on the no-op (pre-13.2 node updates run in the +// caller's context). +CUresult enter_context(const ContextHandle& h_context, CUcontext* previous, int* changed) noexcept { + *previous = nullptr; + *changed = 0; + clear_last_error_detail(); + CUcontext target = as_cu(h_context); + if (!target) { + return CUDA_SUCCESS; + } + + GILReleaseGuard gil; + CUresult status = p_cuCtxGetCurrent(previous); + if (status != CUDA_SUCCESS || *previous == target) { + return status; + } + status = p_cuCtxSetCurrent(target); + *changed = status == CUDA_SUCCESS; + return status; +} + +// Restore the caller's context. Returns the restoration status. +CUresult restore_context(CUcontext previous) noexcept { + if (context_restore_fault != CUDA_SUCCESS) { + // Test hook: behave as if cuCtxSetCurrent(previous) failed, leaving the + // target context current exactly as a real failure would. + CUresult fault = context_restore_fault; + context_restore_fault = CUDA_SUCCESS; + return fault; + } + GILReleaseGuard gil; + return p_cuCtxSetCurrent(previous); +} +// Restore the previous context and preserve an earlier operation error. The +// operation error, if any, is returned; otherwise the restoration status is. +// Either way a restoration failure is recorded as the detail of the returned +// status, so the eventual CUDAError explains it (see take_last_error_detail()). +CUresult exit_context(CUcontext previous, int changed, CUresult operation_status) noexcept { + CUresult restore_status = changed ? restore_context(previous) : CUDA_SUCCESS; + if (restore_status == CUDA_SUCCESS) { + return operation_status; + } + note_context_not_restored(previous, operation_status, restore_status); + return operation_status != CUDA_SUCCESS ? operation_status : restore_status; +} +} // namespace detail + +// Synchronize the provided context. +CUresult context_synchronize(const ContextHandle& h_context) noexcept { + GILReleaseGuard gil; + return invoke_in_context(h_context, []() noexcept { + return p_cuCtxSynchronize(); + }); +} + +// Query the stream priority range for the provided context. +CUresult context_get_stream_priority_range(const ContextHandle& h_context, + int* least_priority, + int* greatest_priority) noexcept { + GILReleaseGuard gil; + return invoke_in_context(h_context, [&]() noexcept { + return p_cuCtxGetStreamPriorityRange(least_priority, greatest_priority); + }); +} + +// Query the device of the provided context. +CUresult context_get_device(const ContextHandle& h_context, CUdevice* device) noexcept { + return invoke_in_context(h_context, [&]() noexcept { + return p_cuCtxGetDevice(device); + }); +} + +// ============================================================================ +// Context Handles +// ============================================================================ + +namespace { +struct ContextBox { + CUcontext resource; + GreenCtxHandle h_green_ctx; +}; + +struct GreenCtxBox { + CUgreenCtx resource; +}; + +static const ContextBox* get_box(const ContextHandle& h) noexcept { + const CUcontext* p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) - offsetof(ContextBox, resource) + ); +} + +// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) +static HandleRegistry context_registry; + +// Create a context handle reference, with optional green context as source. +ContextHandle create_context_handle_ref(CUcontext ctx, GreenCtxHandle h_green_ctx) { + if (!ctx) { + return {}; + } + if (auto h = context_registry.lookup(ctx)) { + return h; + } + auto box = std::shared_ptr( + new ContextBox{ctx, std::move(h_green_ctx)}, + [](const ContextBox* b) { + context_registry.unregister_handle(b->resource); + delete b; + } + ); + ContextHandle h(box, &box->resource); + context_registry.register_handle(ctx, h); + return h; +} +} // namespace + +ContextHandle create_context_handle_ref(CUcontext ctx) { + return create_context_handle_ref(ctx, {}); +} + +ContextHandle create_context_handle_from_green_ctx(const GreenCtxHandle& h_green_ctx) { + GILReleaseGuard gil; + if (!h_green_ctx) { + return {}; + } + if (!p_cuCtxFromGreenCtx) { + err = CUDA_ERROR_NOT_SUPPORTED; + return {}; + } + + CUcontext ctx = nullptr; + if (CUDA_SUCCESS != (err = p_cuCtxFromGreenCtx(&ctx, as_cu(h_green_ctx)))) { + return {}; + } + + return create_context_handle_ref(ctx, h_green_ctx); +} + +GreenCtxHandle get_context_green_ctx(const ContextHandle& h) noexcept { + if (!h) { + return {}; + } + return get_box(h)->h_green_ctx; +} + +GreenCtxHandle create_green_ctx_handle(CUdevResource* resources, unsigned int nbResources, + CUdevice dev, unsigned int flags) { + GILReleaseGuard gil; + if (!p_cuDevResourceGenerateDesc || !p_cuGreenCtxCreate || !p_cuGreenCtxDestroy) { + err = CUDA_ERROR_NOT_SUPPORTED; + return {}; + } + + CUdevResourceDesc desc = nullptr; + if (CUDA_SUCCESS != (err = p_cuDevResourceGenerateDesc(&desc, resources, nbResources))) { + return {}; + } + + CUgreenCtx green_ctx = nullptr; + if (CUDA_SUCCESS != (err = p_cuGreenCtxCreate(&green_ctx, desc, dev, flags))) { + return {}; + } + + auto box = std::shared_ptr( + new GreenCtxBox{green_ctx}, + [](const GreenCtxBox* b) { + GILReleaseGuard gil; + pw_cuGreenCtxDestroy(b->resource); + delete b; + } + ); + return GreenCtxHandle(box, &box->resource); +} + +GreenCtxHandle create_green_ctx_handle_ref(CUgreenCtx green_ctx) { + if (!green_ctx) { + return {}; + } + auto box = std::make_shared(GreenCtxBox{green_ctx}); + return GreenCtxHandle(box, &box->resource); +} + +// Thread-local cache of primary contexts indexed by device ID +static thread_local std::vector primary_context_cache; + +ContextHandle get_primary_context(int device_id) { + // Check thread-local cache + if (static_cast(device_id) < primary_context_cache.size()) { + if (auto cached = primary_context_cache[device_id]) { + return cached; + } + } + + // Cache miss - acquire primary context from driver + GILReleaseGuard gil; + CUcontext ctx; + if (CUDA_SUCCESS != (err = p_cuDevicePrimaryCtxRetain(&ctx, device_id))) { + return {}; + } + + auto box = std::shared_ptr( + new ContextBox{ctx, {}}, + [device_id](const ContextBox* b) { + context_registry.unregister_handle(b->resource); + GILReleaseGuard gil; + p_cuDevicePrimaryCtxRelease(device_id); + delete b; + } + ); + auto h = ContextHandle(box, &box->resource); + context_registry.register_handle(ctx, h); + + // Update cache + if (static_cast(device_id) >= primary_context_cache.size()) { + primary_context_cache.resize(device_id + 1); + } + primary_context_cache[device_id] = h; + return h; +} + +ContextHandle get_current_context() { + GILReleaseGuard gil; + CUcontext ctx = nullptr; + if (CUDA_SUCCESS != (err = p_cuCtxGetCurrent(&ctx))) { + return {}; + } + if (!ctx) { + return {}; // No current context (not an error) + } + return create_context_handle_ref(ctx); +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/context_scope.hpp b/cuda_core/cuda/core/_cpp/rt/context_scope.hpp new file mode 100644 index 00000000000..5ae22e15788 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/context_scope.hpp @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "types.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include +#include +#include + +namespace cuda_core::rt::detail { + +// Implemented in context.cpp +CUresult enter_context(const ContextHandle& h_context, CUcontext* previous, int* changed) noexcept; +// Implemented in context.cpp +CUresult restore_context(CUcontext previous) noexcept; +// Implemented in context.cpp +CUresult exit_context(CUcontext previous, int changed, CUresult operation_status) noexcept; + +// Require a callable to be invocable without throwing. +#define ASSERT_NOTHROW_INVOCABLE(...) \ + static_assert(std::is_nothrow_invocable_v<__VA_ARGS__>, "operation must be noexcept") + +// Run an operation with the requested context current. +template +CUresult invoke_in_context(const ContextHandle& h_context, Fn&& operation, Args&&... args) noexcept { + ASSERT_NOTHROW_INVOCABLE(Fn&&, Args&&...); + if (!h_context) { + return CUDA_ERROR_INVALID_CONTEXT; + } + CUcontext previous = nullptr; + int changed = 0; + CUresult status = enter_context(h_context, &previous, &changed); + if (status == CUDA_SUCCESS) { + status = std::invoke(std::forward(operation), std::forward(args)...); + } + return exit_context(previous, changed, status); +} + +// Run a creation operation and undo it if context restoration fails. +// Context-independent undo always runs. Context-sensitive undo runs only +// after verifying that the target context remains current; otherwise the +// resource leaks rather than risking cleanup in the wrong context. +template +CUresult invoke_in_context_or_undo(const ContextHandle& h_context, Fn&& operation, + Undo&& undo, bool undo_requires_target_context) noexcept { + ASSERT_NOTHROW_INVOCABLE(Fn&&); + ASSERT_NOTHROW_INVOCABLE(Undo&&); + if (!h_context) { + return CUDA_ERROR_INVALID_CONTEXT; + } + CUcontext previous = nullptr; + int changed = 0; + CUresult status = enter_context(h_context, &previous, &changed); + if (status != CUDA_SUCCESS) { + return status; + } + status = std::invoke(std::forward(operation)); + CUresult composite = exit_context(previous, changed, status); + if (status == CUDA_SUCCESS && composite != CUDA_SUCCESS) { + bool undo_ok = true; + if (undo_requires_target_context) { + CUcontext current = nullptr; + undo_ok = p_cuCtxGetCurrent(¤t) == CUDA_SUCCESS + && current == as_cu(h_context); + } + if (undo_ok) { + std::invoke(std::forward(undo)); + } else { + report_cuda_error( + "cuCtxSetCurrent (restoring the caller's context)", composite, + "failed; cleanup of the new resource skipped because its context " + "is no longer current (resource leaked)"); + } + } + return composite; +} + +// Run cleanup with the requested context current. Warn and skip the operation +// if activation fails, and independently warn on operation or restoration +// failure. Return the operation or activation status; restoration never +// changes the return value. +template +CUresult cleanup_in_context(const ContextHandle& h_context, const char* name, + Fn&& operation, Args&&... args) noexcept { + ASSERT_NOTHROW_INVOCABLE(Fn&&, Args&&...); + CUcontext previous = nullptr; + int changed = 0; + CUresult status = enter_context(h_context, &previous, &changed); + if (status != CUDA_SUCCESS) { + report_cuda_error(name, status, + "skipped (context activation failed; resource leaked)"); + } else { + status = std::invoke(std::forward(operation), std::forward(args)...); + if (status != CUDA_SUCCESS) { + report_cuda_error(name, status); + } + } + CUresult restore = exit_context(previous, changed, CUDA_SUCCESS); + if (restore != CUDA_SUCCESS) { + // Nothing is raised here, so the detail exit_context recorded has no + // exception to attach to: report it and drop the detail. + report_cuda_error(name, restore, "failed while restoring the caller's context"); + clear_last_error_detail(); + } + return status; +} + +#undef ASSERT_NOTHROW_INVOCABLE + +} // namespace cuda_core::rt::detail diff --git a/cuda_core/cuda/core/_cpp/rt/driver_api.cpp b/cuda_core/cuda/core/_cpp/rt/driver_api.cpp new file mode 100644 index 00000000000..860a29a3554 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/driver_api.cpp @@ -0,0 +1,164 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "driver_api.hpp" +#include +#include + +namespace cuda_core::rt { + +// ============================================================================ +// CUDA driver function pointers +// +// These are populated by _rt.pyx at module import time using +// function pointers extracted from cuda.bindings.cydriver.__pyx_capi__. +// ============================================================================ + +decltype(&cuGetErrorName) p_cuGetErrorName = nullptr; +decltype(&cuGetErrorString) p_cuGetErrorString = nullptr; + +decltype(&cuDevicePrimaryCtxRetain) p_cuDevicePrimaryCtxRetain = nullptr; +decltype(&cuDevicePrimaryCtxRelease) p_cuDevicePrimaryCtxRelease = nullptr; +decltype(&cuCtxGetCurrent) p_cuCtxGetCurrent = nullptr; +decltype(&cuCtxSetCurrent) p_cuCtxSetCurrent = nullptr; +decltype(&cuCtxSynchronize) p_cuCtxSynchronize = nullptr; +decltype(&cuCtxGetStreamPriorityRange) p_cuCtxGetStreamPriorityRange = nullptr; +decltype(&cuCtxGetDevice) p_cuCtxGetDevice = nullptr; +decltype(&cuGraphNodeSetParams) p_cuGraphNodeSetParams = nullptr; +decltype(&cuGreenCtxCreate) p_cuGreenCtxCreate = nullptr; +decltype(&cuGreenCtxDestroy) p_cuGreenCtxDestroy = nullptr; +decltype(&cuCtxFromGreenCtx) p_cuCtxFromGreenCtx = nullptr; +decltype(&cuDevResourceGenerateDesc) p_cuDevResourceGenerateDesc = nullptr; + +decltype(&cuGreenCtxStreamCreate) p_cuGreenCtxStreamCreate = nullptr; + +decltype(&cuStreamCreateWithPriority) p_cuStreamCreateWithPriority = nullptr; +decltype(&cuStreamDestroy) p_cuStreamDestroy = nullptr; +decltype(&cuStreamGetCtx) p_cuStreamGetCtx = nullptr; + +decltype(&cuEventCreate) p_cuEventCreate = nullptr; +decltype(&cuEventDestroy) p_cuEventDestroy = nullptr; +decltype(&cuIpcOpenEventHandle) p_cuIpcOpenEventHandle = nullptr; + +decltype(&cuDeviceGetCount) p_cuDeviceGetCount = nullptr; + +decltype(&cuMemPoolSetAccess) p_cuMemPoolSetAccess = nullptr; +decltype(&cuMemPoolDestroy) p_cuMemPoolDestroy = nullptr; +decltype(&cuMemPoolCreate) p_cuMemPoolCreate = nullptr; +decltype(&cuDeviceGetMemPool) p_cuDeviceGetMemPool = nullptr; +decltype(&cuMemPoolImportFromShareableHandle) p_cuMemPoolImportFromShareableHandle = nullptr; + +decltype(&cuMemAllocFromPoolAsync) p_cuMemAllocFromPoolAsync = nullptr; +decltype(&cuMemAllocAsync) p_cuMemAllocAsync = nullptr; +decltype(&cuMemAlloc) p_cuMemAlloc = nullptr; +decltype(&cuMemAllocHost) p_cuMemAllocHost = nullptr; + +decltype(&cuMemFreeAsync) p_cuMemFreeAsync = nullptr; +decltype(&cuMemFree) p_cuMemFree = nullptr; +decltype(&cuMemFreeHost) p_cuMemFreeHost = nullptr; + +decltype(&cuMemPoolImportPointer) p_cuMemPoolImportPointer = nullptr; + +decltype(&cuLibraryLoadFromFile) p_cuLibraryLoadFromFile = nullptr; +decltype(&cuLibraryLoadData) p_cuLibraryLoadData = nullptr; +decltype(&cuLibraryUnload) p_cuLibraryUnload = nullptr; +decltype(&cuLibraryGetKernel) p_cuLibraryGetKernel = nullptr; + +// Graph +decltype(&cuGraphDestroy) p_cuGraphDestroy = nullptr; +decltype(&cuGraphInstantiateWithParams) p_cuGraphInstantiateWithParams = nullptr; +decltype(&cuGraphExecUpdate) p_cuGraphExecUpdate = nullptr; +decltype(&cuGraphExecDestroy) p_cuGraphExecDestroy = nullptr; +decltype(&cuUserObjectCreate) p_cuUserObjectCreate = nullptr; +decltype(&cuUserObjectRelease) p_cuUserObjectRelease = nullptr; +decltype(&cuGraphRetainUserObject) p_cuGraphRetainUserObject = nullptr; +decltype(&cuGraphReleaseUserObject) p_cuGraphReleaseUserObject = nullptr; +decltype(&cuGraphNodeFindInClone) p_cuGraphNodeFindInClone = nullptr; +decltype(&cuGraphChildGraphNodeGetGraph) p_cuGraphChildGraphNodeGetGraph = nullptr; + +// Linker +decltype(&cuLinkDestroy) p_cuLinkDestroy = nullptr; + +// GL interop pointers +decltype(&cuGraphicsUnmapResources) p_cuGraphicsUnmapResources = nullptr; +decltype(&cuGraphicsUnregisterResource) p_cuGraphicsUnregisterResource = nullptr; + +decltype(&cuArray3DCreate) p_cuArray3DCreate = nullptr; +decltype(&cuArrayDestroy) p_cuArrayDestroy = nullptr; +decltype(&cuMipmappedArrayCreate) p_cuMipmappedArrayCreate = nullptr; +decltype(&cuMipmappedArrayDestroy) p_cuMipmappedArrayDestroy = nullptr; +decltype(&cuMipmappedArrayGetLevel) p_cuMipmappedArrayGetLevel = nullptr; +decltype(&cuTexObjectCreate) p_cuTexObjectCreate = nullptr; +decltype(&cuTexObjectDestroy) p_cuTexObjectDestroy = nullptr; +decltype(&cuSurfObjectCreate) p_cuSurfObjectCreate = nullptr; +decltype(&cuSurfObjectDestroy) p_cuSurfObjectDestroy = nullptr; + +// SM resource split (13.1+ — may be null on older drivers/bindings) +#if CUDA_VERSION >= 13010 +decltype(&cuDevSmResourceSplit) p_cuDevSmResourceSplit = nullptr; +#else +void* p_cuDevSmResourceSplit = nullptr; +#endif + +// cuMemcpyWithAttributesAsync (13.2+ — may be null on older drivers/bindings) +#if CUDA_VERSION >= 13020 +decltype(&cuMemcpyWithAttributesAsync) p_cuMemcpyWithAttributesAsync = nullptr; +#else +void* p_cuMemcpyWithAttributesAsync = nullptr; +#endif + +// NVRTC function pointers +decltype(&nvrtcDestroyProgram) p_nvrtcDestroyProgram = nullptr; + +// NVVM function pointers (may be null if NVVM is not available) +NvvmDestroyProgramFn p_nvvmDestroyProgram = nullptr; + +// nvJitLink function pointers (may be null if nvJitLink is not available) +NvJitLinkDestroyFn p_nvJitLinkDestroy = nullptr; + +// ============================================================================ +// SM resource split wrapper +// ============================================================================ + +CUresult sm_resource_split(CUdevResource* result, unsigned int nbGroups, + const CUdevResource* input, CUdevResource* remainder, + unsigned int flags, void* groupParams) { +#if CUDA_VERSION >= 13010 + if (!p_cuDevSmResourceSplit) { + return CUDA_ERROR_NOT_SUPPORTED; + } + return p_cuDevSmResourceSplit( + result, nbGroups, input, remainder, flags, + static_cast(groupParams)); +#else + return CUDA_ERROR_NOT_SUPPORTED; +#endif +} + +bool has_sm_resource_split() noexcept { + return p_cuDevSmResourceSplit != nullptr; +} + +// ============================================================================ +// cuMemcpyWithAttributesAsync wrapper +// ============================================================================ + +CUresult memcpy_with_attributes_async(CUdeviceptr dst, CUdeviceptr src, size_t size, + void* attr, CUstream hStream) { +#if CUDA_VERSION >= 13020 + if (!p_cuMemcpyWithAttributesAsync) { + return CUDA_ERROR_NOT_SUPPORTED; + } + return p_cuMemcpyWithAttributesAsync( + dst, src, size, static_cast(attr), hStream); +#else + return CUDA_ERROR_NOT_SUPPORTED; +#endif +} + +bool has_memcpy_with_attributes_async() noexcept { + return p_cuMemcpyWithAttributesAsync != nullptr; +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/driver_api.hpp b/cuda_core/cuda/core/_cpp/rt/driver_api.hpp new file mode 100644 index 00000000000..87ed3bd7906 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/driver_api.hpp @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "types.hpp" +#include +#include +#include + +namespace cuda_core::rt { + +// ============================================================================ +// CUDA driver function pointers +// +// These are populated by _rt.pyx at module import time using +// function pointers extracted from cuda.bindings.cydriver.__pyx_capi__. +// ============================================================================ + +extern decltype(&cuGetErrorName) p_cuGetErrorName; +extern decltype(&cuGetErrorString) p_cuGetErrorString; + +extern decltype(&cuDevicePrimaryCtxRetain) p_cuDevicePrimaryCtxRetain; +extern decltype(&cuDevicePrimaryCtxRelease) p_cuDevicePrimaryCtxRelease; +extern decltype(&cuCtxGetCurrent) p_cuCtxGetCurrent; +extern decltype(&cuCtxSetCurrent) p_cuCtxSetCurrent; +extern decltype(&cuCtxSynchronize) p_cuCtxSynchronize; +extern decltype(&cuCtxGetStreamPriorityRange) p_cuCtxGetStreamPriorityRange; +extern decltype(&cuCtxGetDevice) p_cuCtxGetDevice; +extern decltype(&cuGraphNodeSetParams) p_cuGraphNodeSetParams; +extern decltype(&cuGreenCtxCreate) p_cuGreenCtxCreate; +extern decltype(&cuGreenCtxDestroy) p_cuGreenCtxDestroy; +extern decltype(&cuCtxFromGreenCtx) p_cuCtxFromGreenCtx; +extern decltype(&cuDevResourceGenerateDesc) p_cuDevResourceGenerateDesc; + +extern decltype(&cuGreenCtxStreamCreate) p_cuGreenCtxStreamCreate; + +extern decltype(&cuStreamCreateWithPriority) p_cuStreamCreateWithPriority; +extern decltype(&cuStreamDestroy) p_cuStreamDestroy; +extern decltype(&cuStreamGetCtx) p_cuStreamGetCtx; + +extern decltype(&cuEventCreate) p_cuEventCreate; +extern decltype(&cuEventDestroy) p_cuEventDestroy; +extern decltype(&cuIpcOpenEventHandle) p_cuIpcOpenEventHandle; + +extern decltype(&cuDeviceGetCount) p_cuDeviceGetCount; + +extern decltype(&cuMemPoolSetAccess) p_cuMemPoolSetAccess; +extern decltype(&cuMemPoolDestroy) p_cuMemPoolDestroy; +extern decltype(&cuMemPoolCreate) p_cuMemPoolCreate; +extern decltype(&cuDeviceGetMemPool) p_cuDeviceGetMemPool; +extern decltype(&cuMemPoolImportFromShareableHandle) p_cuMemPoolImportFromShareableHandle; + +extern decltype(&cuMemAllocFromPoolAsync) p_cuMemAllocFromPoolAsync; +extern decltype(&cuMemAllocAsync) p_cuMemAllocAsync; +extern decltype(&cuMemAlloc) p_cuMemAlloc; +extern decltype(&cuMemAllocHost) p_cuMemAllocHost; + +extern decltype(&cuMemFreeAsync) p_cuMemFreeAsync; +extern decltype(&cuMemFree) p_cuMemFree; +extern decltype(&cuMemFreeHost) p_cuMemFreeHost; + +extern decltype(&cuMemPoolImportPointer) p_cuMemPoolImportPointer; + +// Library +extern decltype(&cuLibraryLoadFromFile) p_cuLibraryLoadFromFile; +extern decltype(&cuLibraryLoadData) p_cuLibraryLoadData; +extern decltype(&cuLibraryUnload) p_cuLibraryUnload; +extern decltype(&cuLibraryGetKernel) p_cuLibraryGetKernel; + +// Graph +extern decltype(&cuGraphDestroy) p_cuGraphDestroy; +extern decltype(&cuGraphInstantiateWithParams) p_cuGraphInstantiateWithParams; +extern decltype(&cuGraphExecUpdate) p_cuGraphExecUpdate; +extern decltype(&cuGraphExecDestroy) p_cuGraphExecDestroy; +extern decltype(&cuUserObjectCreate) p_cuUserObjectCreate; +extern decltype(&cuUserObjectRelease) p_cuUserObjectRelease; +extern decltype(&cuGraphRetainUserObject) p_cuGraphRetainUserObject; +extern decltype(&cuGraphReleaseUserObject) p_cuGraphReleaseUserObject; +extern decltype(&cuGraphNodeFindInClone) p_cuGraphNodeFindInClone; +extern decltype(&cuGraphChildGraphNodeGetGraph) p_cuGraphChildGraphNodeGetGraph; + +// Linker +extern decltype(&cuLinkDestroy) p_cuLinkDestroy; + +// Graphics interop +extern decltype(&cuGraphicsUnmapResources) p_cuGraphicsUnmapResources; +extern decltype(&cuGraphicsUnregisterResource) p_cuGraphicsUnregisterResource; + +// Texture / surface / array (PR #467) +extern decltype(&cuArray3DCreate) p_cuArray3DCreate; +extern decltype(&cuArrayDestroy) p_cuArrayDestroy; +extern decltype(&cuMipmappedArrayCreate) p_cuMipmappedArrayCreate; +extern decltype(&cuMipmappedArrayDestroy) p_cuMipmappedArrayDestroy; +extern decltype(&cuMipmappedArrayGetLevel) p_cuMipmappedArrayGetLevel; +extern decltype(&cuTexObjectCreate) p_cuTexObjectCreate; +extern decltype(&cuTexObjectDestroy) p_cuTexObjectDestroy; +extern decltype(&cuSurfObjectCreate) p_cuSurfObjectCreate; +extern decltype(&cuSurfObjectDestroy) p_cuSurfObjectDestroy; + +// SM resource split (13.1+ — may be null on older drivers/bindings) +#if CUDA_VERSION >= 13010 +extern decltype(&cuDevSmResourceSplit) p_cuDevSmResourceSplit; +#else +// cuDevSmResourceSplit doesn't exist in CUDA < 13.1 headers, so use a +// void* placeholder. The pointer is always null when built against 12.x. +extern void* p_cuDevSmResourceSplit; +#endif + +// cuMemcpyWithAttributesAsync (13.2+ — may be null on older drivers/bindings) +#if CUDA_VERSION >= 13020 +extern decltype(&cuMemcpyWithAttributesAsync) p_cuMemcpyWithAttributesAsync; +#else +// cuMemcpyWithAttributesAsync doesn't exist in CUDA < 13.2 headers, so use a +// void* placeholder. The pointer is always null when built against older CUDA. +extern void* p_cuMemcpyWithAttributesAsync; +#endif + +// ============================================================================ +// NVRTC function pointers +// +// These are populated by _rt.pyx at module import time using +// function pointers extracted from cuda.bindings.cynvrtc.__pyx_capi__. +// ============================================================================ + +extern decltype(&nvrtcDestroyProgram) p_nvrtcDestroyProgram; + +// ============================================================================ +// NVVM function pointers +// +// These are populated by _rt.pyx at module import time using +// function pointers extracted from cuda.bindings.cynvvm.__pyx_capi__. +// Note: May be null if NVVM is not available at runtime. +// ============================================================================ + +// Function pointer type for nvvmDestroyProgram (avoids nvvm.h dependency) +// Signature: nvvmResult nvvmDestroyProgram(nvvmProgram *prog) +using NvvmDestroyProgramFn = int (*)(nvvmProgram*); +extern NvvmDestroyProgramFn p_nvvmDestroyProgram; + +// ============================================================================ +// nvJitLink function pointers +// +// These are populated by _rt.pyx at module import time using +// function pointers extracted from cuda.bindings.cynvjitlink.__pyx_capi__. +// Note: May be null if nvJitLink is not available at runtime. +// ============================================================================ + +// Function pointer type for nvJitLinkDestroy (avoids nvJitLink.h dependency) +// Signature: nvJitLinkResult nvJitLinkDestroy(nvJitLinkHandle *handle) +using NvJitLinkDestroyFn = int (*)(nvJitLink_t*); +extern NvJitLinkDestroyFn p_nvJitLinkDestroy; + +// ============================================================================ +// SM resource split wrapper (13.1+) +// +// Calls through p_cuDevSmResourceSplit if available, otherwise returns +// CUDA_ERROR_NOT_SUPPORTED. This avoids a direct Cython cimport of the +// cydriver cdef function, which would fail at module init on cuda-bindings +// < 13.1 (see https://github.com/NVIDIA/cuda-python/issues/2063). +// ============================================================================ + +// groupParams is void* so the Cython declaration doesn't reference +// CU_DEV_SM_RESOURCE_GROUP_PARAMS (absent from cuda-bindings 13.0 .pxd). +CUresult sm_resource_split(CUdevResource* result, unsigned int nbGroups, + const CUdevResource* input, CUdevResource* remainder, + unsigned int flags, void* groupParams); + +// Returns true if the cuDevSmResourceSplit function pointer is available. +bool has_sm_resource_split() noexcept; + +// ============================================================================ +// cuMemcpyWithAttributesAsync wrapper (13.2+) +// +// Calls through p_cuMemcpyWithAttributesAsync if available, otherwise returns +// CUDA_ERROR_NOT_SUPPORTED. This avoids a direct Cython cimport of the +// cydriver cdef function, which would fail at module init on cuda-bindings +// < 13.2 (see https://github.com/NVIDIA/cuda-python/issues/2063). +// ============================================================================ + +// attr is void* so the Cython declaration doesn't reference CUmemcpyAttributes +// (absent from cuda-bindings built against CUDA < 12.8). The C++ side casts it. +CUresult memcpy_with_attributes_async(CUdeviceptr dst, CUdeviceptr src, size_t size, + void* attr, CUstream hStream); + +// Returns true if the cuMemcpyWithAttributesAsync function pointer is available. +bool has_memcpy_with_attributes_async() noexcept; + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/error.cpp b/cuda_core/cuda/core/_cpp/rt/error.cpp new file mode 100644 index 00000000000..c0f7caf32d4 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/error.cpp @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "error.hpp" +#include "driver_api.hpp" +#include "internal.hpp" +#include +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +// ---------------------------------------------------------------------------- +// Non-propagating error reporting +// +// Deleters, CUDA callbacks and other non-propagating paths cannot raise. They +// report through report_cuda_error()/report_message(), which emit a +// cuda.core.CUDAWarning when the interpreter is usable and fall back to stderr +// otherwise. See docs/source/error_handling.rst for the policy. +// ---------------------------------------------------------------------------- + +namespace { +// Thread-local detail attached to the next raised CUDAError with a matching +// status (see take_last_error_detail()). Written only by propagating helpers. +// The taken copy stays valid until the next take on the same thread. +thread_local char last_error_detail[512] = {0}; +thread_local char taken_error_detail[512] = {0}; +thread_local CUresult last_error_detail_status = CUDA_SUCCESS; +} // namespace + +namespace detail { +// Format " : : " for a failed CUDA call. +void format_cuda_error(char* buffer, size_t size, const char* operation, CUresult status, + const char* detail) noexcept { + const char* error_name = nullptr; + const char* error_description = nullptr; + bool decoded = p_cuGetErrorName && p_cuGetErrorString + && p_cuGetErrorName(status, &error_name) == CUDA_SUCCESS + && p_cuGetErrorString(status, &error_description) == CUDA_SUCCESS; + const char* outcome = detail ? detail : "failed"; + if (decoded) { + std::snprintf(buffer, size, "%s %s: %s: %s", operation, outcome, error_name, error_description); + } else { + std::snprintf(buffer, size, "%s %s (CUDA error %d)", operation, outcome, static_cast(status)); + } +} +} // namespace detail + +// Report a failed non-CUDA call (NVRTC, NVVM, nvJitLink) from a path that +// cannot raise. +void report_status_code(const char* operation, long code) noexcept { + char message[256]; + std::snprintf(message, sizeof(message), "%s failed (status %ld)", operation, code); + report_message(message); +} + +// Report a failed CUDA call from a path that cannot raise. CUDA_ERROR_DEINITIALIZED +// is not reported: it means the driver is shutting down, which makes cleanup +// failures expected and uninteresting. +void report_cuda_error(const char* operation, CUresult status, const char* detail) noexcept { + if (status == CUDA_SUCCESS || status == CUDA_ERROR_DEINITIALIZED) { + return; + } + char message[512]; + format_cuda_error(message, sizeof(message), operation, status, detail); + report_message(message); +} + +const char* take_last_error_detail(CUresult status) noexcept { + if (!last_error_detail[0] || status != last_error_detail_status) { + return nullptr; + } + std::memcpy(taken_error_detail, last_error_detail, sizeof(taken_error_detail)); + clear_last_error_detail(); + return taken_error_detail; +} + +void clear_last_error_detail() noexcept { + last_error_detail[0] = 0; + last_error_detail_status = CUDA_SUCCESS; +} + +namespace detail { +// Record that the caller's context was not restored as the detail of the +// CUresult about to be returned and raised: the operation status if the +// operation failed too, else the restoration status. For a double failure the +// detail also names the restoration error, which the raised error does not. +void note_context_not_restored(CUcontext previous, CUresult operation_status, + CUresult restore_status) noexcept { + CUcontext current = nullptr; + if (p_cuCtxGetCurrent(¤t) != CUDA_SUCCESS) { + current = nullptr; + } + char cause[128] = {0}; + if (operation_status != CUDA_SUCCESS) { + const char* error_name = nullptr; + if (p_cuGetErrorName && p_cuGetErrorName(restore_status, &error_name) == CUDA_SUCCESS) { + std::snprintf(cause, sizeof(cause), " after this failure (cuCtxSetCurrent: %s)", error_name); + } else { + std::snprintf(cause, sizeof(cause), " after this failure (cuCtxSetCurrent: CUDA error %d)", + static_cast(restore_status)); + } + } + std::snprintf(last_error_detail, sizeof(last_error_detail), + "the calling thread's CUDA context (%#llx) could not be restored%s; " + "context %#llx is now current. Call Device.set_current() before issuing " + "further CUDA work on this thread", + static_cast(reinterpret_cast(previous)), + cause, + static_cast(reinterpret_cast(current))); + last_error_detail_status = operation_status != CUDA_SUCCESS ? operation_status : restore_status; +} +} // namespace detail + +// ============================================================================ +// Thread-local error handling +// ============================================================================ + +// Thread-local status of the most recent CUDA API call in this module. +thread_local CUresult err = CUDA_SUCCESS; + +// Return and clear the calling thread's most recent CUDA error. +CUresult get_last_error() noexcept { + CUresult e = err; + err = CUDA_SUCCESS; + return e; +} + +// Return the calling thread's most recent CUDA error without clearing it. +CUresult peek_last_error() noexcept { + return err; +} + +void clear_last_error() noexcept { + err = CUDA_SUCCESS; +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/error.hpp b/cuda_core/cuda/core/_cpp/rt/error.hpp new file mode 100644 index 00000000000..2be565084a6 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/error.hpp @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +namespace cuda_core::rt { + +// ============================================================================ +// Thread-local error handling +// ============================================================================ + +// Get and clear the last CUDA error (like cudaGetLastError) +CUresult get_last_error() noexcept; + +// Get the last CUDA error without clearing it (like cudaPeekAtLastError) +CUresult peek_last_error() noexcept; + +// Explicitly clear the last error +void clear_last_error() noexcept; + +// Thread-local status of the most recent CUDA API call in this module. Defined +// in error.cpp; every family source writes it. +extern thread_local CUresult err; + +// ============================================================================ +// Non-propagating error reporting +// +// Paths that cannot raise (shared_ptr deleters, CUDA callbacks, __dealloc__) +// report failures through these functions instead of discarding them. They +// emit a cuda.core.CUDAWarning when the interpreter can be used and write to +// stderr otherwise; they never raise. See docs/source/error_handling.rst. +// ============================================================================ + +// Report a failed CUDA call. `detail` replaces the default "failed" wording, +// e.g. "skipped (context activation failed; resource leaked)". +// CUDA_ERROR_DEINITIALIZED (driver shutting down) is never reported. +void report_cuda_error(const char* operation, CUresult status, const char* detail = nullptr) noexcept; + +// Report a message that is not tied to a CUresult. +// Implemented in py_report.cpp +void report_message(const char* message) noexcept; + +// Report a failed NVRTC/NVVM/nvJitLink call by raw status code. +void report_status_code(const char* operation, long code) noexcept; + +// Attach a failed CUDA call to the Python exception currently being handled +// (PEP 678 note, Python 3.11+): for rollback failures inside `except` blocks +// whose original exception is about to be re-raised. When no exception is +// being handled or notes are unavailable, falls back to report_cuda_error(). +// Implemented in py_report.cpp +void note_or_report_cuda_error(const char* operation, CUresult status, const char* detail = nullptr) noexcept; + +// Detail recorded by a context-scoped helper for the CUresult it is about to +// return, e.g. that the caller's context could not be restored. The Cython +// error path attaches it to the raised CUDAError as a note. Thread-local and +// keyed by status: take_ returns the detail (valid until the next take on this +// thread) and clears it when `status` is the CUresult it was recorded for, and +// returns nullptr otherwise, so a detail whose status was never raised cannot +// attach to an unrelated error. +const char* take_last_error_detail(CUresult status) noexcept; +void clear_last_error_detail() noexcept; + +// Tests only: make the next context restoration on this thread fail with +// `status`, leaving the target context current as a real failure would. +// Implemented in context.cpp +void set_context_restore_fault_for_testing(CUresult status) noexcept; + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/event.cpp b/cuda_core/cuda/core/_cpp/rt/event.cpp new file mode 100644 index 00000000000..7514f8b8e41 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/event.cpp @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "context_scope.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +// ============================================================================ +// Event Handles +// ============================================================================ + +namespace { +struct EventBox { + CUevent resource; + bool timing_enabled; + bool is_blocking_sync; + bool ipc_enabled; + int device_id; + ContextHandle h_context; +}; +} // namespace + +static const EventBox* get_box(const EventHandle& h) { + const CUevent* p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) - offsetof(EventBox, resource) + ); +} + +bool get_event_timing_enabled(const EventHandle& h) noexcept { + return h ? get_box(h)->timing_enabled : false; +} + +bool get_event_is_blocking_sync(const EventHandle& h) noexcept { + return h ? get_box(h)->is_blocking_sync : false; +} + +bool get_event_ipc_enabled(const EventHandle& h) noexcept { + return h ? get_box(h)->ipc_enabled : false; +} + +int get_event_device_id(const EventHandle& h) noexcept { + return h ? get_box(h)->device_id : -1; +} + +// Return the context retained by an event handle. +ContextHandle get_event_context(const EventHandle& h) noexcept { + return h ? get_box(h)->h_context : ContextHandle{}; +} + +// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) +static HandleRegistry event_registry; + +EventHandle create_event_handle(const ContextHandle& h_ctx, unsigned int flags, + bool timing_enabled, bool is_blocking_sync, + bool ipc_enabled, int device_id) { + GILReleaseGuard gil; + CUevent event = nullptr; + err = invoke_in_context_or_undo( + h_ctx, + [&]() noexcept { return p_cuEventCreate(&event, flags); }, + [&]() noexcept { pw_cuEventDestroy(event); }, + /*undo_requires_target_context=*/false); + if (err != CUDA_SUCCESS) { + return {}; + } + + auto box = std::shared_ptr( + new EventBox{event, timing_enabled, is_blocking_sync, ipc_enabled, device_id, h_ctx}, + [](const EventBox* b) { + event_registry.unregister_handle(b->resource); + GILReleaseGuard gil; + pw_cuEventDestroy(b->resource); + delete b; + } + ); + EventHandle h(box, &box->resource); + event_registry.register_handle(event, h); + return h; +} + +EventHandle create_event_handle_for_stream(CUstream stream, unsigned int flags) { + // Resolve the stream's owning context (for default-stream tokens this is + // the current context, per cuStreamGetCtx) and create the event there, so + // it can be recorded on `stream` no matter which context is current. + CUcontext ctx = nullptr; + { + GILReleaseGuard gil; + err = p_cuStreamGetCtx(stream, &ctx); + } + if (err != CUDA_SUCCESS) { + return {}; + } + if (!ctx) { + err = CUDA_ERROR_INVALID_CONTEXT; + return {}; + } + return create_event_handle(create_context_handle_ref(ctx), flags, false, false, false, -1); +} + +EventHandle create_event_handle_ref(CUevent event) { + if (auto h = event_registry.lookup(event)) { + return h; + } + auto box = std::make_shared(EventBox{event, false, false, false, -1, {}}); + return EventHandle(box, &box->resource); +} + +EventHandle create_event_handle_ipc(const CUipcEventHandle& ipc_handle, + bool is_blocking_sync) { + GILReleaseGuard gil; + CUevent event; + if (CUDA_SUCCESS != (err = p_cuIpcOpenEventHandle(&event, ipc_handle))) { + return {}; + } + + auto box = std::shared_ptr( + new EventBox{event, false, is_blocking_sync, true, -1, {}}, + [](const EventBox* b) { + event_registry.unregister_handle(b->resource); + GILReleaseGuard gil; + pw_cuEventDestroy(b->resource); + delete b; + } + ); + EventHandle h(box, &box->resource); + event_registry.register_handle(event, h); + return h; +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/graph.cpp b/cuda_core/cuda/core/_cpp/rt/graph.cpp new file mode 100644 index 00000000000..4f6105dfdaa --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/graph.cpp @@ -0,0 +1,758 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "context_scope.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +// Set a graph node's parameters with h_context current (an empty handle runs in +// the caller's context). Returns the cuGraphNodeSetParams status. A failure to +// restore the caller's context is returned separately in *restore_status so the +// caller can publish the metadata that depends on the successful update before +// raising it; if the update itself failed, its status is returned with the +// restoration failure recorded as its detail and *restore_status is CUDA_SUCCESS. +CUresult graph_node_set_params(CUgraphNode node, CUgraphNodeParams* params, + const ContextHandle& h_context, + CUresult* restore_status) noexcept { + *restore_status = CUDA_SUCCESS; + if (!p_cuGraphNodeSetParams) { + return CUDA_ERROR_NOT_SUPPORTED; + } + CUcontext previous = nullptr; + int changed = 0; + CUresult status = enter_context(h_context, &previous, &changed); + if (status != CUDA_SUCCESS) { + return status; + } + { + GILReleaseGuard gil; + status = p_cuGraphNodeSetParams(node, params); + } + if (!changed) { + return status; + } + CUresult restored = restore_context(previous); + if (restored == CUDA_SUCCESS) { + return status; + } + note_context_not_restored(previous, status, restored); + if (status == CUDA_SUCCESS) { + *restore_status = restored; + } + return status; +} + +// ============================================================================ +// Graph Handles +// ============================================================================ + +namespace { + +struct NodeAttachment; +using GraphAttachmentMap = std::map; + +struct GraphHierarchy; + +// Standard-layout alias target for GraphHandle. +struct GraphBoxBase { + CUgraph resource = nullptr; +}; + +// Canonical state for one CUgraph. Its GraphHandle aliases resource, whose +// address remains stable for the lifetime of the hierarchy. +struct GraphBox : GraphBoxBase { + GraphHierarchy* hierarchy = nullptr; // Non-owning back-reference. + GraphBox* parent = nullptr; // Null for the root graph. + CUgraphNode owner_node = nullptr; // Node in parent that owns this graph. + GraphAttachmentMap attachments; // Non-owning attachment index. + HandleRegistry node_handles; + + GraphBox( + CUgraph resource_, + GraphHierarchy* hierarchy_, + GraphBox* parent_ = nullptr, + CUgraphNode owner_node_ = nullptr) noexcept + : GraphBoxBase{resource_}, + hierarchy(hierarchy_), + parent(parent_), + owner_node(owner_node_) {} +}; + +// Shared owner of stable GraphBox storage. Every GraphHandle aliases the same +// control block, so any graph handle keeps the entire hierarchy alive. +struct GraphHierarchy { + std::list graphs; // Parent boxes precede their descendants. + std::list graveyard; // Retired child graph tombstones. + + GraphBox* root() noexcept { + return graphs.empty() ? nullptr : &graphs.front(); + } +}; + +// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) +using GraphRegistry = HandleRegistry; +static GraphRegistry graph_registry; + +// Immutable resource owners for one version of a graph node's parameters. +// Inheriting DeferredCleanupItem lets CUDA's user-object destructor enqueue +// the payload without destroying owners on the callback thread. +struct NodeAttachment : DeferredCleanupItem { + CUuserObject object = nullptr; + std::array owners; + + NodeAttachment(OpaqueHandle owner0, OpaqueHandle owner1) + : owners{std::move(owner0), std::move(owner1)} {} +}; + +// shared_ptr deleters for the payloads that need one. Typed handles convert to +// OpaqueHandle by assignment and reuse their own control block, so they need no +// deleter here. The Python deleter follows the owner-release pattern used by +// the stream/deviceptr handles above. +void py_deleter(const void* p) noexcept { + GILAcquireGuard gil; + if (gil.acquired()) { + Py_DECREF(const_cast(static_cast(p))); + } +} + +void free_deleter(const void* p) noexcept { + std::free(const_cast(p)); +} + +GraphBox* get_box(const GraphHandle& h) noexcept { + auto* value = reinterpret_cast(h.get()); + return const_cast( + static_cast(value)); +} + +// Rekey a staged attachment map from source nodes to their cloned nodes. +// The caller must release the GIL before calling this function. +CUresult rekey_attachments( + GraphAttachmentMap& attachments, CUgraph cloned_graph) { + if (!cloned_graph) { + return CUDA_ERROR_INVALID_VALUE; + } + if (!p_cuGraphNodeFindInClone) { + return CUDA_ERROR_NOT_SUPPORTED; + } + + GraphAttachmentMap remapped; + while (!attachments.empty()) { + auto attachment = attachments.extract(attachments.begin()); + CUgraphNode cloned_node = nullptr; + CUresult status = p_cuGraphNodeFindInClone( + &cloned_node, attachment.key(), cloned_graph); + if (status != CUDA_SUCCESS) { + return status; + } + attachment.key() = cloned_node; + if (!remapped.insert(std::move(attachment)).inserted) { + return CUDA_ERROR_INVALID_VALUE; + } + } + attachments.swap(remapped); + return CUDA_SUCCESS; +} + +struct StagedGraphMetadata { + const GraphBox* source; + GraphBox* clone; + GraphAttachmentMap* attachments; +}; +using StagedGraphMetadataList = std::vector; + +// Copy a source hierarchy into detached metadata before CUDA mutation. +void stage_graph_metadata( + const GraphBox& source, + GraphBox& clone, + GraphAttachmentMap& attachments, + std::list& subgraphs, + StagedGraphMetadataList& staged) { + attachments = source.attachments; + staged.push_back({&source, &clone, &attachments}); + + for (const GraphBox& source_child : source.hierarchy->graphs) { + if (source_child.parent != &source || !source_child.resource) { + continue; + } + GraphBox& cloned_child = subgraphs.emplace_back( + nullptr, + clone.hierarchy, + &clone, + nullptr); + stage_graph_metadata( + source_child, + cloned_child, + cloned_child.attachments, + subgraphs, + staged); + } +} + +// Bind staged metadata to a CUDA-cloned hierarchy. The root clone resource +// must be populated before entry. The caller must release the GIL. +CUresult rekey_graph_metadata( + StagedGraphMetadataList& staged) { + if (!p_cuGraphNodeFindInClone || !p_cuGraphChildGraphNodeGetGraph) { + return CUDA_ERROR_NOT_SUPPORTED; + } + + CUresult status; + for (size_t i = 0; i < staged.size(); ++i) { + const GraphBox& source = *staged[i].source; + GraphBox& clone = *staged[i].clone; + if (i != 0) { + CUgraphNode cloned_owner = nullptr; + status = p_cuGraphNodeFindInClone( + &cloned_owner, + source.owner_node, + clone.parent->resource); + if (status == CUDA_SUCCESS) { + status = p_cuGraphChildGraphNodeGetGraph( + cloned_owner, &clone.resource); + } + if (status != CUDA_SUCCESS) { + return status; + } + clone.owner_node = cloned_owner; + } + + status = rekey_attachments( + *staged[i].attachments, clone.resource); + if (status != CUDA_SUCCESS) { + return status; + } + } + return CUDA_SUCCESS; +} + +} // namespace + +OpaqueHandle make_opaque_py(PyObject* obj) { + Py_INCREF(obj); + return OpaqueHandle(static_cast(obj), py_deleter); +} + +OpaqueHandle make_opaque_malloc(void* buf) { + return OpaqueHandle(static_cast(buf), free_deleter); +} + +// State held by PreparedAttachment between preparation and commit. It keeps the +// graph alive, tracks the graph-retained replacement, and holds a preallocated +// map entry so commit cannot allocate. Destroying PreparedAttachment rolls back +// the staged user-object retain unless graph_commit_attachment publishes it. +struct PreparedAttachmentState { + GraphHandle h_graph; + NodeAttachment* replacement = nullptr; + GraphAttachmentMap::node_type replacement_entry; + + explicit PreparedAttachmentState(GraphHandle h_graph_) + : h_graph(std::move(h_graph_)) {} +}; + +void rollback_prepared_attachment( + PreparedAttachmentState* state) noexcept { + if (!state) { + return; + } + if (state->replacement) { + GraphBox* box = get_box(state->h_graph); + if (box->resource) { + GILReleaseGuard gil; + pw_cuGraphReleaseUserObject( + box->resource, state->replacement->object, 1); + } + } + delete state; +} + +// Detached metadata for a replacement embedded graph hierarchy. Preparation +// copies every attachment map and allocates every GraphBox before CUDA destroys +// the old embedded graph. Commit only rekeys and publishes it. +struct PreparedChildGraphUpdateState { + GraphHandle h_parent; + GraphHandle h_source; + GraphBox* old_root = nullptr; + CUgraphNode owner_node = nullptr; + std::list replacement; + StagedGraphMetadataList staged; + std::vector handles; + + PreparedChildGraphUpdateState( + GraphHandle h_parent_, + GraphHandle h_source_, + GraphBox* old_root_, + CUgraphNode owner_node_) + : h_parent(std::move(h_parent_)), + h_source(std::move(h_source_)), + old_root(old_root_), + owner_node(owner_node_) {} +}; + +GraphHandle create_graph_handle(CUgraph graph) { + if (!graph) { + return {}; + } + + auto hierarchy = std::shared_ptr( + new GraphHierarchy{}, + [](GraphHierarchy* hierarchy) { + for (const GraphBox& box : hierarchy->graphs) { + if (box.resource) { + graph_registry.unregister_handle(box.resource); + } + } + GraphBox* root = hierarchy->root(); + if (root && root->resource) { + GILReleaseGuard gil; + pw_cuGraphDestroy(root->resource); + } + retry_deferred_cleanup(); + delete hierarchy; + } + ); + GraphBox& root = hierarchy->graphs.emplace_back( + graph, hierarchy.get()); + + GraphHandle h_graph(hierarchy, &root.resource); + graph_registry.register_handle(graph, h_graph); + return h_graph; +} + +GraphHandle create_child_graph_handle( + CUgraph child_graph, const GraphHandle& h_parent, + CUgraphNode owner_node) { + if (!child_graph || !h_parent || !owner_node) { + return {}; + } + if (GraphHandle h_graph = graph_registry.lookup(child_graph)) { + return h_graph; + } + + GraphBox* parent = get_box(h_parent); + GraphHierarchy* hierarchy = parent->hierarchy; + GraphBox& child = hierarchy->graphs.emplace_back( + child_graph, hierarchy, parent, owner_node); + + GraphHandle h_child(h_parent, &child.resource); + graph_registry.register_handle(child_graph, h_child); + return h_child; +} + +CUresult graph_prepare_child_graph_update( + const GraphHandle& h_parent, + const GraphHandle& h_old_child, + CUgraphNode owner_node, + const GraphHandle& h_source, + PreparedChildGraphUpdate* out_prepared) { + if (!h_parent || !h_old_child || !owner_node || + !h_source || !out_prepared) { + return CUDA_ERROR_INVALID_VALUE; + } + out_prepared->reset(); + + GraphBox* parent = get_box(h_parent); + GraphBox* old_root = get_box(h_old_child); + GraphBox* source = get_box(h_source); + // A source from the destination hierarchy can include the old embedded + // subtree whose raw node keys CUDA destroys during replacement. + if (!parent->resource || !old_root->resource || !source->resource || + old_root->parent != parent || + old_root->owner_node != owner_node || + source->hierarchy == parent->hierarchy) { + return CUDA_ERROR_INVALID_VALUE; + } + + PreparedChildGraphUpdate prepared = + std::make_shared( + h_parent, h_source, old_root, owner_node); + + GraphBox& replacement_root = + prepared->replacement.emplace_back( + nullptr, parent->hierarchy, parent, owner_node); + stage_graph_metadata( + *source, + replacement_root, + replacement_root.attachments, + prepared->replacement, + prepared->staged); + + const size_t graph_count = prepared->staged.size(); + prepared->handles.reserve(graph_count); + for (const StagedGraphMetadata& graph : prepared->staged) { + prepared->handles.emplace_back( + h_parent, &graph.clone->resource); + } + + *out_prepared = std::move(prepared); + return CUDA_SUCCESS; +} + +void publish_child_graph_update( + PreparedChildGraphUpdateState& state, + GraphHandle* out_child) { + GraphBox* parent = get_box(state.h_parent); + parent->hierarchy->graphs.splice( + parent->hierarchy->graphs.end(), state.replacement); + *out_child = state.handles.front(); + graph_registry.register_handles(state.handles); +} + +CUresult graph_commit_child_graph_update( + PreparedChildGraphUpdate& prepared, + GraphHandle* out_child) { + if (!prepared || !out_child) { + return CUDA_ERROR_INVALID_VALUE; + } + out_child->reset(); + + PreparedChildGraphUpdateState& state = *prepared; + GraphBox* parent = get_box(state.h_parent); + if (!parent->resource || !state.old_root->resource) { + prepared.reset(); + return CUDA_ERROR_INVALID_VALUE; + } + + CUresult status = CUDA_ERROR_NOT_SUPPORTED; + CUgraph cloned_root = nullptr; + if (p_cuGraphChildGraphNodeGetGraph) { + GILReleaseGuard gil; + status = p_cuGraphChildGraphNodeGetGraph( + state.owner_node, &cloned_root); + if (status == CUDA_SUCCESS) { + state.staged.front().clone->resource = cloned_root; + status = rekey_graph_metadata(state.staged); + } + } + + // CUDA has already destroyed the old embedded graph. No replacement + // metadata is visible yet, so this selects only the old generation. + invalidate_child_graph_state( + state.h_parent, state.owner_node); + + if (status != CUDA_SUCCESS) { + prepared.reset(); + throw std::runtime_error( + "failed to update graph metadata after child graph replacement"); + } + + publish_child_graph_update(state, out_child); + prepared.reset(); + return status; +} + +CUresult graph_get_attachment( + const GraphHandle& h_graph, CUgraphNode node, + OpaqueHandle* owner0, OpaqueHandle* owner1) { + if (!h_graph || !node || (!owner0 && !owner1)) { + return CUDA_ERROR_INVALID_VALUE; + } + if (owner0) { + owner0->reset(); + } + if (owner1) { + owner1->reset(); + } + + GraphBox* box = get_box(h_graph); + if (!box->resource) { + return CUDA_ERROR_INVALID_VALUE; + } + auto it = box->attachments.find(node); + if (it != box->attachments.end()) { + if (owner0) { + *owner0 = it->second->owners[0]; + } + if (owner1) { + *owner1 = it->second->owners[1]; + } + } + return CUDA_SUCCESS; +} + +CUresult graph_prepare_attachment( + const GraphHandle& h_graph, + OpaqueHandle owner0, + OpaqueHandle owner1, + PreparedAttachment* out_prepared) { + if (!out_prepared) { + return CUDA_ERROR_INVALID_VALUE; + } + out_prepared->reset(); + if (!h_graph) { + return CUDA_ERROR_INVALID_VALUE; + } + + GraphBox* box = get_box(h_graph); + if (!box->resource) { + return CUDA_ERROR_INVALID_VALUE; + } + if (!p_cuGraphReleaseUserObject) { + return CUDA_ERROR_NOT_SUPPORTED; + } + + PreparedAttachment prepared( + new PreparedAttachmentState(h_graph), + PreparedAttachmentDeleter{rollback_prepared_attachment}); + if (owner0 || owner1) { + if (!p_cuUserObjectCreate || !p_cuUserObjectRelease || + !p_cuGraphRetainUserObject) { + return CUDA_ERROR_NOT_SUPPORTED; + } + + ensure_deferred_cleanup_ready(); + prepared->replacement = new NodeAttachment( + std::move(owner0), std::move(owner1)); + GraphAttachmentMap staged; + try { + staged.emplace(nullptr, prepared->replacement); + prepared->replacement_entry = + staged.extract(staged.begin()); + } catch (...) { + delete prepared->replacement; + prepared->replacement = nullptr; + throw; + } + auto* cleanup_item = + static_cast( + prepared->replacement); + + CUuserObject object = nullptr; + CUresult status; + { + GILReleaseGuard gil; + status = p_cuUserObjectCreate( + &object, cleanup_item, + reinterpret_cast(enqueue_cleanup), + 1, CU_USER_OBJECT_NO_DESTRUCTOR_SYNC); + if (status != CUDA_SUCCESS) { + prepared->replacement_entry.mapped() = nullptr; + delete prepared->replacement; + prepared->replacement = nullptr; + return status; + } + prepared->replacement->object = object; + status = p_cuGraphRetainUserObject( + box->resource, object, 1, CU_GRAPH_USER_OBJECT_MOVE); + if (status != CUDA_SUCCESS) { + prepared->replacement_entry.mapped() = nullptr; + prepared->replacement = nullptr; + pw_cuUserObjectRelease(object, 1); + return status; + } + } + } + + *out_prepared = std::move(prepared); + return CUDA_SUCCESS; +} + +CUresult graph_commit_attachment( + PreparedAttachment& prepared, + CUgraphNode node) { + if (!prepared) { + return CUDA_ERROR_INVALID_VALUE; + } + + GraphHandle h_graph = prepared->h_graph; + GraphBox* box = get_box(h_graph); + if (!box->resource || (!node && !prepared->replacement)) { + delete prepared.release(); + return CUDA_ERROR_INVALID_VALUE; + } + if (!node) { + delete prepared.release(); + return CUDA_SUCCESS; + } + + // Publish the replacement or removal before releasing the previous graph + // reference; that release can make the previous payload eligible for + // destruction. + NodeAttachment* previous = nullptr; + auto it = box->attachments.find(node); + if (it == box->attachments.end()) { + if (prepared->replacement) { + prepared->replacement_entry.key() = node; + auto result = box->attachments.insert( + std::move(prepared->replacement_entry)); + if (!result.inserted) { + prepared->replacement_entry = + std::move(result.node); + delete prepared.release(); + return CUDA_ERROR_INVALID_VALUE; + } + } + } else { + previous = it->second; + if (prepared->replacement) { + it->second = prepared->replacement; + } else { + box->attachments.erase(it); + } + } + + delete prepared.release(); + if (!previous) { + return CUDA_SUCCESS; + } + GILReleaseGuard gil; + return p_cuGraphReleaseUserObject( + box->resource, previous->object, 1); +} + +CUresult graph_clone_attachments( + const GraphHandle& h_clone, + const GraphHandle& h_source) { + if (!h_clone || !h_source) { + return CUDA_ERROR_INVALID_VALUE; + } + + GraphBox* clone = get_box(h_clone); + GraphBox* source = get_box(h_source); + if (!clone->resource || !source->resource || + !clone->attachments.empty()) { + return CUDA_ERROR_INVALID_VALUE; + } + + // Build and rekey the clone metadata off-hierarchy so a CUDA mapping error + // cannot partially publish it. + GraphAttachmentMap attachments; + std::list subgraphs; + StagedGraphMetadataList staged; + stage_graph_metadata( + *source, *clone, attachments, subgraphs, staged); + + std::vector handles; + handles.reserve(subgraphs.size()); + for (GraphBox& graph : subgraphs) { + handles.emplace_back(h_clone, &graph.resource); + } + + CUresult status; + { + GILReleaseGuard gil; + status = rekey_graph_metadata(staged); + } + if (status != CUDA_SUCCESS) { + return status; + } + + clone->attachments.swap(attachments); + if (subgraphs.empty()) { + return CUDA_SUCCESS; + } + + clone->hierarchy->graphs.splice( + clone->hierarchy->graphs.end(), subgraphs); + graph_registry.register_handles(handles); + return CUDA_SUCCESS; +} + +namespace { +struct GraphNodeBox { + mutable CUgraphNode resource; + GraphHandle h_graph; +}; +} // namespace + +static const GraphNodeBox* get_box(const GraphNodeHandle& h) { + const CUgraphNode* p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) - offsetof(GraphNodeBox, resource) + ); +} + +// graphs is ordered parent-before-child. Nulling a selected box marks its +// later descendants, whose parent pointers remain valid after list splicing. +// This permits one allocation-free sweep of the hierarchy. +void invalidate_child_graph_state( + const GraphHandle& h_parent, + CUgraphNode owner_node) noexcept { + if (!h_parent || !owner_node) { + return; + } + + GraphBox* parent = get_box(h_parent); + if (!parent->resource) { + return; + } + GraphHierarchy& hierarchy = *parent->hierarchy; + for (auto it = hierarchy.graphs.begin(); + it != hierarchy.graphs.end();) { + auto graph = it++; + bool is_owned_root = graph->parent == parent && + graph->owner_node == owner_node; + bool is_descendant = graph->parent && + !graph->parent->resource; + if (!is_owned_root && !is_descendant) { + continue; + } + + // Empty node_handles and invalidate each one. + for (auto& entry : graph->node_handles.drain()) { + if (GraphNodeHandle h_node = entry.second.lock()) { + get_box(h_node)->resource = nullptr; + } + } + graph_registry.unregister_handle(graph->resource); + graph->resource = nullptr; + graph->attachments.clear(); + hierarchy.graveyard.splice( + hierarchy.graveyard.end(), hierarchy.graphs, graph); + } +} + +GraphNodeHandle create_graph_node_handle(CUgraphNode node, const GraphHandle& h_graph) { + if (!node) { + auto box = std::make_shared( + GraphNodeBox{nullptr, h_graph}); + return GraphNodeHandle(box, &box->resource); + } + + GraphBox* graph = get_box(h_graph); + return graph->node_handles.get_or_create( + node, + [node, &h_graph] { + auto box = std::make_shared( + GraphNodeBox{node, h_graph}); + return GraphNodeHandle(box, &box->resource); + }); +} + +GraphHandle graph_node_get_graph(const GraphNodeHandle& h) noexcept { + return h ? get_box(h)->h_graph : GraphHandle{}; +} + +void invalidate_graph_node(const GraphNodeHandle& h) noexcept { + if (!h) { + return; + } + + const GraphNodeBox* node_box = get_box(h); + CUgraphNode node = node_box->resource; + if (!node) { + return; + } + GraphBox* graph = get_box(node_box->h_graph); + graph->node_handles.unregister_handle(node); + node_box->resource = nullptr; +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/graph_exec.cpp b/cuda_core/cuda/core/_cpp/rt/graph_exec.cpp new file mode 100644 index 00000000000..dbf09cf77e0 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/graph_exec.cpp @@ -0,0 +1,293 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +// ============================================================================ +// Graph Exec Handles +// ============================================================================ + +namespace { + +// Append-only owners introduced by individual executable-node updates. CUDA +// owns this payload through a user object propagated into the CUgraphExec. +struct ExecAttachments : DeferredCleanupItem { + CUuserObject object = nullptr; + std::vector owners; +}; + +struct GraphExecBox { + CUgraphExec resource = nullptr; + ExecAttachments* attachments = nullptr; // Non-owning. + + ~GraphExecBox() noexcept { + if (resource) { + GILReleaseGuard gil; + pw_cuGraphExecDestroy(resource); + } + // The accumulator fields may be dangling after exec destruction. + retry_deferred_cleanup(); + } +}; + +GraphExecBox* get_exec_box(const GraphExecHandle& h) noexcept { + return const_cast( + reinterpret_cast(h.get())); +} + +GraphExecHandle make_graph_exec_handle( + CUgraphExec graph_exec, ExecAttachments* attachments) { + struct RawGraphExecGuard { + CUgraphExec resource; + + ~RawGraphExecGuard() noexcept { + if (resource) { + GILReleaseGuard gil; + pw_cuGraphExecDestroy(resource); + } + retry_deferred_cleanup(); + } + } guard{graph_exec}; + + auto box = std::make_shared(); + box->resource = graph_exec; + box->attachments = attachments; + guard.resource = nullptr; + return GraphExecHandle(box, &box->resource); +} + +// Holds a fresh accumulator retained on the source graph across a CUDA call +// that propagates user objects into an exec. Releasing drops the source's +// reference: after successful propagation the exec keeps the accumulator +// alive, and otherwise this drops its last reference. +struct ExecAttachmentStaging { + GraphHandle h_source; + ExecAttachments* accumulator = nullptr; + + ~ExecAttachmentStaging() noexcept { + report_cuda_error("cuGraphReleaseUserObject", release(), + "failed while dropping a staged graph attachment"); + } + + CUresult release() noexcept { + if (!h_source || !accumulator) { + return CUDA_SUCCESS; + } + const CUuserObject object = accumulator->object; + const GraphHandle source = std::move(h_source); + accumulator = nullptr; + GILReleaseGuard gil; + return p_cuGraphReleaseUserObject(*source, object, 1); + } +}; + +// Create an accumulator and retain it on h_source, so that a following +// instantiation or whole-graph update propagates a reference into the exec. +CUresult stage_exec_attachments( + const GraphHandle& h_source, ExecAttachmentStaging* out_staging) { + if (!p_cuUserObjectCreate || !p_cuUserObjectRelease || + !p_cuGraphRetainUserObject || !p_cuGraphReleaseUserObject) { + return CUDA_ERROR_NOT_SUPPORTED; + } + + ensure_deferred_cleanup_ready(); + auto* accumulator = new ExecAttachments; + + CUuserObject object = nullptr; + CUresult status; + { + GILReleaseGuard gil; + status = p_cuUserObjectCreate( + &object, + static_cast(accumulator), + reinterpret_cast(enqueue_cleanup), + 1, + CU_USER_OBJECT_NO_DESTRUCTOR_SYNC); + if (status != CUDA_SUCCESS) { + delete accumulator; + return status; + } + accumulator->object = object; + status = p_cuGraphRetainUserObject( + *h_source, object, 1, CU_GRAPH_USER_OBJECT_MOVE); + if (status != CUDA_SUCCESS) { + // Dropping the last reference retires the accumulator. + pw_cuUserObjectRelease(object, 1); + return status; + } + } + + out_staging->h_source = h_source; + out_staging->accumulator = accumulator; + return CUDA_SUCCESS; +} + +} // namespace + +// State held by PreparedExecAttachment between preparation and commit. It keeps +// the exec alive and remembers the accumulator size before the append, so that +// rollback can drop owners staged for a mutation that CUDA rejected. +struct PreparedExecAttachmentState { + GraphExecHandle h_exec; + ExecAttachments* attachments = nullptr; + size_t original_size = 0; + + PreparedExecAttachmentState( + GraphExecHandle h_exec_, + ExecAttachments* attachments_, + size_t original_size_) + : h_exec(std::move(h_exec_)), + attachments(attachments_), + original_size(original_size_) {} +}; + +void rollback_prepared_exec_attachment( + PreparedExecAttachmentState* state) noexcept { + if (!state) { + return; + } + if (state->attachments) { + while (state->attachments->owners.size() > state->original_size) { + state->attachments->owners.pop_back(); + } + } + delete state; +} + +GraphExecHandle create_graph_exec_handle( + const GraphHandle& h_source, + CUDA_GRAPH_INSTANTIATE_PARAMS* params) { + if (!h_source || !*h_source || !params) { + err = CUDA_ERROR_INVALID_VALUE; + return {}; + } + if (!p_cuGraphInstantiateWithParams) { + err = CUDA_ERROR_NOT_SUPPORTED; + return {}; + } + + ExecAttachmentStaging staging; + if (CUDA_SUCCESS != (err = stage_exec_attachments(h_source, &staging))) { + return {}; + } + + CUgraphExec graph_exec = nullptr; + { + GILReleaseGuard gil; + err = p_cuGraphInstantiateWithParams(&graph_exec, *h_source, params); + } + if (err != CUDA_SUCCESS) { + return {}; + } + // CUDA can report a specific failure while returning success. The exec is + // then unusable, so it stays unadopted for the caller to diagnose from + // params->result_out. + if (params->result_out != CUDA_GRAPH_INSTANTIATE_SUCCESS) { + return {}; + } + if (!graph_exec) { + err = CUDA_ERROR_INVALID_VALUE; + return {}; + } + + GraphExecHandle h_exec = make_graph_exec_handle( + graph_exec, staging.accumulator); + if (CUDA_SUCCESS != (err = staging.release())) { + return {}; + } + return h_exec; +} + +CUresult graph_exec_update( + const GraphExecHandle& h_exec, + const GraphHandle& h_source, + CUgraphExecUpdateResultInfo* result_info) { + if (!h_exec || !h_source || !*h_source || !result_info) { + return CUDA_ERROR_INVALID_VALUE; + } + if (!p_cuGraphExecUpdate) { + return CUDA_ERROR_NOT_SUPPORTED; + } + + GraphExecBox* box = get_exec_box(h_exec); + if (!box->resource) { + return CUDA_ERROR_INVALID_VALUE; + } + + ExecAttachmentStaging staging; + CUresult status = stage_exec_attachments(h_source, &staging); + if (status != CUDA_SUCCESS) { + return status; + } + + { + GILReleaseGuard gil; + status = p_cuGraphExecUpdate(box->resource, *h_source, result_info); + } + if (status != CUDA_SUCCESS) { + return status; + } + + // CUDA may already have retired the old accumulator. Publish the new one + // before releasing the source graph's temporary reference. + box->attachments = staging.accumulator; + return staging.release(); +} + +CUresult graph_prepare_exec_attachment( + const GraphExecHandle& h_exec, + OpaqueHandle owner0, + OpaqueHandle owner1, + PreparedExecAttachment* out_prepared) { + if (!out_prepared) { + return CUDA_ERROR_INVALID_VALUE; + } + out_prepared->reset(); + if (!h_exec) { + return CUDA_ERROR_INVALID_VALUE; + } + + GraphExecBox* box = get_exec_box(h_exec); + if (!box->resource || !box->attachments) { + return CUDA_ERROR_INVALID_VALUE; + } + + ExecAttachments* attachments = box->attachments; + const size_t original_size = attachments->owners.size(); + const size_t additions = + static_cast(static_cast(owner0)) + + static_cast(static_cast(owner1)); + // Reserve before staging so that rollback and commit cannot allocate. + attachments->owners.reserve(original_size + additions); + PreparedExecAttachment prepared( + new PreparedExecAttachmentState(h_exec, attachments, original_size), + PreparedExecAttachmentDeleter{rollback_prepared_exec_attachment}); + if (owner0) { + attachments->owners.emplace_back(std::move(owner0)); + } + if (owner1) { + attachments->owners.emplace_back(std::move(owner1)); + } + *out_prepared = std::move(prepared); + return CUDA_SUCCESS; +} + +void graph_commit_exec_attachment( + PreparedExecAttachment& prepared) noexcept { + delete prepared.release(); +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/handles.hpp b/cuda_core/cuda/core/_cpp/rt/handles.hpp new file mode 100644 index 00000000000..043ddd7ef88 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/handles.hpp @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +// Consumer umbrella, named only by _rt.pxd: the handle types, the inline +// accessors and the Python seam. Everything else reaches consumers through +// __pyx_capi__, never through a header. + +#include "py.hpp" +#include "types.hpp" diff --git a/cuda_core/cuda/core/_cpp/rt/internal.hpp b/cuda_core/cuda/core/_cpp/rt/internal.hpp new file mode 100644 index 00000000000..7033c884722 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/internal.hpp @@ -0,0 +1,176 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "types.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cuda_core::rt::detail { + +// Implemented in error.cpp +void format_cuda_error(char* buffer, size_t size, const char* operation, CUresult status, + const char* detail) noexcept; +// Implemented in error.cpp +void note_context_not_restored(CUcontext previous, CUresult operation_status, + CUresult restore_status) noexcept; + +// Store a stream and any state needed to preserve deallocation ordering. +struct DeallocationStream { + StreamHandle h_stream; + std::thread::id ptds_tid{}; +}; + +// Implemented in stream.cpp +ContextHandle deallocation_context(const DeallocationStream& stream) noexcept; +// Implemented in stream.cpp +bool make_deallocation_stream(const StreamHandle& h, DeallocationStream& out) noexcept; + +// Decorate a status-returning cleanup call to report whenever it fails. CUDA +// calls (CUresult) are reported with the error name and description; NVRTC, +// NVVM and nvJitLink calls (integer status codes) with the raw code. +template +class WarnOnFailure { +public: + explicit WarnOnFailure(const char* operation) noexcept : operation_(operation) {} + + template + auto operator()(Args&&... args) const noexcept { + auto status = Function(std::forward(args)...); + report(status); + return status; + } + +private: + void report(CUresult status) const noexcept { + report_cuda_error(operation_, status); + } + + template + void report(Status status) const noexcept { + if (static_cast(status) != 0) { + report_status_code(operation_, static_cast(status)); + } + } + + const char* operation_; +}; + +// Warning-decorated CUDA operations used by non-throwing cleanup paths. +const WarnOnFailure pw_cuStreamDestroy{"cuStreamDestroy"}; +const WarnOnFailure pw_cuEventDestroy{"cuEventDestroy"}; +const WarnOnFailure pw_cuMemFree{"cuMemFree"}; +const WarnOnFailure pw_cuMemFreeAsync{"cuMemFreeAsync"}; +const WarnOnFailure pw_cuArrayDestroy{"cuArrayDestroy"}; +const WarnOnFailure pw_cuMipmappedArrayDestroy{"cuMipmappedArrayDestroy"}; +const WarnOnFailure pw_cuTexObjectDestroy{"cuTexObjectDestroy"}; +const WarnOnFailure pw_cuSurfObjectDestroy{"cuSurfObjectDestroy"}; +const WarnOnFailure pw_cuGreenCtxDestroy{"cuGreenCtxDestroy"}; +const WarnOnFailure pw_cuMemPoolDestroy{"cuMemPoolDestroy"}; +const WarnOnFailure pw_cuMemFreeHost{"cuMemFreeHost"}; +const WarnOnFailure pw_cuGraphDestroy{"cuGraphDestroy"}; +const WarnOnFailure pw_cuGraphExecDestroy{"cuGraphExecDestroy"}; +const WarnOnFailure pw_cuGraphicsUnregisterResource{"cuGraphicsUnregisterResource"}; +const WarnOnFailure pw_cuLinkDestroy{"cuLinkDestroy"}; +const WarnOnFailure pw_cuUserObjectRelease{"cuUserObjectRelease"}; +const WarnOnFailure pw_cuGraphReleaseUserObject{"cuGraphReleaseUserObject"}; +const WarnOnFailure pw_nvrtcDestroyProgram{"nvrtcDestroyProgram"}; +const WarnOnFailure pw_nvvmDestroyProgram{"nvvmDestroyProgram"}; +const WarnOnFailure pw_nvJitLinkDestroy{"nvJitLinkDestroy"}; + +// Intrusive base for payloads transferred out of CUDA's callback. +struct DeferredCleanupItem { + DeferredCleanupItem* next = nullptr; + virtual ~DeferredCleanupItem() noexcept = default; +}; + +// Implemented in py_deferred_cleanup.cpp +void ensure_deferred_cleanup_ready(); +// Implemented in py_deferred_cleanup.cpp +void enqueue_cleanup(void* item) noexcept; + +// ============================================================================ +// Handle reverse-lookup registry +// +// Maps raw CUDA handles (CUevent, CUkernel, etc.) back to their owning +// shared_ptr so that _ref constructors can recover full metadata. +// Uses weak_ptr to avoid preventing destruction. +// ============================================================================ + +template> +class HandleRegistry { +public: + using MapType = std::unordered_map, Hash>; + + void register_handle(const Key& key, const Handle& h) { + std::lock_guard lock(mutex_); + map_[key] = h; + } + + void unregister_handle(const Key& key) noexcept { + std::lock_guard lock(mutex_); + map_.erase(key); + } + + void register_handles(const std::vector& handles) { + std::lock_guard lock(mutex_); + for (const Handle& h : handles) { + if (h) { + map_[*h] = h; + } + } + } + + Handle lookup(const Key& key) { + std::lock_guard lock(mutex_); + auto it = map_.find(key); + if (it != map_.end()) { + if (auto h = it->second.lock()) { + return h; + } + map_.erase(it); + } + return {}; + } + + template + Handle get_or_create(const Key& key, Factory&& create) { + std::lock_guard lock(mutex_); + auto it = map_.find(key); + if (it != map_.end()) { + if (Handle h = it->second.lock()) { + return h; + } + map_.erase(it); + } + + Handle h = create(); + if (h) { + map_[key] = h; + } + return h; + } + + MapType drain() noexcept { + std::lock_guard lock(mutex_); + MapType extracted; + extracted.swap(map_); + return extracted; + } + +private: + std::mutex mutex_; + MapType map_; +}; + +} // namespace cuda_core::rt::detail diff --git a/cuda_core/cuda/core/_cpp/rt/memory.cpp b/cuda_core/cuda/core/_cpp/rt/memory.cpp new file mode 100644 index 00000000000..7e5b0ab34a4 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/memory.cpp @@ -0,0 +1,497 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "context_scope.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include +#include +#include +#include +#include +#ifndef _WIN32 +#include +#endif + +namespace cuda_core::rt { + +using namespace detail; + +// ============================================================================ +// Memory Pool Handles +// ============================================================================ + +namespace { +struct MemoryPoolBox { + CUmemoryPool resource; +}; +} // namespace + +// Helper to clear peer access before destroying a memory pool. +// Works around nvbug 5698116: recycled pool handles inherit peer access state. +// Must be noexcept since it's called from a shared_ptr deleter. +static void clear_mempool_peer_access(CUmemoryPool pool) noexcept { + try { + int device_count = 0; + if (p_cuDeviceGetCount(&device_count) != CUDA_SUCCESS || device_count <= 0) { + return; + } + + std::vector clear_access(device_count); + for (int i = 0; i < device_count; ++i) { + clear_access[i].location.type = CU_MEM_LOCATION_TYPE_DEVICE; + clear_access[i].location.id = i; + clear_access[i].flags = CU_MEM_ACCESS_FLAGS_PROT_NONE; + } + p_cuMemPoolSetAccess(pool, clear_access.data(), device_count); // Best effort + } catch (...) { + // Swallow exceptions - this is best-effort cleanup in destructor context + } +} + +static MemoryPoolHandle wrap_mempool_owned(CUmemoryPool pool) { + auto box = std::shared_ptr( + new MemoryPoolBox{pool}, + [](const MemoryPoolBox* b) { + GILReleaseGuard gil; + clear_mempool_peer_access(b->resource); + pw_cuMemPoolDestroy(b->resource); + delete b; + } + ); + return MemoryPoolHandle(box, &box->resource); +} + +MemoryPoolHandle create_mempool_handle(const CUmemPoolProps& props) { + GILReleaseGuard gil; + CUmemoryPool pool; + if (CUDA_SUCCESS != (err = p_cuMemPoolCreate(&pool, &props))) { + return {}; + } + return wrap_mempool_owned(pool); +} + +MemoryPoolHandle create_mempool_handle_ref(CUmemoryPool pool) { + auto box = std::make_shared(MemoryPoolBox{pool}); + return MemoryPoolHandle(box, &box->resource); +} + +MemoryPoolHandle get_device_mempool(int device_id) { + GILReleaseGuard gil; + CUmemoryPool pool; + if (CUDA_SUCCESS != (err = p_cuDeviceGetMemPool(&pool, device_id))) { + return {}; + } + return create_mempool_handle_ref(pool); +} + +MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type) { + GILReleaseGuard gil; + CUmemoryPool pool; + auto handle_ptr = reinterpret_cast(static_cast(fd)); + if (CUDA_SUCCESS != (err = p_cuMemPoolImportFromShareableHandle(&pool, handle_ptr, handle_type, 0))) { + return {}; + } + return wrap_mempool_owned(pool); +} + +// ============================================================================ +// Device Pointer Handles +// ============================================================================ + +namespace { +struct DevicePtrBox { + CUdeviceptr resource; + // Mutable so set_deallocation_stream() can update free ordering through a + // const DevicePtrHandle. Built with make_deallocation_stream so default- + // stream tokens carry a bound context. + mutable DeallocationStream deallocation; +}; +} // namespace + +// Recovers the owning DevicePtrBox from the aliased CUdeviceptr pointer. +// This works because DevicePtrHandle is a shared_ptr alias pointing to +// &box->resource, so we can compute the containing struct using offsetof. +// The const_cast is safe because we only use this to access the mutable +// deallocation member or in the deleter (where the box is being destroyed). +static DevicePtrBox* get_box(const DevicePtrHandle& h) { + const CUdeviceptr* p = h.get(); + return reinterpret_cast( + reinterpret_cast(const_cast(p)) - offsetof(DevicePtrBox, resource) + ); +} + +// Return the stream that orders a device pointer's deallocation. +StreamHandle deallocation_stream(const DevicePtrHandle& h) noexcept { + return get_box(h)->deallocation.h_stream; +} + +// Replace the stream that orders a device pointer's deallocation. +CUresult set_deallocation_stream(const DevicePtrHandle& h, const StreamHandle& h_stream) noexcept { + if (!h) { + return CUDA_ERROR_INVALID_VALUE; + } + DeallocationStream ds; + if (!make_deallocation_stream(h_stream, ds)) { + return err != CUDA_SUCCESS ? err : CUDA_ERROR_INVALID_CONTEXT; + } + get_box(h)->deallocation = std::move(ds); + return CUDA_SUCCESS; +} + +DevicePtrHandle deviceptr_alloc_from_pool(size_t size, const MemoryPoolHandle& h_pool, const StreamHandle& h_stream) { + GILReleaseGuard gil; + CUdeviceptr ptr; + if (CUDA_SUCCESS != (err = p_cuMemAllocFromPoolAsync(&ptr, size, *h_pool, as_cu(h_stream)))) { + return {}; + } + + DeallocationStream ds; + if (!make_deallocation_stream(h_stream, ds)) { + pw_cuMemFreeAsync(ptr, as_cu(h_stream)); + return {}; + } + + auto box = std::shared_ptr( + new DevicePtrBox{ptr, std::move(ds)}, + [h_pool](DevicePtrBox* b) { + GILReleaseGuard gil; + const DeallocationStream& stream = b->deallocation; + cleanup_in_context( + deallocation_context(stream), "cuMemFreeAsync", + [&]() noexcept { + return p_cuMemFreeAsync( + b->resource, as_cu(stream.h_stream)); + }); + delete b; + } + ); + return DevicePtrHandle(box, &box->resource); +} + +DevicePtrHandle deviceptr_alloc_async(size_t size, const StreamHandle& h_stream) { + GILReleaseGuard gil; + CUdeviceptr ptr; + if (CUDA_SUCCESS != (err = p_cuMemAllocAsync(&ptr, size, as_cu(h_stream)))) { + return {}; + } + + DeallocationStream ds; + if (!make_deallocation_stream(h_stream, ds)) { + pw_cuMemFreeAsync(ptr, as_cu(h_stream)); + return {}; + } + + auto box = std::shared_ptr( + new DevicePtrBox{ptr, std::move(ds)}, + [](DevicePtrBox* b) { + GILReleaseGuard gil; + const DeallocationStream& stream = b->deallocation; + cleanup_in_context( + deallocation_context(stream), "cuMemFreeAsync", + [&]() noexcept { + return p_cuMemFreeAsync( + b->resource, as_cu(stream.h_stream)); + }); + delete b; + } + ); + return DevicePtrHandle(box, &box->resource); +} + +// Allocate device memory synchronously with the provided context current. +CUresult deviceptr_alloc_raw(CUdeviceptr* ptr, size_t size, + const ContextHandle& h_context) noexcept { + GILReleaseGuard gil; + return invoke_in_context_or_undo( + h_context, + [&]() noexcept { return p_cuMemAlloc(ptr, size); }, + [&]() noexcept { pw_cuMemFree(*ptr); }, + /*undo_requires_target_context=*/false); +} + +DevicePtrHandle deviceptr_alloc_host(size_t size) { + GILReleaseGuard gil; + void* ptr; + if (CUDA_SUCCESS != (err = p_cuMemAllocHost(&ptr, size))) { + return {}; + } + + auto box = std::shared_ptr( + new DevicePtrBox{reinterpret_cast(ptr), DeallocationStream{}}, + [](DevicePtrBox* b) { + GILReleaseGuard gil; + pw_cuMemFreeHost(reinterpret_cast(b->resource)); + delete b; + } + ); + return DevicePtrHandle(box, &box->resource); +} + +DevicePtrHandle deviceptr_create_ref(CUdeviceptr ptr) { + auto box = std::make_shared(DevicePtrBox{ptr, DeallocationStream{}}); + return DevicePtrHandle(box, &box->resource); +} + +DevicePtrHandle deviceptr_create_with_owner(CUdeviceptr ptr, PyObject* owner) { + if (!owner) { + return deviceptr_create_ref(ptr); + } + // GIL required when owner is provided + GILAcquireGuard gil; + if (!gil.acquired()) { + // Python finalizing - fall back to ref version (no owner tracking) + return deviceptr_create_ref(ptr); + } + Py_INCREF(owner); + auto box = std::shared_ptr( + new DevicePtrBox{ptr, DeallocationStream{}}, + [owner](DevicePtrBox* b) { + GILAcquireGuard gil; + if (gil.acquired()) { + Py_DECREF(owner); + } + delete b; + } + ); + return DevicePtrHandle(box, &box->resource); +} + +DevicePtrHandle deviceptr_create_mapped_graphics( + CUdeviceptr ptr, + const GraphicsResourceHandle& h_resource, + const StreamHandle& h_stream +) { + DeallocationStream ds; + if (!make_deallocation_stream(h_stream, ds)) { + return {}; + } + auto box = std::shared_ptr( + new DevicePtrBox{ptr, std::move(ds)}, + [h_resource](DevicePtrBox* b) { + GILReleaseGuard gil; + CUgraphicsResource resource = as_cu(h_resource); + const DeallocationStream& stream = b->deallocation; + cleanup_in_context( + deallocation_context(stream), "cuGraphicsUnmapResources", + [&]() noexcept { + return p_cuGraphicsUnmapResources( + 1, &resource, as_cu(stream.h_stream)); + }); + delete b; + } + ); + return DevicePtrHandle(box, &box->resource); +} + +// ============================================================================ +// MemoryResource-owned Device Pointer Handles +// ============================================================================ + +static MRDeallocCallback mr_dealloc_cb = nullptr; + +void register_mr_dealloc_callback(MRDeallocCallback cb) { + mr_dealloc_cb = cb; +} + +DevicePtrHandle deviceptr_create_with_mr(CUdeviceptr ptr, size_t size, PyObject* mr) { + if (!mr) { + return deviceptr_create_ref(ptr); + } + // GIL required when mr is provided + GILAcquireGuard gil; + if (!gil.acquired()) { + return deviceptr_create_ref(ptr); + } + Py_INCREF(mr); + auto box = std::shared_ptr( + new DevicePtrBox{ptr, DeallocationStream{}}, + [mr, size](DevicePtrBox* b) { + GILAcquireGuard gil; + if (gil.acquired()) { + if (mr_dealloc_cb) { + const DeallocationStream& stream = b->deallocation; + cleanup_in_context( + deallocation_context(stream), "MemoryResource.deallocate", + [&]() noexcept { + mr_dealloc_cb(mr, b->resource, size, stream.h_stream); + return CUDA_SUCCESS; + }); + } + Py_DECREF(mr); + } + delete b; + } + ); + return DevicePtrHandle(box, &box->resource); +} + +// ============================================================================ +// IPC Pointer Cache +// ============================================================================ +// This cache handles duplicate IPC imports, which behave differently depending +// on the memory type: +// +// 1. Memory pool allocations (DeviceMemoryResource): +// Multiple imports of the same allocation succeed and return duplicate +// pointers. However, the driver has a reference counting bug (nvbug 5570902) +// where the first cuMemFreeAsync incorrectly unmaps the memory even when +// imported multiple times. A driver fix is expected. +// +// 2. Pinned memory allocations (PinnedMemoryResource): +// Duplicate imports result in CUDA_ERROR_ALREADY_MAPPED. +// +// The cache solves both issues by checking the cache before calling +// cuMemPoolImportPointer and returning the existing handle for duplicate +// imports. This provides a consistent user experience where the same IPC +// descriptor can be imported multiple times regardless of memory type. +// +// The cache key is the export_data bytes (CUmemPoolPtrExportData), not the +// returned pointer, because we must check before calling the driver API. + + +// TODO: When driver fix for nvbug 5570902 is available, consider whether +// the cache is still needed for memory pool allocations (it will still be +// needed for pinned memory). +static bool use_ipc_ptr_cache() { + return true; +} + +namespace { +// Wrapper for CUmemPoolPtrExportData to use as map key +struct ExportDataKey { + CUmemPoolPtrExportData data; + + bool operator==(const ExportDataKey& other) const { + return std::memcmp(&data, &other.data, sizeof(data)) == 0; + } +}; + +struct ExportDataKeyHash { + std::size_t operator()(const ExportDataKey& key) const { + // Simple hash of the bytes + std::size_t h = 0; + const auto* bytes = reinterpret_cast(&key.data); + for (std::size_t i = 0; i < sizeof(key.data); ++i) { + h = h * 31 + bytes[i]; + } + return h; + } +}; + +} + +static HandleRegistry ipc_ptr_cache; +static std::mutex ipc_import_mutex; + +DevicePtrHandle deviceptr_import_ipc(const MemoryPoolHandle& h_pool, const void* export_data, const StreamHandle& h_stream) { + auto data = const_cast( + reinterpret_cast(export_data)); + + if (use_ipc_ptr_cache()) { + ExportDataKey key; + std::memcpy(&key.data, data, sizeof(key.data)); + + std::lock_guard lock(ipc_import_mutex); + + if (auto h = ipc_ptr_cache.lookup(key)) { + return h; + } + + GILReleaseGuard gil; + CUdeviceptr ptr; + if (CUDA_SUCCESS != (err = p_cuMemPoolImportPointer(&ptr, *h_pool, data))) { + return {}; + } + + DeallocationStream ds; + if (!make_deallocation_stream(h_stream, ds)) { + pw_cuMemFreeAsync(ptr, as_cu(h_stream)); + return {}; + } + + auto box = std::shared_ptr( + new DevicePtrBox{ptr, std::move(ds)}, + [h_pool, key](DevicePtrBox* b) { + ipc_ptr_cache.unregister_handle(key); + GILReleaseGuard gil; + const DeallocationStream& stream = b->deallocation; + cleanup_in_context( + deallocation_context(stream), "cuMemFreeAsync", + [&]() noexcept { + return p_cuMemFreeAsync( + b->resource, as_cu(stream.h_stream)); + }); + delete b; + } + ); + DevicePtrHandle h(box, &box->resource); + ipc_ptr_cache.register_handle(key, h); + return h; + + } else { + GILReleaseGuard gil; + CUdeviceptr ptr; + if (CUDA_SUCCESS != (err = p_cuMemPoolImportPointer(&ptr, *h_pool, data))) { + return {}; + } + + DeallocationStream ds; + if (!make_deallocation_stream(h_stream, ds)) { + pw_cuMemFreeAsync(ptr, as_cu(h_stream)); + return {}; + } + + auto box = std::shared_ptr( + new DevicePtrBox{ptr, std::move(ds)}, + [h_pool](DevicePtrBox* b) { + GILReleaseGuard gil; + const DeallocationStream& stream = b->deallocation; + cleanup_in_context( + deallocation_context(stream), "cuMemFreeAsync", + [&]() noexcept { + return p_cuMemFreeAsync( + b->resource, as_cu(stream.h_stream)); + }); + delete b; + } + ); + return DevicePtrHandle(box, &box->resource); + } +} + +// ============================================================================ +// File Descriptor Handles +// ============================================================================ + +FileDescriptorHandle create_fd_handle(int fd) { +#ifdef _WIN32 + throw std::runtime_error("create_fd_handle is not supported on Windows"); +#else + return FileDescriptorHandle( + new int(fd), + [](const int* p) { + if (::close(*p) != 0) { + report_message("close() failed for an IPC file descriptor; the descriptor may have leaked"); + } + delete p; + } + ); +#endif +} + +FileDescriptorHandle create_fd_handle_ref(int fd) { +#ifdef _WIN32 + throw std::runtime_error("create_fd_handle_ref is not supported on Windows"); +#else + return std::make_shared(fd); +#endif +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/program.cpp b/cuda_core/cuda/core/_cpp/rt/program.cpp new file mode 100644 index 00000000000..74c6455264a --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/program.cpp @@ -0,0 +1,243 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +// ============================================================================ +// Library Handles +// ============================================================================ + +namespace { +struct LibraryBox { + CUlibrary resource; +}; +} // namespace + +LibraryHandle create_library_handle_from_file(const char* path) { + GILReleaseGuard gil; + CUlibrary library; + if (CUDA_SUCCESS != (err = p_cuLibraryLoadFromFile(&library, path, nullptr, nullptr, 0, nullptr, nullptr, 0))) { + return {}; + } + + auto box = std::shared_ptr( + new LibraryBox{library}, + [](const LibraryBox* b) { + GILReleaseGuard gil; + // TODO: re-enable once LibraryBox tracks its owning context + // p_cuLibraryUnload(b->resource); + delete b; + } + ); + return LibraryHandle(box, &box->resource); +} + +LibraryHandle create_library_handle_from_data(const void* data) { + GILReleaseGuard gil; + CUlibrary library; + if (CUDA_SUCCESS != (err = p_cuLibraryLoadData(&library, data, nullptr, nullptr, 0, nullptr, nullptr, 0))) { + return {}; + } + + auto box = std::shared_ptr( + new LibraryBox{library}, + [](const LibraryBox* b) { + GILReleaseGuard gil; + // TODO: re-enable once LibraryBox tracks its owning context + // p_cuLibraryUnload(b->resource); + delete b; + } + ); + return LibraryHandle(box, &box->resource); +} + +LibraryHandle create_library_handle_ref(CUlibrary library) { + auto box = std::make_shared(LibraryBox{library}); + return LibraryHandle(box, &box->resource); +} + +// ============================================================================ +// Kernel Handles +// ============================================================================ + +namespace { +struct KernelBox { + CUkernel resource; + LibraryHandle h_library; +}; +} // namespace + +static const KernelBox* get_box(const KernelHandle& h) { + const CUkernel* p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) - offsetof(KernelBox, resource) + ); +} + +// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) +static HandleRegistry kernel_registry; + +KernelHandle create_kernel_handle(const LibraryHandle& h_library, const char* name) { + GILReleaseGuard gil; + CUkernel kernel; + if (CUDA_SUCCESS != (err = p_cuLibraryGetKernel(&kernel, *h_library, name))) { + return {}; + } + + auto box = std::make_shared(KernelBox{kernel, h_library}); + KernelHandle h(box, &box->resource); + kernel_registry.register_handle(kernel, h); + return h; +} + +KernelHandle create_kernel_handle_ref(CUkernel kernel) { + if (auto h = kernel_registry.lookup(kernel)) { + return h; + } + auto box = std::make_shared(KernelBox{kernel, {}}); + return KernelHandle(box, &box->resource); +} + +LibraryHandle get_kernel_library(const KernelHandle& h) noexcept { + if (!h) return {}; + return get_box(h)->h_library; +} + +// ============================================================================ +// NVRTC Program Handles +// ============================================================================ + +namespace { +struct NvrtcProgramBox { + nvrtcProgram resource; +}; +} // namespace + +NvrtcProgramHandle create_nvrtc_program_handle(nvrtcProgram prog) { + auto box = std::shared_ptr( + new NvrtcProgramBox{prog}, + [](NvrtcProgramBox* b) { + // Note: nvrtcDestroyProgram takes nvrtcProgram* and nulls it, + // but we're deleting the box anyway so nulling is harmless. + if (p_nvrtcDestroyProgram) { + GILReleaseGuard gil; + pw_nvrtcDestroyProgram(&b->resource); + } + delete b; + } + ); + return NvrtcProgramHandle(box, &box->resource); +} + +NvrtcProgramHandle create_nvrtc_program_handle_ref(nvrtcProgram prog) { + auto box = std::make_shared(NvrtcProgramBox{prog}); + return NvrtcProgramHandle(box, &box->resource); +} + +// ============================================================================ +// NVVM Program Handles +// ============================================================================ + +namespace { +struct NvvmProgramBox { + NvvmProgramValue resource; +}; +} // namespace + +NvvmProgramHandle create_nvvm_program_handle(nvvmProgram prog) { + auto box = std::shared_ptr( + new NvvmProgramBox{{prog}}, + [](NvvmProgramBox* b) { + // Note: nvvmDestroyProgram takes nvvmProgram* and nulls it, + // but we're deleting the box anyway so nulling is harmless. + // If NVVM is not available, the function pointer is null. + if (p_nvvmDestroyProgram) { + GILReleaseGuard gil; + pw_nvvmDestroyProgram(&b->resource.raw); + } + delete b; + } + ); + return NvvmProgramHandle(box, &box->resource); +} + +NvvmProgramHandle create_nvvm_program_handle_ref(nvvmProgram prog) { + auto box = std::make_shared(NvvmProgramBox{{prog}}); + return NvvmProgramHandle(box, &box->resource); +} + +// ============================================================================ +// nvJitLink Handles +// ============================================================================ + +namespace { +struct NvJitLinkBox { + NvJitLinkValue resource; +}; +} // namespace + +NvJitLinkHandle create_nvjitlink_handle(nvJitLink_t handle) { + auto box = std::shared_ptr( + new NvJitLinkBox{{handle}}, + [](NvJitLinkBox* b) { + // Note: nvJitLinkDestroy takes nvJitLinkHandle* and nulls it, + // but we're deleting the box anyway so nulling is harmless. + // If nvJitLink is not available, the function pointer is null. + if (p_nvJitLinkDestroy) { + GILReleaseGuard gil; + pw_nvJitLinkDestroy(&b->resource.raw); + } + delete b; + } + ); + return NvJitLinkHandle(box, &box->resource); +} + +NvJitLinkHandle create_nvjitlink_handle_ref(nvJitLink_t handle) { + auto box = std::make_shared(NvJitLinkBox{{handle}}); + return NvJitLinkHandle(box, &box->resource); +} + +// ============================================================================ +// cuLink Handles +// ============================================================================ + +namespace { +struct CuLinkBox { + CUlinkState resource; +}; +} // namespace + +CuLinkHandle create_culink_handle(CUlinkState state) { + auto box = std::shared_ptr( + new CuLinkBox{state}, + [](CuLinkBox* b) { + // cuLinkDestroy takes CUlinkState by value (not pointer). + if (p_cuLinkDestroy) { + GILReleaseGuard gil; + pw_cuLinkDestroy(b->resource); + } + delete b; + } + ); + return CuLinkHandle(box, &box->resource); +} + +CuLinkHandle create_culink_handle_ref(CUlinkState state) { + auto box = std::make_shared(CuLinkBox{state}); + return CuLinkHandle(box, &box->resource); +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/py.hpp b/cuda_core/cuda/core/_cpp/rt/py.hpp new file mode 100644 index 00000000000..b606398317e --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/py.hpp @@ -0,0 +1,253 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include "types.hpp" +#include +#include + +namespace cuda_core::rt { + +#if PY_VERSION_HEX < 0x030D0000 +extern "C" int _Py_IsFinalizing(void); +#endif + +// Best-effort probe for interpreter shutdown. +// +// In CPython this is not a hard guarantee: finalization can begin after this +// returns false but before a later PyGILState_Ensure() or other Python C-API +// call. +// +// If that race is lost on a non-finalizer thread, CPython's behavior is +// version-dependent: on older supported versions (3.10-3.13) it may abruptly +// terminate the current thread (historically via PyThread_exit_thread(), +// without normal C++ unwinding), while on newer versions (3.14+) it may hang +// the thread until process exit. +// +// We still use this check because the policy in this layer is to avoid Python +// work once shutdown is underway and accept an intentional leak or skipped +// Python conversion in that edge case rather than add more complex deferral +// machinery. +inline bool py_is_finalizing() noexcept { +#if PY_VERSION_HEX >= 0x030D0000 + return Py_IsFinalizing(); +#else + return _Py_IsFinalizing() != 0; +#endif +} + +// Conditionally release the GIL while calling into the CUDA driver. +class GILReleaseGuard { +public: + GILReleaseGuard() noexcept { + if (!Py_IsInitialized() || py_is_finalizing()) { + return; + } + if (PyGILState_Check()) { + tstate_ = PyEval_SaveThread(); + } + } + + ~GILReleaseGuard() { + if (tstate_) { + PyEval_RestoreThread(tstate_); + } + } + + GILReleaseGuard(const GILReleaseGuard&) = delete; + GILReleaseGuard& operator=(const GILReleaseGuard&) = delete; + +private: + PyThreadState* tstate_ = nullptr; +}; + +// Helper to acquire the GIL when we might not hold it. +// Use in C++ destructors that need to manipulate Python objects. +class GILAcquireGuard { +public: + GILAcquireGuard() : acquired_(false) { + // Don't try to acquire GIL if Python is finalizing + if (!Py_IsInitialized() || py_is_finalizing()) { + return; + } + gstate_ = PyGILState_Ensure(); + acquired_ = true; + } + + ~GILAcquireGuard() { + if (acquired_) { + PyGILState_Release(gstate_); + } + } + + bool acquired() const { return acquired_; } + + // Non-copyable, non-movable + GILAcquireGuard(const GILAcquireGuard&) = delete; + GILAcquireGuard& operator=(const GILAcquireGuard&) = delete; + +private: + PyGILState_STATE gstate_; + bool acquired_; +}; + +// as_py() - convert handle to Python wrapper object (returns new reference) +namespace detail { +// n.b. class lookup is not cached to avoid deadlock hazard, see DESIGN.md +inline PyObject* make_py(const char* module_name, const char* class_name, std::intptr_t value) noexcept { + if (py_is_finalizing()) { + Py_RETURN_NONE; + } + PyObject* mod = PyImport_ImportModule(module_name); + if (!mod) return nullptr; + PyObject* cls = PyObject_GetAttrString(mod, class_name); + Py_DECREF(mod); + if (!cls) return nullptr; + PyObject* result = PyObject_CallFunction(cls, "L", value); + Py_DECREF(cls); + return result; +} +} // namespace detail + +inline PyObject* as_py(const ContextHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUcontext", as_intptr(h)); +} + +inline PyObject* as_py(const GreenCtxHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUgreenCtx", as_intptr(h)); +} + +inline PyObject* as_py(const StreamHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUstream", as_intptr(h)); +} + +inline PyObject* as_py(const EventHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUevent", as_intptr(h)); +} + +inline PyObject* as_py(const MemoryPoolHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUmemoryPool", as_intptr(h)); +} + +inline PyObject* as_py(const DevicePtrHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUdeviceptr", as_intptr(h)); +} + +inline PyObject* as_py(const LibraryHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUlibrary", as_intptr(h)); +} + +inline PyObject* as_py(const CUmodule& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUmodule", as_intptr(h)); +} + +inline PyObject* as_py(const KernelHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUkernel", as_intptr(h)); +} + +inline PyObject* as_py(const GraphHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUgraph", as_intptr(h)); +} + +inline PyObject* as_py(const GraphExecHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUgraphExec", as_intptr(h)); +} + +inline PyObject* as_py(const GraphNodeHandle& h) noexcept { + if (!as_intptr(h)) { + Py_RETURN_NONE; + } + return detail::make_py("cuda.bindings.driver", "CUgraphNode", as_intptr(h)); +} + +inline PyObject* as_py(const NvrtcProgramHandle& h) noexcept { + return detail::make_py("cuda.bindings.nvrtc", "nvrtcProgram", as_intptr(h)); +} + +inline PyObject* as_py(const NvvmProgramHandle& h) noexcept { + // NVVM bindings use raw integers, not wrapper classes + return PyLong_FromSsize_t(as_intptr(h)); +} + +inline PyObject* as_py(const NvJitLinkHandle& h) noexcept { + // nvJitLink bindings use raw integers, not wrapper classes + return PyLong_FromSsize_t(as_intptr(h)); +} + +inline PyObject* as_py(const CuLinkHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUlinkState", as_intptr(h)); +} + +inline PyObject* as_py(const GraphicsResourceHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUgraphicsResource", as_intptr(h)); +} + +inline PyObject* as_py(const FileDescriptorHandle& h) noexcept { + return PyLong_FromSsize_t(as_intptr(h)); +} + +inline PyObject* as_py(const OpaqueArrayHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUarray", as_intptr(h)); +} + +inline PyObject* as_py(const MipmappedArrayHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUmipmappedArray", as_intptr(h)); +} + +inline PyObject* as_py(const TexObjectHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUtexObject", as_intptr(h)); +} + +inline PyObject* as_py(const SurfObjectHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUsurfObject", as_intptr(h)); +} + +// ============================================================================ +// Python-coupled API: the prototypes that take or return PyObject* +// ============================================================================ + +// Register the Python warning category used by report_* (cuda.core.CUDAWarning). +// Implemented in py_report.cpp +void register_warning_category(PyObject* category) noexcept; + +// Create a non-owning stream handle that prevents a Python owner from being GC'd. +// The owner's refcount is incremented; decremented when handle is released. +// The owner is responsible for keeping the stream's context alive. +// Implemented in stream.cpp +StreamHandle create_stream_handle_with_owner(CUstream stream, PyObject* owner); + +// Create a non-owning device pointer handle that prevents a Python owner from being GC'd. +// The owner's refcount is incremented; decremented when handle is released. +// The pointer will NOT be freed when the handle is released. +// If owner is nullptr, equivalent to deviceptr_create_ref. +// Implemented in memory.cpp +DevicePtrHandle deviceptr_create_with_owner(CUdeviceptr ptr, PyObject* owner); + +// Callback type for MemoryResource deallocation. +// Called from the shared_ptr deleter when a handle created via +// deviceptr_create_with_mr is destroyed. The implementation is responsible +// for converting raw C types to Python objects and calling +// mr.deallocate(ptr, size, stream). +using MRDeallocCallback = void (*)(PyObject* mr, CUdeviceptr ptr, + size_t size, const StreamHandle& stream); + +// Register the MR deallocation callback. +// Implemented in memory.cpp +void register_mr_dealloc_callback(MRDeallocCallback cb); + +// Create a device pointer handle whose destructor calls mr.deallocate() +// via the registered callback. The mr's refcount is incremented and +// decremented when the handle is released. +// If mr is nullptr, equivalent to deviceptr_create_ref. +// Implemented in memory.cpp +DevicePtrHandle deviceptr_create_with_mr(CUdeviceptr ptr, size_t size, PyObject* mr); + +// Build an OpaqueHandle from a Python object: increments its refcount now and +// decrements it (under the GIL) on release. The caller must hold the GIL. +// Implemented in graph.cpp +OpaqueHandle make_opaque_py(PyObject* obj); + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/py_deferred_cleanup.cpp b/cuda_core/cuda/core/_cpp/rt/py_deferred_cleanup.cpp new file mode 100644 index 00000000000..126a90879b1 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/py_deferred_cleanup.cpp @@ -0,0 +1,157 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "internal.hpp" +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +// ============================================================================ +// CUDA user-object deferred cleanup +// +// CUDA invokes a user-object destructor on an internal thread where CUDA +// calls are forbidden. Payload cleanup can release resource handles whose +// deleters call CUDA, so the callback only transfers a preallocated intrusive +// node to this process-lifetime queue. One coalesced pending call drains all +// queued payloads from Python's main thread. +// ============================================================================ + +namespace { +// Process-lifetime MPSC queue that drains payloads from Python's main thread. +class DeferredCleanupQueue { +public: + // Transfer one preallocated cleanup item from a producer to the queue. + void enqueue(DeferredCleanupItem* item) noexcept { + DeferredCleanupItem* head = head_.load(std::memory_order_relaxed); + do { + item->next = head; + } while (!head_.compare_exchange_weak( + head, item, std::memory_order_release, std::memory_order_relaxed)); + schedule(); + } + + // Permanently disable pending-call scheduling during interpreter shutdown. + void stop() noexcept { + accepting_.store(false, std::memory_order_release); + } + + // Reattempt scheduling after Py_AddPendingCall() found CPython's bounded + // pending-call queue full and left payloads queued for a later safe entry. + void retry_schedule() noexcept { + schedule(); + } + +private: + // Adapt queue draining to CPython's int (*)(void*) callback ABI. + static int pending_call(void* arg) noexcept { + static_cast(arg)->drain(); + return 0; + } + + // Coalesce all queued work behind at most one CPython pending call. + void schedule() noexcept { + if (!accepting_.load(std::memory_order_acquire)) { + return; + } + if (!Py_IsInitialized() || py_is_finalizing()) { + stop(); + return; + } + if (!head_.load(std::memory_order_acquire)) { + return; + } + bool expected = false; + if (!scheduled_.compare_exchange_strong( + expected, true, std::memory_order_acq_rel, + std::memory_order_relaxed)) { + return; + } + if (Py_AddPendingCall(&DeferredCleanupQueue::pending_call, this) != 0) { + // Keep every payload queued. A later enqueue or safe cuda-core + // entry can retry without blocking CUDA's callback thread. + scheduled_.store(false, std::memory_order_release); + } + } + + // Detach and destroy all queued payloads from Python's main thread. + void drain() noexcept { + if (!Py_IsInitialized() || py_is_finalizing()) { + stop(); + scheduled_.store(false, std::memory_order_release); + return; // Intentionally leak intact payloads during shutdown. + } + + while (DeferredCleanupItem* list = + head_.exchange(nullptr, std::memory_order_acquire)) { + while (list) { + DeferredCleanupItem* next = list->next; + delete list; + list = next; + } + } + + scheduled_.store(false, std::memory_order_release); + if (head_.load(std::memory_order_acquire)) { + schedule(); + } + } + + // Head of the intrusive multi-producer, single-consumer payload stack. + std::atomic head_{nullptr}; + // True while one cuda-core drain callback is pending or executing. + std::atomic scheduled_{false}; + // False once shutdown begins, causing later payloads to be leaked safely. + std::atomic accepting_{true}; +}; + +// Published once at module initialization and intentionally never freed. +std::atomic deferred_cleanup_queue{nullptr}; +} // namespace + +namespace detail { +void ensure_deferred_cleanup_ready() { + DeferredCleanupQueue* queue = + deferred_cleanup_queue.load(std::memory_order_acquire); + if (!queue) { + throw std::runtime_error("deferred cleanup is not initialized"); + } + queue->retry_schedule(); +} + +// CUDA's CUhostFn ABI is void (*)(void*); recover and enqueue the cleanup item. +void enqueue_cleanup(void* item) noexcept { + auto* cleanup = static_cast(item); + if (DeferredCleanupQueue* queue = + deferred_cleanup_queue.load(std::memory_order_acquire)) { + queue->enqueue(cleanup); + } +} +} // namespace detail + +// Module initialization calls this once with the GIL held, which serializes +// the check, allocation, and publication below. +void initialize_deferred_cleanup() { + if (deferred_cleanup_queue.load(std::memory_order_acquire)) { + return; + } + auto* queue = new DeferredCleanupQueue(); + deferred_cleanup_queue.store(queue, std::memory_order_release); +} + +void retry_deferred_cleanup() noexcept { + if (!Py_IsInitialized() || py_is_finalizing()) { + return; + } + if (DeferredCleanupQueue* queue = + deferred_cleanup_queue.load(std::memory_order_acquire)) { + queue->retry_schedule(); + } +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/py_report.cpp b/cuda_core/cuda/core/_cpp/rt/py_report.cpp new file mode 100644 index 00000000000..a2e9badd029 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/py_report.cpp @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +namespace { +// Warning category registered by _utils/cuda_utils.pyx (cuda.core.CUDAWarning). +std::atomic warning_category{nullptr}; +} // namespace + +// Report a message that could not be raised. Emits cuda.core.CUDAWarning via +// the Python warnings machinery; if that itself fails (for example because the +// warning was promoted to an error), the failure is written as an unraisable +// exception, the CPython convention for exceptions in destructors. Falls back +// to stderr when the interpreter cannot be used. +void report_message(const char* message) noexcept { + PyObject* category = warning_category.load(std::memory_order_acquire); + if (category && Py_IsInitialized() && !py_is_finalizing()) { + GILAcquireGuard gil; + if (gil.acquired()) { + // Deleters can run while a Python exception is propagating; keep it. +#if PY_VERSION_HEX >= 0x030C0000 + PyObject* pending = PyErr_GetRaisedException(); +#else + PyObject *pending_type, *pending_value, *pending_tb; + PyErr_Fetch(&pending_type, &pending_value, &pending_tb); +#endif + if (PyErr_WarnEx(category, message, 1) != 0) { + PyObject* subject = PyUnicode_FromString(message); + PyErr_WriteUnraisable(subject); + Py_XDECREF(subject); + } +#if PY_VERSION_HEX >= 0x030C0000 + PyErr_SetRaisedException(pending); +#else + PyErr_Restore(pending_type, pending_value, pending_tb); +#endif + return; + } + } + std::fprintf(stderr, "%s\n", message); + std::fflush(stderr); +} + +void register_warning_category(PyObject* category) noexcept { + warning_category.store(category, std::memory_order_release); +} + +namespace { +// Attach `message` as a PEP 678 note to the exception currently being handled. +// Returns false when there is none or the interpreter cannot be used. +bool add_note_to_handled_exception(const char* message) noexcept { +#if PY_VERSION_HEX >= 0x030B0000 + if (!Py_IsInitialized() || py_is_finalizing()) { + return false; + } + GILAcquireGuard gil; + if (!gil.acquired()) { + return false; + } + PyObject* exc = PyErr_GetHandledException(); + if (!exc) { + return false; + } + PyObject* result = PyObject_CallMethod(exc, "add_note", "s", message); + Py_DECREF(exc); + if (!result) { + PyErr_Clear(); + return false; + } + Py_DECREF(result); + return true; +#else + (void)message; + return false; +#endif +} +} // namespace + +void note_or_report_cuda_error(const char* operation, CUresult status, const char* detail) noexcept { + if (status == CUDA_SUCCESS || status == CUDA_ERROR_DEINITIALIZED) { + return; + } + char message[512]; + format_cuda_error(message, sizeof(message), operation, status, detail); + if (!add_note_to_handled_exception(message)) { + report_message(message); + } +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/rt.hpp b/cuda_core/cuda/core/_cpp/rt/rt.hpp new file mode 100644 index 00000000000..7b843017225 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/rt.hpp @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +// Module umbrella, named only by _rt.pyx. + +#include "py.hpp" +#include "types.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "api.hpp" diff --git a/cuda_core/cuda/core/_cpp/rt/stream.cpp b/cuda_core/cuda/core/_cpp/rt/stream.cpp new file mode 100644 index 00000000000..eb17b428f11 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/stream.cpp @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "context_scope.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +namespace { +// Return whether a stream handle needs a current context to resolve it. +bool is_default_stream(CUstream stream) noexcept { + return stream == nullptr || stream == CU_STREAM_LEGACY || stream == CU_STREAM_PER_THREAD; +} +} // namespace + +namespace detail { +// Return the context a deallocation-stream token must run under. Real streams +// resolve their own context; default-stream tokens use the context bound at +// allocation time. Warn when PTDS deallocation crosses host threads. +ContextHandle deallocation_context(const DeallocationStream& stream) noexcept { + if (!is_default_stream(as_cu(stream.h_stream))) { + return {}; + } + if (stream.ptds_tid != std::thread::id{} + && stream.ptds_tid != std::this_thread::get_id()) { + report_message( + "Buffer deallocation for a per-thread default stream " + "is running on a different host thread than the one that recorded " + "the deallocation stream; ordering relative to the allocating " + "thread's PTDS is not preserved"); + } + return get_stream_context(stream.h_stream); +} +} // namespace detail + +// ============================================================================ +// Stream Handles +// ============================================================================ + +namespace { +struct StreamBox { + CUstream resource; + ContextHandle h_context; +}; + +static const StreamBox* get_box(const StreamHandle& h) noexcept { + const CUstream* p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) - offsetof(StreamBox, resource) + ); +} + +// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) +static HandleRegistry stream_registry; +} // namespace + +StreamHandle create_stream_handle(const ContextHandle& h_ctx, unsigned int flags, int priority) { + GILReleaseGuard gil; + CUstream stream = nullptr; + GreenCtxHandle h_green = get_context_green_ctx(h_ctx); + if (h_green) { + err = p_cuGreenCtxStreamCreate + ? p_cuGreenCtxStreamCreate(&stream, as_cu(h_green), flags, priority) + : CUDA_ERROR_NOT_SUPPORTED; + } else { + err = invoke_in_context_or_undo( + h_ctx, + [&]() noexcept { return p_cuStreamCreateWithPriority(&stream, flags, priority); }, + [&]() noexcept { pw_cuStreamDestroy(stream); }, + /*undo_requires_target_context=*/false); + } + if (err != CUDA_SUCCESS) { + return {}; + } + + auto box = std::shared_ptr( + new StreamBox{stream, h_ctx}, + [](const StreamBox* b) { + stream_registry.unregister_handle(b->resource); + GILReleaseGuard gil; + pw_cuStreamDestroy(b->resource); + delete b; + } + ); + StreamHandle h(box, &box->resource); + stream_registry.register_handle(stream, h); + return h; +} + +StreamHandle create_stream_handle_ref(CUstream stream) { + if (auto h = stream_registry.lookup(stream)) { + return h; + } + auto box = std::shared_ptr( + new StreamBox{stream, {}}, + [](const StreamBox* b) { + stream_registry.unregister_handle(b->resource); + delete b; + } + ); + StreamHandle h(box, &box->resource); + stream_registry.register_handle(stream, h); + return h; +} + +StreamHandle create_stream_handle_with_owner(CUstream stream, PyObject* owner) { + if (auto h = stream_registry.lookup(stream)) { + // Reuse handles that already carry structural context metadata, e.g. + // cuda-core-owned streams. + if (get_box(h)->h_context) { + return h; + } + } + if (!owner) { + return create_stream_handle_ref(stream); + } + // GIL required when owner is provided + GILAcquireGuard gil; + if (!gil.acquired()) { + // Python finalizing - fall back to ref version (no owner tracking) + return create_stream_handle_ref(stream); + } + Py_INCREF(owner); + // Owner-backed handles are NOT registered in the stream registry to avoid + // corruption when multiple owners wrap the same CUstream (each stacks its + // own Py_INCREF/Py_DECREF independently). + auto box = std::shared_ptr( + new StreamBox{stream, {}}, + [owner](const StreamBox* b) { + GILAcquireGuard gil; + if (gil.acquired()) { + Py_DECREF(owner); + } + delete b; + } + ); + return StreamHandle(box, &box->resource); +} + +// Return the context retained by a stream handle. +ContextHandle get_stream_context(const StreamHandle& h) noexcept { + return h ? get_box(h)->h_context : ContextHandle{}; +} + +StreamHandle get_legacy_stream() { + static StreamHandle handle = create_stream_handle_ref(CU_STREAM_LEGACY); + return handle; +} + +StreamHandle get_per_thread_stream() { + static StreamHandle handle = create_stream_handle_ref(CU_STREAM_PER_THREAD); + return handle; +} + +StreamHandle create_context_bound_legacy_stream(const ContextHandle& h_context) { + if (!h_context) { + return {}; + } + // Default deleter: this handle never owns CU_STREAM_LEGACY, so nothing + // needs to run when the last reference is released. + auto box = std::make_shared(StreamBox{CU_STREAM_LEGACY, h_context}); + return StreamHandle(box, &box->resource); +} + +// ============================================================================ +// Deallocation streams +// +// A DeallocationStream is a StreamHandle used for ordering frees. It differs +// from an ordinary StreamHandle only for default-stream tokens, for which it +// stores the (de)allocation context. Ordinarily, the LEGACY and PER_THREAD +// default streams resolve to whichever context is active at the time they are +// used, but for storing deallocation recipes we need to pin the context. With +// the PER_THREAD token, it is not possible to restore the original stream when +// deallocation runs on a different thread. Therefore, in that case the +// allocating host thread id is also stored so that cross-thread frees can be +// detected and warnings can be issued. +// ============================================================================ + +namespace detail { +// Real streams are copied unchanged. Default-stream tokens without an embedded +// context are bound to the current context. Returns false (and sets err) when a +// default-stream token cannot be bound because no context is current. +bool make_deallocation_stream( + const StreamHandle& h, DeallocationStream& out) noexcept { + out = {}; + if (!h) { + return true; + } + + const CUstream stream = as_cu(h); + if (!is_default_stream(stream)) { + out = DeallocationStream{h, {}}; + return true; + } + + StreamHandle h_bound = h; + if (!get_stream_context(h)) { + ContextHandle h_ctx = get_current_context(); + if (!h_ctx) { + if (err == CUDA_SUCCESS) { + err = CUDA_ERROR_INVALID_CONTEXT; + } + return false; + } + // Do not register in stream_registry: the token value alone is not + // a unique stream identity (context is part of the meaning). + auto box = std::shared_ptr( + new StreamBox{stream, h_ctx}); + h_bound = StreamHandle(box, &box->resource); + } + + std::thread::id ptds_tid{}; + if (stream == CU_STREAM_PER_THREAD) { + ptds_tid = std::this_thread::get_id(); + } + out = DeallocationStream{std::move(h_bound), ptds_tid}; + return true; +} +} // namespace detail + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/texture.cpp b/cuda_core/cuda/core/_cpp/rt/texture.cpp new file mode 100644 index 00000000000..11680a7e6bc --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/texture.cpp @@ -0,0 +1,265 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "context_scope.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +// ============================================================================ +// Graphics Resource Handles +// ============================================================================ + +namespace { +struct GraphicsResourceBox { + CUgraphicsResource resource; +}; +} // namespace + +GraphicsResourceHandle create_graphics_resource_handle(CUgraphicsResource resource) { + auto box = std::shared_ptr( + new GraphicsResourceBox{resource}, + [](const GraphicsResourceBox* b) { + GILReleaseGuard gil; + pw_cuGraphicsUnregisterResource(b->resource); + delete b; + } + ); + return GraphicsResourceHandle(box, &box->resource); +} + +// ============================================================================ +// Array / mipmapped-array / texture / surface handles (PR #467) +// ============================================================================ + +namespace { +struct ArrayBox { + CUarray resource; + // Non-null only for a mipmap-level view: keeps the parent mipmap (the real + // owner of the level's storage) alive for as long as the level is held. + MipmappedArrayHandle h_parent; + ContextHandle h_context; +}; + +struct MipmappedArrayBox { + CUmipmappedArray resource; + ContextHandle h_context; +}; + +// Texture and surface objects are per-context pool indices. Destroying one +// with the wrong context current can silently succeed without freeing it or +// can free an unrelated object, so destruction must enter the creating +// context. Handle-based resources resolve their own context and must not. +struct TexObjectBox { + // Tagged so TexObjectHandle is a distinct C++ type from DevicePtrHandle / + // SurfObjectHandle (all wrap `unsigned long long`). + TexObjectValue resource; + // Type-erased backing dependency (OpaqueArrayHandle / MipmappedArrayHandle / + // DevicePtrHandle). The texture's resource is a union; we only need to keep + // whichever backing it was built from alive, never to dereference it. + std::shared_ptr h_backing; + ContextHandle h_context; +}; + +struct SurfObjectBox { + SurfObjectValue resource; + OpaqueArrayHandle h_array; // surfaces are always array-backed + ContextHandle h_context; +}; + +// Recover an array's owning box from its aliased resource pointer. +const ArrayBox* get_box(const OpaqueArrayHandle& h) noexcept { + const CUarray* p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) - offsetof(ArrayBox, resource)); +} + +// Recover a mipmapped array's owning box from its aliased resource pointer. +const MipmappedArrayBox* get_box(const MipmappedArrayHandle& h) noexcept { + const CUmipmappedArray* p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) + - offsetof(MipmappedArrayBox, resource)); +} + +// Wrap an array with shared owning-destruction behavior. +static OpaqueArrayHandle wrap_array_owned(CUarray arr, ContextHandle h_context) { + auto box = std::shared_ptr( + new ArrayBox{arr, {}, std::move(h_context)}, + [](const ArrayBox* b) { + GILReleaseGuard gil; + pw_cuArrayDestroy(b->resource); + delete b; + } + ); + return OpaqueArrayHandle(box, &box->resource); +} + +} // namespace + +OpaqueArrayHandle create_array_handle(const ContextHandle& h_context, const CUDA_ARRAY3D_DESCRIPTOR& desc) { + GILReleaseGuard gil; + CUarray arr = nullptr; + err = invoke_in_context_or_undo( + h_context, + [&]() noexcept { return p_cuArray3DCreate(&arr, &desc); }, + [&]() noexcept { pw_cuArrayDestroy(arr); }, + /*undo_requires_target_context=*/false); + if (err != CUDA_SUCCESS) { + return {}; + } + return wrap_array_owned(arr, h_context); +} + +OpaqueArrayHandle create_array_handle_ref(CUarray arr) { + if (!arr) { + return {}; + } + auto box = std::make_shared(ArrayBox{arr, {}, {}}); + return OpaqueArrayHandle(box, &box->resource); +} + +OpaqueArrayHandle create_array_handle_owning(CUarray arr) { + if (!arr) { + return {}; + } + return wrap_array_owned(arr, {}); +} + +// Return the context retained by an array handle. +ContextHandle get_array_context(const OpaqueArrayHandle& h) noexcept { + return h ? get_box(h)->h_context : ContextHandle{}; +} + +OpaqueArrayHandle create_array_level_handle(const MipmappedArrayHandle& h_mip, unsigned int level) { + GILReleaseGuard gil; + CUarray arr; + ContextHandle h_context = h_mip ? get_box(h_mip)->h_context : ContextHandle{}; + if (CUDA_SUCCESS != (err = p_cuMipmappedArrayGetLevel(&arr, as_cu(h_mip), level))) { + return {}; + } + // Non-owning level view: storage belongs to the mipmap. Embed the mipmap + // handle so the parent outlives this level; the deleter does not destroy. + auto box = std::shared_ptr( + new ArrayBox{arr, h_mip, h_context}, + [](const ArrayBox* b) { delete b; } + ); + return OpaqueArrayHandle(box, &box->resource); +} + +MipmappedArrayHandle create_mipmapped_array_handle(const ContextHandle& h_context, + const CUDA_ARRAY3D_DESCRIPTOR& desc, + unsigned int num_levels) { + GILReleaseGuard gil; + CUmipmappedArray mip = nullptr; + err = invoke_in_context_or_undo( + h_context, + [&]() noexcept { return p_cuMipmappedArrayCreate(&mip, &desc, num_levels); }, + [&]() noexcept { pw_cuMipmappedArrayDestroy(mip); }, + /*undo_requires_target_context=*/false); + if (err != CUDA_SUCCESS) { + return {}; + } + auto box = std::shared_ptr( + new MipmappedArrayBox{mip, h_context}, + [](const MipmappedArrayBox* b) { + GILReleaseGuard gil; + pw_cuMipmappedArrayDestroy(b->resource); + delete b; + } + ); + return MipmappedArrayHandle(box, &box->resource); +} + +// Return the context retained by a mipmapped array handle. +ContextHandle get_mipmapped_array_context(const MipmappedArrayHandle& h) noexcept { + return h ? get_box(h)->h_context : ContextHandle{}; +} + +namespace { +TexObjectHandle make_tex_object_handle(const CUDA_RESOURCE_DESC& res, + const CUDA_TEXTURE_DESC& tex, + std::shared_ptr h_backing, + const ContextHandle& h_context) { + GILReleaseGuard gil; + CUtexObject obj = 0; + err = invoke_in_context_or_undo( + h_context, + [&]() noexcept { return p_cuTexObjectCreate(&obj, &res, &tex, nullptr); }, + [&]() noexcept { pw_cuTexObjectDestroy(obj); }, + /*undo_requires_target_context=*/true); + if (err != CUDA_SUCCESS) { + return {}; + } + auto box = std::shared_ptr( + new TexObjectBox{TexObjectValue{obj}, std::move(h_backing), h_context}, + [](const TexObjectBox* b) { + GILReleaseGuard gil; + cleanup_in_context(b->h_context, "cuTexObjectDestroy", [&]() noexcept { + return p_cuTexObjectDestroy(b->resource.raw); + }); + delete b; + } + ); + return TexObjectHandle(box, &box->resource); +} +} // namespace + +TexObjectHandle create_tex_object_handle_array(const ContextHandle& h_context, + const CUDA_RESOURCE_DESC& res, + const CUDA_TEXTURE_DESC& tex, + const OpaqueArrayHandle& h_backing) { + return make_tex_object_handle(res, tex, h_backing, h_context); +} + +TexObjectHandle create_tex_object_handle_mipmap(const ContextHandle& h_context, + const CUDA_RESOURCE_DESC& res, + const CUDA_TEXTURE_DESC& tex, + const MipmappedArrayHandle& h_backing) { + return make_tex_object_handle(res, tex, h_backing, h_context); +} + +TexObjectHandle create_tex_object_handle_linear(const ContextHandle& h_context, + const CUDA_RESOURCE_DESC& res, + const CUDA_TEXTURE_DESC& tex, + const DevicePtrHandle& h_backing) { + return make_tex_object_handle(res, tex, h_backing, h_context); +} + +SurfObjectHandle create_surf_object_handle(const ContextHandle& h_context, + const CUDA_RESOURCE_DESC& res, + const OpaqueArrayHandle& h_backing) { + GILReleaseGuard gil; + CUsurfObject obj = 0; + err = invoke_in_context_or_undo( + h_context, + [&]() noexcept { return p_cuSurfObjectCreate(&obj, &res); }, + [&]() noexcept { pw_cuSurfObjectDestroy(obj); }, + /*undo_requires_target_context=*/true); + if (err != CUDA_SUCCESS) { + return {}; + } + auto box = std::shared_ptr( + new SurfObjectBox{SurfObjectValue{obj}, h_backing, h_context}, + [](const SurfObjectBox* b) { + GILReleaseGuard gil; + cleanup_in_context(b->h_context, "cuSurfObjectDestroy", [&]() noexcept { + return p_cuSurfObjectDestroy(b->resource.raw); + }); + delete b; + } + ); + return SurfObjectHandle(box, &box->resource); +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/types.hpp b/cuda_core/cuda/core/_cpp/rt/types.hpp new file mode 100644 index 00000000000..59389f78fca --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/types.hpp @@ -0,0 +1,294 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include + +// Forward declaration for NVVM - avoids nvvm.h dependency +// Use void* to match cuda.bindings.cynvvm's typedef +using nvvmProgram = void*; + +// Forward declaration for nvJitLink - avoids nvJitLink.h dependency +// Use void* to match cuda.bindings.cynvjitlink's typedef +using nvJitLink_t = void*; + +namespace cuda_core::rt { + +// ============================================================================ +// TaggedHandle - make void*-based handle types distinct for overloading +// +// Both nvvmProgram and nvJitLink_t are void*, so shared_ptr +// would be the same C++ type for both. TaggedHandle wraps the raw +// value with a unique tag type, making each shared_ptr type distinct. +// ============================================================================ + +template +struct TaggedHandle { + T raw; +}; + +using NvvmProgramValue = TaggedHandle; +using NvJitLinkValue = TaggedHandle; + +// CUtexObject, CUsurfObject and CUdeviceptr are all `unsigned long long`, so +// shared_ptr et al. would be the *same* C++ type as +// DevicePtrHandle (and each other), collapsing the as_cu/as_intptr/as_py +// overload sets. Tag them to keep each handle type distinct, exactly as the +// NVVM / nvJitLink handles above do. +using TexObjectValue = TaggedHandle; +using SurfObjectValue = TaggedHandle; + +// ============================================================================ +// Handle type aliases - expose only the raw CUDA resource +// ============================================================================ + +using ContextHandle = std::shared_ptr; +using GreenCtxHandle = std::shared_ptr; +using StreamHandle = std::shared_ptr; +using EventHandle = std::shared_ptr; +using MemoryPoolHandle = std::shared_ptr; +using LibraryHandle = std::shared_ptr; +using KernelHandle = std::shared_ptr; +using GraphHandle = std::shared_ptr; +using GraphExecHandle = std::shared_ptr; +using GraphNodeHandle = std::shared_ptr; +using GraphicsResourceHandle = std::shared_ptr; +using NvrtcProgramHandle = std::shared_ptr; +using NvvmProgramHandle = std::shared_ptr; +using NvJitLinkHandle = std::shared_ptr; +using CuLinkHandle = std::shared_ptr; +using FileDescriptorHandle = std::shared_ptr; +using OpaqueArrayHandle = std::shared_ptr; +using MipmappedArrayHandle = std::shared_ptr; +using TexObjectHandle = std::shared_ptr; +using SurfObjectHandle = std::shared_ptr; + +using DevicePtrHandle = std::shared_ptr; + +// Type-erased shared owner of an attached resource. Typed handles such as +// EventHandle and KernelHandle convert to OpaqueHandle by assignment, reusing +// their existing control block; the helpers below build OpaqueHandles for the +// two cases that need a custom deleter. +using OpaqueHandle = std::shared_ptr; + +struct PreparedAttachmentState; +using PreparedAttachmentRollback = + void (*)(PreparedAttachmentState*) noexcept; +struct PreparedAttachmentDeleter { + PreparedAttachmentRollback rollback = nullptr; + + void operator()(PreparedAttachmentState* state) const noexcept { + rollback(state); + } +}; +using PreparedAttachment = + std::unique_ptr; + +struct PreparedChildGraphUpdateState; +// Opaque unpublished hierarchy transaction; releasing it discards staged +// metadata unless graph_commit_child_graph_update publishes the replacement. +using PreparedChildGraphUpdate = + std::shared_ptr; + +struct PreparedExecAttachmentState; +using PreparedExecAttachmentRollback = + void (*)(PreparedExecAttachmentState*) noexcept; +struct PreparedExecAttachmentDeleter { + PreparedExecAttachmentRollback rollback = nullptr; + + void operator()(PreparedExecAttachmentState* state) const noexcept { + rollback(state); + } +}; +// Opaque append transaction. Releasing it rolls back newly appended owners +// unless graph_commit_exec_attachment has kept them. +using PreparedExecAttachment = + std::unique_ptr; + +// ============================================================================ +// Overloaded helper functions to extract raw resources from handles +// ============================================================================ + +// as_cu() - extract the raw CUDA handle +inline CUcontext as_cu(const ContextHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUgreenCtx as_cu(const GreenCtxHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUstream as_cu(const StreamHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUevent as_cu(const EventHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUmemoryPool as_cu(const MemoryPoolHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUdeviceptr as_cu(const DevicePtrHandle& h) noexcept { + return h ? *h : 0; +} + +inline CUlibrary as_cu(const LibraryHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUmodule as_cu(const CUmodule& h) noexcept { + return h; +} + +inline CUkernel as_cu(const KernelHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUgraph as_cu(const GraphHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUgraphExec as_cu(const GraphExecHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUgraphNode as_cu(const GraphNodeHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUgraphicsResource as_cu(const GraphicsResourceHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline nvrtcProgram as_cu(const NvrtcProgramHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline nvvmProgram as_cu(const NvvmProgramHandle& h) noexcept { + return h ? h->raw : nullptr; +} + +inline nvJitLink_t as_cu(const NvJitLinkHandle& h) noexcept { + return h ? h->raw : nullptr; +} + +inline CUlinkState as_cu(const CuLinkHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUarray as_cu(const OpaqueArrayHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUmipmappedArray as_cu(const MipmappedArrayHandle& h) noexcept { + return h ? *h : nullptr; +} + +// CUtexObject / CUsurfObject are integer-valued (like CUdeviceptr); null is 0. +// The raw value lives in the tagged wrapper's `raw` field. +inline CUtexObject as_cu(const TexObjectHandle& h) noexcept { + return h ? h->raw : 0; +} + +inline CUsurfObject as_cu(const SurfObjectHandle& h) noexcept { + return h ? h->raw : 0; +} + +// as_intptr() - extract handle as intptr_t for Python interop +// Using signed intptr_t per C standard convention and issue #1342 +inline std::intptr_t as_intptr(const ContextHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const GreenCtxHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const StreamHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const EventHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const MemoryPoolHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const DevicePtrHandle& h) noexcept { + return static_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const LibraryHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const CUmodule& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const KernelHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const GraphHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const GraphExecHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const GraphNodeHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const GraphicsResourceHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const NvrtcProgramHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const NvvmProgramHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const NvJitLinkHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const CuLinkHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const FileDescriptorHandle& h) noexcept { + return h ? static_cast(*h) : -1; +} + +inline std::intptr_t as_intptr(const OpaqueArrayHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const MipmappedArrayHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const TexObjectHandle& h) noexcept { + return static_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const SurfObjectHandle& h) noexcept { + return static_cast(as_cu(h)); +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_device.pyx b/cuda_core/cuda/core/_device.pyx index a7e7d59e04a..40b09d6c761 100644 --- a/cuda_core/cuda/core/_device.pyx +++ b/cuda_core/cuda/core/_device.pyx @@ -18,7 +18,7 @@ from cuda.core._device_resources cimport DeviceResources, SMResource, WorkqueueR from cuda.core._event cimport Event as cyEvent from cuda.core._event import Event, EventOptions from cuda.core._memory._buffer cimport Buffer, MemoryResource -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( ContextHandle, GreenCtxHandle, create_context_handle_ref, @@ -1315,8 +1315,12 @@ class Device: HANDLE_RETURN(cydriver.cuCtxGetCurrent(&prev_ctx)) if prev_ctx != NULL: HANDLE_RETURN(cydriver.cuCtxGetDevice(&prev_dev)) - HANDLE_RETURN(cydriver.cuCtxPopCurrent(&prev_ctx)) - HANDLE_RETURN(cydriver.cuCtxPushCurrent(curr_ctx)) + # cuCtxSetCurrent replaces the top of the thread's context stack + # in one driver call (or binds ctx when nothing is current), so + # a failure leaves the previous context current instead of + # leaving the thread with no context, as a failed pop-then-push + # would. + HANDLE_RETURN(cydriver.cuCtxSetCurrent(curr_ctx)) self._has_inited = True self._context = ctx # Store owning context reference if prev_ctx != NULL: diff --git a/cuda_core/cuda/core/_device_resources.pxd b/cuda_core/cuda/core/_device_resources.pxd index d618c24cf10..212e3cac9b1 100644 --- a/cuda_core/cuda/core/_device_resources.pxd +++ b/cuda_core/cuda/core/_device_resources.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ContextHandle, GreenCtxHandle +from cuda.core._rt cimport ContextHandle, GreenCtxHandle cdef class SMResource: diff --git a/cuda_core/cuda/core/_device_resources.pyx b/cuda_core/cuda/core/_device_resources.pyx index 15ca6c56685..c9fe33c4fd7 100644 --- a/cuda_core/cuda/core/_device_resources.pyx +++ b/cuda_core/cuda/core/_device_resources.pyx @@ -17,7 +17,7 @@ from libc.stdlib cimport free, malloc from libc.string cimport memset from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ContextHandle, GreenCtxHandle, as_cu, get_context_green_ctx +from cuda.core._rt cimport ContextHandle, GreenCtxHandle, as_cu, get_context_green_ctx from cuda.core._utils.cuda_utils cimport check_or_create_options, HANDLE_RETURN from cuda.core._utils.cuda_utils import is_sequence from cuda.core._utils.version cimport cy_binding_version, cy_driver_version @@ -226,7 +226,7 @@ cdef inline unsigned int _to_sm_count(object value) except? 0: IF CUDA_CORE_BUILD_MAJOR >= 13: - from cuda.core._resource_handles cimport sm_resource_split, has_sm_resource_split + from cuda.core._rt cimport sm_resource_split, has_sm_resource_split cdef int _structured_split_checked = 0 diff --git a/cuda_core/cuda/core/_event.pxd b/cuda_core/cuda/core/_event.pxd index c1ab008d5e1..aa4726dc383 100644 --- a/cuda_core/cuda/core/_event.pxd +++ b/cuda_core/cuda/core/_event.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ContextHandle, EventHandle +from cuda.core._rt cimport ContextHandle, EventHandle cdef class Event: diff --git a/cuda_core/cuda/core/_event.pyx b/cuda_core/cuda/core/_event.pyx index dcbb55ba5a4..965c4b878eb 100644 --- a/cuda_core/cuda/core/_event.pyx +++ b/cuda_core/cuda/core/_event.pyx @@ -9,7 +9,7 @@ from libc.stddef cimport size_t from libc.string cimport memcpy from cuda.bindings cimport cydriver from cuda.core._context cimport Context -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( ContextHandle, EventHandle, create_event_handle, diff --git a/cuda_core/cuda/core/_graphics.pxd b/cuda_core/cuda/core/_graphics.pxd index 520a366bbde..d8d4838d111 100644 --- a/cuda_core/cuda/core/_graphics.pxd +++ b/cuda_core/cuda/core/_graphics.pxd @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -from cuda.core._resource_handles cimport GraphicsResourceHandle +from cuda.core._rt cimport GraphicsResourceHandle cdef class GraphicsResource: diff --git a/cuda_core/cuda/core/_graphics.pyx b/cuda_core/cuda/core/_graphics.pyx index b8945bb0d71..6e5ea061f50 100644 --- a/cuda_core/cuda/core/_graphics.pyx +++ b/cuda_core/cuda/core/_graphics.pyx @@ -7,7 +7,7 @@ from __future__ import annotations from typing import Sequence from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( create_graphics_resource_handle, deviceptr_create_mapped_graphics, as_cu, diff --git a/cuda_core/cuda/core/_launcher.pyx b/cuda_core/cuda/core/_launcher.pyx index 036189790e0..e5947b48be2 100644 --- a/cuda_core/cuda/core/_launcher.pyx +++ b/cuda_core/cuda/core/_launcher.pyx @@ -9,7 +9,7 @@ from cuda.bindings cimport cydriver from cuda.core._launch_config cimport LaunchConfig from cuda.core._kernel_arg_handler cimport ParamHolder from cuda.core._module cimport Kernel -from cuda.core._resource_handles cimport as_cu +from cuda.core._rt cimport as_cu from cuda.core._stream cimport Stream_accept, Stream from cuda.core._utils.cuda_utils cimport ( check_or_create_options, diff --git a/cuda_core/cuda/core/_linker.pxd b/cuda_core/cuda/core/_linker.pxd index 1b7d39fd1d4..63fa517fb8e 100644 --- a/cuda_core/cuda/core/_linker.pxd +++ b/cuda_core/cuda/core/_linker.pxd @@ -6,7 +6,7 @@ from libcpp.vector cimport vector from cuda.bindings cimport cydriver -from ._resource_handles cimport NvJitLinkHandle, CuLinkHandle +from ._rt cimport NvJitLinkHandle, CuLinkHandle cdef class Linker: diff --git a/cuda_core/cuda/core/_linker.pyx b/cuda_core/cuda/core/_linker.pyx index f104e2d158b..3e68d24afa3 100644 --- a/cuda_core/cuda/core/_linker.pyx +++ b/cuda_core/cuda/core/_linker.pyx @@ -16,7 +16,7 @@ from libcpp.vector cimport vector from cuda.bindings cimport cydriver from cuda.bindings cimport cynvjitlink -from ._resource_handles cimport ( +from ._rt cimport ( as_cu, as_py, create_culink_handle, diff --git a/cuda_core/cuda/core/_memory/_buffer.pxd b/cuda_core/cuda/core/_memory/_buffer.pxd index 75d94c91d58..e929210601f 100644 --- a/cuda_core/cuda/core/_memory/_buffer.pxd +++ b/cuda_core/cuda/core/_memory/_buffer.pxd @@ -5,7 +5,7 @@ from libcpp cimport bool as cpp_bool from libcpp.atomic cimport atomic as std_atomic -from cuda.core._resource_handles cimport DevicePtrHandle +from cuda.core._rt cimport DevicePtrHandle cdef struct _MemAttrs: diff --git a/cuda_core/cuda/core/_memory/_buffer.pyx b/cuda_core/cuda/core/_memory/_buffer.pyx index 52539b8c314..02a8d2078ee 100644 --- a/cuda_core/cuda/core/_memory/_buffer.pyx +++ b/cuda_core/cuda/core/_memory/_buffer.pyx @@ -13,7 +13,7 @@ from cuda.core._memory._device_memory_resource import DeviceMemoryResource from cuda.core._memory._pinned_memory_resource import PinnedMemoryResource from cuda.core._memory._ipc cimport IPCBufferDescriptor, IPCDataForBuffer from cuda.core._memory cimport _ipc -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( DevicePtrHandle, StreamHandle, ContextHandle, @@ -31,16 +31,17 @@ from cuda.core._memory._copy_attributes cimport _with_attributes_available from cuda.core._memory._copy_attributes cimport _to_cu_memcpy_attributes # no-cython-lint IF CUDA_CORE_BUILD_MAJOR >= 13: - from cuda.core._resource_handles cimport memcpy_with_attributes_async + from cuda.core._rt cimport memcpy_with_attributes_async from cuda.core._stream cimport Stream, Stream_accept, Stream_is_legacy_default_token, default_stream from cuda.core._utils.cuda_utils cimport HANDLE_RETURN, _parse_fill_value -import sys +import warnings from collections.abc import Sequence from typing import TYPE_CHECKING from cuda.core._memory._copy_enums import CopyOptions, _reject_unsupported_during_api_call +from cuda.core._utils.cuda_utils import CUDAWarning from cuda.core._utils.pycompat import BufferProtocol from cuda.core._dlpack import classify_dl_device, make_py_capsule from cuda.core._device import Device @@ -59,7 +60,11 @@ cdef void _mr_dealloc_callback( size_t size, const StreamHandle& h_stream, ) noexcept: - """Called by the C++ deleter to deallocate via MemoryResource.deallocate.""" + """Called by the C++ deleter to deallocate via MemoryResource.deallocate. + + Runs from a destructor, so nothing can be raised here; failures are reported + as :class:`~cuda.core.CUDAWarning` (see the error handling policy). + """ cdef Stream stream try: if not h_stream: @@ -71,8 +76,11 @@ cdef void _mr_dealloc_callback( stream = Stream._from_handle(Stream, h_stream) mr.deallocate(int(ptr), size, stream=stream) except Exception as exc: - print(f"Warning: mr.deallocate() failed during Buffer destruction: {exc}", - file=sys.stderr) + warnings.warn( + f"mr.deallocate() failed during Buffer destruction; the allocation may have leaked: {exc}", + CUDAWarning, + stacklevel=2, + ) register_mr_dealloc_callback(_mr_dealloc_callback) diff --git a/cuda_core/cuda/core/_memory/_copy_attributes.pxd b/cuda_core/cuda/core/_memory/_copy_attributes.pxd index 96c213dfec5..3e4ae2e778f 100644 --- a/cuda_core/cuda/core/_memory/_copy_attributes.pxd +++ b/cuda_core/cuda/core/_memory/_copy_attributes.pxd @@ -11,7 +11,7 @@ from cuda.core._utils.version cimport cy_binding_version, cy_driver_version # n IF CUDA_CORE_BUILD_MAJOR >= 13: - from cuda.core._resource_handles cimport has_memcpy_with_attributes_async + from cuda.core._rt cimport has_memcpy_with_attributes_async cdef inline bint _with_attributes_available(): # has_memcpy_with_attributes_async() says whether the installed diff --git a/cuda_core/cuda/core/_memory/_copy_ops.pyx b/cuda_core/cuda/core/_memory/_copy_ops.pyx index e57be2e40a0..be0ecafaed7 100644 --- a/cuda_core/cuda/core/_memory/_copy_ops.pyx +++ b/cuda_core/cuda/core/_memory/_copy_ops.pyx @@ -12,7 +12,7 @@ IF CUDA_CORE_BUILD_MAJOR >= 13: from cuda.bindings cimport cydriver from cuda.core._memory._buffer cimport Buffer, Buffer_coerce_batch from cuda.core._memory._copy_attributes cimport _to_cu_memcpy_attributes # no-cython-lint -from cuda.core._resource_handles cimport as_cu +from cuda.core._rt cimport as_cu from cuda.core._stream cimport Stream, Stream_accept, Stream_is_legacy_default_token from cuda.core._utils.cuda_utils cimport HANDLE_RETURN diff --git a/cuda_core/cuda/core/_memory/_device_memory_resource.pyx b/cuda_core/cuda/core/_memory/_device_memory_resource.pyx index 62dc4f9e747..8a0fed73eee 100644 --- a/cuda_core/cuda/core/_memory/_device_memory_resource.pyx +++ b/cuda_core/cuda/core/_memory/_device_memory_resource.pyx @@ -11,7 +11,7 @@ from cuda.core._memory._memory_pool cimport ( ) from cuda.core._memory cimport _ipc from cuda.core._memory._ipc cimport IPCAllocationHandle -from cuda.core._resource_handles cimport as_cu, get_device_mempool, get_last_error +from cuda.core._rt cimport as_cu, get_device_mempool, get_last_error from cuda.core._utils.cuda_utils cimport ( check_or_create_options, HANDLE_RETURN, diff --git a/cuda_core/cuda/core/_memory/_graph_memory_resource.pyx b/cuda_core/cuda/core/_memory/_graph_memory_resource.pyx index 67ecf97f58c..9d26798c630 100644 --- a/cuda_core/cuda/core/_memory/_graph_memory_resource.pyx +++ b/cuda_core/cuda/core/_memory/_graph_memory_resource.pyx @@ -8,7 +8,7 @@ from libc.stdint cimport intptr_t from cuda.bindings cimport cydriver from cuda.core._memory._buffer cimport Buffer, Buffer_from_deviceptr_handle, MemoryResource -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( DevicePtrHandle, deviceptr_alloc_async, get_last_error, diff --git a/cuda_core/cuda/core/_memory/_ipc.pxd b/cuda_core/cuda/core/_memory/_ipc.pxd index b912ace0035..5321d48fe13 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pxd +++ b/cuda_core/cuda/core/_memory/_ipc.pxd @@ -5,7 +5,7 @@ from cuda.bindings cimport cydriver from cuda.core._memory._buffer cimport Buffer from cuda.core._memory._memory_pool cimport _MemPool -from cuda.core._resource_handles cimport FileDescriptorHandle +from cuda.core._rt cimport FileDescriptorHandle # Holds _MemPool objects imported by this process. This enables diff --git a/cuda_core/cuda/core/_memory/_ipc.pyx b/cuda_core/cuda/core/_memory/_ipc.pyx index ae8db6589b4..7923c21fc00 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pyx +++ b/cuda_core/cuda/core/_memory/_ipc.pyx @@ -9,7 +9,7 @@ from cuda.bindings cimport cydriver from cuda.core._memory._buffer cimport Buffer, Buffer_check_open, Buffer_from_deviceptr_handle from cuda.core._memory._memory_pool cimport _MemPool, MP_check_open from cuda.core._stream cimport Stream, Stream_accept -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( DevicePtrHandle, create_fd_handle, create_mempool_handle_ipc, diff --git a/cuda_core/cuda/core/_memory/_managed_memory_ops.pyx b/cuda_core/cuda/core/_memory/_managed_memory_ops.pyx index e433ccf1e38..61caa59ce1f 100644 --- a/cuda_core/cuda/core/_memory/_managed_memory_ops.pyx +++ b/cuda_core/cuda/core/_memory/_managed_memory_ops.pyx @@ -18,7 +18,7 @@ from cuda.core._memory._buffer cimport Buffer, Buffer_check_open, Buffer_coerce_ # need a pragma to be seen as used. from cuda.core._memory._location cimport cumemlocation_from_id # no-cython-lint from cuda.core._memory._location cimport to_cumemlocation # no-cython-lint -from cuda.core._resource_handles cimport as_cu +from cuda.core._rt cimport as_cu from cuda.core._stream cimport Stream, Stream_accept from cuda.core._utils.cuda_utils cimport HANDLE_RETURN diff --git a/cuda_core/cuda/core/_memory/_memory_pool.pxd b/cuda_core/cuda/core/_memory/_memory_pool.pxd index 23e5eba588e..b85e5182e3b 100644 --- a/cuda_core/cuda/core/_memory/_memory_pool.pxd +++ b/cuda_core/cuda/core/_memory/_memory_pool.pxd @@ -5,7 +5,7 @@ from cuda.bindings cimport cydriver from cuda.core._memory._buffer cimport Buffer, MemoryResource from cuda.core._memory._ipc cimport IPCDataForMR -from cuda.core._resource_handles cimport MemoryPoolHandle +from cuda.core._rt cimport MemoryPoolHandle from cuda.core._stream cimport Stream diff --git a/cuda_core/cuda/core/_memory/_memory_pool.pyx b/cuda_core/cuda/core/_memory/_memory_pool.pyx index 6e5d26e7df8..c0d2b010c79 100644 --- a/cuda_core/cuda/core/_memory/_memory_pool.pyx +++ b/cuda_core/cuda/core/_memory/_memory_pool.pyx @@ -16,7 +16,7 @@ from cuda.core._memory cimport _ipc # does not evaluate compile-time IF blocks, so it needs a pragma to be seen as used. from cuda.core._memory._location cimport cumemlocation_from_id # no-cython-lint from cuda.core._stream cimport Stream_accept, Stream -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( MemoryPoolHandle, DevicePtrHandle, create_mempool_handle, @@ -25,7 +25,7 @@ from cuda.core._resource_handles cimport ( as_cu, as_py, ) -from cuda.core._resource_handles cimport create_mempool_handle_ref # no-cython-lint +from cuda.core._rt cimport create_mempool_handle_ref # no-cython-lint from cuda.core._utils.cuda_utils cimport ( HANDLE_RETURN, diff --git a/cuda_core/cuda/core/_memory/_peer_access_utils.pyx b/cuda_core/cuda/core/_memory/_peer_access_utils.pyx index 21f88258f57..be3426d8a69 100644 --- a/cuda_core/cuda/core/_memory/_peer_access_utils.pyx +++ b/cuda_core/cuda/core/_memory/_peer_access_utils.pyx @@ -14,7 +14,7 @@ from cuda.bindings cimport cydriver from cuda.core._memory._device_memory_resource cimport DeviceMemoryResource from cuda.core._memory._location cimport cumemlocation_from_id from cuda.core._memory._memory_pool cimport MP_check_open -from cuda.core._resource_handles cimport as_cu +from cuda.core._rt cimport as_cu from cuda.core._utils.cuda_utils cimport HANDLE_RETURN from cpython.mem cimport PyMem_Malloc, PyMem_Free from libcpp.vector cimport vector diff --git a/cuda_core/cuda/core/_memory/_synchronous_memory_resource.pyx b/cuda_core/cuda/core/_memory/_synchronous_memory_resource.pyx index f02f38f69b1..e5438797664 100644 --- a/cuda_core/cuda/core/_memory/_synchronous_memory_resource.pyx +++ b/cuda_core/cuda/core/_memory/_synchronous_memory_resource.pyx @@ -9,7 +9,7 @@ from libc.stdint cimport uintptr_t from cuda.bindings cimport cydriver from cuda.core._context cimport Context from cuda.core._memory._buffer cimport Buffer, MemoryResource -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( ContextHandle, create_context_bound_legacy_stream, deviceptr_alloc_raw, diff --git a/cuda_core/cuda/core/_memoryview.pyx b/cuda_core/cuda/core/_memoryview.pyx index 129cb91711b..5ce1cc1faa8 100644 --- a/cuda_core/cuda/core/_memoryview.pyx +++ b/cuda_core/cuda/core/_memoryview.pyx @@ -24,7 +24,7 @@ if TYPE_CHECKING: import numpy from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( EventHandle, create_event_handle_for_stream, as_cu, diff --git a/cuda_core/cuda/core/_module.pxd b/cuda_core/cuda/core/_module.pxd index 5e9d08fc13f..05a070a8afb 100644 --- a/cuda_core/cuda/core/_module.pxd +++ b/cuda_core/cuda/core/_module.pxd @@ -5,7 +5,7 @@ from libcpp.mutex cimport py_safe_once_flag from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport LibraryHandle, KernelHandle +from cuda.core._rt cimport LibraryHandle, KernelHandle cdef class ObjectCode cdef class Kernel diff --git a/cuda_core/cuda/core/_module.pyx b/cuda_core/cuda/core/_module.pyx index a350f14887f..fbaaef9e1e8 100644 --- a/cuda_core/cuda/core/_module.pyx +++ b/cuda_core/cuda/core/_module.pyx @@ -16,7 +16,7 @@ from cuda.core._launch_config cimport LaunchConfig from cuda.core._launch_config import LaunchConfig from cuda.core._stream cimport Stream, Stream_accept from cuda.core._program import ObjectCodeFormatType -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( LibraryHandle, KernelHandle, create_library_handle_from_file, diff --git a/cuda_core/cuda/core/_program.pxd b/cuda_core/cuda/core/_program.pxd index e1cbaa6fbda..88aedbb0b1c 100644 --- a/cuda_core/cuda/core/_program.pxd +++ b/cuda_core/cuda/core/_program.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from ._linker cimport Linker -from ._resource_handles cimport NvrtcProgramHandle, NvvmProgramHandle +from ._rt cimport NvrtcProgramHandle, NvvmProgramHandle cdef class Program: diff --git a/cuda_core/cuda/core/_program.pyx b/cuda_core/cuda/core/_program.pyx index 9ea624f8e83..bf6c16e19f7 100644 --- a/cuda_core/cuda/core/_program.pyx +++ b/cuda_core/cuda/core/_program.pyx @@ -26,7 +26,7 @@ from cuda.pathfinder._optional_cuda_import import _optional_cuda_import from libcpp.vector cimport vector -from ._resource_handles cimport ( +from ._rt cimport ( as_cu, as_py, create_nvrtc_program_handle, diff --git a/cuda_core/cuda/core/_resource_handles.pxd b/cuda_core/cuda/core/_rt.pxd similarity index 92% rename from cuda_core/cuda/core/_resource_handles.pxd rename to cuda_core/cuda/core/_rt.pxd index fe075b6414b..e0772a70444 100644 --- a/cuda_core/cuda/core/_resource_handles.pxd +++ b/cuda_core/cuda/core/_rt.pxd @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 +from cpython.object cimport PyObject from libc.stddef cimport size_t from libc.stdint cimport intptr_t @@ -17,7 +18,7 @@ from cuda.bindings cimport cynvjitlink # Handle type aliases and inline helpers (declared from C++ header) # ============================================================================= -cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": +cdef extern from "_cpp/rt/handles.hpp" namespace "cuda_core::rt": # Handle types ctypedef shared_ptr[const cydriver.CUcontext] ContextHandle ctypedef shared_ptr[const cydriver.CUgreenCtx] GreenCtxHandle @@ -35,9 +36,9 @@ cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": # NvvmProgramValue and NvJitLinkValue are TaggedHandle # instantiations that make each shared_ptr type distinct for overloading. - cppclass NvvmProgramValue "cuda_core::NvvmProgramValue": + cppclass NvvmProgramValue "cuda_core::rt::NvvmProgramValue": pass - cppclass NvJitLinkValue "cuda_core::NvJitLinkValue": + cppclass NvJitLinkValue "cuda_core::rt::NvJitLinkValue": pass ctypedef shared_ptr[const NvvmProgramValue] NvvmProgramHandle ctypedef shared_ptr[const NvJitLinkValue] NvJitLinkHandle @@ -50,9 +51,9 @@ cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": # CUtexObject / CUsurfObject are both `unsigned long long` (as is CUdeviceptr), # so they are wrapped in distinct tagged value types to keep each handle's # as_cu/as_intptr/as_py overloads distinct. - cppclass TexObjectValue "cuda_core::TexObjectValue": + cppclass TexObjectValue "cuda_core::rt::TexObjectValue": pass - cppclass SurfObjectValue "cuda_core::SurfObjectValue": + cppclass SurfObjectValue "cuda_core::rt::SurfObjectValue": pass ctypedef shared_ptr[const TexObjectValue] TexObjectHandle ctypedef shared_ptr[const SurfObjectValue] SurfObjectHandle @@ -158,9 +159,9 @@ cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": # ============================================================================= -# Wrapper function declarations (implemented in _resource_handles.pyx) +# Wrapper function declarations (implemented in _rt.pyx) # -# Consumer modules cimport these. Calls go through _resource_handles.so. +# Consumer modules cimport these. Calls go through _rt.so. # ============================================================================= # Thread-local error handling @@ -168,6 +169,18 @@ cdef cydriver.CUresult get_last_error() noexcept nogil cdef cydriver.CUresult peek_last_error() noexcept nogil cdef void clear_last_error() noexcept nogil +# Non-propagating error reporting (never raises; emits cuda.core.CUDAWarning +# when possible, else writes to stderr) +cdef void register_warning_category(PyObject* category) noexcept +cdef void report_cuda_error( + const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil +cdef void report_message(const char* message) noexcept nogil +cdef void report_status_code(const char* operation, long code) noexcept nogil +cdef void note_or_report_cuda_error( + const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil +cdef const char* take_last_error_detail(cydriver.CUresult status) noexcept nogil +cdef void clear_last_error_detail() noexcept nogil + # Context handles cdef ContextHandle create_context_handle_ref(cydriver.CUcontext ctx) except+ nogil cdef ContextHandle create_context_handle_from_green_ctx(const GreenCtxHandle& h_green_ctx) except+ nogil @@ -184,13 +197,17 @@ cdef cydriver.CUresult context_get_stream_priority_range( const ContextHandle& h_context, int* least_priority, int* greatest_priority) noexcept nogil +cdef cydriver.CUresult context_get_device( + const ContextHandle& h_context, cydriver.CUdevice* device) noexcept nogil +cdef cydriver.CUresult graph_node_set_params( + cydriver.CUgraphNode node, cydriver.CUgraphNodeParams* params, + const ContextHandle& h_context, cydriver.CUresult* restore_status) noexcept nogil # Stream handles cdef StreamHandle create_stream_handle( const ContextHandle& h_ctx, unsigned int flags, int priority) except+ nogil cdef StreamHandle create_stream_handle_ref(cydriver.CUstream stream) except+ nogil cdef StreamHandle create_stream_handle_with_owner(cydriver.CUstream stream, object owner) except+ nogil -cdef void py_object_user_object_destroy(void* py_object) noexcept nogil cdef void retry_deferred_cleanup() noexcept cdef ContextHandle get_stream_context(const StreamHandle& h) noexcept nogil cdef StreamHandle get_legacy_stream() except+ nogil diff --git a/cuda_core/cuda/core/_resource_handles.pyi b/cuda_core/cuda/core/_rt.pyi similarity index 70% rename from cuda_core/cuda/core/_resource_handles.pyi rename to cuda_core/cuda/core/_rt.pyi index 66cbf80761a..d25c3ad5b56 100644 --- a/cuda_core/cuda/core/_resource_handles.pyi +++ b/cuda_core/cuda/core/_rt.pyi @@ -1,4 +1,4 @@ -# This file was generated by stubgen-pyx v0.2.22 from cuda_core/cuda/core/_resource_handles.pyx +# This file was generated by stubgen-pyx v0.2.22 from cuda_core/cuda/core/_rt.pyx from typing import Callable @@ -41,3 +41,18 @@ PreparedAttachmentDeleter: TypeAlias = Incomplete PreparedChildGraphUpdateState: TypeAlias = Incomplete PreparedExecAttachmentState: TypeAlias = Incomplete PreparedExecAttachmentDeleter: TypeAlias = Incomplete + +def _set_context_restore_fault_for_testing(status: int): + """Make the next context restoration on this thread fail with ``status``. + + Test hook for the context save/restore paths in the handle layer. The + injected failure leaves the target context current, exactly as a failing + ``cuCtxSetCurrent`` would, so callers must restore the context themselves. + """ +def _note_or_report_cuda_error_for_testing(status: int): + """Attach a failed CUDA call to the exception being handled, or report it. + + Test hook for ``note_or_report_cuda_error()``. Called inside an ``except`` + block it adds a note to the exception being handled (Python 3.11+); anywhere + else it emits a ``CUDAWarning``. + """ diff --git a/cuda_core/cuda/core/_resource_handles.pyx b/cuda_core/cuda/core/_rt.pyx similarity index 61% rename from cuda_core/cuda/core/_resource_handles.pyx rename to cuda_core/cuda/core/_rt.pyx index 0f8d6e15cde..be1c493439c 100644 --- a/cuda_core/cuda/core/_resource_handles.pyx +++ b/cuda_core/cuda/core/_rt.pyx @@ -2,14 +2,19 @@ # # SPDX-License-Identifier: Apache-2.0 -# This module compiles _cpp/resource_handles.cpp into a shared library. -# Consumer modules cimport the functions declared in _resource_handles.pxd. -# Since there is only one copy of the C++ code (in this .so), all static and -# thread-local state is shared correctly across all consumer modules. +# This module compiles the C++ under _cpp/rt/ into one shared library. +# Consumer modules cimport the functions declared in _rt.pxd. Since there is +# only one copy of the C++ code (in this .so), all static and thread-local +# state is shared correctly across all consumer modules. +# +# "rt" is short for runtime: this is cuda.core's runtime support layer +# (resource handles, the driver function-pointer table, error reporting and +# deferred cleanup). It is unrelated to the CUDA Runtime API (cudart). # # The cdef extern from declarations below satisfy the .pxd declarations directly, # without needing separate wrapper functions. +from cpython.object cimport PyObject from cpython.pycapsule cimport PyCapsule_GetName, PyCapsule_GetPointer from libc.stddef cimport size_t @@ -24,281 +29,295 @@ import cuda.bindings.cynvvm as cynvvm import cuda.bindings.cynvjitlink as cynvjitlink # ============================================================================= -# C++ function declarations (non-inline, implemented in resource_handles.cpp) +# C++ function declarations (non-inline, implemented under _cpp/rt/) # -# These declarations satisfy the cdef function declarations in _resource_handles.pxd. +# These declarations satisfy the cdef function declarations in _rt.pxd. # Consumer modules cimport these functions and calls go through this .so. # ============================================================================= -cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": +cdef extern from "_cpp/rt/rt.hpp" namespace "cuda_core::rt": # Thread-local error handling - cydriver.CUresult get_last_error "cuda_core::get_last_error" () noexcept nogil - cydriver.CUresult peek_last_error "cuda_core::peek_last_error" () noexcept nogil - void clear_last_error "cuda_core::clear_last_error" () noexcept nogil + cydriver.CUresult get_last_error "cuda_core::rt::get_last_error" () noexcept nogil + cydriver.CUresult peek_last_error "cuda_core::rt::peek_last_error" () noexcept nogil + void clear_last_error "cuda_core::rt::clear_last_error" () noexcept nogil + + # Non-propagating error reporting + void register_warning_category "cuda_core::rt::register_warning_category" ( + PyObject* category) noexcept + void report_cuda_error "cuda_core::rt::report_cuda_error" ( + const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil + void report_message "cuda_core::rt::report_message" (const char* message) noexcept nogil + void report_status_code "cuda_core::rt::report_status_code" ( + const char* operation, long code) noexcept nogil + void note_or_report_cuda_error "cuda_core::rt::note_or_report_cuda_error" ( + const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil + # Alias for calls made from this module: calling the pxd-declared name here + # would make Cython emit a conflicting static prototype for it. + void _note_or_report_cuda_error_local "cuda_core::rt::note_or_report_cuda_error" ( + const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil + const char* take_last_error_detail "cuda_core::rt::take_last_error_detail" ( + cydriver.CUresult status) noexcept nogil + void clear_last_error_detail "cuda_core::rt::clear_last_error_detail" () noexcept nogil + void set_context_restore_fault_for_testing "cuda_core::rt::set_context_restore_fault_for_testing" ( + cydriver.CUresult status) noexcept nogil # Context handles - ContextHandle create_context_handle_ref "cuda_core::create_context_handle_ref" ( + ContextHandle create_context_handle_ref "cuda_core::rt::create_context_handle_ref" ( cydriver.CUcontext ctx) except+ nogil - ContextHandle create_context_handle_from_green_ctx "cuda_core::create_context_handle_from_green_ctx" ( + ContextHandle create_context_handle_from_green_ctx "cuda_core::rt::create_context_handle_from_green_ctx" ( const GreenCtxHandle& h_green_ctx) except+ nogil - GreenCtxHandle get_context_green_ctx "cuda_core::get_context_green_ctx" ( + GreenCtxHandle get_context_green_ctx "cuda_core::rt::get_context_green_ctx" ( const ContextHandle& h) noexcept nogil - GreenCtxHandle create_green_ctx_handle "cuda_core::create_green_ctx_handle" ( + GreenCtxHandle create_green_ctx_handle "cuda_core::rt::create_green_ctx_handle" ( cydriver.CUdevResource* resources, unsigned int nbResources, cydriver.CUdevice dev, unsigned int flags) except+ nogil - GreenCtxHandle create_green_ctx_handle_ref "cuda_core::create_green_ctx_handle_ref" ( + GreenCtxHandle create_green_ctx_handle_ref "cuda_core::rt::create_green_ctx_handle_ref" ( cydriver.CUgreenCtx ctx) except+ nogil - ContextHandle get_primary_context "cuda_core::get_primary_context" ( + ContextHandle get_primary_context "cuda_core::rt::get_primary_context" ( int device_id) except+ nogil - ContextHandle get_current_context "cuda_core::get_current_context" () except+ nogil - cydriver.CUresult context_synchronize "cuda_core::context_synchronize" ( + ContextHandle get_current_context "cuda_core::rt::get_current_context" () except+ nogil + cydriver.CUresult context_synchronize "cuda_core::rt::context_synchronize" ( const ContextHandle& h_context) noexcept nogil - cydriver.CUresult context_get_stream_priority_range "cuda_core::context_get_stream_priority_range" ( + cydriver.CUresult context_get_stream_priority_range "cuda_core::rt::context_get_stream_priority_range" ( const ContextHandle& h_context, int* least_priority, int* greatest_priority) noexcept nogil + cydriver.CUresult context_get_device "cuda_core::rt::context_get_device" ( + const ContextHandle& h_context, cydriver.CUdevice* device) noexcept nogil + cydriver.CUresult graph_node_set_params "cuda_core::rt::graph_node_set_params" ( + cydriver.CUgraphNode node, cydriver.CUgraphNodeParams* params, + const ContextHandle& h_context, cydriver.CUresult* restore_status) noexcept nogil # Stream handles - StreamHandle create_stream_handle "cuda_core::create_stream_handle" ( + StreamHandle create_stream_handle "cuda_core::rt::create_stream_handle" ( const ContextHandle& h_ctx, unsigned int flags, int priority) except+ nogil - StreamHandle create_stream_handle_ref "cuda_core::create_stream_handle_ref" ( + StreamHandle create_stream_handle_ref "cuda_core::rt::create_stream_handle_ref" ( cydriver.CUstream stream) except+ nogil - StreamHandle create_stream_handle_with_owner "cuda_core::create_stream_handle_with_owner" ( + StreamHandle create_stream_handle_with_owner "cuda_core::rt::create_stream_handle_with_owner" ( cydriver.CUstream stream, object owner) except+ nogil - void py_object_user_object_destroy "cuda_core::py_object_user_object_destroy" ( - void* py_object) noexcept nogil - void initialize_deferred_cleanup "cuda_core::initialize_deferred_cleanup" () except+ - void retry_deferred_cleanup "cuda_core::retry_deferred_cleanup" () noexcept - ContextHandle get_stream_context "cuda_core::get_stream_context" ( + void initialize_deferred_cleanup "cuda_core::rt::initialize_deferred_cleanup" () except+ + void retry_deferred_cleanup "cuda_core::rt::retry_deferred_cleanup" () noexcept + ContextHandle get_stream_context "cuda_core::rt::get_stream_context" ( const StreamHandle& h) noexcept nogil - StreamHandle get_legacy_stream "cuda_core::get_legacy_stream" () except+ nogil - StreamHandle get_per_thread_stream "cuda_core::get_per_thread_stream" () except+ nogil - StreamHandle create_context_bound_legacy_stream "cuda_core::create_context_bound_legacy_stream" ( + StreamHandle get_legacy_stream "cuda_core::rt::get_legacy_stream" () except+ nogil + StreamHandle get_per_thread_stream "cuda_core::rt::get_per_thread_stream" () except+ nogil + StreamHandle create_context_bound_legacy_stream "cuda_core::rt::create_context_bound_legacy_stream" ( const ContextHandle& h_context) except+ nogil # Event handles (note: _create_event_handle* are internal due to C++ overloading) - EventHandle create_event_handle "cuda_core::create_event_handle" ( + EventHandle create_event_handle "cuda_core::rt::create_event_handle" ( const ContextHandle& h_ctx, unsigned int flags, bint timing_enabled, bint is_blocking_sync, bint ipc_enabled, int device_id) except+ nogil - EventHandle create_event_handle_for_stream "cuda_core::create_event_handle_for_stream" ( + EventHandle create_event_handle_for_stream "cuda_core::rt::create_event_handle_for_stream" ( cydriver.CUstream stream, unsigned int flags) except+ nogil - EventHandle create_event_handle_ref "cuda_core::create_event_handle_ref" ( + EventHandle create_event_handle_ref "cuda_core::rt::create_event_handle_ref" ( cydriver.CUevent event) except+ nogil - EventHandle create_event_handle_ipc "cuda_core::create_event_handle_ipc" ( + EventHandle create_event_handle_ipc "cuda_core::rt::create_event_handle_ipc" ( const cydriver.CUipcEventHandle& ipc_handle, bint is_blocking_sync) except+ nogil # Event metadata getters - bint get_event_timing_enabled "cuda_core::get_event_timing_enabled" ( + bint get_event_timing_enabled "cuda_core::rt::get_event_timing_enabled" ( const EventHandle& h) noexcept nogil - bint get_event_is_blocking_sync "cuda_core::get_event_is_blocking_sync" ( + bint get_event_is_blocking_sync "cuda_core::rt::get_event_is_blocking_sync" ( const EventHandle& h) noexcept nogil - bint get_event_ipc_enabled "cuda_core::get_event_ipc_enabled" ( + bint get_event_ipc_enabled "cuda_core::rt::get_event_ipc_enabled" ( const EventHandle& h) noexcept nogil - int get_event_device_id "cuda_core::get_event_device_id" ( + int get_event_device_id "cuda_core::rt::get_event_device_id" ( const EventHandle& h) noexcept nogil - ContextHandle get_event_context "cuda_core::get_event_context" ( + ContextHandle get_event_context "cuda_core::rt::get_event_context" ( const EventHandle& h) noexcept nogil # Memory pool handles - MemoryPoolHandle create_mempool_handle "cuda_core::create_mempool_handle" ( + MemoryPoolHandle create_mempool_handle "cuda_core::rt::create_mempool_handle" ( const cydriver.CUmemPoolProps& props) except+ nogil - MemoryPoolHandle create_mempool_handle_ref "cuda_core::create_mempool_handle_ref" ( + MemoryPoolHandle create_mempool_handle_ref "cuda_core::rt::create_mempool_handle_ref" ( cydriver.CUmemoryPool pool) except+ nogil - MemoryPoolHandle get_device_mempool "cuda_core::get_device_mempool" ( + MemoryPoolHandle get_device_mempool "cuda_core::rt::get_device_mempool" ( int device_id) except+ nogil - MemoryPoolHandle create_mempool_handle_ipc "cuda_core::create_mempool_handle_ipc" ( + MemoryPoolHandle create_mempool_handle_ipc "cuda_core::rt::create_mempool_handle_ipc" ( int fd, cydriver.CUmemAllocationHandleType handle_type) except+ nogil # Device pointer handles - DevicePtrHandle deviceptr_alloc_from_pool "cuda_core::deviceptr_alloc_from_pool" ( + DevicePtrHandle deviceptr_alloc_from_pool "cuda_core::rt::deviceptr_alloc_from_pool" ( size_t size, const MemoryPoolHandle& h_pool, const StreamHandle& h_stream) except+ nogil - DevicePtrHandle deviceptr_alloc_async "cuda_core::deviceptr_alloc_async" ( + DevicePtrHandle deviceptr_alloc_async "cuda_core::rt::deviceptr_alloc_async" ( size_t size, const StreamHandle& h_stream) except+ nogil - cydriver.CUresult deviceptr_alloc_raw "cuda_core::deviceptr_alloc_raw" ( + cydriver.CUresult deviceptr_alloc_raw "cuda_core::rt::deviceptr_alloc_raw" ( cydriver.CUdeviceptr* ptr, size_t size, const ContextHandle& h_context) noexcept nogil - DevicePtrHandle deviceptr_alloc_host "cuda_core::deviceptr_alloc_host" (size_t size) except+ nogil - DevicePtrHandle deviceptr_create_ref "cuda_core::deviceptr_create_ref" ( + DevicePtrHandle deviceptr_alloc_host "cuda_core::rt::deviceptr_alloc_host" (size_t size) except+ nogil + DevicePtrHandle deviceptr_create_ref "cuda_core::rt::deviceptr_create_ref" ( cydriver.CUdeviceptr ptr) except+ nogil - DevicePtrHandle deviceptr_create_with_owner "cuda_core::deviceptr_create_with_owner" ( + DevicePtrHandle deviceptr_create_with_owner "cuda_core::rt::deviceptr_create_with_owner" ( cydriver.CUdeviceptr ptr, object owner) except+ nogil - DevicePtrHandle deviceptr_create_mapped_graphics "cuda_core::deviceptr_create_mapped_graphics" ( + DevicePtrHandle deviceptr_create_mapped_graphics "cuda_core::rt::deviceptr_create_mapped_graphics" ( cydriver.CUdeviceptr ptr, const GraphicsResourceHandle& h_resource, const StreamHandle& h_stream) except+ nogil # MR deallocation callback - void register_mr_dealloc_callback "cuda_core::register_mr_dealloc_callback" ( + void register_mr_dealloc_callback "cuda_core::rt::register_mr_dealloc_callback" ( MRDeallocCallback cb) noexcept - DevicePtrHandle deviceptr_create_with_mr "cuda_core::deviceptr_create_with_mr" ( + DevicePtrHandle deviceptr_create_with_mr "cuda_core::rt::deviceptr_create_with_mr" ( cydriver.CUdeviceptr ptr, size_t size, object mr) except+ nogil - DevicePtrHandle deviceptr_import_ipc "cuda_core::deviceptr_import_ipc" ( + DevicePtrHandle deviceptr_import_ipc "cuda_core::rt::deviceptr_import_ipc" ( const MemoryPoolHandle& h_pool, const void* export_data, const StreamHandle& h_stream) except+ nogil - StreamHandle deallocation_stream "cuda_core::deallocation_stream" ( + StreamHandle deallocation_stream "cuda_core::rt::deallocation_stream" ( const DevicePtrHandle& h) noexcept nogil - cydriver.CUresult set_deallocation_stream "cuda_core::set_deallocation_stream" ( + cydriver.CUresult set_deallocation_stream "cuda_core::rt::set_deallocation_stream" ( const DevicePtrHandle& h, const StreamHandle& h_stream) noexcept nogil # Library handles - LibraryHandle create_library_handle_from_file "cuda_core::create_library_handle_from_file" ( + LibraryHandle create_library_handle_from_file "cuda_core::rt::create_library_handle_from_file" ( const char* path) except+ nogil - LibraryHandle create_library_handle_from_data "cuda_core::create_library_handle_from_data" ( + LibraryHandle create_library_handle_from_data "cuda_core::rt::create_library_handle_from_data" ( const void* data) except+ nogil - LibraryHandle create_library_handle_ref "cuda_core::create_library_handle_ref" ( + LibraryHandle create_library_handle_ref "cuda_core::rt::create_library_handle_ref" ( cydriver.CUlibrary library) except+ nogil # Kernel handles - KernelHandle create_kernel_handle "cuda_core::create_kernel_handle" ( + KernelHandle create_kernel_handle "cuda_core::rt::create_kernel_handle" ( const LibraryHandle& h_library, const char* name) except+ nogil - KernelHandle create_kernel_handle_ref "cuda_core::create_kernel_handle_ref" ( + KernelHandle create_kernel_handle_ref "cuda_core::rt::create_kernel_handle_ref" ( cydriver.CUkernel kernel) except+ nogil - LibraryHandle get_kernel_library "cuda_core::get_kernel_library" ( + LibraryHandle get_kernel_library "cuda_core::rt::get_kernel_library" ( const KernelHandle& h) noexcept nogil # Graph handles - GraphHandle create_graph_handle "cuda_core::create_graph_handle" ( + GraphHandle create_graph_handle "cuda_core::rt::create_graph_handle" ( cydriver.CUgraph graph) except+ nogil - GraphHandle create_child_graph_handle "cuda_core::create_child_graph_handle" ( + GraphHandle create_child_graph_handle "cuda_core::rt::create_child_graph_handle" ( cydriver.CUgraph child_graph, const GraphHandle& h_parent, cydriver.CUgraphNode owner_node) except+ nogil # Graph node attachments - OpaqueHandle make_opaque_py "cuda_core::make_opaque_py" (object obj) except+ - OpaqueHandle make_opaque_malloc "cuda_core::make_opaque_malloc" (void* buf) except+ - cydriver.CUresult graph_get_attachment "cuda_core::graph_get_attachment" ( + OpaqueHandle make_opaque_py "cuda_core::rt::make_opaque_py" (object obj) except+ + OpaqueHandle make_opaque_malloc "cuda_core::rt::make_opaque_malloc" (void* buf) except+ + cydriver.CUresult graph_get_attachment "cuda_core::rt::graph_get_attachment" ( const GraphHandle& h_graph, cydriver.CUgraphNode node, OpaqueHandle* owner0, OpaqueHandle* owner1) except+ - cydriver.CUresult graph_prepare_attachment "cuda_core::graph_prepare_attachment" ( + cydriver.CUresult graph_prepare_attachment "cuda_core::rt::graph_prepare_attachment" ( const GraphHandle& h_graph, OpaqueHandle owner0, OpaqueHandle owner1, PreparedAttachment* out_prepared) except+ - cydriver.CUresult graph_commit_attachment "cuda_core::graph_commit_attachment" ( + cydriver.CUresult graph_commit_attachment "cuda_core::rt::graph_commit_attachment" ( PreparedAttachment& prepared, cydriver.CUgraphNode node) except+ - cydriver.CUresult graph_clone_attachments "cuda_core::graph_clone_attachments" ( + cydriver.CUresult graph_clone_attachments "cuda_core::rt::graph_clone_attachments" ( const GraphHandle& h_clone, const GraphHandle& h_source) except+ - cydriver.CUresult graph_prepare_child_graph_update "cuda_core::graph_prepare_child_graph_update" ( + cydriver.CUresult graph_prepare_child_graph_update "cuda_core::rt::graph_prepare_child_graph_update" ( const GraphHandle& h_parent, const GraphHandle& h_old_child, cydriver.CUgraphNode owner_node, const GraphHandle& h_source, PreparedChildGraphUpdate* out_prepared) except+ - cydriver.CUresult graph_commit_child_graph_update "cuda_core::graph_commit_child_graph_update" ( + cydriver.CUresult graph_commit_child_graph_update "cuda_core::rt::graph_commit_child_graph_update" ( PreparedChildGraphUpdate& prepared, GraphHandle* out_child) except+ - void invalidate_child_graph_state "cuda_core::invalidate_child_graph_state" ( + void invalidate_child_graph_state "cuda_core::rt::invalidate_child_graph_state" ( const GraphHandle& h_parent, cydriver.CUgraphNode owner_node) noexcept # Graph exec handles - GraphExecHandle create_graph_exec_handle "cuda_core::create_graph_exec_handle" ( + GraphExecHandle create_graph_exec_handle "cuda_core::rt::create_graph_exec_handle" ( const GraphHandle& h_source, cydriver.CUDA_GRAPH_INSTANTIATE_PARAMS* params) except+ - cydriver.CUresult graph_exec_update "cuda_core::graph_exec_update" ( + cydriver.CUresult graph_exec_update "cuda_core::rt::graph_exec_update" ( const GraphExecHandle& h_exec, const GraphHandle& h_source, cydriver.CUgraphExecUpdateResultInfo* result_info) except+ - cydriver.CUresult graph_prepare_exec_attachment "cuda_core::graph_prepare_exec_attachment" ( + cydriver.CUresult graph_prepare_exec_attachment "cuda_core::rt::graph_prepare_exec_attachment" ( const GraphExecHandle& h_exec, OpaqueHandle owner0, OpaqueHandle owner1, PreparedExecAttachment* out_prepared) except+ - void graph_commit_exec_attachment "cuda_core::graph_commit_exec_attachment" ( + void graph_commit_exec_attachment "cuda_core::rt::graph_commit_exec_attachment" ( PreparedExecAttachment& prepared) noexcept # Graph node handles - GraphNodeHandle create_graph_node_handle "cuda_core::create_graph_node_handle" ( + GraphNodeHandle create_graph_node_handle "cuda_core::rt::create_graph_node_handle" ( cydriver.CUgraphNode node, const GraphHandle& h_graph) except+ nogil - GraphHandle graph_node_get_graph "cuda_core::graph_node_get_graph" ( + GraphHandle graph_node_get_graph "cuda_core::rt::graph_node_get_graph" ( const GraphNodeHandle& h) noexcept nogil - void invalidate_graph_node "cuda_core::invalidate_graph_node" ( + void invalidate_graph_node "cuda_core::rt::invalidate_graph_node" ( const GraphNodeHandle& h) noexcept nogil # Graphics resource handles - GraphicsResourceHandle create_graphics_resource_handle "cuda_core::create_graphics_resource_handle" ( + GraphicsResourceHandle create_graphics_resource_handle "cuda_core::rt::create_graphics_resource_handle" ( cydriver.CUgraphicsResource resource) except+ nogil # NVRTC Program handles - NvrtcProgramHandle create_nvrtc_program_handle "cuda_core::create_nvrtc_program_handle" ( + NvrtcProgramHandle create_nvrtc_program_handle "cuda_core::rt::create_nvrtc_program_handle" ( cynvrtc.nvrtcProgram prog) except+ nogil - NvrtcProgramHandle create_nvrtc_program_handle_ref "cuda_core::create_nvrtc_program_handle_ref" ( + NvrtcProgramHandle create_nvrtc_program_handle_ref "cuda_core::rt::create_nvrtc_program_handle_ref" ( cynvrtc.nvrtcProgram prog) except+ nogil # NVVM Program handles - NvvmProgramHandle create_nvvm_program_handle "cuda_core::create_nvvm_program_handle" ( + NvvmProgramHandle create_nvvm_program_handle "cuda_core::rt::create_nvvm_program_handle" ( cynvvm.nvvmProgram prog) except+ nogil - NvvmProgramHandle create_nvvm_program_handle_ref "cuda_core::create_nvvm_program_handle_ref" ( + NvvmProgramHandle create_nvvm_program_handle_ref "cuda_core::rt::create_nvvm_program_handle_ref" ( cynvvm.nvvmProgram prog) except+ nogil # nvJitLink handles - NvJitLinkHandle create_nvjitlink_handle "cuda_core::create_nvjitlink_handle" ( + NvJitLinkHandle create_nvjitlink_handle "cuda_core::rt::create_nvjitlink_handle" ( cynvjitlink.nvJitLinkHandle handle) except+ nogil - NvJitLinkHandle create_nvjitlink_handle_ref "cuda_core::create_nvjitlink_handle_ref" ( + NvJitLinkHandle create_nvjitlink_handle_ref "cuda_core::rt::create_nvjitlink_handle_ref" ( cynvjitlink.nvJitLinkHandle handle) except+ nogil # cuLink handles - CuLinkHandle create_culink_handle "cuda_core::create_culink_handle" ( + CuLinkHandle create_culink_handle "cuda_core::rt::create_culink_handle" ( cydriver.CUlinkState state) except+ nogil - CuLinkHandle create_culink_handle_ref "cuda_core::create_culink_handle_ref" ( + CuLinkHandle create_culink_handle_ref "cuda_core::rt::create_culink_handle_ref" ( cydriver.CUlinkState state) except+ nogil # File descriptor handles - FileDescriptorHandle create_fd_handle "cuda_core::create_fd_handle" ( + FileDescriptorHandle create_fd_handle "cuda_core::rt::create_fd_handle" ( int fd) except+ nogil - FileDescriptorHandle create_fd_handle_ref "cuda_core::create_fd_handle_ref" ( + FileDescriptorHandle create_fd_handle_ref "cuda_core::rt::create_fd_handle_ref" ( int fd) except+ nogil # SM resource split (13.1+ wrapper — avoids direct cydriver cimport) # groupParams is void* to avoid referencing CU_DEV_SM_RESOURCE_GROUP_PARAMS # (which doesn't exist in cuda-bindings 13.0 .pxd). The C++ side casts it. - cydriver.CUresult sm_resource_split "cuda_core::sm_resource_split" ( + cydriver.CUresult sm_resource_split "cuda_core::rt::sm_resource_split" ( cydriver.CUdevResource* result, unsigned int nbGroups, const cydriver.CUdevResource* input, cydriver.CUdevResource* remainder, unsigned int flags, void* groupParams) nogil - bint has_sm_resource_split "cuda_core::has_sm_resource_split" () noexcept nogil + bint has_sm_resource_split "cuda_core::rt::has_sm_resource_split" () noexcept nogil # cuMemcpyWithAttributesAsync (13.2+ wrapper — avoids direct cydriver cimport) # attr is void* to avoid referencing CUmemcpyAttributes (absent from # cuda-bindings built against CUDA < 12.8). The C++ side casts it. - cydriver.CUresult memcpy_with_attributes_async "cuda_core::memcpy_with_attributes_async" ( + cydriver.CUresult memcpy_with_attributes_async "cuda_core::rt::memcpy_with_attributes_async" ( cydriver.CUdeviceptr dst, cydriver.CUdeviceptr src, size_t size, void* attr, cydriver.CUstream hStream) nogil - bint has_memcpy_with_attributes_async "cuda_core::has_memcpy_with_attributes_async" () noexcept nogil + bint has_memcpy_with_attributes_async "cuda_core::rt::has_memcpy_with_attributes_async" () noexcept nogil # Array / mipmapped-array / texture / surface handles (PR #467) - OpaqueArrayHandle create_array_handle "cuda_core::create_array_handle" ( + OpaqueArrayHandle create_array_handle "cuda_core::rt::create_array_handle" ( const ContextHandle& h_context, const cydriver.CUDA_ARRAY3D_DESCRIPTOR& desc) except+ nogil - OpaqueArrayHandle create_array_handle_ref "cuda_core::create_array_handle_ref" ( + OpaqueArrayHandle create_array_handle_ref "cuda_core::rt::create_array_handle_ref" ( cydriver.CUarray arr) except+ nogil - OpaqueArrayHandle create_array_handle_owning "cuda_core::create_array_handle_owning" ( + OpaqueArrayHandle create_array_handle_owning "cuda_core::rt::create_array_handle_owning" ( cydriver.CUarray arr) except+ nogil - ContextHandle get_array_context "cuda_core::get_array_context" ( + ContextHandle get_array_context "cuda_core::rt::get_array_context" ( const OpaqueArrayHandle& h) noexcept nogil - OpaqueArrayHandle create_array_level_handle "cuda_core::create_array_level_handle" ( + OpaqueArrayHandle create_array_level_handle "cuda_core::rt::create_array_level_handle" ( const MipmappedArrayHandle& h_mip, unsigned int level) except+ nogil - MipmappedArrayHandle create_mipmapped_array_handle "cuda_core::create_mipmapped_array_handle" ( + MipmappedArrayHandle create_mipmapped_array_handle "cuda_core::rt::create_mipmapped_array_handle" ( const ContextHandle& h_context, const cydriver.CUDA_ARRAY3D_DESCRIPTOR& desc, unsigned int num_levels) except+ nogil - ContextHandle get_mipmapped_array_context "cuda_core::get_mipmapped_array_context" ( + ContextHandle get_mipmapped_array_context "cuda_core::rt::get_mipmapped_array_context" ( const MipmappedArrayHandle& h) noexcept nogil - TexObjectHandle create_tex_object_handle_array "cuda_core::create_tex_object_handle_array" ( + TexObjectHandle create_tex_object_handle_array "cuda_core::rt::create_tex_object_handle_array" ( const ContextHandle& h_context, const cydriver.CUDA_RESOURCE_DESC& res, const cydriver.CUDA_TEXTURE_DESC& tex, const OpaqueArrayHandle& h_backing) except+ nogil - TexObjectHandle create_tex_object_handle_mipmap "cuda_core::create_tex_object_handle_mipmap" ( + TexObjectHandle create_tex_object_handle_mipmap "cuda_core::rt::create_tex_object_handle_mipmap" ( const ContextHandle& h_context, const cydriver.CUDA_RESOURCE_DESC& res, const cydriver.CUDA_TEXTURE_DESC& tex, const MipmappedArrayHandle& h_backing) except+ nogil - TexObjectHandle create_tex_object_handle_linear "cuda_core::create_tex_object_handle_linear" ( + TexObjectHandle create_tex_object_handle_linear "cuda_core::rt::create_tex_object_handle_linear" ( const ContextHandle& h_context, const cydriver.CUDA_RESOURCE_DESC& res, const cydriver.CUDA_TEXTURE_DESC& tex, const DevicePtrHandle& h_backing) except+ nogil - SurfObjectHandle create_surf_object_handle "cuda_core::create_surf_object_handle" ( + SurfObjectHandle create_surf_object_handle "cuda_core::rt::create_surf_object_handle" ( const ContextHandle& h_context, const cydriver.CUDA_RESOURCE_DESC& res, const OpaqueArrayHandle& h_backing) except+ nogil -# ============================================================================= -# CUDA Driver API capsule -# -# This provides resolved CUDA driver function pointers to the C++ code. -# ============================================================================= - -cdef const char* _CUDA_DRIVER_API_V1_NAME = b"cuda.core._resource_handles._CUDA_DRIVER_API_V1" - - # ============================================================================= # CUDA driver function pointer initialization # @@ -311,108 +330,110 @@ cdef const char* _CUDA_DRIVER_API_V1_NAME = b"cuda.core._resource_handles._CUDA_ # ============================================================================= # Declare extern variables with reinterpret_cast to allow void* assignment -cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": +cdef extern from "_cpp/rt/rt.hpp" namespace "cuda_core::rt": # Error formatting - void* p_cuGetErrorName "reinterpret_cast(cuda_core::p_cuGetErrorName)" - void* p_cuGetErrorString "reinterpret_cast(cuda_core::p_cuGetErrorString)" + void* p_cuGetErrorName "reinterpret_cast(cuda_core::rt::p_cuGetErrorName)" + void* p_cuGetErrorString "reinterpret_cast(cuda_core::rt::p_cuGetErrorString)" # Context - void* p_cuDevicePrimaryCtxRetain "reinterpret_cast(cuda_core::p_cuDevicePrimaryCtxRetain)" - void* p_cuDevicePrimaryCtxRelease "reinterpret_cast(cuda_core::p_cuDevicePrimaryCtxRelease)" - void* p_cuCtxGetCurrent "reinterpret_cast(cuda_core::p_cuCtxGetCurrent)" - void* p_cuCtxSetCurrent "reinterpret_cast(cuda_core::p_cuCtxSetCurrent)" - void* p_cuCtxSynchronize "reinterpret_cast(cuda_core::p_cuCtxSynchronize)" - void* p_cuCtxGetStreamPriorityRange "reinterpret_cast(cuda_core::p_cuCtxGetStreamPriorityRange)" - void* p_cuGreenCtxCreate "reinterpret_cast(cuda_core::p_cuGreenCtxCreate)" - void* p_cuGreenCtxDestroy "reinterpret_cast(cuda_core::p_cuGreenCtxDestroy)" - void* p_cuCtxFromGreenCtx "reinterpret_cast(cuda_core::p_cuCtxFromGreenCtx)" - void* p_cuDevResourceGenerateDesc "reinterpret_cast(cuda_core::p_cuDevResourceGenerateDesc)" - void* p_cuGreenCtxStreamCreate "reinterpret_cast(cuda_core::p_cuGreenCtxStreamCreate)" + void* p_cuDevicePrimaryCtxRetain "reinterpret_cast(cuda_core::rt::p_cuDevicePrimaryCtxRetain)" + void* p_cuDevicePrimaryCtxRelease "reinterpret_cast(cuda_core::rt::p_cuDevicePrimaryCtxRelease)" + void* p_cuCtxGetCurrent "reinterpret_cast(cuda_core::rt::p_cuCtxGetCurrent)" + void* p_cuCtxSetCurrent "reinterpret_cast(cuda_core::rt::p_cuCtxSetCurrent)" + void* p_cuCtxSynchronize "reinterpret_cast(cuda_core::rt::p_cuCtxSynchronize)" + void* p_cuCtxGetStreamPriorityRange "reinterpret_cast(cuda_core::rt::p_cuCtxGetStreamPriorityRange)" + void* p_cuCtxGetDevice "reinterpret_cast(cuda_core::rt::p_cuCtxGetDevice)" + void* p_cuGraphNodeSetParams "reinterpret_cast(cuda_core::rt::p_cuGraphNodeSetParams)" + void* p_cuGreenCtxCreate "reinterpret_cast(cuda_core::rt::p_cuGreenCtxCreate)" + void* p_cuGreenCtxDestroy "reinterpret_cast(cuda_core::rt::p_cuGreenCtxDestroy)" + void* p_cuCtxFromGreenCtx "reinterpret_cast(cuda_core::rt::p_cuCtxFromGreenCtx)" + void* p_cuDevResourceGenerateDesc "reinterpret_cast(cuda_core::rt::p_cuDevResourceGenerateDesc)" + void* p_cuGreenCtxStreamCreate "reinterpret_cast(cuda_core::rt::p_cuGreenCtxStreamCreate)" # Stream - void* p_cuStreamCreateWithPriority "reinterpret_cast(cuda_core::p_cuStreamCreateWithPriority)" - void* p_cuStreamDestroy "reinterpret_cast(cuda_core::p_cuStreamDestroy)" - void* p_cuStreamGetCtx "reinterpret_cast(cuda_core::p_cuStreamGetCtx)" + void* p_cuStreamCreateWithPriority "reinterpret_cast(cuda_core::rt::p_cuStreamCreateWithPriority)" + void* p_cuStreamDestroy "reinterpret_cast(cuda_core::rt::p_cuStreamDestroy)" + void* p_cuStreamGetCtx "reinterpret_cast(cuda_core::rt::p_cuStreamGetCtx)" # Event - void* p_cuEventCreate "reinterpret_cast(cuda_core::p_cuEventCreate)" - void* p_cuEventDestroy "reinterpret_cast(cuda_core::p_cuEventDestroy)" - void* p_cuIpcOpenEventHandle "reinterpret_cast(cuda_core::p_cuIpcOpenEventHandle)" + void* p_cuEventCreate "reinterpret_cast(cuda_core::rt::p_cuEventCreate)" + void* p_cuEventDestroy "reinterpret_cast(cuda_core::rt::p_cuEventDestroy)" + void* p_cuIpcOpenEventHandle "reinterpret_cast(cuda_core::rt::p_cuIpcOpenEventHandle)" # Device - void* p_cuDeviceGetCount "reinterpret_cast(cuda_core::p_cuDeviceGetCount)" + void* p_cuDeviceGetCount "reinterpret_cast(cuda_core::rt::p_cuDeviceGetCount)" # Memory pool - void* p_cuMemPoolSetAccess "reinterpret_cast(cuda_core::p_cuMemPoolSetAccess)" - void* p_cuMemPoolDestroy "reinterpret_cast(cuda_core::p_cuMemPoolDestroy)" - void* p_cuMemPoolCreate "reinterpret_cast(cuda_core::p_cuMemPoolCreate)" - void* p_cuDeviceGetMemPool "reinterpret_cast(cuda_core::p_cuDeviceGetMemPool)" - void* p_cuMemPoolImportFromShareableHandle "reinterpret_cast(cuda_core::p_cuMemPoolImportFromShareableHandle)" + void* p_cuMemPoolSetAccess "reinterpret_cast(cuda_core::rt::p_cuMemPoolSetAccess)" + void* p_cuMemPoolDestroy "reinterpret_cast(cuda_core::rt::p_cuMemPoolDestroy)" + void* p_cuMemPoolCreate "reinterpret_cast(cuda_core::rt::p_cuMemPoolCreate)" + void* p_cuDeviceGetMemPool "reinterpret_cast(cuda_core::rt::p_cuDeviceGetMemPool)" + void* p_cuMemPoolImportFromShareableHandle "reinterpret_cast(cuda_core::rt::p_cuMemPoolImportFromShareableHandle)" # Memory allocation - void* p_cuMemAllocFromPoolAsync "reinterpret_cast(cuda_core::p_cuMemAllocFromPoolAsync)" - void* p_cuMemAllocAsync "reinterpret_cast(cuda_core::p_cuMemAllocAsync)" - void* p_cuMemAlloc "reinterpret_cast(cuda_core::p_cuMemAlloc)" - void* p_cuMemAllocHost "reinterpret_cast(cuda_core::p_cuMemAllocHost)" + void* p_cuMemAllocFromPoolAsync "reinterpret_cast(cuda_core::rt::p_cuMemAllocFromPoolAsync)" + void* p_cuMemAllocAsync "reinterpret_cast(cuda_core::rt::p_cuMemAllocAsync)" + void* p_cuMemAlloc "reinterpret_cast(cuda_core::rt::p_cuMemAlloc)" + void* p_cuMemAllocHost "reinterpret_cast(cuda_core::rt::p_cuMemAllocHost)" # Memory deallocation - void* p_cuMemFreeAsync "reinterpret_cast(cuda_core::p_cuMemFreeAsync)" - void* p_cuMemFree "reinterpret_cast(cuda_core::p_cuMemFree)" - void* p_cuMemFreeHost "reinterpret_cast(cuda_core::p_cuMemFreeHost)" + void* p_cuMemFreeAsync "reinterpret_cast(cuda_core::rt::p_cuMemFreeAsync)" + void* p_cuMemFree "reinterpret_cast(cuda_core::rt::p_cuMemFree)" + void* p_cuMemFreeHost "reinterpret_cast(cuda_core::rt::p_cuMemFreeHost)" # IPC - void* p_cuMemPoolImportPointer "reinterpret_cast(cuda_core::p_cuMemPoolImportPointer)" + void* p_cuMemPoolImportPointer "reinterpret_cast(cuda_core::rt::p_cuMemPoolImportPointer)" # Library - void* p_cuLibraryLoadFromFile "reinterpret_cast(cuda_core::p_cuLibraryLoadFromFile)" - void* p_cuLibraryLoadData "reinterpret_cast(cuda_core::p_cuLibraryLoadData)" - void* p_cuLibraryUnload "reinterpret_cast(cuda_core::p_cuLibraryUnload)" - void* p_cuLibraryGetKernel "reinterpret_cast(cuda_core::p_cuLibraryGetKernel)" + void* p_cuLibraryLoadFromFile "reinterpret_cast(cuda_core::rt::p_cuLibraryLoadFromFile)" + void* p_cuLibraryLoadData "reinterpret_cast(cuda_core::rt::p_cuLibraryLoadData)" + void* p_cuLibraryUnload "reinterpret_cast(cuda_core::rt::p_cuLibraryUnload)" + void* p_cuLibraryGetKernel "reinterpret_cast(cuda_core::rt::p_cuLibraryGetKernel)" # Graph - void* p_cuGraphDestroy "reinterpret_cast(cuda_core::p_cuGraphDestroy)" - void* p_cuGraphInstantiateWithParams "reinterpret_cast(cuda_core::p_cuGraphInstantiateWithParams)" - void* p_cuGraphExecUpdate "reinterpret_cast(cuda_core::p_cuGraphExecUpdate)" - void* p_cuGraphExecDestroy "reinterpret_cast(cuda_core::p_cuGraphExecDestroy)" - void* p_cuUserObjectCreate "reinterpret_cast(cuda_core::p_cuUserObjectCreate)" - void* p_cuUserObjectRelease "reinterpret_cast(cuda_core::p_cuUserObjectRelease)" - void* p_cuGraphRetainUserObject "reinterpret_cast(cuda_core::p_cuGraphRetainUserObject)" - void* p_cuGraphReleaseUserObject "reinterpret_cast(cuda_core::p_cuGraphReleaseUserObject)" - void* p_cuGraphNodeFindInClone "reinterpret_cast(cuda_core::p_cuGraphNodeFindInClone)" - void* p_cuGraphChildGraphNodeGetGraph "reinterpret_cast(cuda_core::p_cuGraphChildGraphNodeGetGraph)" + void* p_cuGraphDestroy "reinterpret_cast(cuda_core::rt::p_cuGraphDestroy)" + void* p_cuGraphInstantiateWithParams "reinterpret_cast(cuda_core::rt::p_cuGraphInstantiateWithParams)" + void* p_cuGraphExecUpdate "reinterpret_cast(cuda_core::rt::p_cuGraphExecUpdate)" + void* p_cuGraphExecDestroy "reinterpret_cast(cuda_core::rt::p_cuGraphExecDestroy)" + void* p_cuUserObjectCreate "reinterpret_cast(cuda_core::rt::p_cuUserObjectCreate)" + void* p_cuUserObjectRelease "reinterpret_cast(cuda_core::rt::p_cuUserObjectRelease)" + void* p_cuGraphRetainUserObject "reinterpret_cast(cuda_core::rt::p_cuGraphRetainUserObject)" + void* p_cuGraphReleaseUserObject "reinterpret_cast(cuda_core::rt::p_cuGraphReleaseUserObject)" + void* p_cuGraphNodeFindInClone "reinterpret_cast(cuda_core::rt::p_cuGraphNodeFindInClone)" + void* p_cuGraphChildGraphNodeGetGraph "reinterpret_cast(cuda_core::rt::p_cuGraphChildGraphNodeGetGraph)" # Linker - void* p_cuLinkDestroy "reinterpret_cast(cuda_core::p_cuLinkDestroy)" + void* p_cuLinkDestroy "reinterpret_cast(cuda_core::rt::p_cuLinkDestroy)" # Graphics interop - void* p_cuGraphicsUnmapResources "reinterpret_cast(cuda_core::p_cuGraphicsUnmapResources)" - void* p_cuGraphicsUnregisterResource "reinterpret_cast(cuda_core::p_cuGraphicsUnregisterResource)" + void* p_cuGraphicsUnmapResources "reinterpret_cast(cuda_core::rt::p_cuGraphicsUnmapResources)" + void* p_cuGraphicsUnregisterResource "reinterpret_cast(cuda_core::rt::p_cuGraphicsUnregisterResource)" # Texture / surface / array (PR #467) - void* p_cuArray3DCreate "reinterpret_cast(cuda_core::p_cuArray3DCreate)" - void* p_cuArrayDestroy "reinterpret_cast(cuda_core::p_cuArrayDestroy)" - void* p_cuMipmappedArrayCreate "reinterpret_cast(cuda_core::p_cuMipmappedArrayCreate)" - void* p_cuMipmappedArrayDestroy "reinterpret_cast(cuda_core::p_cuMipmappedArrayDestroy)" - void* p_cuMipmappedArrayGetLevel "reinterpret_cast(cuda_core::p_cuMipmappedArrayGetLevel)" - void* p_cuTexObjectCreate "reinterpret_cast(cuda_core::p_cuTexObjectCreate)" - void* p_cuTexObjectDestroy "reinterpret_cast(cuda_core::p_cuTexObjectDestroy)" - void* p_cuSurfObjectCreate "reinterpret_cast(cuda_core::p_cuSurfObjectCreate)" - void* p_cuSurfObjectDestroy "reinterpret_cast(cuda_core::p_cuSurfObjectDestroy)" + void* p_cuArray3DCreate "reinterpret_cast(cuda_core::rt::p_cuArray3DCreate)" + void* p_cuArrayDestroy "reinterpret_cast(cuda_core::rt::p_cuArrayDestroy)" + void* p_cuMipmappedArrayCreate "reinterpret_cast(cuda_core::rt::p_cuMipmappedArrayCreate)" + void* p_cuMipmappedArrayDestroy "reinterpret_cast(cuda_core::rt::p_cuMipmappedArrayDestroy)" + void* p_cuMipmappedArrayGetLevel "reinterpret_cast(cuda_core::rt::p_cuMipmappedArrayGetLevel)" + void* p_cuTexObjectCreate "reinterpret_cast(cuda_core::rt::p_cuTexObjectCreate)" + void* p_cuTexObjectDestroy "reinterpret_cast(cuda_core::rt::p_cuTexObjectDestroy)" + void* p_cuSurfObjectCreate "reinterpret_cast(cuda_core::rt::p_cuSurfObjectCreate)" + void* p_cuSurfObjectDestroy "reinterpret_cast(cuda_core::rt::p_cuSurfObjectDestroy)" # SM resource split (13.1+) - void* p_cuDevSmResourceSplit "reinterpret_cast(cuda_core::p_cuDevSmResourceSplit)" + void* p_cuDevSmResourceSplit "reinterpret_cast(cuda_core::rt::p_cuDevSmResourceSplit)" # cuMemcpyWithAttributesAsync (13.2+) - void* p_cuMemcpyWithAttributesAsync "reinterpret_cast(cuda_core::p_cuMemcpyWithAttributesAsync)" + void* p_cuMemcpyWithAttributesAsync "reinterpret_cast(cuda_core::rt::p_cuMemcpyWithAttributesAsync)" # NVRTC - void* p_nvrtcDestroyProgram "reinterpret_cast(cuda_core::p_nvrtcDestroyProgram)" + void* p_nvrtcDestroyProgram "reinterpret_cast(cuda_core::rt::p_nvrtcDestroyProgram)" # NVVM - void* p_nvvmDestroyProgram "reinterpret_cast(cuda_core::p_nvvmDestroyProgram)" + void* p_nvvmDestroyProgram "reinterpret_cast(cuda_core::rt::p_nvvmDestroyProgram)" # nvJitLink - void* p_nvJitLinkDestroy "reinterpret_cast(cuda_core::p_nvJitLinkDestroy)" + void* p_nvJitLinkDestroy "reinterpret_cast(cuda_core::rt::p_nvJitLinkDestroy)" # Initialize driver function pointers from cydriver.__pyx_capi__ at module load @@ -433,6 +454,7 @@ cdef void _init_driver_fn_pointers() noexcept: global p_cuGetErrorName, p_cuGetErrorString global p_cuDevicePrimaryCtxRetain, p_cuDevicePrimaryCtxRelease, p_cuCtxGetCurrent global p_cuCtxSetCurrent, p_cuCtxSynchronize, p_cuCtxGetStreamPriorityRange + global p_cuCtxGetDevice, p_cuGraphNodeSetParams global p_cuGreenCtxCreate, p_cuGreenCtxDestroy, p_cuCtxFromGreenCtx global p_cuDevResourceGenerateDesc, p_cuGreenCtxStreamCreate global p_cuStreamCreateWithPriority, p_cuStreamDestroy, p_cuStreamGetCtx @@ -469,6 +491,9 @@ cdef void _init_driver_fn_pointers() noexcept: p_cuCtxSetCurrent = _get_driver_fn("cuCtxSetCurrent") p_cuCtxSynchronize = _get_driver_fn("cuCtxSynchronize") p_cuCtxGetStreamPriorityRange = _get_driver_fn("cuCtxGetStreamPriorityRange") + p_cuCtxGetDevice = _get_driver_fn("cuCtxGetDevice") + # Graph node parameter updates need CUDA 12.2+ (checked again at the call site). + p_cuGraphNodeSetParams = _get_optional_driver_fn("cuGraphNodeSetParams") p_cuGreenCtxCreate = _get_optional_driver_fn("cuGreenCtxCreate") p_cuGreenCtxDestroy = _get_optional_driver_fn("cuGreenCtxDestroy") p_cuCtxFromGreenCtx = _get_optional_driver_fn("cuCtxFromGreenCtx") @@ -554,6 +579,27 @@ cdef void _init_driver_fn_pointers() noexcept: _init_driver_fn_pointers() initialize_deferred_cleanup() + +def _set_context_restore_fault_for_testing(int status): + """Make the next context restoration on this thread fail with ``status``. + + Test hook for the context save/restore paths in the handle layer. The + injected failure leaves the target context current, exactly as a failing + ``cuCtxSetCurrent`` would, so callers must restore the context themselves. + """ + set_context_restore_fault_for_testing(status) + + +def _note_or_report_cuda_error_for_testing(int status): + """Attach a failed CUDA call to the exception being handled, or report it. + + Test hook for ``note_or_report_cuda_error()``. Called inside an ``except`` + block it adds a note to the exception being handled (Python 3.11+); anywhere + else it emits a ``CUDAWarning``. + """ + _note_or_report_cuda_error_local( + b"cuTestOperation", status, b"failed while testing") + # ============================================================================= # NVRTC function pointer initialization # ============================================================================= diff --git a/cuda_core/cuda/core/_stream.pxd b/cuda_core/cuda/core/_stream.pxd index 5de11a36761..9ae01151785 100644 --- a/cuda_core/cuda/core/_stream.pxd +++ b/cuda_core/cuda/core/_stream.pxd @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -from cuda.core._resource_handles cimport ContextHandle, StreamHandle +from cuda.core._rt cimport ContextHandle, StreamHandle cdef class Stream: diff --git a/cuda_core/cuda/core/_stream.pyx b/cuda_core/cuda/core/_stream.pyx index e662d67c87f..4c51b32f4f0 100644 --- a/cuda_core/cuda/core/_stream.pyx +++ b/cuda_core/cuda/core/_stream.pyx @@ -27,7 +27,8 @@ from cuda.core._context cimport ( from cuda.core._device_resources cimport DeviceResources from cuda.core._event import Event, EventOptions -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport context_get_device +from cuda.core._rt cimport ( ContextHandle, EventHandle, StreamHandle, @@ -36,7 +37,6 @@ from cuda.core._resource_handles cimport ( create_stream_handle, create_stream_handle_with_owner, context_get_stream_priority_range, - get_current_context, get_last_error, get_legacy_stream, get_per_thread_stream, @@ -564,10 +564,7 @@ cdef inline int Stream_get_ctx(Stream self, ContextHandle* h_context) except?-1 cdef inline int Stream_get_ctx_device(Stream self, ContextHandle* h_context, int* device_id) except?-1: """Resolve the stream's context handle and device ID.""" - cdef cydriver.CUcontext ctx cdef cydriver.CUdevice target_dev - cdef ContextHandle current_context - cdef bint switch_context cdef bint is_default = Stream_is_default_token(self) with nogil: @@ -575,14 +572,9 @@ cdef inline int Stream_get_ctx_device(Stream self, ContextHandle* h_context, int if self._device_id >= 0 and not is_default: device_id[0] = self._device_id else: - # Get device ID from context, switching context temporarily if needed - current_context = get_current_context() - switch_context = (as_cu(current_context) != as_cu(h_context[0])) - if switch_context: - HANDLE_RETURN(cydriver.cuCtxPushCurrent(as_cu(h_context[0]))) - HANDLE_RETURN(cydriver.cuCtxGetDevice(&target_dev)) - if switch_context: - HANDLE_RETURN(cydriver.cuCtxPopCurrent(&ctx)) + # Query the device with the stream's context current. The handle + # layer restores the caller's context, including on failure. + HANDLE_RETURN(context_get_device(h_context[0], &target_dev)) device_id[0] = target_dev if not is_default: self._device_id = device_id[0] diff --git a/cuda_core/cuda/core/_tensor_bridge.pyx b/cuda_core/cuda/core/_tensor_bridge.pyx index ae7a6794507..c7c3cd5c718 100644 --- a/cuda_core/cuda/core/_tensor_bridge.pyx +++ b/cuda_core/cuda/core/_tensor_bridge.pyx @@ -54,7 +54,7 @@ from libc.stdint cimport intptr_t, int8_t, int16_t, int32_t, int64_t, uint8_t from cuda.core._memoryview cimport StridedMemoryView from cuda.core._layout cimport _StridedLayout from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( EventHandle, create_event_handle_for_stream, as_cu, diff --git a/cuda_core/cuda/core/_utils/_weak_handles.pyx b/cuda_core/cuda/core/_utils/_weak_handles.pyx index d9f71e36772..c0c456ed728 100644 --- a/cuda_core/cuda/core/_utils/_weak_handles.pyx +++ b/cuda_core/cuda/core/_utils/_weak_handles.pyx @@ -24,7 +24,7 @@ Python owners via ``make_opaque_py`` are not covered here -- use from cuda.core._memory._buffer cimport Buffer from cuda.core.graph._graph_definition cimport GraphDefinition -from cuda.core._resource_handles cimport OpaqueHandle +from cuda.core._rt cimport OpaqueHandle # Cython cannot spell ``weak_ptr[const void]`` inline (the ``const void`` diff --git a/cuda_core/cuda/core/_utils/cuda_utils.pyi b/cuda_core/cuda/core/_utils/cuda_utils.pyi index 51f992fa238..565200277ba 100644 --- a/cuda_core/cuda/core/_utils/cuda_utils.pyi +++ b/cuda_core/cuda/core/_utils/cuda_utils.pyi @@ -14,6 +14,27 @@ _fork_warning_checked = False class CUDAError(Exception): ... +class CUDAWarning(RuntimeWarning): + """Warning issued when ``cuda.core`` hits a CUDA error it cannot raise. + + ``cuda.core`` raises exceptions for failures in ordinary calls. Some failures + happen where no exception can propagate: while a resource is released by the + garbage collector or by a CUDA callback, including the driver calls that + switch and restore the CUDA context around such a release. Those failures + are reported as this warning instead, and the affected resource may have + leaked. + + Filter on this category to make such failures fatal in tests:: + + warnings.filterwarnings("error", category=cuda.core.CUDAWarning) + + Because the report comes from a destructor, an escalated warning cannot be + raised into user code; it is delivered through :func:`sys.unraisablehook` + (which pytest surfaces as ``PytestUnraisableExceptionWarning``). + + .. versionadded:: 1.3.0 + """ + class NVRTCError(CUDAError): ... class ComputeCapability(NamedTuple): diff --git a/cuda_core/cuda/core/_utils/cuda_utils.pyx b/cuda_core/cuda/core/_utils/cuda_utils.pyx index ce75746de56..7c9c352fb72 100644 --- a/cuda_core/cuda/core/_utils/cuda_utils.pyx +++ b/cuda_core/cuda/core/_utils/cuda_utils.pyx @@ -27,6 +27,10 @@ from cuda.bindings.nvjitlink import nvJitLinkError from cpython.buffer cimport PyObject_GetBuffer, PyBuffer_Release, Py_buffer, PyBUF_SIMPLE from cuda.bindings cimport cynvrtc, cynvvm, cynvjitlink +from cuda.core._rt cimport ( + register_warning_category, + take_last_error_detail, +) from cuda.core._utils.driver_cu_result_explanations import DRIVER_CU_RESULT_EXPLANATIONS from cuda.core._utils.runtime_cuda_error_explanations import RUNTIME_CUDA_ERROR_EXPLANATIONS @@ -36,6 +40,32 @@ class CUDAError(Exception): pass +class CUDAWarning(RuntimeWarning): + """Warning issued when ``cuda.core`` hits a CUDA error it cannot raise. + + ``cuda.core`` raises exceptions for failures in ordinary calls. Some failures + happen where no exception can propagate: while a resource is released by the + garbage collector or by a CUDA callback, including the driver calls that + switch and restore the CUDA context around such a release. Those failures + are reported as this warning instead, and the affected resource may have + leaked. + + Filter on this category to make such failures fatal in tests:: + + warnings.filterwarnings("error", category=cuda.core.CUDAWarning) + + Because the report comes from a destructor, an escalated warning cannot be + raised into user code; it is delivered through :func:`sys.unraisablehook` + (which pytest surfaces as ``PytestUnraisableExceptionWarning``). + + .. versionadded:: 1.3.0 + """ + + +# Route the C++ handle layer's non-propagating reports through this category. +register_warning_category(CUDAWarning) + + class NVRTCError(CUDAError): pass @@ -131,23 +161,40 @@ cdef object _RUNTIME_SUCCESS = runtime.cudaError_t.cudaSuccess cdef object _NVRTC_SUCCESS = nvrtc.nvrtcResult.NVRTC_SUCCESS +cdef inline void _attach_detail(exc, str detail): + # PEP 678 notes (Python 3.11+) keep the detail separable from the message; + # older interpreters get it appended to the message instead. + add_note = getattr(exc, "add_note", None) + if add_note is not None: + add_note(detail) + else: + exc.args = (f"{exc.args[0]} ({detail})", *exc.args[1:]) + + cpdef inline int _check_driver_error(cydriver.CUresult error) except?-1 nogil: if error == cydriver.CUresult.CUDA_SUCCESS: return 0 cdef const char* name + cdef const char* desc + # A context-scoped helper in the handle layer may have recorded why this + # status needs more explanation (e.g. the caller's context was not restored). + cdef const char* detail = take_last_error_detail(error) name_err = cydriver.cuGetErrorName(error, &name) if name_err != cydriver.CUresult.CUDA_SUCCESS: raise CUDAError(f"UNEXPECTED ERROR CODE: {error}") + desc_err = cydriver.cuGetErrorString(error, &desc) with gil: # TODO: consider lower this to Cython expl = DRIVER_CU_RESULT_EXPLANATIONS.get(int(error)) if expl is not None: - raise CUDAError(f"{name.decode()}: {expl}") - cdef const char* desc - desc_err = cydriver.cuGetErrorString(error, &desc) - if desc_err != cydriver.CUresult.CUDA_SUCCESS: - raise CUDAError(f"{name.decode()}") - raise CUDAError(f"{name.decode()}: {desc.decode()}") + exc = CUDAError(f"{name.decode()}: {expl}") + elif desc_err != cydriver.CUresult.CUDA_SUCCESS: + exc = CUDAError(name.decode()) + else: + exc = CUDAError(f"{name.decode()}: {desc.decode()}") + if detail != NULL: + _attach_detail(exc, detail.decode()) + raise exc cpdef inline int _check_runtime_error(error) except?-1: diff --git a/cuda_core/cuda/core/graph/_adjacency_set_proxy.pyx b/cuda_core/cuda/core/graph/_adjacency_set_proxy.pyx index 8919067d0b0..2c2c67e5005 100644 --- a/cuda_core/cuda/core/graph/_adjacency_set_proxy.pyx +++ b/cuda_core/cuda/core/graph/_adjacency_set_proxy.pyx @@ -8,7 +8,7 @@ from libc.stddef cimport size_t from libcpp.vector cimport vector from cuda.bindings cimport cydriver from cuda.core.graph._graph_node cimport GraphNode, GN_check_valid -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( GraphHandle, GraphNodeHandle, as_cu, diff --git a/cuda_core/cuda/core/graph/_graph_builder.pxd b/cuda_core/cuda/core/graph/_graph_builder.pxd index eb75e6bd44a..f61701a5970 100644 --- a/cuda_core/cuda/core/graph/_graph_builder.pxd +++ b/cuda_core/cuda/core/graph/_graph_builder.pxd @@ -4,7 +4,7 @@ from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport GraphExecHandle, GraphHandle, StreamHandle +from cuda.core._rt cimport GraphExecHandle, GraphHandle, StreamHandle from cuda.core._stream cimport Stream diff --git a/cuda_core/cuda/core/graph/_graph_builder.pyx b/cuda_core/cuda/core/graph/_graph_builder.pyx index 071fff38386..941cd989958 100644 --- a/cuda_core/cuda/core/graph/_graph_builder.pyx +++ b/cuda_core/cuda/core/graph/_graph_builder.pyx @@ -20,7 +20,8 @@ from cuda.core.graph._subclasses cimport ( ExecutableGraphNode, create_executable_node_view, ) -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport note_or_report_cuda_error, report_cuda_error +from cuda.core._rt cimport ( GraphExecHandle, GraphHandle, OpaqueHandle, @@ -860,6 +861,12 @@ cdef class GraphBuilder: if rollback_status == cydriver.CUDA_SUCCESS: invalidate_child_graph_state( self._h_graph, c_new_node) + else: + # The original exception propagates with the failed rollback + # attached as a note (error handling policy). + note_or_report_cuda_error( + b"cuGraphDestroyNode", rollback_status, + b"failed while rolling back a child graph node; the node remains in the graph") raise deps_info_update = [[new_node]] + [None] * (len(deps_info_out) - 1) @@ -990,8 +997,8 @@ cdef inline int GB_end_capture_if_needed(GraphBuilder gb, bint check_status) exc capture. A FORKED builder must not call cuStreamEndCapture: the driver requires forked streams to be joined first. - check_status=True checks the driver return (close()); False ignores it - (__dealloc__). + check_status=True raises on a driver error (close()); False reports it as + a CUDAWarning instead, because nothing can be raised from __dealloc__. """ cdef cydriver.CUgraph c_graph cdef cydriver.CUresult err @@ -1002,6 +1009,10 @@ cdef inline int GB_end_capture_if_needed(GraphBuilder gb, bint check_status) exc err = cydriver.cuStreamEndCapture(c_stream, &c_graph) if check_status: HANDLE_RETURN(err) + else: + report_cuda_error( + b"cuStreamEndCapture", err, + b"failed while releasing a GraphBuilder that was still building") return 0 diff --git a/cuda_core/cuda/core/graph/_graph_definition.pxd b/cuda_core/cuda/core/graph/_graph_definition.pxd index 634c2ba2580..ea64be805f8 100644 --- a/cuda_core/cuda/core/graph/_graph_definition.pxd +++ b/cuda_core/cuda/core/graph/_graph_definition.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport GraphHandle, as_intptr +from cuda.core._rt cimport GraphHandle, as_intptr cdef class GraphCondition: diff --git a/cuda_core/cuda/core/graph/_graph_definition.pyx b/cuda_core/cuda/core/graph/_graph_definition.pyx index c1a3999bc08..1c02255ba7c 100644 --- a/cuda_core/cuda/core/graph/_graph_definition.pyx +++ b/cuda_core/cuda/core/graph/_graph_definition.pyx @@ -14,7 +14,7 @@ from libcpp.vector cimport vector from cuda.bindings cimport cydriver from cuda.core.graph._graph_node cimport GraphNode -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( GraphHandle, as_cu, as_intptr, diff --git a/cuda_core/cuda/core/graph/_graph_node.pxd b/cuda_core/cuda/core/graph/_graph_node.pxd index ef7d1ff0643..b34885fde68 100644 --- a/cuda_core/cuda/core/graph/_graph_node.pxd +++ b/cuda_core/cuda/core/graph/_graph_node.pxd @@ -5,7 +5,7 @@ from libc.stddef cimport size_t from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( GraphHandle, GraphNodeHandle, OpaqueHandle, diff --git a/cuda_core/cuda/core/graph/_graph_node.pyx b/cuda_core/cuda/core/graph/_graph_node.pyx index 7295d786089..085dd616920 100644 --- a/cuda_core/cuda/core/graph/_graph_node.pyx +++ b/cuda_core/cuda/core/graph/_graph_node.pyx @@ -45,7 +45,8 @@ from cuda.core.graph._subclasses cimport ( SwitchNode, WhileNode, ) -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport note_or_report_cuda_error +from cuda.core._rt cimport ( GraphHandle, GraphNodeHandle, OpaqueHandle, @@ -1114,6 +1115,12 @@ cdef inline ChildGraphNode GN_embed(GraphNode self, GraphDefinition child_def): rollback_status = cydriver.cuGraphDestroyNode(new_node) if rollback_status == cydriver.CUDA_SUCCESS: invalidate_child_graph_state(h_graph, new_node) + else: + # The original exception propagates with the failed rollback + # attached as a note (error handling policy). + note_or_report_cuda_error( + b"cuGraphDestroyNode", rollback_status, + b"failed while rolling back a child graph node; the node remains in the graph") raise return _registered(ChildGraphNode._create_with_params( diff --git a/cuda_core/cuda/core/graph/_host_callback.pxd b/cuda_core/cuda/core/graph/_host_callback.pxd index fc77809c846..1724c6fcc2c 100644 --- a/cuda_core/cuda/core/graph/_host_callback.pxd +++ b/cuda_core/cuda/core/graph/_host_callback.pxd @@ -4,7 +4,7 @@ from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport OpaqueHandle +from cuda.core._rt cimport OpaqueHandle cdef bint _is_py_host_trampoline(cydriver.CUhostFn fn) noexcept nogil diff --git a/cuda_core/cuda/core/graph/_host_callback.pyx b/cuda_core/cuda/core/graph/_host_callback.pyx index 4fb48f0d6ec..de974ab257f 100644 --- a/cuda_core/cuda/core/graph/_host_callback.pyx +++ b/cuda_core/cuda/core/graph/_host_callback.pyx @@ -8,7 +8,7 @@ from libc.string cimport memcpy as c_memcpy from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( OpaqueHandle, make_opaque_malloc, make_opaque_py, diff --git a/cuda_core/cuda/core/graph/_subclasses.pxd b/cuda_core/cuda/core/graph/_subclasses.pxd index 7f92eafe7e8..c85f5e3f201 100644 --- a/cuda_core/cuda/core/graph/_subclasses.pxd +++ b/cuda_core/cuda/core/graph/_subclasses.pxd @@ -7,7 +7,7 @@ from libc.stddef cimport size_t from cuda.bindings cimport cydriver from cuda.core.graph._graph_definition cimport GraphCondition from cuda.core.graph._graph_node cimport GraphNode -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( EventHandle, GraphExecHandle, GraphHandle, diff --git a/cuda_core/cuda/core/graph/_subclasses.pyx b/cuda_core/cuda/core/graph/_subclasses.pyx index 2201f7babf0..0f77cabfb39 100644 --- a/cuda_core/cuda/core/graph/_subclasses.pyx +++ b/cuda_core/cuda/core/graph/_subclasses.pyx @@ -29,7 +29,12 @@ from cuda.core.graph._graph_node cimport ( _init_memcpy_params, _resolve_memcpy_operand, ) -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( + ContextHandle, + create_context_handle_ref, + graph_node_set_params, +) +from cuda.core._rt cimport ( EventHandle, GraphExecHandle, GraphHandle, @@ -124,26 +129,25 @@ cdef void _set_definition_node_params( if node == NULL: raise RuntimeError("GraphNode has been destroyed") _require_graph_node_update_support() - cdef cydriver.CUcontext previous_ctx = NULL - cdef bint restore_ctx = False cdef PreparedAttachment prepared + cdef ContextHandle h_update_ctx + cdef cydriver.CUresult status + cdef cydriver.CUresult restore_status = cydriver.CUresult.CUDA_SUCCESS HANDLE_RETURN(graph_prepare_attachment( h_graph, owner0, owner1, &prepared)) if update_ctx != NULL: - with nogil: - HANDLE_RETURN(cydriver.cuCtxGetCurrent(&previous_ctx)) - if previous_ctx != update_ctx: - HANDLE_RETURN(cydriver.cuCtxSetCurrent(update_ctx)) - restore_ctx = True + h_update_ctx = create_context_handle_ref(update_ctx) + with nogil: + status = graph_node_set_params(node, params, h_update_ctx, &restore_status) + HANDLE_RETURN(status) + # The driver node now references the new owners. Publish their attachment + # before raising anything else: an exception here would roll back the + # prepared retention and leave the node pointing at released resources. try: - with nogil: - HANDLE_RETURN(cydriver.cuGraphNodeSetParams(node, params)) + HANDLE_RETURN(graph_commit_attachment(prepared, node)) finally: - if restore_ctx: - with nogil: - HANDLE_RETURN(cydriver.cuCtxSetCurrent(previous_ctx)) - HANDLE_RETURN(graph_commit_attachment(prepared, node)) + HANDLE_RETURN(restore_status) cdef void _set_executable_node_params( diff --git a/cuda_core/cuda/core/texture/_array.pxd b/cuda_core/cuda/core/texture/_array.pxd index ceb64e2401c..48a392c7a91 100644 --- a/cuda_core/cuda/core/texture/_array.pxd +++ b/cuda_core/cuda/core/texture/_array.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport OpaqueArrayHandle +from cuda.core._rt cimport OpaqueArrayHandle cdef class OpaqueArray: diff --git a/cuda_core/cuda/core/texture/_array.pyx b/cuda_core/cuda/core/texture/_array.pyx index e5fc3f6c9e2..b9677077bfd 100644 --- a/cuda_core/cuda/core/texture/_array.pyx +++ b/cuda_core/cuda/core/texture/_array.pyx @@ -11,7 +11,7 @@ from libc.string cimport memset from cuda.bindings cimport cydriver from cuda.core._context cimport Context from cuda.core._memory._buffer cimport Buffer, Buffer_check_open -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( OpaqueArrayHandle, as_cu, as_intptr, diff --git a/cuda_core/cuda/core/texture/_mipmapped_array.pxd b/cuda_core/cuda/core/texture/_mipmapped_array.pxd index 281c7140b05..00233fafa40 100644 --- a/cuda_core/cuda/core/texture/_mipmapped_array.pxd +++ b/cuda_core/cuda/core/texture/_mipmapped_array.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport MipmappedArrayHandle +from cuda.core._rt cimport MipmappedArrayHandle cdef class MipmappedArray: diff --git a/cuda_core/cuda/core/texture/_mipmapped_array.pyx b/cuda_core/cuda/core/texture/_mipmapped_array.pyx index 8d6bf5a2589..2bdf9d92937 100644 --- a/cuda_core/cuda/core/texture/_mipmapped_array.pyx +++ b/cuda_core/cuda/core/texture/_mipmapped_array.pyx @@ -13,7 +13,7 @@ from cuda.core.texture._array import ( _validate_array_shape, _validate_format_channels, ) -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( OpaqueArrayHandle, MipmappedArrayHandle, as_intptr, diff --git a/cuda_core/cuda/core/texture/_surface.pxd b/cuda_core/cuda/core/texture/_surface.pxd index d6702061e79..09e8e961be1 100644 --- a/cuda_core/cuda/core/texture/_surface.pxd +++ b/cuda_core/cuda/core/texture/_surface.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport SurfObjectHandle +from cuda.core._rt cimport SurfObjectHandle cdef class SurfaceObject: diff --git a/cuda_core/cuda/core/texture/_surface.pyx b/cuda_core/cuda/core/texture/_surface.pyx index 790ce048ecd..da1ee6c25b0 100644 --- a/cuda_core/cuda/core/texture/_surface.pyx +++ b/cuda_core/cuda/core/texture/_surface.pyx @@ -9,7 +9,7 @@ from libc.string cimport memset from cuda.bindings cimport cydriver from cuda.core._context cimport Context from cuda.core.texture._array cimport OpaqueArray, OpaqueArray_check_open -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( ContextHandle, SurfObjectHandle, as_cu, diff --git a/cuda_core/cuda/core/texture/_texture.pxd b/cuda_core/cuda/core/texture/_texture.pxd index 6d0871fe848..3a689f593c9 100644 --- a/cuda_core/cuda/core/texture/_texture.pxd +++ b/cuda_core/cuda/core/texture/_texture.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport TexObjectHandle +from cuda.core._rt cimport TexObjectHandle cdef class TextureObject: diff --git a/cuda_core/cuda/core/texture/_texture.pyx b/cuda_core/cuda/core/texture/_texture.pyx index 28ddf2d6aa8..dd6be36c34b 100644 --- a/cuda_core/cuda/core/texture/_texture.pyx +++ b/cuda_core/cuda/core/texture/_texture.pyx @@ -19,7 +19,7 @@ from cuda.core.texture._array import ( from cuda.core._memory._buffer cimport Buffer, Buffer_check_open from cuda.core.texture._mipmapped_array cimport MipmappedArray, MipmappedArray_check_open from cuda.core.texture._mipmapped_array import MipmappedArray as _PyMipmappedArray -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( ContextHandle, TexObjectHandle, as_cu, diff --git a/cuda_core/docs/source/api.rst b/cuda_core/docs/source/api.rst index 5ee34d34f54..9fb6a8eb718 100644 --- a/cuda_core/docs/source/api.rst +++ b/cuda_core/docs/source/api.rst @@ -312,6 +312,21 @@ The associated enumerations — alongside the other ``cuda.core`` enumerations. +Errors and warnings +------------------- + +Failed CUDA calls raise exceptions; see :doc:`error_handling` for the +guarantees an exception provides and for the situations in which a failure is +reported as a warning instead. + +.. currentmodule:: cuda.core + +.. autosummary:: + :toctree: generated/ + + CUDAWarning + + CUDA process checkpointing -------------------------- diff --git a/cuda_core/docs/source/error_handling.rst b/cuda_core/docs/source/error_handling.rst new file mode 100644 index 00000000000..ad37c76422e --- /dev/null +++ b/cuda_core/docs/source/error_handling.rst @@ -0,0 +1,133 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +.. currentmodule:: cuda.core + +Error Handling +============== + +``cuda.core`` reports failures with Python exceptions. This page describes what +an exception from ``cuda.core`` guarantees about the state it leaves behind, +what happens when a failure occurs where no exception can be raised, and the +few situations in which ``cuda.core`` cannot fully undo a failed operation. + +Exceptions +---------- + +A CUDA driver, runtime, NVRTC, NVVM or nvJitLink call that fails raises an +exception (``CUDAError`` for driver and runtime failures) whose message contains +the CUDA error name and its description. Invalid arguments and misuse raise the +usual Python exception types (``TypeError``, ``ValueError``, ``RuntimeError``). + +When a ``cuda.core`` call raises, the following hold: + +- A call that creates a resource creates nothing. If a later step of the call + fails after the resource was created, the resource is destroyed before the + exception propagates. +- The calling thread's current CUDA context is the one that was current when + the call began. The only method that changes the current context on purpose + is :meth:`Device.set_current`; every other method that must run in a + different context restores the caller's context before returning, whether it + succeeds or fails. See `Context restoration failures`_ for the one case in + which the driver refuses to restore it. +- Objects that were modified by a call that failed midway remain usable and + consistent, but some operations do not have an all-or-nothing outcome. Their + documentation says so where it applies (for example the graph mutation + methods that add several driver edges). + +``cuda.core`` does not swallow driver errors. When a second failure occurs +while an exception is being raised, for example the caller's context cannot be +restored after a failed call, or the rollback of a partially built graph node +fails, the second failure is attached to the exception as a note +(:meth:`BaseException.add_note`), which appears in the traceback and in +``__notes__``. Python 3.10 has no exception notes; there the information is +appended to the message when ``cuda.core`` constructs the exception, and +reported as described in the next section otherwise. + +Failures that cannot be raised +------------------------------ + +Some ``cuda.core`` code runs where no Python exception can propagate: + +- resources released by the garbage collector or by the deferred cleanup of + CUDA graphs, and the CUDA driver calls those releases make, including the + context switch and restoration around such a release; +- callbacks invoked by CUDA. + +A CUDA error in one of these places is reported as a :class:`CUDAWarning`. The +message names the failed driver call and the CUDA error. The warning means the +affected resource may have leaked; ``cuda.core`` never leaves a resource in use +by CUDA with its memory released (it prefers a leak to a dangling pointer). + +:class:`CUDAWarning` derives from :class:`RuntimeWarning`, so it is shown by +default and can be filtered like any other warning. To make these failures +loud in a test suite:: + + import warnings + import cuda.core + + warnings.filterwarnings("error", category=cuda.core.CUDAWarning) + +Because the report comes from a destructor or callback, an escalated warning is +delivered through :func:`sys.unraisablehook` rather than raised into user code. +pytest reports it as ``PytestUnraisableExceptionWarning``, which its +``-W error`` option turns into a test failure. + +``CUDA_ERROR_DEINITIALIZED`` is not reported. It means the CUDA driver is +shutting down, which happens during process exit; cleanup failures at that +point are expected and there is nothing left to clean up. + +Context restoration failures +---------------------------- + +Methods that run in a context other than the current one, such as +:meth:`Device.create_stream` when another device is current, switch the current +context, perform the driver call, and switch back. Restoring the caller's +context can fail only when the driver is shutting down +(``CUDA_ERROR_DEINITIALIZED``), when the caller's context was destroyed in the +meantime (``CUDA_ERROR_INVALID_CONTEXT``), or when the driver is reporting an +earlier, unrecoverable kernel fault (see `Sticky errors`_). None of these can +be fixed by retrying, so ``cuda.core`` does not retry. + +When restoration fails in an ordinary call, the resource created by the call is +destroyed and a ``CUDAError`` is raised for the failed ``cuCtxSetCurrent``, +with a note stating that the caller's context could not be restored and which +context is now current. If the call itself failed as well, its own error is +raised and the restoration failure is the note. Call +:meth:`Device.set_current` before issuing further CUDA work on that thread. + +When restoration fails inside a destructor or callback, a :class:`CUDAWarning` +is issued and the thread keeps the context that the cleanup used. + +Sticky errors +------------- + +Some CUDA errors mark the process as unusable for further CUDA work, for +example ``CUDA_ERROR_ILLEGAL_ADDRESS`` or ``CUDA_ERROR_LAUNCH_FAILED`` after a +kernel fault. The CUDA documentation calls for the process to be terminated and +relaunched after such an error, and every later CUDA call returns the same +error. Because these faults are detected asynchronously, the call that first +raises the error is often unrelated to the kernel that caused it. + +``cuda.core`` raises these errors like any other and does not attempt to +recover from them. It does not terminate the process for you: the exception +carries the Python traceback of the call that observed the fault, and your +application decides how to shut down. + +Interpreter shutdown +-------------------- + +Once the interpreter starts finalizing, ``cuda.core`` no longer touches Python +objects from CUDA callbacks or destructors. Resources whose release would +require Python at that point are intentionally leaked; the operating system and +the driver reclaim them when the process exits. Release all ``cuda.core`` +objects explicitly (with ``close()`` or a ``with`` block) if their deterministic +release matters. + +Process termination +------------------- + +``cuda.core`` does not abort the process in response to a CUDA error, including +errors that cannot be raised, and including failures to restore the caller's +context. Aborting is reserved for an internal invariant violation where +continuing could corrupt memory. diff --git a/cuda_core/docs/source/index.rst b/cuda_core/docs/source/index.rst index 34c0933ffb8..373e6b61665 100644 --- a/cuda_core/docs/source/index.rst +++ b/cuda_core/docs/source/index.rst @@ -16,6 +16,7 @@ Welcome to the documentation for ``cuda.core``. examples interoperability concurrency + error_handling api api_nvml environment_variables diff --git a/cuda_core/docs/source/release/1.3.0-notes.rst b/cuda_core/docs/source/release/1.3.0-notes.rst index a1432459e58..53a3c778066 100644 --- a/cuda_core/docs/source/release/1.3.0-notes.rst +++ b/cuda_core/docs/source/release/1.3.0-notes.rst @@ -18,6 +18,14 @@ New features completion of the asynchronous prefetch operation. (`#2109 `__) +- Added :class:`CUDAWarning`, the warning category ``cuda.core`` uses for CUDA + errors that cannot be raised, such as a failed driver call while a resource + is released by the garbage collector. Filter on it with + ``warnings.filterwarnings("error", category=cuda.core.CUDAWarning)`` to make + such failures loud. The new :doc:`error handling <../error_handling>` page + documents what an exception from ``cuda.core`` guarantees, how failures that + cannot be raised are reported, and how context restoration failures and + sticky CUDA errors are handled. Fixes and enhancements ---------------------- @@ -48,3 +56,45 @@ Fixes and enhancements ``RuntimeError``. :attr:`Buffer.device_id` on such a buffer returns ``-1`` as well, which also lets a pinned buffer back a linear or pitched texture resource. + +- Cleanup failures are now reported as :class:`CUDAWarning` instead of being + written to ``stderr`` with ``print`` or ``fprintf``, so they can be filtered, + captured with :func:`warnings.catch_warnings`, and escalated. Failures of + ``cuStreamDestroy``, ``cuEventDestroy``, ``cuMemFree``, ``cuMemFreeAsync``, + ``cuMemFreeHost``, ``cuMemPoolDestroy``, ``cuGreenCtxDestroy``, + ``cuGraphDestroy``, ``cuGraphExecDestroy``, ``cuGraphicsUnregisterResource``, + ``cuLinkDestroy``, ``cuArrayDestroy``, ``cuMipmappedArrayDestroy``, + ``cuTexObjectDestroy``, ``cuSurfObjectDestroy``, user-object releases and the + NVRTC, NVVM and nvJitLink destroy calls made from destructors were previously + discarded; they are now reported. ``CUDA_ERROR_DEINITIALIZED`` (the driver is + shutting down) is not reported. Test code that matched the old ``stderr`` + text uses ``pytest.warns(CUDAWarning)`` instead. + +- When a :class:`Device` method has to run in the device's context and the + caller's context cannot be restored afterwards, the created resource is + destroyed and the raised ``CUDAError`` now carries a note (Python 3.11+; + appended to the message on 3.10) stating that the caller's context could not + be restored and which context is current. Previously the error named only + the driver status of the failed ``cuCtxSetCurrent`` call. If the call itself + failed as well, its error is raised and the restoration failure is the note. + A restoration failure during resource cleanup is reported as + :class:`CUDAWarning`. + +- Updating a memcpy or memset graph node whose context differs from the current + one no longer risks a dangling node parameter when the caller's context + cannot be restored after the update: the resources referenced by the new + parameters are now retained before the restoration failure is raised. + +- :meth:`Device.set_current` with an explicit :class:`Context` now switches + contexts with a single driver call, so a failure leaves the previous context + current instead of leaving the thread with no context. It also works when no + context is current, returning ``None``. + +- A failed rollback of a partially embedded child graph node is now attached + as a note to the exception that triggered the rollback (reported as + :class:`CUDAWarning` on Python 3.10), and a failed ``cuStreamEndCapture`` + made when a still-building :class:`~graph.GraphBuilder` is garbage collected + is now reported as :class:`CUDAWarning`; both were silent. + +- :attr:`Stream.device` and related queries on a stream whose context is not + current now restore the caller's context even when the device query fails. diff --git a/cuda_core/pyproject.toml b/cuda_core/pyproject.toml index 3eac3a4c0af..3df8536a32b 100644 --- a/cuda_core/pyproject.toml +++ b/cuda_core/pyproject.toml @@ -108,7 +108,7 @@ include-package-data = false [tool.setuptools.package-data] "*" = ["*.pxd", "*.pyi", "py.typed"] "cuda.core._include" = ["*.h", "*.hpp"] -"cuda.core._cpp" = ["*.h", "*.hpp"] +"cuda.core._cpp" = ["**/*.h", "**/*.hpp"] [tool.setuptools.dynamic] readme = { file = ["DESCRIPTION.rst"], content-type = "text/x-rst" } diff --git a/cuda_core/setup.py b/cuda_core/setup.py index c66050fe50a..b19a99c8d2c 100644 --- a/cuda_core/setup.py +++ b/cuda_core/setup.py @@ -2,7 +2,10 @@ # # SPDX-License-Identifier: Apache-2.0 +import contextlib import os +from concurrent.futures import ThreadPoolExecutor +from distutils.ccompiler import CCompiler from pathlib import Path import build_hooks # our build backend @@ -77,10 +80,62 @@ def _configure_windows_tensor_bridge(self): raise RuntimeError(f"Failed to find extension {_TENSOR_BRIDGE_EXT_NAME!r} for Windows build.") + @contextlib.contextmanager + def _parallel_source_compilation(self): + """Compile the sources of every extension through one shared thread pool. + + setuptools runs extensions in parallel (self.parallel) but compiles the + sources of one extension serially, so a multi-source extension such as + cuda.core._rt (a dozen .cpp files) becomes the critical path. This + mirrors CCompiler.compile() and fans its per-object _compile() calls out + to a pool shared by all extensions, so at most `nthreads` compiler + processes run at once. It applies only to compilers that still use + CCompiler.compile(), which drives the per-object _compile() hook (the + Unix family); MSVC overrides compile() wholesale and keeps the stock path. + """ + compiler = self.compiler + if nthreads <= 1 or type(compiler).compile is not CCompiler.compile: + yield + return + stock_compile = compiler.compile + with ThreadPoolExecutor(max_workers=nthreads) as pool: + + def compile( + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None, + ): + macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile( + output_dir, macros, include_dirs, sources, depends, extra_postargs + ) + cc_args = compiler._get_cc_args(pp_opts, debug, extra_preargs) + + def compile_one(obj): + try: + src, ext = build[obj] + except KeyError: + return # up to date + compiler._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + list(pool.map(compile_one, objects)) # re-raises the first failure + return objects + + compiler.compile = compile + try: + yield + finally: + compiler.compile = stock_compile + def build_extensions(self): self.parallel = nthreads self._configure_windows_tensor_bridge() - super().build_extensions() + with self._parallel_source_compilation(): + super().build_extensions() build_hooks.record_build_major() diff --git a/cuda_core/tests/helpers/contexts.py b/cuda_core/tests/helpers/contexts.py index 7ee01bb255f..f613f0ac336 100644 --- a/cuda_core/tests/helpers/contexts.py +++ b/cuda_core/tests/helpers/contexts.py @@ -1,18 +1,36 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import warnings from contextlib import contextmanager +from cuda.core import CUDAWarning from cuda.core._utils.cuda_utils import driver, handle_return __all__ = [ "assert_device_operations_use_bound_context", + "assert_no_cuda_warning", "current_context_handle", "no_current_context", "use_context", ] +@contextmanager +def assert_no_cuda_warning(): + """Fail if a :class:`CUDAWarning` is issued inside the block. + + Cleanup paths cannot raise, so a driver failure there surfaces only as a + warning; this makes such a failure a test failure. Tests using it must be + marked ``thread_unsafe``: warning capture is process-global. + """ + with warnings.catch_warnings(record=True) as records: + warnings.simplefilter("always", CUDAWarning) + yield + cuda_warnings = [str(record.message) for record in records if issubclass(record.category, CUDAWarning)] + assert not cuda_warnings, f"unexpected CUDAWarning(s): {cuda_warnings}" + + def current_context_handle(): """Return the current CUDA context handle, or zero if none is current.""" return int(handle_return(driver.cuCtxGetCurrent())) diff --git a/cuda_core/tests/test_build_hooks.py b/cuda_core/tests/test_build_hooks.py index 2f1b3211781..27c057e0297 100644 --- a/cuda_core/tests/test_build_hooks.py +++ b/cuda_core/tests/test_build_hooks.py @@ -21,6 +21,8 @@ import os import sys import tempfile +import threading +from distutils.ccompiler import CCompiler from pathlib import Path from unittest import mock @@ -325,3 +327,127 @@ def test_flag_set_forces_rebuild(self, monkeypatch): def test_flag_clear_leaves_default(self, monkeypatch): assert not self._finalized_build_ext(False, monkeypatch).force + + +class TestExtensionSources: + """_extension_sources: a directory of .cpp files, a single legacy .cpp, or nothing.""" + + @pytest.fixture + def tree(self, tmp_path, monkeypatch): + core = tmp_path / "cuda" / "core" + cpp = core / "_cpp" + (cpp / "a" / "nested").mkdir(parents=True) + (cpp / "d").mkdir() + for name in ("_a.pyx", "_b.pyx", "_c.pyx", "_d.pyx"): + (core / name).write_text("") + for name in ("a/x.cpp", "a/y.cpp", "a/nested/z.cpp", "a/notes.md", "b.cpp"): + (cpp / name).write_text("") + monkeypatch.chdir(tmp_path) + + @pytest.mark.agent_authored(model="claude-fable-5-1") + def test_directory_of_sources(self, tree): + a = os.path.join("cuda", "core", "_cpp", "a") + assert build_hooks._extension_sources("_a") == [ + "cuda/core/_a.pyx", + os.path.join(a, "nested", "z.cpp"), + os.path.join(a, "x.cpp"), + os.path.join(a, "y.cpp"), + ] + + @pytest.mark.agent_authored(model="claude-fable-5-1") + def test_legacy_single_file_and_no_cpp(self, tree): + assert build_hooks._extension_sources("_b") == [ + "cuda/core/_b.pyx", + os.path.join("cuda", "core", "_cpp", "b.cpp"), + ] + assert build_hooks._extension_sources("_c") == ["cuda/core/_c.pyx"] + + @pytest.mark.agent_authored(model="claude-fable-5-1") + def test_empty_directory_is_an_error(self, tree): + with pytest.raises(RuntimeError, match="no .cpp files"): + build_hooks._extension_sources("_d") + + +class TestExtensionDepends: + """_extension_depends: every header under a directory-form module's + _cpp//, the same list for every extension (see its docstring).""" + + @pytest.mark.agent_authored(model="claude-fable-5-1") + def test_headers_under_module_directories_only(self, tmp_path, monkeypatch): + cpp = tmp_path / "cuda" / "core" / "_cpp" + (cpp / "a" / "nested").mkdir(parents=True) + for name in ("a/x.hpp", "a/nested/y.h", "a/z.cpp", "a/notes.md", "top.hpp", "b.cpp"): + (cpp / name).write_text("") + monkeypatch.chdir(tmp_path) + a = os.path.join("cuda", "core", "_cpp", "a") + assert build_hooks._extension_depends() == [os.path.join(a, "nested", "y.h"), os.path.join(a, "x.hpp")] + + +class TestParallelSourceCompilation: + """setup.py compiles an extension's sources through one shared thread pool.""" + + class FakeCompiler(CCompiler): + """Uses the stock CCompiler.compile(), like the Unix compilers.""" + + executables = {} + + def __init__(self, fail_on=None): + super().__init__() + self.compiled = [] + self.fail_on = fail_on + self.lock = threading.Lock() + + def _setup_compile(self, outdir, macros, incdirs, sources, depends, extra): + extra = [] if extra is None else extra # as distutils does + objects = [source + ".o" for source in sources] + return macros, objects, extra, ["-Dpp"], {obj: (src, ".cpp") for obj, src in zip(objects, sources)} + + def _get_cc_args(self, pp_opts, debug, before): + return ["-c", *pp_opts] + + def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): + if src == self.fail_on: + raise RuntimeError(f"{src} failed") + with self.lock: + self.compiled.append((obj, src, ext, tuple(cc_args), tuple(extra_postargs), tuple(pp_opts))) + + class MsvcLikeCompiler(FakeCompiler): + """Overrides compile() wholesale, like MSVCCompiler.""" + + def compile(self, *args, **kwargs): + return "stock" + + def _build_ext(self, monkeypatch, nthreads, compiler): + from setuptools.dist import Distribution + + setup_py = _load_setup_py(monkeypatch) + monkeypatch.setattr(setup_py, "nthreads", nthreads) + cmd = setup_py.build_ext(Distribution({"name": "cuda-core", "version": "0"})) + cmd.compiler = compiler + return cmd + + @pytest.mark.agent_authored(model="claude-fable-5-1") + def test_every_source_compiles_once_and_the_object_order_is_kept(self, monkeypatch): + cmd = self._build_ext(monkeypatch, 4, self.FakeCompiler()) + sources = [f"rt/{name}.cpp" for name in "abcdef"] + with cmd._parallel_source_compilation(): + objects = cmd.compiler.compile(sources, output_dir="tmp", extra_postargs=["-O2"], depends=["x.hpp"]) + assert objects == [source + ".o" for source in sources] + assert sorted(entry[0] for entry in cmd.compiler.compiled) == sorted(objects) + assert {entry[2:] for entry in cmd.compiler.compiled} == {(".cpp", ("-c", "-Dpp"), ("-O2",), ("-Dpp",))} + assert cmd.compiler.compile.__func__ is CCompiler.compile # restored on exit + + @pytest.mark.agent_authored(model="claude-fable-5-1") + def test_a_failing_source_fails_the_extension(self, monkeypatch): + cmd = self._build_ext(monkeypatch, 4, self.FakeCompiler(fail_on="rt/c.cpp")) + with cmd._parallel_source_compilation(), pytest.raises(RuntimeError, match="rt/c.cpp failed"): + cmd.compiler.compile([f"rt/{name}.cpp" for name in "abcdef"]) + + @pytest.mark.agent_authored(model="claude-fable-5-1") + def test_serial_builds_and_compilers_without_the_hook_keep_the_stock_path(self, monkeypatch): + cmd = self._build_ext(monkeypatch, 1, self.FakeCompiler()) + with cmd._parallel_source_compilation(): + assert cmd.compiler.compile.__func__ is CCompiler.compile + cmd = self._build_ext(monkeypatch, 4, self.MsvcLikeCompiler()) + with cmd._parallel_source_compilation(): + assert cmd.compiler.compile(["a.cpp"]) == "stock" diff --git a/cuda_core/tests/test_error_handling.py b/cuda_core/tests/test_error_handling.py new file mode 100644 index 00000000000..44e8331f597 --- /dev/null +++ b/cuda_core/tests/test_error_handling.py @@ -0,0 +1,282 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the error handling policy (docs/source/error_handling.rst). + +Ordinary calls raise and leave the caller's context untouched; failures that +cannot be raised are reported as CUDAWarning; a failure to restore the caller's +context is raised (or reported) with an explanation rather than swallowed or +turned into a process abort; and a secondary failure that occurs while an +exception is being raised is attached to that exception as a note. Restoration +failures are injected with the handle layer's test hook, which leaves the target +context current exactly as a real ``cuCtxSetCurrent`` failure would, so every +test here restores the context stack itself. +""" + +import ctypes +import sys +from contextlib import contextmanager + +import pytest +from helpers.constants import POOL_SIZE +from helpers.contexts import assert_no_cuda_warning, current_context_handle + +import cuda.core +from cuda.core import ( + CUDAWarning, + DeviceMemoryResource, + DeviceMemoryResourceOptions, + LegacyPinnedMemoryResource, +) +from cuda.core._memory._synchronous_memory_resource import _SynchronousMemoryResource +from cuda.core._rt import ( + _note_or_report_cuda_error_for_testing, + _set_context_restore_fault_for_testing, +) +from cuda.core._stream import default_stream +from cuda.core._utils.cuda_utils import CUDAError, driver, handle_return +from cuda.core._utils.version import binding_version, driver_version +from cuda.core.graph import GraphDefinition + +INVALID_CONTEXT = int(driver.CUresult.CUDA_ERROR_INVALID_CONTEXT) +INVALID_VALUE = int(driver.CUresult.CUDA_ERROR_INVALID_VALUE) +DEINITIALIZED = int(driver.CUresult.CUDA_ERROR_DEINITIALIZED) + +# PEP 678 exception notes; on 3.10 the same information lands in the message. +HAS_NOTES = sys.version_info >= (3, 11) + +thread_unsafe_context_fault = pytest.mark.thread_unsafe( + reason="injects a thread-local restoration fault and mutates the CUDA context stack" +) +thread_unsafe_warning_capture = pytest.mark.thread_unsafe(reason="warning capture is process-global") + + +def error_text(exc): + """The message plus any notes, wherever the detail lives on this interpreter.""" + return "\n".join([str(exc), *getattr(exc, "__notes__", [])]) + + +@contextmanager +def no_context_with_restore_fault(status=INVALID_CONTEXT): + """Pop the current context and make the next restoration fail with ``status``. + + On exit, drop whatever the failed restoration left current, clear an unused + fault, and push the popped context back. + """ + previous = handle_return(driver.cuCtxPopCurrent()) + assert current_context_handle() == 0 + _set_context_restore_fault_for_testing(status) + try: + yield + finally: + _set_context_restore_fault_for_testing(0) + handle_return(driver.cuCtxSetCurrent(driver.CUcontext(0))) + handle_return(driver.cuCtxPushCurrent(previous)) + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_cudawarning_is_public_and_shown_by_default(): + assert "CUDAWarning" in cuda.core.__all__ + assert issubclass(CUDAWarning, RuntimeWarning) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_create_stream_raises_when_context_cannot_be_restored(init_cuda): + """Creation is undone and the error explains the context state; no abort, no warning.""" + dev = init_cuda + with no_context_with_restore_fault(): + with assert_no_cuda_warning(), pytest.raises(CUDAError) as excinfo: + dev.create_stream() + text = error_text(excinfo.value) + assert "could not be restored" in text + assert "CUDA_ERROR_INVALID_CONTEXT" in text + assert "Device.set_current()" in text + if HAS_NOTES: + # The explanation is a note, separable from the driver error message. + assert "could not be restored" not in str(excinfo.value) + assert any("could not be restored" in note for note in excinfo.value.__notes__) + # As documented, a failed restoration leaves the device's context current. + assert current_context_handle() == int(dev.context.handle) + assert current_context_handle() == int(dev.context.handle) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_sync_raises_when_context_cannot_be_restored(init_cuda): + """A context-scoped call without a created resource raises the same explanation.""" + dev = init_cuda + with no_context_with_restore_fault(): + with pytest.raises(CUDAError) as excinfo: + dev.sync() + assert "could not be restored" in error_text(excinfo.value) + assert current_context_handle() == int(dev.context.handle) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_failed_call_raises_its_own_error_with_the_restore_failure_attached(init_cuda): + """When the call and the restoration both fail, the call's error is raised and the + restoration failure is attached to it; nothing is reported out of band.""" + dev = init_cuda + mr = _SynchronousMemoryResource(dev.device_id) + with no_context_with_restore_fault(): + with assert_no_cuda_warning(), pytest.raises(CUDAError) as excinfo: + mr.allocate(1 << 62) + message = str(excinfo.value) + text = error_text(excinfo.value) + # The allocation failure is the primary error, not the restoration failure. + assert not message.startswith("CUDA_ERROR_INVALID_CONTEXT") + assert "could not be restored after this failure" in text + assert "cuCtxSetCurrent: CUDA_ERROR_INVALID_CONTEXT" in text + assert "Device.set_current()" in text + assert current_context_handle() == int(dev.context.handle) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_unused_restore_fault_does_not_fire_without_a_context_switch(init_cuda): + """The hook only affects restorations; a call in the current context never restores.""" + dev = init_cuda + _set_context_restore_fault_for_testing(INVALID_CONTEXT) + try: + stream = dev.create_stream() + stream.close() + finally: + _set_context_restore_fault_for_testing(0) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_cleanup_reports_restore_failure_as_warning(mempool_device): + """A restoration failure inside a destructor cannot raise, so it is reported.""" + dev = mempool_device + mr = DeviceMemoryResource(dev, DeviceMemoryResourceOptions(max_size=POOL_SIZE)) + # A default-stream deallocation records the allocating context, so freeing + # with no context current switches to it and must switch back. + buf = mr.allocate(256, stream=default_stream()) + with no_context_with_restore_fault(): + with pytest.warns(CUDAWarning, match="restoring the caller's context") as records: + buf.close() + assert any("CUDA_ERROR_INVALID_CONTEXT" in str(record.message) for record in records) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_escalated_cudawarning_from_cleanup_is_not_a_crash(mempool_device): + """With CUDAWarning promoted to an error, a destructor-path report cannot be raised. + + It is delivered through sys.unraisablehook instead; the process continues and + the resource release still runs. pytest surfaces the hook as a warning, so the + hook is replaced here to keep the test's own outcome deterministic. + """ + import sys + import warnings + + dev = mempool_device + mr = DeviceMemoryResource(dev, DeviceMemoryResourceOptions(max_size=POOL_SIZE)) + buf = mr.allocate(256, stream=default_stream()) + unraisable = [] + previous_hook = sys.unraisablehook + sys.unraisablehook = unraisable.append + try: + with no_context_with_restore_fault(), warnings.catch_warnings(): + warnings.simplefilter("error", CUDAWarning) + buf.close() + finally: + sys.unraisablehook = previous_hook + assert len(unraisable) == 1 + assert issubclass(unraisable[0].exc_type, CUDAWarning) + assert "restoring the caller's context" in str(unraisable[0].exc_value) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_set_current_with_context_works_without_a_current_context(init_cuda): + """set_current(ctx) binds in one driver call; no previous context means None.""" + dev = init_cuda + ctx = dev.context + previous = handle_return(driver.cuCtxPopCurrent()) + try: + assert current_context_handle() == 0 + assert dev.set_current(ctx) is None + assert current_context_handle() == int(ctx.handle) + finally: + # Leave exactly one context on the stack, as the fixture expects. + handle_return(driver.cuCtxSetCurrent(driver.CUcontext(0))) + handle_return(driver.cuCtxPushCurrent(previous)) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_memset_update_keeps_new_owners_alive_when_context_cannot_be_restored(device_x2): + """The node's new parameters stay valid: the attachment is published before the + restoration failure is raised, so the updated graph instantiates and runs.""" + if driver_version() < (13, 2, 0) or binding_version() < (13, 2, 0): + pytest.skip("node contexts are only recorded by cuGraphNodeGetParams on CUDA 13.2+") + node_dev, other_dev = device_x2 + node_dev.set_current() + memory_resource = LegacyPinnedMemoryResource() + dst = memory_resource.allocate(4) + replacement = memory_resource.allocate(4) + graph_def = GraphDefinition() + node = graph_def.memset(dst, 0x11, 4) + + # Updating from another device's context switches to the node's context and + # must switch back; make that restoration fail. + other_dev.set_current() + _set_context_restore_fault_for_testing(INVALID_CONTEXT) + try: + with pytest.raises(CUDAError) as excinfo: + node.update(dst=replacement, value=0x22) + assert "could not be restored" in error_text(excinfo.value) + finally: + _set_context_restore_fault_for_testing(0) + node_dev.set_current() + + def as_bytes(buffer): + return (ctypes.c_uint8 * 4).from_address(int(buffer.handle)) + + as_bytes(dst)[:] = [0] * 4 + as_bytes(replacement)[:] = [0] * 4 + graph = graph_def.instantiate() + stream = node_dev.create_stream() + graph.launch(stream) + stream.sync() + # The driver applied the update, and the replacement buffer it references + # is still retained by the graph rather than dangling. + assert list(as_bytes(replacement)) == [0x22] * 4 + assert list(as_bytes(dst)) == [0] * 4 + graph.close() + stream.close() + + +@thread_unsafe_warning_capture +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_rollback_failure_is_attached_to_the_propagating_exception(): + """A failed rollback inside an except block becomes a note on the exception being + handled (Python 3.11+); on 3.10, or with no exception being handled, it is reported + as a CUDAWarning. CUDA_ERROR_DEINITIALIZED is neither attached nor reported.""" + with pytest.raises(RuntimeError) as excinfo: + try: + raise RuntimeError("primary failure") + except RuntimeError: + if HAS_NOTES: + with assert_no_cuda_warning(): + _note_or_report_cuda_error_for_testing(INVALID_VALUE) + else: + with pytest.warns(CUDAWarning, match="cuTestOperation failed while testing"): + _note_or_report_cuda_error_for_testing(INVALID_VALUE) + with assert_no_cuda_warning(): + _note_or_report_cuda_error_for_testing(DEINITIALIZED) + raise + exc = excinfo.value + assert str(exc) == "primary failure" + if HAS_NOTES: + assert len(exc.__notes__) == 1 + assert "cuTestOperation failed while testing: CUDA_ERROR_INVALID_VALUE" in exc.__notes__[0] + else: + assert not hasattr(exc, "__notes__") + # With no exception being handled there is nothing to attach to. + with pytest.warns(CUDAWarning, match="cuTestOperation failed while testing"): + _note_or_report_cuda_error_for_testing(INVALID_VALUE) diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index befc3031e90..a4a026e7dca 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -27,7 +27,7 @@ ) from helpers.child_processes import child_timeout_sec, kill_subprocesses from helpers.constants import POOL_SIZE -from helpers.contexts import current_context_handle, no_current_context +from helpers.contexts import assert_no_cuda_warning, current_context_handle, no_current_context from helpers.memory import ( create_managed_memory_resource_or_skip, create_pinned_memory_resource_or_xfail, @@ -37,6 +37,7 @@ from cuda.core import ( Buffer, + CUDAWarning, Device, DeviceMemoryResource, DeviceMemoryResourceOptions, @@ -824,9 +825,10 @@ def test_from_handle_mr_explicit_stream_without_current_context(buffer_type): ] +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="claude-fable-5-1") @pytest.mark.parametrize("mr_cls", _HOST_ONLY_MRS) -def test_from_handle_host_only_mr_without_current_context(mr_cls, capfd): +def test_from_handle_host_only_mr_without_current_context(mr_cls): """Host-only memory needs no current context to create or free a Buffer.""" device = Device() device.set_current() @@ -836,28 +838,30 @@ def test_from_handle_host_only_mr_without_current_context(mr_cls, capfd): assert int(previous) != 0 try: assert int(handle_return(driver.cuCtxGetCurrent())) == 0 - buf = mr.allocate(64) - assert buf.is_host_accessible - buf.close() + with assert_no_cuda_warning(): + buf = mr.allocate(64) + assert buf.is_host_accessible + buf.close() assert int(handle_return(driver.cuCtxGetCurrent())) == 0 finally: handle_return(driver.cuCtxSetCurrent(previous)) - assert "Warning" not in capfd.readouterr().err - def _host_only_child_main(mr_cls): """Allocate and free host-only memory in a process that never initialized CUDA.""" - buf = mr_cls().allocate(64) - assert buf.is_host_accessible - buf.close() + # Warnings do not cross processes: check for a CUDAWarning here, where a + # failed assertion becomes a non-zero exit code for the parent to see. + with assert_no_cuda_warning(): + buf = mr_cls().allocate(64) + assert buf.is_host_accessible + buf.close() err, _ = driver.cuCtxGetCurrent() assert err == driver.CUresult.CUDA_ERROR_NOT_INITIALIZED, err @pytest.mark.agent_authored(model="claude-fable-5-1") @pytest.mark.parametrize("mr_cls", _HOST_ONLY_MRS) -def test_from_handle_host_only_mr_without_cuda_init(mr_cls, capfd): +def test_from_handle_host_only_mr_without_cuda_init(mr_cls): """Host-only buffers work in a spawned process that never initializes CUDA.""" process = mp.Process(target=_host_only_child_main, args=(mr_cls,)) process.start() @@ -865,26 +869,27 @@ def test_from_handle_host_only_mr_without_cuda_init(mr_cls, capfd): survivors = kill_subprocesses(process) assert not survivors, "child did not exit within timeout" assert process.exitcode == 0, f"child exited with {process.exitcode}" - assert "Warning" not in capfd.readouterr().err +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="gpt-5.6") -def test_mr_deallocation_failure_warns(capfd): - """Destructor-path MR failures are contained and reported.""" +def test_mr_deallocation_failure_warns(): + """Destructor-path MR failures are contained and reported as CUDAWarning.""" device = Device() device.set_current() FailingMR, _ = make_instrumented_memory_resource(deallocate_error=RuntimeError("expected deallocation failure")) buf = Buffer.from_handle(1, 1024, mr=FailingMR(device)) - buf.close() - assert ( - "Warning: mr.deallocate() failed during Buffer destruction: expected deallocation failure" - ) in capfd.readouterr().err + with pytest.warns( + CUDAWarning, match=r"mr\.deallocate\(\) failed during Buffer destruction.*expected deallocation failure" + ): + buf.close() +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="cursor-grok-4.5") @pytest.mark.parametrize("replace_stream", [False, True]) -def test_mr_deallocation_without_current_context(init_cuda, capsys, replace_stream): +def test_mr_deallocation_without_current_context(init_cuda, replace_stream): """MR-backed Buffer teardown activates the recorded context when none is current.""" TrackingMR, telemetry = make_instrumented_memory_resource(DummyDeviceMemoryResource, track_active=True) mr = TrackingMR(init_cuda) @@ -895,16 +900,17 @@ def test_mr_deallocation_without_current_context(init_cuda, capsys, replace_stre with no_current_context(): assert current_context_handle() == 0 - buf.close(stream) + with assert_no_cuda_warning(): + buf.close(stream) assert len(telemetry["active"]) == 0 assert current_context_handle() == 0 - assert "mr.deallocate() failed" not in capsys.readouterr().err +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="cursor-grok-4.5") @pytest.mark.parametrize("replace_stream", [False, True]) -def test_mr_deallocation_with_foreign_context(device_x2, capsys, replace_stream): +def test_mr_deallocation_with_foreign_context(device_x2, replace_stream): """MR-backed Buffer teardown switches away from an unrelated current context.""" alloc_dev, foreign_dev = device_x2 alloc_dev.set_current() @@ -921,11 +927,11 @@ def test_mr_deallocation_with_foreign_context(device_x2, capsys, replace_stream) assert foreign_ctx != alloc_ctx try: - buf.close(stream) + with assert_no_cuda_warning(): + buf.close(stream) assert len(telemetry["active"]) == 0 assert current_context_handle() == foreign_ctx - assert "mr.deallocate() failed" not in capsys.readouterr().err finally: alloc_dev.set_current() @@ -945,8 +951,9 @@ def test_mr_deallocate_raises_on_driver_error(mempool_device): mr.deallocate(0xDEADBEEF, 256, stream=stream) +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="cursor-grok-4.5") -def test_pool_buffer_deallocates_without_current_context(mempool_device, capfd): +def test_pool_buffer_deallocates_without_current_context(mempool_device): """Pool Buffer.close frees on the recorded stream with no current context.""" dev = mempool_device stream = dev.create_stream() @@ -959,18 +966,17 @@ def test_pool_buffer_deallocates_without_current_context(mempool_device, capfd): with no_current_context(): assert current_context_handle() == 0 - buf.close() + with assert_no_cuda_warning(): + buf.close() stream.sync() assert mr.attributes.used_mem_current < used_after_alloc assert current_context_handle() == 0 - err = capfd.readouterr().err - assert "cuMemFreeAsync failed" not in err - assert "mr.deallocate() failed" not in err +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="cursor-grok-4.5") -def test_pool_buffer_deallocates_with_foreign_context(mempool_device_x2, capfd): +def test_pool_buffer_deallocates_with_foreign_context(mempool_device_x2): """Pool Buffer.close frees under the recorded context while another is current.""" alloc_dev, foreign_dev = mempool_device_x2 alloc_dev.set_current() @@ -988,7 +994,8 @@ def test_pool_buffer_deallocates_with_foreign_context(mempool_device_x2, capfd): assert foreign_ctx != alloc_ctx try: - buf.close() + with assert_no_cuda_warning(): + buf.close() assert current_context_handle() == foreign_ctx # Observe the free on the allocation device, then restore the foreign context. @@ -996,9 +1003,6 @@ def test_pool_buffer_deallocates_with_foreign_context(mempool_device_x2, capfd): stream.sync() assert mr.attributes.used_mem_current < used_after_alloc foreign_dev.set_current() - - err = capfd.readouterr().err - assert "cuMemFreeAsync failed" not in err finally: alloc_dev.set_current() @@ -2354,8 +2358,9 @@ def test_synchronous_memory_resource_restores_context_after_failure(device_x2): assert current_context_handle() == current_context +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="claude-sonnet-5") -def test_synchronous_memory_resource_default_stream_deallocates_in_own_context(device_x2, capsys): +def test_synchronous_memory_resource_default_stream_deallocates_in_own_context(device_x2): """Buffer teardown with no explicit stream frees in the resource's own context, not whatever context happens to be current at close() time.""" from cuda.core._memory._synchronous_memory_resource import _SynchronousMemoryResource @@ -2370,13 +2375,14 @@ def test_synchronous_memory_resource_default_stream_deallocates_in_own_context(d buf = mr.allocate(64) # no explicit stream: records a context-bound default token assert current_context_handle() == current_context - buf.close() # no explicit stream: reuses the recorded token + with assert_no_cuda_warning(): + buf.close() # no explicit stream: reuses the recorded token assert current_context_handle() == current_context - assert capsys.readouterr().err == "" +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="claude-sonnet-5") -def test_synchronous_memory_resource_allocate_without_current_context(device_x2, capsys): +def test_synchronous_memory_resource_allocate_without_current_context(device_x2): """allocate()/close() with no explicit stream succeed with no context current, instead of raising or leaking the allocation (#2311).""" from cuda.core._memory._synchronous_memory_resource import _SynchronousMemoryResource @@ -2386,14 +2392,12 @@ def test_synchronous_memory_resource_allocate_without_current_context(device_x2, mr = _SynchronousMemoryResource(alloc_dev.device_id, alloc_dev.context) current_dev.set_current() - with no_current_context(): + with no_current_context(), assert_no_cuda_warning(): buf = mr.allocate(64) assert current_context_handle() == 0 buf.close() assert current_context_handle() == 0 - assert capsys.readouterr().err == "" - @pytest.mark.parametrize( ("method", "spec", "match"), diff --git a/cuda_core/tests/test_rt_layout.py b/cuda_core/tests/test_rt_layout.py new file mode 100644 index 00000000000..53158370968 --- /dev/null +++ b/cuda_core/tests/test_rt_layout.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Layout rules of cuda/core/_cpp/rt/, the C++ behind cuda.core._rt. + +Source-tree properties only: no compiler, no GPU. Run with --noconftest, since +conftest.py imports the compiled package: + + pytest tests/test_rt_layout.py -v --noconftest + +Why the rules exist: _rt.pxd names handles.hpp, and Cython compiles every +cimporting extension against a copy of that header placed in its build +directory, next to copies of the extension's `depends`. So handles.hpp may pull +in only what thirty-odd consumer extensions can safely compile: types, +templates and inline functions, through file-relative includes. Everything with +storage or a body lives behind rt.hpp, which only _rt.pyx names. +""" + +import re +from pathlib import Path + +import pytest + +CORE = Path(__file__).resolve().parent.parent / "cuda" / "core" +RT = CORE / "_cpp" / "rt" +HEADERS = sorted(RT.glob("*.hpp")) +SOURCES = sorted(RT.glob("*.cpp")) +UMBRELLAS = {"rt.hpp", "handles.hpp"} +PYTHON_TOKEN = re.compile(r"\bPy[A-Z_]\w*|\bPyObject\b|Python\.h") +QUOTED_INCLUDE = re.compile(r'^\s*#\s*include\s+"([^"]+)"', re.M) + + +def read(path): + return path.read_text(encoding="utf-8") + + +def quoted_includes(path): + return QUOTED_INCLUDE.findall(read(path)) + + +def include_closure(path): + seen = [] + todo = [path] + while todo: + current = todo.pop() + if current in seen: + continue + seen.append(current) + todo.extend(current.parent / name for name in quoted_includes(current)) + return sorted(seen) + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_python_h_is_spelled_only_in_py_hpp(): + spellers = [p.name for p in HEADERS + SOURCES if "" in read(p)] + assert spellers == ["py.hpp"] + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_neutral_files_do_not_name_python(): + """Every header except the seam and the umbrellas, and every source that does + not include py.hpp, compiles without a Python include path.""" + neutral_sources = [p for p in SOURCES if "py.hpp" not in quoted_includes(p)] + assert {p.name for p in neutral_sources} == {"driver_api.cpp", "error.cpp"} + neutral = [p for p in HEADERS if p.name not in UMBRELLAS | {"py.hpp"}] + neutral_sources + offenders = {p.name: PYTHON_TOKEN.findall(read(p)) for p in neutral} + assert {name: hits for name, hits in offenders.items() if hits} == {} + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_in_tree_includes_are_bare_sibling_names_that_exist(): + for path in HEADERS + SOURCES: + for name in quoted_includes(path): + assert "/" not in name, f"{path.name} includes {name!r}; in-tree includes are bare sibling names" + assert (RT / name).is_file(), f"{path.name} includes {name!r}, which does not exist" + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_umbrellas_are_named_only_by_their_cython_file(): + for path in HEADERS + SOURCES: + assert UMBRELLAS.isdisjoint(quoted_includes(path)), f"{path.name} includes an umbrella" + named = {} + for path in sorted(list(CORE.rglob("*.pyx")) + list(CORE.rglob("*.pxd"))): + for header in re.findall(r'cdef extern from "(_cpp/rt/[^"]+)"', read(path)): + named.setdefault(header, set()).add(path.name) + assert named == {"_cpp/rt/rt.hpp": {"_rt.pyx"}, "_cpp/rt/handles.hpp": {"_rt.pxd"}} + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_consumer_closure_is_types_and_the_python_seam(): + closure = {p.name for p in include_closure(RT / "handles.hpp")} + assert closure == {"handles.hpp", "py.hpp", "types.hpp"} + # Consumers are RTLD_LOCAL extensions that cannot link to _rt: nothing with storage. + for name in sorted(closure): + text = read(RT / name) + assert not re.search(r'^extern (?!"C")', text, re.M), f"{name} declares an extern variable" + assert not re.search(r"^(static|thread_local)\b", text, re.M), f"{name} defines storage" + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_pxd_functions_are_not_called_by_name_inside_the_module(): + """Cython emits a static prototype for each cdef function the .pxd declares, so + calling one by that name from _rt.pyx clashes with the extern C++ declaration. + The module calls through an alias with a different Cython name instead.""" + names = re.findall(r"^cdef\s+(?:[\w:.*& \[\]]+?\s)?(\w+)\(", read(CORE / "_rt.pxd"), re.M) + assert len(names) > 90 + pyx = re.sub(r'""".*?"""', "", read(CORE / "_rt.pyx"), flags=re.S) + pyx = re.sub(r"#[^\n]*", "", pyx) + code = "\n".join(line for line in pyx.split("\n") if '"cuda_core::rt::' not in line) + assert {name for name in names if re.search(rf"\b{name}\(", code)} == set()