From 81b31250e94996f9e8e0e3ee0a30374ade11a59f Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Wed, 2 Sep 2026 17:14:12 -0700 Subject: [PATCH 01/14] cuda.core: define the error handling policy and report failures that cannot be raised Write down how cuda.core handles CUDA failures (docs/source/error_handling.rst for users, a "Failure handling" section in AGENTS.md and _cpp/DESIGN.md for contributors) and bring the code into line with it: - Add cuda.core.CUDAWarning, emitted for CUDA errors that cannot be raised (destructors, CUDA callbacks, cleanup after an earlier failure). The C++ handle layer reports through one helper that uses the Python warnings machinery when the interpreter is usable, delivers an escalated warning as an unraisable exception, and falls back to stderr otherwise. CUDA_ERROR_DEINITIALIZED is not reported. - Wrap every destroy call made from a deleter (pw_*) so its failure is reported instead of discarded, including memory pools, green contexts, graphs, graph execs, graphics resources, the linker, user objects, the NVRTC/NVVM/nvJitLink handles and file descriptors; release the GIL around the compiler-handle destroys like the CUDA ones. - When the caller's context cannot be restored after a successful operation, undo the creation and raise a CUDAError that says which context is current; report the same failure as a warning in deleters; report a skipped context-sensitive undo instead of leaking silently. - Add context_get_device and graph_node_set_params so Stream_get_ctx_device and _set_definition_node_params stop hand-rolling cuCtxPush/Pop/SetCurrent. The node update now publishes its attachment before raising a restoration failure, closing a window that left the node referencing released owners. - Device.set_current(ctx) switches with a single cuCtxSetCurrent, so a failure leaves the previous context current and the call works without one. - Report failed cuStreamEndCapture in GraphBuilder.__dealloc__ and failed child-graph rollbacks; warn from _mr_dealloc_callback instead of printing. - Add a test hook that makes the next context restoration fail, tests for the policy, and release notes for 1.3.0. Co-Authored-By: Claude Fable 5.1 --- cuda_core/AGENTS.md | 56 +++ cuda_core/cuda/core/__init__.py | 2 + cuda_core/cuda/core/_cpp/DESIGN.md | 31 ++ cuda_core/cuda/core/_cpp/resource_handles.cpp | 334 ++++++++++++++---- cuda_core/cuda/core/_cpp/resource_handles.hpp | 52 +++ cuda_core/cuda/core/_device.pyx | 8 +- cuda_core/cuda/core/_memory/_buffer.pyx | 23 +- cuda_core/cuda/core/_resource_handles.pxd | 16 + cuda_core/cuda/core/_resource_handles.pyi | 8 + cuda_core/cuda/core/_resource_handles.pyx | 35 ++ cuda_core/cuda/core/_stream.pyx | 16 +- cuda_core/cuda/core/_utils/cuda_utils.pyi | 20 ++ cuda_core/cuda/core/_utils/cuda_utils.pyx | 45 ++- cuda_core/cuda/core/graph/_graph_builder.pyx | 15 +- cuda_core/cuda/core/graph/_graph_node.pyx | 7 + cuda_core/cuda/core/graph/_subclasses.pyx | 30 +- cuda_core/docs/source/api.rst | 15 + cuda_core/docs/source/error_handling.rst | 127 +++++++ cuda_core/docs/source/index.rst | 1 + cuda_core/docs/source/release/1.3.0-notes.rst | 51 +++ cuda_core/tests/helpers/contexts.py | 18 + cuda_core/tests/test_error_handling.py | 207 +++++++++++ cuda_core/tests/test_memory.py | 48 +-- 23 files changed, 1033 insertions(+), 132 deletions(-) create mode 100644 cuda_core/docs/source/error_handling.rst create mode 100644 cuda_core/tests/test_error_handling.py diff --git a/cuda_core/AGENTS.md b/cuda_core/AGENTS.md index 9d80ab74aaa..1e7c8da1077 100644 --- a/cuda_core/AGENTS.md +++ b/cuda_core/AGENTS.md @@ -101,6 +101,62 @@ and agents should flag violations. (kernel arguments, memcpy/memset operands, `dst_owner`/`src_owner`, and host-callback closures) inherit this contract. +## Failure handling + +The user-facing contract lives in `docs/source/error_handling.rst`; the rules +below are for contributors. Reviewers and agents should flag violations. + +- **Raise by default**: any failure on a path where an exception can propagate + raises. Driver statuses go through `HANDLE_RETURN` (Cython) or are returned as + `CUresult` from the C++ handle layer and then `HANDLE_RETURN`ed; never + replace a `CUresult` with a generic `RuntimeError`, and drain + `get_last_error()` immediately after a handle constructor returns empty so a + stale status cannot be misattributed later. +- **Guarantees**: a call that creates a resource must create nothing when it + raises (undo the creation if a later step fails). Every call except + `Device.set_current` must leave the calling thread's current context as it + found it. Do not hand-roll `cuCtxPush/Pop/SetCurrent` sequences in Cython; use + the handle layer's scoped-context helpers (`invoke_in_context`, + `invoke_in_context_or_undo`, `cleanup_in_context`, `context_get_device`, + `graph_node_set_params`) so the failure handling exists in one place. +- **Publish before you raise**: when a driver mutation has succeeded and a later + step can still fail, commit whatever keeps that mutation memory-safe (for + example the graph attachment that retains a node's new owners) before raising + the later error. Rolling back the retention of a live mutation creates a + dangling reference. When ownership cannot be established, retain the + resources anyway (leak) rather than release them; a leak is always preferred + to a use-after-free. +- **Non-propagating paths never raise and never discard a status**: shared_ptr + deleters, `__dealloc__`, CUDA callbacks and cleanup after a failure report + through one channel, `report_cuda_error()` / `report_message()` in C++ (the + `pw_*` wrappers) or `warnings.warn(..., CUDAWarning)` in Cython and Python, + which emits `cuda.core.CUDAWarning`. No `print(file=sys.stderr)` and no + `fprintf` outside that helper. `CUDA_ERROR_DEINITIALIZED` is filtered by the + helper because it means the driver is shutting down. +- **Rollback failure**: the original exception propagates; the failed rollback + is reported out of band (or chained with `raise ... from` when a second + exception must be raised). Bare `except:` is acceptable only for + rollback-then-`raise` blocks. +- **Finalization**: once `py_is_finalizing()` is true, do no Python work from + destructors or callbacks and accept the leak (see + `_cpp/resource_handles.hpp` and `_cpp/GRAPH_ATTACHMENTS.md`). +- **Aborting**: `std::abort` (or any process termination) is reserved for an + internal invariant violation where continuing could corrupt memory or produce + silently wrong results *and* no leak-based fallback exists. A failed CUDA + call, including a failed context restoration, never qualifies: raise or + report instead. There is currently no such path; if one is ever needed it + must go through a single helper that writes a diagnostic (call, CUDA error, + invariant, "please report") to stderr before aborting, must never trigger + during interpreter finalization or for driver-shutdown errors, and must be + called out in the docs and release notes. An *implicit* abort (an exception + escaping a `noexcept` function or a deleter, including `std::bad_alloc` from + an allocation inside `noexcept` code) is a bug (#1489, #2417), not a policy + choice: `noexcept` helpers must not allocate, or must catch what they call. +- **Testing**: inject restoration failures with + `cuda.core._resource_handles._set_context_restore_fault_for_testing`; assert + reports with `pytest.warns(CUDAWarning)` or `warnings.catch_warnings`, never + by matching stderr text. + ## API design guidelines These are some API design guidelines we try to follow when adding new APIs to diff --git a/cuda_core/cuda/core/__init__.py b/cuda_core/cuda/core/__init__.py index 7864ae794ca..149fe327f12 100644 --- a/cuda_core/cuda/core/__init__.py +++ b/cuda_core/cuda/core/__init__.py @@ -102,8 +102,10 @@ class _PatchedProperty(metaclass=_PatchedPropMeta): from cuda.core._stream import __all__ as _stream_all from cuda.core._tensor_map import * from cuda.core._tensor_map import __all__ as _tensor_map_all +from cuda.core._utils.cuda_utils import CUDAWarning __all__ = [ + "CUDAWarning", *_context_all, *_device_all, *_device_resources_all, diff --git a/cuda_core/cuda/core/_cpp/DESIGN.md b/cuda_core/cuda/core/_cpp/DESIGN.md index 6615f21c4ba..8e1a55a34a3 100644 --- a/cuda_core/cuda/core/_cpp/DESIGN.md +++ b/cuda_core/cuda/core/_cpp/DESIGN.md @@ -275,6 +275,37 @@ Related functions: - `peek_last_error()`: Returns the error without clearing it - `clear_last_error()`: Clears the error state +Some functions return a `CUresult` directly instead of a handle (for example +`context_synchronize`, `context_get_device`, `graph_node_set_params`). Their +callers `HANDLE_RETURN` the value. + +### Context-scoped operations + +Operations that must run in a specific context use `invoke_in_context` / +`invoke_in_context_or_undo` (propagating paths) and `cleanup_in_context` +(deleters). They switch the current context, run the operation, and restore the +caller's context. When restoration fails after the operation succeeded, the +creation is undone and the restoration status is returned; the helper also +records a thread-local detail (`take_last_error_detail()`) that the Cython error +path appends to the raised `CUDAError`, so the user learns that the caller's +context was not restored and which context is current. When both the operation +and the restoration fail, the operation status is returned and the restoration +failure is reported out of band. Tests inject restoration failures with +`set_context_restore_fault_for_testing()`. + +### Reporting from non-propagating paths + +Deleters, CUDA callbacks and cleanup-after-failure cannot raise. They report +through `report_cuda_error()` / `report_message()` (the `pw_*` wrappers +decorate destroy calls with it), which emit a `cuda.core.CUDAWarning` through +the Python warnings machinery when the interpreter is usable, deliver an +escalated warning as an unraisable exception, and fall back to stderr when the +GIL cannot be taken (for example during finalization). `CUDA_ERROR_DEINITIALIZED` +is never reported because it means the driver is shutting down. No status is +discarded silently anywhere in this layer, and nothing in this layer terminates +the process; see `docs/source/error_handling.rst` and the "Failure handling" +section of `AGENTS.md` for the policy. + ## Usage from Cython ```cython diff --git a/cuda_core/cuda/core/_cpp/resource_handles.cpp b/cuda_core/cuda/core/_cpp/resource_handles.cpp index 46f7b019379..75d2d27bc7d 100644 --- a/cuda_core/cuda/core/_cpp/resource_handles.cpp +++ b/cuda_core/cuda/core/_cpp/resource_handles.cpp @@ -45,6 +45,8 @@ decltype(&cuCtxGetCurrent) p_cuCtxGetCurrent = nullptr; decltype(&cuCtxSetCurrent) p_cuCtxSetCurrent = nullptr; decltype(&cuCtxSynchronize) p_cuCtxSynchronize = nullptr; decltype(&cuCtxGetStreamPriorityRange) p_cuCtxGetStreamPriorityRange = nullptr; +decltype(&cuCtxGetDevice) p_cuCtxGetDevice = nullptr; +decltype(&cuGraphNodeSetParams) p_cuGraphNodeSetParams = nullptr; decltype(&cuGreenCtxCreate) p_cuGreenCtxCreate = nullptr; decltype(&cuGreenCtxDestroy) p_cuGreenCtxDestroy = nullptr; decltype(&cuCtxFromGreenCtx) p_cuCtxFromGreenCtx = nullptr; @@ -197,7 +199,120 @@ class GILAcquireGuard { bool acquired_; }; -void warn_on_cuda_error(const char* operation, CUresult status, const char* detail = nullptr) noexcept; +// ---------------------------------------------------------------------------- +// Non-propagating error reporting +// +// Deleters, CUDA callbacks and other non-propagating paths cannot raise. They +// report through report_cuda_error()/report_message(), which emit a +// cuda.core.CUDAWarning when the interpreter is usable and fall back to stderr +// otherwise. See docs/source/error_handling.rst for the policy. +// ---------------------------------------------------------------------------- + +// Warning category registered by _resource_handles.pyx (cuda.core.CUDAWarning). +std::atomic warning_category{nullptr}; + +// Thread-local detail attached to the next raised CUDAError (see +// take_last_error_detail()). Written only by propagating helpers. The taken +// copy stays valid until the next take on the same thread. +thread_local char last_error_detail[256] = {0}; +thread_local char taken_error_detail[256] = {0}; + +// Thread-local fault injected into the next context restoration (tests only). +thread_local CUresult context_restore_fault = CUDA_SUCCESS; + +// Format " : : " for a failed CUDA call. +void format_cuda_error(char* buffer, size_t size, const char* operation, CUresult status, + const char* detail) noexcept { + const char* error_name = nullptr; + const char* error_description = nullptr; + bool decoded = p_cuGetErrorName && p_cuGetErrorString + && p_cuGetErrorName(status, &error_name) == CUDA_SUCCESS + && p_cuGetErrorString(status, &error_description) == CUDA_SUCCESS; + const char* outcome = detail ? detail : "failed"; + if (decoded) { + std::snprintf(buffer, size, "%s %s: %s: %s", operation, outcome, error_name, error_description); + } else { + std::snprintf(buffer, size, "%s %s (CUDA error %d)", operation, outcome, static_cast(status)); + } +} + +} // namespace + +// Report a message that could not be raised. Emits cuda.core.CUDAWarning via +// the Python warnings machinery; if that itself fails (for example because the +// warning was promoted to an error), the failure is written as an unraisable +// exception, the CPython convention for exceptions in destructors. Falls back +// to stderr when the interpreter cannot be used. +void report_message(const char* message) noexcept { + PyObject* category = warning_category.load(std::memory_order_acquire); + if (category && Py_IsInitialized() && !py_is_finalizing()) { + GILAcquireGuard gil; + if (gil.acquired()) { + // Deleters can run while a Python exception is propagating; keep it. +#if PY_VERSION_HEX >= 0x030C0000 + PyObject* pending = PyErr_GetRaisedException(); +#else + PyObject *pending_type, *pending_value, *pending_tb; + PyErr_Fetch(&pending_type, &pending_value, &pending_tb); +#endif + if (PyErr_WarnEx(category, message, 1) != 0) { + PyObject* subject = PyUnicode_FromString(message); + PyErr_WriteUnraisable(subject); + Py_XDECREF(subject); + } +#if PY_VERSION_HEX >= 0x030C0000 + PyErr_SetRaisedException(pending); +#else + PyErr_Restore(pending_type, pending_value, pending_tb); +#endif + return; + } + } + std::fprintf(stderr, "%s\n", message); +} + +// Report a failed non-CUDA call (NVRTC, NVVM, nvJitLink) from a path that +// cannot raise. +void report_status_code(const char* operation, long code) noexcept { + char message[256]; + std::snprintf(message, sizeof(message), "%s failed (status %ld)", operation, code); + report_message(message); +} + +void register_warning_category(PyObject* category) noexcept { + warning_category.store(category, std::memory_order_release); +} + +// Report a failed CUDA call from a path that cannot raise. CUDA_ERROR_DEINITIALIZED +// is not reported: it means the driver is shutting down, which makes cleanup +// failures expected and uninteresting. +void report_cuda_error(const char* operation, CUresult status, const char* detail) noexcept { + if (status == CUDA_SUCCESS || status == CUDA_ERROR_DEINITIALIZED) { + return; + } + char message[512]; + format_cuda_error(message, sizeof(message), operation, status, detail); + report_message(message); +} + +const char* take_last_error_detail() noexcept { + if (!last_error_detail[0]) { + return nullptr; + } + std::memcpy(taken_error_detail, last_error_detail, sizeof(taken_error_detail)); + last_error_detail[0] = 0; + return taken_error_detail; +} + +void clear_last_error_detail() noexcept { + last_error_detail[0] = 0; +} + +void set_context_restore_fault_for_testing(CUresult status) noexcept { + context_restore_fault = status; +} + +namespace { // Make a context current and record the state needed to restore it. // An empty handle is a no-op: the operation runs in the caller's current @@ -205,6 +320,7 @@ void warn_on_cuda_error(const char* operation, CUresult status, const char* deta CUresult enter_context(const ContextHandle& h_context, CUcontext* previous, int* changed) noexcept { *previous = nullptr; *changed = 0; + clear_last_error_detail(); CUcontext target = as_cu(h_context); if (!target) { return CUDA_SUCCESS; @@ -220,17 +336,49 @@ CUresult enter_context(const ContextHandle& h_context, CUcontext* previous, int* return status; } -// Restore the previous context and preserve an earlier operation error. +// Restore the caller's context. Returns the restoration status. +CUresult restore_context(CUcontext previous) noexcept { + if (context_restore_fault != CUDA_SUCCESS) { + // Test hook: behave as if cuCtxSetCurrent(previous) failed, leaving the + // target context current exactly as a real failure would. + CUresult fault = context_restore_fault; + context_restore_fault = CUDA_SUCCESS; + return fault; + } + GILReleaseGuard gil; + return p_cuCtxSetCurrent(previous); +} + +// Record why the CUresult about to be returned should be explained further +// when it is raised as a CUDAError: the caller's context was not restored. +void note_context_not_restored(CUcontext previous) noexcept { + CUcontext current = nullptr; + if (p_cuCtxGetCurrent(¤t) != CUDA_SUCCESS) { + current = nullptr; + } + std::snprintf(last_error_detail, sizeof(last_error_detail), + "the calling thread's CUDA context (%#llx) could not be restored; " + "context %#llx is now current. Call Device.set_current() before issuing " + "further CUDA work on this thread", + static_cast(reinterpret_cast(previous)), + static_cast(reinterpret_cast(current))); +} + +// Restore the previous context and preserve an earlier operation error. The +// operation error, if any, is returned; a restoration failure is then reported +// out of band. Otherwise the restoration status is returned, annotated for the +// eventual CUDAError. CUresult exit_context(CUcontext previous, int changed, CUresult operation_status) noexcept { - CUresult restore_status = CUDA_SUCCESS; - if (changed) { - GILReleaseGuard gil; - restore_status = p_cuCtxSetCurrent(previous); + CUresult restore_status = changed ? restore_context(previous) : CUDA_SUCCESS; + if (restore_status == CUDA_SUCCESS) { + return operation_status; } - if (operation_status != CUDA_SUCCESS && restore_status != CUDA_SUCCESS) { - warn_on_cuda_error("cuCtxSetCurrent (restoring the caller's context)", restore_status); + if (operation_status != CUDA_SUCCESS) { + report_cuda_error("cuCtxSetCurrent (restoring the caller's context)", restore_status); + return operation_status; } - return operation_status != CUDA_SUCCESS ? operation_status : restore_status; + note_context_not_restored(previous); + return restore_status; } // Require a callable to be invocable without throwing. @@ -257,12 +405,11 @@ ContextHandle deallocation_context(const DeallocationStream& stream) noexcept { } if (stream.ptds_tid != std::thread::id{} && stream.ptds_tid != std::this_thread::get_id()) { - std::fprintf( - stderr, - "Warning: Buffer deallocation for a per-thread default stream " + report_message( + "Buffer deallocation for a per-thread default stream " "is running on a different host thread than the one that recorded " "the deallocation stream; ordering relative to the allocating " - "thread's PTDS is not preserved\n"); + "thread's PTDS is not preserved"); } return get_stream_context(stream.h_stream); } @@ -313,7 +460,7 @@ CUresult invoke_in_context_or_undo(const ContextHandle& h_context, Fn&& operatio if (undo_ok) { std::invoke(std::forward(undo)); } else { - warn_on_cuda_error( + report_cuda_error( "cuCtxSetCurrent (restoring the caller's context)", composite, "failed; cleanup of the new resource skipped because its context " "is no longer current (resource leaked)"); @@ -322,32 +469,6 @@ CUresult invoke_in_context_or_undo(const ContextHandle& h_context, Fn&& operatio return composite; } -// Write a warning that includes the CUDA error name and description. -void warn_on_cuda_error(const char* operation, CUresult status, const char* detail) noexcept { - const char* error_name = nullptr; - const char* error_description = nullptr; - CUresult name_status = p_cuGetErrorName(status, &error_name); - CUresult description_status = p_cuGetErrorString(status, &error_description); - - if (name_status == CUDA_SUCCESS && description_status == CUDA_SUCCESS) { - if (detail) { - std::fprintf(stderr, "Warning: %s %s: %s: %s\n", - operation, detail, error_name, error_description); - } else { - std::fprintf(stderr, "Warning: %s failed: %s: %s\n", - operation, error_name, error_description); - } - } else { - if (detail) { - std::fprintf(stderr, "Warning: %s %s (CUDA error %d)\n", - operation, detail, static_cast(status)); - } else { - std::fprintf(stderr, "Warning: %s failed (CUDA error %d)\n", - operation, static_cast(status)); - } - } -} - // Run cleanup with the requested context current. Warn and skip the operation // if activation fails, and independently warn on operation or restoration // failure. Return the operation or activation status; restoration never @@ -360,39 +481,50 @@ CUresult cleanup_in_context(const ContextHandle& h_context, const char* name, int changed = 0; CUresult status = enter_context(h_context, &previous, &changed); if (status != CUDA_SUCCESS) { - warn_on_cuda_error(name, status, + report_cuda_error(name, status, "skipped (context activation failed; resource leaked)"); } else { status = std::invoke(std::forward(operation), std::forward(args)...); if (status != CUDA_SUCCESS) { - warn_on_cuda_error(name, status); + report_cuda_error(name, status); } } CUresult restore = exit_context(previous, changed, CUDA_SUCCESS); if (restore != CUDA_SUCCESS) { - warn_on_cuda_error(name, restore, "failed while restoring the caller's context"); + report_cuda_error(name, restore, "failed while restoring the caller's context"); } return status; } #undef ASSERT_NOTHROW_INVOCABLE -// Decorate a CUDA operation to warn whenever it returns an error. +// Decorate a status-returning cleanup call to report whenever it fails. CUDA +// calls (CUresult) are reported with the error name and description; NVRTC, +// NVVM and nvJitLink calls (integer status codes) with the raw code. template class WarnOnFailure { public: explicit WarnOnFailure(const char* operation) noexcept : operation_(operation) {} template - CUresult operator()(Args&&... args) const noexcept { - CUresult status = Function(std::forward(args)...); - if (status != CUDA_SUCCESS) { - warn_on_cuda_error(operation_, status); - } + auto operator()(Args&&... args) const noexcept { + auto status = Function(std::forward(args)...); + report(status); return status; } private: + void report(CUresult status) const noexcept { + report_cuda_error(operation_, status); + } + + template + void report(Status status) const noexcept { + if (static_cast(status) != 0) { + report_status_code(operation_, static_cast(status)); + } + } + const char* operation_; }; @@ -405,6 +537,18 @@ const WarnOnFailure pw_cuArrayDestroy{"cuArrayDestroy"}; const WarnOnFailure pw_cuMipmappedArrayDestroy{"cuMipmappedArrayDestroy"}; const WarnOnFailure pw_cuTexObjectDestroy{"cuTexObjectDestroy"}; const WarnOnFailure pw_cuSurfObjectDestroy{"cuSurfObjectDestroy"}; +const WarnOnFailure pw_cuGreenCtxDestroy{"cuGreenCtxDestroy"}; +const WarnOnFailure pw_cuMemPoolDestroy{"cuMemPoolDestroy"}; +const WarnOnFailure pw_cuMemFreeHost{"cuMemFreeHost"}; +const WarnOnFailure pw_cuGraphDestroy{"cuGraphDestroy"}; +const WarnOnFailure pw_cuGraphExecDestroy{"cuGraphExecDestroy"}; +const WarnOnFailure pw_cuGraphicsUnregisterResource{"cuGraphicsUnregisterResource"}; +const WarnOnFailure pw_cuLinkDestroy{"cuLinkDestroy"}; +const WarnOnFailure pw_cuUserObjectRelease{"cuUserObjectRelease"}; +const WarnOnFailure pw_cuGraphReleaseUserObject{"cuGraphReleaseUserObject"}; +const WarnOnFailure pw_nvrtcDestroyProgram{"nvrtcDestroyProgram"}; +const WarnOnFailure pw_nvvmDestroyProgram{"nvvmDestroyProgram"}; +const WarnOnFailure pw_nvJitLinkDestroy{"nvJitLinkDestroy"}; } // namespace @@ -426,6 +570,52 @@ CUresult context_get_stream_priority_range(const ContextHandle& h_context, }); } +// Query the device of the provided context. +CUresult context_get_device(const ContextHandle& h_context, CUdevice* device) noexcept { + return invoke_in_context(h_context, [&]() noexcept { + return p_cuCtxGetDevice(device); + }); +} + +// Set a graph node's parameters with h_context current (an empty handle runs in +// the caller's context). Returns the cuGraphNodeSetParams status. A failure to +// restore the caller's context is returned separately in *restore_status so the +// caller can publish the metadata that depends on the successful update before +// raising it; if the update itself failed, a restoration failure is reported +// out of band and *restore_status is CUDA_SUCCESS. +CUresult graph_node_set_params(CUgraphNode node, CUgraphNodeParams* params, + const ContextHandle& h_context, + CUresult* restore_status) noexcept { + *restore_status = CUDA_SUCCESS; + if (!p_cuGraphNodeSetParams) { + return CUDA_ERROR_NOT_SUPPORTED; + } + CUcontext previous = nullptr; + int changed = 0; + CUresult status = enter_context(h_context, &previous, &changed); + if (status != CUDA_SUCCESS) { + return status; + } + { + GILReleaseGuard gil; + status = p_cuGraphNodeSetParams(node, params); + } + if (!changed) { + return status; + } + CUresult restored = restore_context(previous); + if (restored == CUDA_SUCCESS) { + return status; + } + if (status != CUDA_SUCCESS) { + report_cuda_error("cuCtxSetCurrent (restoring the caller's context)", restored); + return status; + } + note_context_not_restored(previous); + *restore_status = restored; + return status; +} + // ============================================================================ // CUDA user-object deferred cleanup // @@ -767,7 +957,7 @@ GreenCtxHandle create_green_ctx_handle(CUdevResource* resources, unsigned int nb new GreenCtxBox{green_ctx}, [](const GreenCtxBox* b) { GILReleaseGuard gil; - p_cuGreenCtxDestroy(b->resource); + pw_cuGreenCtxDestroy(b->resource); delete b; } ); @@ -1187,7 +1377,7 @@ static MemoryPoolHandle wrap_mempool_owned(CUmemoryPool pool) { [](const MemoryPoolBox* b) { GILReleaseGuard gil; clear_mempool_peer_access(b->resource); - p_cuMemPoolDestroy(b->resource); + pw_cuMemPoolDestroy(b->resource); delete b; } ); @@ -1353,7 +1543,7 @@ DevicePtrHandle deviceptr_alloc_host(size_t size) { new DevicePtrBox{reinterpret_cast(ptr), DeallocationStream{}}, [](DevicePtrBox* b) { GILReleaseGuard gil; - p_cuMemFreeHost(reinterpret_cast(b->resource)); + pw_cuMemFreeHost(reinterpret_cast(b->resource)); delete b; } ); @@ -1909,7 +2099,7 @@ void rollback_prepared_attachment( GraphBox* box = get_box(state->h_graph); if (box->resource) { GILReleaseGuard gil; - p_cuGraphReleaseUserObject( + pw_cuGraphReleaseUserObject( box->resource, state->replacement->object, 1); } } @@ -1955,7 +2145,7 @@ GraphHandle create_graph_handle(CUgraph graph) { GraphBox* root = hierarchy->root(); if (root && root->resource) { GILReleaseGuard gil; - p_cuGraphDestroy(root->resource); + pw_cuGraphDestroy(root->resource); } retry_deferred_cleanup(); delete hierarchy; @@ -2187,7 +2377,7 @@ CUresult graph_prepare_attachment( if (status != CUDA_SUCCESS) { prepared->replacement_entry.mapped() = nullptr; prepared->replacement = nullptr; - p_cuUserObjectRelease(object, 1); + pw_cuUserObjectRelease(object, 1); return status; } } @@ -2318,7 +2508,7 @@ struct GraphExecBox { ~GraphExecBox() noexcept { if (resource) { GILReleaseGuard gil; - p_cuGraphExecDestroy(resource); + pw_cuGraphExecDestroy(resource); } // The accumulator fields may be dangling after exec destruction. retry_deferred_cleanup(); @@ -2338,7 +2528,7 @@ GraphExecHandle make_graph_exec_handle( ~RawGraphExecGuard() noexcept { if (resource) { GILReleaseGuard gil; - p_cuGraphExecDestroy(resource); + pw_cuGraphExecDestroy(resource); } retry_deferred_cleanup(); } @@ -2360,7 +2550,8 @@ struct ExecAttachmentStaging { ExecAttachments* accumulator = nullptr; ~ExecAttachmentStaging() noexcept { - release(); + report_cuda_error("cuGraphReleaseUserObject", release(), + "failed while dropping a staged graph attachment"); } CUresult release() noexcept { @@ -2406,7 +2597,7 @@ CUresult stage_exec_attachments( *h_source, object, 1, CU_GRAPH_USER_OBJECT_MOVE); if (status != CUDA_SUCCESS) { // Dropping the last reference retires the accumulator. - p_cuUserObjectRelease(object, 1); + pw_cuUserObjectRelease(object, 1); return status; } } @@ -2676,7 +2867,7 @@ GraphicsResourceHandle create_graphics_resource_handle(CUgraphicsResource resour new GraphicsResourceBox{resource}, [](const GraphicsResourceBox* b) { GILReleaseGuard gil; - p_cuGraphicsUnregisterResource(b->resource); + pw_cuGraphicsUnregisterResource(b->resource); delete b; } ); @@ -2699,8 +2890,10 @@ NvrtcProgramHandle create_nvrtc_program_handle(nvrtcProgram prog) { [](NvrtcProgramBox* b) { // Note: nvrtcDestroyProgram takes nvrtcProgram* and nulls it, // but we're deleting the box anyway so nulling is harmless. - // Errors are ignored (standard destructor practice). - p_nvrtcDestroyProgram(&b->resource); + if (p_nvrtcDestroyProgram) { + GILReleaseGuard gil; + pw_nvrtcDestroyProgram(&b->resource); + } delete b; } ); @@ -2730,7 +2923,8 @@ NvvmProgramHandle create_nvvm_program_handle(nvvmProgram prog) { // but we're deleting the box anyway so nulling is harmless. // If NVVM is not available, the function pointer is null. if (p_nvvmDestroyProgram) { - p_nvvmDestroyProgram(&b->resource.raw); + GILReleaseGuard gil; + pw_nvvmDestroyProgram(&b->resource.raw); } delete b; } @@ -2761,7 +2955,8 @@ NvJitLinkHandle create_nvjitlink_handle(nvJitLink_t handle) { // but we're deleting the box anyway so nulling is harmless. // If nvJitLink is not available, the function pointer is null. if (p_nvJitLinkDestroy) { - p_nvJitLinkDestroy(&b->resource.raw); + GILReleaseGuard gil; + pw_nvJitLinkDestroy(&b->resource.raw); } delete b; } @@ -2789,9 +2984,9 @@ CuLinkHandle create_culink_handle(CUlinkState state) { new CuLinkBox{state}, [](CuLinkBox* b) { // cuLinkDestroy takes CUlinkState by value (not pointer). - // Errors are ignored (standard destructor practice). if (p_cuLinkDestroy) { - p_cuLinkDestroy(b->resource); + GILReleaseGuard gil; + pw_cuLinkDestroy(b->resource); } delete b; } @@ -2814,7 +3009,12 @@ FileDescriptorHandle create_fd_handle(int fd) { #else return FileDescriptorHandle( new int(fd), - [](const int* p) { ::close(*p); delete p; } + [](const int* p) { + if (::close(*p) != 0) { + report_message("close() failed for an IPC file descriptor; the descriptor may have leaked"); + } + delete p; + } ); #endif } diff --git a/cuda_core/cuda/core/_cpp/resource_handles.hpp b/cuda_core/cuda/core/_cpp/resource_handles.hpp index 419710ea0d9..069e3ec8319 100644 --- a/cuda_core/cuda/core/_cpp/resource_handles.hpp +++ b/cuda_core/cuda/core/_cpp/resource_handles.hpp @@ -57,6 +57,41 @@ CUresult peek_last_error() noexcept; // Explicitly clear the last error void clear_last_error() noexcept; +// ============================================================================ +// Non-propagating error reporting +// +// Paths that cannot raise (shared_ptr deleters, CUDA callbacks, __dealloc__) +// report failures through these functions instead of discarding them. They +// emit a cuda.core.CUDAWarning when the interpreter can be used and write to +// stderr otherwise; they never raise. See docs/source/error_handling.rst. +// ============================================================================ + +// Register the Python warning category used by report_* (cuda.core.CUDAWarning). +void register_warning_category(PyObject* category) noexcept; + +// Report a failed CUDA call. `detail` replaces the default "failed" wording, +// e.g. "skipped (context activation failed; resource leaked)". +// CUDA_ERROR_DEINITIALIZED (driver shutting down) is never reported. +void report_cuda_error(const char* operation, CUresult status, const char* detail = nullptr) noexcept; + +// Report a message that is not tied to a CUresult. +void report_message(const char* message) noexcept; + +// Report a failed NVRTC/NVVM/nvJitLink call by raw status code. +void report_status_code(const char* operation, long code) noexcept; + +// Detail recorded by a context-scoped helper for the CUresult it is about to +// return, e.g. that the caller's context could not be restored. The Cython +// error path appends it to the raised CUDAError. Thread-local; take_ returns +// the detail (valid until the next take on this thread) and clears it, or +// nullptr when none is recorded. +const char* take_last_error_detail() noexcept; +void clear_last_error_detail() noexcept; + +// Tests only: make the next context restoration on this thread fail with +// `status`, leaving the target context current as a real failure would. +void set_context_restore_fault_for_testing(CUresult status) noexcept; + // ============================================================================ // CUDA driver function pointers // @@ -73,6 +108,8 @@ extern decltype(&cuCtxGetCurrent) p_cuCtxGetCurrent; extern decltype(&cuCtxSetCurrent) p_cuCtxSetCurrent; extern decltype(&cuCtxSynchronize) p_cuCtxSynchronize; extern decltype(&cuCtxGetStreamPriorityRange) p_cuCtxGetStreamPriorityRange; +extern decltype(&cuCtxGetDevice) p_cuCtxGetDevice; +extern decltype(&cuGraphNodeSetParams) p_cuGraphNodeSetParams; extern decltype(&cuGreenCtxCreate) p_cuGreenCtxCreate; extern decltype(&cuGreenCtxDestroy) p_cuGreenCtxDestroy; extern decltype(&cuCtxFromGreenCtx) p_cuCtxFromGreenCtx; @@ -263,6 +300,21 @@ CUresult context_get_stream_priority_range( int* least_priority, int* greatest_priority) noexcept; +// Query the device of the provided context. +// Returns CUDA_ERROR_INVALID_CONTEXT for an empty handle. +CUresult context_get_device(const ContextHandle& h_context, CUdevice* device) noexcept; + +// Call cuGraphNodeSetParams with h_context current (empty handle: the caller's +// context). Returns the update status; *restore_status receives a failure to +// restore the caller's context after a successful update, which the caller +// raises only after publishing the metadata that depends on the update. +// Returns CUDA_ERROR_NOT_SUPPORTED when the driver lacks cuGraphNodeSetParams. +CUresult graph_node_set_params( + CUgraphNode node, + CUgraphNodeParams* params, + const ContextHandle& h_context, + CUresult* restore_status) noexcept; + // ============================================================================ // Stream handle functions // ============================================================================ diff --git a/cuda_core/cuda/core/_device.pyx b/cuda_core/cuda/core/_device.pyx index a7e7d59e04a..170bedc0034 100644 --- a/cuda_core/cuda/core/_device.pyx +++ b/cuda_core/cuda/core/_device.pyx @@ -1315,8 +1315,12 @@ class Device: HANDLE_RETURN(cydriver.cuCtxGetCurrent(&prev_ctx)) if prev_ctx != NULL: HANDLE_RETURN(cydriver.cuCtxGetDevice(&prev_dev)) - HANDLE_RETURN(cydriver.cuCtxPopCurrent(&prev_ctx)) - HANDLE_RETURN(cydriver.cuCtxPushCurrent(curr_ctx)) + # cuCtxSetCurrent replaces the top of the thread's context stack + # in one driver call (or binds ctx when nothing is current), so + # a failure leaves the previous context current instead of + # leaving the thread with no context, as a failed pop-then-push + # would. + HANDLE_RETURN(cydriver.cuCtxSetCurrent(curr_ctx)) self._has_inited = True self._context = ctx # Store owning context reference if prev_ctx != NULL: diff --git a/cuda_core/cuda/core/_memory/_buffer.pyx b/cuda_core/cuda/core/_memory/_buffer.pyx index 2484ad82b00..a24035c68d4 100644 --- a/cuda_core/cuda/core/_memory/_buffer.pyx +++ b/cuda_core/cuda/core/_memory/_buffer.pyx @@ -36,11 +36,12 @@ IF CUDA_CORE_BUILD_MAJOR >= 13: from cuda.core._stream cimport Stream, Stream_accept, Stream_is_legacy_default_token, default_stream from cuda.core._utils.cuda_utils cimport HANDLE_RETURN, _parse_fill_value -import sys +import warnings from collections.abc import Sequence from typing import TYPE_CHECKING from cuda.core._memory._copy_enums import CopyOptions, _reject_unsupported_during_api_call +from cuda.core._utils.cuda_utils import CUDAWarning from cuda.core._utils.pycompat import BufferProtocol from cuda.core._dlpack import classify_dl_device, make_py_capsule from cuda.core._device import Device @@ -59,25 +60,33 @@ cdef void _mr_dealloc_callback( size_t size, const StreamHandle& h_stream, ) noexcept: - """Called by the C++ deleter to deallocate via MemoryResource.deallocate.""" + """Called by the C++ deleter to deallocate via MemoryResource.deallocate. + + Runs from a destructor, so nothing can be raised here; failures are reported + as :class:`~cuda.core.CUDAWarning` (see the error handling policy). + """ cdef Stream stream try: if not h_stream: - print( - "Warning: no deallocation stream was recorded; falling back to " + warnings.warn( + "no deallocation stream was recorded; falling back to " "the default stream for mr.deallocate() during Buffer " "destruction. This is an internal cuda-core error; please " "report it with your CUDA driver, CUDA Toolkit, and " "cuda-python versions.", - file=sys.stderr, + CUDAWarning, + stacklevel=2, ) stream = default_stream() else: stream = Stream._from_handle(Stream, h_stream) mr.deallocate(int(ptr), size, stream=stream) except Exception as exc: - print(f"Warning: mr.deallocate() failed during Buffer destruction: {exc}", - file=sys.stderr) + warnings.warn( + f"mr.deallocate() failed during Buffer destruction; the allocation may have leaked: {exc}", + CUDAWarning, + stacklevel=2, + ) register_mr_dealloc_callback(_mr_dealloc_callback) diff --git a/cuda_core/cuda/core/_resource_handles.pxd b/cuda_core/cuda/core/_resource_handles.pxd index fe075b6414b..37f4bcb0435 100644 --- a/cuda_core/cuda/core/_resource_handles.pxd +++ b/cuda_core/cuda/core/_resource_handles.pxd @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 +from cpython.object cimport PyObject from libc.stddef cimport size_t from libc.stdint cimport intptr_t @@ -168,6 +169,16 @@ cdef cydriver.CUresult get_last_error() noexcept nogil cdef cydriver.CUresult peek_last_error() noexcept nogil cdef void clear_last_error() noexcept nogil +# Non-propagating error reporting (never raises; emits cuda.core.CUDAWarning +# when possible, else writes to stderr) +cdef void register_warning_category(PyObject* category) noexcept +cdef void report_cuda_error( + const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil +cdef void report_message(const char* message) noexcept nogil +cdef void report_status_code(const char* operation, long code) noexcept nogil +cdef const char* take_last_error_detail() noexcept nogil +cdef void clear_last_error_detail() noexcept nogil + # Context handles cdef ContextHandle create_context_handle_ref(cydriver.CUcontext ctx) except+ nogil cdef ContextHandle create_context_handle_from_green_ctx(const GreenCtxHandle& h_green_ctx) except+ nogil @@ -184,6 +195,11 @@ cdef cydriver.CUresult context_get_stream_priority_range( const ContextHandle& h_context, int* least_priority, int* greatest_priority) noexcept nogil +cdef cydriver.CUresult context_get_device( + const ContextHandle& h_context, cydriver.CUdevice* device) noexcept nogil +cdef cydriver.CUresult graph_node_set_params( + cydriver.CUgraphNode node, cydriver.CUgraphNodeParams* params, + const ContextHandle& h_context, cydriver.CUresult* restore_status) noexcept nogil # Stream handles cdef StreamHandle create_stream_handle( diff --git a/cuda_core/cuda/core/_resource_handles.pyi b/cuda_core/cuda/core/_resource_handles.pyi index 66cbf80761a..c44bcd46a03 100644 --- a/cuda_core/cuda/core/_resource_handles.pyi +++ b/cuda_core/cuda/core/_resource_handles.pyi @@ -41,3 +41,11 @@ PreparedAttachmentDeleter: TypeAlias = Incomplete PreparedChildGraphUpdateState: TypeAlias = Incomplete PreparedExecAttachmentState: TypeAlias = Incomplete PreparedExecAttachmentDeleter: TypeAlias = Incomplete + +def _set_context_restore_fault_for_testing(status: int): + """Make the next context restoration on this thread fail with ``status``. + + Test hook for the context save/restore paths in the handle layer. The + injected failure leaves the target context current, exactly as a failing + ``cuCtxSetCurrent`` would, so callers must restore the context themselves. + """ diff --git a/cuda_core/cuda/core/_resource_handles.pyx b/cuda_core/cuda/core/_resource_handles.pyx index 0f8d6e15cde..09692f9d9a2 100644 --- a/cuda_core/cuda/core/_resource_handles.pyx +++ b/cuda_core/cuda/core/_resource_handles.pyx @@ -10,6 +10,7 @@ # The cdef extern from declarations below satisfy the .pxd declarations directly, # without needing separate wrapper functions. +from cpython.object cimport PyObject from cpython.pycapsule cimport PyCapsule_GetName, PyCapsule_GetPointer from libc.stddef cimport size_t @@ -36,6 +37,19 @@ cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": cydriver.CUresult peek_last_error "cuda_core::peek_last_error" () noexcept nogil void clear_last_error "cuda_core::clear_last_error" () noexcept nogil + # Non-propagating error reporting + void register_warning_category "cuda_core::register_warning_category" ( + PyObject* category) noexcept + void report_cuda_error "cuda_core::report_cuda_error" ( + const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil + void report_message "cuda_core::report_message" (const char* message) noexcept nogil + void report_status_code "cuda_core::report_status_code" ( + const char* operation, long code) noexcept nogil + const char* take_last_error_detail "cuda_core::take_last_error_detail" () noexcept nogil + void clear_last_error_detail "cuda_core::clear_last_error_detail" () noexcept nogil + void set_context_restore_fault_for_testing "cuda_core::set_context_restore_fault_for_testing" ( + cydriver.CUresult status) noexcept nogil + # Context handles ContextHandle create_context_handle_ref "cuda_core::create_context_handle_ref" ( cydriver.CUcontext ctx) except+ nogil @@ -57,6 +71,11 @@ cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": const ContextHandle& h_context, int* least_priority, int* greatest_priority) noexcept nogil + cydriver.CUresult context_get_device "cuda_core::context_get_device" ( + const ContextHandle& h_context, cydriver.CUdevice* device) noexcept nogil + cydriver.CUresult graph_node_set_params "cuda_core::graph_node_set_params" ( + cydriver.CUgraphNode node, cydriver.CUgraphNodeParams* params, + const ContextHandle& h_context, cydriver.CUresult* restore_status) noexcept nogil # Stream handles StreamHandle create_stream_handle "cuda_core::create_stream_handle" ( @@ -323,6 +342,8 @@ cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": void* p_cuCtxSetCurrent "reinterpret_cast(cuda_core::p_cuCtxSetCurrent)" void* p_cuCtxSynchronize "reinterpret_cast(cuda_core::p_cuCtxSynchronize)" void* p_cuCtxGetStreamPriorityRange "reinterpret_cast(cuda_core::p_cuCtxGetStreamPriorityRange)" + void* p_cuCtxGetDevice "reinterpret_cast(cuda_core::p_cuCtxGetDevice)" + void* p_cuGraphNodeSetParams "reinterpret_cast(cuda_core::p_cuGraphNodeSetParams)" void* p_cuGreenCtxCreate "reinterpret_cast(cuda_core::p_cuGreenCtxCreate)" void* p_cuGreenCtxDestroy "reinterpret_cast(cuda_core::p_cuGreenCtxDestroy)" void* p_cuCtxFromGreenCtx "reinterpret_cast(cuda_core::p_cuCtxFromGreenCtx)" @@ -433,6 +454,7 @@ cdef void _init_driver_fn_pointers() noexcept: global p_cuGetErrorName, p_cuGetErrorString global p_cuDevicePrimaryCtxRetain, p_cuDevicePrimaryCtxRelease, p_cuCtxGetCurrent global p_cuCtxSetCurrent, p_cuCtxSynchronize, p_cuCtxGetStreamPriorityRange + global p_cuCtxGetDevice, p_cuGraphNodeSetParams global p_cuGreenCtxCreate, p_cuGreenCtxDestroy, p_cuCtxFromGreenCtx global p_cuDevResourceGenerateDesc, p_cuGreenCtxStreamCreate global p_cuStreamCreateWithPriority, p_cuStreamDestroy, p_cuStreamGetCtx @@ -469,6 +491,9 @@ cdef void _init_driver_fn_pointers() noexcept: p_cuCtxSetCurrent = _get_driver_fn("cuCtxSetCurrent") p_cuCtxSynchronize = _get_driver_fn("cuCtxSynchronize") p_cuCtxGetStreamPriorityRange = _get_driver_fn("cuCtxGetStreamPriorityRange") + p_cuCtxGetDevice = _get_driver_fn("cuCtxGetDevice") + # Graph node parameter updates need CUDA 12.2+ (checked again at the call site). + p_cuGraphNodeSetParams = _get_optional_driver_fn("cuGraphNodeSetParams") p_cuGreenCtxCreate = _get_optional_driver_fn("cuGreenCtxCreate") p_cuGreenCtxDestroy = _get_optional_driver_fn("cuGreenCtxDestroy") p_cuCtxFromGreenCtx = _get_optional_driver_fn("cuCtxFromGreenCtx") @@ -554,6 +579,16 @@ cdef void _init_driver_fn_pointers() noexcept: _init_driver_fn_pointers() initialize_deferred_cleanup() + +def _set_context_restore_fault_for_testing(int status): + """Make the next context restoration on this thread fail with ``status``. + + Test hook for the context save/restore paths in the handle layer. The + injected failure leaves the target context current, exactly as a failing + ``cuCtxSetCurrent`` would, so callers must restore the context themselves. + """ + set_context_restore_fault_for_testing(status) + # ============================================================================= # NVRTC function pointer initialization # ============================================================================= diff --git a/cuda_core/cuda/core/_stream.pyx b/cuda_core/cuda/core/_stream.pyx index e662d67c87f..916a6eb01fe 100644 --- a/cuda_core/cuda/core/_stream.pyx +++ b/cuda_core/cuda/core/_stream.pyx @@ -27,6 +27,7 @@ from cuda.core._context cimport ( from cuda.core._device_resources cimport DeviceResources from cuda.core._event import Event, EventOptions +from cuda.core._resource_handles cimport context_get_device from cuda.core._resource_handles cimport ( ContextHandle, EventHandle, @@ -36,7 +37,6 @@ from cuda.core._resource_handles cimport ( create_stream_handle, create_stream_handle_with_owner, context_get_stream_priority_range, - get_current_context, get_last_error, get_legacy_stream, get_per_thread_stream, @@ -564,10 +564,7 @@ cdef inline int Stream_get_ctx(Stream self, ContextHandle* h_context) except?-1 cdef inline int Stream_get_ctx_device(Stream self, ContextHandle* h_context, int* device_id) except?-1: """Resolve the stream's context handle and device ID.""" - cdef cydriver.CUcontext ctx cdef cydriver.CUdevice target_dev - cdef ContextHandle current_context - cdef bint switch_context cdef bint is_default = Stream_is_default_token(self) with nogil: @@ -575,14 +572,9 @@ cdef inline int Stream_get_ctx_device(Stream self, ContextHandle* h_context, int if self._device_id >= 0 and not is_default: device_id[0] = self._device_id else: - # Get device ID from context, switching context temporarily if needed - current_context = get_current_context() - switch_context = (as_cu(current_context) != as_cu(h_context[0])) - if switch_context: - HANDLE_RETURN(cydriver.cuCtxPushCurrent(as_cu(h_context[0]))) - HANDLE_RETURN(cydriver.cuCtxGetDevice(&target_dev)) - if switch_context: - HANDLE_RETURN(cydriver.cuCtxPopCurrent(&ctx)) + # Query the device with the stream's context current. The handle + # layer restores the caller's context, including on failure. + HANDLE_RETURN(context_get_device(h_context[0], &target_dev)) device_id[0] = target_dev if not is_default: self._device_id = device_id[0] diff --git a/cuda_core/cuda/core/_utils/cuda_utils.pyi b/cuda_core/cuda/core/_utils/cuda_utils.pyi index 51f992fa238..b190e5967d0 100644 --- a/cuda_core/cuda/core/_utils/cuda_utils.pyi +++ b/cuda_core/cuda/core/_utils/cuda_utils.pyi @@ -14,6 +14,26 @@ _fork_warning_checked = False class CUDAError(Exception): ... +class CUDAWarning(RuntimeWarning): + """Warning issued when ``cuda.core`` hits a CUDA error it cannot raise. + + ``cuda.core`` raises exceptions for failures in ordinary calls. Some failures + happen where no exception can propagate: while a resource is released by the + garbage collector or by a CUDA callback, or while a context switch is undone + after the requested operation already succeeded. Those failures are reported + as this warning instead, and the affected resource may have leaked. + + Filter on this category to make such failures fatal in tests:: + + warnings.filterwarnings("error", category=cuda.core.CUDAWarning) + + Because the report comes from a destructor, an escalated warning cannot be + raised into user code; it is delivered through :func:`sys.unraisablehook` + (which pytest surfaces as ``PytestUnraisableExceptionWarning``). + + .. versionadded:: 1.3.0 + """ + class NVRTCError(CUDAError): ... class ComputeCapability(NamedTuple): diff --git a/cuda_core/cuda/core/_utils/cuda_utils.pyx b/cuda_core/cuda/core/_utils/cuda_utils.pyx index ce75746de56..38fc42027df 100644 --- a/cuda_core/cuda/core/_utils/cuda_utils.pyx +++ b/cuda_core/cuda/core/_utils/cuda_utils.pyx @@ -27,6 +27,10 @@ from cuda.bindings.nvjitlink import nvJitLinkError from cpython.buffer cimport PyObject_GetBuffer, PyBuffer_Release, Py_buffer, PyBUF_SIMPLE from cuda.bindings cimport cynvrtc, cynvvm, cynvjitlink +from cuda.core._resource_handles cimport ( + register_warning_category, + take_last_error_detail, +) from cuda.core._utils.driver_cu_result_explanations import DRIVER_CU_RESULT_EXPLANATIONS from cuda.core._utils.runtime_cuda_error_explanations import RUNTIME_CUDA_ERROR_EXPLANATIONS @@ -36,6 +40,31 @@ class CUDAError(Exception): pass +class CUDAWarning(RuntimeWarning): + """Warning issued when ``cuda.core`` hits a CUDA error it cannot raise. + + ``cuda.core`` raises exceptions for failures in ordinary calls. Some failures + happen where no exception can propagate: while a resource is released by the + garbage collector or by a CUDA callback, or while a context switch is undone + after the requested operation already succeeded. Those failures are reported + as this warning instead, and the affected resource may have leaked. + + Filter on this category to make such failures fatal in tests:: + + warnings.filterwarnings("error", category=cuda.core.CUDAWarning) + + Because the report comes from a destructor, an escalated warning cannot be + raised into user code; it is delivered through :func:`sys.unraisablehook` + (which pytest surfaces as ``PytestUnraisableExceptionWarning``). + + .. versionadded:: 1.3.0 + """ + + +# Route the C++ handle layer's non-propagating reports through this category. +register_warning_category(CUDAWarning) + + class NVRTCError(CUDAError): pass @@ -135,19 +164,23 @@ cpdef inline int _check_driver_error(cydriver.CUresult error) except?-1 nogil: if error == cydriver.CUresult.CUDA_SUCCESS: return 0 cdef const char* name + cdef const char* desc + # A context-scoped helper in the handle layer may have recorded why this + # status needs more explanation (e.g. the caller's context was not restored). + cdef const char* detail = take_last_error_detail() name_err = cydriver.cuGetErrorName(error, &name) if name_err != cydriver.CUresult.CUDA_SUCCESS: raise CUDAError(f"UNEXPECTED ERROR CODE: {error}") + desc_err = cydriver.cuGetErrorString(error, &desc) with gil: + suffix = f" ({detail.decode()})" if detail != NULL else "" # TODO: consider lower this to Cython expl = DRIVER_CU_RESULT_EXPLANATIONS.get(int(error)) if expl is not None: - raise CUDAError(f"{name.decode()}: {expl}") - cdef const char* desc - desc_err = cydriver.cuGetErrorString(error, &desc) - if desc_err != cydriver.CUresult.CUDA_SUCCESS: - raise CUDAError(f"{name.decode()}") - raise CUDAError(f"{name.decode()}: {desc.decode()}") + raise CUDAError(f"{name.decode()}: {expl}{suffix}") + if desc_err != cydriver.CUresult.CUDA_SUCCESS: + raise CUDAError(f"{name.decode()}{suffix}") + raise CUDAError(f"{name.decode()}: {desc.decode()}{suffix}") cpdef inline int _check_runtime_error(error) except?-1: diff --git a/cuda_core/cuda/core/graph/_graph_builder.pyx b/cuda_core/cuda/core/graph/_graph_builder.pyx index 071fff38386..7033f90b239 100644 --- a/cuda_core/cuda/core/graph/_graph_builder.pyx +++ b/cuda_core/cuda/core/graph/_graph_builder.pyx @@ -20,6 +20,7 @@ from cuda.core.graph._subclasses cimport ( ExecutableGraphNode, create_executable_node_view, ) +from cuda.core._resource_handles cimport report_cuda_error from cuda.core._resource_handles cimport ( GraphExecHandle, GraphHandle, @@ -860,6 +861,12 @@ cdef class GraphBuilder: if rollback_status == cydriver.CUDA_SUCCESS: invalidate_child_graph_state( self._h_graph, c_new_node) + else: + # The original exception propagates; the failed rollback is + # reported out of band (error handling policy). + report_cuda_error( + b"cuGraphDestroyNode", rollback_status, + b"failed while rolling back a child graph node; the node remains in the graph") raise deps_info_update = [[new_node]] + [None] * (len(deps_info_out) - 1) @@ -990,8 +997,8 @@ cdef inline int GB_end_capture_if_needed(GraphBuilder gb, bint check_status) exc capture. A FORKED builder must not call cuStreamEndCapture: the driver requires forked streams to be joined first. - check_status=True checks the driver return (close()); False ignores it - (__dealloc__). + check_status=True raises on a driver error (close()); False reports it as + a CUDAWarning instead, because nothing can be raised from __dealloc__. """ cdef cydriver.CUgraph c_graph cdef cydriver.CUresult err @@ -1002,6 +1009,10 @@ cdef inline int GB_end_capture_if_needed(GraphBuilder gb, bint check_status) exc err = cydriver.cuStreamEndCapture(c_stream, &c_graph) if check_status: HANDLE_RETURN(err) + else: + report_cuda_error( + b"cuStreamEndCapture", err, + b"failed while releasing a GraphBuilder that was still building") return 0 diff --git a/cuda_core/cuda/core/graph/_graph_node.pyx b/cuda_core/cuda/core/graph/_graph_node.pyx index 7295d786089..831c0f30b69 100644 --- a/cuda_core/cuda/core/graph/_graph_node.pyx +++ b/cuda_core/cuda/core/graph/_graph_node.pyx @@ -45,6 +45,7 @@ from cuda.core.graph._subclasses cimport ( SwitchNode, WhileNode, ) +from cuda.core._resource_handles cimport report_cuda_error from cuda.core._resource_handles cimport ( GraphHandle, GraphNodeHandle, @@ -1114,6 +1115,12 @@ cdef inline ChildGraphNode GN_embed(GraphNode self, GraphDefinition child_def): rollback_status = cydriver.cuGraphDestroyNode(new_node) if rollback_status == cydriver.CUDA_SUCCESS: invalidate_child_graph_state(h_graph, new_node) + else: + # The original exception propagates; the failed rollback is + # reported out of band (error handling policy). + report_cuda_error( + b"cuGraphDestroyNode", rollback_status, + b"failed while rolling back a child graph node; the node remains in the graph") raise return _registered(ChildGraphNode._create_with_params( diff --git a/cuda_core/cuda/core/graph/_subclasses.pyx b/cuda_core/cuda/core/graph/_subclasses.pyx index 2201f7babf0..4967983d947 100644 --- a/cuda_core/cuda/core/graph/_subclasses.pyx +++ b/cuda_core/cuda/core/graph/_subclasses.pyx @@ -29,6 +29,11 @@ from cuda.core.graph._graph_node cimport ( _init_memcpy_params, _resolve_memcpy_operand, ) +from cuda.core._resource_handles cimport ( + ContextHandle, + create_context_handle_ref, + graph_node_set_params, +) from cuda.core._resource_handles cimport ( EventHandle, GraphExecHandle, @@ -124,26 +129,25 @@ cdef void _set_definition_node_params( if node == NULL: raise RuntimeError("GraphNode has been destroyed") _require_graph_node_update_support() - cdef cydriver.CUcontext previous_ctx = NULL - cdef bint restore_ctx = False cdef PreparedAttachment prepared + cdef ContextHandle h_update_ctx + cdef cydriver.CUresult status + cdef cydriver.CUresult restore_status = cydriver.CUresult.CUDA_SUCCESS HANDLE_RETURN(graph_prepare_attachment( h_graph, owner0, owner1, &prepared)) if update_ctx != NULL: - with nogil: - HANDLE_RETURN(cydriver.cuCtxGetCurrent(&previous_ctx)) - if previous_ctx != update_ctx: - HANDLE_RETURN(cydriver.cuCtxSetCurrent(update_ctx)) - restore_ctx = True + h_update_ctx = create_context_handle_ref(update_ctx) + with nogil: + status = graph_node_set_params(node, params, h_update_ctx, &restore_status) + HANDLE_RETURN(status) + # The driver node now references the new owners. Publish their attachment + # before raising anything else: an exception here would roll back the + # prepared retention and leave the node pointing at released resources. try: - with nogil: - HANDLE_RETURN(cydriver.cuGraphNodeSetParams(node, params)) + HANDLE_RETURN(graph_commit_attachment(prepared, node)) finally: - if restore_ctx: - with nogil: - HANDLE_RETURN(cydriver.cuCtxSetCurrent(previous_ctx)) - HANDLE_RETURN(graph_commit_attachment(prepared, node)) + HANDLE_RETURN(restore_status) cdef void _set_executable_node_params( diff --git a/cuda_core/docs/source/api.rst b/cuda_core/docs/source/api.rst index 5ee34d34f54..140b69c0cc8 100644 --- a/cuda_core/docs/source/api.rst +++ b/cuda_core/docs/source/api.rst @@ -298,6 +298,21 @@ DLPack zero-copy interop. Data is moved in and out only by copying — use TextureObject SurfaceObject + +Errors and warnings +------------------- + +Failed CUDA calls raise exceptions; see :doc:`error_handling` for the +guarantees an exception provides and for the situations in which a failure is +reported as a warning instead. + +.. currentmodule:: cuda.core + +.. autosummary:: + :toctree: generated/ + + CUDAWarning + :template: dataclass.rst OpaqueArrayOptions diff --git a/cuda_core/docs/source/error_handling.rst b/cuda_core/docs/source/error_handling.rst new file mode 100644 index 00000000000..a6761aadd24 --- /dev/null +++ b/cuda_core/docs/source/error_handling.rst @@ -0,0 +1,127 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +.. currentmodule:: cuda.core + +Error Handling +============== + +``cuda.core`` reports failures with Python exceptions. This page describes what +an exception from ``cuda.core`` guarantees about the state it leaves behind, +what happens when a failure occurs where no exception can be raised, and the +few situations in which ``cuda.core`` cannot fully undo a failed operation. + +Exceptions +---------- + +A CUDA driver, runtime, NVRTC, NVVM or nvJitLink call that fails raises an +exception (``CUDAError`` for driver and runtime failures) whose message contains +the CUDA error name and its description. Invalid arguments and misuse raise the +usual Python exception types (``TypeError``, ``ValueError``, ``RuntimeError``). + +When a ``cuda.core`` call raises, the following hold: + +- A call that creates a resource creates nothing. If a later step of the call + fails after the resource was created, the resource is destroyed before the + exception propagates. +- The calling thread's current CUDA context is the one that was current when + the call began. The only method that changes the current context on purpose + is :meth:`Device.set_current`; every other method that must run in a + different context restores the caller's context before returning, whether it + succeeds or fails. See `Context restoration failures`_ for the one case in + which the driver refuses to restore it. +- Objects that were modified by a call that failed midway remain usable and + consistent, but some operations do not have an all-or-nothing outcome. Their + documentation says so where it applies (for example the graph mutation + methods that add several driver edges). + +``cuda.core`` does not swallow driver errors. A failure that would otherwise be +hidden, for example because it occurred while another exception was already +propagating, is reported as described in the next section. + +Failures that cannot be raised +------------------------------ + +Some ``cuda.core`` code runs where no Python exception can propagate: + +- resources released by the garbage collector or by the deferred cleanup of + CUDA graphs, and the CUDA driver calls those releases make; +- callbacks invoked by CUDA; +- cleanup performed after an operation has already failed, such as rolling back + a partially built graph node or restoring the caller's CUDA context. + +A CUDA error in one of these places is reported as a :class:`CUDAWarning`. The +message names the failed driver call and the CUDA error. The warning means the +affected resource may have leaked; ``cuda.core`` never leaves a resource in use +by CUDA with its memory released (it prefers a leak to a dangling pointer). + +:class:`CUDAWarning` derives from :class:`RuntimeWarning`, so it is shown by +default and can be filtered like any other warning. To make these failures +loud in a test suite:: + + import warnings + import cuda.core + + warnings.filterwarnings("error", category=cuda.core.CUDAWarning) + +Because the report comes from a destructor or callback, an escalated warning is +delivered through :func:`sys.unraisablehook` rather than raised into user code. +pytest reports it as ``PytestUnraisableExceptionWarning``, which its +``-W error`` option turns into a test failure. + +``CUDA_ERROR_DEINITIALIZED`` is not reported. It means the CUDA driver is +shutting down, which happens during process exit; cleanup failures at that +point are expected and there is nothing left to clean up. + +Context restoration failures +---------------------------- + +Methods that run in a context other than the current one, such as +:meth:`Device.create_stream` when another device is current, switch the current +context, perform the driver call, and switch back. Restoring the caller's +context can fail only when the driver is shutting down +(``CUDA_ERROR_DEINITIALIZED``), when the caller's context was destroyed in the +meantime (``CUDA_ERROR_INVALID_CONTEXT``), or when the driver is reporting an +earlier, unrecoverable kernel fault (see `Sticky errors`_). None of these can +be fixed by retrying, so ``cuda.core`` does not retry. + +When restoration fails in an ordinary call, the resource created by the call is +destroyed and a ``CUDAError`` is raised whose message states that the caller's +context could not be restored and which context is now current. Call +:meth:`Device.set_current` before issuing further CUDA work on that thread. + +When restoration fails inside a destructor or callback, a :class:`CUDAWarning` +is issued and the thread keeps the context that the cleanup used. + +Sticky errors +------------- + +Some CUDA errors mark the process as unusable for further CUDA work, for +example ``CUDA_ERROR_ILLEGAL_ADDRESS`` or ``CUDA_ERROR_LAUNCH_FAILED`` after a +kernel fault. The CUDA documentation calls for the process to be terminated and +relaunched after such an error, and every later CUDA call returns the same +error. Because these faults are detected asynchronously, the call that first +raises the error is often unrelated to the kernel that caused it. + +``cuda.core`` raises these errors like any other and does not attempt to +recover from them. It does not terminate the process for you: the exception +carries the Python traceback of the call that observed the fault, and your +application decides how to shut down. + +Interpreter shutdown +-------------------- + +Once the interpreter starts finalizing, ``cuda.core`` no longer touches Python +objects from CUDA callbacks or destructors. Resources whose release would +require Python at that point are intentionally leaked; the operating system and +the driver reclaim them when the process exits. Release all ``cuda.core`` +objects explicitly (with ``close()`` or a ``with`` block) if their deterministic +release matters. + +Process termination +------------------- + +``cuda.core`` does not abort the process in response to a CUDA error, including +errors that cannot be raised, and including failures to restore the caller's +context. Aborting is reserved for an internal invariant violation where +continuing could corrupt memory, and no such code path exists in this release. diff --git a/cuda_core/docs/source/index.rst b/cuda_core/docs/source/index.rst index 34c0933ffb8..373e6b61665 100644 --- a/cuda_core/docs/source/index.rst +++ b/cuda_core/docs/source/index.rst @@ -16,6 +16,7 @@ Welcome to the documentation for ``cuda.core``. examples interoperability concurrency + error_handling api api_nvml environment_variables diff --git a/cuda_core/docs/source/release/1.3.0-notes.rst b/cuda_core/docs/source/release/1.3.0-notes.rst index 37ff06b34e5..3e94b19a77b 100644 --- a/cuda_core/docs/source/release/1.3.0-notes.rst +++ b/cuda_core/docs/source/release/1.3.0-notes.rst @@ -6,6 +6,18 @@ ``cuda.core`` 1.3.0 Release Notes ================================== +New features +------------ + +- Added :class:`CUDAWarning`, the warning category ``cuda.core`` uses for CUDA + errors that cannot be raised, such as a failed driver call while a resource + is released by the garbage collector. Filter on it with + ``warnings.filterwarnings("error", category=cuda.core.CUDAWarning)`` to make + such failures loud. The new :doc:`error handling <../error_handling>` page + documents what an exception from ``cuda.core`` guarantees, how failures that + cannot be raised are reported, and how context restoration failures and + sticky CUDA errors are handled. + Fixes and enhancements ---------------------- @@ -35,3 +47,42 @@ Fixes and enhancements ``RuntimeError``. :attr:`Buffer.device_id` on such a buffer returns ``-1`` as well, which also lets a pinned buffer back a linear or pitched texture resource. + +- Cleanup failures are now reported as :class:`CUDAWarning` instead of being + written to ``stderr`` with ``print`` or ``fprintf``, so they can be filtered, + captured with :func:`warnings.catch_warnings`, and escalated. Failures of + ``cuStreamDestroy``, ``cuEventDestroy``, ``cuMemFree``, ``cuMemFreeAsync``, + ``cuMemFreeHost``, ``cuMemPoolDestroy``, ``cuGreenCtxDestroy``, + ``cuGraphDestroy``, ``cuGraphExecDestroy``, ``cuGraphicsUnregisterResource``, + ``cuLinkDestroy``, ``cuArrayDestroy``, ``cuMipmappedArrayDestroy``, + ``cuTexObjectDestroy``, ``cuSurfObjectDestroy``, user-object releases and the + NVRTC, NVVM and nvJitLink destroy calls made from destructors were previously + discarded; they are now reported. ``CUDA_ERROR_DEINITIALIZED`` (the driver is + shutting down) is not reported. Test code that matched the old ``stderr`` + text uses ``pytest.warns(CUDAWarning)`` instead. + +- When a :class:`Device` method has to run in the device's context and the + caller's context cannot be restored afterwards, the created resource is + destroyed and the raised ``CUDAError`` now states that the caller's context + could not be restored and which context is current. Previously the error + named only the driver status of the failed ``cuCtxSetCurrent`` call. A + restoration failure during resource cleanup is reported as + :class:`CUDAWarning`. + +- Updating a memcpy or memset graph node whose context differs from the current + one no longer risks a dangling node parameter when the caller's context + cannot be restored after the update: the resources referenced by the new + parameters are now retained before the restoration failure is raised. + +- :meth:`Device.set_current` with an explicit :class:`Context` now switches + contexts with a single driver call, so a failure leaves the previous context + current instead of leaving the thread with no context. It also works when no + context is current, returning ``None``. + +- Failed rollbacks of a partially embedded child graph node and failed + ``cuStreamEndCapture`` calls made when a still-building + :class:`~graph.GraphBuilder` is garbage collected are now reported as + :class:`CUDAWarning`; both were silent. + +- :attr:`Stream.device` and related queries on a stream whose context is not + current now restore the caller's context even when the device query fails. diff --git a/cuda_core/tests/helpers/contexts.py b/cuda_core/tests/helpers/contexts.py index 7ee01bb255f..f613f0ac336 100644 --- a/cuda_core/tests/helpers/contexts.py +++ b/cuda_core/tests/helpers/contexts.py @@ -1,18 +1,36 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import warnings from contextlib import contextmanager +from cuda.core import CUDAWarning from cuda.core._utils.cuda_utils import driver, handle_return __all__ = [ "assert_device_operations_use_bound_context", + "assert_no_cuda_warning", "current_context_handle", "no_current_context", "use_context", ] +@contextmanager +def assert_no_cuda_warning(): + """Fail if a :class:`CUDAWarning` is issued inside the block. + + Cleanup paths cannot raise, so a driver failure there surfaces only as a + warning; this makes such a failure a test failure. Tests using it must be + marked ``thread_unsafe``: warning capture is process-global. + """ + with warnings.catch_warnings(record=True) as records: + warnings.simplefilter("always", CUDAWarning) + yield + cuda_warnings = [str(record.message) for record in records if issubclass(record.category, CUDAWarning)] + assert not cuda_warnings, f"unexpected CUDAWarning(s): {cuda_warnings}" + + def current_context_handle(): """Return the current CUDA context handle, or zero if none is current.""" return int(handle_return(driver.cuCtxGetCurrent())) diff --git a/cuda_core/tests/test_error_handling.py b/cuda_core/tests/test_error_handling.py new file mode 100644 index 00000000000..e716b001f66 --- /dev/null +++ b/cuda_core/tests/test_error_handling.py @@ -0,0 +1,207 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the error handling policy (docs/source/error_handling.rst). + +Ordinary calls raise and leave the caller's context untouched; failures that +cannot be raised are reported as CUDAWarning; and a failure to restore the +caller's context is raised (or reported) with an explanation rather than +swallowed or turned into a process abort. Restoration failures are injected with +the handle layer's test hook, which leaves the target context current exactly as +a real ``cuCtxSetCurrent`` failure would, so every test here restores the +context stack itself. +""" + +import ctypes +from contextlib import contextmanager + +import pytest +from helpers.constants import POOL_SIZE +from helpers.contexts import assert_no_cuda_warning, current_context_handle + +import cuda.core +from cuda.core import ( + CUDAWarning, + DeviceMemoryResource, + DeviceMemoryResourceOptions, + LegacyPinnedMemoryResource, +) +from cuda.core._resource_handles import _set_context_restore_fault_for_testing +from cuda.core._stream import default_stream +from cuda.core._utils.cuda_utils import CUDAError, driver, handle_return +from cuda.core._utils.version import binding_version, driver_version +from cuda.core.graph import GraphDefinition + +INVALID_CONTEXT = int(driver.CUresult.CUDA_ERROR_INVALID_CONTEXT) + +thread_unsafe_context_fault = pytest.mark.thread_unsafe( + reason="injects a thread-local restoration fault and mutates the CUDA context stack" +) + + +@contextmanager +def no_context_with_restore_fault(status=INVALID_CONTEXT): + """Pop the current context and make the next restoration fail with ``status``. + + On exit, drop whatever the failed restoration left current, clear an unused + fault, and push the popped context back. + """ + previous = handle_return(driver.cuCtxPopCurrent()) + assert current_context_handle() == 0 + _set_context_restore_fault_for_testing(status) + try: + yield + finally: + _set_context_restore_fault_for_testing(0) + handle_return(driver.cuCtxSetCurrent(driver.CUcontext(0))) + handle_return(driver.cuCtxPushCurrent(previous)) + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_cudawarning_is_public_and_shown_by_default(): + assert "CUDAWarning" in cuda.core.__all__ + assert issubclass(CUDAWarning, RuntimeWarning) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_create_stream_raises_when_context_cannot_be_restored(init_cuda): + """Creation is undone and the error explains the context state; no abort, no warning.""" + dev = init_cuda + with no_context_with_restore_fault(): + with assert_no_cuda_warning(), pytest.raises(CUDAError, match="could not be restored") as excinfo: + dev.create_stream() + message = str(excinfo.value) + assert "CUDA_ERROR_INVALID_CONTEXT" in message + assert "Device.set_current()" in message + # As documented, a failed restoration leaves the device's context current. + assert current_context_handle() == int(dev.context.handle) + assert current_context_handle() == int(dev.context.handle) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_sync_raises_when_context_cannot_be_restored(init_cuda): + """A context-scoped call without a created resource raises the same explanation.""" + dev = init_cuda + with no_context_with_restore_fault(): + with pytest.raises(CUDAError, match="could not be restored"): + dev.sync() + assert current_context_handle() == int(dev.context.handle) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_unused_restore_fault_does_not_fire_without_a_context_switch(init_cuda): + """The hook only affects restorations; a call in the current context never restores.""" + dev = init_cuda + _set_context_restore_fault_for_testing(INVALID_CONTEXT) + try: + stream = dev.create_stream() + stream.close() + finally: + _set_context_restore_fault_for_testing(0) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_cleanup_reports_restore_failure_as_warning(mempool_device): + """A restoration failure inside a destructor cannot raise, so it is reported.""" + dev = mempool_device + mr = DeviceMemoryResource(dev, DeviceMemoryResourceOptions(max_size=POOL_SIZE)) + # A default-stream deallocation records the allocating context, so freeing + # with no context current switches to it and must switch back. + buf = mr.allocate(256, stream=default_stream()) + with no_context_with_restore_fault(): + with pytest.warns(CUDAWarning, match="restoring the caller's context") as records: + buf.close() + assert any("CUDA_ERROR_INVALID_CONTEXT" in str(record.message) for record in records) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_escalated_cudawarning_from_cleanup_is_not_a_crash(mempool_device): + """With CUDAWarning promoted to an error, a destructor-path report cannot be raised. + + It is delivered through sys.unraisablehook instead; the process continues and + the resource release still runs. pytest surfaces the hook as a warning, so the + hook is replaced here to keep the test's own outcome deterministic. + """ + import sys + import warnings + + dev = mempool_device + mr = DeviceMemoryResource(dev, DeviceMemoryResourceOptions(max_size=POOL_SIZE)) + buf = mr.allocate(256, stream=default_stream()) + unraisable = [] + previous_hook = sys.unraisablehook + sys.unraisablehook = unraisable.append + try: + with no_context_with_restore_fault(), warnings.catch_warnings(): + warnings.simplefilter("error", CUDAWarning) + buf.close() + finally: + sys.unraisablehook = previous_hook + assert len(unraisable) == 1 + assert issubclass(unraisable[0].exc_type, CUDAWarning) + assert "restoring the caller's context" in str(unraisable[0].exc_value) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_set_current_with_context_works_without_a_current_context(init_cuda): + """set_current(ctx) binds in one driver call; no previous context means None.""" + dev = init_cuda + ctx = dev.context + previous = handle_return(driver.cuCtxPopCurrent()) + try: + assert current_context_handle() == 0 + assert dev.set_current(ctx) is None + assert current_context_handle() == int(ctx.handle) + finally: + # Leave exactly one context on the stack, as the fixture expects. + handle_return(driver.cuCtxSetCurrent(driver.CUcontext(0))) + handle_return(driver.cuCtxPushCurrent(previous)) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_memset_update_keeps_new_owners_alive_when_context_cannot_be_restored(device_x2): + """The node's new parameters stay valid: the attachment is published before the + restoration failure is raised, so the updated graph instantiates and runs.""" + if driver_version() < (13, 2, 0) or binding_version() < (13, 2, 0): + pytest.skip("node contexts are only recorded by cuGraphNodeGetParams on CUDA 13.2+") + node_dev, other_dev = device_x2 + node_dev.set_current() + memory_resource = LegacyPinnedMemoryResource() + dst = memory_resource.allocate(4) + replacement = memory_resource.allocate(4) + graph_def = GraphDefinition() + node = graph_def.memset(dst, 0x11, 4) + + # Updating from another device's context switches to the node's context and + # must switch back; make that restoration fail. + other_dev.set_current() + _set_context_restore_fault_for_testing(INVALID_CONTEXT) + try: + with pytest.raises(CUDAError, match="could not be restored"): + node.update(dst=replacement, value=0x22) + finally: + _set_context_restore_fault_for_testing(0) + node_dev.set_current() + + def as_bytes(buffer): + return (ctypes.c_uint8 * 4).from_address(int(buffer.handle)) + + as_bytes(dst)[:] = [0] * 4 + as_bytes(replacement)[:] = [0] * 4 + graph = graph_def.instantiate() + stream = node_dev.create_stream() + graph.launch(stream) + stream.sync() + # The driver applied the update, and the replacement buffer it references + # is still retained by the graph rather than dangling. + assert list(as_bytes(replacement)) == [0x22] * 4 + assert list(as_bytes(dst)) == [0] * 4 + graph.close() + stream.close() diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index 769d780b36b..661e540c308 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -23,7 +23,7 @@ thread_unsafe_on_windows, ) from helpers.constants import POOL_SIZE -from helpers.contexts import current_context_handle, no_current_context +from helpers.contexts import assert_no_cuda_warning, current_context_handle, no_current_context from helpers.memory import ( create_managed_memory_resource_or_skip, create_pinned_memory_resource_or_xfail, @@ -33,6 +33,7 @@ from cuda.core import ( Buffer, + CUDAWarning, Device, DeviceMemoryResource, DeviceMemoryResourceOptions, @@ -779,23 +780,25 @@ def test_from_handle_mr_explicit_stream_without_current_context(buffer_type): assert telemetry["deallocations"][-1]["stream"].handle == stream.handle +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="gpt-5.6") -def test_mr_deallocation_failure_warns(capfd): - """Destructor-path MR failures are contained and reported.""" +def test_mr_deallocation_failure_warns(): + """Destructor-path MR failures are contained and reported as CUDAWarning.""" device = Device() device.set_current() FailingMR, _ = make_instrumented_memory_resource(deallocate_error=RuntimeError("expected deallocation failure")) buf = Buffer.from_handle(1, 1024, mr=FailingMR(device)) - buf.close() - assert ( - "Warning: mr.deallocate() failed during Buffer destruction: expected deallocation failure" - ) in capfd.readouterr().err + with pytest.warns( + CUDAWarning, match=r"mr\.deallocate\(\) failed during Buffer destruction.*expected deallocation failure" + ): + buf.close() +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="cursor-grok-4.5") @pytest.mark.parametrize("replace_stream", [False, True]) -def test_mr_deallocation_without_current_context(init_cuda, capsys, replace_stream): +def test_mr_deallocation_without_current_context(init_cuda, replace_stream): """MR-backed Buffer teardown activates the recorded context when none is current.""" TrackingMR, telemetry = make_instrumented_memory_resource(DummyDeviceMemoryResource, track_active=True) mr = TrackingMR(init_cuda) @@ -806,16 +809,17 @@ def test_mr_deallocation_without_current_context(init_cuda, capsys, replace_stre with no_current_context(): assert current_context_handle() == 0 - buf.close(stream) + with assert_no_cuda_warning(): + buf.close(stream) assert len(telemetry["active"]) == 0 assert current_context_handle() == 0 - assert "mr.deallocate() failed" not in capsys.readouterr().err +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="cursor-grok-4.5") @pytest.mark.parametrize("replace_stream", [False, True]) -def test_mr_deallocation_with_foreign_context(device_x2, capsys, replace_stream): +def test_mr_deallocation_with_foreign_context(device_x2, replace_stream): """MR-backed Buffer teardown switches away from an unrelated current context.""" alloc_dev, foreign_dev = device_x2 alloc_dev.set_current() @@ -832,11 +836,11 @@ def test_mr_deallocation_with_foreign_context(device_x2, capsys, replace_stream) assert foreign_ctx != alloc_ctx try: - buf.close(stream) + with assert_no_cuda_warning(): + buf.close(stream) assert len(telemetry["active"]) == 0 assert current_context_handle() == foreign_ctx - assert "mr.deallocate() failed" not in capsys.readouterr().err finally: alloc_dev.set_current() @@ -856,8 +860,9 @@ def test_mr_deallocate_raises_on_driver_error(mempool_device): mr.deallocate(0xDEADBEEF, 256, stream=stream) +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="cursor-grok-4.5") -def test_pool_buffer_deallocates_without_current_context(mempool_device, capfd): +def test_pool_buffer_deallocates_without_current_context(mempool_device): """Pool Buffer.close frees on the recorded stream with no current context.""" dev = mempool_device stream = dev.create_stream() @@ -870,18 +875,17 @@ def test_pool_buffer_deallocates_without_current_context(mempool_device, capfd): with no_current_context(): assert current_context_handle() == 0 - buf.close() + with assert_no_cuda_warning(): + buf.close() stream.sync() assert mr.attributes.used_mem_current < used_after_alloc assert current_context_handle() == 0 - err = capfd.readouterr().err - assert "cuMemFreeAsync failed" not in err - assert "mr.deallocate() failed" not in err +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="cursor-grok-4.5") -def test_pool_buffer_deallocates_with_foreign_context(mempool_device_x2, capfd): +def test_pool_buffer_deallocates_with_foreign_context(mempool_device_x2): """Pool Buffer.close frees under the recorded context while another is current.""" alloc_dev, foreign_dev = mempool_device_x2 alloc_dev.set_current() @@ -899,7 +903,8 @@ def test_pool_buffer_deallocates_with_foreign_context(mempool_device_x2, capfd): assert foreign_ctx != alloc_ctx try: - buf.close() + with assert_no_cuda_warning(): + buf.close() assert current_context_handle() == foreign_ctx # Observe the free on the allocation device, then restore the foreign context. @@ -907,9 +912,6 @@ def test_pool_buffer_deallocates_with_foreign_context(mempool_device_x2, capfd): stream.sync() assert mr.attributes.used_mem_current < used_after_alloc foreign_dev.set_current() - - err = capfd.readouterr().err - assert "cuMemFreeAsync failed" not in err finally: alloc_dev.set_current() From 963e0b63e3b09ca70e7563d57830744b23a6a3a1 Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Thu, 3 Sep 2026 07:07:33 -0700 Subject: [PATCH 02/14] cuda.core: keep the texture autosummary contiguous in api.rst The "Errors and warnings" section was inserted between the texture classes and the texture option dataclasses, which moved OpaqueArrayOptions, MipmappedArrayOptions and TextureObjectOptions under cuda.core in the docs index and failed test_api_docs_consistency on every CI platform. Place the section after the texture section instead. Co-Authored-By: Claude Fable 5.1 --- cuda_core/docs/source/api.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/cuda_core/docs/source/api.rst b/cuda_core/docs/source/api.rst index 140b69c0cc8..9fb6a8eb718 100644 --- a/cuda_core/docs/source/api.rst +++ b/cuda_core/docs/source/api.rst @@ -298,6 +298,19 @@ DLPack zero-copy interop. Data is moved in and out only by copying — use TextureObject SurfaceObject + :template: dataclass.rst + + OpaqueArrayOptions + MipmappedArrayOptions + TextureObjectOptions + +The associated enumerations — +:class:`~cuda.core.typing.ArrayFormatType`, +:class:`~cuda.core.typing.AddressModeType`, +:class:`~cuda.core.typing.FilterModeType`, and +:class:`~cuda.core.typing.ReadModeType` — live in :mod:`cuda.core.typing` +alongside the other ``cuda.core`` enumerations. + Errors and warnings ------------------- @@ -313,19 +326,6 @@ reported as a warning instead. CUDAWarning - :template: dataclass.rst - - OpaqueArrayOptions - MipmappedArrayOptions - TextureObjectOptions - -The associated enumerations — -:class:`~cuda.core.typing.ArrayFormatType`, -:class:`~cuda.core.typing.AddressModeType`, -:class:`~cuda.core.typing.FilterModeType`, and -:class:`~cuda.core.typing.ReadModeType` — live in :mod:`cuda.core.typing` -alongside the other ``cuda.core`` enumerations. - CUDA process checkpointing -------------------------- From 7e2ac8c80c5a60a25d7195a8f612e4296e598b36 Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Thu, 3 Sep 2026 09:39:53 -0700 Subject: [PATCH 03/14] cuda.core: attach secondary failures to the propagating exception as notes Review follow-ups on the error-handling policy: - A failure that happens while an exception is being raised is no longer reported out of band. When both an operation and the restoration of the caller's context fail, the operation's CUDAError is raised with the restoration failure attached; when only the restoration fails, its error is raised with the context explanation attached. The attachment is a PEP 678 note on Python 3.11+ and is appended to the message on 3.10. The thread-local detail is keyed to the status it was recorded for, so it cannot attach to an unrelated error if that status is never raised. - A failed rollback inside a Cython `except` block is attached to the exception being handled through note_or_report_cuda_error(), which falls back to a CUDAWarning when nothing is being handled or notes are unavailable. - Reporting stays reserved for destructors and CUDA callbacks; CUDAWarning's docstring and the docs say so. - DESIGN.md explains the two status conventions of the C++ layer (handle factories use thread-local err, everything else returns CUresult) and the abort-helper guidance in AGENTS.md asks for a faulthandler-style traceback. - Drop the release-relative "in this release" wording from the stable docs. Co-Authored-By: Claude Fable 5.1 --- cuda_core/AGENTS.md | 16 ++- cuda_core/cuda/core/_cpp/DESIGN.md | 39 ++++-- cuda_core/cuda/core/_cpp/resource_handles.cpp | 114 +++++++++++++----- cuda_core/cuda/core/_cpp/resource_handles.hpp | 16 ++- cuda_core/cuda/core/_resource_handles.pxd | 4 +- cuda_core/cuda/core/_resource_handles.pyi | 7 ++ cuda_core/cuda/core/_resource_handles.pyx | 20 ++- cuda_core/cuda/core/_utils/cuda_utils.pyi | 7 +- cuda_core/cuda/core/_utils/cuda_utils.pyx | 32 +++-- cuda_core/cuda/core/graph/_graph_builder.pyx | 8 +- cuda_core/cuda/core/graph/_graph_node.pyx | 8 +- cuda_core/docs/source/error_handling.rst | 26 ++-- cuda_core/docs/source/release/1.3.0-notes.rst | 19 +-- cuda_core/tests/test_error_handling.py | 101 ++++++++++++++-- 14 files changed, 314 insertions(+), 103 deletions(-) diff --git a/cuda_core/AGENTS.md b/cuda_core/AGENTS.md index 1e7c8da1077..fa1b2ce5c51 100644 --- a/cuda_core/AGENTS.md +++ b/cuda_core/AGENTS.md @@ -127,16 +127,18 @@ below are for contributors. Reviewers and agents should flag violations. resources anyway (leak) rather than release them; a leak is always preferred to a use-after-free. - **Non-propagating paths never raise and never discard a status**: shared_ptr - deleters, `__dealloc__`, CUDA callbacks and cleanup after a failure report - through one channel, `report_cuda_error()` / `report_message()` in C++ (the + deleters, `__dealloc__` and CUDA callbacks report through one channel, `report_cuda_error()` / `report_message()` in C++ (the `pw_*` wrappers) or `warnings.warn(..., CUDAWarning)` in Cython and Python, which emits `cuda.core.CUDAWarning`. No `print(file=sys.stderr)` and no `fprintf` outside that helper. `CUDA_ERROR_DEINITIALIZED` is filtered by the helper because it means the driver is shutting down. - **Rollback failure**: the original exception propagates; the failed rollback - is reported out of band (or chained with `raise ... from` when a second - exception must be raised). Bare `except:` is acceptable only for - rollback-then-`raise` blocks. + is attached to it with `note_or_report_cuda_error()` (a PEP 678 note on + Python 3.11+, reported out-of-band on 3.10), or chained with + `raise ... from` when a second exception must be raised. Catching everything + (bare `except:` or `except BaseException:`) is acceptable only for + rollback-then-`raise` blocks, where the rollback must also run for + `KeyboardInterrupt`. - **Finalization**: once `py_is_finalizing()` is true, do no Python work from destructors or callbacks and accept the leak (see `_cpp/resource_handles.hpp` and `_cpp/GRAPH_ATTACHMENTS.md`). @@ -146,7 +148,9 @@ below are for contributors. Reviewers and agents should flag violations. call, including a failed context restoration, never qualifies: raise or report instead. There is currently no such path; if one is ever needed it must go through a single helper that writes a diagnostic (call, CUDA error, - invariant, "please report") to stderr before aborting, must never trigger + invariant, "please report") and a Python traceback of all threads to stderr + before aborting (as `faulthandler` does, via the GIL-free + `_Py_DumpTracebackThreads`; no Python-object work), must never trigger during interpreter finalization or for driver-shutdown errors, and must be called out in the docs and release notes. An *implicit* abort (an exception escaping a `noexcept` function or a deleter, including `std::bad_alloc` from diff --git a/cuda_core/cuda/core/_cpp/DESIGN.md b/cuda_core/cuda/core/_cpp/DESIGN.md index 8e1a55a34a3..d0f2fdcc799 100644 --- a/cuda_core/cuda/core/_cpp/DESIGN.md +++ b/cuda_core/cuda/core/_cpp/DESIGN.md @@ -275,9 +275,16 @@ Related functions: - `peek_last_error()`: Returns the error without clearing it - `clear_last_error()`: Clears the error state -Some functions return a `CUresult` directly instead of a handle (for example -`context_synchronize`, `context_get_device`, `graph_node_set_params`). Their -callers `HANDLE_RETURN` the value. +The C++ layer never raises Python exceptions: it runs `nogil` and `noexcept`, +and is called from deleters, CUDA callbacks and GIL-released code where raising +is impossible. Status is turned into `CUDAError` in one place, `HANDLE_RETURN` +in the Cython layer. Which status convention a function uses is decided by its +return value. Factories return the handle, so their status goes to thread-local +`err` and is read with `get_last_error()`. Functions that do not produce a +handle (`context_synchronize`, `context_get_device`, `graph_node_set_params`, +the `graph_*_attachment` family, `deviceptr_alloc_raw`) return the `CUresult` +directly and deliver results through out-parameters, mirroring the driver API; +their callers `HANDLE_RETURN` the value. The two conventions never mix. ### Context-scoped operations @@ -285,19 +292,22 @@ Operations that must run in a specific context use `invoke_in_context` / `invoke_in_context_or_undo` (propagating paths) and `cleanup_in_context` (deleters). They switch the current context, run the operation, and restore the caller's context. When restoration fails after the operation succeeded, the -creation is undone and the restoration status is returned; the helper also -records a thread-local detail (`take_last_error_detail()`) that the Cython error -path appends to the raised `CUDAError`, so the user learns that the caller's -context was not restored and which context is current. When both the operation -and the restoration fail, the operation status is returned and the restoration -failure is reported out of band. Tests inject restoration failures with +creation is undone and the restoration status is returned. When both fail, the +operation status is returned. Either way the helper records a thread-local +detail keyed to the returned status (`take_last_error_detail(status)`) that +`_check_driver_error` attaches to the raised `CUDAError` as a PEP 678 note +(appended to the message on Python 3.10), so the user learns that the caller's +context was not restored, which context is current and, for a double failure, +why restoration failed. Keying the detail to its status keeps it from attaching +to an unrelated error if the caller never raises that status; `enter_context` +clears any stale detail. Tests inject restoration failures with `set_context_restore_fault_for_testing()`. ### Reporting from non-propagating paths -Deleters, CUDA callbacks and cleanup-after-failure cannot raise. They report -through `report_cuda_error()` / `report_message()` (the `pw_*` wrappers -decorate destroy calls with it), which emit a `cuda.core.CUDAWarning` through +Deleters and CUDA callbacks cannot raise. They report through +`report_cuda_error()` / `report_message()` (the `pw_*` wrappers decorate +destroy calls with it), which emit a `cuda.core.CUDAWarning` through the Python warnings machinery when the interpreter is usable, deliver an escalated warning as an unraisable exception, and fall back to stderr when the GIL cannot be taken (for example during finalization). `CUDA_ERROR_DEINITIALIZED` @@ -306,6 +316,11 @@ discarded silently anywhere in this layer, and nothing in this layer terminates the process; see `docs/source/error_handling.rst` and the "Failure handling" section of `AGENTS.md` for the policy. +A rollback that fails inside a Cython `except` block is not a non-propagating +path: `note_or_report_cuda_error()` attaches it as a note to the exception being +handled (`PyErr_GetHandledException`, Python 3.11+) and falls back to a report +only when there is no such exception or notes are unavailable. + ## Usage from Cython ```cython diff --git a/cuda_core/cuda/core/_cpp/resource_handles.cpp b/cuda_core/cuda/core/_cpp/resource_handles.cpp index 75d2d27bc7d..fc05d376487 100644 --- a/cuda_core/cuda/core/_cpp/resource_handles.cpp +++ b/cuda_core/cuda/core/_cpp/resource_handles.cpp @@ -211,11 +211,12 @@ class GILAcquireGuard { // Warning category registered by _resource_handles.pyx (cuda.core.CUDAWarning). std::atomic warning_category{nullptr}; -// Thread-local detail attached to the next raised CUDAError (see -// take_last_error_detail()). Written only by propagating helpers. The taken -// copy stays valid until the next take on the same thread. -thread_local char last_error_detail[256] = {0}; -thread_local char taken_error_detail[256] = {0}; +// Thread-local detail attached to the next raised CUDAError with a matching +// status (see take_last_error_detail()). Written only by propagating helpers. +// The taken copy stays valid until the next take on the same thread. +thread_local char last_error_detail[512] = {0}; +thread_local char taken_error_detail[512] = {0}; +thread_local CUresult last_error_detail_status = CUDA_SUCCESS; // Thread-local fault injected into the next context restoration (tests only). thread_local CUresult context_restore_fault = CUDA_SUCCESS; @@ -295,17 +296,62 @@ void report_cuda_error(const char* operation, CUresult status, const char* detai report_message(message); } -const char* take_last_error_detail() noexcept { - if (!last_error_detail[0]) { +namespace { + +// Attach `message` as a PEP 678 note to the exception currently being handled. +// Returns false when there is none or the interpreter cannot be used. +bool add_note_to_handled_exception(const char* message) noexcept { +#if PY_VERSION_HEX >= 0x030B0000 + if (!Py_IsInitialized() || py_is_finalizing()) { + return false; + } + GILAcquireGuard gil; + if (!gil.acquired()) { + return false; + } + PyObject* exc = PyErr_GetHandledException(); + if (!exc) { + return false; + } + PyObject* result = PyObject_CallMethod(exc, "add_note", "s", message); + Py_DECREF(exc); + if (!result) { + PyErr_Clear(); + return false; + } + Py_DECREF(result); + return true; +#else + (void)message; + return false; +#endif +} + +} // namespace + +void note_or_report_cuda_error(const char* operation, CUresult status, const char* detail) noexcept { + if (status == CUDA_SUCCESS || status == CUDA_ERROR_DEINITIALIZED) { + return; + } + char message[512]; + format_cuda_error(message, sizeof(message), operation, status, detail); + if (!add_note_to_handled_exception(message)) { + report_message(message); + } +} + +const char* take_last_error_detail(CUresult status) noexcept { + if (!last_error_detail[0] || status != last_error_detail_status) { return nullptr; } std::memcpy(taken_error_detail, last_error_detail, sizeof(taken_error_detail)); - last_error_detail[0] = 0; + clear_last_error_detail(); return taken_error_detail; } void clear_last_error_detail() noexcept { last_error_detail[0] = 0; + last_error_detail_status = CUDA_SUCCESS; } void set_context_restore_fault_for_testing(CUresult status) noexcept { @@ -349,36 +395,47 @@ CUresult restore_context(CUcontext previous) noexcept { return p_cuCtxSetCurrent(previous); } -// Record why the CUresult about to be returned should be explained further -// when it is raised as a CUDAError: the caller's context was not restored. -void note_context_not_restored(CUcontext previous) noexcept { +// Record that the caller's context was not restored as the detail of the +// CUresult about to be returned and raised: the operation status if the +// operation failed too, else the restoration status. For a double failure the +// detail also names the restoration error, which the raised error does not. +void note_context_not_restored(CUcontext previous, CUresult operation_status, + CUresult restore_status) noexcept { CUcontext current = nullptr; if (p_cuCtxGetCurrent(¤t) != CUDA_SUCCESS) { current = nullptr; } + char cause[128] = {0}; + if (operation_status != CUDA_SUCCESS) { + const char* error_name = nullptr; + if (p_cuGetErrorName && p_cuGetErrorName(restore_status, &error_name) == CUDA_SUCCESS) { + std::snprintf(cause, sizeof(cause), " after this failure (cuCtxSetCurrent: %s)", error_name); + } else { + std::snprintf(cause, sizeof(cause), " after this failure (cuCtxSetCurrent: CUDA error %d)", + static_cast(restore_status)); + } + } std::snprintf(last_error_detail, sizeof(last_error_detail), - "the calling thread's CUDA context (%#llx) could not be restored; " + "the calling thread's CUDA context (%#llx) could not be restored%s; " "context %#llx is now current. Call Device.set_current() before issuing " "further CUDA work on this thread", static_cast(reinterpret_cast(previous)), + cause, static_cast(reinterpret_cast(current))); + last_error_detail_status = operation_status != CUDA_SUCCESS ? operation_status : restore_status; } // Restore the previous context and preserve an earlier operation error. The -// operation error, if any, is returned; a restoration failure is then reported -// out of band. Otherwise the restoration status is returned, annotated for the -// eventual CUDAError. +// operation error, if any, is returned; otherwise the restoration status is. +// Either way a restoration failure is recorded as the detail of the returned +// status, so the eventual CUDAError explains it (see take_last_error_detail()). CUresult exit_context(CUcontext previous, int changed, CUresult operation_status) noexcept { CUresult restore_status = changed ? restore_context(previous) : CUDA_SUCCESS; if (restore_status == CUDA_SUCCESS) { return operation_status; } - if (operation_status != CUDA_SUCCESS) { - report_cuda_error("cuCtxSetCurrent (restoring the caller's context)", restore_status); - return operation_status; - } - note_context_not_restored(previous); - return restore_status; + note_context_not_restored(previous, operation_status, restore_status); + return operation_status != CUDA_SUCCESS ? operation_status : restore_status; } // Require a callable to be invocable without throwing. @@ -491,7 +548,10 @@ CUresult cleanup_in_context(const ContextHandle& h_context, const char* name, } CUresult restore = exit_context(previous, changed, CUDA_SUCCESS); if (restore != CUDA_SUCCESS) { + // Nothing is raised here, so the detail exit_context recorded has no + // exception to attach to: report it and drop the detail. report_cuda_error(name, restore, "failed while restoring the caller's context"); + clear_last_error_detail(); } return status; } @@ -581,8 +641,8 @@ CUresult context_get_device(const ContextHandle& h_context, CUdevice* device) no // the caller's context). Returns the cuGraphNodeSetParams status. A failure to // restore the caller's context is returned separately in *restore_status so the // caller can publish the metadata that depends on the successful update before -// raising it; if the update itself failed, a restoration failure is reported -// out of band and *restore_status is CUDA_SUCCESS. +// raising it; if the update itself failed, its status is returned with the +// restoration failure recorded as its detail and *restore_status is CUDA_SUCCESS. CUresult graph_node_set_params(CUgraphNode node, CUgraphNodeParams* params, const ContextHandle& h_context, CUresult* restore_status) noexcept { @@ -607,12 +667,10 @@ CUresult graph_node_set_params(CUgraphNode node, CUgraphNodeParams* params, if (restored == CUDA_SUCCESS) { return status; } - if (status != CUDA_SUCCESS) { - report_cuda_error("cuCtxSetCurrent (restoring the caller's context)", restored); - return status; + note_context_not_restored(previous, status, restored); + if (status == CUDA_SUCCESS) { + *restore_status = restored; } - note_context_not_restored(previous); - *restore_status = restored; return status; } diff --git a/cuda_core/cuda/core/_cpp/resource_handles.hpp b/cuda_core/cuda/core/_cpp/resource_handles.hpp index 069e3ec8319..5ad65143659 100644 --- a/cuda_core/cuda/core/_cpp/resource_handles.hpp +++ b/cuda_core/cuda/core/_cpp/resource_handles.hpp @@ -80,12 +80,20 @@ void report_message(const char* message) noexcept; // Report a failed NVRTC/NVVM/nvJitLink call by raw status code. void report_status_code(const char* operation, long code) noexcept; +// Attach a failed CUDA call to the Python exception currently being handled +// (PEP 678 note, Python 3.11+): for rollback failures inside `except` blocks +// whose original exception is about to be re-raised. When no exception is +// being handled or notes are unavailable, falls back to report_cuda_error(). +void note_or_report_cuda_error(const char* operation, CUresult status, const char* detail = nullptr) noexcept; + // Detail recorded by a context-scoped helper for the CUresult it is about to // return, e.g. that the caller's context could not be restored. The Cython -// error path appends it to the raised CUDAError. Thread-local; take_ returns -// the detail (valid until the next take on this thread) and clears it, or -// nullptr when none is recorded. -const char* take_last_error_detail() noexcept; +// error path attaches it to the raised CUDAError as a note. Thread-local and +// keyed by status: take_ returns the detail (valid until the next take on this +// thread) and clears it when `status` is the CUresult it was recorded for, and +// returns nullptr otherwise, so a detail whose status was never raised cannot +// attach to an unrelated error. +const char* take_last_error_detail(CUresult status) noexcept; void clear_last_error_detail() noexcept; // Tests only: make the next context restoration on this thread fail with diff --git a/cuda_core/cuda/core/_resource_handles.pxd b/cuda_core/cuda/core/_resource_handles.pxd index 37f4bcb0435..339e2610b0e 100644 --- a/cuda_core/cuda/core/_resource_handles.pxd +++ b/cuda_core/cuda/core/_resource_handles.pxd @@ -176,7 +176,9 @@ cdef void report_cuda_error( const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil cdef void report_message(const char* message) noexcept nogil cdef void report_status_code(const char* operation, long code) noexcept nogil -cdef const char* take_last_error_detail() noexcept nogil +cdef void note_or_report_cuda_error( + const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil +cdef const char* take_last_error_detail(cydriver.CUresult status) noexcept nogil cdef void clear_last_error_detail() noexcept nogil # Context handles diff --git a/cuda_core/cuda/core/_resource_handles.pyi b/cuda_core/cuda/core/_resource_handles.pyi index c44bcd46a03..882ccdc483d 100644 --- a/cuda_core/cuda/core/_resource_handles.pyi +++ b/cuda_core/cuda/core/_resource_handles.pyi @@ -49,3 +49,10 @@ def _set_context_restore_fault_for_testing(status: int): injected failure leaves the target context current, exactly as a failing ``cuCtxSetCurrent`` would, so callers must restore the context themselves. """ +def _note_or_report_cuda_error_for_testing(status: int): + """Attach a failed CUDA call to the exception being handled, or report it. + + Test hook for ``note_or_report_cuda_error()``. Called inside an ``except`` + block it adds a note to the exception being handled (Python 3.11+); anywhere + else it emits a ``CUDAWarning``. + """ diff --git a/cuda_core/cuda/core/_resource_handles.pyx b/cuda_core/cuda/core/_resource_handles.pyx index 09692f9d9a2..692ae7368e5 100644 --- a/cuda_core/cuda/core/_resource_handles.pyx +++ b/cuda_core/cuda/core/_resource_handles.pyx @@ -45,7 +45,14 @@ cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": void report_message "cuda_core::report_message" (const char* message) noexcept nogil void report_status_code "cuda_core::report_status_code" ( const char* operation, long code) noexcept nogil - const char* take_last_error_detail "cuda_core::take_last_error_detail" () noexcept nogil + void note_or_report_cuda_error "cuda_core::note_or_report_cuda_error" ( + const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil + # Alias for calls made from this module: calling the pxd-declared name here + # would make Cython emit a conflicting static prototype for it. + void _note_or_report_cuda_error_local "cuda_core::note_or_report_cuda_error" ( + const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil + const char* take_last_error_detail "cuda_core::take_last_error_detail" ( + cydriver.CUresult status) noexcept nogil void clear_last_error_detail "cuda_core::clear_last_error_detail" () noexcept nogil void set_context_restore_fault_for_testing "cuda_core::set_context_restore_fault_for_testing" ( cydriver.CUresult status) noexcept nogil @@ -589,6 +596,17 @@ def _set_context_restore_fault_for_testing(int status): """ set_context_restore_fault_for_testing(status) + +def _note_or_report_cuda_error_for_testing(int status): + """Attach a failed CUDA call to the exception being handled, or report it. + + Test hook for ``note_or_report_cuda_error()``. Called inside an ``except`` + block it adds a note to the exception being handled (Python 3.11+); anywhere + else it emits a ``CUDAWarning``. + """ + _note_or_report_cuda_error_local( + b"cuTestOperation", status, b"failed while testing") + # ============================================================================= # NVRTC function pointer initialization # ============================================================================= diff --git a/cuda_core/cuda/core/_utils/cuda_utils.pyi b/cuda_core/cuda/core/_utils/cuda_utils.pyi index b190e5967d0..565200277ba 100644 --- a/cuda_core/cuda/core/_utils/cuda_utils.pyi +++ b/cuda_core/cuda/core/_utils/cuda_utils.pyi @@ -19,9 +19,10 @@ class CUDAWarning(RuntimeWarning): ``cuda.core`` raises exceptions for failures in ordinary calls. Some failures happen where no exception can propagate: while a resource is released by the - garbage collector or by a CUDA callback, or while a context switch is undone - after the requested operation already succeeded. Those failures are reported - as this warning instead, and the affected resource may have leaked. + garbage collector or by a CUDA callback, including the driver calls that + switch and restore the CUDA context around such a release. Those failures + are reported as this warning instead, and the affected resource may have + leaked. Filter on this category to make such failures fatal in tests:: diff --git a/cuda_core/cuda/core/_utils/cuda_utils.pyx b/cuda_core/cuda/core/_utils/cuda_utils.pyx index 38fc42027df..b6f33112953 100644 --- a/cuda_core/cuda/core/_utils/cuda_utils.pyx +++ b/cuda_core/cuda/core/_utils/cuda_utils.pyx @@ -45,9 +45,10 @@ class CUDAWarning(RuntimeWarning): ``cuda.core`` raises exceptions for failures in ordinary calls. Some failures happen where no exception can propagate: while a resource is released by the - garbage collector or by a CUDA callback, or while a context switch is undone - after the requested operation already succeeded. Those failures are reported - as this warning instead, and the affected resource may have leaked. + garbage collector or by a CUDA callback, including the driver calls that + switch and restore the CUDA context around such a release. Those failures + are reported as this warning instead, and the affected resource may have + leaked. Filter on this category to make such failures fatal in tests:: @@ -160,6 +161,16 @@ cdef object _RUNTIME_SUCCESS = runtime.cudaError_t.cudaSuccess cdef object _NVRTC_SUCCESS = nvrtc.nvrtcResult.NVRTC_SUCCESS +cdef inline void _attach_detail(exc, str detail): + # PEP 678 notes (Python 3.11+) keep the detail separable from the message; + # older interpreters get it appended to the message instead. + add_note = getattr(exc, "add_note", None) + if add_note is not None: + add_note(detail) + else: + exc.args = (f"{exc.args[0]} ({detail})", *exc.args[1:]) + + cpdef inline int _check_driver_error(cydriver.CUresult error) except?-1 nogil: if error == cydriver.CUresult.CUDA_SUCCESS: return 0 @@ -167,20 +178,23 @@ cpdef inline int _check_driver_error(cydriver.CUresult error) except?-1 nogil: cdef const char* desc # A context-scoped helper in the handle layer may have recorded why this # status needs more explanation (e.g. the caller's context was not restored). - cdef const char* detail = take_last_error_detail() + cdef const char* detail = take_last_error_detail(error) name_err = cydriver.cuGetErrorName(error, &name) if name_err != cydriver.CUresult.CUDA_SUCCESS: raise CUDAError(f"UNEXPECTED ERROR CODE: {error}") desc_err = cydriver.cuGetErrorString(error, &desc) with gil: - suffix = f" ({detail.decode()})" if detail != NULL else "" # TODO: consider lower this to Cython expl = DRIVER_CU_RESULT_EXPLANATIONS.get(int(error)) if expl is not None: - raise CUDAError(f"{name.decode()}: {expl}{suffix}") - if desc_err != cydriver.CUresult.CUDA_SUCCESS: - raise CUDAError(f"{name.decode()}{suffix}") - raise CUDAError(f"{name.decode()}: {desc.decode()}{suffix}") + exc = CUDAError(f"{name.decode()}: {expl}") + elif desc_err != cydriver.CUresult.CUDA_SUCCESS: + exc = CUDAError(name.decode()) + else: + exc = CUDAError(f"{name.decode()}: {desc.decode()}") + if detail != NULL: + _attach_detail(exc, detail.decode()) + raise exc cpdef inline int _check_runtime_error(error) except?-1: diff --git a/cuda_core/cuda/core/graph/_graph_builder.pyx b/cuda_core/cuda/core/graph/_graph_builder.pyx index 7033f90b239..98faf09eec3 100644 --- a/cuda_core/cuda/core/graph/_graph_builder.pyx +++ b/cuda_core/cuda/core/graph/_graph_builder.pyx @@ -20,7 +20,7 @@ from cuda.core.graph._subclasses cimport ( ExecutableGraphNode, create_executable_node_view, ) -from cuda.core._resource_handles cimport report_cuda_error +from cuda.core._resource_handles cimport note_or_report_cuda_error, report_cuda_error from cuda.core._resource_handles cimport ( GraphExecHandle, GraphHandle, @@ -862,9 +862,9 @@ cdef class GraphBuilder: invalidate_child_graph_state( self._h_graph, c_new_node) else: - # The original exception propagates; the failed rollback is - # reported out of band (error handling policy). - report_cuda_error( + # The original exception propagates with the failed rollback + # attached as a note (error handling policy). + note_or_report_cuda_error( b"cuGraphDestroyNode", rollback_status, b"failed while rolling back a child graph node; the node remains in the graph") raise diff --git a/cuda_core/cuda/core/graph/_graph_node.pyx b/cuda_core/cuda/core/graph/_graph_node.pyx index 831c0f30b69..d072971615d 100644 --- a/cuda_core/cuda/core/graph/_graph_node.pyx +++ b/cuda_core/cuda/core/graph/_graph_node.pyx @@ -45,7 +45,7 @@ from cuda.core.graph._subclasses cimport ( SwitchNode, WhileNode, ) -from cuda.core._resource_handles cimport report_cuda_error +from cuda.core._resource_handles cimport note_or_report_cuda_error from cuda.core._resource_handles cimport ( GraphHandle, GraphNodeHandle, @@ -1116,9 +1116,9 @@ cdef inline ChildGraphNode GN_embed(GraphNode self, GraphDefinition child_def): if rollback_status == cydriver.CUDA_SUCCESS: invalidate_child_graph_state(h_graph, new_node) else: - # The original exception propagates; the failed rollback is - # reported out of band (error handling policy). - report_cuda_error( + # The original exception propagates with the failed rollback + # attached as a note (error handling policy). + note_or_report_cuda_error( b"cuGraphDestroyNode", rollback_status, b"failed while rolling back a child graph node; the node remains in the graph") raise diff --git a/cuda_core/docs/source/error_handling.rst b/cuda_core/docs/source/error_handling.rst index a6761aadd24..ad37c76422e 100644 --- a/cuda_core/docs/source/error_handling.rst +++ b/cuda_core/docs/source/error_handling.rst @@ -35,9 +35,14 @@ When a ``cuda.core`` call raises, the following hold: documentation says so where it applies (for example the graph mutation methods that add several driver edges). -``cuda.core`` does not swallow driver errors. A failure that would otherwise be -hidden, for example because it occurred while another exception was already -propagating, is reported as described in the next section. +``cuda.core`` does not swallow driver errors. When a second failure occurs +while an exception is being raised, for example the caller's context cannot be +restored after a failed call, or the rollback of a partially built graph node +fails, the second failure is attached to the exception as a note +(:meth:`BaseException.add_note`), which appears in the traceback and in +``__notes__``. Python 3.10 has no exception notes; there the information is +appended to the message when ``cuda.core`` constructs the exception, and +reported as described in the next section otherwise. Failures that cannot be raised ------------------------------ @@ -45,10 +50,9 @@ Failures that cannot be raised Some ``cuda.core`` code runs where no Python exception can propagate: - resources released by the garbage collector or by the deferred cleanup of - CUDA graphs, and the CUDA driver calls those releases make; -- callbacks invoked by CUDA; -- cleanup performed after an operation has already failed, such as rolling back - a partially built graph node or restoring the caller's CUDA context. + CUDA graphs, and the CUDA driver calls those releases make, including the + context switch and restoration around such a release; +- callbacks invoked by CUDA. A CUDA error in one of these places is reported as a :class:`CUDAWarning`. The message names the failed driver call and the CUDA error. The warning means the @@ -86,8 +90,10 @@ earlier, unrecoverable kernel fault (see `Sticky errors`_). None of these can be fixed by retrying, so ``cuda.core`` does not retry. When restoration fails in an ordinary call, the resource created by the call is -destroyed and a ``CUDAError`` is raised whose message states that the caller's -context could not be restored and which context is now current. Call +destroyed and a ``CUDAError`` is raised for the failed ``cuCtxSetCurrent``, +with a note stating that the caller's context could not be restored and which +context is now current. If the call itself failed as well, its own error is +raised and the restoration failure is the note. Call :meth:`Device.set_current` before issuing further CUDA work on that thread. When restoration fails inside a destructor or callback, a :class:`CUDAWarning` @@ -124,4 +130,4 @@ Process termination ``cuda.core`` does not abort the process in response to a CUDA error, including errors that cannot be raised, and including failures to restore the caller's context. Aborting is reserved for an internal invariant violation where -continuing could corrupt memory, and no such code path exists in this release. +continuing could corrupt memory. diff --git a/cuda_core/docs/source/release/1.3.0-notes.rst b/cuda_core/docs/source/release/1.3.0-notes.rst index 3e94b19a77b..2eb19e5b22a 100644 --- a/cuda_core/docs/source/release/1.3.0-notes.rst +++ b/cuda_core/docs/source/release/1.3.0-notes.rst @@ -63,10 +63,12 @@ Fixes and enhancements - When a :class:`Device` method has to run in the device's context and the caller's context cannot be restored afterwards, the created resource is - destroyed and the raised ``CUDAError`` now states that the caller's context - could not be restored and which context is current. Previously the error - named only the driver status of the failed ``cuCtxSetCurrent`` call. A - restoration failure during resource cleanup is reported as + destroyed and the raised ``CUDAError`` now carries a note (Python 3.11+; + appended to the message on 3.10) stating that the caller's context could not + be restored and which context is current. Previously the error named only + the driver status of the failed ``cuCtxSetCurrent`` call. If the call itself + failed as well, its error is raised and the restoration failure is the note. + A restoration failure during resource cleanup is reported as :class:`CUDAWarning`. - Updating a memcpy or memset graph node whose context differs from the current @@ -79,10 +81,11 @@ Fixes and enhancements current instead of leaving the thread with no context. It also works when no context is current, returning ``None``. -- Failed rollbacks of a partially embedded child graph node and failed - ``cuStreamEndCapture`` calls made when a still-building - :class:`~graph.GraphBuilder` is garbage collected are now reported as - :class:`CUDAWarning`; both were silent. +- A failed rollback of a partially embedded child graph node is now attached + as a note to the exception that triggered the rollback (reported as + :class:`CUDAWarning` on Python 3.10), and a failed ``cuStreamEndCapture`` + made when a still-building :class:`~graph.GraphBuilder` is garbage collected + is now reported as :class:`CUDAWarning`; both were silent. - :attr:`Stream.device` and related queries on a stream whose context is not current now restore the caller's context even when the device query fails. diff --git a/cuda_core/tests/test_error_handling.py b/cuda_core/tests/test_error_handling.py index e716b001f66..b0dbc10d9df 100644 --- a/cuda_core/tests/test_error_handling.py +++ b/cuda_core/tests/test_error_handling.py @@ -4,15 +4,17 @@ """Tests for the error handling policy (docs/source/error_handling.rst). Ordinary calls raise and leave the caller's context untouched; failures that -cannot be raised are reported as CUDAWarning; and a failure to restore the -caller's context is raised (or reported) with an explanation rather than -swallowed or turned into a process abort. Restoration failures are injected with -the handle layer's test hook, which leaves the target context current exactly as -a real ``cuCtxSetCurrent`` failure would, so every test here restores the -context stack itself. +cannot be raised are reported as CUDAWarning; a failure to restore the caller's +context is raised (or reported) with an explanation rather than swallowed or +turned into a process abort; and a secondary failure that occurs while an +exception is being raised is attached to that exception as a note. Restoration +failures are injected with the handle layer's test hook, which leaves the target +context current exactly as a real ``cuCtxSetCurrent`` failure would, so every +test here restores the context stack itself. """ import ctypes +import sys from contextlib import contextmanager import pytest @@ -26,17 +28,32 @@ DeviceMemoryResourceOptions, LegacyPinnedMemoryResource, ) -from cuda.core._resource_handles import _set_context_restore_fault_for_testing +from cuda.core._memory._device_memory_resource import _SynchronousMemoryResource +from cuda.core._resource_handles import ( + _note_or_report_cuda_error_for_testing, + _set_context_restore_fault_for_testing, +) from cuda.core._stream import default_stream from cuda.core._utils.cuda_utils import CUDAError, driver, handle_return from cuda.core._utils.version import binding_version, driver_version from cuda.core.graph import GraphDefinition INVALID_CONTEXT = int(driver.CUresult.CUDA_ERROR_INVALID_CONTEXT) +INVALID_VALUE = int(driver.CUresult.CUDA_ERROR_INVALID_VALUE) +DEINITIALIZED = int(driver.CUresult.CUDA_ERROR_DEINITIALIZED) + +# PEP 678 exception notes; on 3.10 the same information lands in the message. +HAS_NOTES = sys.version_info >= (3, 11) thread_unsafe_context_fault = pytest.mark.thread_unsafe( reason="injects a thread-local restoration fault and mutates the CUDA context stack" ) +thread_unsafe_warning_capture = pytest.mark.thread_unsafe(reason="warning capture is process-global") + + +def error_text(exc): + """The message plus any notes, wherever the detail lives on this interpreter.""" + return "\n".join([str(exc), *getattr(exc, "__notes__", [])]) @contextmanager @@ -69,11 +86,16 @@ def test_create_stream_raises_when_context_cannot_be_restored(init_cuda): """Creation is undone and the error explains the context state; no abort, no warning.""" dev = init_cuda with no_context_with_restore_fault(): - with assert_no_cuda_warning(), pytest.raises(CUDAError, match="could not be restored") as excinfo: + with assert_no_cuda_warning(), pytest.raises(CUDAError) as excinfo: dev.create_stream() - message = str(excinfo.value) - assert "CUDA_ERROR_INVALID_CONTEXT" in message - assert "Device.set_current()" in message + text = error_text(excinfo.value) + assert "could not be restored" in text + assert "CUDA_ERROR_INVALID_CONTEXT" in text + assert "Device.set_current()" in text + if HAS_NOTES: + # The explanation is a note, separable from the driver error message. + assert "could not be restored" not in str(excinfo.value) + assert any("could not be restored" in note for note in excinfo.value.__notes__) # As documented, a failed restoration leaves the device's context current. assert current_context_handle() == int(dev.context.handle) assert current_context_handle() == int(dev.context.handle) @@ -85,8 +107,29 @@ def test_sync_raises_when_context_cannot_be_restored(init_cuda): """A context-scoped call without a created resource raises the same explanation.""" dev = init_cuda with no_context_with_restore_fault(): - with pytest.raises(CUDAError, match="could not be restored"): + with pytest.raises(CUDAError) as excinfo: dev.sync() + assert "could not be restored" in error_text(excinfo.value) + assert current_context_handle() == int(dev.context.handle) + + +@thread_unsafe_context_fault +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_failed_call_raises_its_own_error_with_the_restore_failure_attached(init_cuda): + """When the call and the restoration both fail, the call's error is raised and the + restoration failure is attached to it; nothing is reported out of band.""" + dev = init_cuda + mr = _SynchronousMemoryResource(dev.device_id) + with no_context_with_restore_fault(): + with assert_no_cuda_warning(), pytest.raises(CUDAError) as excinfo: + mr.allocate(1 << 62) + message = str(excinfo.value) + text = error_text(excinfo.value) + # The allocation failure is the primary error, not the restoration failure. + assert not message.startswith("CUDA_ERROR_INVALID_CONTEXT") + assert "could not be restored after this failure" in text + assert "cuCtxSetCurrent: CUDA_ERROR_INVALID_CONTEXT" in text + assert "Device.set_current()" in text assert current_context_handle() == int(dev.context.handle) @@ -184,8 +227,9 @@ def test_memset_update_keeps_new_owners_alive_when_context_cannot_be_restored(de other_dev.set_current() _set_context_restore_fault_for_testing(INVALID_CONTEXT) try: - with pytest.raises(CUDAError, match="could not be restored"): + with pytest.raises(CUDAError) as excinfo: node.update(dst=replacement, value=0x22) + assert "could not be restored" in error_text(excinfo.value) finally: _set_context_restore_fault_for_testing(0) node_dev.set_current() @@ -205,3 +249,34 @@ def as_bytes(buffer): assert list(as_bytes(dst)) == [0] * 4 graph.close() stream.close() + + +@thread_unsafe_warning_capture +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_rollback_failure_is_attached_to_the_propagating_exception(): + """A failed rollback inside an except block becomes a note on the exception being + handled (Python 3.11+); on 3.10, or with no exception being handled, it is reported + as a CUDAWarning. CUDA_ERROR_DEINITIALIZED is neither attached nor reported.""" + with pytest.raises(RuntimeError) as excinfo: + try: + raise RuntimeError("primary failure") + except RuntimeError: + if HAS_NOTES: + with assert_no_cuda_warning(): + _note_or_report_cuda_error_for_testing(INVALID_VALUE) + else: + with pytest.warns(CUDAWarning, match="cuTestOperation failed while testing"): + _note_or_report_cuda_error_for_testing(INVALID_VALUE) + with assert_no_cuda_warning(): + _note_or_report_cuda_error_for_testing(DEINITIALIZED) + raise + exc = excinfo.value + assert str(exc) == "primary failure" + if HAS_NOTES: + assert len(exc.__notes__) == 1 + assert "cuTestOperation failed while testing: CUDA_ERROR_INVALID_VALUE" in exc.__notes__[0] + else: + assert not hasattr(exc, "__notes__") + # With no exception being handled there is nothing to attach to. + with pytest.warns(CUDAWarning, match="cuTestOperation failed while testing"): + _note_or_report_cuda_error_for_testing(INVALID_VALUE) From c5399ce35e93ec99e3f08af24e874affcd084f8c Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Fri, 4 Sep 2026 11:30:08 -0700 Subject: [PATCH 04/14] cuda.core: follow the review of #2750 and flush the stderr fallback Rebased onto the reviewed head of #2750. Adjustments the rebase needed: - The review's warning for an undo skipped after a failed context restoration is routed through report_cuda_error(), so it carries the CUDA status and becomes a CUDAWarning like every other non-raising report. - invoke_in_context and invoke_in_context_or_undo now reject empty handles themselves, so context_get_device drops its own guard like the other helpers did; enter_context's no-op for empty handles is documented as used only by graph_node_set_params. - _SynchronousMemoryResource moved to its own module; the error-handling test imports it from there. The review's two teardown tests asserted that stderr stayed empty; under the policy a teardown failure is a CUDAWarning, so they assert that no CUDAWarning is issued instead (and are marked thread_unsafe because warning capture is process-global). - report_message() flushes stderr after its last-resort fprintf, so the text is not lost if the process dies right after (review comment). Co-Authored-By: Claude Fable 5.1 --- cuda_core/cuda/core/_cpp/resource_handles.cpp | 6 +++++- cuda_core/tests/test_error_handling.py | 2 +- cuda_core/tests/test_memory.py | 14 +++++++------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/cuda_core/cuda/core/_cpp/resource_handles.cpp b/cuda_core/cuda/core/_cpp/resource_handles.cpp index fc05d376487..8e455c73854 100644 --- a/cuda_core/cuda/core/_cpp/resource_handles.cpp +++ b/cuda_core/cuda/core/_cpp/resource_handles.cpp @@ -270,6 +270,7 @@ void report_message(const char* message) noexcept { } } std::fprintf(stderr, "%s\n", message); + std::fflush(stderr); } // Report a failed non-CUDA call (NVRTC, NVVM, nvJitLink) from a path that @@ -362,7 +363,10 @@ namespace { // Make a context current and record the state needed to restore it. // An empty handle is a no-op: the operation runs in the caller's current -// context, and nothing is restored on exit. +// context, and nothing is restored on exit. invoke_in_context and +// invoke_in_context_or_undo reject empty handles before getting here; only +// graph_node_set_params relies on the no-op (pre-13.2 node updates run in the +// caller's context). CUresult enter_context(const ContextHandle& h_context, CUcontext* previous, int* changed) noexcept { *previous = nullptr; *changed = 0; diff --git a/cuda_core/tests/test_error_handling.py b/cuda_core/tests/test_error_handling.py index b0dbc10d9df..caed0beea66 100644 --- a/cuda_core/tests/test_error_handling.py +++ b/cuda_core/tests/test_error_handling.py @@ -28,7 +28,7 @@ DeviceMemoryResourceOptions, LegacyPinnedMemoryResource, ) -from cuda.core._memory._device_memory_resource import _SynchronousMemoryResource +from cuda.core._memory._synchronous_memory_resource import _SynchronousMemoryResource from cuda.core._resource_handles import ( _note_or_report_cuda_error_for_testing, _set_context_restore_fault_for_testing, diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index 661e540c308..ccf064df767 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -2264,8 +2264,9 @@ def test_synchronous_memory_resource_restores_context_after_failure(device_x2): assert current_context_handle() == current_context +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="claude-sonnet-5") -def test_synchronous_memory_resource_default_stream_deallocates_in_own_context(device_x2, capsys): +def test_synchronous_memory_resource_default_stream_deallocates_in_own_context(device_x2): """Buffer teardown with no explicit stream frees in the resource's own context, not whatever context happens to be current at close() time.""" from cuda.core._memory._synchronous_memory_resource import _SynchronousMemoryResource @@ -2280,13 +2281,14 @@ def test_synchronous_memory_resource_default_stream_deallocates_in_own_context(d buf = mr.allocate(64) # no explicit stream: records a context-bound default token assert current_context_handle() == current_context - buf.close() # no explicit stream: reuses the recorded token + with assert_no_cuda_warning(): + buf.close() # no explicit stream: reuses the recorded token assert current_context_handle() == current_context - assert capsys.readouterr().err == "" +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="claude-sonnet-5") -def test_synchronous_memory_resource_allocate_without_current_context(device_x2, capsys): +def test_synchronous_memory_resource_allocate_without_current_context(device_x2): """allocate()/close() with no explicit stream succeed with no context current, instead of raising or leaking the allocation (#2311).""" from cuda.core._memory._synchronous_memory_resource import _SynchronousMemoryResource @@ -2296,14 +2298,12 @@ def test_synchronous_memory_resource_allocate_without_current_context(device_x2, mr = _SynchronousMemoryResource(alloc_dev.device_id, alloc_dev.context) current_dev.set_current() - with no_current_context(): + with no_current_context(), assert_no_cuda_warning(): buf = mr.allocate(64) assert current_context_handle() == 0 buf.close() assert current_context_handle() == 0 - assert capsys.readouterr().err == "" - @pytest.mark.parametrize( ("method", "spec", "match"), From e30c01f895b189daa6f712ca471797f561620160 Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Tue, 8 Sep 2026 09:45:34 -0700 Subject: [PATCH 05/14] cuda.core tests: check host-only buffer teardown for CUDAWarning, not stderr The host-only Buffer tests from #2773 asserted that nothing containing "Warning" reached stderr. Under the error handling policy a teardown failure is a CUDAWarning, not stderr text, so that assertion no longer checks anything. Use assert_no_cuda_warning() around allocate/close instead (marked thread_unsafe, as warning capture is process-global). The spawned-process variant checks inside the child, since warnings do not cross processes; a failure surfaces as the non-zero exit code the parent already asserts on. Co-Authored-By: Claude Fable 5.1 --- cuda_core/tests/test_memory.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index 6902936e1d3..5706d68df4a 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -824,9 +824,10 @@ def test_from_handle_mr_explicit_stream_without_current_context(buffer_type): ] +@pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") @pytest.mark.agent_authored(model="claude-fable-5-1") @pytest.mark.parametrize("mr_cls", _HOST_ONLY_MRS) -def test_from_handle_host_only_mr_without_current_context(mr_cls, capfd): +def test_from_handle_host_only_mr_without_current_context(mr_cls): """Host-only memory needs no current context to create or free a Buffer.""" device = Device() device.set_current() @@ -836,28 +837,30 @@ def test_from_handle_host_only_mr_without_current_context(mr_cls, capfd): assert int(previous) != 0 try: assert int(handle_return(driver.cuCtxGetCurrent())) == 0 - buf = mr.allocate(64) - assert buf.is_host_accessible - buf.close() + with assert_no_cuda_warning(): + buf = mr.allocate(64) + assert buf.is_host_accessible + buf.close() assert int(handle_return(driver.cuCtxGetCurrent())) == 0 finally: handle_return(driver.cuCtxSetCurrent(previous)) - assert "Warning" not in capfd.readouterr().err - def _host_only_child_main(mr_cls): """Allocate and free host-only memory in a process that never initialized CUDA.""" - buf = mr_cls().allocate(64) - assert buf.is_host_accessible - buf.close() + # Warnings do not cross processes: check for a CUDAWarning here, where a + # failed assertion becomes a non-zero exit code for the parent to see. + with assert_no_cuda_warning(): + buf = mr_cls().allocate(64) + assert buf.is_host_accessible + buf.close() err, _ = driver.cuCtxGetCurrent() assert err == driver.CUresult.CUDA_ERROR_NOT_INITIALIZED, err @pytest.mark.agent_authored(model="claude-fable-5-1") @pytest.mark.parametrize("mr_cls", _HOST_ONLY_MRS) -def test_from_handle_host_only_mr_without_cuda_init(mr_cls, capfd): +def test_from_handle_host_only_mr_without_cuda_init(mr_cls): """Host-only buffers work in a spawned process that never initializes CUDA.""" process = mp.Process(target=_host_only_child_main, args=(mr_cls,)) process.start() @@ -865,7 +868,6 @@ def test_from_handle_host_only_mr_without_cuda_init(mr_cls, capfd): survivors = kill_subprocesses(process) assert not survivors, "child did not exit within timeout" assert process.exitcode == 0, f"child exited with {process.exitcode}" - assert "Warning" not in capfd.readouterr().err @pytest.mark.thread_unsafe(reason="records process-global warnings and mutates the context stack") From cb5784ea073f5e4d84e9bffc28537bda8c3a795b Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Wed, 9 Sep 2026 19:35:39 -0700 Subject: [PATCH 06/14] cuda.core: directory-aware C++ build rule and drop the dead top-level copy from the merged wheel build_hooks.py maps a Cython module to its C++ by name. It now also accepts a directory: every .cpp under cuda/core/_cpp// compiles into the one extension for _.pyx, with the legacy single file _cpp/.cpp kept as the fallback (tensor_map is unchanged). With no such directory in the tree yet, the sources are exactly today's, so this part is inert on its own. The cuda.core._cpp package-data globs become recursive so headers in nested directories ship, and .gitignore stops ignoring .cpp files under cuda/core/_cpp/ so new sources are visible to git. ci/tools/merge_cuda_core_wheels.py stops retaining a third, top-level copy of _resource_handles and the top-level _cpp/ and _include/ headers in the merged cu12+cu13 wheel. cuda/core/__init__.py rewrites __path__ to the versioned subpackage before any import reaches them, so that copy (about 308 KB uncompressed in the 1.2.0 wheel) was never imported; the comment defending it referred to an import removed in #1463. A step in build-wheel.yml now asserts that the merged wheel's top level holds only __init__.py, _version.py and the two versioned trees. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/build-wheel.yml | 21 ++++++++++++++++ .gitignore | 1 + ci/tools/merge_cuda_core_wheels.py | 11 +++----- cuda_core/build_hooks.py | 30 ++++++++++++---------- cuda_core/pyproject.toml | 2 +- cuda_core/tests/test_build_hooks.py | 39 +++++++++++++++++++++++++++++ 6 files changed, 83 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index 14cebd633d0..01091a7dd79 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -696,6 +696,27 @@ jobs: "${{ env.CUDA_CORE_ARTIFACTS_DIR }}"/cu"${BUILD_PREV_CUDA_MAJOR}"/cuda_core*.whl \ --output-dir "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" + - name: Check merged cuda.core wheel layout + if: ${{ env.BUILD_CORE == 'true' }} + run: | + # cuda/core/__init__.py rewrites __path__ to the active cuXX/ tree, so + # only the entry points and the two versioned trees belong at the top + # level of cuda/core/; anything else is a dead copy. + python - "${{ env.CUDA_CORE_ARTIFACTS_DIR }}"/cuda_core*.whl <<'EOF' + import os + import sys + import zipfile + + prefix = "cuda/core/" + members = zipfile.ZipFile(sys.argv[1]).namelist() + names = {n[len(prefix) :].split("/")[0] for n in members if n.startswith(prefix)} + names.discard("") + expected = {"__init__.py", "_version.py"} + expected |= {f"cu{os.environ[v]}" for v in ("BUILD_CUDA_MAJOR", "BUILD_PREV_CUDA_MAJOR")} + assert names == expected, f"unexpected top-level entries under cuda/core/: {sorted(names ^ expected)}" + print("merged wheel top level:", sorted(names)) + EOF + - name: Check cuda.core wheel if: ${{ env.BUILD_CORE == 'true' }} run: | diff --git a/.gitignore b/.gitignore index 91c3b74a105..338c72735d4 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ __pycache__/ !*_impl.cpp !cuda_bindings/cuda/bindings/_lib/param_packer.cpp !cuda_bindings/cuda/bindings/_bindings/loader.cpp +!cuda_core/cuda/core/_cpp/**/*.cpp cache_driver cache_runtime cache_nvrtc diff --git a/ci/tools/merge_cuda_core_wheels.py b/ci/tools/merge_cuda_core_wheels.py index 23a8a21289f..47af0751186 100644 --- a/ci/tools/merge_cuda_core_wheels.py +++ b/ci/tools/merge_cuda_core_wheels.py @@ -145,25 +145,22 @@ def merge_wheels(wheels: list[Path], output_dir: Path, show_wheel_contents: bool os.truncate(versioned_dir / "__init__.py", 0) print("\n=== Removing files from cuda/core/ directory ===", file=sys.stderr) + # Only what cuda/core/__init__.py uses before it rewrites __path__ to the + # versioned subpackage stays at top level: it imports _version, then + # redirects every later import into cu12/ or cu13/. Anything else left at + # top level is a dead copy that nothing imports. items_to_keep = ( "__init__.py", "_version.py", - "_include", - "_cpp", # Headers for Cython development "cu12", "cu13", ) - # _resource_handles is shared (not CUDA-version-specific) and must stay - # at top level. It's imported early in __init__.py before versioned code. - items_to_keep_prefix = ("_resource_handles",) all_items = os.scandir(base_wheel / base_dir) removed_count = 0 for f in all_items: f_abspath = f.path if f.name in items_to_keep: continue - if any(f.name.startswith(prefix) for prefix in items_to_keep_prefix): - continue if f.is_dir(): print(f" Removing directory: {f.name}", file=sys.stderr) shutil.rmtree(f_abspath) diff --git a/cuda_core/build_hooks.py b/cuda_core/build_hooks.py index fc112b5a74e..01f6c756bc8 100644 --- a/cuda_core/build_hooks.py +++ b/cuda_core/build_hooks.py @@ -183,6 +183,22 @@ def _relativize_extension_sources(extensions) -> None: ] +def _extension_sources(mod_name): + """The module's .pyx plus its C++, if any: every .cpp under + cuda/core/_cpp//, or the single legacy file cuda/core/_cpp/.cpp. + Example: _tensor_map.pyx compiles _cpp/tensor_map.cpp.""" + sources = [f"cuda/core/{mod_name}.pyx"] + cpp_stem = os.path.join("cuda", "core", "_cpp", mod_name.lstrip("_")) + if os.path.isdir(cpp_stem): + cpp_sources = sorted(glob.glob(os.path.join(cpp_stem, "**", "*.cpp"), recursive=True)) + if not cpp_sources: + raise RuntimeError(f"{cpp_stem}/ exists but contains no .cpp files") + sources.extend(cpp_sources) + elif os.path.isfile(cpp_stem + ".cpp"): + sources.append(cpp_stem + ".cpp") + return sources + + def _build_cuda_core(debug=False): # Customizing the build hooks is needed because we must defer cythonization until cuda-bindings, # now a required build-time dependency that's dynamically installed via the other hook below, @@ -227,18 +243,6 @@ def module_names(): continue yield mod - def get_sources(mod_name): - """Get source files for a module, including any .cpp files.""" - sources = [f"cuda/core/{mod_name}.pyx"] - - # Add module-specific .cpp file from _cpp/ directory if it exists - # Example: _resource_handles.pyx finds _cpp/resource_handles.cpp. - cpp_file = f"cuda/core/_cpp/{mod_name.lstrip('_')}.cpp" - if os.path.exists(cpp_file): - sources.append(cpp_file) - - return sources - all_include_dirs = [os.path.join(cuda_path, "include")] extra_compile_args = [] extra_link_args = [] @@ -264,7 +268,7 @@ def get_sources(mod_name): ext_modules = tuple( Extension( f"cuda.core.{mod.replace(os.path.sep, '.')}", - sources=get_sources(mod), + sources=_extension_sources(mod), include_dirs=[ "cuda/core/_include", "cuda/core/_cpp", diff --git a/cuda_core/pyproject.toml b/cuda_core/pyproject.toml index 3eac3a4c0af..3df8536a32b 100644 --- a/cuda_core/pyproject.toml +++ b/cuda_core/pyproject.toml @@ -108,7 +108,7 @@ include-package-data = false [tool.setuptools.package-data] "*" = ["*.pxd", "*.pyi", "py.typed"] "cuda.core._include" = ["*.h", "*.hpp"] -"cuda.core._cpp" = ["*.h", "*.hpp"] +"cuda.core._cpp" = ["**/*.h", "**/*.hpp"] [tool.setuptools.dynamic] readme = { file = ["DESCRIPTION.rst"], content-type = "text/x-rst" } diff --git a/cuda_core/tests/test_build_hooks.py b/cuda_core/tests/test_build_hooks.py index 2f1b3211781..b59e059548a 100644 --- a/cuda_core/tests/test_build_hooks.py +++ b/cuda_core/tests/test_build_hooks.py @@ -325,3 +325,42 @@ def test_flag_set_forces_rebuild(self, monkeypatch): def test_flag_clear_leaves_default(self, monkeypatch): assert not self._finalized_build_ext(False, monkeypatch).force + + +class TestExtensionSources: + """_extension_sources: a directory of .cpp files, a single legacy .cpp, or nothing.""" + + @pytest.fixture + def tree(self, tmp_path, monkeypatch): + core = tmp_path / "cuda" / "core" + cpp = core / "_cpp" + (cpp / "a" / "nested").mkdir(parents=True) + (cpp / "d").mkdir() + for name in ("_a.pyx", "_b.pyx", "_c.pyx", "_d.pyx"): + (core / name).write_text("") + for name in ("a/x.cpp", "a/y.cpp", "a/nested/z.cpp", "a/notes.md", "b.cpp"): + (cpp / name).write_text("") + monkeypatch.chdir(tmp_path) + + @pytest.mark.agent_authored(model="claude-fable-5-1") + def test_directory_of_sources(self, tree): + a = os.path.join("cuda", "core", "_cpp", "a") + assert build_hooks._extension_sources("_a") == [ + "cuda/core/_a.pyx", + os.path.join(a, "nested", "z.cpp"), + os.path.join(a, "x.cpp"), + os.path.join(a, "y.cpp"), + ] + + @pytest.mark.agent_authored(model="claude-fable-5-1") + def test_legacy_single_file_and_no_cpp(self, tree): + assert build_hooks._extension_sources("_b") == [ + "cuda/core/_b.pyx", + os.path.join("cuda", "core", "_cpp", "b.cpp"), + ] + assert build_hooks._extension_sources("_c") == ["cuda/core/_c.pyx"] + + @pytest.mark.agent_authored(model="claude-fable-5-1") + def test_empty_directory_is_an_error(self, tree): + with pytest.raises(RuntimeError, match="no .cpp files"): + build_hooks._extension_sources("_d") From 28b73d2efbff7acefc4f506e4643905ee07696eb Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Fri, 11 Sep 2026 10:39:48 -0700 Subject: [PATCH 07/14] cuda.core: keep the merged-wheel layout check in the merge script; use pathlib Review follow-up. The top-level layout assertion moves from build-wheel.yml into ci/tools/merge_cuda_core_wheels.py, which now derives the kept cuNN directories from its input wheels and raises if anything else remains under cuda/core/ after the removal. _extension_sources uses pathlib. --- .github/workflows/build-wheel.yml | 21 --------------------- ci/tools/merge_cuda_core_wheels.py | 18 ++++++++++-------- cuda_core/build_hooks.py | 10 +++++----- 3 files changed, 15 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index 01091a7dd79..14cebd633d0 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -696,27 +696,6 @@ jobs: "${{ env.CUDA_CORE_ARTIFACTS_DIR }}"/cu"${BUILD_PREV_CUDA_MAJOR}"/cuda_core*.whl \ --output-dir "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" - - name: Check merged cuda.core wheel layout - if: ${{ env.BUILD_CORE == 'true' }} - run: | - # cuda/core/__init__.py rewrites __path__ to the active cuXX/ tree, so - # only the entry points and the two versioned trees belong at the top - # level of cuda/core/; anything else is a dead copy. - python - "${{ env.CUDA_CORE_ARTIFACTS_DIR }}"/cuda_core*.whl <<'EOF' - import os - import sys - import zipfile - - prefix = "cuda/core/" - members = zipfile.ZipFile(sys.argv[1]).namelist() - names = {n[len(prefix) :].split("/")[0] for n in members if n.startswith(prefix)} - names.discard("") - expected = {"__init__.py", "_version.py"} - expected |= {f"cu{os.environ[v]}" for v in ("BUILD_CUDA_MAJOR", "BUILD_PREV_CUDA_MAJOR")} - assert names == expected, f"unexpected top-level entries under cuda/core/: {sorted(names ^ expected)}" - print("merged wheel top level:", sorted(names)) - EOF - - name: Check cuda.core wheel if: ${{ env.BUILD_CORE == 'true' }} run: | diff --git a/ci/tools/merge_cuda_core_wheels.py b/ci/tools/merge_cuda_core_wheels.py index 47af0751186..6e7f51e73b6 100644 --- a/ci/tools/merge_cuda_core_wheels.py +++ b/ci/tools/merge_cuda_core_wheels.py @@ -133,9 +133,11 @@ def merge_wheels(wheels: list[Path], output_dir: Path, show_wheel_contents: bool # Copy version-specific directories from each wheel into versioned subdirectories base_dir = Path("cuda") / "core" + versioned_dirs = set() for i, wheel_dir in enumerate(extracted_wheels): cuda_version = wheels[i].name.split(".cu")[1].split(".")[0] versioned_dir = base_wheel / base_dir / f"cu{cuda_version}" + versioned_dirs.add(versioned_dir.name) # Copy entire directory tree from source wheel to versioned directory print(f" Copying {wheel_dir / base_dir} to {versioned_dir}", file=sys.stderr) @@ -147,14 +149,9 @@ def merge_wheels(wheels: list[Path], output_dir: Path, show_wheel_contents: bool print("\n=== Removing files from cuda/core/ directory ===", file=sys.stderr) # Only what cuda/core/__init__.py uses before it rewrites __path__ to the # versioned subpackage stays at top level: it imports _version, then - # redirects every later import into cu12/ or cu13/. Anything else left at - # top level is a dead copy that nothing imports. - items_to_keep = ( - "__init__.py", - "_version.py", - "cu12", - "cu13", - ) + # redirects every later import into the versioned tree. Anything else + # left at top level is a dead copy that nothing imports. + items_to_keep = {"__init__.py", "_version.py", *versioned_dirs} all_items = os.scandir(base_wheel / base_dir) removed_count = 0 for f in all_items: @@ -169,6 +166,11 @@ def merge_wheels(wheels: list[Path], output_dir: Path, show_wheel_contents: bool os.remove(f_abspath) removed_count += 1 print(f"Removed {removed_count} items from cuda/core/ directory", file=sys.stderr) + remaining = {entry.name for entry in os.scandir(base_wheel / base_dir)} + if remaining != items_to_keep: + raise RuntimeError( + f"unexpected top level under cuda/core/: {sorted(remaining)} (expected {sorted(items_to_keep)})" + ) # Repack the merged wheel output_dir.mkdir(parents=True, exist_ok=True) diff --git a/cuda_core/build_hooks.py b/cuda_core/build_hooks.py index 01f6c756bc8..f105c92005a 100644 --- a/cuda_core/build_hooks.py +++ b/cuda_core/build_hooks.py @@ -188,14 +188,14 @@ def _extension_sources(mod_name): cuda/core/_cpp//, or the single legacy file cuda/core/_cpp/.cpp. Example: _tensor_map.pyx compiles _cpp/tensor_map.cpp.""" sources = [f"cuda/core/{mod_name}.pyx"] - cpp_stem = os.path.join("cuda", "core", "_cpp", mod_name.lstrip("_")) - if os.path.isdir(cpp_stem): - cpp_sources = sorted(glob.glob(os.path.join(cpp_stem, "**", "*.cpp"), recursive=True)) + cpp_stem = Path("cuda", "core", "_cpp", mod_name.lstrip("_")) + if cpp_stem.is_dir(): + cpp_sources = sorted(str(path) for path in cpp_stem.rglob("*.cpp")) if not cpp_sources: raise RuntimeError(f"{cpp_stem}/ exists but contains no .cpp files") sources.extend(cpp_sources) - elif os.path.isfile(cpp_stem + ".cpp"): - sources.append(cpp_stem + ".cpp") + elif cpp_stem.with_suffix(".cpp").is_file(): + sources.append(str(cpp_stem.with_suffix(".cpp"))) return sources From bf92d4f22f77f8d510be8d084f3da5973c055196 Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Fri, 11 Sep 2026 10:32:46 -0700 Subject: [PATCH 08/14] cuda.core: rename _resource_handles to _rt, no code motion The module holds cuda.core's runtime support layer: resource handles, the driver function-pointer table, error reporting and deferred cleanup. Rename it to _rt and move its C++ under _cpp/rt/ in namespace cuda_core::rt. Every function keeps its body, signature and symbol name; only the namespace qualifier and the file names change. - git mv _resource_handles.{pyx,pxd,pyi} to _rt.*; resource_handles.{hpp,cpp} to _cpp/rt/rt.{hpp,cpp}; the three design notes to _cpp/rt/. - The 53 cimporting Cython files change only their cimport line. - Delete _CUDA_DRIVER_API_V1_NAME, a capsule-name constant nothing reads. - Regenerate the stub (stubgen-pyx). The dynamic symbol table of the built extension equals the old one under s/cuda_core::/cuda_core::rt::/, and __pyx_capi__ has the same keys. --- cuda_core/cuda/core/_context.pxd | 2 +- cuda_core/cuda/core/_context.pyx | 2 +- cuda_core/cuda/core/_cpp/{ => rt}/DESIGN.md | 0 .../core/_cpp/{ => rt}/GRAPH_ATTACHMENTS.md | 0 .../core/_cpp/{ => rt}/REGISTRY_DESIGN.md | 0 .../_cpp/{resource_handles.cpp => rt/rt.cpp} | 10 +- .../_cpp/{resource_handles.hpp => rt/rt.hpp} | 12 +- cuda_core/cuda/core/_device.pyx | 2 +- cuda_core/cuda/core/_device_resources.pxd | 2 +- cuda_core/cuda/core/_device_resources.pyx | 4 +- cuda_core/cuda/core/_event.pxd | 2 +- cuda_core/cuda/core/_event.pyx | 2 +- cuda_core/cuda/core/_graphics.pxd | 2 +- cuda_core/cuda/core/_graphics.pyx | 2 +- cuda_core/cuda/core/_launcher.pyx | 2 +- cuda_core/cuda/core/_linker.pxd | 2 +- cuda_core/cuda/core/_linker.pyx | 2 +- cuda_core/cuda/core/_memory/_buffer.pxd | 2 +- cuda_core/cuda/core/_memory/_buffer.pyx | 4 +- .../cuda/core/_memory/_copy_attributes.pxd | 2 +- cuda_core/cuda/core/_memory/_copy_ops.pyx | 2 +- .../core/_memory/_device_memory_resource.pyx | 2 +- .../core/_memory/_graph_memory_resource.pyx | 2 +- cuda_core/cuda/core/_memory/_ipc.pxd | 2 +- cuda_core/cuda/core/_memory/_ipc.pyx | 2 +- .../cuda/core/_memory/_managed_memory_ops.pyx | 2 +- cuda_core/cuda/core/_memory/_memory_pool.pxd | 2 +- cuda_core/cuda/core/_memory/_memory_pool.pyx | 4 +- .../cuda/core/_memory/_peer_access_utils.pyx | 2 +- .../_memory/_synchronous_memory_resource.pyx | 2 +- cuda_core/cuda/core/_memoryview.pyx | 2 +- cuda_core/cuda/core/_module.pxd | 2 +- cuda_core/cuda/core/_module.pyx | 2 +- cuda_core/cuda/core/_program.pxd | 2 +- cuda_core/cuda/core/_program.pyx | 2 +- .../core/{_resource_handles.pxd => _rt.pxd} | 14 +- .../core/{_resource_handles.pyi => _rt.pyi} | 2 +- .../core/{_resource_handles.pyx => _rt.pyx} | 377 +++++++++--------- cuda_core/cuda/core/_stream.pxd | 2 +- cuda_core/cuda/core/_stream.pyx | 4 +- cuda_core/cuda/core/_tensor_bridge.pyx | 2 +- cuda_core/cuda/core/_utils/_weak_handles.pyx | 2 +- cuda_core/cuda/core/_utils/cuda_utils.pyx | 2 +- .../cuda/core/graph/_adjacency_set_proxy.pyx | 2 +- cuda_core/cuda/core/graph/_graph_builder.pxd | 2 +- cuda_core/cuda/core/graph/_graph_builder.pyx | 4 +- .../cuda/core/graph/_graph_definition.pxd | 2 +- .../cuda/core/graph/_graph_definition.pyx | 2 +- cuda_core/cuda/core/graph/_graph_node.pxd | 2 +- cuda_core/cuda/core/graph/_graph_node.pyx | 4 +- cuda_core/cuda/core/graph/_host_callback.pxd | 2 +- cuda_core/cuda/core/graph/_host_callback.pyx | 2 +- cuda_core/cuda/core/graph/_subclasses.pxd | 2 +- cuda_core/cuda/core/graph/_subclasses.pyx | 4 +- cuda_core/cuda/core/texture/_array.pxd | 2 +- cuda_core/cuda/core/texture/_array.pyx | 2 +- .../cuda/core/texture/_mipmapped_array.pxd | 2 +- .../cuda/core/texture/_mipmapped_array.pyx | 2 +- cuda_core/cuda/core/texture/_surface.pxd | 2 +- cuda_core/cuda/core/texture/_surface.pyx | 2 +- cuda_core/cuda/core/texture/_texture.pxd | 2 +- cuda_core/cuda/core/texture/_texture.pyx | 2 +- cuda_core/tests/test_error_handling.py | 2 +- 63 files changed, 267 insertions(+), 272 deletions(-) rename cuda_core/cuda/core/_cpp/{ => rt}/DESIGN.md (100%) rename cuda_core/cuda/core/_cpp/{ => rt}/GRAPH_ATTACHMENTS.md (100%) rename cuda_core/cuda/core/_cpp/{ => rt}/REGISTRY_DESIGN.md (100%) rename cuda_core/cuda/core/_cpp/{resource_handles.cpp => rt/rt.cpp} (99%) rename cuda_core/cuda/core/_cpp/{resource_handles.hpp => rt/rt.hpp} (99%) rename cuda_core/cuda/core/{_resource_handles.pxd => _rt.pxd} (97%) rename cuda_core/cuda/core/{_resource_handles.pyi => _rt.pyi} (98%) rename cuda_core/cuda/core/{_resource_handles.pyx => _rt.pyx} (67%) diff --git a/cuda_core/cuda/core/_context.pxd b/cuda_core/cuda/core/_context.pxd index 078c0c33415..3476fffb322 100644 --- a/cuda_core/cuda/core/_context.pxd +++ b/cuda_core/cuda/core/_context.pxd @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -from cuda.core._resource_handles cimport ContextHandle, GreenCtxHandle +from cuda.core._rt cimport ContextHandle, GreenCtxHandle cdef class Context: """Cython declaration for Context class. diff --git a/cuda_core/cuda/core/_context.pyx b/cuda_core/cuda/core/_context.pyx index 6855d0cc255..7f5fba9f80f 100644 --- a/cuda_core/cuda/core/_context.pyx +++ b/cuda_core/cuda/core/_context.pyx @@ -13,7 +13,7 @@ import cython from cuda.bindings cimport cydriver from cuda.core._device_resources cimport DeviceResources, SMResource, WorkqueueResource from cuda.core._device_resources import SMResource, WorkqueueResource -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( ContextHandle, GreenCtxHandle, as_cu, diff --git a/cuda_core/cuda/core/_cpp/DESIGN.md b/cuda_core/cuda/core/_cpp/rt/DESIGN.md similarity index 100% rename from cuda_core/cuda/core/_cpp/DESIGN.md rename to cuda_core/cuda/core/_cpp/rt/DESIGN.md diff --git a/cuda_core/cuda/core/_cpp/GRAPH_ATTACHMENTS.md b/cuda_core/cuda/core/_cpp/rt/GRAPH_ATTACHMENTS.md similarity index 100% rename from cuda_core/cuda/core/_cpp/GRAPH_ATTACHMENTS.md rename to cuda_core/cuda/core/_cpp/rt/GRAPH_ATTACHMENTS.md diff --git a/cuda_core/cuda/core/_cpp/REGISTRY_DESIGN.md b/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md similarity index 100% rename from cuda_core/cuda/core/_cpp/REGISTRY_DESIGN.md rename to cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md diff --git a/cuda_core/cuda/core/_cpp/resource_handles.cpp b/cuda_core/cuda/core/_cpp/rt/rt.cpp similarity index 99% rename from cuda_core/cuda/core/_cpp/resource_handles.cpp rename to cuda_core/cuda/core/_cpp/rt/rt.cpp index 8e455c73854..41af411a412 100644 --- a/cuda_core/cuda/core/_cpp/resource_handles.cpp +++ b/cuda_core/cuda/core/_cpp/rt/rt.cpp @@ -4,7 +4,7 @@ #include -#include "resource_handles.hpp" +#include "rt.hpp" #include #include #include @@ -27,12 +27,12 @@ #include #endif -namespace cuda_core { +namespace cuda_core::rt { // ============================================================================ // CUDA driver function pointers // -// These are populated by _resource_handles.pyx at module import time using +// These are populated by _rt.pyx at module import time using // function pointers extracted from cuda.bindings.cydriver.__pyx_capi__. // ============================================================================ @@ -208,7 +208,7 @@ class GILAcquireGuard { // otherwise. See docs/source/error_handling.rst for the policy. // ---------------------------------------------------------------------------- -// Warning category registered by _resource_handles.pyx (cuda.core.CUDAWarning). +// Warning category registered by _utils/cuda_utils.pyx (cuda.core.CUDAWarning). std::atomic warning_category{nullptr}; // Thread-local detail attached to the next raised CUDAError with a matching @@ -3357,4 +3357,4 @@ bool has_memcpy_with_attributes_async() noexcept { return p_cuMemcpyWithAttributesAsync != nullptr; } -} // namespace cuda_core +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/resource_handles.hpp b/cuda_core/cuda/core/_cpp/rt/rt.hpp similarity index 99% rename from cuda_core/cuda/core/_cpp/resource_handles.hpp rename to cuda_core/cuda/core/_cpp/rt/rt.hpp index 5ad65143659..bbb370b5089 100644 --- a/cuda_core/cuda/core/_cpp/resource_handles.hpp +++ b/cuda_core/cuda/core/_cpp/rt/rt.hpp @@ -18,7 +18,7 @@ using nvvmProgram = void*; // Use void* to match cuda.bindings.cynvjitlink's typedef using nvJitLink_t = void*; -namespace cuda_core { +namespace cuda_core::rt { // ============================================================================ // TaggedHandle - make void*-based handle types distinct for overloading @@ -103,7 +103,7 @@ void set_context_restore_fault_for_testing(CUresult status) noexcept; // ============================================================================ // CUDA driver function pointers // -// These are populated by _resource_handles.pyx at module import time using +// These are populated by _rt.pyx at module import time using // function pointers extracted from cuda.bindings.cydriver.__pyx_capi__. // ============================================================================ @@ -209,7 +209,7 @@ extern void* p_cuMemcpyWithAttributesAsync; // ============================================================================ // NVRTC function pointers // -// These are populated by _resource_handles.pyx at module import time using +// These are populated by _rt.pyx at module import time using // function pointers extracted from cuda.bindings.cynvrtc.__pyx_capi__. // ============================================================================ @@ -218,7 +218,7 @@ extern decltype(&nvrtcDestroyProgram) p_nvrtcDestroyProgram; // ============================================================================ // NVVM function pointers // -// These are populated by _resource_handles.pyx at module import time using +// These are populated by _rt.pyx at module import time using // function pointers extracted from cuda.bindings.cynvvm.__pyx_capi__. // Note: May be null if NVVM is not available at runtime. // ============================================================================ @@ -231,7 +231,7 @@ extern NvvmDestroyProgramFn p_nvvmDestroyProgram; // ============================================================================ // nvJitLink function pointers // -// These are populated by _resource_handles.pyx at module import time using +// These are populated by _rt.pyx at module import time using // function pointers extracted from cuda.bindings.cynvjitlink.__pyx_capi__. // Note: May be null if nvJitLink is not available at runtime. // ============================================================================ @@ -1236,4 +1236,4 @@ CUresult memcpy_with_attributes_async(CUdeviceptr dst, CUdeviceptr src, size_t s // Returns true if the cuMemcpyWithAttributesAsync function pointer is available. bool has_memcpy_with_attributes_async() noexcept; -} // namespace cuda_core +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_device.pyx b/cuda_core/cuda/core/_device.pyx index 170bedc0034..40b09d6c761 100644 --- a/cuda_core/cuda/core/_device.pyx +++ b/cuda_core/cuda/core/_device.pyx @@ -18,7 +18,7 @@ from cuda.core._device_resources cimport DeviceResources, SMResource, WorkqueueR from cuda.core._event cimport Event as cyEvent from cuda.core._event import Event, EventOptions from cuda.core._memory._buffer cimport Buffer, MemoryResource -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( ContextHandle, GreenCtxHandle, create_context_handle_ref, diff --git a/cuda_core/cuda/core/_device_resources.pxd b/cuda_core/cuda/core/_device_resources.pxd index d618c24cf10..212e3cac9b1 100644 --- a/cuda_core/cuda/core/_device_resources.pxd +++ b/cuda_core/cuda/core/_device_resources.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ContextHandle, GreenCtxHandle +from cuda.core._rt cimport ContextHandle, GreenCtxHandle cdef class SMResource: diff --git a/cuda_core/cuda/core/_device_resources.pyx b/cuda_core/cuda/core/_device_resources.pyx index 15ca6c56685..c9fe33c4fd7 100644 --- a/cuda_core/cuda/core/_device_resources.pyx +++ b/cuda_core/cuda/core/_device_resources.pyx @@ -17,7 +17,7 @@ from libc.stdlib cimport free, malloc from libc.string cimport memset from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ContextHandle, GreenCtxHandle, as_cu, get_context_green_ctx +from cuda.core._rt cimport ContextHandle, GreenCtxHandle, as_cu, get_context_green_ctx from cuda.core._utils.cuda_utils cimport check_or_create_options, HANDLE_RETURN from cuda.core._utils.cuda_utils import is_sequence from cuda.core._utils.version cimport cy_binding_version, cy_driver_version @@ -226,7 +226,7 @@ cdef inline unsigned int _to_sm_count(object value) except? 0: IF CUDA_CORE_BUILD_MAJOR >= 13: - from cuda.core._resource_handles cimport sm_resource_split, has_sm_resource_split + from cuda.core._rt cimport sm_resource_split, has_sm_resource_split cdef int _structured_split_checked = 0 diff --git a/cuda_core/cuda/core/_event.pxd b/cuda_core/cuda/core/_event.pxd index c1ab008d5e1..aa4726dc383 100644 --- a/cuda_core/cuda/core/_event.pxd +++ b/cuda_core/cuda/core/_event.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ContextHandle, EventHandle +from cuda.core._rt cimport ContextHandle, EventHandle cdef class Event: diff --git a/cuda_core/cuda/core/_event.pyx b/cuda_core/cuda/core/_event.pyx index dcbb55ba5a4..965c4b878eb 100644 --- a/cuda_core/cuda/core/_event.pyx +++ b/cuda_core/cuda/core/_event.pyx @@ -9,7 +9,7 @@ from libc.stddef cimport size_t from libc.string cimport memcpy from cuda.bindings cimport cydriver from cuda.core._context cimport Context -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( ContextHandle, EventHandle, create_event_handle, diff --git a/cuda_core/cuda/core/_graphics.pxd b/cuda_core/cuda/core/_graphics.pxd index 520a366bbde..d8d4838d111 100644 --- a/cuda_core/cuda/core/_graphics.pxd +++ b/cuda_core/cuda/core/_graphics.pxd @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -from cuda.core._resource_handles cimport GraphicsResourceHandle +from cuda.core._rt cimport GraphicsResourceHandle cdef class GraphicsResource: diff --git a/cuda_core/cuda/core/_graphics.pyx b/cuda_core/cuda/core/_graphics.pyx index b8945bb0d71..6e5ea061f50 100644 --- a/cuda_core/cuda/core/_graphics.pyx +++ b/cuda_core/cuda/core/_graphics.pyx @@ -7,7 +7,7 @@ from __future__ import annotations from typing import Sequence from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( create_graphics_resource_handle, deviceptr_create_mapped_graphics, as_cu, diff --git a/cuda_core/cuda/core/_launcher.pyx b/cuda_core/cuda/core/_launcher.pyx index 036189790e0..e5947b48be2 100644 --- a/cuda_core/cuda/core/_launcher.pyx +++ b/cuda_core/cuda/core/_launcher.pyx @@ -9,7 +9,7 @@ from cuda.bindings cimport cydriver from cuda.core._launch_config cimport LaunchConfig from cuda.core._kernel_arg_handler cimport ParamHolder from cuda.core._module cimport Kernel -from cuda.core._resource_handles cimport as_cu +from cuda.core._rt cimport as_cu from cuda.core._stream cimport Stream_accept, Stream from cuda.core._utils.cuda_utils cimport ( check_or_create_options, diff --git a/cuda_core/cuda/core/_linker.pxd b/cuda_core/cuda/core/_linker.pxd index 1b7d39fd1d4..63fa517fb8e 100644 --- a/cuda_core/cuda/core/_linker.pxd +++ b/cuda_core/cuda/core/_linker.pxd @@ -6,7 +6,7 @@ from libcpp.vector cimport vector from cuda.bindings cimport cydriver -from ._resource_handles cimport NvJitLinkHandle, CuLinkHandle +from ._rt cimport NvJitLinkHandle, CuLinkHandle cdef class Linker: diff --git a/cuda_core/cuda/core/_linker.pyx b/cuda_core/cuda/core/_linker.pyx index f104e2d158b..3e68d24afa3 100644 --- a/cuda_core/cuda/core/_linker.pyx +++ b/cuda_core/cuda/core/_linker.pyx @@ -16,7 +16,7 @@ from libcpp.vector cimport vector from cuda.bindings cimport cydriver from cuda.bindings cimport cynvjitlink -from ._resource_handles cimport ( +from ._rt cimport ( as_cu, as_py, create_culink_handle, diff --git a/cuda_core/cuda/core/_memory/_buffer.pxd b/cuda_core/cuda/core/_memory/_buffer.pxd index 75d94c91d58..e929210601f 100644 --- a/cuda_core/cuda/core/_memory/_buffer.pxd +++ b/cuda_core/cuda/core/_memory/_buffer.pxd @@ -5,7 +5,7 @@ from libcpp cimport bool as cpp_bool from libcpp.atomic cimport atomic as std_atomic -from cuda.core._resource_handles cimport DevicePtrHandle +from cuda.core._rt cimport DevicePtrHandle cdef struct _MemAttrs: diff --git a/cuda_core/cuda/core/_memory/_buffer.pyx b/cuda_core/cuda/core/_memory/_buffer.pyx index 48d81681c74..02a8d2078ee 100644 --- a/cuda_core/cuda/core/_memory/_buffer.pyx +++ b/cuda_core/cuda/core/_memory/_buffer.pyx @@ -13,7 +13,7 @@ from cuda.core._memory._device_memory_resource import DeviceMemoryResource from cuda.core._memory._pinned_memory_resource import PinnedMemoryResource from cuda.core._memory._ipc cimport IPCBufferDescriptor, IPCDataForBuffer from cuda.core._memory cimport _ipc -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( DevicePtrHandle, StreamHandle, ContextHandle, @@ -31,7 +31,7 @@ from cuda.core._memory._copy_attributes cimport _with_attributes_available from cuda.core._memory._copy_attributes cimport _to_cu_memcpy_attributes # no-cython-lint IF CUDA_CORE_BUILD_MAJOR >= 13: - from cuda.core._resource_handles cimport memcpy_with_attributes_async + from cuda.core._rt cimport memcpy_with_attributes_async from cuda.core._stream cimport Stream, Stream_accept, Stream_is_legacy_default_token, default_stream from cuda.core._utils.cuda_utils cimport HANDLE_RETURN, _parse_fill_value diff --git a/cuda_core/cuda/core/_memory/_copy_attributes.pxd b/cuda_core/cuda/core/_memory/_copy_attributes.pxd index 96c213dfec5..3e4ae2e778f 100644 --- a/cuda_core/cuda/core/_memory/_copy_attributes.pxd +++ b/cuda_core/cuda/core/_memory/_copy_attributes.pxd @@ -11,7 +11,7 @@ from cuda.core._utils.version cimport cy_binding_version, cy_driver_version # n IF CUDA_CORE_BUILD_MAJOR >= 13: - from cuda.core._resource_handles cimport has_memcpy_with_attributes_async + from cuda.core._rt cimport has_memcpy_with_attributes_async cdef inline bint _with_attributes_available(): # has_memcpy_with_attributes_async() says whether the installed diff --git a/cuda_core/cuda/core/_memory/_copy_ops.pyx b/cuda_core/cuda/core/_memory/_copy_ops.pyx index e57be2e40a0..be0ecafaed7 100644 --- a/cuda_core/cuda/core/_memory/_copy_ops.pyx +++ b/cuda_core/cuda/core/_memory/_copy_ops.pyx @@ -12,7 +12,7 @@ IF CUDA_CORE_BUILD_MAJOR >= 13: from cuda.bindings cimport cydriver from cuda.core._memory._buffer cimport Buffer, Buffer_coerce_batch from cuda.core._memory._copy_attributes cimport _to_cu_memcpy_attributes # no-cython-lint -from cuda.core._resource_handles cimport as_cu +from cuda.core._rt cimport as_cu from cuda.core._stream cimport Stream, Stream_accept, Stream_is_legacy_default_token from cuda.core._utils.cuda_utils cimport HANDLE_RETURN diff --git a/cuda_core/cuda/core/_memory/_device_memory_resource.pyx b/cuda_core/cuda/core/_memory/_device_memory_resource.pyx index 62dc4f9e747..8a0fed73eee 100644 --- a/cuda_core/cuda/core/_memory/_device_memory_resource.pyx +++ b/cuda_core/cuda/core/_memory/_device_memory_resource.pyx @@ -11,7 +11,7 @@ from cuda.core._memory._memory_pool cimport ( ) from cuda.core._memory cimport _ipc from cuda.core._memory._ipc cimport IPCAllocationHandle -from cuda.core._resource_handles cimport as_cu, get_device_mempool, get_last_error +from cuda.core._rt cimport as_cu, get_device_mempool, get_last_error from cuda.core._utils.cuda_utils cimport ( check_or_create_options, HANDLE_RETURN, diff --git a/cuda_core/cuda/core/_memory/_graph_memory_resource.pyx b/cuda_core/cuda/core/_memory/_graph_memory_resource.pyx index 67ecf97f58c..9d26798c630 100644 --- a/cuda_core/cuda/core/_memory/_graph_memory_resource.pyx +++ b/cuda_core/cuda/core/_memory/_graph_memory_resource.pyx @@ -8,7 +8,7 @@ from libc.stdint cimport intptr_t from cuda.bindings cimport cydriver from cuda.core._memory._buffer cimport Buffer, Buffer_from_deviceptr_handle, MemoryResource -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( DevicePtrHandle, deviceptr_alloc_async, get_last_error, diff --git a/cuda_core/cuda/core/_memory/_ipc.pxd b/cuda_core/cuda/core/_memory/_ipc.pxd index b912ace0035..5321d48fe13 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pxd +++ b/cuda_core/cuda/core/_memory/_ipc.pxd @@ -5,7 +5,7 @@ from cuda.bindings cimport cydriver from cuda.core._memory._buffer cimport Buffer from cuda.core._memory._memory_pool cimport _MemPool -from cuda.core._resource_handles cimport FileDescriptorHandle +from cuda.core._rt cimport FileDescriptorHandle # Holds _MemPool objects imported by this process. This enables diff --git a/cuda_core/cuda/core/_memory/_ipc.pyx b/cuda_core/cuda/core/_memory/_ipc.pyx index ae8db6589b4..7923c21fc00 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pyx +++ b/cuda_core/cuda/core/_memory/_ipc.pyx @@ -9,7 +9,7 @@ from cuda.bindings cimport cydriver from cuda.core._memory._buffer cimport Buffer, Buffer_check_open, Buffer_from_deviceptr_handle from cuda.core._memory._memory_pool cimport _MemPool, MP_check_open from cuda.core._stream cimport Stream, Stream_accept -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( DevicePtrHandle, create_fd_handle, create_mempool_handle_ipc, diff --git a/cuda_core/cuda/core/_memory/_managed_memory_ops.pyx b/cuda_core/cuda/core/_memory/_managed_memory_ops.pyx index e433ccf1e38..61caa59ce1f 100644 --- a/cuda_core/cuda/core/_memory/_managed_memory_ops.pyx +++ b/cuda_core/cuda/core/_memory/_managed_memory_ops.pyx @@ -18,7 +18,7 @@ from cuda.core._memory._buffer cimport Buffer, Buffer_check_open, Buffer_coerce_ # need a pragma to be seen as used. from cuda.core._memory._location cimport cumemlocation_from_id # no-cython-lint from cuda.core._memory._location cimport to_cumemlocation # no-cython-lint -from cuda.core._resource_handles cimport as_cu +from cuda.core._rt cimport as_cu from cuda.core._stream cimport Stream, Stream_accept from cuda.core._utils.cuda_utils cimport HANDLE_RETURN diff --git a/cuda_core/cuda/core/_memory/_memory_pool.pxd b/cuda_core/cuda/core/_memory/_memory_pool.pxd index 23e5eba588e..b85e5182e3b 100644 --- a/cuda_core/cuda/core/_memory/_memory_pool.pxd +++ b/cuda_core/cuda/core/_memory/_memory_pool.pxd @@ -5,7 +5,7 @@ from cuda.bindings cimport cydriver from cuda.core._memory._buffer cimport Buffer, MemoryResource from cuda.core._memory._ipc cimport IPCDataForMR -from cuda.core._resource_handles cimport MemoryPoolHandle +from cuda.core._rt cimport MemoryPoolHandle from cuda.core._stream cimport Stream diff --git a/cuda_core/cuda/core/_memory/_memory_pool.pyx b/cuda_core/cuda/core/_memory/_memory_pool.pyx index 6e5d26e7df8..c0d2b010c79 100644 --- a/cuda_core/cuda/core/_memory/_memory_pool.pyx +++ b/cuda_core/cuda/core/_memory/_memory_pool.pyx @@ -16,7 +16,7 @@ from cuda.core._memory cimport _ipc # does not evaluate compile-time IF blocks, so it needs a pragma to be seen as used. from cuda.core._memory._location cimport cumemlocation_from_id # no-cython-lint from cuda.core._stream cimport Stream_accept, Stream -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( MemoryPoolHandle, DevicePtrHandle, create_mempool_handle, @@ -25,7 +25,7 @@ from cuda.core._resource_handles cimport ( as_cu, as_py, ) -from cuda.core._resource_handles cimport create_mempool_handle_ref # no-cython-lint +from cuda.core._rt cimport create_mempool_handle_ref # no-cython-lint from cuda.core._utils.cuda_utils cimport ( HANDLE_RETURN, diff --git a/cuda_core/cuda/core/_memory/_peer_access_utils.pyx b/cuda_core/cuda/core/_memory/_peer_access_utils.pyx index 21f88258f57..be3426d8a69 100644 --- a/cuda_core/cuda/core/_memory/_peer_access_utils.pyx +++ b/cuda_core/cuda/core/_memory/_peer_access_utils.pyx @@ -14,7 +14,7 @@ from cuda.bindings cimport cydriver from cuda.core._memory._device_memory_resource cimport DeviceMemoryResource from cuda.core._memory._location cimport cumemlocation_from_id from cuda.core._memory._memory_pool cimport MP_check_open -from cuda.core._resource_handles cimport as_cu +from cuda.core._rt cimport as_cu from cuda.core._utils.cuda_utils cimport HANDLE_RETURN from cpython.mem cimport PyMem_Malloc, PyMem_Free from libcpp.vector cimport vector diff --git a/cuda_core/cuda/core/_memory/_synchronous_memory_resource.pyx b/cuda_core/cuda/core/_memory/_synchronous_memory_resource.pyx index f02f38f69b1..e5438797664 100644 --- a/cuda_core/cuda/core/_memory/_synchronous_memory_resource.pyx +++ b/cuda_core/cuda/core/_memory/_synchronous_memory_resource.pyx @@ -9,7 +9,7 @@ from libc.stdint cimport uintptr_t from cuda.bindings cimport cydriver from cuda.core._context cimport Context from cuda.core._memory._buffer cimport Buffer, MemoryResource -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( ContextHandle, create_context_bound_legacy_stream, deviceptr_alloc_raw, diff --git a/cuda_core/cuda/core/_memoryview.pyx b/cuda_core/cuda/core/_memoryview.pyx index 129cb91711b..5ce1cc1faa8 100644 --- a/cuda_core/cuda/core/_memoryview.pyx +++ b/cuda_core/cuda/core/_memoryview.pyx @@ -24,7 +24,7 @@ if TYPE_CHECKING: import numpy from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( EventHandle, create_event_handle_for_stream, as_cu, diff --git a/cuda_core/cuda/core/_module.pxd b/cuda_core/cuda/core/_module.pxd index 5e9d08fc13f..05a070a8afb 100644 --- a/cuda_core/cuda/core/_module.pxd +++ b/cuda_core/cuda/core/_module.pxd @@ -5,7 +5,7 @@ from libcpp.mutex cimport py_safe_once_flag from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport LibraryHandle, KernelHandle +from cuda.core._rt cimport LibraryHandle, KernelHandle cdef class ObjectCode cdef class Kernel diff --git a/cuda_core/cuda/core/_module.pyx b/cuda_core/cuda/core/_module.pyx index a350f14887f..fbaaef9e1e8 100644 --- a/cuda_core/cuda/core/_module.pyx +++ b/cuda_core/cuda/core/_module.pyx @@ -16,7 +16,7 @@ from cuda.core._launch_config cimport LaunchConfig from cuda.core._launch_config import LaunchConfig from cuda.core._stream cimport Stream, Stream_accept from cuda.core._program import ObjectCodeFormatType -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( LibraryHandle, KernelHandle, create_library_handle_from_file, diff --git a/cuda_core/cuda/core/_program.pxd b/cuda_core/cuda/core/_program.pxd index e1cbaa6fbda..88aedbb0b1c 100644 --- a/cuda_core/cuda/core/_program.pxd +++ b/cuda_core/cuda/core/_program.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from ._linker cimport Linker -from ._resource_handles cimport NvrtcProgramHandle, NvvmProgramHandle +from ._rt cimport NvrtcProgramHandle, NvvmProgramHandle cdef class Program: diff --git a/cuda_core/cuda/core/_program.pyx b/cuda_core/cuda/core/_program.pyx index 9ea624f8e83..bf6c16e19f7 100644 --- a/cuda_core/cuda/core/_program.pyx +++ b/cuda_core/cuda/core/_program.pyx @@ -26,7 +26,7 @@ from cuda.pathfinder._optional_cuda_import import _optional_cuda_import from libcpp.vector cimport vector -from ._resource_handles cimport ( +from ._rt cimport ( as_cu, as_py, create_nvrtc_program_handle, diff --git a/cuda_core/cuda/core/_resource_handles.pxd b/cuda_core/cuda/core/_rt.pxd similarity index 97% rename from cuda_core/cuda/core/_resource_handles.pxd rename to cuda_core/cuda/core/_rt.pxd index 339e2610b0e..eb90c5ef529 100644 --- a/cuda_core/cuda/core/_resource_handles.pxd +++ b/cuda_core/cuda/core/_rt.pxd @@ -18,7 +18,7 @@ from cuda.bindings cimport cynvjitlink # Handle type aliases and inline helpers (declared from C++ header) # ============================================================================= -cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": +cdef extern from "_cpp/rt/rt.hpp" namespace "cuda_core::rt": # Handle types ctypedef shared_ptr[const cydriver.CUcontext] ContextHandle ctypedef shared_ptr[const cydriver.CUgreenCtx] GreenCtxHandle @@ -36,9 +36,9 @@ cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": # NvvmProgramValue and NvJitLinkValue are TaggedHandle # instantiations that make each shared_ptr type distinct for overloading. - cppclass NvvmProgramValue "cuda_core::NvvmProgramValue": + cppclass NvvmProgramValue "cuda_core::rt::NvvmProgramValue": pass - cppclass NvJitLinkValue "cuda_core::NvJitLinkValue": + cppclass NvJitLinkValue "cuda_core::rt::NvJitLinkValue": pass ctypedef shared_ptr[const NvvmProgramValue] NvvmProgramHandle ctypedef shared_ptr[const NvJitLinkValue] NvJitLinkHandle @@ -51,9 +51,9 @@ cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": # CUtexObject / CUsurfObject are both `unsigned long long` (as is CUdeviceptr), # so they are wrapped in distinct tagged value types to keep each handle's # as_cu/as_intptr/as_py overloads distinct. - cppclass TexObjectValue "cuda_core::TexObjectValue": + cppclass TexObjectValue "cuda_core::rt::TexObjectValue": pass - cppclass SurfObjectValue "cuda_core::SurfObjectValue": + cppclass SurfObjectValue "cuda_core::rt::SurfObjectValue": pass ctypedef shared_ptr[const TexObjectValue] TexObjectHandle ctypedef shared_ptr[const SurfObjectValue] SurfObjectHandle @@ -159,9 +159,9 @@ cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": # ============================================================================= -# Wrapper function declarations (implemented in _resource_handles.pyx) +# Wrapper function declarations (implemented in _rt.pyx) # -# Consumer modules cimport these. Calls go through _resource_handles.so. +# Consumer modules cimport these. Calls go through _rt.so. # ============================================================================= # Thread-local error handling diff --git a/cuda_core/cuda/core/_resource_handles.pyi b/cuda_core/cuda/core/_rt.pyi similarity index 98% rename from cuda_core/cuda/core/_resource_handles.pyi rename to cuda_core/cuda/core/_rt.pyi index 882ccdc483d..d25c3ad5b56 100644 --- a/cuda_core/cuda/core/_resource_handles.pyi +++ b/cuda_core/cuda/core/_rt.pyi @@ -1,4 +1,4 @@ -# This file was generated by stubgen-pyx v0.2.22 from cuda_core/cuda/core/_resource_handles.pyx +# This file was generated by stubgen-pyx v0.2.22 from cuda_core/cuda/core/_rt.pyx from typing import Callable diff --git a/cuda_core/cuda/core/_resource_handles.pyx b/cuda_core/cuda/core/_rt.pyx similarity index 67% rename from cuda_core/cuda/core/_resource_handles.pyx rename to cuda_core/cuda/core/_rt.pyx index 692ae7368e5..c42c0ccaf66 100644 --- a/cuda_core/cuda/core/_resource_handles.pyx +++ b/cuda_core/cuda/core/_rt.pyx @@ -2,10 +2,14 @@ # # SPDX-License-Identifier: Apache-2.0 -# This module compiles _cpp/resource_handles.cpp into a shared library. -# Consumer modules cimport the functions declared in _resource_handles.pxd. -# Since there is only one copy of the C++ code (in this .so), all static and -# thread-local state is shared correctly across all consumer modules. +# This module compiles the C++ under _cpp/rt/ into one shared library. +# Consumer modules cimport the functions declared in _rt.pxd. Since there is +# only one copy of the C++ code (in this .so), all static and thread-local +# state is shared correctly across all consumer modules. +# +# "rt" is short for runtime: this is cuda.core's runtime support layer +# (resource handles, the driver function-pointer table, error reporting and +# deferred cleanup). It is unrelated to the CUDA Runtime API (cudart). # # The cdef extern from declarations below satisfy the .pxd declarations directly, # without needing separate wrapper functions. @@ -25,306 +29,297 @@ import cuda.bindings.cynvvm as cynvvm import cuda.bindings.cynvjitlink as cynvjitlink # ============================================================================= -# C++ function declarations (non-inline, implemented in resource_handles.cpp) +# C++ function declarations (non-inline, implemented under _cpp/rt/) # -# These declarations satisfy the cdef function declarations in _resource_handles.pxd. +# These declarations satisfy the cdef function declarations in _rt.pxd. # Consumer modules cimport these functions and calls go through this .so. # ============================================================================= -cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": +cdef extern from "_cpp/rt/rt.hpp" namespace "cuda_core::rt": # Thread-local error handling - cydriver.CUresult get_last_error "cuda_core::get_last_error" () noexcept nogil - cydriver.CUresult peek_last_error "cuda_core::peek_last_error" () noexcept nogil - void clear_last_error "cuda_core::clear_last_error" () noexcept nogil + cydriver.CUresult get_last_error "cuda_core::rt::get_last_error" () noexcept nogil + cydriver.CUresult peek_last_error "cuda_core::rt::peek_last_error" () noexcept nogil + void clear_last_error "cuda_core::rt::clear_last_error" () noexcept nogil # Non-propagating error reporting - void register_warning_category "cuda_core::register_warning_category" ( + void register_warning_category "cuda_core::rt::register_warning_category" ( PyObject* category) noexcept - void report_cuda_error "cuda_core::report_cuda_error" ( + void report_cuda_error "cuda_core::rt::report_cuda_error" ( const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil - void report_message "cuda_core::report_message" (const char* message) noexcept nogil - void report_status_code "cuda_core::report_status_code" ( + void report_message "cuda_core::rt::report_message" (const char* message) noexcept nogil + void report_status_code "cuda_core::rt::report_status_code" ( const char* operation, long code) noexcept nogil - void note_or_report_cuda_error "cuda_core::note_or_report_cuda_error" ( + void note_or_report_cuda_error "cuda_core::rt::note_or_report_cuda_error" ( const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil # Alias for calls made from this module: calling the pxd-declared name here # would make Cython emit a conflicting static prototype for it. - void _note_or_report_cuda_error_local "cuda_core::note_or_report_cuda_error" ( + void _note_or_report_cuda_error_local "cuda_core::rt::note_or_report_cuda_error" ( const char* operation, cydriver.CUresult status, const char* detail) noexcept nogil - const char* take_last_error_detail "cuda_core::take_last_error_detail" ( + const char* take_last_error_detail "cuda_core::rt::take_last_error_detail" ( cydriver.CUresult status) noexcept nogil - void clear_last_error_detail "cuda_core::clear_last_error_detail" () noexcept nogil - void set_context_restore_fault_for_testing "cuda_core::set_context_restore_fault_for_testing" ( + void clear_last_error_detail "cuda_core::rt::clear_last_error_detail" () noexcept nogil + void set_context_restore_fault_for_testing "cuda_core::rt::set_context_restore_fault_for_testing" ( cydriver.CUresult status) noexcept nogil # Context handles - ContextHandle create_context_handle_ref "cuda_core::create_context_handle_ref" ( + ContextHandle create_context_handle_ref "cuda_core::rt::create_context_handle_ref" ( cydriver.CUcontext ctx) except+ nogil - ContextHandle create_context_handle_from_green_ctx "cuda_core::create_context_handle_from_green_ctx" ( + ContextHandle create_context_handle_from_green_ctx "cuda_core::rt::create_context_handle_from_green_ctx" ( const GreenCtxHandle& h_green_ctx) except+ nogil - GreenCtxHandle get_context_green_ctx "cuda_core::get_context_green_ctx" ( + GreenCtxHandle get_context_green_ctx "cuda_core::rt::get_context_green_ctx" ( const ContextHandle& h) noexcept nogil - GreenCtxHandle create_green_ctx_handle "cuda_core::create_green_ctx_handle" ( + GreenCtxHandle create_green_ctx_handle "cuda_core::rt::create_green_ctx_handle" ( cydriver.CUdevResource* resources, unsigned int nbResources, cydriver.CUdevice dev, unsigned int flags) except+ nogil - GreenCtxHandle create_green_ctx_handle_ref "cuda_core::create_green_ctx_handle_ref" ( + GreenCtxHandle create_green_ctx_handle_ref "cuda_core::rt::create_green_ctx_handle_ref" ( cydriver.CUgreenCtx ctx) except+ nogil - ContextHandle get_primary_context "cuda_core::get_primary_context" ( + ContextHandle get_primary_context "cuda_core::rt::get_primary_context" ( int device_id) except+ nogil - ContextHandle get_current_context "cuda_core::get_current_context" () except+ nogil - cydriver.CUresult context_synchronize "cuda_core::context_synchronize" ( + ContextHandle get_current_context "cuda_core::rt::get_current_context" () except+ nogil + cydriver.CUresult context_synchronize "cuda_core::rt::context_synchronize" ( const ContextHandle& h_context) noexcept nogil - cydriver.CUresult context_get_stream_priority_range "cuda_core::context_get_stream_priority_range" ( + cydriver.CUresult context_get_stream_priority_range "cuda_core::rt::context_get_stream_priority_range" ( const ContextHandle& h_context, int* least_priority, int* greatest_priority) noexcept nogil - cydriver.CUresult context_get_device "cuda_core::context_get_device" ( + cydriver.CUresult context_get_device "cuda_core::rt::context_get_device" ( const ContextHandle& h_context, cydriver.CUdevice* device) noexcept nogil - cydriver.CUresult graph_node_set_params "cuda_core::graph_node_set_params" ( + cydriver.CUresult graph_node_set_params "cuda_core::rt::graph_node_set_params" ( cydriver.CUgraphNode node, cydriver.CUgraphNodeParams* params, const ContextHandle& h_context, cydriver.CUresult* restore_status) noexcept nogil # Stream handles - StreamHandle create_stream_handle "cuda_core::create_stream_handle" ( + StreamHandle create_stream_handle "cuda_core::rt::create_stream_handle" ( const ContextHandle& h_ctx, unsigned int flags, int priority) except+ nogil - StreamHandle create_stream_handle_ref "cuda_core::create_stream_handle_ref" ( + StreamHandle create_stream_handle_ref "cuda_core::rt::create_stream_handle_ref" ( cydriver.CUstream stream) except+ nogil - StreamHandle create_stream_handle_with_owner "cuda_core::create_stream_handle_with_owner" ( + StreamHandle create_stream_handle_with_owner "cuda_core::rt::create_stream_handle_with_owner" ( cydriver.CUstream stream, object owner) except+ nogil - void py_object_user_object_destroy "cuda_core::py_object_user_object_destroy" ( + void py_object_user_object_destroy "cuda_core::rt::py_object_user_object_destroy" ( void* py_object) noexcept nogil - void initialize_deferred_cleanup "cuda_core::initialize_deferred_cleanup" () except+ - void retry_deferred_cleanup "cuda_core::retry_deferred_cleanup" () noexcept - ContextHandle get_stream_context "cuda_core::get_stream_context" ( + void initialize_deferred_cleanup "cuda_core::rt::initialize_deferred_cleanup" () except+ + void retry_deferred_cleanup "cuda_core::rt::retry_deferred_cleanup" () noexcept + ContextHandle get_stream_context "cuda_core::rt::get_stream_context" ( const StreamHandle& h) noexcept nogil - StreamHandle get_legacy_stream "cuda_core::get_legacy_stream" () except+ nogil - StreamHandle get_per_thread_stream "cuda_core::get_per_thread_stream" () except+ nogil - StreamHandle create_context_bound_legacy_stream "cuda_core::create_context_bound_legacy_stream" ( + StreamHandle get_legacy_stream "cuda_core::rt::get_legacy_stream" () except+ nogil + StreamHandle get_per_thread_stream "cuda_core::rt::get_per_thread_stream" () except+ nogil + StreamHandle create_context_bound_legacy_stream "cuda_core::rt::create_context_bound_legacy_stream" ( const ContextHandle& h_context) except+ nogil # Event handles (note: _create_event_handle* are internal due to C++ overloading) - EventHandle create_event_handle "cuda_core::create_event_handle" ( + EventHandle create_event_handle "cuda_core::rt::create_event_handle" ( const ContextHandle& h_ctx, unsigned int flags, bint timing_enabled, bint is_blocking_sync, bint ipc_enabled, int device_id) except+ nogil - EventHandle create_event_handle_for_stream "cuda_core::create_event_handle_for_stream" ( + EventHandle create_event_handle_for_stream "cuda_core::rt::create_event_handle_for_stream" ( cydriver.CUstream stream, unsigned int flags) except+ nogil - EventHandle create_event_handle_ref "cuda_core::create_event_handle_ref" ( + EventHandle create_event_handle_ref "cuda_core::rt::create_event_handle_ref" ( cydriver.CUevent event) except+ nogil - EventHandle create_event_handle_ipc "cuda_core::create_event_handle_ipc" ( + EventHandle create_event_handle_ipc "cuda_core::rt::create_event_handle_ipc" ( const cydriver.CUipcEventHandle& ipc_handle, bint is_blocking_sync) except+ nogil # Event metadata getters - bint get_event_timing_enabled "cuda_core::get_event_timing_enabled" ( + bint get_event_timing_enabled "cuda_core::rt::get_event_timing_enabled" ( const EventHandle& h) noexcept nogil - bint get_event_is_blocking_sync "cuda_core::get_event_is_blocking_sync" ( + bint get_event_is_blocking_sync "cuda_core::rt::get_event_is_blocking_sync" ( const EventHandle& h) noexcept nogil - bint get_event_ipc_enabled "cuda_core::get_event_ipc_enabled" ( + bint get_event_ipc_enabled "cuda_core::rt::get_event_ipc_enabled" ( const EventHandle& h) noexcept nogil - int get_event_device_id "cuda_core::get_event_device_id" ( + int get_event_device_id "cuda_core::rt::get_event_device_id" ( const EventHandle& h) noexcept nogil - ContextHandle get_event_context "cuda_core::get_event_context" ( + ContextHandle get_event_context "cuda_core::rt::get_event_context" ( const EventHandle& h) noexcept nogil # Memory pool handles - MemoryPoolHandle create_mempool_handle "cuda_core::create_mempool_handle" ( + MemoryPoolHandle create_mempool_handle "cuda_core::rt::create_mempool_handle" ( const cydriver.CUmemPoolProps& props) except+ nogil - MemoryPoolHandle create_mempool_handle_ref "cuda_core::create_mempool_handle_ref" ( + MemoryPoolHandle create_mempool_handle_ref "cuda_core::rt::create_mempool_handle_ref" ( cydriver.CUmemoryPool pool) except+ nogil - MemoryPoolHandle get_device_mempool "cuda_core::get_device_mempool" ( + MemoryPoolHandle get_device_mempool "cuda_core::rt::get_device_mempool" ( int device_id) except+ nogil - MemoryPoolHandle create_mempool_handle_ipc "cuda_core::create_mempool_handle_ipc" ( + MemoryPoolHandle create_mempool_handle_ipc "cuda_core::rt::create_mempool_handle_ipc" ( int fd, cydriver.CUmemAllocationHandleType handle_type) except+ nogil # Device pointer handles - DevicePtrHandle deviceptr_alloc_from_pool "cuda_core::deviceptr_alloc_from_pool" ( + DevicePtrHandle deviceptr_alloc_from_pool "cuda_core::rt::deviceptr_alloc_from_pool" ( size_t size, const MemoryPoolHandle& h_pool, const StreamHandle& h_stream) except+ nogil - DevicePtrHandle deviceptr_alloc_async "cuda_core::deviceptr_alloc_async" ( + DevicePtrHandle deviceptr_alloc_async "cuda_core::rt::deviceptr_alloc_async" ( size_t size, const StreamHandle& h_stream) except+ nogil - cydriver.CUresult deviceptr_alloc_raw "cuda_core::deviceptr_alloc_raw" ( + cydriver.CUresult deviceptr_alloc_raw "cuda_core::rt::deviceptr_alloc_raw" ( cydriver.CUdeviceptr* ptr, size_t size, const ContextHandle& h_context) noexcept nogil - DevicePtrHandle deviceptr_alloc_host "cuda_core::deviceptr_alloc_host" (size_t size) except+ nogil - DevicePtrHandle deviceptr_create_ref "cuda_core::deviceptr_create_ref" ( + DevicePtrHandle deviceptr_alloc_host "cuda_core::rt::deviceptr_alloc_host" (size_t size) except+ nogil + DevicePtrHandle deviceptr_create_ref "cuda_core::rt::deviceptr_create_ref" ( cydriver.CUdeviceptr ptr) except+ nogil - DevicePtrHandle deviceptr_create_with_owner "cuda_core::deviceptr_create_with_owner" ( + DevicePtrHandle deviceptr_create_with_owner "cuda_core::rt::deviceptr_create_with_owner" ( cydriver.CUdeviceptr ptr, object owner) except+ nogil - DevicePtrHandle deviceptr_create_mapped_graphics "cuda_core::deviceptr_create_mapped_graphics" ( + DevicePtrHandle deviceptr_create_mapped_graphics "cuda_core::rt::deviceptr_create_mapped_graphics" ( cydriver.CUdeviceptr ptr, const GraphicsResourceHandle& h_resource, const StreamHandle& h_stream) except+ nogil # MR deallocation callback - void register_mr_dealloc_callback "cuda_core::register_mr_dealloc_callback" ( + void register_mr_dealloc_callback "cuda_core::rt::register_mr_dealloc_callback" ( MRDeallocCallback cb) noexcept - DevicePtrHandle deviceptr_create_with_mr "cuda_core::deviceptr_create_with_mr" ( + DevicePtrHandle deviceptr_create_with_mr "cuda_core::rt::deviceptr_create_with_mr" ( cydriver.CUdeviceptr ptr, size_t size, object mr) except+ nogil - DevicePtrHandle deviceptr_import_ipc "cuda_core::deviceptr_import_ipc" ( + DevicePtrHandle deviceptr_import_ipc "cuda_core::rt::deviceptr_import_ipc" ( const MemoryPoolHandle& h_pool, const void* export_data, const StreamHandle& h_stream) except+ nogil - StreamHandle deallocation_stream "cuda_core::deallocation_stream" ( + StreamHandle deallocation_stream "cuda_core::rt::deallocation_stream" ( const DevicePtrHandle& h) noexcept nogil - cydriver.CUresult set_deallocation_stream "cuda_core::set_deallocation_stream" ( + cydriver.CUresult set_deallocation_stream "cuda_core::rt::set_deallocation_stream" ( const DevicePtrHandle& h, const StreamHandle& h_stream) noexcept nogil # Library handles - LibraryHandle create_library_handle_from_file "cuda_core::create_library_handle_from_file" ( + LibraryHandle create_library_handle_from_file "cuda_core::rt::create_library_handle_from_file" ( const char* path) except+ nogil - LibraryHandle create_library_handle_from_data "cuda_core::create_library_handle_from_data" ( + LibraryHandle create_library_handle_from_data "cuda_core::rt::create_library_handle_from_data" ( const void* data) except+ nogil - LibraryHandle create_library_handle_ref "cuda_core::create_library_handle_ref" ( + LibraryHandle create_library_handle_ref "cuda_core::rt::create_library_handle_ref" ( cydriver.CUlibrary library) except+ nogil # Kernel handles - KernelHandle create_kernel_handle "cuda_core::create_kernel_handle" ( + KernelHandle create_kernel_handle "cuda_core::rt::create_kernel_handle" ( const LibraryHandle& h_library, const char* name) except+ nogil - KernelHandle create_kernel_handle_ref "cuda_core::create_kernel_handle_ref" ( + KernelHandle create_kernel_handle_ref "cuda_core::rt::create_kernel_handle_ref" ( cydriver.CUkernel kernel) except+ nogil - LibraryHandle get_kernel_library "cuda_core::get_kernel_library" ( + LibraryHandle get_kernel_library "cuda_core::rt::get_kernel_library" ( const KernelHandle& h) noexcept nogil # Graph handles - GraphHandle create_graph_handle "cuda_core::create_graph_handle" ( + GraphHandle create_graph_handle "cuda_core::rt::create_graph_handle" ( cydriver.CUgraph graph) except+ nogil - GraphHandle create_child_graph_handle "cuda_core::create_child_graph_handle" ( + GraphHandle create_child_graph_handle "cuda_core::rt::create_child_graph_handle" ( cydriver.CUgraph child_graph, const GraphHandle& h_parent, cydriver.CUgraphNode owner_node) except+ nogil # Graph node attachments - OpaqueHandle make_opaque_py "cuda_core::make_opaque_py" (object obj) except+ - OpaqueHandle make_opaque_malloc "cuda_core::make_opaque_malloc" (void* buf) except+ - cydriver.CUresult graph_get_attachment "cuda_core::graph_get_attachment" ( + OpaqueHandle make_opaque_py "cuda_core::rt::make_opaque_py" (object obj) except+ + OpaqueHandle make_opaque_malloc "cuda_core::rt::make_opaque_malloc" (void* buf) except+ + cydriver.CUresult graph_get_attachment "cuda_core::rt::graph_get_attachment" ( const GraphHandle& h_graph, cydriver.CUgraphNode node, OpaqueHandle* owner0, OpaqueHandle* owner1) except+ - cydriver.CUresult graph_prepare_attachment "cuda_core::graph_prepare_attachment" ( + cydriver.CUresult graph_prepare_attachment "cuda_core::rt::graph_prepare_attachment" ( const GraphHandle& h_graph, OpaqueHandle owner0, OpaqueHandle owner1, PreparedAttachment* out_prepared) except+ - cydriver.CUresult graph_commit_attachment "cuda_core::graph_commit_attachment" ( + cydriver.CUresult graph_commit_attachment "cuda_core::rt::graph_commit_attachment" ( PreparedAttachment& prepared, cydriver.CUgraphNode node) except+ - cydriver.CUresult graph_clone_attachments "cuda_core::graph_clone_attachments" ( + cydriver.CUresult graph_clone_attachments "cuda_core::rt::graph_clone_attachments" ( const GraphHandle& h_clone, const GraphHandle& h_source) except+ - cydriver.CUresult graph_prepare_child_graph_update "cuda_core::graph_prepare_child_graph_update" ( + cydriver.CUresult graph_prepare_child_graph_update "cuda_core::rt::graph_prepare_child_graph_update" ( const GraphHandle& h_parent, const GraphHandle& h_old_child, cydriver.CUgraphNode owner_node, const GraphHandle& h_source, PreparedChildGraphUpdate* out_prepared) except+ - cydriver.CUresult graph_commit_child_graph_update "cuda_core::graph_commit_child_graph_update" ( + cydriver.CUresult graph_commit_child_graph_update "cuda_core::rt::graph_commit_child_graph_update" ( PreparedChildGraphUpdate& prepared, GraphHandle* out_child) except+ - void invalidate_child_graph_state "cuda_core::invalidate_child_graph_state" ( + void invalidate_child_graph_state "cuda_core::rt::invalidate_child_graph_state" ( const GraphHandle& h_parent, cydriver.CUgraphNode owner_node) noexcept # Graph exec handles - GraphExecHandle create_graph_exec_handle "cuda_core::create_graph_exec_handle" ( + GraphExecHandle create_graph_exec_handle "cuda_core::rt::create_graph_exec_handle" ( const GraphHandle& h_source, cydriver.CUDA_GRAPH_INSTANTIATE_PARAMS* params) except+ - cydriver.CUresult graph_exec_update "cuda_core::graph_exec_update" ( + cydriver.CUresult graph_exec_update "cuda_core::rt::graph_exec_update" ( const GraphExecHandle& h_exec, const GraphHandle& h_source, cydriver.CUgraphExecUpdateResultInfo* result_info) except+ - cydriver.CUresult graph_prepare_exec_attachment "cuda_core::graph_prepare_exec_attachment" ( + cydriver.CUresult graph_prepare_exec_attachment "cuda_core::rt::graph_prepare_exec_attachment" ( const GraphExecHandle& h_exec, OpaqueHandle owner0, OpaqueHandle owner1, PreparedExecAttachment* out_prepared) except+ - void graph_commit_exec_attachment "cuda_core::graph_commit_exec_attachment" ( + void graph_commit_exec_attachment "cuda_core::rt::graph_commit_exec_attachment" ( PreparedExecAttachment& prepared) noexcept # Graph node handles - GraphNodeHandle create_graph_node_handle "cuda_core::create_graph_node_handle" ( + GraphNodeHandle create_graph_node_handle "cuda_core::rt::create_graph_node_handle" ( cydriver.CUgraphNode node, const GraphHandle& h_graph) except+ nogil - GraphHandle graph_node_get_graph "cuda_core::graph_node_get_graph" ( + GraphHandle graph_node_get_graph "cuda_core::rt::graph_node_get_graph" ( const GraphNodeHandle& h) noexcept nogil - void invalidate_graph_node "cuda_core::invalidate_graph_node" ( + void invalidate_graph_node "cuda_core::rt::invalidate_graph_node" ( const GraphNodeHandle& h) noexcept nogil # Graphics resource handles - GraphicsResourceHandle create_graphics_resource_handle "cuda_core::create_graphics_resource_handle" ( + GraphicsResourceHandle create_graphics_resource_handle "cuda_core::rt::create_graphics_resource_handle" ( cydriver.CUgraphicsResource resource) except+ nogil # NVRTC Program handles - NvrtcProgramHandle create_nvrtc_program_handle "cuda_core::create_nvrtc_program_handle" ( + NvrtcProgramHandle create_nvrtc_program_handle "cuda_core::rt::create_nvrtc_program_handle" ( cynvrtc.nvrtcProgram prog) except+ nogil - NvrtcProgramHandle create_nvrtc_program_handle_ref "cuda_core::create_nvrtc_program_handle_ref" ( + NvrtcProgramHandle create_nvrtc_program_handle_ref "cuda_core::rt::create_nvrtc_program_handle_ref" ( cynvrtc.nvrtcProgram prog) except+ nogil # NVVM Program handles - NvvmProgramHandle create_nvvm_program_handle "cuda_core::create_nvvm_program_handle" ( + NvvmProgramHandle create_nvvm_program_handle "cuda_core::rt::create_nvvm_program_handle" ( cynvvm.nvvmProgram prog) except+ nogil - NvvmProgramHandle create_nvvm_program_handle_ref "cuda_core::create_nvvm_program_handle_ref" ( + NvvmProgramHandle create_nvvm_program_handle_ref "cuda_core::rt::create_nvvm_program_handle_ref" ( cynvvm.nvvmProgram prog) except+ nogil # nvJitLink handles - NvJitLinkHandle create_nvjitlink_handle "cuda_core::create_nvjitlink_handle" ( + NvJitLinkHandle create_nvjitlink_handle "cuda_core::rt::create_nvjitlink_handle" ( cynvjitlink.nvJitLinkHandle handle) except+ nogil - NvJitLinkHandle create_nvjitlink_handle_ref "cuda_core::create_nvjitlink_handle_ref" ( + NvJitLinkHandle create_nvjitlink_handle_ref "cuda_core::rt::create_nvjitlink_handle_ref" ( cynvjitlink.nvJitLinkHandle handle) except+ nogil # cuLink handles - CuLinkHandle create_culink_handle "cuda_core::create_culink_handle" ( + CuLinkHandle create_culink_handle "cuda_core::rt::create_culink_handle" ( cydriver.CUlinkState state) except+ nogil - CuLinkHandle create_culink_handle_ref "cuda_core::create_culink_handle_ref" ( + CuLinkHandle create_culink_handle_ref "cuda_core::rt::create_culink_handle_ref" ( cydriver.CUlinkState state) except+ nogil # File descriptor handles - FileDescriptorHandle create_fd_handle "cuda_core::create_fd_handle" ( + FileDescriptorHandle create_fd_handle "cuda_core::rt::create_fd_handle" ( int fd) except+ nogil - FileDescriptorHandle create_fd_handle_ref "cuda_core::create_fd_handle_ref" ( + FileDescriptorHandle create_fd_handle_ref "cuda_core::rt::create_fd_handle_ref" ( int fd) except+ nogil # SM resource split (13.1+ wrapper — avoids direct cydriver cimport) # groupParams is void* to avoid referencing CU_DEV_SM_RESOURCE_GROUP_PARAMS # (which doesn't exist in cuda-bindings 13.0 .pxd). The C++ side casts it. - cydriver.CUresult sm_resource_split "cuda_core::sm_resource_split" ( + cydriver.CUresult sm_resource_split "cuda_core::rt::sm_resource_split" ( cydriver.CUdevResource* result, unsigned int nbGroups, const cydriver.CUdevResource* input, cydriver.CUdevResource* remainder, unsigned int flags, void* groupParams) nogil - bint has_sm_resource_split "cuda_core::has_sm_resource_split" () noexcept nogil + bint has_sm_resource_split "cuda_core::rt::has_sm_resource_split" () noexcept nogil # cuMemcpyWithAttributesAsync (13.2+ wrapper — avoids direct cydriver cimport) # attr is void* to avoid referencing CUmemcpyAttributes (absent from # cuda-bindings built against CUDA < 12.8). The C++ side casts it. - cydriver.CUresult memcpy_with_attributes_async "cuda_core::memcpy_with_attributes_async" ( + cydriver.CUresult memcpy_with_attributes_async "cuda_core::rt::memcpy_with_attributes_async" ( cydriver.CUdeviceptr dst, cydriver.CUdeviceptr src, size_t size, void* attr, cydriver.CUstream hStream) nogil - bint has_memcpy_with_attributes_async "cuda_core::has_memcpy_with_attributes_async" () noexcept nogil + bint has_memcpy_with_attributes_async "cuda_core::rt::has_memcpy_with_attributes_async" () noexcept nogil # Array / mipmapped-array / texture / surface handles (PR #467) - OpaqueArrayHandle create_array_handle "cuda_core::create_array_handle" ( + OpaqueArrayHandle create_array_handle "cuda_core::rt::create_array_handle" ( const ContextHandle& h_context, const cydriver.CUDA_ARRAY3D_DESCRIPTOR& desc) except+ nogil - OpaqueArrayHandle create_array_handle_ref "cuda_core::create_array_handle_ref" ( + OpaqueArrayHandle create_array_handle_ref "cuda_core::rt::create_array_handle_ref" ( cydriver.CUarray arr) except+ nogil - OpaqueArrayHandle create_array_handle_owning "cuda_core::create_array_handle_owning" ( + OpaqueArrayHandle create_array_handle_owning "cuda_core::rt::create_array_handle_owning" ( cydriver.CUarray arr) except+ nogil - ContextHandle get_array_context "cuda_core::get_array_context" ( + ContextHandle get_array_context "cuda_core::rt::get_array_context" ( const OpaqueArrayHandle& h) noexcept nogil - OpaqueArrayHandle create_array_level_handle "cuda_core::create_array_level_handle" ( + OpaqueArrayHandle create_array_level_handle "cuda_core::rt::create_array_level_handle" ( const MipmappedArrayHandle& h_mip, unsigned int level) except+ nogil - MipmappedArrayHandle create_mipmapped_array_handle "cuda_core::create_mipmapped_array_handle" ( + MipmappedArrayHandle create_mipmapped_array_handle "cuda_core::rt::create_mipmapped_array_handle" ( const ContextHandle& h_context, const cydriver.CUDA_ARRAY3D_DESCRIPTOR& desc, unsigned int num_levels) except+ nogil - ContextHandle get_mipmapped_array_context "cuda_core::get_mipmapped_array_context" ( + ContextHandle get_mipmapped_array_context "cuda_core::rt::get_mipmapped_array_context" ( const MipmappedArrayHandle& h) noexcept nogil - TexObjectHandle create_tex_object_handle_array "cuda_core::create_tex_object_handle_array" ( + TexObjectHandle create_tex_object_handle_array "cuda_core::rt::create_tex_object_handle_array" ( const ContextHandle& h_context, const cydriver.CUDA_RESOURCE_DESC& res, const cydriver.CUDA_TEXTURE_DESC& tex, const OpaqueArrayHandle& h_backing) except+ nogil - TexObjectHandle create_tex_object_handle_mipmap "cuda_core::create_tex_object_handle_mipmap" ( + TexObjectHandle create_tex_object_handle_mipmap "cuda_core::rt::create_tex_object_handle_mipmap" ( const ContextHandle& h_context, const cydriver.CUDA_RESOURCE_DESC& res, const cydriver.CUDA_TEXTURE_DESC& tex, const MipmappedArrayHandle& h_backing) except+ nogil - TexObjectHandle create_tex_object_handle_linear "cuda_core::create_tex_object_handle_linear" ( + TexObjectHandle create_tex_object_handle_linear "cuda_core::rt::create_tex_object_handle_linear" ( const ContextHandle& h_context, const cydriver.CUDA_RESOURCE_DESC& res, const cydriver.CUDA_TEXTURE_DESC& tex, const DevicePtrHandle& h_backing) except+ nogil - SurfObjectHandle create_surf_object_handle "cuda_core::create_surf_object_handle" ( + SurfObjectHandle create_surf_object_handle "cuda_core::rt::create_surf_object_handle" ( const ContextHandle& h_context, const cydriver.CUDA_RESOURCE_DESC& res, const OpaqueArrayHandle& h_backing) except+ nogil -# ============================================================================= -# CUDA Driver API capsule -# -# This provides resolved CUDA driver function pointers to the C++ code. -# ============================================================================= - -cdef const char* _CUDA_DRIVER_API_V1_NAME = b"cuda.core._resource_handles._CUDA_DRIVER_API_V1" - - # ============================================================================= # CUDA driver function pointer initialization # @@ -337,110 +332,110 @@ cdef const char* _CUDA_DRIVER_API_V1_NAME = b"cuda.core._resource_handles._CUDA_ # ============================================================================= # Declare extern variables with reinterpret_cast to allow void* assignment -cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": +cdef extern from "_cpp/rt/rt.hpp" namespace "cuda_core::rt": # Error formatting - void* p_cuGetErrorName "reinterpret_cast(cuda_core::p_cuGetErrorName)" - void* p_cuGetErrorString "reinterpret_cast(cuda_core::p_cuGetErrorString)" + void* p_cuGetErrorName "reinterpret_cast(cuda_core::rt::p_cuGetErrorName)" + void* p_cuGetErrorString "reinterpret_cast(cuda_core::rt::p_cuGetErrorString)" # Context - void* p_cuDevicePrimaryCtxRetain "reinterpret_cast(cuda_core::p_cuDevicePrimaryCtxRetain)" - void* p_cuDevicePrimaryCtxRelease "reinterpret_cast(cuda_core::p_cuDevicePrimaryCtxRelease)" - void* p_cuCtxGetCurrent "reinterpret_cast(cuda_core::p_cuCtxGetCurrent)" - void* p_cuCtxSetCurrent "reinterpret_cast(cuda_core::p_cuCtxSetCurrent)" - void* p_cuCtxSynchronize "reinterpret_cast(cuda_core::p_cuCtxSynchronize)" - void* p_cuCtxGetStreamPriorityRange "reinterpret_cast(cuda_core::p_cuCtxGetStreamPriorityRange)" - void* p_cuCtxGetDevice "reinterpret_cast(cuda_core::p_cuCtxGetDevice)" - void* p_cuGraphNodeSetParams "reinterpret_cast(cuda_core::p_cuGraphNodeSetParams)" - void* p_cuGreenCtxCreate "reinterpret_cast(cuda_core::p_cuGreenCtxCreate)" - void* p_cuGreenCtxDestroy "reinterpret_cast(cuda_core::p_cuGreenCtxDestroy)" - void* p_cuCtxFromGreenCtx "reinterpret_cast(cuda_core::p_cuCtxFromGreenCtx)" - void* p_cuDevResourceGenerateDesc "reinterpret_cast(cuda_core::p_cuDevResourceGenerateDesc)" - void* p_cuGreenCtxStreamCreate "reinterpret_cast(cuda_core::p_cuGreenCtxStreamCreate)" + void* p_cuDevicePrimaryCtxRetain "reinterpret_cast(cuda_core::rt::p_cuDevicePrimaryCtxRetain)" + void* p_cuDevicePrimaryCtxRelease "reinterpret_cast(cuda_core::rt::p_cuDevicePrimaryCtxRelease)" + void* p_cuCtxGetCurrent "reinterpret_cast(cuda_core::rt::p_cuCtxGetCurrent)" + void* p_cuCtxSetCurrent "reinterpret_cast(cuda_core::rt::p_cuCtxSetCurrent)" + void* p_cuCtxSynchronize "reinterpret_cast(cuda_core::rt::p_cuCtxSynchronize)" + void* p_cuCtxGetStreamPriorityRange "reinterpret_cast(cuda_core::rt::p_cuCtxGetStreamPriorityRange)" + void* p_cuCtxGetDevice "reinterpret_cast(cuda_core::rt::p_cuCtxGetDevice)" + void* p_cuGraphNodeSetParams "reinterpret_cast(cuda_core::rt::p_cuGraphNodeSetParams)" + void* p_cuGreenCtxCreate "reinterpret_cast(cuda_core::rt::p_cuGreenCtxCreate)" + void* p_cuGreenCtxDestroy "reinterpret_cast(cuda_core::rt::p_cuGreenCtxDestroy)" + void* p_cuCtxFromGreenCtx "reinterpret_cast(cuda_core::rt::p_cuCtxFromGreenCtx)" + void* p_cuDevResourceGenerateDesc "reinterpret_cast(cuda_core::rt::p_cuDevResourceGenerateDesc)" + void* p_cuGreenCtxStreamCreate "reinterpret_cast(cuda_core::rt::p_cuGreenCtxStreamCreate)" # Stream - void* p_cuStreamCreateWithPriority "reinterpret_cast(cuda_core::p_cuStreamCreateWithPriority)" - void* p_cuStreamDestroy "reinterpret_cast(cuda_core::p_cuStreamDestroy)" - void* p_cuStreamGetCtx "reinterpret_cast(cuda_core::p_cuStreamGetCtx)" + void* p_cuStreamCreateWithPriority "reinterpret_cast(cuda_core::rt::p_cuStreamCreateWithPriority)" + void* p_cuStreamDestroy "reinterpret_cast(cuda_core::rt::p_cuStreamDestroy)" + void* p_cuStreamGetCtx "reinterpret_cast(cuda_core::rt::p_cuStreamGetCtx)" # Event - void* p_cuEventCreate "reinterpret_cast(cuda_core::p_cuEventCreate)" - void* p_cuEventDestroy "reinterpret_cast(cuda_core::p_cuEventDestroy)" - void* p_cuIpcOpenEventHandle "reinterpret_cast(cuda_core::p_cuIpcOpenEventHandle)" + void* p_cuEventCreate "reinterpret_cast(cuda_core::rt::p_cuEventCreate)" + void* p_cuEventDestroy "reinterpret_cast(cuda_core::rt::p_cuEventDestroy)" + void* p_cuIpcOpenEventHandle "reinterpret_cast(cuda_core::rt::p_cuIpcOpenEventHandle)" # Device - void* p_cuDeviceGetCount "reinterpret_cast(cuda_core::p_cuDeviceGetCount)" + void* p_cuDeviceGetCount "reinterpret_cast(cuda_core::rt::p_cuDeviceGetCount)" # Memory pool - void* p_cuMemPoolSetAccess "reinterpret_cast(cuda_core::p_cuMemPoolSetAccess)" - void* p_cuMemPoolDestroy "reinterpret_cast(cuda_core::p_cuMemPoolDestroy)" - void* p_cuMemPoolCreate "reinterpret_cast(cuda_core::p_cuMemPoolCreate)" - void* p_cuDeviceGetMemPool "reinterpret_cast(cuda_core::p_cuDeviceGetMemPool)" - void* p_cuMemPoolImportFromShareableHandle "reinterpret_cast(cuda_core::p_cuMemPoolImportFromShareableHandle)" + void* p_cuMemPoolSetAccess "reinterpret_cast(cuda_core::rt::p_cuMemPoolSetAccess)" + void* p_cuMemPoolDestroy "reinterpret_cast(cuda_core::rt::p_cuMemPoolDestroy)" + void* p_cuMemPoolCreate "reinterpret_cast(cuda_core::rt::p_cuMemPoolCreate)" + void* p_cuDeviceGetMemPool "reinterpret_cast(cuda_core::rt::p_cuDeviceGetMemPool)" + void* p_cuMemPoolImportFromShareableHandle "reinterpret_cast(cuda_core::rt::p_cuMemPoolImportFromShareableHandle)" # Memory allocation - void* p_cuMemAllocFromPoolAsync "reinterpret_cast(cuda_core::p_cuMemAllocFromPoolAsync)" - void* p_cuMemAllocAsync "reinterpret_cast(cuda_core::p_cuMemAllocAsync)" - void* p_cuMemAlloc "reinterpret_cast(cuda_core::p_cuMemAlloc)" - void* p_cuMemAllocHost "reinterpret_cast(cuda_core::p_cuMemAllocHost)" + void* p_cuMemAllocFromPoolAsync "reinterpret_cast(cuda_core::rt::p_cuMemAllocFromPoolAsync)" + void* p_cuMemAllocAsync "reinterpret_cast(cuda_core::rt::p_cuMemAllocAsync)" + void* p_cuMemAlloc "reinterpret_cast(cuda_core::rt::p_cuMemAlloc)" + void* p_cuMemAllocHost "reinterpret_cast(cuda_core::rt::p_cuMemAllocHost)" # Memory deallocation - void* p_cuMemFreeAsync "reinterpret_cast(cuda_core::p_cuMemFreeAsync)" - void* p_cuMemFree "reinterpret_cast(cuda_core::p_cuMemFree)" - void* p_cuMemFreeHost "reinterpret_cast(cuda_core::p_cuMemFreeHost)" + void* p_cuMemFreeAsync "reinterpret_cast(cuda_core::rt::p_cuMemFreeAsync)" + void* p_cuMemFree "reinterpret_cast(cuda_core::rt::p_cuMemFree)" + void* p_cuMemFreeHost "reinterpret_cast(cuda_core::rt::p_cuMemFreeHost)" # IPC - void* p_cuMemPoolImportPointer "reinterpret_cast(cuda_core::p_cuMemPoolImportPointer)" + void* p_cuMemPoolImportPointer "reinterpret_cast(cuda_core::rt::p_cuMemPoolImportPointer)" # Library - void* p_cuLibraryLoadFromFile "reinterpret_cast(cuda_core::p_cuLibraryLoadFromFile)" - void* p_cuLibraryLoadData "reinterpret_cast(cuda_core::p_cuLibraryLoadData)" - void* p_cuLibraryUnload "reinterpret_cast(cuda_core::p_cuLibraryUnload)" - void* p_cuLibraryGetKernel "reinterpret_cast(cuda_core::p_cuLibraryGetKernel)" + void* p_cuLibraryLoadFromFile "reinterpret_cast(cuda_core::rt::p_cuLibraryLoadFromFile)" + void* p_cuLibraryLoadData "reinterpret_cast(cuda_core::rt::p_cuLibraryLoadData)" + void* p_cuLibraryUnload "reinterpret_cast(cuda_core::rt::p_cuLibraryUnload)" + void* p_cuLibraryGetKernel "reinterpret_cast(cuda_core::rt::p_cuLibraryGetKernel)" # Graph - void* p_cuGraphDestroy "reinterpret_cast(cuda_core::p_cuGraphDestroy)" - void* p_cuGraphInstantiateWithParams "reinterpret_cast(cuda_core::p_cuGraphInstantiateWithParams)" - void* p_cuGraphExecUpdate "reinterpret_cast(cuda_core::p_cuGraphExecUpdate)" - void* p_cuGraphExecDestroy "reinterpret_cast(cuda_core::p_cuGraphExecDestroy)" - void* p_cuUserObjectCreate "reinterpret_cast(cuda_core::p_cuUserObjectCreate)" - void* p_cuUserObjectRelease "reinterpret_cast(cuda_core::p_cuUserObjectRelease)" - void* p_cuGraphRetainUserObject "reinterpret_cast(cuda_core::p_cuGraphRetainUserObject)" - void* p_cuGraphReleaseUserObject "reinterpret_cast(cuda_core::p_cuGraphReleaseUserObject)" - void* p_cuGraphNodeFindInClone "reinterpret_cast(cuda_core::p_cuGraphNodeFindInClone)" - void* p_cuGraphChildGraphNodeGetGraph "reinterpret_cast(cuda_core::p_cuGraphChildGraphNodeGetGraph)" + void* p_cuGraphDestroy "reinterpret_cast(cuda_core::rt::p_cuGraphDestroy)" + void* p_cuGraphInstantiateWithParams "reinterpret_cast(cuda_core::rt::p_cuGraphInstantiateWithParams)" + void* p_cuGraphExecUpdate "reinterpret_cast(cuda_core::rt::p_cuGraphExecUpdate)" + void* p_cuGraphExecDestroy "reinterpret_cast(cuda_core::rt::p_cuGraphExecDestroy)" + void* p_cuUserObjectCreate "reinterpret_cast(cuda_core::rt::p_cuUserObjectCreate)" + void* p_cuUserObjectRelease "reinterpret_cast(cuda_core::rt::p_cuUserObjectRelease)" + void* p_cuGraphRetainUserObject "reinterpret_cast(cuda_core::rt::p_cuGraphRetainUserObject)" + void* p_cuGraphReleaseUserObject "reinterpret_cast(cuda_core::rt::p_cuGraphReleaseUserObject)" + void* p_cuGraphNodeFindInClone "reinterpret_cast(cuda_core::rt::p_cuGraphNodeFindInClone)" + void* p_cuGraphChildGraphNodeGetGraph "reinterpret_cast(cuda_core::rt::p_cuGraphChildGraphNodeGetGraph)" # Linker - void* p_cuLinkDestroy "reinterpret_cast(cuda_core::p_cuLinkDestroy)" + void* p_cuLinkDestroy "reinterpret_cast(cuda_core::rt::p_cuLinkDestroy)" # Graphics interop - void* p_cuGraphicsUnmapResources "reinterpret_cast(cuda_core::p_cuGraphicsUnmapResources)" - void* p_cuGraphicsUnregisterResource "reinterpret_cast(cuda_core::p_cuGraphicsUnregisterResource)" + void* p_cuGraphicsUnmapResources "reinterpret_cast(cuda_core::rt::p_cuGraphicsUnmapResources)" + void* p_cuGraphicsUnregisterResource "reinterpret_cast(cuda_core::rt::p_cuGraphicsUnregisterResource)" # Texture / surface / array (PR #467) - void* p_cuArray3DCreate "reinterpret_cast(cuda_core::p_cuArray3DCreate)" - void* p_cuArrayDestroy "reinterpret_cast(cuda_core::p_cuArrayDestroy)" - void* p_cuMipmappedArrayCreate "reinterpret_cast(cuda_core::p_cuMipmappedArrayCreate)" - void* p_cuMipmappedArrayDestroy "reinterpret_cast(cuda_core::p_cuMipmappedArrayDestroy)" - void* p_cuMipmappedArrayGetLevel "reinterpret_cast(cuda_core::p_cuMipmappedArrayGetLevel)" - void* p_cuTexObjectCreate "reinterpret_cast(cuda_core::p_cuTexObjectCreate)" - void* p_cuTexObjectDestroy "reinterpret_cast(cuda_core::p_cuTexObjectDestroy)" - void* p_cuSurfObjectCreate "reinterpret_cast(cuda_core::p_cuSurfObjectCreate)" - void* p_cuSurfObjectDestroy "reinterpret_cast(cuda_core::p_cuSurfObjectDestroy)" + void* p_cuArray3DCreate "reinterpret_cast(cuda_core::rt::p_cuArray3DCreate)" + void* p_cuArrayDestroy "reinterpret_cast(cuda_core::rt::p_cuArrayDestroy)" + void* p_cuMipmappedArrayCreate "reinterpret_cast(cuda_core::rt::p_cuMipmappedArrayCreate)" + void* p_cuMipmappedArrayDestroy "reinterpret_cast(cuda_core::rt::p_cuMipmappedArrayDestroy)" + void* p_cuMipmappedArrayGetLevel "reinterpret_cast(cuda_core::rt::p_cuMipmappedArrayGetLevel)" + void* p_cuTexObjectCreate "reinterpret_cast(cuda_core::rt::p_cuTexObjectCreate)" + void* p_cuTexObjectDestroy "reinterpret_cast(cuda_core::rt::p_cuTexObjectDestroy)" + void* p_cuSurfObjectCreate "reinterpret_cast(cuda_core::rt::p_cuSurfObjectCreate)" + void* p_cuSurfObjectDestroy "reinterpret_cast(cuda_core::rt::p_cuSurfObjectDestroy)" # SM resource split (13.1+) - void* p_cuDevSmResourceSplit "reinterpret_cast(cuda_core::p_cuDevSmResourceSplit)" + void* p_cuDevSmResourceSplit "reinterpret_cast(cuda_core::rt::p_cuDevSmResourceSplit)" # cuMemcpyWithAttributesAsync (13.2+) - void* p_cuMemcpyWithAttributesAsync "reinterpret_cast(cuda_core::p_cuMemcpyWithAttributesAsync)" + void* p_cuMemcpyWithAttributesAsync "reinterpret_cast(cuda_core::rt::p_cuMemcpyWithAttributesAsync)" # NVRTC - void* p_nvrtcDestroyProgram "reinterpret_cast(cuda_core::p_nvrtcDestroyProgram)" + void* p_nvrtcDestroyProgram "reinterpret_cast(cuda_core::rt::p_nvrtcDestroyProgram)" # NVVM - void* p_nvvmDestroyProgram "reinterpret_cast(cuda_core::p_nvvmDestroyProgram)" + void* p_nvvmDestroyProgram "reinterpret_cast(cuda_core::rt::p_nvvmDestroyProgram)" # nvJitLink - void* p_nvJitLinkDestroy "reinterpret_cast(cuda_core::p_nvJitLinkDestroy)" + void* p_nvJitLinkDestroy "reinterpret_cast(cuda_core::rt::p_nvJitLinkDestroy)" # Initialize driver function pointers from cydriver.__pyx_capi__ at module load diff --git a/cuda_core/cuda/core/_stream.pxd b/cuda_core/cuda/core/_stream.pxd index 5de11a36761..9ae01151785 100644 --- a/cuda_core/cuda/core/_stream.pxd +++ b/cuda_core/cuda/core/_stream.pxd @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -from cuda.core._resource_handles cimport ContextHandle, StreamHandle +from cuda.core._rt cimport ContextHandle, StreamHandle cdef class Stream: diff --git a/cuda_core/cuda/core/_stream.pyx b/cuda_core/cuda/core/_stream.pyx index 916a6eb01fe..4c51b32f4f0 100644 --- a/cuda_core/cuda/core/_stream.pyx +++ b/cuda_core/cuda/core/_stream.pyx @@ -27,8 +27,8 @@ from cuda.core._context cimport ( from cuda.core._device_resources cimport DeviceResources from cuda.core._event import Event, EventOptions -from cuda.core._resource_handles cimport context_get_device -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport context_get_device +from cuda.core._rt cimport ( ContextHandle, EventHandle, StreamHandle, diff --git a/cuda_core/cuda/core/_tensor_bridge.pyx b/cuda_core/cuda/core/_tensor_bridge.pyx index ae7a6794507..c7c3cd5c718 100644 --- a/cuda_core/cuda/core/_tensor_bridge.pyx +++ b/cuda_core/cuda/core/_tensor_bridge.pyx @@ -54,7 +54,7 @@ from libc.stdint cimport intptr_t, int8_t, int16_t, int32_t, int64_t, uint8_t from cuda.core._memoryview cimport StridedMemoryView from cuda.core._layout cimport _StridedLayout from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( EventHandle, create_event_handle_for_stream, as_cu, diff --git a/cuda_core/cuda/core/_utils/_weak_handles.pyx b/cuda_core/cuda/core/_utils/_weak_handles.pyx index d9f71e36772..c0c456ed728 100644 --- a/cuda_core/cuda/core/_utils/_weak_handles.pyx +++ b/cuda_core/cuda/core/_utils/_weak_handles.pyx @@ -24,7 +24,7 @@ Python owners via ``make_opaque_py`` are not covered here -- use from cuda.core._memory._buffer cimport Buffer from cuda.core.graph._graph_definition cimport GraphDefinition -from cuda.core._resource_handles cimport OpaqueHandle +from cuda.core._rt cimport OpaqueHandle # Cython cannot spell ``weak_ptr[const void]`` inline (the ``const void`` diff --git a/cuda_core/cuda/core/_utils/cuda_utils.pyx b/cuda_core/cuda/core/_utils/cuda_utils.pyx index b6f33112953..7c9c352fb72 100644 --- a/cuda_core/cuda/core/_utils/cuda_utils.pyx +++ b/cuda_core/cuda/core/_utils/cuda_utils.pyx @@ -27,7 +27,7 @@ from cuda.bindings.nvjitlink import nvJitLinkError from cpython.buffer cimport PyObject_GetBuffer, PyBuffer_Release, Py_buffer, PyBUF_SIMPLE from cuda.bindings cimport cynvrtc, cynvvm, cynvjitlink -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( register_warning_category, take_last_error_detail, ) diff --git a/cuda_core/cuda/core/graph/_adjacency_set_proxy.pyx b/cuda_core/cuda/core/graph/_adjacency_set_proxy.pyx index 8919067d0b0..2c2c67e5005 100644 --- a/cuda_core/cuda/core/graph/_adjacency_set_proxy.pyx +++ b/cuda_core/cuda/core/graph/_adjacency_set_proxy.pyx @@ -8,7 +8,7 @@ from libc.stddef cimport size_t from libcpp.vector cimport vector from cuda.bindings cimport cydriver from cuda.core.graph._graph_node cimport GraphNode, GN_check_valid -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( GraphHandle, GraphNodeHandle, as_cu, diff --git a/cuda_core/cuda/core/graph/_graph_builder.pxd b/cuda_core/cuda/core/graph/_graph_builder.pxd index eb75e6bd44a..f61701a5970 100644 --- a/cuda_core/cuda/core/graph/_graph_builder.pxd +++ b/cuda_core/cuda/core/graph/_graph_builder.pxd @@ -4,7 +4,7 @@ from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport GraphExecHandle, GraphHandle, StreamHandle +from cuda.core._rt cimport GraphExecHandle, GraphHandle, StreamHandle from cuda.core._stream cimport Stream diff --git a/cuda_core/cuda/core/graph/_graph_builder.pyx b/cuda_core/cuda/core/graph/_graph_builder.pyx index 98faf09eec3..941cd989958 100644 --- a/cuda_core/cuda/core/graph/_graph_builder.pyx +++ b/cuda_core/cuda/core/graph/_graph_builder.pyx @@ -20,8 +20,8 @@ from cuda.core.graph._subclasses cimport ( ExecutableGraphNode, create_executable_node_view, ) -from cuda.core._resource_handles cimport note_or_report_cuda_error, report_cuda_error -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport note_or_report_cuda_error, report_cuda_error +from cuda.core._rt cimport ( GraphExecHandle, GraphHandle, OpaqueHandle, diff --git a/cuda_core/cuda/core/graph/_graph_definition.pxd b/cuda_core/cuda/core/graph/_graph_definition.pxd index 634c2ba2580..ea64be805f8 100644 --- a/cuda_core/cuda/core/graph/_graph_definition.pxd +++ b/cuda_core/cuda/core/graph/_graph_definition.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport GraphHandle, as_intptr +from cuda.core._rt cimport GraphHandle, as_intptr cdef class GraphCondition: diff --git a/cuda_core/cuda/core/graph/_graph_definition.pyx b/cuda_core/cuda/core/graph/_graph_definition.pyx index c1a3999bc08..1c02255ba7c 100644 --- a/cuda_core/cuda/core/graph/_graph_definition.pyx +++ b/cuda_core/cuda/core/graph/_graph_definition.pyx @@ -14,7 +14,7 @@ from libcpp.vector cimport vector from cuda.bindings cimport cydriver from cuda.core.graph._graph_node cimport GraphNode -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( GraphHandle, as_cu, as_intptr, diff --git a/cuda_core/cuda/core/graph/_graph_node.pxd b/cuda_core/cuda/core/graph/_graph_node.pxd index ef7d1ff0643..b34885fde68 100644 --- a/cuda_core/cuda/core/graph/_graph_node.pxd +++ b/cuda_core/cuda/core/graph/_graph_node.pxd @@ -5,7 +5,7 @@ from libc.stddef cimport size_t from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( GraphHandle, GraphNodeHandle, OpaqueHandle, diff --git a/cuda_core/cuda/core/graph/_graph_node.pyx b/cuda_core/cuda/core/graph/_graph_node.pyx index d072971615d..085dd616920 100644 --- a/cuda_core/cuda/core/graph/_graph_node.pyx +++ b/cuda_core/cuda/core/graph/_graph_node.pyx @@ -45,8 +45,8 @@ from cuda.core.graph._subclasses cimport ( SwitchNode, WhileNode, ) -from cuda.core._resource_handles cimport note_or_report_cuda_error -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport note_or_report_cuda_error +from cuda.core._rt cimport ( GraphHandle, GraphNodeHandle, OpaqueHandle, diff --git a/cuda_core/cuda/core/graph/_host_callback.pxd b/cuda_core/cuda/core/graph/_host_callback.pxd index fc77809c846..1724c6fcc2c 100644 --- a/cuda_core/cuda/core/graph/_host_callback.pxd +++ b/cuda_core/cuda/core/graph/_host_callback.pxd @@ -4,7 +4,7 @@ from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport OpaqueHandle +from cuda.core._rt cimport OpaqueHandle cdef bint _is_py_host_trampoline(cydriver.CUhostFn fn) noexcept nogil diff --git a/cuda_core/cuda/core/graph/_host_callback.pyx b/cuda_core/cuda/core/graph/_host_callback.pyx index 4fb48f0d6ec..de974ab257f 100644 --- a/cuda_core/cuda/core/graph/_host_callback.pyx +++ b/cuda_core/cuda/core/graph/_host_callback.pyx @@ -8,7 +8,7 @@ from libc.string cimport memcpy as c_memcpy from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( OpaqueHandle, make_opaque_malloc, make_opaque_py, diff --git a/cuda_core/cuda/core/graph/_subclasses.pxd b/cuda_core/cuda/core/graph/_subclasses.pxd index 7f92eafe7e8..c85f5e3f201 100644 --- a/cuda_core/cuda/core/graph/_subclasses.pxd +++ b/cuda_core/cuda/core/graph/_subclasses.pxd @@ -7,7 +7,7 @@ from libc.stddef cimport size_t from cuda.bindings cimport cydriver from cuda.core.graph._graph_definition cimport GraphCondition from cuda.core.graph._graph_node cimport GraphNode -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( EventHandle, GraphExecHandle, GraphHandle, diff --git a/cuda_core/cuda/core/graph/_subclasses.pyx b/cuda_core/cuda/core/graph/_subclasses.pyx index 4967983d947..0f77cabfb39 100644 --- a/cuda_core/cuda/core/graph/_subclasses.pyx +++ b/cuda_core/cuda/core/graph/_subclasses.pyx @@ -29,12 +29,12 @@ from cuda.core.graph._graph_node cimport ( _init_memcpy_params, _resolve_memcpy_operand, ) -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( ContextHandle, create_context_handle_ref, graph_node_set_params, ) -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( EventHandle, GraphExecHandle, GraphHandle, diff --git a/cuda_core/cuda/core/texture/_array.pxd b/cuda_core/cuda/core/texture/_array.pxd index ceb64e2401c..48a392c7a91 100644 --- a/cuda_core/cuda/core/texture/_array.pxd +++ b/cuda_core/cuda/core/texture/_array.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport OpaqueArrayHandle +from cuda.core._rt cimport OpaqueArrayHandle cdef class OpaqueArray: diff --git a/cuda_core/cuda/core/texture/_array.pyx b/cuda_core/cuda/core/texture/_array.pyx index e5fc3f6c9e2..b9677077bfd 100644 --- a/cuda_core/cuda/core/texture/_array.pyx +++ b/cuda_core/cuda/core/texture/_array.pyx @@ -11,7 +11,7 @@ from libc.string cimport memset from cuda.bindings cimport cydriver from cuda.core._context cimport Context from cuda.core._memory._buffer cimport Buffer, Buffer_check_open -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( OpaqueArrayHandle, as_cu, as_intptr, diff --git a/cuda_core/cuda/core/texture/_mipmapped_array.pxd b/cuda_core/cuda/core/texture/_mipmapped_array.pxd index 281c7140b05..00233fafa40 100644 --- a/cuda_core/cuda/core/texture/_mipmapped_array.pxd +++ b/cuda_core/cuda/core/texture/_mipmapped_array.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport MipmappedArrayHandle +from cuda.core._rt cimport MipmappedArrayHandle cdef class MipmappedArray: diff --git a/cuda_core/cuda/core/texture/_mipmapped_array.pyx b/cuda_core/cuda/core/texture/_mipmapped_array.pyx index 8d6bf5a2589..2bdf9d92937 100644 --- a/cuda_core/cuda/core/texture/_mipmapped_array.pyx +++ b/cuda_core/cuda/core/texture/_mipmapped_array.pyx @@ -13,7 +13,7 @@ from cuda.core.texture._array import ( _validate_array_shape, _validate_format_channels, ) -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( OpaqueArrayHandle, MipmappedArrayHandle, as_intptr, diff --git a/cuda_core/cuda/core/texture/_surface.pxd b/cuda_core/cuda/core/texture/_surface.pxd index d6702061e79..09e8e961be1 100644 --- a/cuda_core/cuda/core/texture/_surface.pxd +++ b/cuda_core/cuda/core/texture/_surface.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport SurfObjectHandle +from cuda.core._rt cimport SurfObjectHandle cdef class SurfaceObject: diff --git a/cuda_core/cuda/core/texture/_surface.pyx b/cuda_core/cuda/core/texture/_surface.pyx index 790ce048ecd..da1ee6c25b0 100644 --- a/cuda_core/cuda/core/texture/_surface.pyx +++ b/cuda_core/cuda/core/texture/_surface.pyx @@ -9,7 +9,7 @@ from libc.string cimport memset from cuda.bindings cimport cydriver from cuda.core._context cimport Context from cuda.core.texture._array cimport OpaqueArray, OpaqueArray_check_open -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( ContextHandle, SurfObjectHandle, as_cu, diff --git a/cuda_core/cuda/core/texture/_texture.pxd b/cuda_core/cuda/core/texture/_texture.pxd index 6d0871fe848..3a689f593c9 100644 --- a/cuda_core/cuda/core/texture/_texture.pxd +++ b/cuda_core/cuda/core/texture/_texture.pxd @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from cuda.bindings cimport cydriver -from cuda.core._resource_handles cimport TexObjectHandle +from cuda.core._rt cimport TexObjectHandle cdef class TextureObject: diff --git a/cuda_core/cuda/core/texture/_texture.pyx b/cuda_core/cuda/core/texture/_texture.pyx index 28ddf2d6aa8..dd6be36c34b 100644 --- a/cuda_core/cuda/core/texture/_texture.pyx +++ b/cuda_core/cuda/core/texture/_texture.pyx @@ -19,7 +19,7 @@ from cuda.core.texture._array import ( from cuda.core._memory._buffer cimport Buffer, Buffer_check_open from cuda.core.texture._mipmapped_array cimport MipmappedArray, MipmappedArray_check_open from cuda.core.texture._mipmapped_array import MipmappedArray as _PyMipmappedArray -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( ContextHandle, TexObjectHandle, as_cu, diff --git a/cuda_core/tests/test_error_handling.py b/cuda_core/tests/test_error_handling.py index caed0beea66..44e8331f597 100644 --- a/cuda_core/tests/test_error_handling.py +++ b/cuda_core/tests/test_error_handling.py @@ -29,7 +29,7 @@ LegacyPinnedMemoryResource, ) from cuda.core._memory._synchronous_memory_resource import _SynchronousMemoryResource -from cuda.core._resource_handles import ( +from cuda.core._rt import ( _note_or_report_cuda_error_for_testing, _set_context_restore_fault_for_testing, ) From 41dd419f4b643f627f7f1984951520ac91b71e80 Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Fri, 11 Sep 2026 10:34:30 -0700 Subject: [PATCH 09/14] cuda.core: remove the dead py_object_user_object_destroy A Py_DECREF callback shaped for cuUserObjectCreate, superseded by the make_opaque_py ownership path and called from nowhere. Its four declarations go: the C++ prototype and body, the .pxd cdef and the .pyx extern declaration. --- cuda_core/cuda/core/_cpp/rt/rt.cpp | 11 ----------- cuda_core/cuda/core/_cpp/rt/rt.hpp | 4 ---- cuda_core/cuda/core/_rt.pxd | 1 - cuda_core/cuda/core/_rt.pyx | 2 -- 4 files changed, 18 deletions(-) diff --git a/cuda_core/cuda/core/_cpp/rt/rt.cpp b/cuda_core/cuda/core/_cpp/rt/rt.cpp index 41af411a412..75bdf52fec1 100644 --- a/cuda_core/cuda/core/_cpp/rt/rt.cpp +++ b/cuda_core/cuda/core/_cpp/rt/rt.cpp @@ -1188,17 +1188,6 @@ StreamHandle create_stream_handle_with_owner(CUstream stream, PyObject* owner) { return StreamHandle(box, &box->resource); } -void py_object_user_object_destroy(void* py_object) noexcept { - if (!py_object) { - return; - } - GILAcquireGuard gil; - if (!gil.acquired()) { - return; - } - Py_DECREF(reinterpret_cast(py_object)); -} - // Return the context retained by a stream handle. ContextHandle get_stream_context(const StreamHandle& h) noexcept { return h ? get_box(h)->h_context : ContextHandle{}; diff --git a/cuda_core/cuda/core/_cpp/rt/rt.hpp b/cuda_core/cuda/core/_cpp/rt/rt.hpp index bbb370b5089..83c69c3d993 100644 --- a/cuda_core/cuda/core/_cpp/rt/rt.hpp +++ b/cuda_core/cuda/core/_cpp/rt/rt.hpp @@ -344,10 +344,6 @@ StreamHandle create_stream_handle_ref(CUstream stream); // The owner is responsible for keeping the stream's context alive. StreamHandle create_stream_handle_with_owner(CUstream stream, PyObject* owner); -// Destroy a Python-backed CUDA user object by decref'ing it when safe. -// If Python is finalized or finalizing, the object is intentionally leaked. -void py_object_user_object_destroy(void* py_object) noexcept; - // Initialize the process-lifetime CUDA user-object cleanup queue. Called once // from module initialization while Python is fully initialized. void initialize_deferred_cleanup(); diff --git a/cuda_core/cuda/core/_rt.pxd b/cuda_core/cuda/core/_rt.pxd index eb90c5ef529..d93bc5e2bee 100644 --- a/cuda_core/cuda/core/_rt.pxd +++ b/cuda_core/cuda/core/_rt.pxd @@ -208,7 +208,6 @@ cdef StreamHandle create_stream_handle( const ContextHandle& h_ctx, unsigned int flags, int priority) except+ nogil cdef StreamHandle create_stream_handle_ref(cydriver.CUstream stream) except+ nogil cdef StreamHandle create_stream_handle_with_owner(cydriver.CUstream stream, object owner) except+ nogil -cdef void py_object_user_object_destroy(void* py_object) noexcept nogil cdef void retry_deferred_cleanup() noexcept cdef ContextHandle get_stream_context(const StreamHandle& h) noexcept nogil cdef StreamHandle get_legacy_stream() except+ nogil diff --git a/cuda_core/cuda/core/_rt.pyx b/cuda_core/cuda/core/_rt.pyx index c42c0ccaf66..be1c493439c 100644 --- a/cuda_core/cuda/core/_rt.pyx +++ b/cuda_core/cuda/core/_rt.pyx @@ -95,8 +95,6 @@ cdef extern from "_cpp/rt/rt.hpp" namespace "cuda_core::rt": cydriver.CUstream stream) except+ nogil StreamHandle create_stream_handle_with_owner "cuda_core::rt::create_stream_handle_with_owner" ( cydriver.CUstream stream, object owner) except+ nogil - void py_object_user_object_destroy "cuda_core::rt::py_object_user_object_destroy" ( - void* py_object) noexcept nogil void initialize_deferred_cleanup "cuda_core::rt::initialize_deferred_cleanup" () except+ void retry_deferred_cleanup "cuda_core::rt::retry_deferred_cleanup" () noexcept ContextHandle get_stream_context "cuda_core::rt::get_stream_context" ( From 0f3931aef3f67770ebcf4cff61e709ad21351824 Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Fri, 11 Sep 2026 10:41:35 -0700 Subject: [PATCH 10/14] cuda.core build: depend on module-directory headers; compile an extension's sources in parallel Two build changes the C++ split needs. Every extension now lists the headers under cuda/core/_cpp// as `depends`. A cimporting extension compiles against the header its .pxd names, and cythonize copies each `depends` entry into its build directory, so the copied header finds its sibling includes beside it. Listing the whole directory avoids parsing includes; every extension rebuilds when one of these headers changes, exactly as editing the one monolithic header did. setuptools compiles the sources of one extension serially and parallelizes only across extensions, so a multi-source extension becomes the critical path. build_ext now fans the per-object compile calls of every extension out to one shared thread pool of `nthreads` workers. MSVC keeps the stock path. --- cuda_core/build_hooks.py | 29 +++++++++++ cuda_core/setup.py | 55 +++++++++++++++++++- cuda_core/tests/test_build_hooks.py | 80 +++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 1 deletion(-) diff --git a/cuda_core/build_hooks.py b/cuda_core/build_hooks.py index f105c92005a..47bd4e7c66e 100644 --- a/cuda_core/build_hooks.py +++ b/cuda_core/build_hooks.py @@ -199,6 +199,29 @@ def _extension_sources(mod_name): return sources +def _extension_depends(): + """Headers whose edits must rebuild an extension: every header under a + directory-form module's cuda/core/_cpp// (a single-file module has + none). + + The same list serves every extension. A module that cimports a + directory-form module compiles against the header its .pxd names, and + cythonize copies each `depends` entry into its build directory before + compiling, so the copied header finds its sibling includes beside it + (quoted includes resolve next to the copy, not in the source tree). + Listing the whole directory keeps the rule free of include parsing; the + cost is that every extension rebuilds when any of these headers changes, + exactly as editing the one monolithic header did before the split.""" + cpp = Path("cuda", "core", "_cpp") + return sorted( + str(path) + for module_dir in cpp.iterdir() + if module_dir.is_dir() + for path in module_dir.rglob("*") + if path.suffix in (".h", ".hpp") + ) + + def _build_cuda_core(debug=False): # Customizing the build hooks is needed because we must defer cythonization until cuda-bindings, # now a required build-time dependency that's dynamically installed via the other hook below, @@ -265,10 +288,12 @@ def module_names(): # related to free-threading builds. extra_compile_args += ["-DCYTHON_TRACE_NOGIL=1", "-DCYTHON_USE_SYS_MONITORING=0"] + depends = _extension_depends() ext_modules = tuple( Extension( f"cuda.core.{mod.replace(os.path.sep, '.')}", sources=_extension_sources(mod), + depends=depends, include_dirs=[ "cuda/core/_include", "cuda/core/_cpp", @@ -299,6 +324,10 @@ def module_names(): # CUDA_PYTHON_COVERAGE deliberately generates in-tree so the sources can # be packaged; every other build gets its own per-configuration cache, # anchored alongside the stamp so both resolve the same from any cwd. + # Cython also copies each extension's extern headers and `depends` under + # this directory and compiles against the copies. Copies are refreshed by + # mtime and never deleted, so remove build/ after renaming or deleting a + # header under _cpp/. build_dir="." if COMPILE_FOR_COVERAGE else str(_BUILD_DIR / "cython" / f"cu{cuda_major}"), nthreads=nthreads, compiler_directives=compiler_directives, diff --git a/cuda_core/setup.py b/cuda_core/setup.py index c66050fe50a..19dd2cd7d23 100644 --- a/cuda_core/setup.py +++ b/cuda_core/setup.py @@ -2,7 +2,9 @@ # # SPDX-License-Identifier: Apache-2.0 +import contextlib import os +from concurrent.futures import ThreadPoolExecutor from pathlib import Path import build_hooks # our build backend @@ -77,10 +79,61 @@ def _configure_windows_tensor_bridge(self): raise RuntimeError(f"Failed to find extension {_TENSOR_BRIDGE_EXT_NAME!r} for Windows build.") + @contextlib.contextmanager + def _parallel_source_compilation(self): + """Compile the sources of every extension through one shared thread pool. + + setuptools runs extensions in parallel (self.parallel) but compiles the + sources of one extension serially, so a multi-source extension such as + cuda.core._rt (a dozen .cpp files) becomes the critical path. This + mirrors CCompiler.compile() and fans its per-object _compile() calls out + to a pool shared by all extensions, so at most `nthreads` compiler + processes run at once. MSVC's compiler class has no _compile(); it keeps + the stock path. + """ + compiler = self.compiler + if nthreads <= 1 or not hasattr(compiler, "_compile"): + yield + return + stock_compile = compiler.compile + with ThreadPoolExecutor(max_workers=nthreads) as pool: + + def compile( + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None, + ): + macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile( + output_dir, macros, include_dirs, sources, depends, extra_postargs + ) + cc_args = compiler._get_cc_args(pp_opts, debug, extra_preargs) + + def compile_one(obj): + try: + src, ext = build[obj] + except KeyError: + return # up to date + compiler._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + list(pool.map(compile_one, objects)) # re-raises the first failure + return objects + + compiler.compile = compile + try: + yield + finally: + compiler.compile = stock_compile + def build_extensions(self): self.parallel = nthreads self._configure_windows_tensor_bridge() - super().build_extensions() + with self._parallel_source_compilation(): + super().build_extensions() build_hooks.record_build_major() diff --git a/cuda_core/tests/test_build_hooks.py b/cuda_core/tests/test_build_hooks.py index b59e059548a..8807a4fb05d 100644 --- a/cuda_core/tests/test_build_hooks.py +++ b/cuda_core/tests/test_build_hooks.py @@ -21,6 +21,8 @@ import os import sys import tempfile +import threading +import types from pathlib import Path from unittest import mock @@ -364,3 +366,81 @@ def test_legacy_single_file_and_no_cpp(self, tree): def test_empty_directory_is_an_error(self, tree): with pytest.raises(RuntimeError, match="no .cpp files"): build_hooks._extension_sources("_d") + + +class TestExtensionDepends: + """_extension_depends: every header under a directory-form module's + _cpp//, the same list for every extension (see its docstring).""" + + @pytest.mark.agent_authored(model="claude-fable-5-1") + def test_headers_under_module_directories_only(self, tmp_path, monkeypatch): + cpp = tmp_path / "cuda" / "core" / "_cpp" + (cpp / "a" / "nested").mkdir(parents=True) + for name in ("a/x.hpp", "a/nested/y.h", "a/z.cpp", "a/notes.md", "top.hpp", "b.cpp"): + (cpp / name).write_text("") + monkeypatch.chdir(tmp_path) + a = os.path.join("cuda", "core", "_cpp", "a") + assert build_hooks._extension_depends() == [os.path.join(a, "nested", "y.h"), os.path.join(a, "x.hpp")] + + +class TestParallelSourceCompilation: + """setup.py compiles an extension's sources through one shared thread pool.""" + + class FakeCompiler: + def __init__(self, fail_on=None): + self.compiled = [] + self.fail_on = fail_on + self.lock = threading.Lock() + + def _setup_compile(self, outdir, macros, incdirs, sources, depends, extra): + extra = [] if extra is None else extra # as distutils does + objects = [source + ".o" for source in sources] + return macros, objects, extra, ["-Dpp"], {obj: (src, ".cpp") for obj, src in zip(objects, sources)} + + def _get_cc_args(self, pp_opts, debug, before): + return ["-c", *pp_opts] + + def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): + if src == self.fail_on: + raise RuntimeError(f"{src} failed") + with self.lock: + self.compiled.append((obj, src, ext, tuple(cc_args), tuple(extra_postargs), tuple(pp_opts))) + + def compile(self, *args, **kwargs): + return "stock" + + def _build_ext(self, monkeypatch, nthreads, compiler): + from setuptools.dist import Distribution + + setup_py = _load_setup_py(monkeypatch) + monkeypatch.setattr(setup_py, "nthreads", nthreads) + cmd = setup_py.build_ext(Distribution({"name": "cuda-core", "version": "0"})) + cmd.compiler = compiler + return cmd + + @pytest.mark.agent_authored(model="claude-fable-5-1") + def test_every_source_compiles_once_and_the_object_order_is_kept(self, monkeypatch): + cmd = self._build_ext(monkeypatch, 4, self.FakeCompiler()) + sources = [f"rt/{name}.cpp" for name in "abcdef"] + with cmd._parallel_source_compilation(): + objects = cmd.compiler.compile(sources, output_dir="tmp", extra_postargs=["-O2"], depends=["x.hpp"]) + assert objects == [source + ".o" for source in sources] + assert sorted(entry[0] for entry in cmd.compiler.compiled) == sorted(objects) + assert {entry[2:] for entry in cmd.compiler.compiled} == {(".cpp", ("-c", "-Dpp"), ("-O2",), ("-Dpp",))} + assert cmd.compiler.compile(sources) == "stock" # restored on exit + + @pytest.mark.agent_authored(model="claude-fable-5-1") + def test_a_failing_source_fails_the_extension(self, monkeypatch): + cmd = self._build_ext(monkeypatch, 4, self.FakeCompiler(fail_on="rt/c.cpp")) + with cmd._parallel_source_compilation(), pytest.raises(RuntimeError, match="rt/c.cpp failed"): + cmd.compiler.compile([f"rt/{name}.cpp" for name in "abcdef"]) + + @pytest.mark.agent_authored(model="claude-fable-5-1") + def test_serial_builds_and_compilers_without_the_hook_keep_the_stock_path(self, monkeypatch): + cmd = self._build_ext(monkeypatch, 1, self.FakeCompiler()) + with cmd._parallel_source_compilation(): + assert cmd.compiler.compile(["a.cpp"]) == "stock" + msvc_like = types.SimpleNamespace(compile=self.FakeCompiler().compile) # no _compile() + cmd = self._build_ext(monkeypatch, 4, msvc_like) + with cmd._parallel_source_compilation(): + assert cmd.compiler.compile(["a.cpp"]) == "stock" From 69ac578bb9fbad159701aaf3de3cc6113e940510 Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Fri, 11 Sep 2026 10:46:23 -0700 Subject: [PATCH 11/14] cuda.core: split rt.hpp into the _cpp/rt/ headers The monolithic header becomes: types.hpp (handle aliases, tagged values, Prepared* types, the inline as_cu/as_intptr accessors), py.hpp (the one file that includes : py_is_finalizing, make_py and as_py, and the prototypes that take or return PyObject*), driver_api.hpp (the p_* table and the version-gated shims), error.hpp (thread-local error state and the non-propagating reporting API), api.hpp (every other prototype, one banner per resource family), plus two umbrellas: rt.hpp, named only by _rt.pyx, and handles.hpp, named only by _rt.pxd, whose include closure is types.hpp and py.hpp. Every declaration moves verbatim; the only additions are the file boilerplate, one banner in py.hpp and `// Implemented in ` lines on the prototypes whose body lives outside their family source. The generator (anchored on the monolith's text) and its check mode live in the maintainer's notes; the dynamic symbol table and __pyx_capi__ of the built extension are unchanged. --- cuda_core/cuda/core/_cpp/rt/api.hpp | 553 +++++++++ cuda_core/cuda/core/_cpp/rt/driver_api.hpp | 190 +++ cuda_core/cuda/core/_cpp/rt/error.hpp | 67 ++ cuda_core/cuda/core/_cpp/rt/handles.hpp | 12 + cuda_core/cuda/core/_cpp/rt/py.hpp | 198 ++++ cuda_core/cuda/core/_cpp/rt/rt.hpp | 1234 +------------------- cuda_core/cuda/core/_cpp/rt/types.hpp | 294 +++++ cuda_core/cuda/core/_rt.pxd | 2 +- 8 files changed, 1321 insertions(+), 1229 deletions(-) create mode 100644 cuda_core/cuda/core/_cpp/rt/api.hpp create mode 100644 cuda_core/cuda/core/_cpp/rt/driver_api.hpp create mode 100644 cuda_core/cuda/core/_cpp/rt/error.hpp create mode 100644 cuda_core/cuda/core/_cpp/rt/handles.hpp create mode 100644 cuda_core/cuda/core/_cpp/rt/py.hpp create mode 100644 cuda_core/cuda/core/_cpp/rt/types.hpp diff --git a/cuda_core/cuda/core/_cpp/rt/api.hpp b/cuda_core/cuda/core/_cpp/rt/api.hpp new file mode 100644 index 00000000000..c52e0851cd1 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/api.hpp @@ -0,0 +1,553 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "types.hpp" +#include +#include +#include + +namespace cuda_core::rt { + +// ============================================================================ +// Context handle functions +// ============================================================================ + +// Function to create a non-owning context handle (references existing context). +ContextHandle create_context_handle_ref(CUcontext ctx); + +// Create a context handle for the CUcontext view of the provided green context. +// The returned ContextHandle keeps the green context alive, but the CUcontext +// view is non-owning and is not destroyed independently. +ContextHandle create_context_handle_from_green_ctx(const GreenCtxHandle& h_green_ctx); + +// Return the green context dependency associated with a ContextHandle, if any. +GreenCtxHandle get_context_green_ctx(const ContextHandle& h) noexcept; + +// Create an owning green context handle from a list of device resources. +GreenCtxHandle create_green_ctx_handle(CUdevResource* resources, unsigned int nbResources, + CUdevice dev, unsigned int flags); + +// Create a non-owning green context handle. +GreenCtxHandle create_green_ctx_handle_ref(CUgreenCtx ctx); + +// Get handle to the primary context for a device (with thread-local caching) +// Returns empty handle on error (caller must check) +ContextHandle get_primary_context(int device_id); + +// Get handle to the current CUDA context +// Returns empty handle if no context is current (caller must check) +ContextHandle get_current_context(); + +// Synchronize the provided context. Releases the GIL around the driver call. +// Returns CUDA_ERROR_INVALID_CONTEXT for an empty handle. +CUresult context_synchronize(const ContextHandle& h_context) noexcept; + +// Query the stream priority range for the provided context. +// Returns CUDA_ERROR_INVALID_CONTEXT for an empty handle. +CUresult context_get_stream_priority_range( + const ContextHandle& h_context, + int* least_priority, + int* greatest_priority) noexcept; + +// Query the device of the provided context. +// Returns CUDA_ERROR_INVALID_CONTEXT for an empty handle. +CUresult context_get_device(const ContextHandle& h_context, CUdevice* device) noexcept; + +// Call cuGraphNodeSetParams with h_context current (empty handle: the caller's +// context). Returns the update status; *restore_status receives a failure to +// restore the caller's context after a successful update, which the caller +// raises only after publishing the metadata that depends on the update. +// Returns CUDA_ERROR_NOT_SUPPORTED when the driver lacks cuGraphNodeSetParams. +// Implemented in graph.cpp +CUresult graph_node_set_params( + CUgraphNode node, + CUgraphNodeParams* params, + const ContextHandle& h_context, + CUresult* restore_status) noexcept; + +// ============================================================================ +// Stream handle functions +// ============================================================================ + +// Create an owning stream handle by calling cuStreamCreateWithPriority. +// The stream structurally depends on the provided context handle. +// When the last reference is released, cuStreamDestroy is called automatically. +// Returns empty handle on error (caller must check). +StreamHandle create_stream_handle(const ContextHandle& h_ctx, unsigned int flags, int priority); + +// Create a non-owning stream handle (references existing stream). +// Use for borrowed streams (from foreign code) or built-in streams. +// The stream will NOT be destroyed when the handle is released. +// Caller is responsible for keeping the stream's context alive. +StreamHandle create_stream_handle_ref(CUstream stream); + +// Initialize the process-lifetime CUDA user-object cleanup queue. Called once +// from module initialization while Python is fully initialized. +// Implemented in py_deferred_cleanup.cpp +void initialize_deferred_cleanup(); +// Implemented in py_deferred_cleanup.cpp +void retry_deferred_cleanup() noexcept; + +// Return the context dependency associated with a stream handle, if any. +ContextHandle get_stream_context(const StreamHandle& h) noexcept; + +// Get non-owning handle to the legacy default stream (CU_STREAM_LEGACY) +// Note: Legacy stream has no specific context dependency. +StreamHandle get_legacy_stream(); + +// Get non-owning handle to the per-thread default stream (CU_STREAM_PER_THREAD) +// Note: Per-thread stream has no specific context dependency. +StreamHandle get_per_thread_stream(); + +// Wrap CU_STREAM_LEGACY with an explicit context, bypassing the "bind to +// whatever is current" resolution that a bare default-stream token uses (see +// make_deallocation_stream). Lets a resource that always operates in one +// known context (e.g. a synchronous, non-pooled allocator) record a correct +// deallocation context without requiring that context to be current when the +// token is created. Returns an empty handle for an empty h_context. +StreamHandle create_context_bound_legacy_stream(const ContextHandle& h_context); + +// ============================================================================ +// Event handle functions +// ============================================================================ + +// Create an owning event handle by calling cuEventCreate. +// The event structurally depends on the provided context handle. +// Metadata fields are stored in the EventBox for later retrieval. +// When the last reference is released, cuEventDestroy is called automatically. +// Returns empty handle on error (caller must check). +EventHandle create_event_handle(const ContextHandle& h_ctx, unsigned int flags, + bool timing_enabled, bool is_blocking_sync, + bool ipc_enabled, int device_id); + +// Create an owning event in the context that owns `stream`, so it can be +// recorded on that stream regardless of which context is current. Default- +// stream tokens resolve to the current context (cuStreamGetCtx semantics). +// Use for temporary ordering events that are created and destroyed in the +// same scope; the handle carries no device id. +// When the last reference is released, cuEventDestroy is called automatically. +// Returns empty handle on error (caller must check). +EventHandle create_event_handle_for_stream(CUstream stream, unsigned int flags); + +// Create an owning event handle from an IPC handle. +// The originating process owns the event and its context. +// When the last reference is released, cuEventDestroy is called automatically. +// Returns empty handle on error (caller must check). +EventHandle create_event_handle_ipc(const CUipcEventHandle& ipc_handle, + bool is_blocking_sync); + +// Create a non-owning event handle (references existing event). +// Use for events that are managed by the CUDA graph or another owner. +// The event will NOT be destroyed when the handle is released. +// Metadata defaults to unknown (timing_enabled=false, device_id=-1). +EventHandle create_event_handle_ref(CUevent event); + +// Event metadata accessors (read from EventBox via pointer arithmetic) +bool get_event_timing_enabled(const EventHandle& h) noexcept; +bool get_event_is_blocking_sync(const EventHandle& h) noexcept; +bool get_event_ipc_enabled(const EventHandle& h) noexcept; +int get_event_device_id(const EventHandle& h) noexcept; +ContextHandle get_event_context(const EventHandle& h) noexcept; + +// ============================================================================ +// Memory pool handle functions +// ============================================================================ + +// Create an owning memory pool handle by calling cuMemPoolCreate. +// Memory pools are device-scoped (not context-scoped). +// When the last reference is released, cuMemPoolDestroy is called automatically. +// Returns empty handle on error (caller must check). +MemoryPoolHandle create_mempool_handle(const CUmemPoolProps& props); + +// Create a non-owning memory pool handle (references existing pool). +// Use for device default/current pools that are managed by the driver. +// The pool will NOT be destroyed when the handle is released. +MemoryPoolHandle create_mempool_handle_ref(CUmemoryPool pool); + +// Get non-owning handle to the current memory pool for a device. +// Returns empty handle on error (caller must check). +MemoryPoolHandle get_device_mempool(int device_id); + +// Create an owning memory pool handle from an IPC import. +// The file descriptor is NOT owned by this handle (caller manages FD separately). +// When the last reference is released, cuMemPoolDestroy is called automatically. +// Returns empty handle on error (caller must check). +MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type); + +// ============================================================================ +// Device pointer handle functions +// ============================================================================ + +// Allocate device memory from a pool asynchronously via cuMemAllocFromPoolAsync. +// The pointer structurally depends on the provided pool handle (captured in deleter). +// When the last reference is released, cuMemFreeAsync is called on the stored stream. +// Returns empty handle on error (caller must check). +DevicePtrHandle deviceptr_alloc_from_pool( + size_t size, + const MemoryPoolHandle& h_pool, + const StreamHandle& h_stream); + +// Allocate device memory asynchronously via cuMemAllocAsync. +// When the last reference is released, cuMemFreeAsync is called on the stored stream. +// Returns empty handle on error (caller must check). +DevicePtrHandle deviceptr_alloc_async(size_t size, const StreamHandle& h_stream); + +// Allocate device memory synchronously via cuMemAlloc with the provided +// context current. The caller owns the pointer and releases it with cuMemFree. +// Returns CUDA_ERROR_INVALID_CONTEXT for an empty handle. +CUresult deviceptr_alloc_raw(CUdeviceptr* ptr, size_t size, + const ContextHandle& h_context) noexcept; + +// Allocate pinned host memory via cuMemAllocHost. +// When the last reference is released, cuMemFreeHost is called. +// Returns empty handle on error (caller must check). +DevicePtrHandle deviceptr_alloc_host(size_t size); + +// Create a non-owning device pointer handle (references existing pointer). +// Use for foreign pointers (e.g., from external libraries). +// The pointer will NOT be freed when the handle is released. +DevicePtrHandle deviceptr_create_ref(CUdeviceptr ptr); + +// Create a device pointer handle for a mapped graphics resource. +// The pointer structurally depends on the provided graphics resource handle. +// When the last reference is released, cuGraphicsUnmapResources is called on +// the stored stream, then the graphics resource may be unregistered when its +// own handle is released. +DevicePtrHandle deviceptr_create_mapped_graphics( + CUdeviceptr ptr, + const GraphicsResourceHandle& h_resource, + const StreamHandle& h_stream); + +// Import a device pointer from IPC via cuMemPoolImportPointer. +// When the last reference is released, cuMemFreeAsync is called on the stored stream. +// Note: Does not yet implement reference counting for nvbug 5570902. +// On error, returns empty handle and sets thread-local error (use get_last_error()). +DevicePtrHandle deviceptr_import_ipc( + const MemoryPoolHandle& h_pool, + const void* export_data, + const StreamHandle& h_stream); + +// Access the deallocation stream for a device pointer handle (read-only). +// For non-owning handles, the stream is not used but can still be accessed. +StreamHandle deallocation_stream(const DevicePtrHandle& h) noexcept; + +// Set the deallocation stream for a device pointer handle. +// Returns CUDA_ERROR_INVALID_CONTEXT when a default-stream token cannot be +// bound because no CUDA context is current. +CUresult set_deallocation_stream( + const DevicePtrHandle& h, const StreamHandle& h_stream) noexcept; + +// ============================================================================ +// Library handle functions +// ============================================================================ + +// Create an owning library handle by loading from a file path. +// When the last reference is released, cuLibraryUnload is called automatically. +// Returns empty handle on error (caller must check). +LibraryHandle create_library_handle_from_file(const char* path); + +// Create an owning library handle by loading from memory data. +// The driver makes an internal copy of the data; caller can free it after return. +// When the last reference is released, cuLibraryUnload is called automatically. +// Returns empty handle on error (caller must check). +LibraryHandle create_library_handle_from_data(const void* data); + +// Create a non-owning library handle (references existing library). +// Use for borrowed libraries (e.g., from foreign code). +// The library will NOT be unloaded when the handle is released. +LibraryHandle create_library_handle_ref(CUlibrary library); + +// ============================================================================ +// Kernel handle functions +// ============================================================================ + +// Get a kernel from a library by name. +// The kernel structurally depends on the provided library handle. +// Kernels have no explicit destroy - their lifetime is tied to the library. +// Returns empty handle on error (caller must check). +KernelHandle create_kernel_handle(const LibraryHandle& h_library, const char* name); + +// Create a kernel handle from a raw CUkernel. +// If the kernel is already managed (in the registry), returns the owning +// handle with library dependency. Otherwise returns a non-owning ref. +KernelHandle create_kernel_handle_ref(CUkernel kernel); + +// Get the library handle associated with a kernel (from KernelBox). +// Returns empty handle if the kernel has no library dependency. +LibraryHandle get_kernel_library(const KernelHandle& h) noexcept; + +// ============================================================================ +// Graph handle functions +// ============================================================================ + +// Create the owning handle for a root graph and its hierarchy. +GraphHandle create_graph_handle(CUgraph graph); + +// Create the canonical handle for a graph whose CUDA lifetime is owned by a +// node in h_parent. +GraphHandle create_child_graph_handle( + CUgraph child_graph, const GraphHandle& h_parent, CUgraphNode owner_node); + +// ============================================================================ +// Graph node attachments +// +// Each resource-bearing node has one attachment with an immutable owner bundle, +// retained on its CUgraph as a CUDA user object. +// +// Attachment mutations use prepare -> CUDA mutation -> commit. Preparation +// graph-retains a replacement and preallocates its map entry when needed; an +// empty replacement stages removal. Dropping an uncommitted PreparedAttachment +// rolls back any staged retain. Commit updates metadata before releasing the +// previous graph reference. +// graph_get_attachment lets callers carry unchanged owners into partial +// updates. The clone and invalidation helpers synchronize non-owning metadata +// after CUDA copies or destroys graph state. +// ============================================================================ + +// Build an OpaqueHandle from a malloc'd buffer: std::free on release. +OpaqueHandle make_opaque_malloc(void* buf); + +// Copy requested owners from node's current attachment. Pass nullptr to ignore +// either owner; a missing attachment produces empty handles. +CUresult graph_get_attachment( + const GraphHandle& h_graph, + CUgraphNode node, + OpaqueHandle* owner0, + OpaqueHandle* owner1); + +// Create and graph-retain a replacement attachment before a CUDA mutation. +// Destruction rolls the prepared attachment back unless it is committed. +CUresult graph_prepare_attachment( + const GraphHandle& h_graph, + OpaqueHandle owner0, + OpaqueHandle owner1, + PreparedAttachment* out_prepared); + +// Publish a prepared attachment after the CUDA mutation succeeds. A null node +// retains the attachment anonymously without publishing node metadata. +CUresult graph_commit_attachment( + PreparedAttachment& prepared, + CUgraphNode node); + +// Copy attachment metadata from a source graph hierarchy into its CUDA clone. +CUresult graph_clone_attachments( + const GraphHandle& h_clone, + const GraphHandle& h_source); + +// Stage a complete metadata replacement before CUDA replaces an embedded +// graph. Dropping the prepared state leaves the current hierarchy unchanged. +CUresult graph_prepare_child_graph_update( + const GraphHandle& h_parent, + const GraphHandle& h_old_child, + CUgraphNode owner_node, + const GraphHandle& h_source, + PreparedChildGraphUpdate* out_prepared); + +// Rekey staged metadata to CUDA's replacement clone, retire the old embedded +// hierarchy, and publish the replacement handle. +CUresult graph_commit_child_graph_update( + PreparedChildGraphUpdate& prepared, + GraphHandle* out_child); + +// Invalidate cuda.core state for child graphs CUDA destroyed with owner_node. +void invalidate_child_graph_state( + const GraphHandle& h_parent, + CUgraphNode owner_node) noexcept; + +// ============================================================================ +// Graph exec handle functions +// ============================================================================ + +// Create an owning exec handle by calling cuGraphInstantiateWithParams. +// A fresh attachment accumulator is retained on h_source first, because CUDA +// propagates user object references only at instantiation; an exec cannot +// receive them afterwards. The exec is the sole owner once this returns. +// When the last reference is released, cuGraphExecDestroy is called +// automatically. +// Returns empty handle on error (caller must check). The caller reads +// params->result_out for the specific instantiation failure and +// get_last_error() for a driver status. +GraphExecHandle create_graph_exec_handle( + const GraphHandle& h_source, + CUDA_GRAPH_INSTANTIATE_PARAMS* params); + +// Update h_exec in place by calling cuGraphExecUpdate, and publish a fresh +// accumulator when CUDA accepts the update. Writes result_info for the caller. +CUresult graph_exec_update( + const GraphExecHandle& h_exec, + const GraphHandle& h_source, + CUgraphExecUpdateResultInfo* result_info); + +// Append owners before an executable-node mutation. The accumulator grows +// because CUDA cannot attach user objects to an exec after instantiation, so +// old owners stay reachable. Dropping the transaction restores the accumulator +// to its original size. +CUresult graph_prepare_exec_attachment( + const GraphExecHandle& h_exec, + OpaqueHandle owner0, + OpaqueHandle owner1, + PreparedExecAttachment* out_prepared); + +// Keep the owners added by graph_prepare_exec_attachment. +void graph_commit_exec_attachment( + PreparedExecAttachment& prepared) noexcept; + +// ============================================================================ +// Graph node handle functions +// ============================================================================ + +// Create a node handle. Nodes are owned by their parent graph (not +// independently destroyable). The GraphHandle dependency ensures the +// graph outlives any node reference. +GraphNodeHandle create_graph_node_handle(CUgraphNode node, const GraphHandle& h_graph); + +// Extract the owning graph handle from a node handle. +GraphHandle graph_node_get_graph(const GraphNodeHandle& h) noexcept; + +// Zero the CUgraphNode resource inside the handle, marking it invalid. +void invalidate_graph_node(const GraphNodeHandle& h) noexcept; + +// ============================================================================ +// Graphics resource handle functions +// ============================================================================ + +// Create an owning graphics resource handle. +// When the last reference is released, cuGraphicsUnregisterResource is called automatically. +// Use for CUgraphicsResource handles obtained from cuGraphicsGLRegisterBuffer etc. +GraphicsResourceHandle create_graphics_resource_handle(CUgraphicsResource resource); + +// ============================================================================ +// NVRTC Program handle functions +// ============================================================================ + +// Create an owning NVRTC program handle. +// When the last reference is released, nvrtcDestroyProgram is called. +// Use this to wrap a program created via nvrtcCreateProgram. +NvrtcProgramHandle create_nvrtc_program_handle(nvrtcProgram prog); + +// Create a non-owning NVRTC program handle (references existing program). +// The program will NOT be destroyed when the handle is released. +NvrtcProgramHandle create_nvrtc_program_handle_ref(nvrtcProgram prog); + +// ============================================================================ +// NVVM Program handle functions +// ============================================================================ + +// Create an owning NVVM program handle. +// When the last reference is released, nvvmDestroyProgram is called. +// Use this to wrap a program created via nvvmCreateProgram. +// Note: If NVVM is not available (p_nvvmDestroyProgram is null), the deleter is a no-op. +NvvmProgramHandle create_nvvm_program_handle(nvvmProgram prog); + +// Create a non-owning NVVM program handle (references existing program). +// The program will NOT be destroyed when the handle is released. +NvvmProgramHandle create_nvvm_program_handle_ref(nvvmProgram prog); + +// ============================================================================ +// nvJitLink handle functions +// ============================================================================ + +// Create an owning nvJitLink handle. +// When the last reference is released, nvJitLinkDestroy is called. +// Use this to wrap a handle created via nvJitLinkCreate. +// Note: If nvJitLink is not available (p_nvJitLinkDestroy is null), the deleter is a no-op. +NvJitLinkHandle create_nvjitlink_handle(nvJitLink_t handle); + +// Create a non-owning nvJitLink handle (references existing handle). +// The handle will NOT be destroyed when the last reference is released. +NvJitLinkHandle create_nvjitlink_handle_ref(nvJitLink_t handle); + +// ============================================================================ +// cuLink handle functions +// ============================================================================ + +// Create an owning cuLink handle. +// When the last reference is released, cuLinkDestroy is called. +// Use this to wrap a CUlinkState created via cuLinkCreate. +CuLinkHandle create_culink_handle(CUlinkState state); + +// Create a non-owning cuLink handle (references existing CUlinkState). +// The handle will NOT be destroyed when the last reference is released. +CuLinkHandle create_culink_handle_ref(CUlinkState state); + +// ============================================================================ +// File descriptor handle functions +// ============================================================================ + +// Create an owning file descriptor handle. +// When the last reference is released, POSIX close() is called. +FileDescriptorHandle create_fd_handle(int fd); + +// Create a non-owning file descriptor handle (caller manages the fd). +FileDescriptorHandle create_fd_handle_ref(int fd); + +// ============================================================================ +// Array / mipmapped-array / texture / surface handle functions (PR #467) +// +// These resources are managed exactly like every other cuda.core resource: +// the owning handle's deleter calls the matching cu*Destroy with the GIL +// released, structural dependencies are embedded in the box (so a backing +// resource always outlives a texture/surface/level built on it), and +// creation returns an empty handle + thread-local error on failure. +// ============================================================================ + +// Create an owning CUDA array via cuArray3DCreate. +// When the last reference is released, cuArrayDestroy is called automatically. +// Returns empty handle on error (caller must check). +OpaqueArrayHandle create_array_handle(const ContextHandle& h_context, const CUDA_ARRAY3D_DESCRIPTOR& desc); + +// Create a non-owning array handle (references an existing CUarray). +// Use for arrays owned elsewhere (e.g. graphics interop). Never destroyed here. +OpaqueArrayHandle create_array_handle_ref(CUarray arr); + +// Create an owning array handle adopting an existing CUarray. +// When the last reference is released, cuArrayDestroy is called automatically. +OpaqueArrayHandle create_array_handle_owning(CUarray arr); + +// Return the context dependency associated with an array, if known. +ContextHandle get_array_context(const OpaqueArrayHandle& h) noexcept; + +// Create a non-owning handle to a mipmap level via cuMipmappedArrayGetLevel. +// The level CUarray is owned by the mipmap; the parent MipmappedArrayHandle is +// embedded in the box so it outlives the level view. No destroy in the deleter. +// Returns empty handle on error (caller must check). +OpaqueArrayHandle create_array_level_handle(const MipmappedArrayHandle& h_mip, unsigned int level); + +// Create an owning mipmapped array via cuMipmappedArrayCreate. +// When the last reference is released, cuMipmappedArrayDestroy is called. +// Returns empty handle on error (caller must check). +MipmappedArrayHandle create_mipmapped_array_handle(const ContextHandle& h_context, + const CUDA_ARRAY3D_DESCRIPTOR& desc, + unsigned int num_levels); + +// Return the context dependency associated with a mipmapped array, if known. +ContextHandle get_mipmapped_array_context(const MipmappedArrayHandle& h) noexcept; + +// Create an owning texture object via cuTexObjectCreate, embedding the backing +// resource handle (array / mipmapped array / linear-or-pitch2d device pointer) +// so the backing always outlives the texture. cuTexObjectDestroy runs in the +// deleter. Returns empty handle on error (caller must check). +TexObjectHandle create_tex_object_handle_array(const ContextHandle& h_context, + const CUDA_RESOURCE_DESC& res, + const CUDA_TEXTURE_DESC& tex, + const OpaqueArrayHandle& h_backing); +TexObjectHandle create_tex_object_handle_mipmap(const ContextHandle& h_context, + const CUDA_RESOURCE_DESC& res, + const CUDA_TEXTURE_DESC& tex, + const MipmappedArrayHandle& h_backing); +TexObjectHandle create_tex_object_handle_linear(const ContextHandle& h_context, + const CUDA_RESOURCE_DESC& res, + const CUDA_TEXTURE_DESC& tex, + const DevicePtrHandle& h_backing); + +// Create an owning surface object via cuSurfObjectCreate, embedding the backing +// array handle so it outlives the surface. cuSurfObjectDestroy runs in the +// deleter. Returns empty handle on error (caller must check). +SurfObjectHandle create_surf_object_handle(const ContextHandle& h_context, + const CUDA_RESOURCE_DESC& res, + const OpaqueArrayHandle& h_backing); + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/driver_api.hpp b/cuda_core/cuda/core/_cpp/rt/driver_api.hpp new file mode 100644 index 00000000000..87ed3bd7906 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/driver_api.hpp @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "types.hpp" +#include +#include +#include + +namespace cuda_core::rt { + +// ============================================================================ +// CUDA driver function pointers +// +// These are populated by _rt.pyx at module import time using +// function pointers extracted from cuda.bindings.cydriver.__pyx_capi__. +// ============================================================================ + +extern decltype(&cuGetErrorName) p_cuGetErrorName; +extern decltype(&cuGetErrorString) p_cuGetErrorString; + +extern decltype(&cuDevicePrimaryCtxRetain) p_cuDevicePrimaryCtxRetain; +extern decltype(&cuDevicePrimaryCtxRelease) p_cuDevicePrimaryCtxRelease; +extern decltype(&cuCtxGetCurrent) p_cuCtxGetCurrent; +extern decltype(&cuCtxSetCurrent) p_cuCtxSetCurrent; +extern decltype(&cuCtxSynchronize) p_cuCtxSynchronize; +extern decltype(&cuCtxGetStreamPriorityRange) p_cuCtxGetStreamPriorityRange; +extern decltype(&cuCtxGetDevice) p_cuCtxGetDevice; +extern decltype(&cuGraphNodeSetParams) p_cuGraphNodeSetParams; +extern decltype(&cuGreenCtxCreate) p_cuGreenCtxCreate; +extern decltype(&cuGreenCtxDestroy) p_cuGreenCtxDestroy; +extern decltype(&cuCtxFromGreenCtx) p_cuCtxFromGreenCtx; +extern decltype(&cuDevResourceGenerateDesc) p_cuDevResourceGenerateDesc; + +extern decltype(&cuGreenCtxStreamCreate) p_cuGreenCtxStreamCreate; + +extern decltype(&cuStreamCreateWithPriority) p_cuStreamCreateWithPriority; +extern decltype(&cuStreamDestroy) p_cuStreamDestroy; +extern decltype(&cuStreamGetCtx) p_cuStreamGetCtx; + +extern decltype(&cuEventCreate) p_cuEventCreate; +extern decltype(&cuEventDestroy) p_cuEventDestroy; +extern decltype(&cuIpcOpenEventHandle) p_cuIpcOpenEventHandle; + +extern decltype(&cuDeviceGetCount) p_cuDeviceGetCount; + +extern decltype(&cuMemPoolSetAccess) p_cuMemPoolSetAccess; +extern decltype(&cuMemPoolDestroy) p_cuMemPoolDestroy; +extern decltype(&cuMemPoolCreate) p_cuMemPoolCreate; +extern decltype(&cuDeviceGetMemPool) p_cuDeviceGetMemPool; +extern decltype(&cuMemPoolImportFromShareableHandle) p_cuMemPoolImportFromShareableHandle; + +extern decltype(&cuMemAllocFromPoolAsync) p_cuMemAllocFromPoolAsync; +extern decltype(&cuMemAllocAsync) p_cuMemAllocAsync; +extern decltype(&cuMemAlloc) p_cuMemAlloc; +extern decltype(&cuMemAllocHost) p_cuMemAllocHost; + +extern decltype(&cuMemFreeAsync) p_cuMemFreeAsync; +extern decltype(&cuMemFree) p_cuMemFree; +extern decltype(&cuMemFreeHost) p_cuMemFreeHost; + +extern decltype(&cuMemPoolImportPointer) p_cuMemPoolImportPointer; + +// Library +extern decltype(&cuLibraryLoadFromFile) p_cuLibraryLoadFromFile; +extern decltype(&cuLibraryLoadData) p_cuLibraryLoadData; +extern decltype(&cuLibraryUnload) p_cuLibraryUnload; +extern decltype(&cuLibraryGetKernel) p_cuLibraryGetKernel; + +// Graph +extern decltype(&cuGraphDestroy) p_cuGraphDestroy; +extern decltype(&cuGraphInstantiateWithParams) p_cuGraphInstantiateWithParams; +extern decltype(&cuGraphExecUpdate) p_cuGraphExecUpdate; +extern decltype(&cuGraphExecDestroy) p_cuGraphExecDestroy; +extern decltype(&cuUserObjectCreate) p_cuUserObjectCreate; +extern decltype(&cuUserObjectRelease) p_cuUserObjectRelease; +extern decltype(&cuGraphRetainUserObject) p_cuGraphRetainUserObject; +extern decltype(&cuGraphReleaseUserObject) p_cuGraphReleaseUserObject; +extern decltype(&cuGraphNodeFindInClone) p_cuGraphNodeFindInClone; +extern decltype(&cuGraphChildGraphNodeGetGraph) p_cuGraphChildGraphNodeGetGraph; + +// Linker +extern decltype(&cuLinkDestroy) p_cuLinkDestroy; + +// Graphics interop +extern decltype(&cuGraphicsUnmapResources) p_cuGraphicsUnmapResources; +extern decltype(&cuGraphicsUnregisterResource) p_cuGraphicsUnregisterResource; + +// Texture / surface / array (PR #467) +extern decltype(&cuArray3DCreate) p_cuArray3DCreate; +extern decltype(&cuArrayDestroy) p_cuArrayDestroy; +extern decltype(&cuMipmappedArrayCreate) p_cuMipmappedArrayCreate; +extern decltype(&cuMipmappedArrayDestroy) p_cuMipmappedArrayDestroy; +extern decltype(&cuMipmappedArrayGetLevel) p_cuMipmappedArrayGetLevel; +extern decltype(&cuTexObjectCreate) p_cuTexObjectCreate; +extern decltype(&cuTexObjectDestroy) p_cuTexObjectDestroy; +extern decltype(&cuSurfObjectCreate) p_cuSurfObjectCreate; +extern decltype(&cuSurfObjectDestroy) p_cuSurfObjectDestroy; + +// SM resource split (13.1+ — may be null on older drivers/bindings) +#if CUDA_VERSION >= 13010 +extern decltype(&cuDevSmResourceSplit) p_cuDevSmResourceSplit; +#else +// cuDevSmResourceSplit doesn't exist in CUDA < 13.1 headers, so use a +// void* placeholder. The pointer is always null when built against 12.x. +extern void* p_cuDevSmResourceSplit; +#endif + +// cuMemcpyWithAttributesAsync (13.2+ — may be null on older drivers/bindings) +#if CUDA_VERSION >= 13020 +extern decltype(&cuMemcpyWithAttributesAsync) p_cuMemcpyWithAttributesAsync; +#else +// cuMemcpyWithAttributesAsync doesn't exist in CUDA < 13.2 headers, so use a +// void* placeholder. The pointer is always null when built against older CUDA. +extern void* p_cuMemcpyWithAttributesAsync; +#endif + +// ============================================================================ +// NVRTC function pointers +// +// These are populated by _rt.pyx at module import time using +// function pointers extracted from cuda.bindings.cynvrtc.__pyx_capi__. +// ============================================================================ + +extern decltype(&nvrtcDestroyProgram) p_nvrtcDestroyProgram; + +// ============================================================================ +// NVVM function pointers +// +// These are populated by _rt.pyx at module import time using +// function pointers extracted from cuda.bindings.cynvvm.__pyx_capi__. +// Note: May be null if NVVM is not available at runtime. +// ============================================================================ + +// Function pointer type for nvvmDestroyProgram (avoids nvvm.h dependency) +// Signature: nvvmResult nvvmDestroyProgram(nvvmProgram *prog) +using NvvmDestroyProgramFn = int (*)(nvvmProgram*); +extern NvvmDestroyProgramFn p_nvvmDestroyProgram; + +// ============================================================================ +// nvJitLink function pointers +// +// These are populated by _rt.pyx at module import time using +// function pointers extracted from cuda.bindings.cynvjitlink.__pyx_capi__. +// Note: May be null if nvJitLink is not available at runtime. +// ============================================================================ + +// Function pointer type for nvJitLinkDestroy (avoids nvJitLink.h dependency) +// Signature: nvJitLinkResult nvJitLinkDestroy(nvJitLinkHandle *handle) +using NvJitLinkDestroyFn = int (*)(nvJitLink_t*); +extern NvJitLinkDestroyFn p_nvJitLinkDestroy; + +// ============================================================================ +// SM resource split wrapper (13.1+) +// +// Calls through p_cuDevSmResourceSplit if available, otherwise returns +// CUDA_ERROR_NOT_SUPPORTED. This avoids a direct Cython cimport of the +// cydriver cdef function, which would fail at module init on cuda-bindings +// < 13.1 (see https://github.com/NVIDIA/cuda-python/issues/2063). +// ============================================================================ + +// groupParams is void* so the Cython declaration doesn't reference +// CU_DEV_SM_RESOURCE_GROUP_PARAMS (absent from cuda-bindings 13.0 .pxd). +CUresult sm_resource_split(CUdevResource* result, unsigned int nbGroups, + const CUdevResource* input, CUdevResource* remainder, + unsigned int flags, void* groupParams); + +// Returns true if the cuDevSmResourceSplit function pointer is available. +bool has_sm_resource_split() noexcept; + +// ============================================================================ +// cuMemcpyWithAttributesAsync wrapper (13.2+) +// +// Calls through p_cuMemcpyWithAttributesAsync if available, otherwise returns +// CUDA_ERROR_NOT_SUPPORTED. This avoids a direct Cython cimport of the +// cydriver cdef function, which would fail at module init on cuda-bindings +// < 13.2 (see https://github.com/NVIDIA/cuda-python/issues/2063). +// ============================================================================ + +// attr is void* so the Cython declaration doesn't reference CUmemcpyAttributes +// (absent from cuda-bindings built against CUDA < 12.8). The C++ side casts it. +CUresult memcpy_with_attributes_async(CUdeviceptr dst, CUdeviceptr src, size_t size, + void* attr, CUstream hStream); + +// Returns true if the cuMemcpyWithAttributesAsync function pointer is available. +bool has_memcpy_with_attributes_async() noexcept; + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/error.hpp b/cuda_core/cuda/core/_cpp/rt/error.hpp new file mode 100644 index 00000000000..febeb6239ab --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/error.hpp @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +namespace cuda_core::rt { + +// ============================================================================ +// Thread-local error handling +// ============================================================================ + +// Get and clear the last CUDA error (like cudaGetLastError) +CUresult get_last_error() noexcept; + +// Get the last CUDA error without clearing it (like cudaPeekAtLastError) +CUresult peek_last_error() noexcept; + +// Explicitly clear the last error +void clear_last_error() noexcept; + +// ============================================================================ +// Non-propagating error reporting +// +// Paths that cannot raise (shared_ptr deleters, CUDA callbacks, __dealloc__) +// report failures through these functions instead of discarding them. They +// emit a cuda.core.CUDAWarning when the interpreter can be used and write to +// stderr otherwise; they never raise. See docs/source/error_handling.rst. +// ============================================================================ + +// Report a failed CUDA call. `detail` replaces the default "failed" wording, +// e.g. "skipped (context activation failed; resource leaked)". +// CUDA_ERROR_DEINITIALIZED (driver shutting down) is never reported. +void report_cuda_error(const char* operation, CUresult status, const char* detail = nullptr) noexcept; + +// Report a message that is not tied to a CUresult. +// Implemented in py_report.cpp +void report_message(const char* message) noexcept; + +// Report a failed NVRTC/NVVM/nvJitLink call by raw status code. +void report_status_code(const char* operation, long code) noexcept; + +// Attach a failed CUDA call to the Python exception currently being handled +// (PEP 678 note, Python 3.11+): for rollback failures inside `except` blocks +// whose original exception is about to be re-raised. When no exception is +// being handled or notes are unavailable, falls back to report_cuda_error(). +// Implemented in py_report.cpp +void note_or_report_cuda_error(const char* operation, CUresult status, const char* detail = nullptr) noexcept; + +// Detail recorded by a context-scoped helper for the CUresult it is about to +// return, e.g. that the caller's context could not be restored. The Cython +// error path attaches it to the raised CUDAError as a note. Thread-local and +// keyed by status: take_ returns the detail (valid until the next take on this +// thread) and clears it when `status` is the CUresult it was recorded for, and +// returns nullptr otherwise, so a detail whose status was never raised cannot +// attach to an unrelated error. +const char* take_last_error_detail(CUresult status) noexcept; +void clear_last_error_detail() noexcept; + +// Tests only: make the next context restoration on this thread fail with +// `status`, leaving the target context current as a real failure would. +// Implemented in context.cpp +void set_context_restore_fault_for_testing(CUresult status) noexcept; + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/handles.hpp b/cuda_core/cuda/core/_cpp/rt/handles.hpp new file mode 100644 index 00000000000..043ddd7ef88 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/handles.hpp @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +// Consumer umbrella, named only by _rt.pxd: the handle types, the inline +// accessors and the Python seam. Everything else reaches consumers through +// __pyx_capi__, never through a header. + +#include "py.hpp" +#include "types.hpp" diff --git a/cuda_core/cuda/core/_cpp/rt/py.hpp b/cuda_core/cuda/core/_cpp/rt/py.hpp new file mode 100644 index 00000000000..65179c1caca --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/py.hpp @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include "types.hpp" +#include +#include + +namespace cuda_core::rt { + +#if PY_VERSION_HEX < 0x030D0000 +extern "C" int _Py_IsFinalizing(void); +#endif + +// Best-effort probe for interpreter shutdown. +// +// In CPython this is not a hard guarantee: finalization can begin after this +// returns false but before a later PyGILState_Ensure() or other Python C-API +// call. +// +// If that race is lost on a non-finalizer thread, CPython's behavior is +// version-dependent: on older supported versions (3.10-3.13) it may abruptly +// terminate the current thread (historically via PyThread_exit_thread(), +// without normal C++ unwinding), while on newer versions (3.14+) it may hang +// the thread until process exit. +// +// We still use this check because the policy in this layer is to avoid Python +// work once shutdown is underway and accept an intentional leak or skipped +// Python conversion in that edge case rather than add more complex deferral +// machinery. +inline bool py_is_finalizing() noexcept { +#if PY_VERSION_HEX >= 0x030D0000 + return Py_IsFinalizing(); +#else + return _Py_IsFinalizing() != 0; +#endif +} + +// as_py() - convert handle to Python wrapper object (returns new reference) +namespace detail { +// n.b. class lookup is not cached to avoid deadlock hazard, see DESIGN.md +inline PyObject* make_py(const char* module_name, const char* class_name, std::intptr_t value) noexcept { + if (py_is_finalizing()) { + Py_RETURN_NONE; + } + PyObject* mod = PyImport_ImportModule(module_name); + if (!mod) return nullptr; + PyObject* cls = PyObject_GetAttrString(mod, class_name); + Py_DECREF(mod); + if (!cls) return nullptr; + PyObject* result = PyObject_CallFunction(cls, "L", value); + Py_DECREF(cls); + return result; +} +} // namespace detail + +inline PyObject* as_py(const ContextHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUcontext", as_intptr(h)); +} + +inline PyObject* as_py(const GreenCtxHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUgreenCtx", as_intptr(h)); +} + +inline PyObject* as_py(const StreamHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUstream", as_intptr(h)); +} + +inline PyObject* as_py(const EventHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUevent", as_intptr(h)); +} + +inline PyObject* as_py(const MemoryPoolHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUmemoryPool", as_intptr(h)); +} + +inline PyObject* as_py(const DevicePtrHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUdeviceptr", as_intptr(h)); +} + +inline PyObject* as_py(const LibraryHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUlibrary", as_intptr(h)); +} + +inline PyObject* as_py(const CUmodule& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUmodule", as_intptr(h)); +} + +inline PyObject* as_py(const KernelHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUkernel", as_intptr(h)); +} + +inline PyObject* as_py(const GraphHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUgraph", as_intptr(h)); +} + +inline PyObject* as_py(const GraphExecHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUgraphExec", as_intptr(h)); +} + +inline PyObject* as_py(const GraphNodeHandle& h) noexcept { + if (!as_intptr(h)) { + Py_RETURN_NONE; + } + return detail::make_py("cuda.bindings.driver", "CUgraphNode", as_intptr(h)); +} + +inline PyObject* as_py(const NvrtcProgramHandle& h) noexcept { + return detail::make_py("cuda.bindings.nvrtc", "nvrtcProgram", as_intptr(h)); +} + +inline PyObject* as_py(const NvvmProgramHandle& h) noexcept { + // NVVM bindings use raw integers, not wrapper classes + return PyLong_FromSsize_t(as_intptr(h)); +} + +inline PyObject* as_py(const NvJitLinkHandle& h) noexcept { + // nvJitLink bindings use raw integers, not wrapper classes + return PyLong_FromSsize_t(as_intptr(h)); +} + +inline PyObject* as_py(const CuLinkHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUlinkState", as_intptr(h)); +} + +inline PyObject* as_py(const GraphicsResourceHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUgraphicsResource", as_intptr(h)); +} + +inline PyObject* as_py(const FileDescriptorHandle& h) noexcept { + return PyLong_FromSsize_t(as_intptr(h)); +} + +inline PyObject* as_py(const OpaqueArrayHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUarray", as_intptr(h)); +} + +inline PyObject* as_py(const MipmappedArrayHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUmipmappedArray", as_intptr(h)); +} + +inline PyObject* as_py(const TexObjectHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUtexObject", as_intptr(h)); +} + +inline PyObject* as_py(const SurfObjectHandle& h) noexcept { + return detail::make_py("cuda.bindings.driver", "CUsurfObject", as_intptr(h)); +} + +// ============================================================================ +// Python-coupled API: the prototypes that take or return PyObject* +// ============================================================================ + +// Register the Python warning category used by report_* (cuda.core.CUDAWarning). +// Implemented in py_report.cpp +void register_warning_category(PyObject* category) noexcept; + +// Create a non-owning stream handle that prevents a Python owner from being GC'd. +// The owner's refcount is incremented; decremented when handle is released. +// The owner is responsible for keeping the stream's context alive. +// Implemented in stream.cpp +StreamHandle create_stream_handle_with_owner(CUstream stream, PyObject* owner); + +// Create a non-owning device pointer handle that prevents a Python owner from being GC'd. +// The owner's refcount is incremented; decremented when handle is released. +// The pointer will NOT be freed when the handle is released. +// If owner is nullptr, equivalent to deviceptr_create_ref. +// Implemented in memory.cpp +DevicePtrHandle deviceptr_create_with_owner(CUdeviceptr ptr, PyObject* owner); + +// Callback type for MemoryResource deallocation. +// Called from the shared_ptr deleter when a handle created via +// deviceptr_create_with_mr is destroyed. The implementation is responsible +// for converting raw C types to Python objects and calling +// mr.deallocate(ptr, size, stream). +using MRDeallocCallback = void (*)(PyObject* mr, CUdeviceptr ptr, + size_t size, const StreamHandle& stream); + +// Register the MR deallocation callback. +// Implemented in memory.cpp +void register_mr_dealloc_callback(MRDeallocCallback cb); + +// Create a device pointer handle whose destructor calls mr.deallocate() +// via the registered callback. The mr's refcount is incremented and +// decremented when the handle is released. +// If mr is nullptr, equivalent to deviceptr_create_ref. +// Implemented in memory.cpp +DevicePtrHandle deviceptr_create_with_mr(CUdeviceptr ptr, size_t size, PyObject* mr); + +// Build an OpaqueHandle from a Python object: increments its refcount now and +// decrements it (under the GIL) on release. The caller must hold the GIL. +// Implemented in graph.cpp +OpaqueHandle make_opaque_py(PyObject* obj); + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/rt.hpp b/cuda_core/cuda/core/_cpp/rt/rt.hpp index 83c69c3d993..7b843017225 100644 --- a/cuda_core/cuda/core/_cpp/rt/rt.hpp +++ b/cuda_core/cuda/core/_cpp/rt/rt.hpp @@ -4,1232 +4,10 @@ #pragma once -#include -#include -#include -#include -#include +// Module umbrella, named only by _rt.pyx. -// Forward declaration for NVVM - avoids nvvm.h dependency -// Use void* to match cuda.bindings.cynvvm's typedef -using nvvmProgram = void*; - -// Forward declaration for nvJitLink - avoids nvJitLink.h dependency -// Use void* to match cuda.bindings.cynvjitlink's typedef -using nvJitLink_t = void*; - -namespace cuda_core::rt { - -// ============================================================================ -// TaggedHandle - make void*-based handle types distinct for overloading -// -// Both nvvmProgram and nvJitLink_t are void*, so shared_ptr -// would be the same C++ type for both. TaggedHandle wraps the raw -// value with a unique tag type, making each shared_ptr type distinct. -// ============================================================================ - -template -struct TaggedHandle { - T raw; -}; - -using NvvmProgramValue = TaggedHandle; -using NvJitLinkValue = TaggedHandle; - -// CUtexObject, CUsurfObject and CUdeviceptr are all `unsigned long long`, so -// shared_ptr et al. would be the *same* C++ type as -// DevicePtrHandle (and each other), collapsing the as_cu/as_intptr/as_py -// overload sets. Tag them to keep each handle type distinct, exactly as the -// NVVM / nvJitLink handles above do. -using TexObjectValue = TaggedHandle; -using SurfObjectValue = TaggedHandle; - -// ============================================================================ -// Thread-local error handling -// ============================================================================ - -// Get and clear the last CUDA error (like cudaGetLastError) -CUresult get_last_error() noexcept; - -// Get the last CUDA error without clearing it (like cudaPeekAtLastError) -CUresult peek_last_error() noexcept; - -// Explicitly clear the last error -void clear_last_error() noexcept; - -// ============================================================================ -// Non-propagating error reporting -// -// Paths that cannot raise (shared_ptr deleters, CUDA callbacks, __dealloc__) -// report failures through these functions instead of discarding them. They -// emit a cuda.core.CUDAWarning when the interpreter can be used and write to -// stderr otherwise; they never raise. See docs/source/error_handling.rst. -// ============================================================================ - -// Register the Python warning category used by report_* (cuda.core.CUDAWarning). -void register_warning_category(PyObject* category) noexcept; - -// Report a failed CUDA call. `detail` replaces the default "failed" wording, -// e.g. "skipped (context activation failed; resource leaked)". -// CUDA_ERROR_DEINITIALIZED (driver shutting down) is never reported. -void report_cuda_error(const char* operation, CUresult status, const char* detail = nullptr) noexcept; - -// Report a message that is not tied to a CUresult. -void report_message(const char* message) noexcept; - -// Report a failed NVRTC/NVVM/nvJitLink call by raw status code. -void report_status_code(const char* operation, long code) noexcept; - -// Attach a failed CUDA call to the Python exception currently being handled -// (PEP 678 note, Python 3.11+): for rollback failures inside `except` blocks -// whose original exception is about to be re-raised. When no exception is -// being handled or notes are unavailable, falls back to report_cuda_error(). -void note_or_report_cuda_error(const char* operation, CUresult status, const char* detail = nullptr) noexcept; - -// Detail recorded by a context-scoped helper for the CUresult it is about to -// return, e.g. that the caller's context could not be restored. The Cython -// error path attaches it to the raised CUDAError as a note. Thread-local and -// keyed by status: take_ returns the detail (valid until the next take on this -// thread) and clears it when `status` is the CUresult it was recorded for, and -// returns nullptr otherwise, so a detail whose status was never raised cannot -// attach to an unrelated error. -const char* take_last_error_detail(CUresult status) noexcept; -void clear_last_error_detail() noexcept; - -// Tests only: make the next context restoration on this thread fail with -// `status`, leaving the target context current as a real failure would. -void set_context_restore_fault_for_testing(CUresult status) noexcept; - -// ============================================================================ -// CUDA driver function pointers -// -// These are populated by _rt.pyx at module import time using -// function pointers extracted from cuda.bindings.cydriver.__pyx_capi__. -// ============================================================================ - -extern decltype(&cuGetErrorName) p_cuGetErrorName; -extern decltype(&cuGetErrorString) p_cuGetErrorString; - -extern decltype(&cuDevicePrimaryCtxRetain) p_cuDevicePrimaryCtxRetain; -extern decltype(&cuDevicePrimaryCtxRelease) p_cuDevicePrimaryCtxRelease; -extern decltype(&cuCtxGetCurrent) p_cuCtxGetCurrent; -extern decltype(&cuCtxSetCurrent) p_cuCtxSetCurrent; -extern decltype(&cuCtxSynchronize) p_cuCtxSynchronize; -extern decltype(&cuCtxGetStreamPriorityRange) p_cuCtxGetStreamPriorityRange; -extern decltype(&cuCtxGetDevice) p_cuCtxGetDevice; -extern decltype(&cuGraphNodeSetParams) p_cuGraphNodeSetParams; -extern decltype(&cuGreenCtxCreate) p_cuGreenCtxCreate; -extern decltype(&cuGreenCtxDestroy) p_cuGreenCtxDestroy; -extern decltype(&cuCtxFromGreenCtx) p_cuCtxFromGreenCtx; -extern decltype(&cuDevResourceGenerateDesc) p_cuDevResourceGenerateDesc; - -extern decltype(&cuGreenCtxStreamCreate) p_cuGreenCtxStreamCreate; - -extern decltype(&cuStreamCreateWithPriority) p_cuStreamCreateWithPriority; -extern decltype(&cuStreamDestroy) p_cuStreamDestroy; -extern decltype(&cuStreamGetCtx) p_cuStreamGetCtx; - -extern decltype(&cuEventCreate) p_cuEventCreate; -extern decltype(&cuEventDestroy) p_cuEventDestroy; -extern decltype(&cuIpcOpenEventHandle) p_cuIpcOpenEventHandle; - -extern decltype(&cuDeviceGetCount) p_cuDeviceGetCount; - -extern decltype(&cuMemPoolSetAccess) p_cuMemPoolSetAccess; -extern decltype(&cuMemPoolDestroy) p_cuMemPoolDestroy; -extern decltype(&cuMemPoolCreate) p_cuMemPoolCreate; -extern decltype(&cuDeviceGetMemPool) p_cuDeviceGetMemPool; -extern decltype(&cuMemPoolImportFromShareableHandle) p_cuMemPoolImportFromShareableHandle; - -extern decltype(&cuMemAllocFromPoolAsync) p_cuMemAllocFromPoolAsync; -extern decltype(&cuMemAllocAsync) p_cuMemAllocAsync; -extern decltype(&cuMemAlloc) p_cuMemAlloc; -extern decltype(&cuMemAllocHost) p_cuMemAllocHost; - -extern decltype(&cuMemFreeAsync) p_cuMemFreeAsync; -extern decltype(&cuMemFree) p_cuMemFree; -extern decltype(&cuMemFreeHost) p_cuMemFreeHost; - -extern decltype(&cuMemPoolImportPointer) p_cuMemPoolImportPointer; - -// Library -extern decltype(&cuLibraryLoadFromFile) p_cuLibraryLoadFromFile; -extern decltype(&cuLibraryLoadData) p_cuLibraryLoadData; -extern decltype(&cuLibraryUnload) p_cuLibraryUnload; -extern decltype(&cuLibraryGetKernel) p_cuLibraryGetKernel; - -// Graph -extern decltype(&cuGraphDestroy) p_cuGraphDestroy; -extern decltype(&cuGraphInstantiateWithParams) p_cuGraphInstantiateWithParams; -extern decltype(&cuGraphExecUpdate) p_cuGraphExecUpdate; -extern decltype(&cuGraphExecDestroy) p_cuGraphExecDestroy; -extern decltype(&cuUserObjectCreate) p_cuUserObjectCreate; -extern decltype(&cuUserObjectRelease) p_cuUserObjectRelease; -extern decltype(&cuGraphRetainUserObject) p_cuGraphRetainUserObject; -extern decltype(&cuGraphReleaseUserObject) p_cuGraphReleaseUserObject; -extern decltype(&cuGraphNodeFindInClone) p_cuGraphNodeFindInClone; -extern decltype(&cuGraphChildGraphNodeGetGraph) p_cuGraphChildGraphNodeGetGraph; - -// Linker -extern decltype(&cuLinkDestroy) p_cuLinkDestroy; - -// Graphics interop -extern decltype(&cuGraphicsUnmapResources) p_cuGraphicsUnmapResources; -extern decltype(&cuGraphicsUnregisterResource) p_cuGraphicsUnregisterResource; - -// Texture / surface / array (PR #467) -extern decltype(&cuArray3DCreate) p_cuArray3DCreate; -extern decltype(&cuArrayDestroy) p_cuArrayDestroy; -extern decltype(&cuMipmappedArrayCreate) p_cuMipmappedArrayCreate; -extern decltype(&cuMipmappedArrayDestroy) p_cuMipmappedArrayDestroy; -extern decltype(&cuMipmappedArrayGetLevel) p_cuMipmappedArrayGetLevel; -extern decltype(&cuTexObjectCreate) p_cuTexObjectCreate; -extern decltype(&cuTexObjectDestroy) p_cuTexObjectDestroy; -extern decltype(&cuSurfObjectCreate) p_cuSurfObjectCreate; -extern decltype(&cuSurfObjectDestroy) p_cuSurfObjectDestroy; - -// SM resource split (13.1+ — may be null on older drivers/bindings) -#if CUDA_VERSION >= 13010 -extern decltype(&cuDevSmResourceSplit) p_cuDevSmResourceSplit; -#else -// cuDevSmResourceSplit doesn't exist in CUDA < 13.1 headers, so use a -// void* placeholder. The pointer is always null when built against 12.x. -extern void* p_cuDevSmResourceSplit; -#endif - -// cuMemcpyWithAttributesAsync (13.2+ — may be null on older drivers/bindings) -#if CUDA_VERSION >= 13020 -extern decltype(&cuMemcpyWithAttributesAsync) p_cuMemcpyWithAttributesAsync; -#else -// cuMemcpyWithAttributesAsync doesn't exist in CUDA < 13.2 headers, so use a -// void* placeholder. The pointer is always null when built against older CUDA. -extern void* p_cuMemcpyWithAttributesAsync; -#endif - -// ============================================================================ -// NVRTC function pointers -// -// These are populated by _rt.pyx at module import time using -// function pointers extracted from cuda.bindings.cynvrtc.__pyx_capi__. -// ============================================================================ - -extern decltype(&nvrtcDestroyProgram) p_nvrtcDestroyProgram; - -// ============================================================================ -// NVVM function pointers -// -// These are populated by _rt.pyx at module import time using -// function pointers extracted from cuda.bindings.cynvvm.__pyx_capi__. -// Note: May be null if NVVM is not available at runtime. -// ============================================================================ - -// Function pointer type for nvvmDestroyProgram (avoids nvvm.h dependency) -// Signature: nvvmResult nvvmDestroyProgram(nvvmProgram *prog) -using NvvmDestroyProgramFn = int (*)(nvvmProgram*); -extern NvvmDestroyProgramFn p_nvvmDestroyProgram; - -// ============================================================================ -// nvJitLink function pointers -// -// These are populated by _rt.pyx at module import time using -// function pointers extracted from cuda.bindings.cynvjitlink.__pyx_capi__. -// Note: May be null if nvJitLink is not available at runtime. -// ============================================================================ - -// Function pointer type for nvJitLinkDestroy (avoids nvJitLink.h dependency) -// Signature: nvJitLinkResult nvJitLinkDestroy(nvJitLinkHandle *handle) -using NvJitLinkDestroyFn = int (*)(nvJitLink_t*); -extern NvJitLinkDestroyFn p_nvJitLinkDestroy; - -// ============================================================================ -// Handle type aliases - expose only the raw CUDA resource -// ============================================================================ - -using ContextHandle = std::shared_ptr; -using GreenCtxHandle = std::shared_ptr; -using StreamHandle = std::shared_ptr; -using EventHandle = std::shared_ptr; -using MemoryPoolHandle = std::shared_ptr; -using LibraryHandle = std::shared_ptr; -using KernelHandle = std::shared_ptr; -using GraphHandle = std::shared_ptr; -using GraphExecHandle = std::shared_ptr; -using GraphNodeHandle = std::shared_ptr; -using GraphicsResourceHandle = std::shared_ptr; -using NvrtcProgramHandle = std::shared_ptr; -using NvvmProgramHandle = std::shared_ptr; -using NvJitLinkHandle = std::shared_ptr; -using CuLinkHandle = std::shared_ptr; -using FileDescriptorHandle = std::shared_ptr; -using OpaqueArrayHandle = std::shared_ptr; -using MipmappedArrayHandle = std::shared_ptr; -using TexObjectHandle = std::shared_ptr; -using SurfObjectHandle = std::shared_ptr; - - -// ============================================================================ -// Context handle functions -// ============================================================================ - -// Function to create a non-owning context handle (references existing context). -ContextHandle create_context_handle_ref(CUcontext ctx); - -// Create a context handle for the CUcontext view of the provided green context. -// The returned ContextHandle keeps the green context alive, but the CUcontext -// view is non-owning and is not destroyed independently. -ContextHandle create_context_handle_from_green_ctx(const GreenCtxHandle& h_green_ctx); - -// Return the green context dependency associated with a ContextHandle, if any. -GreenCtxHandle get_context_green_ctx(const ContextHandle& h) noexcept; - -// Create an owning green context handle from a list of device resources. -GreenCtxHandle create_green_ctx_handle(CUdevResource* resources, unsigned int nbResources, - CUdevice dev, unsigned int flags); - -// Create a non-owning green context handle. -GreenCtxHandle create_green_ctx_handle_ref(CUgreenCtx ctx); - -// Get handle to the primary context for a device (with thread-local caching) -// Returns empty handle on error (caller must check) -ContextHandle get_primary_context(int device_id); - -// Get handle to the current CUDA context -// Returns empty handle if no context is current (caller must check) -ContextHandle get_current_context(); - -// Synchronize the provided context. Releases the GIL around the driver call. -// Returns CUDA_ERROR_INVALID_CONTEXT for an empty handle. -CUresult context_synchronize(const ContextHandle& h_context) noexcept; - -// Query the stream priority range for the provided context. -// Returns CUDA_ERROR_INVALID_CONTEXT for an empty handle. -CUresult context_get_stream_priority_range( - const ContextHandle& h_context, - int* least_priority, - int* greatest_priority) noexcept; - -// Query the device of the provided context. -// Returns CUDA_ERROR_INVALID_CONTEXT for an empty handle. -CUresult context_get_device(const ContextHandle& h_context, CUdevice* device) noexcept; - -// Call cuGraphNodeSetParams with h_context current (empty handle: the caller's -// context). Returns the update status; *restore_status receives a failure to -// restore the caller's context after a successful update, which the caller -// raises only after publishing the metadata that depends on the update. -// Returns CUDA_ERROR_NOT_SUPPORTED when the driver lacks cuGraphNodeSetParams. -CUresult graph_node_set_params( - CUgraphNode node, - CUgraphNodeParams* params, - const ContextHandle& h_context, - CUresult* restore_status) noexcept; - -// ============================================================================ -// Stream handle functions -// ============================================================================ - -// Create an owning stream handle by calling cuStreamCreateWithPriority. -// The stream structurally depends on the provided context handle. -// When the last reference is released, cuStreamDestroy is called automatically. -// Returns empty handle on error (caller must check). -StreamHandle create_stream_handle(const ContextHandle& h_ctx, unsigned int flags, int priority); - -// Create a non-owning stream handle (references existing stream). -// Use for borrowed streams (from foreign code) or built-in streams. -// The stream will NOT be destroyed when the handle is released. -// Caller is responsible for keeping the stream's context alive. -StreamHandle create_stream_handle_ref(CUstream stream); - -// Create a non-owning stream handle that prevents a Python owner from being GC'd. -// The owner's refcount is incremented; decremented when handle is released. -// The owner is responsible for keeping the stream's context alive. -StreamHandle create_stream_handle_with_owner(CUstream stream, PyObject* owner); - -// Initialize the process-lifetime CUDA user-object cleanup queue. Called once -// from module initialization while Python is fully initialized. -void initialize_deferred_cleanup(); -void retry_deferred_cleanup() noexcept; - -// Return the context dependency associated with a stream handle, if any. -ContextHandle get_stream_context(const StreamHandle& h) noexcept; - -// Get non-owning handle to the legacy default stream (CU_STREAM_LEGACY) -// Note: Legacy stream has no specific context dependency. -StreamHandle get_legacy_stream(); - -// Get non-owning handle to the per-thread default stream (CU_STREAM_PER_THREAD) -// Note: Per-thread stream has no specific context dependency. -StreamHandle get_per_thread_stream(); - -// Wrap CU_STREAM_LEGACY with an explicit context, bypassing the "bind to -// whatever is current" resolution that a bare default-stream token uses (see -// make_deallocation_stream). Lets a resource that always operates in one -// known context (e.g. a synchronous, non-pooled allocator) record a correct -// deallocation context without requiring that context to be current when the -// token is created. Returns an empty handle for an empty h_context. -StreamHandle create_context_bound_legacy_stream(const ContextHandle& h_context); - -// ============================================================================ -// Event handle functions -// ============================================================================ - -// Create an owning event handle by calling cuEventCreate. -// The event structurally depends on the provided context handle. -// Metadata fields are stored in the EventBox for later retrieval. -// When the last reference is released, cuEventDestroy is called automatically. -// Returns empty handle on error (caller must check). -EventHandle create_event_handle(const ContextHandle& h_ctx, unsigned int flags, - bool timing_enabled, bool is_blocking_sync, - bool ipc_enabled, int device_id); - -// Create an owning event in the context that owns `stream`, so it can be -// recorded on that stream regardless of which context is current. Default- -// stream tokens resolve to the current context (cuStreamGetCtx semantics). -// Use for temporary ordering events that are created and destroyed in the -// same scope; the handle carries no device id. -// When the last reference is released, cuEventDestroy is called automatically. -// Returns empty handle on error (caller must check). -EventHandle create_event_handle_for_stream(CUstream stream, unsigned int flags); - -// Create an owning event handle from an IPC handle. -// The originating process owns the event and its context. -// When the last reference is released, cuEventDestroy is called automatically. -// Returns empty handle on error (caller must check). -EventHandle create_event_handle_ipc(const CUipcEventHandle& ipc_handle, - bool is_blocking_sync); - -// Create a non-owning event handle (references existing event). -// Use for events that are managed by the CUDA graph or another owner. -// The event will NOT be destroyed when the handle is released. -// Metadata defaults to unknown (timing_enabled=false, device_id=-1). -EventHandle create_event_handle_ref(CUevent event); - -// Event metadata accessors (read from EventBox via pointer arithmetic) -bool get_event_timing_enabled(const EventHandle& h) noexcept; -bool get_event_is_blocking_sync(const EventHandle& h) noexcept; -bool get_event_ipc_enabled(const EventHandle& h) noexcept; -int get_event_device_id(const EventHandle& h) noexcept; -ContextHandle get_event_context(const EventHandle& h) noexcept; - -// ============================================================================ -// Memory pool handle functions -// ============================================================================ - -// Create an owning memory pool handle by calling cuMemPoolCreate. -// Memory pools are device-scoped (not context-scoped). -// When the last reference is released, cuMemPoolDestroy is called automatically. -// Returns empty handle on error (caller must check). -MemoryPoolHandle create_mempool_handle(const CUmemPoolProps& props); - -// Create a non-owning memory pool handle (references existing pool). -// Use for device default/current pools that are managed by the driver. -// The pool will NOT be destroyed when the handle is released. -MemoryPoolHandle create_mempool_handle_ref(CUmemoryPool pool); - -// Get non-owning handle to the current memory pool for a device. -// Returns empty handle on error (caller must check). -MemoryPoolHandle get_device_mempool(int device_id); - -// Create an owning memory pool handle from an IPC import. -// The file descriptor is NOT owned by this handle (caller manages FD separately). -// When the last reference is released, cuMemPoolDestroy is called automatically. -// Returns empty handle on error (caller must check). -MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type); - -// ============================================================================ -// Device pointer handle functions -// ============================================================================ - -using DevicePtrHandle = std::shared_ptr; - -// Allocate device memory from a pool asynchronously via cuMemAllocFromPoolAsync. -// The pointer structurally depends on the provided pool handle (captured in deleter). -// When the last reference is released, cuMemFreeAsync is called on the stored stream. -// Returns empty handle on error (caller must check). -DevicePtrHandle deviceptr_alloc_from_pool( - size_t size, - const MemoryPoolHandle& h_pool, - const StreamHandle& h_stream); - -// Allocate device memory asynchronously via cuMemAllocAsync. -// When the last reference is released, cuMemFreeAsync is called on the stored stream. -// Returns empty handle on error (caller must check). -DevicePtrHandle deviceptr_alloc_async(size_t size, const StreamHandle& h_stream); - -// Allocate device memory synchronously via cuMemAlloc with the provided -// context current. The caller owns the pointer and releases it with cuMemFree. -// Returns CUDA_ERROR_INVALID_CONTEXT for an empty handle. -CUresult deviceptr_alloc_raw(CUdeviceptr* ptr, size_t size, - const ContextHandle& h_context) noexcept; - -// Allocate pinned host memory via cuMemAllocHost. -// When the last reference is released, cuMemFreeHost is called. -// Returns empty handle on error (caller must check). -DevicePtrHandle deviceptr_alloc_host(size_t size); - -// Create a non-owning device pointer handle (references existing pointer). -// Use for foreign pointers (e.g., from external libraries). -// The pointer will NOT be freed when the handle is released. -DevicePtrHandle deviceptr_create_ref(CUdeviceptr ptr); - -// Create a non-owning device pointer handle that prevents a Python owner from being GC'd. -// The owner's refcount is incremented; decremented when handle is released. -// The pointer will NOT be freed when the handle is released. -// If owner is nullptr, equivalent to deviceptr_create_ref. -DevicePtrHandle deviceptr_create_with_owner(CUdeviceptr ptr, PyObject* owner); - -// Create a device pointer handle for a mapped graphics resource. -// The pointer structurally depends on the provided graphics resource handle. -// When the last reference is released, cuGraphicsUnmapResources is called on -// the stored stream, then the graphics resource may be unregistered when its -// own handle is released. -DevicePtrHandle deviceptr_create_mapped_graphics( - CUdeviceptr ptr, - const GraphicsResourceHandle& h_resource, - const StreamHandle& h_stream); - -// Callback type for MemoryResource deallocation. -// Called from the shared_ptr deleter when a handle created via -// deviceptr_create_with_mr is destroyed. The implementation is responsible -// for converting raw C types to Python objects and calling -// mr.deallocate(ptr, size, stream). -using MRDeallocCallback = void (*)(PyObject* mr, CUdeviceptr ptr, - size_t size, const StreamHandle& stream); - -// Register the MR deallocation callback. -void register_mr_dealloc_callback(MRDeallocCallback cb); - -// Create a device pointer handle whose destructor calls mr.deallocate() -// via the registered callback. The mr's refcount is incremented and -// decremented when the handle is released. -// If mr is nullptr, equivalent to deviceptr_create_ref. -DevicePtrHandle deviceptr_create_with_mr(CUdeviceptr ptr, size_t size, PyObject* mr); - -// Import a device pointer from IPC via cuMemPoolImportPointer. -// When the last reference is released, cuMemFreeAsync is called on the stored stream. -// Note: Does not yet implement reference counting for nvbug 5570902. -// On error, returns empty handle and sets thread-local error (use get_last_error()). -DevicePtrHandle deviceptr_import_ipc( - const MemoryPoolHandle& h_pool, - const void* export_data, - const StreamHandle& h_stream); - -// Access the deallocation stream for a device pointer handle (read-only). -// For non-owning handles, the stream is not used but can still be accessed. -StreamHandle deallocation_stream(const DevicePtrHandle& h) noexcept; - -// Set the deallocation stream for a device pointer handle. -// Returns CUDA_ERROR_INVALID_CONTEXT when a default-stream token cannot be -// bound because no CUDA context is current. -CUresult set_deallocation_stream( - const DevicePtrHandle& h, const StreamHandle& h_stream) noexcept; - -// ============================================================================ -// Library handle functions -// ============================================================================ - -// Create an owning library handle by loading from a file path. -// When the last reference is released, cuLibraryUnload is called automatically. -// Returns empty handle on error (caller must check). -LibraryHandle create_library_handle_from_file(const char* path); - -// Create an owning library handle by loading from memory data. -// The driver makes an internal copy of the data; caller can free it after return. -// When the last reference is released, cuLibraryUnload is called automatically. -// Returns empty handle on error (caller must check). -LibraryHandle create_library_handle_from_data(const void* data); - -// Create a non-owning library handle (references existing library). -// Use for borrowed libraries (e.g., from foreign code). -// The library will NOT be unloaded when the handle is released. -LibraryHandle create_library_handle_ref(CUlibrary library); - -// ============================================================================ -// Kernel handle functions -// ============================================================================ - -// Get a kernel from a library by name. -// The kernel structurally depends on the provided library handle. -// Kernels have no explicit destroy - their lifetime is tied to the library. -// Returns empty handle on error (caller must check). -KernelHandle create_kernel_handle(const LibraryHandle& h_library, const char* name); - -// Create a kernel handle from a raw CUkernel. -// If the kernel is already managed (in the registry), returns the owning -// handle with library dependency. Otherwise returns a non-owning ref. -KernelHandle create_kernel_handle_ref(CUkernel kernel); - -// Get the library handle associated with a kernel (from KernelBox). -// Returns empty handle if the kernel has no library dependency. -LibraryHandle get_kernel_library(const KernelHandle& h) noexcept; - -// ============================================================================ -// Graph handle functions -// ============================================================================ - -// Create the owning handle for a root graph and its hierarchy. -GraphHandle create_graph_handle(CUgraph graph); - -// Create the canonical handle for a graph whose CUDA lifetime is owned by a -// node in h_parent. -GraphHandle create_child_graph_handle( - CUgraph child_graph, const GraphHandle& h_parent, CUgraphNode owner_node); - -// ============================================================================ -// Graph node attachments -// -// Each resource-bearing node has one attachment with an immutable owner bundle, -// retained on its CUgraph as a CUDA user object. -// -// Attachment mutations use prepare -> CUDA mutation -> commit. Preparation -// graph-retains a replacement and preallocates its map entry when needed; an -// empty replacement stages removal. Dropping an uncommitted PreparedAttachment -// rolls back any staged retain. Commit updates metadata before releasing the -// previous graph reference. -// graph_get_attachment lets callers carry unchanged owners into partial -// updates. The clone and invalidation helpers synchronize non-owning metadata -// after CUDA copies or destroys graph state. -// ============================================================================ - -// Type-erased shared owner of an attached resource. Typed handles such as -// EventHandle and KernelHandle convert to OpaqueHandle by assignment, reusing -// their existing control block; the helpers below build OpaqueHandles for the -// two cases that need a custom deleter. -using OpaqueHandle = std::shared_ptr; - -// Build an OpaqueHandle from a Python object: increments its refcount now and -// decrements it (under the GIL) on release. The caller must hold the GIL. -OpaqueHandle make_opaque_py(PyObject* obj); - -// Build an OpaqueHandle from a malloc'd buffer: std::free on release. -OpaqueHandle make_opaque_malloc(void* buf); - -struct PreparedAttachmentState; -using PreparedAttachmentRollback = - void (*)(PreparedAttachmentState*) noexcept; -struct PreparedAttachmentDeleter { - PreparedAttachmentRollback rollback = nullptr; - - void operator()(PreparedAttachmentState* state) const noexcept { - rollback(state); - } -}; -using PreparedAttachment = - std::unique_ptr; - -struct PreparedChildGraphUpdateState; -// Opaque unpublished hierarchy transaction; releasing it discards staged -// metadata unless graph_commit_child_graph_update publishes the replacement. -using PreparedChildGraphUpdate = - std::shared_ptr; - -struct PreparedExecAttachmentState; -using PreparedExecAttachmentRollback = - void (*)(PreparedExecAttachmentState*) noexcept; -struct PreparedExecAttachmentDeleter { - PreparedExecAttachmentRollback rollback = nullptr; - - void operator()(PreparedExecAttachmentState* state) const noexcept { - rollback(state); - } -}; -// Opaque append transaction. Releasing it rolls back newly appended owners -// unless graph_commit_exec_attachment has kept them. -using PreparedExecAttachment = - std::unique_ptr; - -// Copy requested owners from node's current attachment. Pass nullptr to ignore -// either owner; a missing attachment produces empty handles. -CUresult graph_get_attachment( - const GraphHandle& h_graph, - CUgraphNode node, - OpaqueHandle* owner0, - OpaqueHandle* owner1); - -// Create and graph-retain a replacement attachment before a CUDA mutation. -// Destruction rolls the prepared attachment back unless it is committed. -CUresult graph_prepare_attachment( - const GraphHandle& h_graph, - OpaqueHandle owner0, - OpaqueHandle owner1, - PreparedAttachment* out_prepared); - -// Publish a prepared attachment after the CUDA mutation succeeds. A null node -// retains the attachment anonymously without publishing node metadata. -CUresult graph_commit_attachment( - PreparedAttachment& prepared, - CUgraphNode node); - -// Copy attachment metadata from a source graph hierarchy into its CUDA clone. -CUresult graph_clone_attachments( - const GraphHandle& h_clone, - const GraphHandle& h_source); - -// Stage a complete metadata replacement before CUDA replaces an embedded -// graph. Dropping the prepared state leaves the current hierarchy unchanged. -CUresult graph_prepare_child_graph_update( - const GraphHandle& h_parent, - const GraphHandle& h_old_child, - CUgraphNode owner_node, - const GraphHandle& h_source, - PreparedChildGraphUpdate* out_prepared); - -// Rekey staged metadata to CUDA's replacement clone, retire the old embedded -// hierarchy, and publish the replacement handle. -CUresult graph_commit_child_graph_update( - PreparedChildGraphUpdate& prepared, - GraphHandle* out_child); - -// Invalidate cuda.core state for child graphs CUDA destroyed with owner_node. -void invalidate_child_graph_state( - const GraphHandle& h_parent, - CUgraphNode owner_node) noexcept; - -// ============================================================================ -// Graph exec handle functions -// ============================================================================ - -// Create an owning exec handle by calling cuGraphInstantiateWithParams. -// A fresh attachment accumulator is retained on h_source first, because CUDA -// propagates user object references only at instantiation; an exec cannot -// receive them afterwards. The exec is the sole owner once this returns. -// When the last reference is released, cuGraphExecDestroy is called -// automatically. -// Returns empty handle on error (caller must check). The caller reads -// params->result_out for the specific instantiation failure and -// get_last_error() for a driver status. -GraphExecHandle create_graph_exec_handle( - const GraphHandle& h_source, - CUDA_GRAPH_INSTANTIATE_PARAMS* params); - -// Update h_exec in place by calling cuGraphExecUpdate, and publish a fresh -// accumulator when CUDA accepts the update. Writes result_info for the caller. -CUresult graph_exec_update( - const GraphExecHandle& h_exec, - const GraphHandle& h_source, - CUgraphExecUpdateResultInfo* result_info); - -// Append owners before an executable-node mutation. The accumulator grows -// because CUDA cannot attach user objects to an exec after instantiation, so -// old owners stay reachable. Dropping the transaction restores the accumulator -// to its original size. -CUresult graph_prepare_exec_attachment( - const GraphExecHandle& h_exec, - OpaqueHandle owner0, - OpaqueHandle owner1, - PreparedExecAttachment* out_prepared); - -// Keep the owners added by graph_prepare_exec_attachment. -void graph_commit_exec_attachment( - PreparedExecAttachment& prepared) noexcept; - -// ============================================================================ -// Graph node handle functions -// ============================================================================ - -// Create a node handle. Nodes are owned by their parent graph (not -// independently destroyable). The GraphHandle dependency ensures the -// graph outlives any node reference. -GraphNodeHandle create_graph_node_handle(CUgraphNode node, const GraphHandle& h_graph); - -// Extract the owning graph handle from a node handle. -GraphHandle graph_node_get_graph(const GraphNodeHandle& h) noexcept; - -// Zero the CUgraphNode resource inside the handle, marking it invalid. -void invalidate_graph_node(const GraphNodeHandle& h) noexcept; - -// ============================================================================ -// Graphics resource handle functions -// ============================================================================ - -// Create an owning graphics resource handle. -// When the last reference is released, cuGraphicsUnregisterResource is called automatically. -// Use for CUgraphicsResource handles obtained from cuGraphicsGLRegisterBuffer etc. -GraphicsResourceHandle create_graphics_resource_handle(CUgraphicsResource resource); - -// ============================================================================ -// NVRTC Program handle functions -// ============================================================================ - -// Create an owning NVRTC program handle. -// When the last reference is released, nvrtcDestroyProgram is called. -// Use this to wrap a program created via nvrtcCreateProgram. -NvrtcProgramHandle create_nvrtc_program_handle(nvrtcProgram prog); - -// Create a non-owning NVRTC program handle (references existing program). -// The program will NOT be destroyed when the handle is released. -NvrtcProgramHandle create_nvrtc_program_handle_ref(nvrtcProgram prog); - -// ============================================================================ -// NVVM Program handle functions -// ============================================================================ - -// Create an owning NVVM program handle. -// When the last reference is released, nvvmDestroyProgram is called. -// Use this to wrap a program created via nvvmCreateProgram. -// Note: If NVVM is not available (p_nvvmDestroyProgram is null), the deleter is a no-op. -NvvmProgramHandle create_nvvm_program_handle(nvvmProgram prog); - -// Create a non-owning NVVM program handle (references existing program). -// The program will NOT be destroyed when the handle is released. -NvvmProgramHandle create_nvvm_program_handle_ref(nvvmProgram prog); - -// ============================================================================ -// nvJitLink handle functions -// ============================================================================ - -// Create an owning nvJitLink handle. -// When the last reference is released, nvJitLinkDestroy is called. -// Use this to wrap a handle created via nvJitLinkCreate. -// Note: If nvJitLink is not available (p_nvJitLinkDestroy is null), the deleter is a no-op. -NvJitLinkHandle create_nvjitlink_handle(nvJitLink_t handle); - -// Create a non-owning nvJitLink handle (references existing handle). -// The handle will NOT be destroyed when the last reference is released. -NvJitLinkHandle create_nvjitlink_handle_ref(nvJitLink_t handle); - -// ============================================================================ -// cuLink handle functions -// ============================================================================ - -// Create an owning cuLink handle. -// When the last reference is released, cuLinkDestroy is called. -// Use this to wrap a CUlinkState created via cuLinkCreate. -CuLinkHandle create_culink_handle(CUlinkState state); - -// Create a non-owning cuLink handle (references existing CUlinkState). -// The handle will NOT be destroyed when the last reference is released. -CuLinkHandle create_culink_handle_ref(CUlinkState state); - -// ============================================================================ -// File descriptor handle functions -// ============================================================================ - -// Create an owning file descriptor handle. -// When the last reference is released, POSIX close() is called. -FileDescriptorHandle create_fd_handle(int fd); - -// Create a non-owning file descriptor handle (caller manages the fd). -FileDescriptorHandle create_fd_handle_ref(int fd); - -// ============================================================================ -// Array / mipmapped-array / texture / surface handle functions (PR #467) -// -// These resources are managed exactly like every other cuda.core resource: -// the owning handle's deleter calls the matching cu*Destroy with the GIL -// released, structural dependencies are embedded in the box (so a backing -// resource always outlives a texture/surface/level built on it), and -// creation returns an empty handle + thread-local error on failure. -// ============================================================================ - -// Create an owning CUDA array via cuArray3DCreate. -// When the last reference is released, cuArrayDestroy is called automatically. -// Returns empty handle on error (caller must check). -OpaqueArrayHandle create_array_handle(const ContextHandle& h_context, const CUDA_ARRAY3D_DESCRIPTOR& desc); - -// Create a non-owning array handle (references an existing CUarray). -// Use for arrays owned elsewhere (e.g. graphics interop). Never destroyed here. -OpaqueArrayHandle create_array_handle_ref(CUarray arr); - -// Create an owning array handle adopting an existing CUarray. -// When the last reference is released, cuArrayDestroy is called automatically. -OpaqueArrayHandle create_array_handle_owning(CUarray arr); - -// Return the context dependency associated with an array, if known. -ContextHandle get_array_context(const OpaqueArrayHandle& h) noexcept; - -// Create a non-owning handle to a mipmap level via cuMipmappedArrayGetLevel. -// The level CUarray is owned by the mipmap; the parent MipmappedArrayHandle is -// embedded in the box so it outlives the level view. No destroy in the deleter. -// Returns empty handle on error (caller must check). -OpaqueArrayHandle create_array_level_handle(const MipmappedArrayHandle& h_mip, unsigned int level); - -// Create an owning mipmapped array via cuMipmappedArrayCreate. -// When the last reference is released, cuMipmappedArrayDestroy is called. -// Returns empty handle on error (caller must check). -MipmappedArrayHandle create_mipmapped_array_handle(const ContextHandle& h_context, - const CUDA_ARRAY3D_DESCRIPTOR& desc, - unsigned int num_levels); - -// Return the context dependency associated with a mipmapped array, if known. -ContextHandle get_mipmapped_array_context(const MipmappedArrayHandle& h) noexcept; - -// Create an owning texture object via cuTexObjectCreate, embedding the backing -// resource handle (array / mipmapped array / linear-or-pitch2d device pointer) -// so the backing always outlives the texture. cuTexObjectDestroy runs in the -// deleter. Returns empty handle on error (caller must check). -TexObjectHandle create_tex_object_handle_array(const ContextHandle& h_context, - const CUDA_RESOURCE_DESC& res, - const CUDA_TEXTURE_DESC& tex, - const OpaqueArrayHandle& h_backing); -TexObjectHandle create_tex_object_handle_mipmap(const ContextHandle& h_context, - const CUDA_RESOURCE_DESC& res, - const CUDA_TEXTURE_DESC& tex, - const MipmappedArrayHandle& h_backing); -TexObjectHandle create_tex_object_handle_linear(const ContextHandle& h_context, - const CUDA_RESOURCE_DESC& res, - const CUDA_TEXTURE_DESC& tex, - const DevicePtrHandle& h_backing); - -// Create an owning surface object via cuSurfObjectCreate, embedding the backing -// array handle so it outlives the surface. cuSurfObjectDestroy runs in the -// deleter. Returns empty handle on error (caller must check). -SurfObjectHandle create_surf_object_handle(const ContextHandle& h_context, - const CUDA_RESOURCE_DESC& res, - const OpaqueArrayHandle& h_backing); - -// ============================================================================ -// Overloaded helper functions to extract raw resources from handles -// ============================================================================ - -// as_cu() - extract the raw CUDA handle -inline CUcontext as_cu(const ContextHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUgreenCtx as_cu(const GreenCtxHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUstream as_cu(const StreamHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUevent as_cu(const EventHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUmemoryPool as_cu(const MemoryPoolHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUdeviceptr as_cu(const DevicePtrHandle& h) noexcept { - return h ? *h : 0; -} - -inline CUlibrary as_cu(const LibraryHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUmodule as_cu(const CUmodule& h) noexcept { - return h; -} - -inline CUkernel as_cu(const KernelHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUgraph as_cu(const GraphHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUgraphExec as_cu(const GraphExecHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUgraphNode as_cu(const GraphNodeHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUgraphicsResource as_cu(const GraphicsResourceHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline nvrtcProgram as_cu(const NvrtcProgramHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline nvvmProgram as_cu(const NvvmProgramHandle& h) noexcept { - return h ? h->raw : nullptr; -} - -inline nvJitLink_t as_cu(const NvJitLinkHandle& h) noexcept { - return h ? h->raw : nullptr; -} - -inline CUlinkState as_cu(const CuLinkHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUarray as_cu(const OpaqueArrayHandle& h) noexcept { - return h ? *h : nullptr; -} - -inline CUmipmappedArray as_cu(const MipmappedArrayHandle& h) noexcept { - return h ? *h : nullptr; -} - -// CUtexObject / CUsurfObject are integer-valued (like CUdeviceptr); null is 0. -// The raw value lives in the tagged wrapper's `raw` field. -inline CUtexObject as_cu(const TexObjectHandle& h) noexcept { - return h ? h->raw : 0; -} - -inline CUsurfObject as_cu(const SurfObjectHandle& h) noexcept { - return h ? h->raw : 0; -} - -// as_intptr() - extract handle as intptr_t for Python interop -// Using signed intptr_t per C standard convention and issue #1342 -inline std::intptr_t as_intptr(const ContextHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const GreenCtxHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const StreamHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const EventHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const MemoryPoolHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const DevicePtrHandle& h) noexcept { - return static_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const LibraryHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const CUmodule& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const KernelHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const GraphHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const GraphExecHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const GraphNodeHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const GraphicsResourceHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const NvrtcProgramHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const NvvmProgramHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const NvJitLinkHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const CuLinkHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const FileDescriptorHandle& h) noexcept { - return h ? static_cast(*h) : -1; -} - -inline std::intptr_t as_intptr(const OpaqueArrayHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const MipmappedArrayHandle& h) noexcept { - return reinterpret_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const TexObjectHandle& h) noexcept { - return static_cast(as_cu(h)); -} - -inline std::intptr_t as_intptr(const SurfObjectHandle& h) noexcept { - return static_cast(as_cu(h)); -} - -// as_py() - convert handle to Python wrapper object (returns new reference) -#if PY_VERSION_HEX < 0x030D0000 -extern "C" int _Py_IsFinalizing(void); -#endif - -// Best-effort probe for interpreter shutdown. -// -// In CPython this is not a hard guarantee: finalization can begin after this -// returns false but before a later PyGILState_Ensure() or other Python C-API -// call. -// -// If that race is lost on a non-finalizer thread, CPython's behavior is -// version-dependent: on older supported versions (3.10-3.13) it may abruptly -// terminate the current thread (historically via PyThread_exit_thread(), -// without normal C++ unwinding), while on newer versions (3.14+) it may hang -// the thread until process exit. -// -// We still use this check because the policy in this layer is to avoid Python -// work once shutdown is underway and accept an intentional leak or skipped -// Python conversion in that edge case rather than add more complex deferral -// machinery. -inline bool py_is_finalizing() noexcept { -#if PY_VERSION_HEX >= 0x030D0000 - return Py_IsFinalizing(); -#else - return _Py_IsFinalizing() != 0; -#endif -} - -namespace detail { -// n.b. class lookup is not cached to avoid deadlock hazard, see DESIGN.md -inline PyObject* make_py(const char* module_name, const char* class_name, std::intptr_t value) noexcept { - if (py_is_finalizing()) { - Py_RETURN_NONE; - } - PyObject* mod = PyImport_ImportModule(module_name); - if (!mod) return nullptr; - PyObject* cls = PyObject_GetAttrString(mod, class_name); - Py_DECREF(mod); - if (!cls) return nullptr; - PyObject* result = PyObject_CallFunction(cls, "L", value); - Py_DECREF(cls); - return result; -} -} // namespace detail - -inline PyObject* as_py(const ContextHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUcontext", as_intptr(h)); -} - -inline PyObject* as_py(const GreenCtxHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUgreenCtx", as_intptr(h)); -} - -inline PyObject* as_py(const StreamHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUstream", as_intptr(h)); -} - -inline PyObject* as_py(const EventHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUevent", as_intptr(h)); -} - -inline PyObject* as_py(const MemoryPoolHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUmemoryPool", as_intptr(h)); -} - -inline PyObject* as_py(const DevicePtrHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUdeviceptr", as_intptr(h)); -} - -inline PyObject* as_py(const LibraryHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUlibrary", as_intptr(h)); -} - -inline PyObject* as_py(const CUmodule& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUmodule", as_intptr(h)); -} - -inline PyObject* as_py(const KernelHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUkernel", as_intptr(h)); -} - -inline PyObject* as_py(const GraphHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUgraph", as_intptr(h)); -} - -inline PyObject* as_py(const GraphExecHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUgraphExec", as_intptr(h)); -} - -inline PyObject* as_py(const GraphNodeHandle& h) noexcept { - if (!as_intptr(h)) { - Py_RETURN_NONE; - } - return detail::make_py("cuda.bindings.driver", "CUgraphNode", as_intptr(h)); -} - -inline PyObject* as_py(const NvrtcProgramHandle& h) noexcept { - return detail::make_py("cuda.bindings.nvrtc", "nvrtcProgram", as_intptr(h)); -} - -inline PyObject* as_py(const NvvmProgramHandle& h) noexcept { - // NVVM bindings use raw integers, not wrapper classes - return PyLong_FromSsize_t(as_intptr(h)); -} - -inline PyObject* as_py(const NvJitLinkHandle& h) noexcept { - // nvJitLink bindings use raw integers, not wrapper classes - return PyLong_FromSsize_t(as_intptr(h)); -} - -inline PyObject* as_py(const CuLinkHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUlinkState", as_intptr(h)); -} - -inline PyObject* as_py(const GraphicsResourceHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUgraphicsResource", as_intptr(h)); -} - -inline PyObject* as_py(const FileDescriptorHandle& h) noexcept { - return PyLong_FromSsize_t(as_intptr(h)); -} - -inline PyObject* as_py(const OpaqueArrayHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUarray", as_intptr(h)); -} - -inline PyObject* as_py(const MipmappedArrayHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUmipmappedArray", as_intptr(h)); -} - -inline PyObject* as_py(const TexObjectHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUtexObject", as_intptr(h)); -} - -inline PyObject* as_py(const SurfObjectHandle& h) noexcept { - return detail::make_py("cuda.bindings.driver", "CUsurfObject", as_intptr(h)); -} - -// ============================================================================ -// SM resource split wrapper (13.1+) -// -// Calls through p_cuDevSmResourceSplit if available, otherwise returns -// CUDA_ERROR_NOT_SUPPORTED. This avoids a direct Cython cimport of the -// cydriver cdef function, which would fail at module init on cuda-bindings -// < 13.1 (see https://github.com/NVIDIA/cuda-python/issues/2063). -// ============================================================================ - -// groupParams is void* so the Cython declaration doesn't reference -// CU_DEV_SM_RESOURCE_GROUP_PARAMS (absent from cuda-bindings 13.0 .pxd). -CUresult sm_resource_split(CUdevResource* result, unsigned int nbGroups, - const CUdevResource* input, CUdevResource* remainder, - unsigned int flags, void* groupParams); - -// Returns true if the cuDevSmResourceSplit function pointer is available. -bool has_sm_resource_split() noexcept; - -// ============================================================================ -// cuMemcpyWithAttributesAsync wrapper (13.2+) -// -// Calls through p_cuMemcpyWithAttributesAsync if available, otherwise returns -// CUDA_ERROR_NOT_SUPPORTED. This avoids a direct Cython cimport of the -// cydriver cdef function, which would fail at module init on cuda-bindings -// < 13.2 (see https://github.com/NVIDIA/cuda-python/issues/2063). -// ============================================================================ - -// attr is void* so the Cython declaration doesn't reference CUmemcpyAttributes -// (absent from cuda-bindings built against CUDA < 12.8). The C++ side casts it. -CUresult memcpy_with_attributes_async(CUdeviceptr dst, CUdeviceptr src, size_t size, - void* attr, CUstream hStream); - -// Returns true if the cuMemcpyWithAttributesAsync function pointer is available. -bool has_memcpy_with_attributes_async() noexcept; - -} // namespace cuda_core::rt +#include "py.hpp" +#include "types.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "api.hpp" diff --git a/cuda_core/cuda/core/_cpp/rt/types.hpp b/cuda_core/cuda/core/_cpp/rt/types.hpp new file mode 100644 index 00000000000..59389f78fca --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/types.hpp @@ -0,0 +1,294 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include + +// Forward declaration for NVVM - avoids nvvm.h dependency +// Use void* to match cuda.bindings.cynvvm's typedef +using nvvmProgram = void*; + +// Forward declaration for nvJitLink - avoids nvJitLink.h dependency +// Use void* to match cuda.bindings.cynvjitlink's typedef +using nvJitLink_t = void*; + +namespace cuda_core::rt { + +// ============================================================================ +// TaggedHandle - make void*-based handle types distinct for overloading +// +// Both nvvmProgram and nvJitLink_t are void*, so shared_ptr +// would be the same C++ type for both. TaggedHandle wraps the raw +// value with a unique tag type, making each shared_ptr type distinct. +// ============================================================================ + +template +struct TaggedHandle { + T raw; +}; + +using NvvmProgramValue = TaggedHandle; +using NvJitLinkValue = TaggedHandle; + +// CUtexObject, CUsurfObject and CUdeviceptr are all `unsigned long long`, so +// shared_ptr et al. would be the *same* C++ type as +// DevicePtrHandle (and each other), collapsing the as_cu/as_intptr/as_py +// overload sets. Tag them to keep each handle type distinct, exactly as the +// NVVM / nvJitLink handles above do. +using TexObjectValue = TaggedHandle; +using SurfObjectValue = TaggedHandle; + +// ============================================================================ +// Handle type aliases - expose only the raw CUDA resource +// ============================================================================ + +using ContextHandle = std::shared_ptr; +using GreenCtxHandle = std::shared_ptr; +using StreamHandle = std::shared_ptr; +using EventHandle = std::shared_ptr; +using MemoryPoolHandle = std::shared_ptr; +using LibraryHandle = std::shared_ptr; +using KernelHandle = std::shared_ptr; +using GraphHandle = std::shared_ptr; +using GraphExecHandle = std::shared_ptr; +using GraphNodeHandle = std::shared_ptr; +using GraphicsResourceHandle = std::shared_ptr; +using NvrtcProgramHandle = std::shared_ptr; +using NvvmProgramHandle = std::shared_ptr; +using NvJitLinkHandle = std::shared_ptr; +using CuLinkHandle = std::shared_ptr; +using FileDescriptorHandle = std::shared_ptr; +using OpaqueArrayHandle = std::shared_ptr; +using MipmappedArrayHandle = std::shared_ptr; +using TexObjectHandle = std::shared_ptr; +using SurfObjectHandle = std::shared_ptr; + +using DevicePtrHandle = std::shared_ptr; + +// Type-erased shared owner of an attached resource. Typed handles such as +// EventHandle and KernelHandle convert to OpaqueHandle by assignment, reusing +// their existing control block; the helpers below build OpaqueHandles for the +// two cases that need a custom deleter. +using OpaqueHandle = std::shared_ptr; + +struct PreparedAttachmentState; +using PreparedAttachmentRollback = + void (*)(PreparedAttachmentState*) noexcept; +struct PreparedAttachmentDeleter { + PreparedAttachmentRollback rollback = nullptr; + + void operator()(PreparedAttachmentState* state) const noexcept { + rollback(state); + } +}; +using PreparedAttachment = + std::unique_ptr; + +struct PreparedChildGraphUpdateState; +// Opaque unpublished hierarchy transaction; releasing it discards staged +// metadata unless graph_commit_child_graph_update publishes the replacement. +using PreparedChildGraphUpdate = + std::shared_ptr; + +struct PreparedExecAttachmentState; +using PreparedExecAttachmentRollback = + void (*)(PreparedExecAttachmentState*) noexcept; +struct PreparedExecAttachmentDeleter { + PreparedExecAttachmentRollback rollback = nullptr; + + void operator()(PreparedExecAttachmentState* state) const noexcept { + rollback(state); + } +}; +// Opaque append transaction. Releasing it rolls back newly appended owners +// unless graph_commit_exec_attachment has kept them. +using PreparedExecAttachment = + std::unique_ptr; + +// ============================================================================ +// Overloaded helper functions to extract raw resources from handles +// ============================================================================ + +// as_cu() - extract the raw CUDA handle +inline CUcontext as_cu(const ContextHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUgreenCtx as_cu(const GreenCtxHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUstream as_cu(const StreamHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUevent as_cu(const EventHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUmemoryPool as_cu(const MemoryPoolHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUdeviceptr as_cu(const DevicePtrHandle& h) noexcept { + return h ? *h : 0; +} + +inline CUlibrary as_cu(const LibraryHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUmodule as_cu(const CUmodule& h) noexcept { + return h; +} + +inline CUkernel as_cu(const KernelHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUgraph as_cu(const GraphHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUgraphExec as_cu(const GraphExecHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUgraphNode as_cu(const GraphNodeHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUgraphicsResource as_cu(const GraphicsResourceHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline nvrtcProgram as_cu(const NvrtcProgramHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline nvvmProgram as_cu(const NvvmProgramHandle& h) noexcept { + return h ? h->raw : nullptr; +} + +inline nvJitLink_t as_cu(const NvJitLinkHandle& h) noexcept { + return h ? h->raw : nullptr; +} + +inline CUlinkState as_cu(const CuLinkHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUarray as_cu(const OpaqueArrayHandle& h) noexcept { + return h ? *h : nullptr; +} + +inline CUmipmappedArray as_cu(const MipmappedArrayHandle& h) noexcept { + return h ? *h : nullptr; +} + +// CUtexObject / CUsurfObject are integer-valued (like CUdeviceptr); null is 0. +// The raw value lives in the tagged wrapper's `raw` field. +inline CUtexObject as_cu(const TexObjectHandle& h) noexcept { + return h ? h->raw : 0; +} + +inline CUsurfObject as_cu(const SurfObjectHandle& h) noexcept { + return h ? h->raw : 0; +} + +// as_intptr() - extract handle as intptr_t for Python interop +// Using signed intptr_t per C standard convention and issue #1342 +inline std::intptr_t as_intptr(const ContextHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const GreenCtxHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const StreamHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const EventHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const MemoryPoolHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const DevicePtrHandle& h) noexcept { + return static_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const LibraryHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const CUmodule& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const KernelHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const GraphHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const GraphExecHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const GraphNodeHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const GraphicsResourceHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const NvrtcProgramHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const NvvmProgramHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const NvJitLinkHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const CuLinkHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const FileDescriptorHandle& h) noexcept { + return h ? static_cast(*h) : -1; +} + +inline std::intptr_t as_intptr(const OpaqueArrayHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const MipmappedArrayHandle& h) noexcept { + return reinterpret_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const TexObjectHandle& h) noexcept { + return static_cast(as_cu(h)); +} + +inline std::intptr_t as_intptr(const SurfObjectHandle& h) noexcept { + return static_cast(as_cu(h)); +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_rt.pxd b/cuda_core/cuda/core/_rt.pxd index d93bc5e2bee..e0772a70444 100644 --- a/cuda_core/cuda/core/_rt.pxd +++ b/cuda_core/cuda/core/_rt.pxd @@ -18,7 +18,7 @@ from cuda.bindings cimport cynvjitlink # Handle type aliases and inline helpers (declared from C++ header) # ============================================================================= -cdef extern from "_cpp/rt/rt.hpp" namespace "cuda_core::rt": +cdef extern from "_cpp/rt/handles.hpp" namespace "cuda_core::rt": # Handle types ctypedef shared_ptr[const cydriver.CUcontext] ContextHandle ctypedef shared_ptr[const cydriver.CUgreenCtx] GreenCtxHandle From c1f90e85a9f86bd1b98c00015f966632bbeec2f1 Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Fri, 11 Sep 2026 10:59:10 -0700 Subject: [PATCH 12/14] cuda.core: split rt.cpp into the _cpp/rt/ sources The monolithic source becomes twelve translation units: one resource family per file (context, stream, event, memory, program, graph, graph_exec, texture), the driver table with its version-gated shims (driver_api.cpp), the error state and non-propagating reporting (error.cpp), and the two Python-coupled bodies (py_report.cpp, py_deferred_cleanup.cpp). Two headers hold the helpers the files share and Cython never names, in namespace cuda_core::rt::detail: context_scope.hpp (enter/restore/exit_context and the invoke_in_context templates) and internal.hpp (HandleRegistry, WarnOnFailure with the pw_* wrappers, DeallocationStream, DeferredCleanupItem and the declarations of the promoted helpers). py.hpp gains the GIL guards; error.hpp declares the thread-local `err` that error.cpp now defines. Every definition moves verbatim. Helpers that were static or in an anonymous namespace and are now called across files become external with a declaration; everything local to one file keeps its anonymous namespace. driver_api.cpp and error.cpp compile without a Python include path. Verification: the generator's check mode reproduces all 21 files from the monoliths; the dynamic symbol table gains exactly the nine promoted detail:: functions and the err object and loses nothing; __pyx_capi__ is unchanged. tests/test_rt_layout.py pins the layout rules. --- cuda_core/cuda/core/_cpp/rt/context.cpp | 267 ++ cuda_core/cuda/core/_cpp/rt/context_scope.hpp | 114 + cuda_core/cuda/core/_cpp/rt/driver_api.cpp | 164 + cuda_core/cuda/core/_cpp/rt/error.cpp | 142 + cuda_core/cuda/core/_cpp/rt/error.hpp | 4 + cuda_core/cuda/core/_cpp/rt/event.cpp | 142 + cuda_core/cuda/core/_cpp/rt/graph.cpp | 758 ++++ cuda_core/cuda/core/_cpp/rt/graph_exec.cpp | 293 ++ cuda_core/cuda/core/_cpp/rt/internal.hpp | 176 + cuda_core/cuda/core/_cpp/rt/memory.cpp | 497 +++ cuda_core/cuda/core/_cpp/rt/program.cpp | 243 ++ cuda_core/cuda/core/_cpp/rt/py.hpp | 55 + .../cuda/core/_cpp/rt/py_deferred_cleanup.cpp | 157 + cuda_core/cuda/core/_cpp/rt/py_report.cpp | 100 + cuda_core/cuda/core/_cpp/rt/rt.cpp | 3349 ----------------- cuda_core/cuda/core/_cpp/rt/stream.cpp | 232 ++ cuda_core/cuda/core/_cpp/rt/texture.cpp | 265 ++ cuda_core/tests/test_rt_layout.py | 111 + 18 files changed, 3720 insertions(+), 3349 deletions(-) create mode 100644 cuda_core/cuda/core/_cpp/rt/context.cpp create mode 100644 cuda_core/cuda/core/_cpp/rt/context_scope.hpp create mode 100644 cuda_core/cuda/core/_cpp/rt/driver_api.cpp create mode 100644 cuda_core/cuda/core/_cpp/rt/error.cpp create mode 100644 cuda_core/cuda/core/_cpp/rt/event.cpp create mode 100644 cuda_core/cuda/core/_cpp/rt/graph.cpp create mode 100644 cuda_core/cuda/core/_cpp/rt/graph_exec.cpp create mode 100644 cuda_core/cuda/core/_cpp/rt/internal.hpp create mode 100644 cuda_core/cuda/core/_cpp/rt/memory.cpp create mode 100644 cuda_core/cuda/core/_cpp/rt/program.cpp create mode 100644 cuda_core/cuda/core/_cpp/rt/py_deferred_cleanup.cpp create mode 100644 cuda_core/cuda/core/_cpp/rt/py_report.cpp delete mode 100644 cuda_core/cuda/core/_cpp/rt/rt.cpp create mode 100644 cuda_core/cuda/core/_cpp/rt/stream.cpp create mode 100644 cuda_core/cuda/core/_cpp/rt/texture.cpp create mode 100644 cuda_core/tests/test_rt_layout.py diff --git a/cuda_core/cuda/core/_cpp/rt/context.cpp b/cuda_core/cuda/core/_cpp/rt/context.cpp new file mode 100644 index 00000000000..cd60db109bb --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/context.cpp @@ -0,0 +1,267 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "context_scope.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +namespace { +// Thread-local fault injected into the next context restoration (tests only). +thread_local CUresult context_restore_fault = CUDA_SUCCESS; +} // namespace + +void set_context_restore_fault_for_testing(CUresult status) noexcept { + context_restore_fault = status; +} + +namespace detail { +// Make a context current and record the state needed to restore it. +// An empty handle is a no-op: the operation runs in the caller's current +// context, and nothing is restored on exit. invoke_in_context and +// invoke_in_context_or_undo reject empty handles before getting here; only +// graph_node_set_params relies on the no-op (pre-13.2 node updates run in the +// caller's context). +CUresult enter_context(const ContextHandle& h_context, CUcontext* previous, int* changed) noexcept { + *previous = nullptr; + *changed = 0; + clear_last_error_detail(); + CUcontext target = as_cu(h_context); + if (!target) { + return CUDA_SUCCESS; + } + + GILReleaseGuard gil; + CUresult status = p_cuCtxGetCurrent(previous); + if (status != CUDA_SUCCESS || *previous == target) { + return status; + } + status = p_cuCtxSetCurrent(target); + *changed = status == CUDA_SUCCESS; + return status; +} + +// Restore the caller's context. Returns the restoration status. +CUresult restore_context(CUcontext previous) noexcept { + if (context_restore_fault != CUDA_SUCCESS) { + // Test hook: behave as if cuCtxSetCurrent(previous) failed, leaving the + // target context current exactly as a real failure would. + CUresult fault = context_restore_fault; + context_restore_fault = CUDA_SUCCESS; + return fault; + } + GILReleaseGuard gil; + return p_cuCtxSetCurrent(previous); +} +// Restore the previous context and preserve an earlier operation error. The +// operation error, if any, is returned; otherwise the restoration status is. +// Either way a restoration failure is recorded as the detail of the returned +// status, so the eventual CUDAError explains it (see take_last_error_detail()). +CUresult exit_context(CUcontext previous, int changed, CUresult operation_status) noexcept { + CUresult restore_status = changed ? restore_context(previous) : CUDA_SUCCESS; + if (restore_status == CUDA_SUCCESS) { + return operation_status; + } + note_context_not_restored(previous, operation_status, restore_status); + return operation_status != CUDA_SUCCESS ? operation_status : restore_status; +} +} // namespace detail + +// Synchronize the provided context. +CUresult context_synchronize(const ContextHandle& h_context) noexcept { + GILReleaseGuard gil; + return invoke_in_context(h_context, []() noexcept { + return p_cuCtxSynchronize(); + }); +} + +// Query the stream priority range for the provided context. +CUresult context_get_stream_priority_range(const ContextHandle& h_context, + int* least_priority, + int* greatest_priority) noexcept { + GILReleaseGuard gil; + return invoke_in_context(h_context, [&]() noexcept { + return p_cuCtxGetStreamPriorityRange(least_priority, greatest_priority); + }); +} + +// Query the device of the provided context. +CUresult context_get_device(const ContextHandle& h_context, CUdevice* device) noexcept { + return invoke_in_context(h_context, [&]() noexcept { + return p_cuCtxGetDevice(device); + }); +} + +// ============================================================================ +// Context Handles +// ============================================================================ + +namespace { +struct ContextBox { + CUcontext resource; + GreenCtxHandle h_green_ctx; +}; + +struct GreenCtxBox { + CUgreenCtx resource; +}; + +static const ContextBox* get_box(const ContextHandle& h) noexcept { + const CUcontext* p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) - offsetof(ContextBox, resource) + ); +} + +// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) +static HandleRegistry context_registry; + +// Create a context handle reference, with optional green context as source. +ContextHandle create_context_handle_ref(CUcontext ctx, GreenCtxHandle h_green_ctx) { + if (!ctx) { + return {}; + } + if (auto h = context_registry.lookup(ctx)) { + return h; + } + auto box = std::shared_ptr( + new ContextBox{ctx, std::move(h_green_ctx)}, + [](const ContextBox* b) { + context_registry.unregister_handle(b->resource); + delete b; + } + ); + ContextHandle h(box, &box->resource); + context_registry.register_handle(ctx, h); + return h; +} +} // namespace + +ContextHandle create_context_handle_ref(CUcontext ctx) { + return create_context_handle_ref(ctx, {}); +} + +ContextHandle create_context_handle_from_green_ctx(const GreenCtxHandle& h_green_ctx) { + GILReleaseGuard gil; + if (!h_green_ctx) { + return {}; + } + if (!p_cuCtxFromGreenCtx) { + err = CUDA_ERROR_NOT_SUPPORTED; + return {}; + } + + CUcontext ctx = nullptr; + if (CUDA_SUCCESS != (err = p_cuCtxFromGreenCtx(&ctx, as_cu(h_green_ctx)))) { + return {}; + } + + return create_context_handle_ref(ctx, h_green_ctx); +} + +GreenCtxHandle get_context_green_ctx(const ContextHandle& h) noexcept { + if (!h) { + return {}; + } + return get_box(h)->h_green_ctx; +} + +GreenCtxHandle create_green_ctx_handle(CUdevResource* resources, unsigned int nbResources, + CUdevice dev, unsigned int flags) { + GILReleaseGuard gil; + if (!p_cuDevResourceGenerateDesc || !p_cuGreenCtxCreate || !p_cuGreenCtxDestroy) { + err = CUDA_ERROR_NOT_SUPPORTED; + return {}; + } + + CUdevResourceDesc desc = nullptr; + if (CUDA_SUCCESS != (err = p_cuDevResourceGenerateDesc(&desc, resources, nbResources))) { + return {}; + } + + CUgreenCtx green_ctx = nullptr; + if (CUDA_SUCCESS != (err = p_cuGreenCtxCreate(&green_ctx, desc, dev, flags))) { + return {}; + } + + auto box = std::shared_ptr( + new GreenCtxBox{green_ctx}, + [](const GreenCtxBox* b) { + GILReleaseGuard gil; + pw_cuGreenCtxDestroy(b->resource); + delete b; + } + ); + return GreenCtxHandle(box, &box->resource); +} + +GreenCtxHandle create_green_ctx_handle_ref(CUgreenCtx green_ctx) { + if (!green_ctx) { + return {}; + } + auto box = std::make_shared(GreenCtxBox{green_ctx}); + return GreenCtxHandle(box, &box->resource); +} + +// Thread-local cache of primary contexts indexed by device ID +static thread_local std::vector primary_context_cache; + +ContextHandle get_primary_context(int device_id) { + // Check thread-local cache + if (static_cast(device_id) < primary_context_cache.size()) { + if (auto cached = primary_context_cache[device_id]) { + return cached; + } + } + + // Cache miss - acquire primary context from driver + GILReleaseGuard gil; + CUcontext ctx; + if (CUDA_SUCCESS != (err = p_cuDevicePrimaryCtxRetain(&ctx, device_id))) { + return {}; + } + + auto box = std::shared_ptr( + new ContextBox{ctx, {}}, + [device_id](const ContextBox* b) { + context_registry.unregister_handle(b->resource); + GILReleaseGuard gil; + p_cuDevicePrimaryCtxRelease(device_id); + delete b; + } + ); + auto h = ContextHandle(box, &box->resource); + context_registry.register_handle(ctx, h); + + // Update cache + if (static_cast(device_id) >= primary_context_cache.size()) { + primary_context_cache.resize(device_id + 1); + } + primary_context_cache[device_id] = h; + return h; +} + +ContextHandle get_current_context() { + GILReleaseGuard gil; + CUcontext ctx = nullptr; + if (CUDA_SUCCESS != (err = p_cuCtxGetCurrent(&ctx))) { + return {}; + } + if (!ctx) { + return {}; // No current context (not an error) + } + return create_context_handle_ref(ctx); +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/context_scope.hpp b/cuda_core/cuda/core/_cpp/rt/context_scope.hpp new file mode 100644 index 00000000000..5ae22e15788 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/context_scope.hpp @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "types.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include +#include +#include + +namespace cuda_core::rt::detail { + +// Implemented in context.cpp +CUresult enter_context(const ContextHandle& h_context, CUcontext* previous, int* changed) noexcept; +// Implemented in context.cpp +CUresult restore_context(CUcontext previous) noexcept; +// Implemented in context.cpp +CUresult exit_context(CUcontext previous, int changed, CUresult operation_status) noexcept; + +// Require a callable to be invocable without throwing. +#define ASSERT_NOTHROW_INVOCABLE(...) \ + static_assert(std::is_nothrow_invocable_v<__VA_ARGS__>, "operation must be noexcept") + +// Run an operation with the requested context current. +template +CUresult invoke_in_context(const ContextHandle& h_context, Fn&& operation, Args&&... args) noexcept { + ASSERT_NOTHROW_INVOCABLE(Fn&&, Args&&...); + if (!h_context) { + return CUDA_ERROR_INVALID_CONTEXT; + } + CUcontext previous = nullptr; + int changed = 0; + CUresult status = enter_context(h_context, &previous, &changed); + if (status == CUDA_SUCCESS) { + status = std::invoke(std::forward(operation), std::forward(args)...); + } + return exit_context(previous, changed, status); +} + +// Run a creation operation and undo it if context restoration fails. +// Context-independent undo always runs. Context-sensitive undo runs only +// after verifying that the target context remains current; otherwise the +// resource leaks rather than risking cleanup in the wrong context. +template +CUresult invoke_in_context_or_undo(const ContextHandle& h_context, Fn&& operation, + Undo&& undo, bool undo_requires_target_context) noexcept { + ASSERT_NOTHROW_INVOCABLE(Fn&&); + ASSERT_NOTHROW_INVOCABLE(Undo&&); + if (!h_context) { + return CUDA_ERROR_INVALID_CONTEXT; + } + CUcontext previous = nullptr; + int changed = 0; + CUresult status = enter_context(h_context, &previous, &changed); + if (status != CUDA_SUCCESS) { + return status; + } + status = std::invoke(std::forward(operation)); + CUresult composite = exit_context(previous, changed, status); + if (status == CUDA_SUCCESS && composite != CUDA_SUCCESS) { + bool undo_ok = true; + if (undo_requires_target_context) { + CUcontext current = nullptr; + undo_ok = p_cuCtxGetCurrent(¤t) == CUDA_SUCCESS + && current == as_cu(h_context); + } + if (undo_ok) { + std::invoke(std::forward(undo)); + } else { + report_cuda_error( + "cuCtxSetCurrent (restoring the caller's context)", composite, + "failed; cleanup of the new resource skipped because its context " + "is no longer current (resource leaked)"); + } + } + return composite; +} + +// Run cleanup with the requested context current. Warn and skip the operation +// if activation fails, and independently warn on operation or restoration +// failure. Return the operation or activation status; restoration never +// changes the return value. +template +CUresult cleanup_in_context(const ContextHandle& h_context, const char* name, + Fn&& operation, Args&&... args) noexcept { + ASSERT_NOTHROW_INVOCABLE(Fn&&, Args&&...); + CUcontext previous = nullptr; + int changed = 0; + CUresult status = enter_context(h_context, &previous, &changed); + if (status != CUDA_SUCCESS) { + report_cuda_error(name, status, + "skipped (context activation failed; resource leaked)"); + } else { + status = std::invoke(std::forward(operation), std::forward(args)...); + if (status != CUDA_SUCCESS) { + report_cuda_error(name, status); + } + } + CUresult restore = exit_context(previous, changed, CUDA_SUCCESS); + if (restore != CUDA_SUCCESS) { + // Nothing is raised here, so the detail exit_context recorded has no + // exception to attach to: report it and drop the detail. + report_cuda_error(name, restore, "failed while restoring the caller's context"); + clear_last_error_detail(); + } + return status; +} + +#undef ASSERT_NOTHROW_INVOCABLE + +} // namespace cuda_core::rt::detail diff --git a/cuda_core/cuda/core/_cpp/rt/driver_api.cpp b/cuda_core/cuda/core/_cpp/rt/driver_api.cpp new file mode 100644 index 00000000000..860a29a3554 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/driver_api.cpp @@ -0,0 +1,164 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "driver_api.hpp" +#include +#include + +namespace cuda_core::rt { + +// ============================================================================ +// CUDA driver function pointers +// +// These are populated by _rt.pyx at module import time using +// function pointers extracted from cuda.bindings.cydriver.__pyx_capi__. +// ============================================================================ + +decltype(&cuGetErrorName) p_cuGetErrorName = nullptr; +decltype(&cuGetErrorString) p_cuGetErrorString = nullptr; + +decltype(&cuDevicePrimaryCtxRetain) p_cuDevicePrimaryCtxRetain = nullptr; +decltype(&cuDevicePrimaryCtxRelease) p_cuDevicePrimaryCtxRelease = nullptr; +decltype(&cuCtxGetCurrent) p_cuCtxGetCurrent = nullptr; +decltype(&cuCtxSetCurrent) p_cuCtxSetCurrent = nullptr; +decltype(&cuCtxSynchronize) p_cuCtxSynchronize = nullptr; +decltype(&cuCtxGetStreamPriorityRange) p_cuCtxGetStreamPriorityRange = nullptr; +decltype(&cuCtxGetDevice) p_cuCtxGetDevice = nullptr; +decltype(&cuGraphNodeSetParams) p_cuGraphNodeSetParams = nullptr; +decltype(&cuGreenCtxCreate) p_cuGreenCtxCreate = nullptr; +decltype(&cuGreenCtxDestroy) p_cuGreenCtxDestroy = nullptr; +decltype(&cuCtxFromGreenCtx) p_cuCtxFromGreenCtx = nullptr; +decltype(&cuDevResourceGenerateDesc) p_cuDevResourceGenerateDesc = nullptr; + +decltype(&cuGreenCtxStreamCreate) p_cuGreenCtxStreamCreate = nullptr; + +decltype(&cuStreamCreateWithPriority) p_cuStreamCreateWithPriority = nullptr; +decltype(&cuStreamDestroy) p_cuStreamDestroy = nullptr; +decltype(&cuStreamGetCtx) p_cuStreamGetCtx = nullptr; + +decltype(&cuEventCreate) p_cuEventCreate = nullptr; +decltype(&cuEventDestroy) p_cuEventDestroy = nullptr; +decltype(&cuIpcOpenEventHandle) p_cuIpcOpenEventHandle = nullptr; + +decltype(&cuDeviceGetCount) p_cuDeviceGetCount = nullptr; + +decltype(&cuMemPoolSetAccess) p_cuMemPoolSetAccess = nullptr; +decltype(&cuMemPoolDestroy) p_cuMemPoolDestroy = nullptr; +decltype(&cuMemPoolCreate) p_cuMemPoolCreate = nullptr; +decltype(&cuDeviceGetMemPool) p_cuDeviceGetMemPool = nullptr; +decltype(&cuMemPoolImportFromShareableHandle) p_cuMemPoolImportFromShareableHandle = nullptr; + +decltype(&cuMemAllocFromPoolAsync) p_cuMemAllocFromPoolAsync = nullptr; +decltype(&cuMemAllocAsync) p_cuMemAllocAsync = nullptr; +decltype(&cuMemAlloc) p_cuMemAlloc = nullptr; +decltype(&cuMemAllocHost) p_cuMemAllocHost = nullptr; + +decltype(&cuMemFreeAsync) p_cuMemFreeAsync = nullptr; +decltype(&cuMemFree) p_cuMemFree = nullptr; +decltype(&cuMemFreeHost) p_cuMemFreeHost = nullptr; + +decltype(&cuMemPoolImportPointer) p_cuMemPoolImportPointer = nullptr; + +decltype(&cuLibraryLoadFromFile) p_cuLibraryLoadFromFile = nullptr; +decltype(&cuLibraryLoadData) p_cuLibraryLoadData = nullptr; +decltype(&cuLibraryUnload) p_cuLibraryUnload = nullptr; +decltype(&cuLibraryGetKernel) p_cuLibraryGetKernel = nullptr; + +// Graph +decltype(&cuGraphDestroy) p_cuGraphDestroy = nullptr; +decltype(&cuGraphInstantiateWithParams) p_cuGraphInstantiateWithParams = nullptr; +decltype(&cuGraphExecUpdate) p_cuGraphExecUpdate = nullptr; +decltype(&cuGraphExecDestroy) p_cuGraphExecDestroy = nullptr; +decltype(&cuUserObjectCreate) p_cuUserObjectCreate = nullptr; +decltype(&cuUserObjectRelease) p_cuUserObjectRelease = nullptr; +decltype(&cuGraphRetainUserObject) p_cuGraphRetainUserObject = nullptr; +decltype(&cuGraphReleaseUserObject) p_cuGraphReleaseUserObject = nullptr; +decltype(&cuGraphNodeFindInClone) p_cuGraphNodeFindInClone = nullptr; +decltype(&cuGraphChildGraphNodeGetGraph) p_cuGraphChildGraphNodeGetGraph = nullptr; + +// Linker +decltype(&cuLinkDestroy) p_cuLinkDestroy = nullptr; + +// GL interop pointers +decltype(&cuGraphicsUnmapResources) p_cuGraphicsUnmapResources = nullptr; +decltype(&cuGraphicsUnregisterResource) p_cuGraphicsUnregisterResource = nullptr; + +decltype(&cuArray3DCreate) p_cuArray3DCreate = nullptr; +decltype(&cuArrayDestroy) p_cuArrayDestroy = nullptr; +decltype(&cuMipmappedArrayCreate) p_cuMipmappedArrayCreate = nullptr; +decltype(&cuMipmappedArrayDestroy) p_cuMipmappedArrayDestroy = nullptr; +decltype(&cuMipmappedArrayGetLevel) p_cuMipmappedArrayGetLevel = nullptr; +decltype(&cuTexObjectCreate) p_cuTexObjectCreate = nullptr; +decltype(&cuTexObjectDestroy) p_cuTexObjectDestroy = nullptr; +decltype(&cuSurfObjectCreate) p_cuSurfObjectCreate = nullptr; +decltype(&cuSurfObjectDestroy) p_cuSurfObjectDestroy = nullptr; + +// SM resource split (13.1+ — may be null on older drivers/bindings) +#if CUDA_VERSION >= 13010 +decltype(&cuDevSmResourceSplit) p_cuDevSmResourceSplit = nullptr; +#else +void* p_cuDevSmResourceSplit = nullptr; +#endif + +// cuMemcpyWithAttributesAsync (13.2+ — may be null on older drivers/bindings) +#if CUDA_VERSION >= 13020 +decltype(&cuMemcpyWithAttributesAsync) p_cuMemcpyWithAttributesAsync = nullptr; +#else +void* p_cuMemcpyWithAttributesAsync = nullptr; +#endif + +// NVRTC function pointers +decltype(&nvrtcDestroyProgram) p_nvrtcDestroyProgram = nullptr; + +// NVVM function pointers (may be null if NVVM is not available) +NvvmDestroyProgramFn p_nvvmDestroyProgram = nullptr; + +// nvJitLink function pointers (may be null if nvJitLink is not available) +NvJitLinkDestroyFn p_nvJitLinkDestroy = nullptr; + +// ============================================================================ +// SM resource split wrapper +// ============================================================================ + +CUresult sm_resource_split(CUdevResource* result, unsigned int nbGroups, + const CUdevResource* input, CUdevResource* remainder, + unsigned int flags, void* groupParams) { +#if CUDA_VERSION >= 13010 + if (!p_cuDevSmResourceSplit) { + return CUDA_ERROR_NOT_SUPPORTED; + } + return p_cuDevSmResourceSplit( + result, nbGroups, input, remainder, flags, + static_cast(groupParams)); +#else + return CUDA_ERROR_NOT_SUPPORTED; +#endif +} + +bool has_sm_resource_split() noexcept { + return p_cuDevSmResourceSplit != nullptr; +} + +// ============================================================================ +// cuMemcpyWithAttributesAsync wrapper +// ============================================================================ + +CUresult memcpy_with_attributes_async(CUdeviceptr dst, CUdeviceptr src, size_t size, + void* attr, CUstream hStream) { +#if CUDA_VERSION >= 13020 + if (!p_cuMemcpyWithAttributesAsync) { + return CUDA_ERROR_NOT_SUPPORTED; + } + return p_cuMemcpyWithAttributesAsync( + dst, src, size, static_cast(attr), hStream); +#else + return CUDA_ERROR_NOT_SUPPORTED; +#endif +} + +bool has_memcpy_with_attributes_async() noexcept { + return p_cuMemcpyWithAttributesAsync != nullptr; +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/error.cpp b/cuda_core/cuda/core/_cpp/rt/error.cpp new file mode 100644 index 00000000000..c0f7caf32d4 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/error.cpp @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "error.hpp" +#include "driver_api.hpp" +#include "internal.hpp" +#include +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +// ---------------------------------------------------------------------------- +// Non-propagating error reporting +// +// Deleters, CUDA callbacks and other non-propagating paths cannot raise. They +// report through report_cuda_error()/report_message(), which emit a +// cuda.core.CUDAWarning when the interpreter is usable and fall back to stderr +// otherwise. See docs/source/error_handling.rst for the policy. +// ---------------------------------------------------------------------------- + +namespace { +// Thread-local detail attached to the next raised CUDAError with a matching +// status (see take_last_error_detail()). Written only by propagating helpers. +// The taken copy stays valid until the next take on the same thread. +thread_local char last_error_detail[512] = {0}; +thread_local char taken_error_detail[512] = {0}; +thread_local CUresult last_error_detail_status = CUDA_SUCCESS; +} // namespace + +namespace detail { +// Format " : : " for a failed CUDA call. +void format_cuda_error(char* buffer, size_t size, const char* operation, CUresult status, + const char* detail) noexcept { + const char* error_name = nullptr; + const char* error_description = nullptr; + bool decoded = p_cuGetErrorName && p_cuGetErrorString + && p_cuGetErrorName(status, &error_name) == CUDA_SUCCESS + && p_cuGetErrorString(status, &error_description) == CUDA_SUCCESS; + const char* outcome = detail ? detail : "failed"; + if (decoded) { + std::snprintf(buffer, size, "%s %s: %s: %s", operation, outcome, error_name, error_description); + } else { + std::snprintf(buffer, size, "%s %s (CUDA error %d)", operation, outcome, static_cast(status)); + } +} +} // namespace detail + +// Report a failed non-CUDA call (NVRTC, NVVM, nvJitLink) from a path that +// cannot raise. +void report_status_code(const char* operation, long code) noexcept { + char message[256]; + std::snprintf(message, sizeof(message), "%s failed (status %ld)", operation, code); + report_message(message); +} + +// Report a failed CUDA call from a path that cannot raise. CUDA_ERROR_DEINITIALIZED +// is not reported: it means the driver is shutting down, which makes cleanup +// failures expected and uninteresting. +void report_cuda_error(const char* operation, CUresult status, const char* detail) noexcept { + if (status == CUDA_SUCCESS || status == CUDA_ERROR_DEINITIALIZED) { + return; + } + char message[512]; + format_cuda_error(message, sizeof(message), operation, status, detail); + report_message(message); +} + +const char* take_last_error_detail(CUresult status) noexcept { + if (!last_error_detail[0] || status != last_error_detail_status) { + return nullptr; + } + std::memcpy(taken_error_detail, last_error_detail, sizeof(taken_error_detail)); + clear_last_error_detail(); + return taken_error_detail; +} + +void clear_last_error_detail() noexcept { + last_error_detail[0] = 0; + last_error_detail_status = CUDA_SUCCESS; +} + +namespace detail { +// Record that the caller's context was not restored as the detail of the +// CUresult about to be returned and raised: the operation status if the +// operation failed too, else the restoration status. For a double failure the +// detail also names the restoration error, which the raised error does not. +void note_context_not_restored(CUcontext previous, CUresult operation_status, + CUresult restore_status) noexcept { + CUcontext current = nullptr; + if (p_cuCtxGetCurrent(¤t) != CUDA_SUCCESS) { + current = nullptr; + } + char cause[128] = {0}; + if (operation_status != CUDA_SUCCESS) { + const char* error_name = nullptr; + if (p_cuGetErrorName && p_cuGetErrorName(restore_status, &error_name) == CUDA_SUCCESS) { + std::snprintf(cause, sizeof(cause), " after this failure (cuCtxSetCurrent: %s)", error_name); + } else { + std::snprintf(cause, sizeof(cause), " after this failure (cuCtxSetCurrent: CUDA error %d)", + static_cast(restore_status)); + } + } + std::snprintf(last_error_detail, sizeof(last_error_detail), + "the calling thread's CUDA context (%#llx) could not be restored%s; " + "context %#llx is now current. Call Device.set_current() before issuing " + "further CUDA work on this thread", + static_cast(reinterpret_cast(previous)), + cause, + static_cast(reinterpret_cast(current))); + last_error_detail_status = operation_status != CUDA_SUCCESS ? operation_status : restore_status; +} +} // namespace detail + +// ============================================================================ +// Thread-local error handling +// ============================================================================ + +// Thread-local status of the most recent CUDA API call in this module. +thread_local CUresult err = CUDA_SUCCESS; + +// Return and clear the calling thread's most recent CUDA error. +CUresult get_last_error() noexcept { + CUresult e = err; + err = CUDA_SUCCESS; + return e; +} + +// Return the calling thread's most recent CUDA error without clearing it. +CUresult peek_last_error() noexcept { + return err; +} + +void clear_last_error() noexcept { + err = CUDA_SUCCESS; +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/error.hpp b/cuda_core/cuda/core/_cpp/rt/error.hpp index febeb6239ab..2be565084a6 100644 --- a/cuda_core/cuda/core/_cpp/rt/error.hpp +++ b/cuda_core/cuda/core/_cpp/rt/error.hpp @@ -21,6 +21,10 @@ CUresult peek_last_error() noexcept; // Explicitly clear the last error void clear_last_error() noexcept; +// Thread-local status of the most recent CUDA API call in this module. Defined +// in error.cpp; every family source writes it. +extern thread_local CUresult err; + // ============================================================================ // Non-propagating error reporting // diff --git a/cuda_core/cuda/core/_cpp/rt/event.cpp b/cuda_core/cuda/core/_cpp/rt/event.cpp new file mode 100644 index 00000000000..7514f8b8e41 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/event.cpp @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "context_scope.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +// ============================================================================ +// Event Handles +// ============================================================================ + +namespace { +struct EventBox { + CUevent resource; + bool timing_enabled; + bool is_blocking_sync; + bool ipc_enabled; + int device_id; + ContextHandle h_context; +}; +} // namespace + +static const EventBox* get_box(const EventHandle& h) { + const CUevent* p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) - offsetof(EventBox, resource) + ); +} + +bool get_event_timing_enabled(const EventHandle& h) noexcept { + return h ? get_box(h)->timing_enabled : false; +} + +bool get_event_is_blocking_sync(const EventHandle& h) noexcept { + return h ? get_box(h)->is_blocking_sync : false; +} + +bool get_event_ipc_enabled(const EventHandle& h) noexcept { + return h ? get_box(h)->ipc_enabled : false; +} + +int get_event_device_id(const EventHandle& h) noexcept { + return h ? get_box(h)->device_id : -1; +} + +// Return the context retained by an event handle. +ContextHandle get_event_context(const EventHandle& h) noexcept { + return h ? get_box(h)->h_context : ContextHandle{}; +} + +// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) +static HandleRegistry event_registry; + +EventHandle create_event_handle(const ContextHandle& h_ctx, unsigned int flags, + bool timing_enabled, bool is_blocking_sync, + bool ipc_enabled, int device_id) { + GILReleaseGuard gil; + CUevent event = nullptr; + err = invoke_in_context_or_undo( + h_ctx, + [&]() noexcept { return p_cuEventCreate(&event, flags); }, + [&]() noexcept { pw_cuEventDestroy(event); }, + /*undo_requires_target_context=*/false); + if (err != CUDA_SUCCESS) { + return {}; + } + + auto box = std::shared_ptr( + new EventBox{event, timing_enabled, is_blocking_sync, ipc_enabled, device_id, h_ctx}, + [](const EventBox* b) { + event_registry.unregister_handle(b->resource); + GILReleaseGuard gil; + pw_cuEventDestroy(b->resource); + delete b; + } + ); + EventHandle h(box, &box->resource); + event_registry.register_handle(event, h); + return h; +} + +EventHandle create_event_handle_for_stream(CUstream stream, unsigned int flags) { + // Resolve the stream's owning context (for default-stream tokens this is + // the current context, per cuStreamGetCtx) and create the event there, so + // it can be recorded on `stream` no matter which context is current. + CUcontext ctx = nullptr; + { + GILReleaseGuard gil; + err = p_cuStreamGetCtx(stream, &ctx); + } + if (err != CUDA_SUCCESS) { + return {}; + } + if (!ctx) { + err = CUDA_ERROR_INVALID_CONTEXT; + return {}; + } + return create_event_handle(create_context_handle_ref(ctx), flags, false, false, false, -1); +} + +EventHandle create_event_handle_ref(CUevent event) { + if (auto h = event_registry.lookup(event)) { + return h; + } + auto box = std::make_shared(EventBox{event, false, false, false, -1, {}}); + return EventHandle(box, &box->resource); +} + +EventHandle create_event_handle_ipc(const CUipcEventHandle& ipc_handle, + bool is_blocking_sync) { + GILReleaseGuard gil; + CUevent event; + if (CUDA_SUCCESS != (err = p_cuIpcOpenEventHandle(&event, ipc_handle))) { + return {}; + } + + auto box = std::shared_ptr( + new EventBox{event, false, is_blocking_sync, true, -1, {}}, + [](const EventBox* b) { + event_registry.unregister_handle(b->resource); + GILReleaseGuard gil; + pw_cuEventDestroy(b->resource); + delete b; + } + ); + EventHandle h(box, &box->resource); + event_registry.register_handle(event, h); + return h; +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/graph.cpp b/cuda_core/cuda/core/_cpp/rt/graph.cpp new file mode 100644 index 00000000000..4f6105dfdaa --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/graph.cpp @@ -0,0 +1,758 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "context_scope.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +// Set a graph node's parameters with h_context current (an empty handle runs in +// the caller's context). Returns the cuGraphNodeSetParams status. A failure to +// restore the caller's context is returned separately in *restore_status so the +// caller can publish the metadata that depends on the successful update before +// raising it; if the update itself failed, its status is returned with the +// restoration failure recorded as its detail and *restore_status is CUDA_SUCCESS. +CUresult graph_node_set_params(CUgraphNode node, CUgraphNodeParams* params, + const ContextHandle& h_context, + CUresult* restore_status) noexcept { + *restore_status = CUDA_SUCCESS; + if (!p_cuGraphNodeSetParams) { + return CUDA_ERROR_NOT_SUPPORTED; + } + CUcontext previous = nullptr; + int changed = 0; + CUresult status = enter_context(h_context, &previous, &changed); + if (status != CUDA_SUCCESS) { + return status; + } + { + GILReleaseGuard gil; + status = p_cuGraphNodeSetParams(node, params); + } + if (!changed) { + return status; + } + CUresult restored = restore_context(previous); + if (restored == CUDA_SUCCESS) { + return status; + } + note_context_not_restored(previous, status, restored); + if (status == CUDA_SUCCESS) { + *restore_status = restored; + } + return status; +} + +// ============================================================================ +// Graph Handles +// ============================================================================ + +namespace { + +struct NodeAttachment; +using GraphAttachmentMap = std::map; + +struct GraphHierarchy; + +// Standard-layout alias target for GraphHandle. +struct GraphBoxBase { + CUgraph resource = nullptr; +}; + +// Canonical state for one CUgraph. Its GraphHandle aliases resource, whose +// address remains stable for the lifetime of the hierarchy. +struct GraphBox : GraphBoxBase { + GraphHierarchy* hierarchy = nullptr; // Non-owning back-reference. + GraphBox* parent = nullptr; // Null for the root graph. + CUgraphNode owner_node = nullptr; // Node in parent that owns this graph. + GraphAttachmentMap attachments; // Non-owning attachment index. + HandleRegistry node_handles; + + GraphBox( + CUgraph resource_, + GraphHierarchy* hierarchy_, + GraphBox* parent_ = nullptr, + CUgraphNode owner_node_ = nullptr) noexcept + : GraphBoxBase{resource_}, + hierarchy(hierarchy_), + parent(parent_), + owner_node(owner_node_) {} +}; + +// Shared owner of stable GraphBox storage. Every GraphHandle aliases the same +// control block, so any graph handle keeps the entire hierarchy alive. +struct GraphHierarchy { + std::list graphs; // Parent boxes precede their descendants. + std::list graveyard; // Retired child graph tombstones. + + GraphBox* root() noexcept { + return graphs.empty() ? nullptr : &graphs.front(); + } +}; + +// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) +using GraphRegistry = HandleRegistry; +static GraphRegistry graph_registry; + +// Immutable resource owners for one version of a graph node's parameters. +// Inheriting DeferredCleanupItem lets CUDA's user-object destructor enqueue +// the payload without destroying owners on the callback thread. +struct NodeAttachment : DeferredCleanupItem { + CUuserObject object = nullptr; + std::array owners; + + NodeAttachment(OpaqueHandle owner0, OpaqueHandle owner1) + : owners{std::move(owner0), std::move(owner1)} {} +}; + +// shared_ptr deleters for the payloads that need one. Typed handles convert to +// OpaqueHandle by assignment and reuse their own control block, so they need no +// deleter here. The Python deleter follows the owner-release pattern used by +// the stream/deviceptr handles above. +void py_deleter(const void* p) noexcept { + GILAcquireGuard gil; + if (gil.acquired()) { + Py_DECREF(const_cast(static_cast(p))); + } +} + +void free_deleter(const void* p) noexcept { + std::free(const_cast(p)); +} + +GraphBox* get_box(const GraphHandle& h) noexcept { + auto* value = reinterpret_cast(h.get()); + return const_cast( + static_cast(value)); +} + +// Rekey a staged attachment map from source nodes to their cloned nodes. +// The caller must release the GIL before calling this function. +CUresult rekey_attachments( + GraphAttachmentMap& attachments, CUgraph cloned_graph) { + if (!cloned_graph) { + return CUDA_ERROR_INVALID_VALUE; + } + if (!p_cuGraphNodeFindInClone) { + return CUDA_ERROR_NOT_SUPPORTED; + } + + GraphAttachmentMap remapped; + while (!attachments.empty()) { + auto attachment = attachments.extract(attachments.begin()); + CUgraphNode cloned_node = nullptr; + CUresult status = p_cuGraphNodeFindInClone( + &cloned_node, attachment.key(), cloned_graph); + if (status != CUDA_SUCCESS) { + return status; + } + attachment.key() = cloned_node; + if (!remapped.insert(std::move(attachment)).inserted) { + return CUDA_ERROR_INVALID_VALUE; + } + } + attachments.swap(remapped); + return CUDA_SUCCESS; +} + +struct StagedGraphMetadata { + const GraphBox* source; + GraphBox* clone; + GraphAttachmentMap* attachments; +}; +using StagedGraphMetadataList = std::vector; + +// Copy a source hierarchy into detached metadata before CUDA mutation. +void stage_graph_metadata( + const GraphBox& source, + GraphBox& clone, + GraphAttachmentMap& attachments, + std::list& subgraphs, + StagedGraphMetadataList& staged) { + attachments = source.attachments; + staged.push_back({&source, &clone, &attachments}); + + for (const GraphBox& source_child : source.hierarchy->graphs) { + if (source_child.parent != &source || !source_child.resource) { + continue; + } + GraphBox& cloned_child = subgraphs.emplace_back( + nullptr, + clone.hierarchy, + &clone, + nullptr); + stage_graph_metadata( + source_child, + cloned_child, + cloned_child.attachments, + subgraphs, + staged); + } +} + +// Bind staged metadata to a CUDA-cloned hierarchy. The root clone resource +// must be populated before entry. The caller must release the GIL. +CUresult rekey_graph_metadata( + StagedGraphMetadataList& staged) { + if (!p_cuGraphNodeFindInClone || !p_cuGraphChildGraphNodeGetGraph) { + return CUDA_ERROR_NOT_SUPPORTED; + } + + CUresult status; + for (size_t i = 0; i < staged.size(); ++i) { + const GraphBox& source = *staged[i].source; + GraphBox& clone = *staged[i].clone; + if (i != 0) { + CUgraphNode cloned_owner = nullptr; + status = p_cuGraphNodeFindInClone( + &cloned_owner, + source.owner_node, + clone.parent->resource); + if (status == CUDA_SUCCESS) { + status = p_cuGraphChildGraphNodeGetGraph( + cloned_owner, &clone.resource); + } + if (status != CUDA_SUCCESS) { + return status; + } + clone.owner_node = cloned_owner; + } + + status = rekey_attachments( + *staged[i].attachments, clone.resource); + if (status != CUDA_SUCCESS) { + return status; + } + } + return CUDA_SUCCESS; +} + +} // namespace + +OpaqueHandle make_opaque_py(PyObject* obj) { + Py_INCREF(obj); + return OpaqueHandle(static_cast(obj), py_deleter); +} + +OpaqueHandle make_opaque_malloc(void* buf) { + return OpaqueHandle(static_cast(buf), free_deleter); +} + +// State held by PreparedAttachment between preparation and commit. It keeps the +// graph alive, tracks the graph-retained replacement, and holds a preallocated +// map entry so commit cannot allocate. Destroying PreparedAttachment rolls back +// the staged user-object retain unless graph_commit_attachment publishes it. +struct PreparedAttachmentState { + GraphHandle h_graph; + NodeAttachment* replacement = nullptr; + GraphAttachmentMap::node_type replacement_entry; + + explicit PreparedAttachmentState(GraphHandle h_graph_) + : h_graph(std::move(h_graph_)) {} +}; + +void rollback_prepared_attachment( + PreparedAttachmentState* state) noexcept { + if (!state) { + return; + } + if (state->replacement) { + GraphBox* box = get_box(state->h_graph); + if (box->resource) { + GILReleaseGuard gil; + pw_cuGraphReleaseUserObject( + box->resource, state->replacement->object, 1); + } + } + delete state; +} + +// Detached metadata for a replacement embedded graph hierarchy. Preparation +// copies every attachment map and allocates every GraphBox before CUDA destroys +// the old embedded graph. Commit only rekeys and publishes it. +struct PreparedChildGraphUpdateState { + GraphHandle h_parent; + GraphHandle h_source; + GraphBox* old_root = nullptr; + CUgraphNode owner_node = nullptr; + std::list replacement; + StagedGraphMetadataList staged; + std::vector handles; + + PreparedChildGraphUpdateState( + GraphHandle h_parent_, + GraphHandle h_source_, + GraphBox* old_root_, + CUgraphNode owner_node_) + : h_parent(std::move(h_parent_)), + h_source(std::move(h_source_)), + old_root(old_root_), + owner_node(owner_node_) {} +}; + +GraphHandle create_graph_handle(CUgraph graph) { + if (!graph) { + return {}; + } + + auto hierarchy = std::shared_ptr( + new GraphHierarchy{}, + [](GraphHierarchy* hierarchy) { + for (const GraphBox& box : hierarchy->graphs) { + if (box.resource) { + graph_registry.unregister_handle(box.resource); + } + } + GraphBox* root = hierarchy->root(); + if (root && root->resource) { + GILReleaseGuard gil; + pw_cuGraphDestroy(root->resource); + } + retry_deferred_cleanup(); + delete hierarchy; + } + ); + GraphBox& root = hierarchy->graphs.emplace_back( + graph, hierarchy.get()); + + GraphHandle h_graph(hierarchy, &root.resource); + graph_registry.register_handle(graph, h_graph); + return h_graph; +} + +GraphHandle create_child_graph_handle( + CUgraph child_graph, const GraphHandle& h_parent, + CUgraphNode owner_node) { + if (!child_graph || !h_parent || !owner_node) { + return {}; + } + if (GraphHandle h_graph = graph_registry.lookup(child_graph)) { + return h_graph; + } + + GraphBox* parent = get_box(h_parent); + GraphHierarchy* hierarchy = parent->hierarchy; + GraphBox& child = hierarchy->graphs.emplace_back( + child_graph, hierarchy, parent, owner_node); + + GraphHandle h_child(h_parent, &child.resource); + graph_registry.register_handle(child_graph, h_child); + return h_child; +} + +CUresult graph_prepare_child_graph_update( + const GraphHandle& h_parent, + const GraphHandle& h_old_child, + CUgraphNode owner_node, + const GraphHandle& h_source, + PreparedChildGraphUpdate* out_prepared) { + if (!h_parent || !h_old_child || !owner_node || + !h_source || !out_prepared) { + return CUDA_ERROR_INVALID_VALUE; + } + out_prepared->reset(); + + GraphBox* parent = get_box(h_parent); + GraphBox* old_root = get_box(h_old_child); + GraphBox* source = get_box(h_source); + // A source from the destination hierarchy can include the old embedded + // subtree whose raw node keys CUDA destroys during replacement. + if (!parent->resource || !old_root->resource || !source->resource || + old_root->parent != parent || + old_root->owner_node != owner_node || + source->hierarchy == parent->hierarchy) { + return CUDA_ERROR_INVALID_VALUE; + } + + PreparedChildGraphUpdate prepared = + std::make_shared( + h_parent, h_source, old_root, owner_node); + + GraphBox& replacement_root = + prepared->replacement.emplace_back( + nullptr, parent->hierarchy, parent, owner_node); + stage_graph_metadata( + *source, + replacement_root, + replacement_root.attachments, + prepared->replacement, + prepared->staged); + + const size_t graph_count = prepared->staged.size(); + prepared->handles.reserve(graph_count); + for (const StagedGraphMetadata& graph : prepared->staged) { + prepared->handles.emplace_back( + h_parent, &graph.clone->resource); + } + + *out_prepared = std::move(prepared); + return CUDA_SUCCESS; +} + +void publish_child_graph_update( + PreparedChildGraphUpdateState& state, + GraphHandle* out_child) { + GraphBox* parent = get_box(state.h_parent); + parent->hierarchy->graphs.splice( + parent->hierarchy->graphs.end(), state.replacement); + *out_child = state.handles.front(); + graph_registry.register_handles(state.handles); +} + +CUresult graph_commit_child_graph_update( + PreparedChildGraphUpdate& prepared, + GraphHandle* out_child) { + if (!prepared || !out_child) { + return CUDA_ERROR_INVALID_VALUE; + } + out_child->reset(); + + PreparedChildGraphUpdateState& state = *prepared; + GraphBox* parent = get_box(state.h_parent); + if (!parent->resource || !state.old_root->resource) { + prepared.reset(); + return CUDA_ERROR_INVALID_VALUE; + } + + CUresult status = CUDA_ERROR_NOT_SUPPORTED; + CUgraph cloned_root = nullptr; + if (p_cuGraphChildGraphNodeGetGraph) { + GILReleaseGuard gil; + status = p_cuGraphChildGraphNodeGetGraph( + state.owner_node, &cloned_root); + if (status == CUDA_SUCCESS) { + state.staged.front().clone->resource = cloned_root; + status = rekey_graph_metadata(state.staged); + } + } + + // CUDA has already destroyed the old embedded graph. No replacement + // metadata is visible yet, so this selects only the old generation. + invalidate_child_graph_state( + state.h_parent, state.owner_node); + + if (status != CUDA_SUCCESS) { + prepared.reset(); + throw std::runtime_error( + "failed to update graph metadata after child graph replacement"); + } + + publish_child_graph_update(state, out_child); + prepared.reset(); + return status; +} + +CUresult graph_get_attachment( + const GraphHandle& h_graph, CUgraphNode node, + OpaqueHandle* owner0, OpaqueHandle* owner1) { + if (!h_graph || !node || (!owner0 && !owner1)) { + return CUDA_ERROR_INVALID_VALUE; + } + if (owner0) { + owner0->reset(); + } + if (owner1) { + owner1->reset(); + } + + GraphBox* box = get_box(h_graph); + if (!box->resource) { + return CUDA_ERROR_INVALID_VALUE; + } + auto it = box->attachments.find(node); + if (it != box->attachments.end()) { + if (owner0) { + *owner0 = it->second->owners[0]; + } + if (owner1) { + *owner1 = it->second->owners[1]; + } + } + return CUDA_SUCCESS; +} + +CUresult graph_prepare_attachment( + const GraphHandle& h_graph, + OpaqueHandle owner0, + OpaqueHandle owner1, + PreparedAttachment* out_prepared) { + if (!out_prepared) { + return CUDA_ERROR_INVALID_VALUE; + } + out_prepared->reset(); + if (!h_graph) { + return CUDA_ERROR_INVALID_VALUE; + } + + GraphBox* box = get_box(h_graph); + if (!box->resource) { + return CUDA_ERROR_INVALID_VALUE; + } + if (!p_cuGraphReleaseUserObject) { + return CUDA_ERROR_NOT_SUPPORTED; + } + + PreparedAttachment prepared( + new PreparedAttachmentState(h_graph), + PreparedAttachmentDeleter{rollback_prepared_attachment}); + if (owner0 || owner1) { + if (!p_cuUserObjectCreate || !p_cuUserObjectRelease || + !p_cuGraphRetainUserObject) { + return CUDA_ERROR_NOT_SUPPORTED; + } + + ensure_deferred_cleanup_ready(); + prepared->replacement = new NodeAttachment( + std::move(owner0), std::move(owner1)); + GraphAttachmentMap staged; + try { + staged.emplace(nullptr, prepared->replacement); + prepared->replacement_entry = + staged.extract(staged.begin()); + } catch (...) { + delete prepared->replacement; + prepared->replacement = nullptr; + throw; + } + auto* cleanup_item = + static_cast( + prepared->replacement); + + CUuserObject object = nullptr; + CUresult status; + { + GILReleaseGuard gil; + status = p_cuUserObjectCreate( + &object, cleanup_item, + reinterpret_cast(enqueue_cleanup), + 1, CU_USER_OBJECT_NO_DESTRUCTOR_SYNC); + if (status != CUDA_SUCCESS) { + prepared->replacement_entry.mapped() = nullptr; + delete prepared->replacement; + prepared->replacement = nullptr; + return status; + } + prepared->replacement->object = object; + status = p_cuGraphRetainUserObject( + box->resource, object, 1, CU_GRAPH_USER_OBJECT_MOVE); + if (status != CUDA_SUCCESS) { + prepared->replacement_entry.mapped() = nullptr; + prepared->replacement = nullptr; + pw_cuUserObjectRelease(object, 1); + return status; + } + } + } + + *out_prepared = std::move(prepared); + return CUDA_SUCCESS; +} + +CUresult graph_commit_attachment( + PreparedAttachment& prepared, + CUgraphNode node) { + if (!prepared) { + return CUDA_ERROR_INVALID_VALUE; + } + + GraphHandle h_graph = prepared->h_graph; + GraphBox* box = get_box(h_graph); + if (!box->resource || (!node && !prepared->replacement)) { + delete prepared.release(); + return CUDA_ERROR_INVALID_VALUE; + } + if (!node) { + delete prepared.release(); + return CUDA_SUCCESS; + } + + // Publish the replacement or removal before releasing the previous graph + // reference; that release can make the previous payload eligible for + // destruction. + NodeAttachment* previous = nullptr; + auto it = box->attachments.find(node); + if (it == box->attachments.end()) { + if (prepared->replacement) { + prepared->replacement_entry.key() = node; + auto result = box->attachments.insert( + std::move(prepared->replacement_entry)); + if (!result.inserted) { + prepared->replacement_entry = + std::move(result.node); + delete prepared.release(); + return CUDA_ERROR_INVALID_VALUE; + } + } + } else { + previous = it->second; + if (prepared->replacement) { + it->second = prepared->replacement; + } else { + box->attachments.erase(it); + } + } + + delete prepared.release(); + if (!previous) { + return CUDA_SUCCESS; + } + GILReleaseGuard gil; + return p_cuGraphReleaseUserObject( + box->resource, previous->object, 1); +} + +CUresult graph_clone_attachments( + const GraphHandle& h_clone, + const GraphHandle& h_source) { + if (!h_clone || !h_source) { + return CUDA_ERROR_INVALID_VALUE; + } + + GraphBox* clone = get_box(h_clone); + GraphBox* source = get_box(h_source); + if (!clone->resource || !source->resource || + !clone->attachments.empty()) { + return CUDA_ERROR_INVALID_VALUE; + } + + // Build and rekey the clone metadata off-hierarchy so a CUDA mapping error + // cannot partially publish it. + GraphAttachmentMap attachments; + std::list subgraphs; + StagedGraphMetadataList staged; + stage_graph_metadata( + *source, *clone, attachments, subgraphs, staged); + + std::vector handles; + handles.reserve(subgraphs.size()); + for (GraphBox& graph : subgraphs) { + handles.emplace_back(h_clone, &graph.resource); + } + + CUresult status; + { + GILReleaseGuard gil; + status = rekey_graph_metadata(staged); + } + if (status != CUDA_SUCCESS) { + return status; + } + + clone->attachments.swap(attachments); + if (subgraphs.empty()) { + return CUDA_SUCCESS; + } + + clone->hierarchy->graphs.splice( + clone->hierarchy->graphs.end(), subgraphs); + graph_registry.register_handles(handles); + return CUDA_SUCCESS; +} + +namespace { +struct GraphNodeBox { + mutable CUgraphNode resource; + GraphHandle h_graph; +}; +} // namespace + +static const GraphNodeBox* get_box(const GraphNodeHandle& h) { + const CUgraphNode* p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) - offsetof(GraphNodeBox, resource) + ); +} + +// graphs is ordered parent-before-child. Nulling a selected box marks its +// later descendants, whose parent pointers remain valid after list splicing. +// This permits one allocation-free sweep of the hierarchy. +void invalidate_child_graph_state( + const GraphHandle& h_parent, + CUgraphNode owner_node) noexcept { + if (!h_parent || !owner_node) { + return; + } + + GraphBox* parent = get_box(h_parent); + if (!parent->resource) { + return; + } + GraphHierarchy& hierarchy = *parent->hierarchy; + for (auto it = hierarchy.graphs.begin(); + it != hierarchy.graphs.end();) { + auto graph = it++; + bool is_owned_root = graph->parent == parent && + graph->owner_node == owner_node; + bool is_descendant = graph->parent && + !graph->parent->resource; + if (!is_owned_root && !is_descendant) { + continue; + } + + // Empty node_handles and invalidate each one. + for (auto& entry : graph->node_handles.drain()) { + if (GraphNodeHandle h_node = entry.second.lock()) { + get_box(h_node)->resource = nullptr; + } + } + graph_registry.unregister_handle(graph->resource); + graph->resource = nullptr; + graph->attachments.clear(); + hierarchy.graveyard.splice( + hierarchy.graveyard.end(), hierarchy.graphs, graph); + } +} + +GraphNodeHandle create_graph_node_handle(CUgraphNode node, const GraphHandle& h_graph) { + if (!node) { + auto box = std::make_shared( + GraphNodeBox{nullptr, h_graph}); + return GraphNodeHandle(box, &box->resource); + } + + GraphBox* graph = get_box(h_graph); + return graph->node_handles.get_or_create( + node, + [node, &h_graph] { + auto box = std::make_shared( + GraphNodeBox{node, h_graph}); + return GraphNodeHandle(box, &box->resource); + }); +} + +GraphHandle graph_node_get_graph(const GraphNodeHandle& h) noexcept { + return h ? get_box(h)->h_graph : GraphHandle{}; +} + +void invalidate_graph_node(const GraphNodeHandle& h) noexcept { + if (!h) { + return; + } + + const GraphNodeBox* node_box = get_box(h); + CUgraphNode node = node_box->resource; + if (!node) { + return; + } + GraphBox* graph = get_box(node_box->h_graph); + graph->node_handles.unregister_handle(node); + node_box->resource = nullptr; +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/graph_exec.cpp b/cuda_core/cuda/core/_cpp/rt/graph_exec.cpp new file mode 100644 index 00000000000..dbf09cf77e0 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/graph_exec.cpp @@ -0,0 +1,293 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +// ============================================================================ +// Graph Exec Handles +// ============================================================================ + +namespace { + +// Append-only owners introduced by individual executable-node updates. CUDA +// owns this payload through a user object propagated into the CUgraphExec. +struct ExecAttachments : DeferredCleanupItem { + CUuserObject object = nullptr; + std::vector owners; +}; + +struct GraphExecBox { + CUgraphExec resource = nullptr; + ExecAttachments* attachments = nullptr; // Non-owning. + + ~GraphExecBox() noexcept { + if (resource) { + GILReleaseGuard gil; + pw_cuGraphExecDestroy(resource); + } + // The accumulator fields may be dangling after exec destruction. + retry_deferred_cleanup(); + } +}; + +GraphExecBox* get_exec_box(const GraphExecHandle& h) noexcept { + return const_cast( + reinterpret_cast(h.get())); +} + +GraphExecHandle make_graph_exec_handle( + CUgraphExec graph_exec, ExecAttachments* attachments) { + struct RawGraphExecGuard { + CUgraphExec resource; + + ~RawGraphExecGuard() noexcept { + if (resource) { + GILReleaseGuard gil; + pw_cuGraphExecDestroy(resource); + } + retry_deferred_cleanup(); + } + } guard{graph_exec}; + + auto box = std::make_shared(); + box->resource = graph_exec; + box->attachments = attachments; + guard.resource = nullptr; + return GraphExecHandle(box, &box->resource); +} + +// Holds a fresh accumulator retained on the source graph across a CUDA call +// that propagates user objects into an exec. Releasing drops the source's +// reference: after successful propagation the exec keeps the accumulator +// alive, and otherwise this drops its last reference. +struct ExecAttachmentStaging { + GraphHandle h_source; + ExecAttachments* accumulator = nullptr; + + ~ExecAttachmentStaging() noexcept { + report_cuda_error("cuGraphReleaseUserObject", release(), + "failed while dropping a staged graph attachment"); + } + + CUresult release() noexcept { + if (!h_source || !accumulator) { + return CUDA_SUCCESS; + } + const CUuserObject object = accumulator->object; + const GraphHandle source = std::move(h_source); + accumulator = nullptr; + GILReleaseGuard gil; + return p_cuGraphReleaseUserObject(*source, object, 1); + } +}; + +// Create an accumulator and retain it on h_source, so that a following +// instantiation or whole-graph update propagates a reference into the exec. +CUresult stage_exec_attachments( + const GraphHandle& h_source, ExecAttachmentStaging* out_staging) { + if (!p_cuUserObjectCreate || !p_cuUserObjectRelease || + !p_cuGraphRetainUserObject || !p_cuGraphReleaseUserObject) { + return CUDA_ERROR_NOT_SUPPORTED; + } + + ensure_deferred_cleanup_ready(); + auto* accumulator = new ExecAttachments; + + CUuserObject object = nullptr; + CUresult status; + { + GILReleaseGuard gil; + status = p_cuUserObjectCreate( + &object, + static_cast(accumulator), + reinterpret_cast(enqueue_cleanup), + 1, + CU_USER_OBJECT_NO_DESTRUCTOR_SYNC); + if (status != CUDA_SUCCESS) { + delete accumulator; + return status; + } + accumulator->object = object; + status = p_cuGraphRetainUserObject( + *h_source, object, 1, CU_GRAPH_USER_OBJECT_MOVE); + if (status != CUDA_SUCCESS) { + // Dropping the last reference retires the accumulator. + pw_cuUserObjectRelease(object, 1); + return status; + } + } + + out_staging->h_source = h_source; + out_staging->accumulator = accumulator; + return CUDA_SUCCESS; +} + +} // namespace + +// State held by PreparedExecAttachment between preparation and commit. It keeps +// the exec alive and remembers the accumulator size before the append, so that +// rollback can drop owners staged for a mutation that CUDA rejected. +struct PreparedExecAttachmentState { + GraphExecHandle h_exec; + ExecAttachments* attachments = nullptr; + size_t original_size = 0; + + PreparedExecAttachmentState( + GraphExecHandle h_exec_, + ExecAttachments* attachments_, + size_t original_size_) + : h_exec(std::move(h_exec_)), + attachments(attachments_), + original_size(original_size_) {} +}; + +void rollback_prepared_exec_attachment( + PreparedExecAttachmentState* state) noexcept { + if (!state) { + return; + } + if (state->attachments) { + while (state->attachments->owners.size() > state->original_size) { + state->attachments->owners.pop_back(); + } + } + delete state; +} + +GraphExecHandle create_graph_exec_handle( + const GraphHandle& h_source, + CUDA_GRAPH_INSTANTIATE_PARAMS* params) { + if (!h_source || !*h_source || !params) { + err = CUDA_ERROR_INVALID_VALUE; + return {}; + } + if (!p_cuGraphInstantiateWithParams) { + err = CUDA_ERROR_NOT_SUPPORTED; + return {}; + } + + ExecAttachmentStaging staging; + if (CUDA_SUCCESS != (err = stage_exec_attachments(h_source, &staging))) { + return {}; + } + + CUgraphExec graph_exec = nullptr; + { + GILReleaseGuard gil; + err = p_cuGraphInstantiateWithParams(&graph_exec, *h_source, params); + } + if (err != CUDA_SUCCESS) { + return {}; + } + // CUDA can report a specific failure while returning success. The exec is + // then unusable, so it stays unadopted for the caller to diagnose from + // params->result_out. + if (params->result_out != CUDA_GRAPH_INSTANTIATE_SUCCESS) { + return {}; + } + if (!graph_exec) { + err = CUDA_ERROR_INVALID_VALUE; + return {}; + } + + GraphExecHandle h_exec = make_graph_exec_handle( + graph_exec, staging.accumulator); + if (CUDA_SUCCESS != (err = staging.release())) { + return {}; + } + return h_exec; +} + +CUresult graph_exec_update( + const GraphExecHandle& h_exec, + const GraphHandle& h_source, + CUgraphExecUpdateResultInfo* result_info) { + if (!h_exec || !h_source || !*h_source || !result_info) { + return CUDA_ERROR_INVALID_VALUE; + } + if (!p_cuGraphExecUpdate) { + return CUDA_ERROR_NOT_SUPPORTED; + } + + GraphExecBox* box = get_exec_box(h_exec); + if (!box->resource) { + return CUDA_ERROR_INVALID_VALUE; + } + + ExecAttachmentStaging staging; + CUresult status = stage_exec_attachments(h_source, &staging); + if (status != CUDA_SUCCESS) { + return status; + } + + { + GILReleaseGuard gil; + status = p_cuGraphExecUpdate(box->resource, *h_source, result_info); + } + if (status != CUDA_SUCCESS) { + return status; + } + + // CUDA may already have retired the old accumulator. Publish the new one + // before releasing the source graph's temporary reference. + box->attachments = staging.accumulator; + return staging.release(); +} + +CUresult graph_prepare_exec_attachment( + const GraphExecHandle& h_exec, + OpaqueHandle owner0, + OpaqueHandle owner1, + PreparedExecAttachment* out_prepared) { + if (!out_prepared) { + return CUDA_ERROR_INVALID_VALUE; + } + out_prepared->reset(); + if (!h_exec) { + return CUDA_ERROR_INVALID_VALUE; + } + + GraphExecBox* box = get_exec_box(h_exec); + if (!box->resource || !box->attachments) { + return CUDA_ERROR_INVALID_VALUE; + } + + ExecAttachments* attachments = box->attachments; + const size_t original_size = attachments->owners.size(); + const size_t additions = + static_cast(static_cast(owner0)) + + static_cast(static_cast(owner1)); + // Reserve before staging so that rollback and commit cannot allocate. + attachments->owners.reserve(original_size + additions); + PreparedExecAttachment prepared( + new PreparedExecAttachmentState(h_exec, attachments, original_size), + PreparedExecAttachmentDeleter{rollback_prepared_exec_attachment}); + if (owner0) { + attachments->owners.emplace_back(std::move(owner0)); + } + if (owner1) { + attachments->owners.emplace_back(std::move(owner1)); + } + *out_prepared = std::move(prepared); + return CUDA_SUCCESS; +} + +void graph_commit_exec_attachment( + PreparedExecAttachment& prepared) noexcept { + delete prepared.release(); +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/internal.hpp b/cuda_core/cuda/core/_cpp/rt/internal.hpp new file mode 100644 index 00000000000..7033c884722 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/internal.hpp @@ -0,0 +1,176 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "types.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cuda_core::rt::detail { + +// Implemented in error.cpp +void format_cuda_error(char* buffer, size_t size, const char* operation, CUresult status, + const char* detail) noexcept; +// Implemented in error.cpp +void note_context_not_restored(CUcontext previous, CUresult operation_status, + CUresult restore_status) noexcept; + +// Store a stream and any state needed to preserve deallocation ordering. +struct DeallocationStream { + StreamHandle h_stream; + std::thread::id ptds_tid{}; +}; + +// Implemented in stream.cpp +ContextHandle deallocation_context(const DeallocationStream& stream) noexcept; +// Implemented in stream.cpp +bool make_deallocation_stream(const StreamHandle& h, DeallocationStream& out) noexcept; + +// Decorate a status-returning cleanup call to report whenever it fails. CUDA +// calls (CUresult) are reported with the error name and description; NVRTC, +// NVVM and nvJitLink calls (integer status codes) with the raw code. +template +class WarnOnFailure { +public: + explicit WarnOnFailure(const char* operation) noexcept : operation_(operation) {} + + template + auto operator()(Args&&... args) const noexcept { + auto status = Function(std::forward(args)...); + report(status); + return status; + } + +private: + void report(CUresult status) const noexcept { + report_cuda_error(operation_, status); + } + + template + void report(Status status) const noexcept { + if (static_cast(status) != 0) { + report_status_code(operation_, static_cast(status)); + } + } + + const char* operation_; +}; + +// Warning-decorated CUDA operations used by non-throwing cleanup paths. +const WarnOnFailure pw_cuStreamDestroy{"cuStreamDestroy"}; +const WarnOnFailure pw_cuEventDestroy{"cuEventDestroy"}; +const WarnOnFailure pw_cuMemFree{"cuMemFree"}; +const WarnOnFailure pw_cuMemFreeAsync{"cuMemFreeAsync"}; +const WarnOnFailure pw_cuArrayDestroy{"cuArrayDestroy"}; +const WarnOnFailure pw_cuMipmappedArrayDestroy{"cuMipmappedArrayDestroy"}; +const WarnOnFailure pw_cuTexObjectDestroy{"cuTexObjectDestroy"}; +const WarnOnFailure pw_cuSurfObjectDestroy{"cuSurfObjectDestroy"}; +const WarnOnFailure pw_cuGreenCtxDestroy{"cuGreenCtxDestroy"}; +const WarnOnFailure pw_cuMemPoolDestroy{"cuMemPoolDestroy"}; +const WarnOnFailure pw_cuMemFreeHost{"cuMemFreeHost"}; +const WarnOnFailure pw_cuGraphDestroy{"cuGraphDestroy"}; +const WarnOnFailure pw_cuGraphExecDestroy{"cuGraphExecDestroy"}; +const WarnOnFailure pw_cuGraphicsUnregisterResource{"cuGraphicsUnregisterResource"}; +const WarnOnFailure pw_cuLinkDestroy{"cuLinkDestroy"}; +const WarnOnFailure pw_cuUserObjectRelease{"cuUserObjectRelease"}; +const WarnOnFailure pw_cuGraphReleaseUserObject{"cuGraphReleaseUserObject"}; +const WarnOnFailure pw_nvrtcDestroyProgram{"nvrtcDestroyProgram"}; +const WarnOnFailure pw_nvvmDestroyProgram{"nvvmDestroyProgram"}; +const WarnOnFailure pw_nvJitLinkDestroy{"nvJitLinkDestroy"}; + +// Intrusive base for payloads transferred out of CUDA's callback. +struct DeferredCleanupItem { + DeferredCleanupItem* next = nullptr; + virtual ~DeferredCleanupItem() noexcept = default; +}; + +// Implemented in py_deferred_cleanup.cpp +void ensure_deferred_cleanup_ready(); +// Implemented in py_deferred_cleanup.cpp +void enqueue_cleanup(void* item) noexcept; + +// ============================================================================ +// Handle reverse-lookup registry +// +// Maps raw CUDA handles (CUevent, CUkernel, etc.) back to their owning +// shared_ptr so that _ref constructors can recover full metadata. +// Uses weak_ptr to avoid preventing destruction. +// ============================================================================ + +template> +class HandleRegistry { +public: + using MapType = std::unordered_map, Hash>; + + void register_handle(const Key& key, const Handle& h) { + std::lock_guard lock(mutex_); + map_[key] = h; + } + + void unregister_handle(const Key& key) noexcept { + std::lock_guard lock(mutex_); + map_.erase(key); + } + + void register_handles(const std::vector& handles) { + std::lock_guard lock(mutex_); + for (const Handle& h : handles) { + if (h) { + map_[*h] = h; + } + } + } + + Handle lookup(const Key& key) { + std::lock_guard lock(mutex_); + auto it = map_.find(key); + if (it != map_.end()) { + if (auto h = it->second.lock()) { + return h; + } + map_.erase(it); + } + return {}; + } + + template + Handle get_or_create(const Key& key, Factory&& create) { + std::lock_guard lock(mutex_); + auto it = map_.find(key); + if (it != map_.end()) { + if (Handle h = it->second.lock()) { + return h; + } + map_.erase(it); + } + + Handle h = create(); + if (h) { + map_[key] = h; + } + return h; + } + + MapType drain() noexcept { + std::lock_guard lock(mutex_); + MapType extracted; + extracted.swap(map_); + return extracted; + } + +private: + std::mutex mutex_; + MapType map_; +}; + +} // namespace cuda_core::rt::detail diff --git a/cuda_core/cuda/core/_cpp/rt/memory.cpp b/cuda_core/cuda/core/_cpp/rt/memory.cpp new file mode 100644 index 00000000000..7e5b0ab34a4 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/memory.cpp @@ -0,0 +1,497 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "context_scope.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include +#include +#include +#include +#include +#ifndef _WIN32 +#include +#endif + +namespace cuda_core::rt { + +using namespace detail; + +// ============================================================================ +// Memory Pool Handles +// ============================================================================ + +namespace { +struct MemoryPoolBox { + CUmemoryPool resource; +}; +} // namespace + +// Helper to clear peer access before destroying a memory pool. +// Works around nvbug 5698116: recycled pool handles inherit peer access state. +// Must be noexcept since it's called from a shared_ptr deleter. +static void clear_mempool_peer_access(CUmemoryPool pool) noexcept { + try { + int device_count = 0; + if (p_cuDeviceGetCount(&device_count) != CUDA_SUCCESS || device_count <= 0) { + return; + } + + std::vector clear_access(device_count); + for (int i = 0; i < device_count; ++i) { + clear_access[i].location.type = CU_MEM_LOCATION_TYPE_DEVICE; + clear_access[i].location.id = i; + clear_access[i].flags = CU_MEM_ACCESS_FLAGS_PROT_NONE; + } + p_cuMemPoolSetAccess(pool, clear_access.data(), device_count); // Best effort + } catch (...) { + // Swallow exceptions - this is best-effort cleanup in destructor context + } +} + +static MemoryPoolHandle wrap_mempool_owned(CUmemoryPool pool) { + auto box = std::shared_ptr( + new MemoryPoolBox{pool}, + [](const MemoryPoolBox* b) { + GILReleaseGuard gil; + clear_mempool_peer_access(b->resource); + pw_cuMemPoolDestroy(b->resource); + delete b; + } + ); + return MemoryPoolHandle(box, &box->resource); +} + +MemoryPoolHandle create_mempool_handle(const CUmemPoolProps& props) { + GILReleaseGuard gil; + CUmemoryPool pool; + if (CUDA_SUCCESS != (err = p_cuMemPoolCreate(&pool, &props))) { + return {}; + } + return wrap_mempool_owned(pool); +} + +MemoryPoolHandle create_mempool_handle_ref(CUmemoryPool pool) { + auto box = std::make_shared(MemoryPoolBox{pool}); + return MemoryPoolHandle(box, &box->resource); +} + +MemoryPoolHandle get_device_mempool(int device_id) { + GILReleaseGuard gil; + CUmemoryPool pool; + if (CUDA_SUCCESS != (err = p_cuDeviceGetMemPool(&pool, device_id))) { + return {}; + } + return create_mempool_handle_ref(pool); +} + +MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type) { + GILReleaseGuard gil; + CUmemoryPool pool; + auto handle_ptr = reinterpret_cast(static_cast(fd)); + if (CUDA_SUCCESS != (err = p_cuMemPoolImportFromShareableHandle(&pool, handle_ptr, handle_type, 0))) { + return {}; + } + return wrap_mempool_owned(pool); +} + +// ============================================================================ +// Device Pointer Handles +// ============================================================================ + +namespace { +struct DevicePtrBox { + CUdeviceptr resource; + // Mutable so set_deallocation_stream() can update free ordering through a + // const DevicePtrHandle. Built with make_deallocation_stream so default- + // stream tokens carry a bound context. + mutable DeallocationStream deallocation; +}; +} // namespace + +// Recovers the owning DevicePtrBox from the aliased CUdeviceptr pointer. +// This works because DevicePtrHandle is a shared_ptr alias pointing to +// &box->resource, so we can compute the containing struct using offsetof. +// The const_cast is safe because we only use this to access the mutable +// deallocation member or in the deleter (where the box is being destroyed). +static DevicePtrBox* get_box(const DevicePtrHandle& h) { + const CUdeviceptr* p = h.get(); + return reinterpret_cast( + reinterpret_cast(const_cast(p)) - offsetof(DevicePtrBox, resource) + ); +} + +// Return the stream that orders a device pointer's deallocation. +StreamHandle deallocation_stream(const DevicePtrHandle& h) noexcept { + return get_box(h)->deallocation.h_stream; +} + +// Replace the stream that orders a device pointer's deallocation. +CUresult set_deallocation_stream(const DevicePtrHandle& h, const StreamHandle& h_stream) noexcept { + if (!h) { + return CUDA_ERROR_INVALID_VALUE; + } + DeallocationStream ds; + if (!make_deallocation_stream(h_stream, ds)) { + return err != CUDA_SUCCESS ? err : CUDA_ERROR_INVALID_CONTEXT; + } + get_box(h)->deallocation = std::move(ds); + return CUDA_SUCCESS; +} + +DevicePtrHandle deviceptr_alloc_from_pool(size_t size, const MemoryPoolHandle& h_pool, const StreamHandle& h_stream) { + GILReleaseGuard gil; + CUdeviceptr ptr; + if (CUDA_SUCCESS != (err = p_cuMemAllocFromPoolAsync(&ptr, size, *h_pool, as_cu(h_stream)))) { + return {}; + } + + DeallocationStream ds; + if (!make_deallocation_stream(h_stream, ds)) { + pw_cuMemFreeAsync(ptr, as_cu(h_stream)); + return {}; + } + + auto box = std::shared_ptr( + new DevicePtrBox{ptr, std::move(ds)}, + [h_pool](DevicePtrBox* b) { + GILReleaseGuard gil; + const DeallocationStream& stream = b->deallocation; + cleanup_in_context( + deallocation_context(stream), "cuMemFreeAsync", + [&]() noexcept { + return p_cuMemFreeAsync( + b->resource, as_cu(stream.h_stream)); + }); + delete b; + } + ); + return DevicePtrHandle(box, &box->resource); +} + +DevicePtrHandle deviceptr_alloc_async(size_t size, const StreamHandle& h_stream) { + GILReleaseGuard gil; + CUdeviceptr ptr; + if (CUDA_SUCCESS != (err = p_cuMemAllocAsync(&ptr, size, as_cu(h_stream)))) { + return {}; + } + + DeallocationStream ds; + if (!make_deallocation_stream(h_stream, ds)) { + pw_cuMemFreeAsync(ptr, as_cu(h_stream)); + return {}; + } + + auto box = std::shared_ptr( + new DevicePtrBox{ptr, std::move(ds)}, + [](DevicePtrBox* b) { + GILReleaseGuard gil; + const DeallocationStream& stream = b->deallocation; + cleanup_in_context( + deallocation_context(stream), "cuMemFreeAsync", + [&]() noexcept { + return p_cuMemFreeAsync( + b->resource, as_cu(stream.h_stream)); + }); + delete b; + } + ); + return DevicePtrHandle(box, &box->resource); +} + +// Allocate device memory synchronously with the provided context current. +CUresult deviceptr_alloc_raw(CUdeviceptr* ptr, size_t size, + const ContextHandle& h_context) noexcept { + GILReleaseGuard gil; + return invoke_in_context_or_undo( + h_context, + [&]() noexcept { return p_cuMemAlloc(ptr, size); }, + [&]() noexcept { pw_cuMemFree(*ptr); }, + /*undo_requires_target_context=*/false); +} + +DevicePtrHandle deviceptr_alloc_host(size_t size) { + GILReleaseGuard gil; + void* ptr; + if (CUDA_SUCCESS != (err = p_cuMemAllocHost(&ptr, size))) { + return {}; + } + + auto box = std::shared_ptr( + new DevicePtrBox{reinterpret_cast(ptr), DeallocationStream{}}, + [](DevicePtrBox* b) { + GILReleaseGuard gil; + pw_cuMemFreeHost(reinterpret_cast(b->resource)); + delete b; + } + ); + return DevicePtrHandle(box, &box->resource); +} + +DevicePtrHandle deviceptr_create_ref(CUdeviceptr ptr) { + auto box = std::make_shared(DevicePtrBox{ptr, DeallocationStream{}}); + return DevicePtrHandle(box, &box->resource); +} + +DevicePtrHandle deviceptr_create_with_owner(CUdeviceptr ptr, PyObject* owner) { + if (!owner) { + return deviceptr_create_ref(ptr); + } + // GIL required when owner is provided + GILAcquireGuard gil; + if (!gil.acquired()) { + // Python finalizing - fall back to ref version (no owner tracking) + return deviceptr_create_ref(ptr); + } + Py_INCREF(owner); + auto box = std::shared_ptr( + new DevicePtrBox{ptr, DeallocationStream{}}, + [owner](DevicePtrBox* b) { + GILAcquireGuard gil; + if (gil.acquired()) { + Py_DECREF(owner); + } + delete b; + } + ); + return DevicePtrHandle(box, &box->resource); +} + +DevicePtrHandle deviceptr_create_mapped_graphics( + CUdeviceptr ptr, + const GraphicsResourceHandle& h_resource, + const StreamHandle& h_stream +) { + DeallocationStream ds; + if (!make_deallocation_stream(h_stream, ds)) { + return {}; + } + auto box = std::shared_ptr( + new DevicePtrBox{ptr, std::move(ds)}, + [h_resource](DevicePtrBox* b) { + GILReleaseGuard gil; + CUgraphicsResource resource = as_cu(h_resource); + const DeallocationStream& stream = b->deallocation; + cleanup_in_context( + deallocation_context(stream), "cuGraphicsUnmapResources", + [&]() noexcept { + return p_cuGraphicsUnmapResources( + 1, &resource, as_cu(stream.h_stream)); + }); + delete b; + } + ); + return DevicePtrHandle(box, &box->resource); +} + +// ============================================================================ +// MemoryResource-owned Device Pointer Handles +// ============================================================================ + +static MRDeallocCallback mr_dealloc_cb = nullptr; + +void register_mr_dealloc_callback(MRDeallocCallback cb) { + mr_dealloc_cb = cb; +} + +DevicePtrHandle deviceptr_create_with_mr(CUdeviceptr ptr, size_t size, PyObject* mr) { + if (!mr) { + return deviceptr_create_ref(ptr); + } + // GIL required when mr is provided + GILAcquireGuard gil; + if (!gil.acquired()) { + return deviceptr_create_ref(ptr); + } + Py_INCREF(mr); + auto box = std::shared_ptr( + new DevicePtrBox{ptr, DeallocationStream{}}, + [mr, size](DevicePtrBox* b) { + GILAcquireGuard gil; + if (gil.acquired()) { + if (mr_dealloc_cb) { + const DeallocationStream& stream = b->deallocation; + cleanup_in_context( + deallocation_context(stream), "MemoryResource.deallocate", + [&]() noexcept { + mr_dealloc_cb(mr, b->resource, size, stream.h_stream); + return CUDA_SUCCESS; + }); + } + Py_DECREF(mr); + } + delete b; + } + ); + return DevicePtrHandle(box, &box->resource); +} + +// ============================================================================ +// IPC Pointer Cache +// ============================================================================ +// This cache handles duplicate IPC imports, which behave differently depending +// on the memory type: +// +// 1. Memory pool allocations (DeviceMemoryResource): +// Multiple imports of the same allocation succeed and return duplicate +// pointers. However, the driver has a reference counting bug (nvbug 5570902) +// where the first cuMemFreeAsync incorrectly unmaps the memory even when +// imported multiple times. A driver fix is expected. +// +// 2. Pinned memory allocations (PinnedMemoryResource): +// Duplicate imports result in CUDA_ERROR_ALREADY_MAPPED. +// +// The cache solves both issues by checking the cache before calling +// cuMemPoolImportPointer and returning the existing handle for duplicate +// imports. This provides a consistent user experience where the same IPC +// descriptor can be imported multiple times regardless of memory type. +// +// The cache key is the export_data bytes (CUmemPoolPtrExportData), not the +// returned pointer, because we must check before calling the driver API. + + +// TODO: When driver fix for nvbug 5570902 is available, consider whether +// the cache is still needed for memory pool allocations (it will still be +// needed for pinned memory). +static bool use_ipc_ptr_cache() { + return true; +} + +namespace { +// Wrapper for CUmemPoolPtrExportData to use as map key +struct ExportDataKey { + CUmemPoolPtrExportData data; + + bool operator==(const ExportDataKey& other) const { + return std::memcmp(&data, &other.data, sizeof(data)) == 0; + } +}; + +struct ExportDataKeyHash { + std::size_t operator()(const ExportDataKey& key) const { + // Simple hash of the bytes + std::size_t h = 0; + const auto* bytes = reinterpret_cast(&key.data); + for (std::size_t i = 0; i < sizeof(key.data); ++i) { + h = h * 31 + bytes[i]; + } + return h; + } +}; + +} + +static HandleRegistry ipc_ptr_cache; +static std::mutex ipc_import_mutex; + +DevicePtrHandle deviceptr_import_ipc(const MemoryPoolHandle& h_pool, const void* export_data, const StreamHandle& h_stream) { + auto data = const_cast( + reinterpret_cast(export_data)); + + if (use_ipc_ptr_cache()) { + ExportDataKey key; + std::memcpy(&key.data, data, sizeof(key.data)); + + std::lock_guard lock(ipc_import_mutex); + + if (auto h = ipc_ptr_cache.lookup(key)) { + return h; + } + + GILReleaseGuard gil; + CUdeviceptr ptr; + if (CUDA_SUCCESS != (err = p_cuMemPoolImportPointer(&ptr, *h_pool, data))) { + return {}; + } + + DeallocationStream ds; + if (!make_deallocation_stream(h_stream, ds)) { + pw_cuMemFreeAsync(ptr, as_cu(h_stream)); + return {}; + } + + auto box = std::shared_ptr( + new DevicePtrBox{ptr, std::move(ds)}, + [h_pool, key](DevicePtrBox* b) { + ipc_ptr_cache.unregister_handle(key); + GILReleaseGuard gil; + const DeallocationStream& stream = b->deallocation; + cleanup_in_context( + deallocation_context(stream), "cuMemFreeAsync", + [&]() noexcept { + return p_cuMemFreeAsync( + b->resource, as_cu(stream.h_stream)); + }); + delete b; + } + ); + DevicePtrHandle h(box, &box->resource); + ipc_ptr_cache.register_handle(key, h); + return h; + + } else { + GILReleaseGuard gil; + CUdeviceptr ptr; + if (CUDA_SUCCESS != (err = p_cuMemPoolImportPointer(&ptr, *h_pool, data))) { + return {}; + } + + DeallocationStream ds; + if (!make_deallocation_stream(h_stream, ds)) { + pw_cuMemFreeAsync(ptr, as_cu(h_stream)); + return {}; + } + + auto box = std::shared_ptr( + new DevicePtrBox{ptr, std::move(ds)}, + [h_pool](DevicePtrBox* b) { + GILReleaseGuard gil; + const DeallocationStream& stream = b->deallocation; + cleanup_in_context( + deallocation_context(stream), "cuMemFreeAsync", + [&]() noexcept { + return p_cuMemFreeAsync( + b->resource, as_cu(stream.h_stream)); + }); + delete b; + } + ); + return DevicePtrHandle(box, &box->resource); + } +} + +// ============================================================================ +// File Descriptor Handles +// ============================================================================ + +FileDescriptorHandle create_fd_handle(int fd) { +#ifdef _WIN32 + throw std::runtime_error("create_fd_handle is not supported on Windows"); +#else + return FileDescriptorHandle( + new int(fd), + [](const int* p) { + if (::close(*p) != 0) { + report_message("close() failed for an IPC file descriptor; the descriptor may have leaked"); + } + delete p; + } + ); +#endif +} + +FileDescriptorHandle create_fd_handle_ref(int fd) { +#ifdef _WIN32 + throw std::runtime_error("create_fd_handle_ref is not supported on Windows"); +#else + return std::make_shared(fd); +#endif +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/program.cpp b/cuda_core/cuda/core/_cpp/rt/program.cpp new file mode 100644 index 00000000000..74c6455264a --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/program.cpp @@ -0,0 +1,243 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +// ============================================================================ +// Library Handles +// ============================================================================ + +namespace { +struct LibraryBox { + CUlibrary resource; +}; +} // namespace + +LibraryHandle create_library_handle_from_file(const char* path) { + GILReleaseGuard gil; + CUlibrary library; + if (CUDA_SUCCESS != (err = p_cuLibraryLoadFromFile(&library, path, nullptr, nullptr, 0, nullptr, nullptr, 0))) { + return {}; + } + + auto box = std::shared_ptr( + new LibraryBox{library}, + [](const LibraryBox* b) { + GILReleaseGuard gil; + // TODO: re-enable once LibraryBox tracks its owning context + // p_cuLibraryUnload(b->resource); + delete b; + } + ); + return LibraryHandle(box, &box->resource); +} + +LibraryHandle create_library_handle_from_data(const void* data) { + GILReleaseGuard gil; + CUlibrary library; + if (CUDA_SUCCESS != (err = p_cuLibraryLoadData(&library, data, nullptr, nullptr, 0, nullptr, nullptr, 0))) { + return {}; + } + + auto box = std::shared_ptr( + new LibraryBox{library}, + [](const LibraryBox* b) { + GILReleaseGuard gil; + // TODO: re-enable once LibraryBox tracks its owning context + // p_cuLibraryUnload(b->resource); + delete b; + } + ); + return LibraryHandle(box, &box->resource); +} + +LibraryHandle create_library_handle_ref(CUlibrary library) { + auto box = std::make_shared(LibraryBox{library}); + return LibraryHandle(box, &box->resource); +} + +// ============================================================================ +// Kernel Handles +// ============================================================================ + +namespace { +struct KernelBox { + CUkernel resource; + LibraryHandle h_library; +}; +} // namespace + +static const KernelBox* get_box(const KernelHandle& h) { + const CUkernel* p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) - offsetof(KernelBox, resource) + ); +} + +// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) +static HandleRegistry kernel_registry; + +KernelHandle create_kernel_handle(const LibraryHandle& h_library, const char* name) { + GILReleaseGuard gil; + CUkernel kernel; + if (CUDA_SUCCESS != (err = p_cuLibraryGetKernel(&kernel, *h_library, name))) { + return {}; + } + + auto box = std::make_shared(KernelBox{kernel, h_library}); + KernelHandle h(box, &box->resource); + kernel_registry.register_handle(kernel, h); + return h; +} + +KernelHandle create_kernel_handle_ref(CUkernel kernel) { + if (auto h = kernel_registry.lookup(kernel)) { + return h; + } + auto box = std::make_shared(KernelBox{kernel, {}}); + return KernelHandle(box, &box->resource); +} + +LibraryHandle get_kernel_library(const KernelHandle& h) noexcept { + if (!h) return {}; + return get_box(h)->h_library; +} + +// ============================================================================ +// NVRTC Program Handles +// ============================================================================ + +namespace { +struct NvrtcProgramBox { + nvrtcProgram resource; +}; +} // namespace + +NvrtcProgramHandle create_nvrtc_program_handle(nvrtcProgram prog) { + auto box = std::shared_ptr( + new NvrtcProgramBox{prog}, + [](NvrtcProgramBox* b) { + // Note: nvrtcDestroyProgram takes nvrtcProgram* and nulls it, + // but we're deleting the box anyway so nulling is harmless. + if (p_nvrtcDestroyProgram) { + GILReleaseGuard gil; + pw_nvrtcDestroyProgram(&b->resource); + } + delete b; + } + ); + return NvrtcProgramHandle(box, &box->resource); +} + +NvrtcProgramHandle create_nvrtc_program_handle_ref(nvrtcProgram prog) { + auto box = std::make_shared(NvrtcProgramBox{prog}); + return NvrtcProgramHandle(box, &box->resource); +} + +// ============================================================================ +// NVVM Program Handles +// ============================================================================ + +namespace { +struct NvvmProgramBox { + NvvmProgramValue resource; +}; +} // namespace + +NvvmProgramHandle create_nvvm_program_handle(nvvmProgram prog) { + auto box = std::shared_ptr( + new NvvmProgramBox{{prog}}, + [](NvvmProgramBox* b) { + // Note: nvvmDestroyProgram takes nvvmProgram* and nulls it, + // but we're deleting the box anyway so nulling is harmless. + // If NVVM is not available, the function pointer is null. + if (p_nvvmDestroyProgram) { + GILReleaseGuard gil; + pw_nvvmDestroyProgram(&b->resource.raw); + } + delete b; + } + ); + return NvvmProgramHandle(box, &box->resource); +} + +NvvmProgramHandle create_nvvm_program_handle_ref(nvvmProgram prog) { + auto box = std::make_shared(NvvmProgramBox{{prog}}); + return NvvmProgramHandle(box, &box->resource); +} + +// ============================================================================ +// nvJitLink Handles +// ============================================================================ + +namespace { +struct NvJitLinkBox { + NvJitLinkValue resource; +}; +} // namespace + +NvJitLinkHandle create_nvjitlink_handle(nvJitLink_t handle) { + auto box = std::shared_ptr( + new NvJitLinkBox{{handle}}, + [](NvJitLinkBox* b) { + // Note: nvJitLinkDestroy takes nvJitLinkHandle* and nulls it, + // but we're deleting the box anyway so nulling is harmless. + // If nvJitLink is not available, the function pointer is null. + if (p_nvJitLinkDestroy) { + GILReleaseGuard gil; + pw_nvJitLinkDestroy(&b->resource.raw); + } + delete b; + } + ); + return NvJitLinkHandle(box, &box->resource); +} + +NvJitLinkHandle create_nvjitlink_handle_ref(nvJitLink_t handle) { + auto box = std::make_shared(NvJitLinkBox{{handle}}); + return NvJitLinkHandle(box, &box->resource); +} + +// ============================================================================ +// cuLink Handles +// ============================================================================ + +namespace { +struct CuLinkBox { + CUlinkState resource; +}; +} // namespace + +CuLinkHandle create_culink_handle(CUlinkState state) { + auto box = std::shared_ptr( + new CuLinkBox{state}, + [](CuLinkBox* b) { + // cuLinkDestroy takes CUlinkState by value (not pointer). + if (p_cuLinkDestroy) { + GILReleaseGuard gil; + pw_cuLinkDestroy(b->resource); + } + delete b; + } + ); + return CuLinkHandle(box, &box->resource); +} + +CuLinkHandle create_culink_handle_ref(CUlinkState state) { + auto box = std::make_shared(CuLinkBox{state}); + return CuLinkHandle(box, &box->resource); +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/py.hpp b/cuda_core/cuda/core/_cpp/rt/py.hpp index 65179c1caca..b606398317e 100644 --- a/cuda_core/cuda/core/_cpp/rt/py.hpp +++ b/cuda_core/cuda/core/_cpp/rt/py.hpp @@ -39,6 +39,61 @@ inline bool py_is_finalizing() noexcept { #endif } +// Conditionally release the GIL while calling into the CUDA driver. +class GILReleaseGuard { +public: + GILReleaseGuard() noexcept { + if (!Py_IsInitialized() || py_is_finalizing()) { + return; + } + if (PyGILState_Check()) { + tstate_ = PyEval_SaveThread(); + } + } + + ~GILReleaseGuard() { + if (tstate_) { + PyEval_RestoreThread(tstate_); + } + } + + GILReleaseGuard(const GILReleaseGuard&) = delete; + GILReleaseGuard& operator=(const GILReleaseGuard&) = delete; + +private: + PyThreadState* tstate_ = nullptr; +}; + +// Helper to acquire the GIL when we might not hold it. +// Use in C++ destructors that need to manipulate Python objects. +class GILAcquireGuard { +public: + GILAcquireGuard() : acquired_(false) { + // Don't try to acquire GIL if Python is finalizing + if (!Py_IsInitialized() || py_is_finalizing()) { + return; + } + gstate_ = PyGILState_Ensure(); + acquired_ = true; + } + + ~GILAcquireGuard() { + if (acquired_) { + PyGILState_Release(gstate_); + } + } + + bool acquired() const { return acquired_; } + + // Non-copyable, non-movable + GILAcquireGuard(const GILAcquireGuard&) = delete; + GILAcquireGuard& operator=(const GILAcquireGuard&) = delete; + +private: + PyGILState_STATE gstate_; + bool acquired_; +}; + // as_py() - convert handle to Python wrapper object (returns new reference) namespace detail { // n.b. class lookup is not cached to avoid deadlock hazard, see DESIGN.md diff --git a/cuda_core/cuda/core/_cpp/rt/py_deferred_cleanup.cpp b/cuda_core/cuda/core/_cpp/rt/py_deferred_cleanup.cpp new file mode 100644 index 00000000000..126a90879b1 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/py_deferred_cleanup.cpp @@ -0,0 +1,157 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "internal.hpp" +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +// ============================================================================ +// CUDA user-object deferred cleanup +// +// CUDA invokes a user-object destructor on an internal thread where CUDA +// calls are forbidden. Payload cleanup can release resource handles whose +// deleters call CUDA, so the callback only transfers a preallocated intrusive +// node to this process-lifetime queue. One coalesced pending call drains all +// queued payloads from Python's main thread. +// ============================================================================ + +namespace { +// Process-lifetime MPSC queue that drains payloads from Python's main thread. +class DeferredCleanupQueue { +public: + // Transfer one preallocated cleanup item from a producer to the queue. + void enqueue(DeferredCleanupItem* item) noexcept { + DeferredCleanupItem* head = head_.load(std::memory_order_relaxed); + do { + item->next = head; + } while (!head_.compare_exchange_weak( + head, item, std::memory_order_release, std::memory_order_relaxed)); + schedule(); + } + + // Permanently disable pending-call scheduling during interpreter shutdown. + void stop() noexcept { + accepting_.store(false, std::memory_order_release); + } + + // Reattempt scheduling after Py_AddPendingCall() found CPython's bounded + // pending-call queue full and left payloads queued for a later safe entry. + void retry_schedule() noexcept { + schedule(); + } + +private: + // Adapt queue draining to CPython's int (*)(void*) callback ABI. + static int pending_call(void* arg) noexcept { + static_cast(arg)->drain(); + return 0; + } + + // Coalesce all queued work behind at most one CPython pending call. + void schedule() noexcept { + if (!accepting_.load(std::memory_order_acquire)) { + return; + } + if (!Py_IsInitialized() || py_is_finalizing()) { + stop(); + return; + } + if (!head_.load(std::memory_order_acquire)) { + return; + } + bool expected = false; + if (!scheduled_.compare_exchange_strong( + expected, true, std::memory_order_acq_rel, + std::memory_order_relaxed)) { + return; + } + if (Py_AddPendingCall(&DeferredCleanupQueue::pending_call, this) != 0) { + // Keep every payload queued. A later enqueue or safe cuda-core + // entry can retry without blocking CUDA's callback thread. + scheduled_.store(false, std::memory_order_release); + } + } + + // Detach and destroy all queued payloads from Python's main thread. + void drain() noexcept { + if (!Py_IsInitialized() || py_is_finalizing()) { + stop(); + scheduled_.store(false, std::memory_order_release); + return; // Intentionally leak intact payloads during shutdown. + } + + while (DeferredCleanupItem* list = + head_.exchange(nullptr, std::memory_order_acquire)) { + while (list) { + DeferredCleanupItem* next = list->next; + delete list; + list = next; + } + } + + scheduled_.store(false, std::memory_order_release); + if (head_.load(std::memory_order_acquire)) { + schedule(); + } + } + + // Head of the intrusive multi-producer, single-consumer payload stack. + std::atomic head_{nullptr}; + // True while one cuda-core drain callback is pending or executing. + std::atomic scheduled_{false}; + // False once shutdown begins, causing later payloads to be leaked safely. + std::atomic accepting_{true}; +}; + +// Published once at module initialization and intentionally never freed. +std::atomic deferred_cleanup_queue{nullptr}; +} // namespace + +namespace detail { +void ensure_deferred_cleanup_ready() { + DeferredCleanupQueue* queue = + deferred_cleanup_queue.load(std::memory_order_acquire); + if (!queue) { + throw std::runtime_error("deferred cleanup is not initialized"); + } + queue->retry_schedule(); +} + +// CUDA's CUhostFn ABI is void (*)(void*); recover and enqueue the cleanup item. +void enqueue_cleanup(void* item) noexcept { + auto* cleanup = static_cast(item); + if (DeferredCleanupQueue* queue = + deferred_cleanup_queue.load(std::memory_order_acquire)) { + queue->enqueue(cleanup); + } +} +} // namespace detail + +// Module initialization calls this once with the GIL held, which serializes +// the check, allocation, and publication below. +void initialize_deferred_cleanup() { + if (deferred_cleanup_queue.load(std::memory_order_acquire)) { + return; + } + auto* queue = new DeferredCleanupQueue(); + deferred_cleanup_queue.store(queue, std::memory_order_release); +} + +void retry_deferred_cleanup() noexcept { + if (!Py_IsInitialized() || py_is_finalizing()) { + return; + } + if (DeferredCleanupQueue* queue = + deferred_cleanup_queue.load(std::memory_order_acquire)) { + queue->retry_schedule(); + } +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/py_report.cpp b/cuda_core/cuda/core/_cpp/rt/py_report.cpp new file mode 100644 index 00000000000..a2e9badd029 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/py_report.cpp @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +namespace { +// Warning category registered by _utils/cuda_utils.pyx (cuda.core.CUDAWarning). +std::atomic warning_category{nullptr}; +} // namespace + +// Report a message that could not be raised. Emits cuda.core.CUDAWarning via +// the Python warnings machinery; if that itself fails (for example because the +// warning was promoted to an error), the failure is written as an unraisable +// exception, the CPython convention for exceptions in destructors. Falls back +// to stderr when the interpreter cannot be used. +void report_message(const char* message) noexcept { + PyObject* category = warning_category.load(std::memory_order_acquire); + if (category && Py_IsInitialized() && !py_is_finalizing()) { + GILAcquireGuard gil; + if (gil.acquired()) { + // Deleters can run while a Python exception is propagating; keep it. +#if PY_VERSION_HEX >= 0x030C0000 + PyObject* pending = PyErr_GetRaisedException(); +#else + PyObject *pending_type, *pending_value, *pending_tb; + PyErr_Fetch(&pending_type, &pending_value, &pending_tb); +#endif + if (PyErr_WarnEx(category, message, 1) != 0) { + PyObject* subject = PyUnicode_FromString(message); + PyErr_WriteUnraisable(subject); + Py_XDECREF(subject); + } +#if PY_VERSION_HEX >= 0x030C0000 + PyErr_SetRaisedException(pending); +#else + PyErr_Restore(pending_type, pending_value, pending_tb); +#endif + return; + } + } + std::fprintf(stderr, "%s\n", message); + std::fflush(stderr); +} + +void register_warning_category(PyObject* category) noexcept { + warning_category.store(category, std::memory_order_release); +} + +namespace { +// Attach `message` as a PEP 678 note to the exception currently being handled. +// Returns false when there is none or the interpreter cannot be used. +bool add_note_to_handled_exception(const char* message) noexcept { +#if PY_VERSION_HEX >= 0x030B0000 + if (!Py_IsInitialized() || py_is_finalizing()) { + return false; + } + GILAcquireGuard gil; + if (!gil.acquired()) { + return false; + } + PyObject* exc = PyErr_GetHandledException(); + if (!exc) { + return false; + } + PyObject* result = PyObject_CallMethod(exc, "add_note", "s", message); + Py_DECREF(exc); + if (!result) { + PyErr_Clear(); + return false; + } + Py_DECREF(result); + return true; +#else + (void)message; + return false; +#endif +} +} // namespace + +void note_or_report_cuda_error(const char* operation, CUresult status, const char* detail) noexcept { + if (status == CUDA_SUCCESS || status == CUDA_ERROR_DEINITIALIZED) { + return; + } + char message[512]; + format_cuda_error(message, sizeof(message), operation, status, detail); + if (!add_note_to_handled_exception(message)) { + report_message(message); + } +} + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/rt.cpp b/cuda_core/cuda/core/_cpp/rt/rt.cpp deleted file mode 100644 index 75bdf52fec1..00000000000 --- a/cuda_core/cuda/core/_cpp/rt/rt.cpp +++ /dev/null @@ -1,3349 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// SPDX-License-Identifier: Apache-2.0 - -#include - -#include "rt.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifndef _WIN32 -#include -#endif - -namespace cuda_core::rt { - -// ============================================================================ -// CUDA driver function pointers -// -// These are populated by _rt.pyx at module import time using -// function pointers extracted from cuda.bindings.cydriver.__pyx_capi__. -// ============================================================================ - -decltype(&cuGetErrorName) p_cuGetErrorName = nullptr; -decltype(&cuGetErrorString) p_cuGetErrorString = nullptr; - -decltype(&cuDevicePrimaryCtxRetain) p_cuDevicePrimaryCtxRetain = nullptr; -decltype(&cuDevicePrimaryCtxRelease) p_cuDevicePrimaryCtxRelease = nullptr; -decltype(&cuCtxGetCurrent) p_cuCtxGetCurrent = nullptr; -decltype(&cuCtxSetCurrent) p_cuCtxSetCurrent = nullptr; -decltype(&cuCtxSynchronize) p_cuCtxSynchronize = nullptr; -decltype(&cuCtxGetStreamPriorityRange) p_cuCtxGetStreamPriorityRange = nullptr; -decltype(&cuCtxGetDevice) p_cuCtxGetDevice = nullptr; -decltype(&cuGraphNodeSetParams) p_cuGraphNodeSetParams = nullptr; -decltype(&cuGreenCtxCreate) p_cuGreenCtxCreate = nullptr; -decltype(&cuGreenCtxDestroy) p_cuGreenCtxDestroy = nullptr; -decltype(&cuCtxFromGreenCtx) p_cuCtxFromGreenCtx = nullptr; -decltype(&cuDevResourceGenerateDesc) p_cuDevResourceGenerateDesc = nullptr; - -decltype(&cuGreenCtxStreamCreate) p_cuGreenCtxStreamCreate = nullptr; - -decltype(&cuStreamCreateWithPriority) p_cuStreamCreateWithPriority = nullptr; -decltype(&cuStreamDestroy) p_cuStreamDestroy = nullptr; -decltype(&cuStreamGetCtx) p_cuStreamGetCtx = nullptr; - -decltype(&cuEventCreate) p_cuEventCreate = nullptr; -decltype(&cuEventDestroy) p_cuEventDestroy = nullptr; -decltype(&cuIpcOpenEventHandle) p_cuIpcOpenEventHandle = nullptr; - -decltype(&cuDeviceGetCount) p_cuDeviceGetCount = nullptr; - -decltype(&cuMemPoolSetAccess) p_cuMemPoolSetAccess = nullptr; -decltype(&cuMemPoolDestroy) p_cuMemPoolDestroy = nullptr; -decltype(&cuMemPoolCreate) p_cuMemPoolCreate = nullptr; -decltype(&cuDeviceGetMemPool) p_cuDeviceGetMemPool = nullptr; -decltype(&cuMemPoolImportFromShareableHandle) p_cuMemPoolImportFromShareableHandle = nullptr; - -decltype(&cuMemAllocFromPoolAsync) p_cuMemAllocFromPoolAsync = nullptr; -decltype(&cuMemAllocAsync) p_cuMemAllocAsync = nullptr; -decltype(&cuMemAlloc) p_cuMemAlloc = nullptr; -decltype(&cuMemAllocHost) p_cuMemAllocHost = nullptr; - -decltype(&cuMemFreeAsync) p_cuMemFreeAsync = nullptr; -decltype(&cuMemFree) p_cuMemFree = nullptr; -decltype(&cuMemFreeHost) p_cuMemFreeHost = nullptr; - -decltype(&cuMemPoolImportPointer) p_cuMemPoolImportPointer = nullptr; - -decltype(&cuLibraryLoadFromFile) p_cuLibraryLoadFromFile = nullptr; -decltype(&cuLibraryLoadData) p_cuLibraryLoadData = nullptr; -decltype(&cuLibraryUnload) p_cuLibraryUnload = nullptr; -decltype(&cuLibraryGetKernel) p_cuLibraryGetKernel = nullptr; - -// Graph -decltype(&cuGraphDestroy) p_cuGraphDestroy = nullptr; -decltype(&cuGraphInstantiateWithParams) p_cuGraphInstantiateWithParams = nullptr; -decltype(&cuGraphExecUpdate) p_cuGraphExecUpdate = nullptr; -decltype(&cuGraphExecDestroy) p_cuGraphExecDestroy = nullptr; -decltype(&cuUserObjectCreate) p_cuUserObjectCreate = nullptr; -decltype(&cuUserObjectRelease) p_cuUserObjectRelease = nullptr; -decltype(&cuGraphRetainUserObject) p_cuGraphRetainUserObject = nullptr; -decltype(&cuGraphReleaseUserObject) p_cuGraphReleaseUserObject = nullptr; -decltype(&cuGraphNodeFindInClone) p_cuGraphNodeFindInClone = nullptr; -decltype(&cuGraphChildGraphNodeGetGraph) p_cuGraphChildGraphNodeGetGraph = nullptr; - -// Linker -decltype(&cuLinkDestroy) p_cuLinkDestroy = nullptr; - -// GL interop pointers -decltype(&cuGraphicsUnmapResources) p_cuGraphicsUnmapResources = nullptr; -decltype(&cuGraphicsUnregisterResource) p_cuGraphicsUnregisterResource = nullptr; - -decltype(&cuArray3DCreate) p_cuArray3DCreate = nullptr; -decltype(&cuArrayDestroy) p_cuArrayDestroy = nullptr; -decltype(&cuMipmappedArrayCreate) p_cuMipmappedArrayCreate = nullptr; -decltype(&cuMipmappedArrayDestroy) p_cuMipmappedArrayDestroy = nullptr; -decltype(&cuMipmappedArrayGetLevel) p_cuMipmappedArrayGetLevel = nullptr; -decltype(&cuTexObjectCreate) p_cuTexObjectCreate = nullptr; -decltype(&cuTexObjectDestroy) p_cuTexObjectDestroy = nullptr; -decltype(&cuSurfObjectCreate) p_cuSurfObjectCreate = nullptr; -decltype(&cuSurfObjectDestroy) p_cuSurfObjectDestroy = nullptr; - -// SM resource split (13.1+ — may be null on older drivers/bindings) -#if CUDA_VERSION >= 13010 -decltype(&cuDevSmResourceSplit) p_cuDevSmResourceSplit = nullptr; -#else -void* p_cuDevSmResourceSplit = nullptr; -#endif - -// cuMemcpyWithAttributesAsync (13.2+ — may be null on older drivers/bindings) -#if CUDA_VERSION >= 13020 -decltype(&cuMemcpyWithAttributesAsync) p_cuMemcpyWithAttributesAsync = nullptr; -#else -void* p_cuMemcpyWithAttributesAsync = nullptr; -#endif - -// NVRTC function pointers -decltype(&nvrtcDestroyProgram) p_nvrtcDestroyProgram = nullptr; - -// NVVM function pointers (may be null if NVVM is not available) -NvvmDestroyProgramFn p_nvvmDestroyProgram = nullptr; - -// nvJitLink function pointers (may be null if nvJitLink is not available) -NvJitLinkDestroyFn p_nvJitLinkDestroy = nullptr; - -// ============================================================================ -// GIL and scoped-context management helpers -// ============================================================================ - -namespace { - -// Conditionally release the GIL while calling into the CUDA driver. -class GILReleaseGuard { -public: - GILReleaseGuard() noexcept { - if (!Py_IsInitialized() || py_is_finalizing()) { - return; - } - if (PyGILState_Check()) { - tstate_ = PyEval_SaveThread(); - } - } - - ~GILReleaseGuard() { - if (tstate_) { - PyEval_RestoreThread(tstate_); - } - } - - GILReleaseGuard(const GILReleaseGuard&) = delete; - GILReleaseGuard& operator=(const GILReleaseGuard&) = delete; - -private: - PyThreadState* tstate_ = nullptr; -}; - -// Helper to acquire the GIL when we might not hold it. -// Use in C++ destructors that need to manipulate Python objects. -class GILAcquireGuard { -public: - GILAcquireGuard() : acquired_(false) { - // Don't try to acquire GIL if Python is finalizing - if (!Py_IsInitialized() || py_is_finalizing()) { - return; - } - gstate_ = PyGILState_Ensure(); - acquired_ = true; - } - - ~GILAcquireGuard() { - if (acquired_) { - PyGILState_Release(gstate_); - } - } - - bool acquired() const { return acquired_; } - - // Non-copyable, non-movable - GILAcquireGuard(const GILAcquireGuard&) = delete; - GILAcquireGuard& operator=(const GILAcquireGuard&) = delete; - -private: - PyGILState_STATE gstate_; - bool acquired_; -}; - -// ---------------------------------------------------------------------------- -// Non-propagating error reporting -// -// Deleters, CUDA callbacks and other non-propagating paths cannot raise. They -// report through report_cuda_error()/report_message(), which emit a -// cuda.core.CUDAWarning when the interpreter is usable and fall back to stderr -// otherwise. See docs/source/error_handling.rst for the policy. -// ---------------------------------------------------------------------------- - -// Warning category registered by _utils/cuda_utils.pyx (cuda.core.CUDAWarning). -std::atomic warning_category{nullptr}; - -// Thread-local detail attached to the next raised CUDAError with a matching -// status (see take_last_error_detail()). Written only by propagating helpers. -// The taken copy stays valid until the next take on the same thread. -thread_local char last_error_detail[512] = {0}; -thread_local char taken_error_detail[512] = {0}; -thread_local CUresult last_error_detail_status = CUDA_SUCCESS; - -// Thread-local fault injected into the next context restoration (tests only). -thread_local CUresult context_restore_fault = CUDA_SUCCESS; - -// Format " : : " for a failed CUDA call. -void format_cuda_error(char* buffer, size_t size, const char* operation, CUresult status, - const char* detail) noexcept { - const char* error_name = nullptr; - const char* error_description = nullptr; - bool decoded = p_cuGetErrorName && p_cuGetErrorString - && p_cuGetErrorName(status, &error_name) == CUDA_SUCCESS - && p_cuGetErrorString(status, &error_description) == CUDA_SUCCESS; - const char* outcome = detail ? detail : "failed"; - if (decoded) { - std::snprintf(buffer, size, "%s %s: %s: %s", operation, outcome, error_name, error_description); - } else { - std::snprintf(buffer, size, "%s %s (CUDA error %d)", operation, outcome, static_cast(status)); - } -} - -} // namespace - -// Report a message that could not be raised. Emits cuda.core.CUDAWarning via -// the Python warnings machinery; if that itself fails (for example because the -// warning was promoted to an error), the failure is written as an unraisable -// exception, the CPython convention for exceptions in destructors. Falls back -// to stderr when the interpreter cannot be used. -void report_message(const char* message) noexcept { - PyObject* category = warning_category.load(std::memory_order_acquire); - if (category && Py_IsInitialized() && !py_is_finalizing()) { - GILAcquireGuard gil; - if (gil.acquired()) { - // Deleters can run while a Python exception is propagating; keep it. -#if PY_VERSION_HEX >= 0x030C0000 - PyObject* pending = PyErr_GetRaisedException(); -#else - PyObject *pending_type, *pending_value, *pending_tb; - PyErr_Fetch(&pending_type, &pending_value, &pending_tb); -#endif - if (PyErr_WarnEx(category, message, 1) != 0) { - PyObject* subject = PyUnicode_FromString(message); - PyErr_WriteUnraisable(subject); - Py_XDECREF(subject); - } -#if PY_VERSION_HEX >= 0x030C0000 - PyErr_SetRaisedException(pending); -#else - PyErr_Restore(pending_type, pending_value, pending_tb); -#endif - return; - } - } - std::fprintf(stderr, "%s\n", message); - std::fflush(stderr); -} - -// Report a failed non-CUDA call (NVRTC, NVVM, nvJitLink) from a path that -// cannot raise. -void report_status_code(const char* operation, long code) noexcept { - char message[256]; - std::snprintf(message, sizeof(message), "%s failed (status %ld)", operation, code); - report_message(message); -} - -void register_warning_category(PyObject* category) noexcept { - warning_category.store(category, std::memory_order_release); -} - -// Report a failed CUDA call from a path that cannot raise. CUDA_ERROR_DEINITIALIZED -// is not reported: it means the driver is shutting down, which makes cleanup -// failures expected and uninteresting. -void report_cuda_error(const char* operation, CUresult status, const char* detail) noexcept { - if (status == CUDA_SUCCESS || status == CUDA_ERROR_DEINITIALIZED) { - return; - } - char message[512]; - format_cuda_error(message, sizeof(message), operation, status, detail); - report_message(message); -} - -namespace { - -// Attach `message` as a PEP 678 note to the exception currently being handled. -// Returns false when there is none or the interpreter cannot be used. -bool add_note_to_handled_exception(const char* message) noexcept { -#if PY_VERSION_HEX >= 0x030B0000 - if (!Py_IsInitialized() || py_is_finalizing()) { - return false; - } - GILAcquireGuard gil; - if (!gil.acquired()) { - return false; - } - PyObject* exc = PyErr_GetHandledException(); - if (!exc) { - return false; - } - PyObject* result = PyObject_CallMethod(exc, "add_note", "s", message); - Py_DECREF(exc); - if (!result) { - PyErr_Clear(); - return false; - } - Py_DECREF(result); - return true; -#else - (void)message; - return false; -#endif -} - -} // namespace - -void note_or_report_cuda_error(const char* operation, CUresult status, const char* detail) noexcept { - if (status == CUDA_SUCCESS || status == CUDA_ERROR_DEINITIALIZED) { - return; - } - char message[512]; - format_cuda_error(message, sizeof(message), operation, status, detail); - if (!add_note_to_handled_exception(message)) { - report_message(message); - } -} - -const char* take_last_error_detail(CUresult status) noexcept { - if (!last_error_detail[0] || status != last_error_detail_status) { - return nullptr; - } - std::memcpy(taken_error_detail, last_error_detail, sizeof(taken_error_detail)); - clear_last_error_detail(); - return taken_error_detail; -} - -void clear_last_error_detail() noexcept { - last_error_detail[0] = 0; - last_error_detail_status = CUDA_SUCCESS; -} - -void set_context_restore_fault_for_testing(CUresult status) noexcept { - context_restore_fault = status; -} - -namespace { - -// Make a context current and record the state needed to restore it. -// An empty handle is a no-op: the operation runs in the caller's current -// context, and nothing is restored on exit. invoke_in_context and -// invoke_in_context_or_undo reject empty handles before getting here; only -// graph_node_set_params relies on the no-op (pre-13.2 node updates run in the -// caller's context). -CUresult enter_context(const ContextHandle& h_context, CUcontext* previous, int* changed) noexcept { - *previous = nullptr; - *changed = 0; - clear_last_error_detail(); - CUcontext target = as_cu(h_context); - if (!target) { - return CUDA_SUCCESS; - } - - GILReleaseGuard gil; - CUresult status = p_cuCtxGetCurrent(previous); - if (status != CUDA_SUCCESS || *previous == target) { - return status; - } - status = p_cuCtxSetCurrent(target); - *changed = status == CUDA_SUCCESS; - return status; -} - -// Restore the caller's context. Returns the restoration status. -CUresult restore_context(CUcontext previous) noexcept { - if (context_restore_fault != CUDA_SUCCESS) { - // Test hook: behave as if cuCtxSetCurrent(previous) failed, leaving the - // target context current exactly as a real failure would. - CUresult fault = context_restore_fault; - context_restore_fault = CUDA_SUCCESS; - return fault; - } - GILReleaseGuard gil; - return p_cuCtxSetCurrent(previous); -} - -// Record that the caller's context was not restored as the detail of the -// CUresult about to be returned and raised: the operation status if the -// operation failed too, else the restoration status. For a double failure the -// detail also names the restoration error, which the raised error does not. -void note_context_not_restored(CUcontext previous, CUresult operation_status, - CUresult restore_status) noexcept { - CUcontext current = nullptr; - if (p_cuCtxGetCurrent(¤t) != CUDA_SUCCESS) { - current = nullptr; - } - char cause[128] = {0}; - if (operation_status != CUDA_SUCCESS) { - const char* error_name = nullptr; - if (p_cuGetErrorName && p_cuGetErrorName(restore_status, &error_name) == CUDA_SUCCESS) { - std::snprintf(cause, sizeof(cause), " after this failure (cuCtxSetCurrent: %s)", error_name); - } else { - std::snprintf(cause, sizeof(cause), " after this failure (cuCtxSetCurrent: CUDA error %d)", - static_cast(restore_status)); - } - } - std::snprintf(last_error_detail, sizeof(last_error_detail), - "the calling thread's CUDA context (%#llx) could not be restored%s; " - "context %#llx is now current. Call Device.set_current() before issuing " - "further CUDA work on this thread", - static_cast(reinterpret_cast(previous)), - cause, - static_cast(reinterpret_cast(current))); - last_error_detail_status = operation_status != CUDA_SUCCESS ? operation_status : restore_status; -} - -// Restore the previous context and preserve an earlier operation error. The -// operation error, if any, is returned; otherwise the restoration status is. -// Either way a restoration failure is recorded as the detail of the returned -// status, so the eventual CUDAError explains it (see take_last_error_detail()). -CUresult exit_context(CUcontext previous, int changed, CUresult operation_status) noexcept { - CUresult restore_status = changed ? restore_context(previous) : CUDA_SUCCESS; - if (restore_status == CUDA_SUCCESS) { - return operation_status; - } - note_context_not_restored(previous, operation_status, restore_status); - return operation_status != CUDA_SUCCESS ? operation_status : restore_status; -} - -// Require a callable to be invocable without throwing. -#define ASSERT_NOTHROW_INVOCABLE(...) \ - static_assert(std::is_nothrow_invocable_v<__VA_ARGS__>, "operation must be noexcept") - -// Store a stream and any state needed to preserve deallocation ordering. -struct DeallocationStream { - StreamHandle h_stream; - std::thread::id ptds_tid{}; -}; - -// Return whether a stream handle needs a current context to resolve it. -bool is_default_stream(CUstream stream) noexcept { - return stream == nullptr || stream == CU_STREAM_LEGACY || stream == CU_STREAM_PER_THREAD; -} - -// Return the context a deallocation-stream token must run under. Real streams -// resolve their own context; default-stream tokens use the context bound at -// allocation time. Warn when PTDS deallocation crosses host threads. -ContextHandle deallocation_context(const DeallocationStream& stream) noexcept { - if (!is_default_stream(as_cu(stream.h_stream))) { - return {}; - } - if (stream.ptds_tid != std::thread::id{} - && stream.ptds_tid != std::this_thread::get_id()) { - report_message( - "Buffer deallocation for a per-thread default stream " - "is running on a different host thread than the one that recorded " - "the deallocation stream; ordering relative to the allocating " - "thread's PTDS is not preserved"); - } - return get_stream_context(stream.h_stream); -} - -// Run an operation with the requested context current. -template -CUresult invoke_in_context(const ContextHandle& h_context, Fn&& operation, Args&&... args) noexcept { - ASSERT_NOTHROW_INVOCABLE(Fn&&, Args&&...); - if (!h_context) { - return CUDA_ERROR_INVALID_CONTEXT; - } - CUcontext previous = nullptr; - int changed = 0; - CUresult status = enter_context(h_context, &previous, &changed); - if (status == CUDA_SUCCESS) { - status = std::invoke(std::forward(operation), std::forward(args)...); - } - return exit_context(previous, changed, status); -} - -// Run a creation operation and undo it if context restoration fails. -// Context-independent undo always runs. Context-sensitive undo runs only -// after verifying that the target context remains current; otherwise the -// resource leaks rather than risking cleanup in the wrong context. -template -CUresult invoke_in_context_or_undo(const ContextHandle& h_context, Fn&& operation, - Undo&& undo, bool undo_requires_target_context) noexcept { - ASSERT_NOTHROW_INVOCABLE(Fn&&); - ASSERT_NOTHROW_INVOCABLE(Undo&&); - if (!h_context) { - return CUDA_ERROR_INVALID_CONTEXT; - } - CUcontext previous = nullptr; - int changed = 0; - CUresult status = enter_context(h_context, &previous, &changed); - if (status != CUDA_SUCCESS) { - return status; - } - status = std::invoke(std::forward(operation)); - CUresult composite = exit_context(previous, changed, status); - if (status == CUDA_SUCCESS && composite != CUDA_SUCCESS) { - bool undo_ok = true; - if (undo_requires_target_context) { - CUcontext current = nullptr; - undo_ok = p_cuCtxGetCurrent(¤t) == CUDA_SUCCESS - && current == as_cu(h_context); - } - if (undo_ok) { - std::invoke(std::forward(undo)); - } else { - report_cuda_error( - "cuCtxSetCurrent (restoring the caller's context)", composite, - "failed; cleanup of the new resource skipped because its context " - "is no longer current (resource leaked)"); - } - } - return composite; -} - -// Run cleanup with the requested context current. Warn and skip the operation -// if activation fails, and independently warn on operation or restoration -// failure. Return the operation or activation status; restoration never -// changes the return value. -template -CUresult cleanup_in_context(const ContextHandle& h_context, const char* name, - Fn&& operation, Args&&... args) noexcept { - ASSERT_NOTHROW_INVOCABLE(Fn&&, Args&&...); - CUcontext previous = nullptr; - int changed = 0; - CUresult status = enter_context(h_context, &previous, &changed); - if (status != CUDA_SUCCESS) { - report_cuda_error(name, status, - "skipped (context activation failed; resource leaked)"); - } else { - status = std::invoke(std::forward(operation), std::forward(args)...); - if (status != CUDA_SUCCESS) { - report_cuda_error(name, status); - } - } - CUresult restore = exit_context(previous, changed, CUDA_SUCCESS); - if (restore != CUDA_SUCCESS) { - // Nothing is raised here, so the detail exit_context recorded has no - // exception to attach to: report it and drop the detail. - report_cuda_error(name, restore, "failed while restoring the caller's context"); - clear_last_error_detail(); - } - return status; -} - -#undef ASSERT_NOTHROW_INVOCABLE - -// Decorate a status-returning cleanup call to report whenever it fails. CUDA -// calls (CUresult) are reported with the error name and description; NVRTC, -// NVVM and nvJitLink calls (integer status codes) with the raw code. -template -class WarnOnFailure { -public: - explicit WarnOnFailure(const char* operation) noexcept : operation_(operation) {} - - template - auto operator()(Args&&... args) const noexcept { - auto status = Function(std::forward(args)...); - report(status); - return status; - } - -private: - void report(CUresult status) const noexcept { - report_cuda_error(operation_, status); - } - - template - void report(Status status) const noexcept { - if (static_cast(status) != 0) { - report_status_code(operation_, static_cast(status)); - } - } - - const char* operation_; -}; - -// Warning-decorated CUDA operations used by non-throwing cleanup paths. -const WarnOnFailure pw_cuStreamDestroy{"cuStreamDestroy"}; -const WarnOnFailure pw_cuEventDestroy{"cuEventDestroy"}; -const WarnOnFailure pw_cuMemFree{"cuMemFree"}; -const WarnOnFailure pw_cuMemFreeAsync{"cuMemFreeAsync"}; -const WarnOnFailure pw_cuArrayDestroy{"cuArrayDestroy"}; -const WarnOnFailure pw_cuMipmappedArrayDestroy{"cuMipmappedArrayDestroy"}; -const WarnOnFailure pw_cuTexObjectDestroy{"cuTexObjectDestroy"}; -const WarnOnFailure pw_cuSurfObjectDestroy{"cuSurfObjectDestroy"}; -const WarnOnFailure pw_cuGreenCtxDestroy{"cuGreenCtxDestroy"}; -const WarnOnFailure pw_cuMemPoolDestroy{"cuMemPoolDestroy"}; -const WarnOnFailure pw_cuMemFreeHost{"cuMemFreeHost"}; -const WarnOnFailure pw_cuGraphDestroy{"cuGraphDestroy"}; -const WarnOnFailure pw_cuGraphExecDestroy{"cuGraphExecDestroy"}; -const WarnOnFailure pw_cuGraphicsUnregisterResource{"cuGraphicsUnregisterResource"}; -const WarnOnFailure pw_cuLinkDestroy{"cuLinkDestroy"}; -const WarnOnFailure pw_cuUserObjectRelease{"cuUserObjectRelease"}; -const WarnOnFailure pw_cuGraphReleaseUserObject{"cuGraphReleaseUserObject"}; -const WarnOnFailure pw_nvrtcDestroyProgram{"nvrtcDestroyProgram"}; -const WarnOnFailure pw_nvvmDestroyProgram{"nvvmDestroyProgram"}; -const WarnOnFailure pw_nvJitLinkDestroy{"nvJitLinkDestroy"}; - -} // namespace - -// Synchronize the provided context. -CUresult context_synchronize(const ContextHandle& h_context) noexcept { - GILReleaseGuard gil; - return invoke_in_context(h_context, []() noexcept { - return p_cuCtxSynchronize(); - }); -} - -// Query the stream priority range for the provided context. -CUresult context_get_stream_priority_range(const ContextHandle& h_context, - int* least_priority, - int* greatest_priority) noexcept { - GILReleaseGuard gil; - return invoke_in_context(h_context, [&]() noexcept { - return p_cuCtxGetStreamPriorityRange(least_priority, greatest_priority); - }); -} - -// Query the device of the provided context. -CUresult context_get_device(const ContextHandle& h_context, CUdevice* device) noexcept { - return invoke_in_context(h_context, [&]() noexcept { - return p_cuCtxGetDevice(device); - }); -} - -// Set a graph node's parameters with h_context current (an empty handle runs in -// the caller's context). Returns the cuGraphNodeSetParams status. A failure to -// restore the caller's context is returned separately in *restore_status so the -// caller can publish the metadata that depends on the successful update before -// raising it; if the update itself failed, its status is returned with the -// restoration failure recorded as its detail and *restore_status is CUDA_SUCCESS. -CUresult graph_node_set_params(CUgraphNode node, CUgraphNodeParams* params, - const ContextHandle& h_context, - CUresult* restore_status) noexcept { - *restore_status = CUDA_SUCCESS; - if (!p_cuGraphNodeSetParams) { - return CUDA_ERROR_NOT_SUPPORTED; - } - CUcontext previous = nullptr; - int changed = 0; - CUresult status = enter_context(h_context, &previous, &changed); - if (status != CUDA_SUCCESS) { - return status; - } - { - GILReleaseGuard gil; - status = p_cuGraphNodeSetParams(node, params); - } - if (!changed) { - return status; - } - CUresult restored = restore_context(previous); - if (restored == CUDA_SUCCESS) { - return status; - } - note_context_not_restored(previous, status, restored); - if (status == CUDA_SUCCESS) { - *restore_status = restored; - } - return status; -} - -// ============================================================================ -// CUDA user-object deferred cleanup -// -// CUDA invokes a user-object destructor on an internal thread where CUDA -// calls are forbidden. Payload cleanup can release resource handles whose -// deleters call CUDA, so the callback only transfers a preallocated intrusive -// node to this process-lifetime queue. One coalesced pending call drains all -// queued payloads from Python's main thread. -// ============================================================================ - -// Intrusive base for payloads transferred out of CUDA's callback. -struct DeferredCleanupItem { - DeferredCleanupItem* next = nullptr; - virtual ~DeferredCleanupItem() noexcept = default; -}; - -namespace { - -// Process-lifetime MPSC queue that drains payloads from Python's main thread. -class DeferredCleanupQueue { -public: - // Transfer one preallocated cleanup item from a producer to the queue. - void enqueue(DeferredCleanupItem* item) noexcept { - DeferredCleanupItem* head = head_.load(std::memory_order_relaxed); - do { - item->next = head; - } while (!head_.compare_exchange_weak( - head, item, std::memory_order_release, std::memory_order_relaxed)); - schedule(); - } - - // Permanently disable pending-call scheduling during interpreter shutdown. - void stop() noexcept { - accepting_.store(false, std::memory_order_release); - } - - // Reattempt scheduling after Py_AddPendingCall() found CPython's bounded - // pending-call queue full and left payloads queued for a later safe entry. - void retry_schedule() noexcept { - schedule(); - } - -private: - // Adapt queue draining to CPython's int (*)(void*) callback ABI. - static int pending_call(void* arg) noexcept { - static_cast(arg)->drain(); - return 0; - } - - // Coalesce all queued work behind at most one CPython pending call. - void schedule() noexcept { - if (!accepting_.load(std::memory_order_acquire)) { - return; - } - if (!Py_IsInitialized() || py_is_finalizing()) { - stop(); - return; - } - if (!head_.load(std::memory_order_acquire)) { - return; - } - bool expected = false; - if (!scheduled_.compare_exchange_strong( - expected, true, std::memory_order_acq_rel, - std::memory_order_relaxed)) { - return; - } - if (Py_AddPendingCall(&DeferredCleanupQueue::pending_call, this) != 0) { - // Keep every payload queued. A later enqueue or safe cuda-core - // entry can retry without blocking CUDA's callback thread. - scheduled_.store(false, std::memory_order_release); - } - } - - // Detach and destroy all queued payloads from Python's main thread. - void drain() noexcept { - if (!Py_IsInitialized() || py_is_finalizing()) { - stop(); - scheduled_.store(false, std::memory_order_release); - return; // Intentionally leak intact payloads during shutdown. - } - - while (DeferredCleanupItem* list = - head_.exchange(nullptr, std::memory_order_acquire)) { - while (list) { - DeferredCleanupItem* next = list->next; - delete list; - list = next; - } - } - - scheduled_.store(false, std::memory_order_release); - if (head_.load(std::memory_order_acquire)) { - schedule(); - } - } - - // Head of the intrusive multi-producer, single-consumer payload stack. - std::atomic head_{nullptr}; - // True while one cuda-core drain callback is pending or executing. - std::atomic scheduled_{false}; - // False once shutdown begins, causing later payloads to be leaked safely. - std::atomic accepting_{true}; -}; - -// Published once at module initialization and intentionally never freed. -std::atomic deferred_cleanup_queue{nullptr}; - -void ensure_deferred_cleanup_ready() { - DeferredCleanupQueue* queue = - deferred_cleanup_queue.load(std::memory_order_acquire); - if (!queue) { - throw std::runtime_error("deferred cleanup is not initialized"); - } - queue->retry_schedule(); -} - -// CUDA's CUhostFn ABI is void (*)(void*); recover and enqueue the cleanup item. -void enqueue_cleanup(void* item) noexcept { - auto* cleanup = static_cast(item); - if (DeferredCleanupQueue* queue = - deferred_cleanup_queue.load(std::memory_order_acquire)) { - queue->enqueue(cleanup); - } -} - -} // namespace - -// Module initialization calls this once with the GIL held, which serializes -// the check, allocation, and publication below. -void initialize_deferred_cleanup() { - if (deferred_cleanup_queue.load(std::memory_order_acquire)) { - return; - } - auto* queue = new DeferredCleanupQueue(); - deferred_cleanup_queue.store(queue, std::memory_order_release); -} - -void retry_deferred_cleanup() noexcept { - if (!Py_IsInitialized() || py_is_finalizing()) { - return; - } - if (DeferredCleanupQueue* queue = - deferred_cleanup_queue.load(std::memory_order_acquire)) { - queue->retry_schedule(); - } -} - -// ============================================================================ -// Handle reverse-lookup registry -// -// Maps raw CUDA handles (CUevent, CUkernel, etc.) back to their owning -// shared_ptr so that _ref constructors can recover full metadata. -// Uses weak_ptr to avoid preventing destruction. -// ============================================================================ - -template> -class HandleRegistry { -public: - using MapType = std::unordered_map, Hash>; - - void register_handle(const Key& key, const Handle& h) { - std::lock_guard lock(mutex_); - map_[key] = h; - } - - void unregister_handle(const Key& key) noexcept { - std::lock_guard lock(mutex_); - map_.erase(key); - } - - void register_handles(const std::vector& handles) { - std::lock_guard lock(mutex_); - for (const Handle& h : handles) { - if (h) { - map_[*h] = h; - } - } - } - - Handle lookup(const Key& key) { - std::lock_guard lock(mutex_); - auto it = map_.find(key); - if (it != map_.end()) { - if (auto h = it->second.lock()) { - return h; - } - map_.erase(it); - } - return {}; - } - - template - Handle get_or_create(const Key& key, Factory&& create) { - std::lock_guard lock(mutex_); - auto it = map_.find(key); - if (it != map_.end()) { - if (Handle h = it->second.lock()) { - return h; - } - map_.erase(it); - } - - Handle h = create(); - if (h) { - map_[key] = h; - } - return h; - } - - MapType drain() noexcept { - std::lock_guard lock(mutex_); - MapType extracted; - extracted.swap(map_); - return extracted; - } - -private: - std::mutex mutex_; - MapType map_; -}; - -// ============================================================================ -// Thread-local error handling -// ============================================================================ - -// Thread-local status of the most recent CUDA API call in this module. -static thread_local CUresult err = CUDA_SUCCESS; - -// Return and clear the calling thread's most recent CUDA error. -CUresult get_last_error() noexcept { - CUresult e = err; - err = CUDA_SUCCESS; - return e; -} - -// Return the calling thread's most recent CUDA error without clearing it. -CUresult peek_last_error() noexcept { - return err; -} - -void clear_last_error() noexcept { - err = CUDA_SUCCESS; -} - -// ============================================================================ -// Context Handles -// ============================================================================ - -namespace { -struct ContextBox { - CUcontext resource; - GreenCtxHandle h_green_ctx; -}; - -struct GreenCtxBox { - CUgreenCtx resource; -}; - -static const ContextBox* get_box(const ContextHandle& h) noexcept { - const CUcontext* p = h.get(); - return reinterpret_cast( - reinterpret_cast(p) - offsetof(ContextBox, resource) - ); -} - -// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) -static HandleRegistry context_registry; - -// Create a context handle reference, with optional green context as source. -ContextHandle create_context_handle_ref(CUcontext ctx, GreenCtxHandle h_green_ctx) { - if (!ctx) { - return {}; - } - if (auto h = context_registry.lookup(ctx)) { - return h; - } - auto box = std::shared_ptr( - new ContextBox{ctx, std::move(h_green_ctx)}, - [](const ContextBox* b) { - context_registry.unregister_handle(b->resource); - delete b; - } - ); - ContextHandle h(box, &box->resource); - context_registry.register_handle(ctx, h); - return h; -} -} // namespace - -ContextHandle create_context_handle_ref(CUcontext ctx) { - return create_context_handle_ref(ctx, {}); -} - -ContextHandle create_context_handle_from_green_ctx(const GreenCtxHandle& h_green_ctx) { - GILReleaseGuard gil; - if (!h_green_ctx) { - return {}; - } - if (!p_cuCtxFromGreenCtx) { - err = CUDA_ERROR_NOT_SUPPORTED; - return {}; - } - - CUcontext ctx = nullptr; - if (CUDA_SUCCESS != (err = p_cuCtxFromGreenCtx(&ctx, as_cu(h_green_ctx)))) { - return {}; - } - - return create_context_handle_ref(ctx, h_green_ctx); -} - -GreenCtxHandle get_context_green_ctx(const ContextHandle& h) noexcept { - if (!h) { - return {}; - } - return get_box(h)->h_green_ctx; -} - -GreenCtxHandle create_green_ctx_handle(CUdevResource* resources, unsigned int nbResources, - CUdevice dev, unsigned int flags) { - GILReleaseGuard gil; - if (!p_cuDevResourceGenerateDesc || !p_cuGreenCtxCreate || !p_cuGreenCtxDestroy) { - err = CUDA_ERROR_NOT_SUPPORTED; - return {}; - } - - CUdevResourceDesc desc = nullptr; - if (CUDA_SUCCESS != (err = p_cuDevResourceGenerateDesc(&desc, resources, nbResources))) { - return {}; - } - - CUgreenCtx green_ctx = nullptr; - if (CUDA_SUCCESS != (err = p_cuGreenCtxCreate(&green_ctx, desc, dev, flags))) { - return {}; - } - - auto box = std::shared_ptr( - new GreenCtxBox{green_ctx}, - [](const GreenCtxBox* b) { - GILReleaseGuard gil; - pw_cuGreenCtxDestroy(b->resource); - delete b; - } - ); - return GreenCtxHandle(box, &box->resource); -} - -GreenCtxHandle create_green_ctx_handle_ref(CUgreenCtx green_ctx) { - if (!green_ctx) { - return {}; - } - auto box = std::make_shared(GreenCtxBox{green_ctx}); - return GreenCtxHandle(box, &box->resource); -} - -// Thread-local cache of primary contexts indexed by device ID -static thread_local std::vector primary_context_cache; - -ContextHandle get_primary_context(int device_id) { - // Check thread-local cache - if (static_cast(device_id) < primary_context_cache.size()) { - if (auto cached = primary_context_cache[device_id]) { - return cached; - } - } - - // Cache miss - acquire primary context from driver - GILReleaseGuard gil; - CUcontext ctx; - if (CUDA_SUCCESS != (err = p_cuDevicePrimaryCtxRetain(&ctx, device_id))) { - return {}; - } - - auto box = std::shared_ptr( - new ContextBox{ctx, {}}, - [device_id](const ContextBox* b) { - context_registry.unregister_handle(b->resource); - GILReleaseGuard gil; - p_cuDevicePrimaryCtxRelease(device_id); - delete b; - } - ); - auto h = ContextHandle(box, &box->resource); - context_registry.register_handle(ctx, h); - - // Update cache - if (static_cast(device_id) >= primary_context_cache.size()) { - primary_context_cache.resize(device_id + 1); - } - primary_context_cache[device_id] = h; - return h; -} - -ContextHandle get_current_context() { - GILReleaseGuard gil; - CUcontext ctx = nullptr; - if (CUDA_SUCCESS != (err = p_cuCtxGetCurrent(&ctx))) { - return {}; - } - if (!ctx) { - return {}; // No current context (not an error) - } - return create_context_handle_ref(ctx); -} - -// ============================================================================ -// Stream Handles -// ============================================================================ - -namespace { -struct StreamBox { - CUstream resource; - ContextHandle h_context; -}; - -static const StreamBox* get_box(const StreamHandle& h) noexcept { - const CUstream* p = h.get(); - return reinterpret_cast( - reinterpret_cast(p) - offsetof(StreamBox, resource) - ); -} - -// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) -static HandleRegistry stream_registry; -} // namespace - -StreamHandle create_stream_handle(const ContextHandle& h_ctx, unsigned int flags, int priority) { - GILReleaseGuard gil; - CUstream stream = nullptr; - GreenCtxHandle h_green = get_context_green_ctx(h_ctx); - if (h_green) { - err = p_cuGreenCtxStreamCreate - ? p_cuGreenCtxStreamCreate(&stream, as_cu(h_green), flags, priority) - : CUDA_ERROR_NOT_SUPPORTED; - } else { - err = invoke_in_context_or_undo( - h_ctx, - [&]() noexcept { return p_cuStreamCreateWithPriority(&stream, flags, priority); }, - [&]() noexcept { pw_cuStreamDestroy(stream); }, - /*undo_requires_target_context=*/false); - } - if (err != CUDA_SUCCESS) { - return {}; - } - - auto box = std::shared_ptr( - new StreamBox{stream, h_ctx}, - [](const StreamBox* b) { - stream_registry.unregister_handle(b->resource); - GILReleaseGuard gil; - pw_cuStreamDestroy(b->resource); - delete b; - } - ); - StreamHandle h(box, &box->resource); - stream_registry.register_handle(stream, h); - return h; -} - -StreamHandle create_stream_handle_ref(CUstream stream) { - if (auto h = stream_registry.lookup(stream)) { - return h; - } - auto box = std::shared_ptr( - new StreamBox{stream, {}}, - [](const StreamBox* b) { - stream_registry.unregister_handle(b->resource); - delete b; - } - ); - StreamHandle h(box, &box->resource); - stream_registry.register_handle(stream, h); - return h; -} - -StreamHandle create_stream_handle_with_owner(CUstream stream, PyObject* owner) { - if (auto h = stream_registry.lookup(stream)) { - // Reuse handles that already carry structural context metadata, e.g. - // cuda-core-owned streams. - if (get_box(h)->h_context) { - return h; - } - } - if (!owner) { - return create_stream_handle_ref(stream); - } - // GIL required when owner is provided - GILAcquireGuard gil; - if (!gil.acquired()) { - // Python finalizing - fall back to ref version (no owner tracking) - return create_stream_handle_ref(stream); - } - Py_INCREF(owner); - // Owner-backed handles are NOT registered in the stream registry to avoid - // corruption when multiple owners wrap the same CUstream (each stacks its - // own Py_INCREF/Py_DECREF independently). - auto box = std::shared_ptr( - new StreamBox{stream, {}}, - [owner](const StreamBox* b) { - GILAcquireGuard gil; - if (gil.acquired()) { - Py_DECREF(owner); - } - delete b; - } - ); - return StreamHandle(box, &box->resource); -} - -// Return the context retained by a stream handle. -ContextHandle get_stream_context(const StreamHandle& h) noexcept { - return h ? get_box(h)->h_context : ContextHandle{}; -} - -StreamHandle get_legacy_stream() { - static StreamHandle handle = create_stream_handle_ref(CU_STREAM_LEGACY); - return handle; -} - -StreamHandle get_per_thread_stream() { - static StreamHandle handle = create_stream_handle_ref(CU_STREAM_PER_THREAD); - return handle; -} - -StreamHandle create_context_bound_legacy_stream(const ContextHandle& h_context) { - if (!h_context) { - return {}; - } - // Default deleter: this handle never owns CU_STREAM_LEGACY, so nothing - // needs to run when the last reference is released. - auto box = std::make_shared(StreamBox{CU_STREAM_LEGACY, h_context}); - return StreamHandle(box, &box->resource); -} - -// ============================================================================ -// Deallocation streams -// -// A DeallocationStream is a StreamHandle used for ordering frees. It differs -// from an ordinary StreamHandle only for default-stream tokens, for which it -// stores the (de)allocation context. Ordinarily, the LEGACY and PER_THREAD -// default streams resolve to whichever context is active at the time they are -// used, but for storing deallocation recipes we need to pin the context. With -// the PER_THREAD token, it is not possible to restore the original stream when -// deallocation runs on a different thread. Therefore, in that case the -// allocating host thread id is also stored so that cross-thread frees can be -// detected and warnings can be issued. -// ============================================================================ - -// Real streams are copied unchanged. Default-stream tokens without an embedded -// context are bound to the current context. Returns false (and sets err) when a -// default-stream token cannot be bound because no context is current. -static bool make_deallocation_stream( - const StreamHandle& h, DeallocationStream& out) noexcept { - out = {}; - if (!h) { - return true; - } - - const CUstream stream = as_cu(h); - if (!is_default_stream(stream)) { - out = DeallocationStream{h, {}}; - return true; - } - - StreamHandle h_bound = h; - if (!get_stream_context(h)) { - ContextHandle h_ctx = get_current_context(); - if (!h_ctx) { - if (err == CUDA_SUCCESS) { - err = CUDA_ERROR_INVALID_CONTEXT; - } - return false; - } - // Do not register in stream_registry: the token value alone is not - // a unique stream identity (context is part of the meaning). - auto box = std::shared_ptr( - new StreamBox{stream, h_ctx}); - h_bound = StreamHandle(box, &box->resource); - } - - std::thread::id ptds_tid{}; - if (stream == CU_STREAM_PER_THREAD) { - ptds_tid = std::this_thread::get_id(); - } - out = DeallocationStream{std::move(h_bound), ptds_tid}; - return true; -} - -// ============================================================================ -// Event Handles -// ============================================================================ - -namespace { -struct EventBox { - CUevent resource; - bool timing_enabled; - bool is_blocking_sync; - bool ipc_enabled; - int device_id; - ContextHandle h_context; -}; -} // namespace - -static const EventBox* get_box(const EventHandle& h) { - const CUevent* p = h.get(); - return reinterpret_cast( - reinterpret_cast(p) - offsetof(EventBox, resource) - ); -} - -bool get_event_timing_enabled(const EventHandle& h) noexcept { - return h ? get_box(h)->timing_enabled : false; -} - -bool get_event_is_blocking_sync(const EventHandle& h) noexcept { - return h ? get_box(h)->is_blocking_sync : false; -} - -bool get_event_ipc_enabled(const EventHandle& h) noexcept { - return h ? get_box(h)->ipc_enabled : false; -} - -int get_event_device_id(const EventHandle& h) noexcept { - return h ? get_box(h)->device_id : -1; -} - -// Return the context retained by an event handle. -ContextHandle get_event_context(const EventHandle& h) noexcept { - return h ? get_box(h)->h_context : ContextHandle{}; -} - -// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) -static HandleRegistry event_registry; - -EventHandle create_event_handle(const ContextHandle& h_ctx, unsigned int flags, - bool timing_enabled, bool is_blocking_sync, - bool ipc_enabled, int device_id) { - GILReleaseGuard gil; - CUevent event = nullptr; - err = invoke_in_context_or_undo( - h_ctx, - [&]() noexcept { return p_cuEventCreate(&event, flags); }, - [&]() noexcept { pw_cuEventDestroy(event); }, - /*undo_requires_target_context=*/false); - if (err != CUDA_SUCCESS) { - return {}; - } - - auto box = std::shared_ptr( - new EventBox{event, timing_enabled, is_blocking_sync, ipc_enabled, device_id, h_ctx}, - [](const EventBox* b) { - event_registry.unregister_handle(b->resource); - GILReleaseGuard gil; - pw_cuEventDestroy(b->resource); - delete b; - } - ); - EventHandle h(box, &box->resource); - event_registry.register_handle(event, h); - return h; -} - -EventHandle create_event_handle_for_stream(CUstream stream, unsigned int flags) { - // Resolve the stream's owning context (for default-stream tokens this is - // the current context, per cuStreamGetCtx) and create the event there, so - // it can be recorded on `stream` no matter which context is current. - CUcontext ctx = nullptr; - { - GILReleaseGuard gil; - err = p_cuStreamGetCtx(stream, &ctx); - } - if (err != CUDA_SUCCESS) { - return {}; - } - if (!ctx) { - err = CUDA_ERROR_INVALID_CONTEXT; - return {}; - } - return create_event_handle(create_context_handle_ref(ctx), flags, false, false, false, -1); -} - -EventHandle create_event_handle_ref(CUevent event) { - if (auto h = event_registry.lookup(event)) { - return h; - } - auto box = std::make_shared(EventBox{event, false, false, false, -1, {}}); - return EventHandle(box, &box->resource); -} - -EventHandle create_event_handle_ipc(const CUipcEventHandle& ipc_handle, - bool is_blocking_sync) { - GILReleaseGuard gil; - CUevent event; - if (CUDA_SUCCESS != (err = p_cuIpcOpenEventHandle(&event, ipc_handle))) { - return {}; - } - - auto box = std::shared_ptr( - new EventBox{event, false, is_blocking_sync, true, -1, {}}, - [](const EventBox* b) { - event_registry.unregister_handle(b->resource); - GILReleaseGuard gil; - pw_cuEventDestroy(b->resource); - delete b; - } - ); - EventHandle h(box, &box->resource); - event_registry.register_handle(event, h); - return h; -} - -// ============================================================================ -// Memory Pool Handles -// ============================================================================ - -namespace { -struct MemoryPoolBox { - CUmemoryPool resource; -}; -} // namespace - -// Helper to clear peer access before destroying a memory pool. -// Works around nvbug 5698116: recycled pool handles inherit peer access state. -// Must be noexcept since it's called from a shared_ptr deleter. -static void clear_mempool_peer_access(CUmemoryPool pool) noexcept { - try { - int device_count = 0; - if (p_cuDeviceGetCount(&device_count) != CUDA_SUCCESS || device_count <= 0) { - return; - } - - std::vector clear_access(device_count); - for (int i = 0; i < device_count; ++i) { - clear_access[i].location.type = CU_MEM_LOCATION_TYPE_DEVICE; - clear_access[i].location.id = i; - clear_access[i].flags = CU_MEM_ACCESS_FLAGS_PROT_NONE; - } - p_cuMemPoolSetAccess(pool, clear_access.data(), device_count); // Best effort - } catch (...) { - // Swallow exceptions - this is best-effort cleanup in destructor context - } -} - -static MemoryPoolHandle wrap_mempool_owned(CUmemoryPool pool) { - auto box = std::shared_ptr( - new MemoryPoolBox{pool}, - [](const MemoryPoolBox* b) { - GILReleaseGuard gil; - clear_mempool_peer_access(b->resource); - pw_cuMemPoolDestroy(b->resource); - delete b; - } - ); - return MemoryPoolHandle(box, &box->resource); -} - -MemoryPoolHandle create_mempool_handle(const CUmemPoolProps& props) { - GILReleaseGuard gil; - CUmemoryPool pool; - if (CUDA_SUCCESS != (err = p_cuMemPoolCreate(&pool, &props))) { - return {}; - } - return wrap_mempool_owned(pool); -} - -MemoryPoolHandle create_mempool_handle_ref(CUmemoryPool pool) { - auto box = std::make_shared(MemoryPoolBox{pool}); - return MemoryPoolHandle(box, &box->resource); -} - -MemoryPoolHandle get_device_mempool(int device_id) { - GILReleaseGuard gil; - CUmemoryPool pool; - if (CUDA_SUCCESS != (err = p_cuDeviceGetMemPool(&pool, device_id))) { - return {}; - } - return create_mempool_handle_ref(pool); -} - -MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type) { - GILReleaseGuard gil; - CUmemoryPool pool; - auto handle_ptr = reinterpret_cast(static_cast(fd)); - if (CUDA_SUCCESS != (err = p_cuMemPoolImportFromShareableHandle(&pool, handle_ptr, handle_type, 0))) { - return {}; - } - return wrap_mempool_owned(pool); -} - -// ============================================================================ -// Device Pointer Handles -// ============================================================================ - -namespace { -struct DevicePtrBox { - CUdeviceptr resource; - // Mutable so set_deallocation_stream() can update free ordering through a - // const DevicePtrHandle. Built with make_deallocation_stream so default- - // stream tokens carry a bound context. - mutable DeallocationStream deallocation; -}; -} // namespace - -// Recovers the owning DevicePtrBox from the aliased CUdeviceptr pointer. -// This works because DevicePtrHandle is a shared_ptr alias pointing to -// &box->resource, so we can compute the containing struct using offsetof. -// The const_cast is safe because we only use this to access the mutable -// deallocation member or in the deleter (where the box is being destroyed). -static DevicePtrBox* get_box(const DevicePtrHandle& h) { - const CUdeviceptr* p = h.get(); - return reinterpret_cast( - reinterpret_cast(const_cast(p)) - offsetof(DevicePtrBox, resource) - ); -} - -// Return the stream that orders a device pointer's deallocation. -StreamHandle deallocation_stream(const DevicePtrHandle& h) noexcept { - return get_box(h)->deallocation.h_stream; -} - -// Replace the stream that orders a device pointer's deallocation. -CUresult set_deallocation_stream(const DevicePtrHandle& h, const StreamHandle& h_stream) noexcept { - if (!h) { - return CUDA_ERROR_INVALID_VALUE; - } - DeallocationStream ds; - if (!make_deallocation_stream(h_stream, ds)) { - return err != CUDA_SUCCESS ? err : CUDA_ERROR_INVALID_CONTEXT; - } - get_box(h)->deallocation = std::move(ds); - return CUDA_SUCCESS; -} - -DevicePtrHandle deviceptr_alloc_from_pool(size_t size, const MemoryPoolHandle& h_pool, const StreamHandle& h_stream) { - GILReleaseGuard gil; - CUdeviceptr ptr; - if (CUDA_SUCCESS != (err = p_cuMemAllocFromPoolAsync(&ptr, size, *h_pool, as_cu(h_stream)))) { - return {}; - } - - DeallocationStream ds; - if (!make_deallocation_stream(h_stream, ds)) { - pw_cuMemFreeAsync(ptr, as_cu(h_stream)); - return {}; - } - - auto box = std::shared_ptr( - new DevicePtrBox{ptr, std::move(ds)}, - [h_pool](DevicePtrBox* b) { - GILReleaseGuard gil; - const DeallocationStream& stream = b->deallocation; - cleanup_in_context( - deallocation_context(stream), "cuMemFreeAsync", - [&]() noexcept { - return p_cuMemFreeAsync( - b->resource, as_cu(stream.h_stream)); - }); - delete b; - } - ); - return DevicePtrHandle(box, &box->resource); -} - -DevicePtrHandle deviceptr_alloc_async(size_t size, const StreamHandle& h_stream) { - GILReleaseGuard gil; - CUdeviceptr ptr; - if (CUDA_SUCCESS != (err = p_cuMemAllocAsync(&ptr, size, as_cu(h_stream)))) { - return {}; - } - - DeallocationStream ds; - if (!make_deallocation_stream(h_stream, ds)) { - pw_cuMemFreeAsync(ptr, as_cu(h_stream)); - return {}; - } - - auto box = std::shared_ptr( - new DevicePtrBox{ptr, std::move(ds)}, - [](DevicePtrBox* b) { - GILReleaseGuard gil; - const DeallocationStream& stream = b->deallocation; - cleanup_in_context( - deallocation_context(stream), "cuMemFreeAsync", - [&]() noexcept { - return p_cuMemFreeAsync( - b->resource, as_cu(stream.h_stream)); - }); - delete b; - } - ); - return DevicePtrHandle(box, &box->resource); -} - -// Allocate device memory synchronously with the provided context current. -CUresult deviceptr_alloc_raw(CUdeviceptr* ptr, size_t size, - const ContextHandle& h_context) noexcept { - GILReleaseGuard gil; - return invoke_in_context_or_undo( - h_context, - [&]() noexcept { return p_cuMemAlloc(ptr, size); }, - [&]() noexcept { pw_cuMemFree(*ptr); }, - /*undo_requires_target_context=*/false); -} - -DevicePtrHandle deviceptr_alloc_host(size_t size) { - GILReleaseGuard gil; - void* ptr; - if (CUDA_SUCCESS != (err = p_cuMemAllocHost(&ptr, size))) { - return {}; - } - - auto box = std::shared_ptr( - new DevicePtrBox{reinterpret_cast(ptr), DeallocationStream{}}, - [](DevicePtrBox* b) { - GILReleaseGuard gil; - pw_cuMemFreeHost(reinterpret_cast(b->resource)); - delete b; - } - ); - return DevicePtrHandle(box, &box->resource); -} - -DevicePtrHandle deviceptr_create_ref(CUdeviceptr ptr) { - auto box = std::make_shared(DevicePtrBox{ptr, DeallocationStream{}}); - return DevicePtrHandle(box, &box->resource); -} - -DevicePtrHandle deviceptr_create_with_owner(CUdeviceptr ptr, PyObject* owner) { - if (!owner) { - return deviceptr_create_ref(ptr); - } - // GIL required when owner is provided - GILAcquireGuard gil; - if (!gil.acquired()) { - // Python finalizing - fall back to ref version (no owner tracking) - return deviceptr_create_ref(ptr); - } - Py_INCREF(owner); - auto box = std::shared_ptr( - new DevicePtrBox{ptr, DeallocationStream{}}, - [owner](DevicePtrBox* b) { - GILAcquireGuard gil; - if (gil.acquired()) { - Py_DECREF(owner); - } - delete b; - } - ); - return DevicePtrHandle(box, &box->resource); -} - -DevicePtrHandle deviceptr_create_mapped_graphics( - CUdeviceptr ptr, - const GraphicsResourceHandle& h_resource, - const StreamHandle& h_stream -) { - DeallocationStream ds; - if (!make_deallocation_stream(h_stream, ds)) { - return {}; - } - auto box = std::shared_ptr( - new DevicePtrBox{ptr, std::move(ds)}, - [h_resource](DevicePtrBox* b) { - GILReleaseGuard gil; - CUgraphicsResource resource = as_cu(h_resource); - const DeallocationStream& stream = b->deallocation; - cleanup_in_context( - deallocation_context(stream), "cuGraphicsUnmapResources", - [&]() noexcept { - return p_cuGraphicsUnmapResources( - 1, &resource, as_cu(stream.h_stream)); - }); - delete b; - } - ); - return DevicePtrHandle(box, &box->resource); -} - -// ============================================================================ -// MemoryResource-owned Device Pointer Handles -// ============================================================================ - -static MRDeallocCallback mr_dealloc_cb = nullptr; - -void register_mr_dealloc_callback(MRDeallocCallback cb) { - mr_dealloc_cb = cb; -} - -DevicePtrHandle deviceptr_create_with_mr(CUdeviceptr ptr, size_t size, PyObject* mr) { - if (!mr) { - return deviceptr_create_ref(ptr); - } - // GIL required when mr is provided - GILAcquireGuard gil; - if (!gil.acquired()) { - return deviceptr_create_ref(ptr); - } - Py_INCREF(mr); - auto box = std::shared_ptr( - new DevicePtrBox{ptr, DeallocationStream{}}, - [mr, size](DevicePtrBox* b) { - GILAcquireGuard gil; - if (gil.acquired()) { - if (mr_dealloc_cb) { - const DeallocationStream& stream = b->deallocation; - cleanup_in_context( - deallocation_context(stream), "MemoryResource.deallocate", - [&]() noexcept { - mr_dealloc_cb(mr, b->resource, size, stream.h_stream); - return CUDA_SUCCESS; - }); - } - Py_DECREF(mr); - } - delete b; - } - ); - return DevicePtrHandle(box, &box->resource); -} - -// ============================================================================ -// IPC Pointer Cache -// ============================================================================ -// This cache handles duplicate IPC imports, which behave differently depending -// on the memory type: -// -// 1. Memory pool allocations (DeviceMemoryResource): -// Multiple imports of the same allocation succeed and return duplicate -// pointers. However, the driver has a reference counting bug (nvbug 5570902) -// where the first cuMemFreeAsync incorrectly unmaps the memory even when -// imported multiple times. A driver fix is expected. -// -// 2. Pinned memory allocations (PinnedMemoryResource): -// Duplicate imports result in CUDA_ERROR_ALREADY_MAPPED. -// -// The cache solves both issues by checking the cache before calling -// cuMemPoolImportPointer and returning the existing handle for duplicate -// imports. This provides a consistent user experience where the same IPC -// descriptor can be imported multiple times regardless of memory type. -// -// The cache key is the export_data bytes (CUmemPoolPtrExportData), not the -// returned pointer, because we must check before calling the driver API. - - -// TODO: When driver fix for nvbug 5570902 is available, consider whether -// the cache is still needed for memory pool allocations (it will still be -// needed for pinned memory). -static bool use_ipc_ptr_cache() { - return true; -} - -namespace { -// Wrapper for CUmemPoolPtrExportData to use as map key -struct ExportDataKey { - CUmemPoolPtrExportData data; - - bool operator==(const ExportDataKey& other) const { - return std::memcmp(&data, &other.data, sizeof(data)) == 0; - } -}; - -struct ExportDataKeyHash { - std::size_t operator()(const ExportDataKey& key) const { - // Simple hash of the bytes - std::size_t h = 0; - const auto* bytes = reinterpret_cast(&key.data); - for (std::size_t i = 0; i < sizeof(key.data); ++i) { - h = h * 31 + bytes[i]; - } - return h; - } -}; - -} - -static HandleRegistry ipc_ptr_cache; -static std::mutex ipc_import_mutex; - -DevicePtrHandle deviceptr_import_ipc(const MemoryPoolHandle& h_pool, const void* export_data, const StreamHandle& h_stream) { - auto data = const_cast( - reinterpret_cast(export_data)); - - if (use_ipc_ptr_cache()) { - ExportDataKey key; - std::memcpy(&key.data, data, sizeof(key.data)); - - std::lock_guard lock(ipc_import_mutex); - - if (auto h = ipc_ptr_cache.lookup(key)) { - return h; - } - - GILReleaseGuard gil; - CUdeviceptr ptr; - if (CUDA_SUCCESS != (err = p_cuMemPoolImportPointer(&ptr, *h_pool, data))) { - return {}; - } - - DeallocationStream ds; - if (!make_deallocation_stream(h_stream, ds)) { - pw_cuMemFreeAsync(ptr, as_cu(h_stream)); - return {}; - } - - auto box = std::shared_ptr( - new DevicePtrBox{ptr, std::move(ds)}, - [h_pool, key](DevicePtrBox* b) { - ipc_ptr_cache.unregister_handle(key); - GILReleaseGuard gil; - const DeallocationStream& stream = b->deallocation; - cleanup_in_context( - deallocation_context(stream), "cuMemFreeAsync", - [&]() noexcept { - return p_cuMemFreeAsync( - b->resource, as_cu(stream.h_stream)); - }); - delete b; - } - ); - DevicePtrHandle h(box, &box->resource); - ipc_ptr_cache.register_handle(key, h); - return h; - - } else { - GILReleaseGuard gil; - CUdeviceptr ptr; - if (CUDA_SUCCESS != (err = p_cuMemPoolImportPointer(&ptr, *h_pool, data))) { - return {}; - } - - DeallocationStream ds; - if (!make_deallocation_stream(h_stream, ds)) { - pw_cuMemFreeAsync(ptr, as_cu(h_stream)); - return {}; - } - - auto box = std::shared_ptr( - new DevicePtrBox{ptr, std::move(ds)}, - [h_pool](DevicePtrBox* b) { - GILReleaseGuard gil; - const DeallocationStream& stream = b->deallocation; - cleanup_in_context( - deallocation_context(stream), "cuMemFreeAsync", - [&]() noexcept { - return p_cuMemFreeAsync( - b->resource, as_cu(stream.h_stream)); - }); - delete b; - } - ); - return DevicePtrHandle(box, &box->resource); - } -} - -// ============================================================================ -// Library Handles -// ============================================================================ - -namespace { -struct LibraryBox { - CUlibrary resource; -}; -} // namespace - -LibraryHandle create_library_handle_from_file(const char* path) { - GILReleaseGuard gil; - CUlibrary library; - if (CUDA_SUCCESS != (err = p_cuLibraryLoadFromFile(&library, path, nullptr, nullptr, 0, nullptr, nullptr, 0))) { - return {}; - } - - auto box = std::shared_ptr( - new LibraryBox{library}, - [](const LibraryBox* b) { - GILReleaseGuard gil; - // TODO: re-enable once LibraryBox tracks its owning context - // p_cuLibraryUnload(b->resource); - delete b; - } - ); - return LibraryHandle(box, &box->resource); -} - -LibraryHandle create_library_handle_from_data(const void* data) { - GILReleaseGuard gil; - CUlibrary library; - if (CUDA_SUCCESS != (err = p_cuLibraryLoadData(&library, data, nullptr, nullptr, 0, nullptr, nullptr, 0))) { - return {}; - } - - auto box = std::shared_ptr( - new LibraryBox{library}, - [](const LibraryBox* b) { - GILReleaseGuard gil; - // TODO: re-enable once LibraryBox tracks its owning context - // p_cuLibraryUnload(b->resource); - delete b; - } - ); - return LibraryHandle(box, &box->resource); -} - -LibraryHandle create_library_handle_ref(CUlibrary library) { - auto box = std::make_shared(LibraryBox{library}); - return LibraryHandle(box, &box->resource); -} - -// ============================================================================ -// Kernel Handles -// ============================================================================ - -namespace { -struct KernelBox { - CUkernel resource; - LibraryHandle h_library; -}; -} // namespace - -static const KernelBox* get_box(const KernelHandle& h) { - const CUkernel* p = h.get(); - return reinterpret_cast( - reinterpret_cast(p) - offsetof(KernelBox, resource) - ); -} - -// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) -static HandleRegistry kernel_registry; - -KernelHandle create_kernel_handle(const LibraryHandle& h_library, const char* name) { - GILReleaseGuard gil; - CUkernel kernel; - if (CUDA_SUCCESS != (err = p_cuLibraryGetKernel(&kernel, *h_library, name))) { - return {}; - } - - auto box = std::make_shared(KernelBox{kernel, h_library}); - KernelHandle h(box, &box->resource); - kernel_registry.register_handle(kernel, h); - return h; -} - -KernelHandle create_kernel_handle_ref(CUkernel kernel) { - if (auto h = kernel_registry.lookup(kernel)) { - return h; - } - auto box = std::make_shared(KernelBox{kernel, {}}); - return KernelHandle(box, &box->resource); -} - -LibraryHandle get_kernel_library(const KernelHandle& h) noexcept { - if (!h) return {}; - return get_box(h)->h_library; -} - -// ============================================================================ -// Graph Handles -// ============================================================================ - -namespace { - -struct NodeAttachment; -using GraphAttachmentMap = std::map; - -struct GraphHierarchy; - -// Standard-layout alias target for GraphHandle. -struct GraphBoxBase { - CUgraph resource = nullptr; -}; - -// Canonical state for one CUgraph. Its GraphHandle aliases resource, whose -// address remains stable for the lifetime of the hierarchy. -struct GraphBox : GraphBoxBase { - GraphHierarchy* hierarchy = nullptr; // Non-owning back-reference. - GraphBox* parent = nullptr; // Null for the root graph. - CUgraphNode owner_node = nullptr; // Node in parent that owns this graph. - GraphAttachmentMap attachments; // Non-owning attachment index. - HandleRegistry node_handles; - - GraphBox( - CUgraph resource_, - GraphHierarchy* hierarchy_, - GraphBox* parent_ = nullptr, - CUgraphNode owner_node_ = nullptr) noexcept - : GraphBoxBase{resource_}, - hierarchy(hierarchy_), - parent(parent_), - owner_node(owner_node_) {} -}; - -// Shared owner of stable GraphBox storage. Every GraphHandle aliases the same -// control block, so any graph handle keeps the entire hierarchy alive. -struct GraphHierarchy { - std::list graphs; // Parent boxes precede their descendants. - std::list graveyard; // Retired child graph tombstones. - - GraphBox* root() noexcept { - return graphs.empty() ? nullptr : &graphs.front(); - } -}; - -// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) -using GraphRegistry = HandleRegistry; -static GraphRegistry graph_registry; - -// Immutable resource owners for one version of a graph node's parameters. -// Inheriting DeferredCleanupItem lets CUDA's user-object destructor enqueue -// the payload without destroying owners on the callback thread. -struct NodeAttachment : DeferredCleanupItem { - CUuserObject object = nullptr; - std::array owners; - - NodeAttachment(OpaqueHandle owner0, OpaqueHandle owner1) - : owners{std::move(owner0), std::move(owner1)} {} -}; - -// shared_ptr deleters for the payloads that need one. Typed handles convert to -// OpaqueHandle by assignment and reuse their own control block, so they need no -// deleter here. The Python deleter follows the owner-release pattern used by -// the stream/deviceptr handles above. -void py_deleter(const void* p) noexcept { - GILAcquireGuard gil; - if (gil.acquired()) { - Py_DECREF(const_cast(static_cast(p))); - } -} - -void free_deleter(const void* p) noexcept { - std::free(const_cast(p)); -} - -GraphBox* get_box(const GraphHandle& h) noexcept { - auto* value = reinterpret_cast(h.get()); - return const_cast( - static_cast(value)); -} - -// Rekey a staged attachment map from source nodes to their cloned nodes. -// The caller must release the GIL before calling this function. -CUresult rekey_attachments( - GraphAttachmentMap& attachments, CUgraph cloned_graph) { - if (!cloned_graph) { - return CUDA_ERROR_INVALID_VALUE; - } - if (!p_cuGraphNodeFindInClone) { - return CUDA_ERROR_NOT_SUPPORTED; - } - - GraphAttachmentMap remapped; - while (!attachments.empty()) { - auto attachment = attachments.extract(attachments.begin()); - CUgraphNode cloned_node = nullptr; - CUresult status = p_cuGraphNodeFindInClone( - &cloned_node, attachment.key(), cloned_graph); - if (status != CUDA_SUCCESS) { - return status; - } - attachment.key() = cloned_node; - if (!remapped.insert(std::move(attachment)).inserted) { - return CUDA_ERROR_INVALID_VALUE; - } - } - attachments.swap(remapped); - return CUDA_SUCCESS; -} - -struct StagedGraphMetadata { - const GraphBox* source; - GraphBox* clone; - GraphAttachmentMap* attachments; -}; -using StagedGraphMetadataList = std::vector; - -// Copy a source hierarchy into detached metadata before CUDA mutation. -void stage_graph_metadata( - const GraphBox& source, - GraphBox& clone, - GraphAttachmentMap& attachments, - std::list& subgraphs, - StagedGraphMetadataList& staged) { - attachments = source.attachments; - staged.push_back({&source, &clone, &attachments}); - - for (const GraphBox& source_child : source.hierarchy->graphs) { - if (source_child.parent != &source || !source_child.resource) { - continue; - } - GraphBox& cloned_child = subgraphs.emplace_back( - nullptr, - clone.hierarchy, - &clone, - nullptr); - stage_graph_metadata( - source_child, - cloned_child, - cloned_child.attachments, - subgraphs, - staged); - } -} - -// Bind staged metadata to a CUDA-cloned hierarchy. The root clone resource -// must be populated before entry. The caller must release the GIL. -CUresult rekey_graph_metadata( - StagedGraphMetadataList& staged) { - if (!p_cuGraphNodeFindInClone || !p_cuGraphChildGraphNodeGetGraph) { - return CUDA_ERROR_NOT_SUPPORTED; - } - - CUresult status; - for (size_t i = 0; i < staged.size(); ++i) { - const GraphBox& source = *staged[i].source; - GraphBox& clone = *staged[i].clone; - if (i != 0) { - CUgraphNode cloned_owner = nullptr; - status = p_cuGraphNodeFindInClone( - &cloned_owner, - source.owner_node, - clone.parent->resource); - if (status == CUDA_SUCCESS) { - status = p_cuGraphChildGraphNodeGetGraph( - cloned_owner, &clone.resource); - } - if (status != CUDA_SUCCESS) { - return status; - } - clone.owner_node = cloned_owner; - } - - status = rekey_attachments( - *staged[i].attachments, clone.resource); - if (status != CUDA_SUCCESS) { - return status; - } - } - return CUDA_SUCCESS; -} - -} // namespace - -OpaqueHandle make_opaque_py(PyObject* obj) { - Py_INCREF(obj); - return OpaqueHandle(static_cast(obj), py_deleter); -} - -OpaqueHandle make_opaque_malloc(void* buf) { - return OpaqueHandle(static_cast(buf), free_deleter); -} - -// State held by PreparedAttachment between preparation and commit. It keeps the -// graph alive, tracks the graph-retained replacement, and holds a preallocated -// map entry so commit cannot allocate. Destroying PreparedAttachment rolls back -// the staged user-object retain unless graph_commit_attachment publishes it. -struct PreparedAttachmentState { - GraphHandle h_graph; - NodeAttachment* replacement = nullptr; - GraphAttachmentMap::node_type replacement_entry; - - explicit PreparedAttachmentState(GraphHandle h_graph_) - : h_graph(std::move(h_graph_)) {} -}; - -void rollback_prepared_attachment( - PreparedAttachmentState* state) noexcept { - if (!state) { - return; - } - if (state->replacement) { - GraphBox* box = get_box(state->h_graph); - if (box->resource) { - GILReleaseGuard gil; - pw_cuGraphReleaseUserObject( - box->resource, state->replacement->object, 1); - } - } - delete state; -} - -// Detached metadata for a replacement embedded graph hierarchy. Preparation -// copies every attachment map and allocates every GraphBox before CUDA destroys -// the old embedded graph. Commit only rekeys and publishes it. -struct PreparedChildGraphUpdateState { - GraphHandle h_parent; - GraphHandle h_source; - GraphBox* old_root = nullptr; - CUgraphNode owner_node = nullptr; - std::list replacement; - StagedGraphMetadataList staged; - std::vector handles; - - PreparedChildGraphUpdateState( - GraphHandle h_parent_, - GraphHandle h_source_, - GraphBox* old_root_, - CUgraphNode owner_node_) - : h_parent(std::move(h_parent_)), - h_source(std::move(h_source_)), - old_root(old_root_), - owner_node(owner_node_) {} -}; - -GraphHandle create_graph_handle(CUgraph graph) { - if (!graph) { - return {}; - } - - auto hierarchy = std::shared_ptr( - new GraphHierarchy{}, - [](GraphHierarchy* hierarchy) { - for (const GraphBox& box : hierarchy->graphs) { - if (box.resource) { - graph_registry.unregister_handle(box.resource); - } - } - GraphBox* root = hierarchy->root(); - if (root && root->resource) { - GILReleaseGuard gil; - pw_cuGraphDestroy(root->resource); - } - retry_deferred_cleanup(); - delete hierarchy; - } - ); - GraphBox& root = hierarchy->graphs.emplace_back( - graph, hierarchy.get()); - - GraphHandle h_graph(hierarchy, &root.resource); - graph_registry.register_handle(graph, h_graph); - return h_graph; -} - -GraphHandle create_child_graph_handle( - CUgraph child_graph, const GraphHandle& h_parent, - CUgraphNode owner_node) { - if (!child_graph || !h_parent || !owner_node) { - return {}; - } - if (GraphHandle h_graph = graph_registry.lookup(child_graph)) { - return h_graph; - } - - GraphBox* parent = get_box(h_parent); - GraphHierarchy* hierarchy = parent->hierarchy; - GraphBox& child = hierarchy->graphs.emplace_back( - child_graph, hierarchy, parent, owner_node); - - GraphHandle h_child(h_parent, &child.resource); - graph_registry.register_handle(child_graph, h_child); - return h_child; -} - -CUresult graph_prepare_child_graph_update( - const GraphHandle& h_parent, - const GraphHandle& h_old_child, - CUgraphNode owner_node, - const GraphHandle& h_source, - PreparedChildGraphUpdate* out_prepared) { - if (!h_parent || !h_old_child || !owner_node || - !h_source || !out_prepared) { - return CUDA_ERROR_INVALID_VALUE; - } - out_prepared->reset(); - - GraphBox* parent = get_box(h_parent); - GraphBox* old_root = get_box(h_old_child); - GraphBox* source = get_box(h_source); - // A source from the destination hierarchy can include the old embedded - // subtree whose raw node keys CUDA destroys during replacement. - if (!parent->resource || !old_root->resource || !source->resource || - old_root->parent != parent || - old_root->owner_node != owner_node || - source->hierarchy == parent->hierarchy) { - return CUDA_ERROR_INVALID_VALUE; - } - - PreparedChildGraphUpdate prepared = - std::make_shared( - h_parent, h_source, old_root, owner_node); - - GraphBox& replacement_root = - prepared->replacement.emplace_back( - nullptr, parent->hierarchy, parent, owner_node); - stage_graph_metadata( - *source, - replacement_root, - replacement_root.attachments, - prepared->replacement, - prepared->staged); - - const size_t graph_count = prepared->staged.size(); - prepared->handles.reserve(graph_count); - for (const StagedGraphMetadata& graph : prepared->staged) { - prepared->handles.emplace_back( - h_parent, &graph.clone->resource); - } - - *out_prepared = std::move(prepared); - return CUDA_SUCCESS; -} - -void publish_child_graph_update( - PreparedChildGraphUpdateState& state, - GraphHandle* out_child) { - GraphBox* parent = get_box(state.h_parent); - parent->hierarchy->graphs.splice( - parent->hierarchy->graphs.end(), state.replacement); - *out_child = state.handles.front(); - graph_registry.register_handles(state.handles); -} - -CUresult graph_commit_child_graph_update( - PreparedChildGraphUpdate& prepared, - GraphHandle* out_child) { - if (!prepared || !out_child) { - return CUDA_ERROR_INVALID_VALUE; - } - out_child->reset(); - - PreparedChildGraphUpdateState& state = *prepared; - GraphBox* parent = get_box(state.h_parent); - if (!parent->resource || !state.old_root->resource) { - prepared.reset(); - return CUDA_ERROR_INVALID_VALUE; - } - - CUresult status = CUDA_ERROR_NOT_SUPPORTED; - CUgraph cloned_root = nullptr; - if (p_cuGraphChildGraphNodeGetGraph) { - GILReleaseGuard gil; - status = p_cuGraphChildGraphNodeGetGraph( - state.owner_node, &cloned_root); - if (status == CUDA_SUCCESS) { - state.staged.front().clone->resource = cloned_root; - status = rekey_graph_metadata(state.staged); - } - } - - // CUDA has already destroyed the old embedded graph. No replacement - // metadata is visible yet, so this selects only the old generation. - invalidate_child_graph_state( - state.h_parent, state.owner_node); - - if (status != CUDA_SUCCESS) { - prepared.reset(); - throw std::runtime_error( - "failed to update graph metadata after child graph replacement"); - } - - publish_child_graph_update(state, out_child); - prepared.reset(); - return status; -} - -CUresult graph_get_attachment( - const GraphHandle& h_graph, CUgraphNode node, - OpaqueHandle* owner0, OpaqueHandle* owner1) { - if (!h_graph || !node || (!owner0 && !owner1)) { - return CUDA_ERROR_INVALID_VALUE; - } - if (owner0) { - owner0->reset(); - } - if (owner1) { - owner1->reset(); - } - - GraphBox* box = get_box(h_graph); - if (!box->resource) { - return CUDA_ERROR_INVALID_VALUE; - } - auto it = box->attachments.find(node); - if (it != box->attachments.end()) { - if (owner0) { - *owner0 = it->second->owners[0]; - } - if (owner1) { - *owner1 = it->second->owners[1]; - } - } - return CUDA_SUCCESS; -} - -CUresult graph_prepare_attachment( - const GraphHandle& h_graph, - OpaqueHandle owner0, - OpaqueHandle owner1, - PreparedAttachment* out_prepared) { - if (!out_prepared) { - return CUDA_ERROR_INVALID_VALUE; - } - out_prepared->reset(); - if (!h_graph) { - return CUDA_ERROR_INVALID_VALUE; - } - - GraphBox* box = get_box(h_graph); - if (!box->resource) { - return CUDA_ERROR_INVALID_VALUE; - } - if (!p_cuGraphReleaseUserObject) { - return CUDA_ERROR_NOT_SUPPORTED; - } - - PreparedAttachment prepared( - new PreparedAttachmentState(h_graph), - PreparedAttachmentDeleter{rollback_prepared_attachment}); - if (owner0 || owner1) { - if (!p_cuUserObjectCreate || !p_cuUserObjectRelease || - !p_cuGraphRetainUserObject) { - return CUDA_ERROR_NOT_SUPPORTED; - } - - ensure_deferred_cleanup_ready(); - prepared->replacement = new NodeAttachment( - std::move(owner0), std::move(owner1)); - GraphAttachmentMap staged; - try { - staged.emplace(nullptr, prepared->replacement); - prepared->replacement_entry = - staged.extract(staged.begin()); - } catch (...) { - delete prepared->replacement; - prepared->replacement = nullptr; - throw; - } - auto* cleanup_item = - static_cast( - prepared->replacement); - - CUuserObject object = nullptr; - CUresult status; - { - GILReleaseGuard gil; - status = p_cuUserObjectCreate( - &object, cleanup_item, - reinterpret_cast(enqueue_cleanup), - 1, CU_USER_OBJECT_NO_DESTRUCTOR_SYNC); - if (status != CUDA_SUCCESS) { - prepared->replacement_entry.mapped() = nullptr; - delete prepared->replacement; - prepared->replacement = nullptr; - return status; - } - prepared->replacement->object = object; - status = p_cuGraphRetainUserObject( - box->resource, object, 1, CU_GRAPH_USER_OBJECT_MOVE); - if (status != CUDA_SUCCESS) { - prepared->replacement_entry.mapped() = nullptr; - prepared->replacement = nullptr; - pw_cuUserObjectRelease(object, 1); - return status; - } - } - } - - *out_prepared = std::move(prepared); - return CUDA_SUCCESS; -} - -CUresult graph_commit_attachment( - PreparedAttachment& prepared, - CUgraphNode node) { - if (!prepared) { - return CUDA_ERROR_INVALID_VALUE; - } - - GraphHandle h_graph = prepared->h_graph; - GraphBox* box = get_box(h_graph); - if (!box->resource || (!node && !prepared->replacement)) { - delete prepared.release(); - return CUDA_ERROR_INVALID_VALUE; - } - if (!node) { - delete prepared.release(); - return CUDA_SUCCESS; - } - - // Publish the replacement or removal before releasing the previous graph - // reference; that release can make the previous payload eligible for - // destruction. - NodeAttachment* previous = nullptr; - auto it = box->attachments.find(node); - if (it == box->attachments.end()) { - if (prepared->replacement) { - prepared->replacement_entry.key() = node; - auto result = box->attachments.insert( - std::move(prepared->replacement_entry)); - if (!result.inserted) { - prepared->replacement_entry = - std::move(result.node); - delete prepared.release(); - return CUDA_ERROR_INVALID_VALUE; - } - } - } else { - previous = it->second; - if (prepared->replacement) { - it->second = prepared->replacement; - } else { - box->attachments.erase(it); - } - } - - delete prepared.release(); - if (!previous) { - return CUDA_SUCCESS; - } - GILReleaseGuard gil; - return p_cuGraphReleaseUserObject( - box->resource, previous->object, 1); -} - -CUresult graph_clone_attachments( - const GraphHandle& h_clone, - const GraphHandle& h_source) { - if (!h_clone || !h_source) { - return CUDA_ERROR_INVALID_VALUE; - } - - GraphBox* clone = get_box(h_clone); - GraphBox* source = get_box(h_source); - if (!clone->resource || !source->resource || - !clone->attachments.empty()) { - return CUDA_ERROR_INVALID_VALUE; - } - - // Build and rekey the clone metadata off-hierarchy so a CUDA mapping error - // cannot partially publish it. - GraphAttachmentMap attachments; - std::list subgraphs; - StagedGraphMetadataList staged; - stage_graph_metadata( - *source, *clone, attachments, subgraphs, staged); - - std::vector handles; - handles.reserve(subgraphs.size()); - for (GraphBox& graph : subgraphs) { - handles.emplace_back(h_clone, &graph.resource); - } - - CUresult status; - { - GILReleaseGuard gil; - status = rekey_graph_metadata(staged); - } - if (status != CUDA_SUCCESS) { - return status; - } - - clone->attachments.swap(attachments); - if (subgraphs.empty()) { - return CUDA_SUCCESS; - } - - clone->hierarchy->graphs.splice( - clone->hierarchy->graphs.end(), subgraphs); - graph_registry.register_handles(handles); - return CUDA_SUCCESS; -} - -// ============================================================================ -// Graph Exec Handles -// ============================================================================ - -namespace { - -// Append-only owners introduced by individual executable-node updates. CUDA -// owns this payload through a user object propagated into the CUgraphExec. -struct ExecAttachments : DeferredCleanupItem { - CUuserObject object = nullptr; - std::vector owners; -}; - -struct GraphExecBox { - CUgraphExec resource = nullptr; - ExecAttachments* attachments = nullptr; // Non-owning. - - ~GraphExecBox() noexcept { - if (resource) { - GILReleaseGuard gil; - pw_cuGraphExecDestroy(resource); - } - // The accumulator fields may be dangling after exec destruction. - retry_deferred_cleanup(); - } -}; - -GraphExecBox* get_exec_box(const GraphExecHandle& h) noexcept { - return const_cast( - reinterpret_cast(h.get())); -} - -GraphExecHandle make_graph_exec_handle( - CUgraphExec graph_exec, ExecAttachments* attachments) { - struct RawGraphExecGuard { - CUgraphExec resource; - - ~RawGraphExecGuard() noexcept { - if (resource) { - GILReleaseGuard gil; - pw_cuGraphExecDestroy(resource); - } - retry_deferred_cleanup(); - } - } guard{graph_exec}; - - auto box = std::make_shared(); - box->resource = graph_exec; - box->attachments = attachments; - guard.resource = nullptr; - return GraphExecHandle(box, &box->resource); -} - -// Holds a fresh accumulator retained on the source graph across a CUDA call -// that propagates user objects into an exec. Releasing drops the source's -// reference: after successful propagation the exec keeps the accumulator -// alive, and otherwise this drops its last reference. -struct ExecAttachmentStaging { - GraphHandle h_source; - ExecAttachments* accumulator = nullptr; - - ~ExecAttachmentStaging() noexcept { - report_cuda_error("cuGraphReleaseUserObject", release(), - "failed while dropping a staged graph attachment"); - } - - CUresult release() noexcept { - if (!h_source || !accumulator) { - return CUDA_SUCCESS; - } - const CUuserObject object = accumulator->object; - const GraphHandle source = std::move(h_source); - accumulator = nullptr; - GILReleaseGuard gil; - return p_cuGraphReleaseUserObject(*source, object, 1); - } -}; - -// Create an accumulator and retain it on h_source, so that a following -// instantiation or whole-graph update propagates a reference into the exec. -CUresult stage_exec_attachments( - const GraphHandle& h_source, ExecAttachmentStaging* out_staging) { - if (!p_cuUserObjectCreate || !p_cuUserObjectRelease || - !p_cuGraphRetainUserObject || !p_cuGraphReleaseUserObject) { - return CUDA_ERROR_NOT_SUPPORTED; - } - - ensure_deferred_cleanup_ready(); - auto* accumulator = new ExecAttachments; - - CUuserObject object = nullptr; - CUresult status; - { - GILReleaseGuard gil; - status = p_cuUserObjectCreate( - &object, - static_cast(accumulator), - reinterpret_cast(enqueue_cleanup), - 1, - CU_USER_OBJECT_NO_DESTRUCTOR_SYNC); - if (status != CUDA_SUCCESS) { - delete accumulator; - return status; - } - accumulator->object = object; - status = p_cuGraphRetainUserObject( - *h_source, object, 1, CU_GRAPH_USER_OBJECT_MOVE); - if (status != CUDA_SUCCESS) { - // Dropping the last reference retires the accumulator. - pw_cuUserObjectRelease(object, 1); - return status; - } - } - - out_staging->h_source = h_source; - out_staging->accumulator = accumulator; - return CUDA_SUCCESS; -} - -} // namespace - -// State held by PreparedExecAttachment between preparation and commit. It keeps -// the exec alive and remembers the accumulator size before the append, so that -// rollback can drop owners staged for a mutation that CUDA rejected. -struct PreparedExecAttachmentState { - GraphExecHandle h_exec; - ExecAttachments* attachments = nullptr; - size_t original_size = 0; - - PreparedExecAttachmentState( - GraphExecHandle h_exec_, - ExecAttachments* attachments_, - size_t original_size_) - : h_exec(std::move(h_exec_)), - attachments(attachments_), - original_size(original_size_) {} -}; - -void rollback_prepared_exec_attachment( - PreparedExecAttachmentState* state) noexcept { - if (!state) { - return; - } - if (state->attachments) { - while (state->attachments->owners.size() > state->original_size) { - state->attachments->owners.pop_back(); - } - } - delete state; -} - -GraphExecHandle create_graph_exec_handle( - const GraphHandle& h_source, - CUDA_GRAPH_INSTANTIATE_PARAMS* params) { - if (!h_source || !*h_source || !params) { - err = CUDA_ERROR_INVALID_VALUE; - return {}; - } - if (!p_cuGraphInstantiateWithParams) { - err = CUDA_ERROR_NOT_SUPPORTED; - return {}; - } - - ExecAttachmentStaging staging; - if (CUDA_SUCCESS != (err = stage_exec_attachments(h_source, &staging))) { - return {}; - } - - CUgraphExec graph_exec = nullptr; - { - GILReleaseGuard gil; - err = p_cuGraphInstantiateWithParams(&graph_exec, *h_source, params); - } - if (err != CUDA_SUCCESS) { - return {}; - } - // CUDA can report a specific failure while returning success. The exec is - // then unusable, so it stays unadopted for the caller to diagnose from - // params->result_out. - if (params->result_out != CUDA_GRAPH_INSTANTIATE_SUCCESS) { - return {}; - } - if (!graph_exec) { - err = CUDA_ERROR_INVALID_VALUE; - return {}; - } - - GraphExecHandle h_exec = make_graph_exec_handle( - graph_exec, staging.accumulator); - if (CUDA_SUCCESS != (err = staging.release())) { - return {}; - } - return h_exec; -} - -CUresult graph_exec_update( - const GraphExecHandle& h_exec, - const GraphHandle& h_source, - CUgraphExecUpdateResultInfo* result_info) { - if (!h_exec || !h_source || !*h_source || !result_info) { - return CUDA_ERROR_INVALID_VALUE; - } - if (!p_cuGraphExecUpdate) { - return CUDA_ERROR_NOT_SUPPORTED; - } - - GraphExecBox* box = get_exec_box(h_exec); - if (!box->resource) { - return CUDA_ERROR_INVALID_VALUE; - } - - ExecAttachmentStaging staging; - CUresult status = stage_exec_attachments(h_source, &staging); - if (status != CUDA_SUCCESS) { - return status; - } - - { - GILReleaseGuard gil; - status = p_cuGraphExecUpdate(box->resource, *h_source, result_info); - } - if (status != CUDA_SUCCESS) { - return status; - } - - // CUDA may already have retired the old accumulator. Publish the new one - // before releasing the source graph's temporary reference. - box->attachments = staging.accumulator; - return staging.release(); -} - -CUresult graph_prepare_exec_attachment( - const GraphExecHandle& h_exec, - OpaqueHandle owner0, - OpaqueHandle owner1, - PreparedExecAttachment* out_prepared) { - if (!out_prepared) { - return CUDA_ERROR_INVALID_VALUE; - } - out_prepared->reset(); - if (!h_exec) { - return CUDA_ERROR_INVALID_VALUE; - } - - GraphExecBox* box = get_exec_box(h_exec); - if (!box->resource || !box->attachments) { - return CUDA_ERROR_INVALID_VALUE; - } - - ExecAttachments* attachments = box->attachments; - const size_t original_size = attachments->owners.size(); - const size_t additions = - static_cast(static_cast(owner0)) + - static_cast(static_cast(owner1)); - // Reserve before staging so that rollback and commit cannot allocate. - attachments->owners.reserve(original_size + additions); - PreparedExecAttachment prepared( - new PreparedExecAttachmentState(h_exec, attachments, original_size), - PreparedExecAttachmentDeleter{rollback_prepared_exec_attachment}); - if (owner0) { - attachments->owners.emplace_back(std::move(owner0)); - } - if (owner1) { - attachments->owners.emplace_back(std::move(owner1)); - } - *out_prepared = std::move(prepared); - return CUDA_SUCCESS; -} - -void graph_commit_exec_attachment( - PreparedExecAttachment& prepared) noexcept { - delete prepared.release(); -} - -namespace { -struct GraphNodeBox { - mutable CUgraphNode resource; - GraphHandle h_graph; -}; -} // namespace - -static const GraphNodeBox* get_box(const GraphNodeHandle& h) { - const CUgraphNode* p = h.get(); - return reinterpret_cast( - reinterpret_cast(p) - offsetof(GraphNodeBox, resource) - ); -} - -// graphs is ordered parent-before-child. Nulling a selected box marks its -// later descendants, whose parent pointers remain valid after list splicing. -// This permits one allocation-free sweep of the hierarchy. -void invalidate_child_graph_state( - const GraphHandle& h_parent, - CUgraphNode owner_node) noexcept { - if (!h_parent || !owner_node) { - return; - } - - GraphBox* parent = get_box(h_parent); - if (!parent->resource) { - return; - } - GraphHierarchy& hierarchy = *parent->hierarchy; - for (auto it = hierarchy.graphs.begin(); - it != hierarchy.graphs.end();) { - auto graph = it++; - bool is_owned_root = graph->parent == parent && - graph->owner_node == owner_node; - bool is_descendant = graph->parent && - !graph->parent->resource; - if (!is_owned_root && !is_descendant) { - continue; - } - - // Empty node_handles and invalidate each one. - for (auto& entry : graph->node_handles.drain()) { - if (GraphNodeHandle h_node = entry.second.lock()) { - get_box(h_node)->resource = nullptr; - } - } - graph_registry.unregister_handle(graph->resource); - graph->resource = nullptr; - graph->attachments.clear(); - hierarchy.graveyard.splice( - hierarchy.graveyard.end(), hierarchy.graphs, graph); - } -} - -GraphNodeHandle create_graph_node_handle(CUgraphNode node, const GraphHandle& h_graph) { - if (!node) { - auto box = std::make_shared( - GraphNodeBox{nullptr, h_graph}); - return GraphNodeHandle(box, &box->resource); - } - - GraphBox* graph = get_box(h_graph); - return graph->node_handles.get_or_create( - node, - [node, &h_graph] { - auto box = std::make_shared( - GraphNodeBox{node, h_graph}); - return GraphNodeHandle(box, &box->resource); - }); -} - -GraphHandle graph_node_get_graph(const GraphNodeHandle& h) noexcept { - return h ? get_box(h)->h_graph : GraphHandle{}; -} - -void invalidate_graph_node(const GraphNodeHandle& h) noexcept { - if (!h) { - return; - } - - const GraphNodeBox* node_box = get_box(h); - CUgraphNode node = node_box->resource; - if (!node) { - return; - } - GraphBox* graph = get_box(node_box->h_graph); - graph->node_handles.unregister_handle(node); - node_box->resource = nullptr; -} - -// ============================================================================ -// Graphics Resource Handles -// ============================================================================ - -namespace { -struct GraphicsResourceBox { - CUgraphicsResource resource; -}; -} // namespace - -GraphicsResourceHandle create_graphics_resource_handle(CUgraphicsResource resource) { - auto box = std::shared_ptr( - new GraphicsResourceBox{resource}, - [](const GraphicsResourceBox* b) { - GILReleaseGuard gil; - pw_cuGraphicsUnregisterResource(b->resource); - delete b; - } - ); - return GraphicsResourceHandle(box, &box->resource); -} - -// ============================================================================ -// NVRTC Program Handles -// ============================================================================ - -namespace { -struct NvrtcProgramBox { - nvrtcProgram resource; -}; -} // namespace - -NvrtcProgramHandle create_nvrtc_program_handle(nvrtcProgram prog) { - auto box = std::shared_ptr( - new NvrtcProgramBox{prog}, - [](NvrtcProgramBox* b) { - // Note: nvrtcDestroyProgram takes nvrtcProgram* and nulls it, - // but we're deleting the box anyway so nulling is harmless. - if (p_nvrtcDestroyProgram) { - GILReleaseGuard gil; - pw_nvrtcDestroyProgram(&b->resource); - } - delete b; - } - ); - return NvrtcProgramHandle(box, &box->resource); -} - -NvrtcProgramHandle create_nvrtc_program_handle_ref(nvrtcProgram prog) { - auto box = std::make_shared(NvrtcProgramBox{prog}); - return NvrtcProgramHandle(box, &box->resource); -} - -// ============================================================================ -// NVVM Program Handles -// ============================================================================ - -namespace { -struct NvvmProgramBox { - NvvmProgramValue resource; -}; -} // namespace - -NvvmProgramHandle create_nvvm_program_handle(nvvmProgram prog) { - auto box = std::shared_ptr( - new NvvmProgramBox{{prog}}, - [](NvvmProgramBox* b) { - // Note: nvvmDestroyProgram takes nvvmProgram* and nulls it, - // but we're deleting the box anyway so nulling is harmless. - // If NVVM is not available, the function pointer is null. - if (p_nvvmDestroyProgram) { - GILReleaseGuard gil; - pw_nvvmDestroyProgram(&b->resource.raw); - } - delete b; - } - ); - return NvvmProgramHandle(box, &box->resource); -} - -NvvmProgramHandle create_nvvm_program_handle_ref(nvvmProgram prog) { - auto box = std::make_shared(NvvmProgramBox{{prog}}); - return NvvmProgramHandle(box, &box->resource); -} - -// ============================================================================ -// nvJitLink Handles -// ============================================================================ - -namespace { -struct NvJitLinkBox { - NvJitLinkValue resource; -}; -} // namespace - -NvJitLinkHandle create_nvjitlink_handle(nvJitLink_t handle) { - auto box = std::shared_ptr( - new NvJitLinkBox{{handle}}, - [](NvJitLinkBox* b) { - // Note: nvJitLinkDestroy takes nvJitLinkHandle* and nulls it, - // but we're deleting the box anyway so nulling is harmless. - // If nvJitLink is not available, the function pointer is null. - if (p_nvJitLinkDestroy) { - GILReleaseGuard gil; - pw_nvJitLinkDestroy(&b->resource.raw); - } - delete b; - } - ); - return NvJitLinkHandle(box, &box->resource); -} - -NvJitLinkHandle create_nvjitlink_handle_ref(nvJitLink_t handle) { - auto box = std::make_shared(NvJitLinkBox{{handle}}); - return NvJitLinkHandle(box, &box->resource); -} - -// ============================================================================ -// cuLink Handles -// ============================================================================ - -namespace { -struct CuLinkBox { - CUlinkState resource; -}; -} // namespace - -CuLinkHandle create_culink_handle(CUlinkState state) { - auto box = std::shared_ptr( - new CuLinkBox{state}, - [](CuLinkBox* b) { - // cuLinkDestroy takes CUlinkState by value (not pointer). - if (p_cuLinkDestroy) { - GILReleaseGuard gil; - pw_cuLinkDestroy(b->resource); - } - delete b; - } - ); - return CuLinkHandle(box, &box->resource); -} - -CuLinkHandle create_culink_handle_ref(CUlinkState state) { - auto box = std::make_shared(CuLinkBox{state}); - return CuLinkHandle(box, &box->resource); -} - -// ============================================================================ -// File Descriptor Handles -// ============================================================================ - -FileDescriptorHandle create_fd_handle(int fd) { -#ifdef _WIN32 - throw std::runtime_error("create_fd_handle is not supported on Windows"); -#else - return FileDescriptorHandle( - new int(fd), - [](const int* p) { - if (::close(*p) != 0) { - report_message("close() failed for an IPC file descriptor; the descriptor may have leaked"); - } - delete p; - } - ); -#endif -} - -FileDescriptorHandle create_fd_handle_ref(int fd) { -#ifdef _WIN32 - throw std::runtime_error("create_fd_handle_ref is not supported on Windows"); -#else - return std::make_shared(fd); -#endif -} - -// ============================================================================ -// Array / mipmapped-array / texture / surface handles (PR #467) -// ============================================================================ - -namespace { -struct ArrayBox { - CUarray resource; - // Non-null only for a mipmap-level view: keeps the parent mipmap (the real - // owner of the level's storage) alive for as long as the level is held. - MipmappedArrayHandle h_parent; - ContextHandle h_context; -}; - -struct MipmappedArrayBox { - CUmipmappedArray resource; - ContextHandle h_context; -}; - -// Texture and surface objects are per-context pool indices. Destroying one -// with the wrong context current can silently succeed without freeing it or -// can free an unrelated object, so destruction must enter the creating -// context. Handle-based resources resolve their own context and must not. -struct TexObjectBox { - // Tagged so TexObjectHandle is a distinct C++ type from DevicePtrHandle / - // SurfObjectHandle (all wrap `unsigned long long`). - TexObjectValue resource; - // Type-erased backing dependency (OpaqueArrayHandle / MipmappedArrayHandle / - // DevicePtrHandle). The texture's resource is a union; we only need to keep - // whichever backing it was built from alive, never to dereference it. - std::shared_ptr h_backing; - ContextHandle h_context; -}; - -struct SurfObjectBox { - SurfObjectValue resource; - OpaqueArrayHandle h_array; // surfaces are always array-backed - ContextHandle h_context; -}; - -// Recover an array's owning box from its aliased resource pointer. -const ArrayBox* get_box(const OpaqueArrayHandle& h) noexcept { - const CUarray* p = h.get(); - return reinterpret_cast( - reinterpret_cast(p) - offsetof(ArrayBox, resource)); -} - -// Recover a mipmapped array's owning box from its aliased resource pointer. -const MipmappedArrayBox* get_box(const MipmappedArrayHandle& h) noexcept { - const CUmipmappedArray* p = h.get(); - return reinterpret_cast( - reinterpret_cast(p) - - offsetof(MipmappedArrayBox, resource)); -} - -// Wrap an array with shared owning-destruction behavior. -static OpaqueArrayHandle wrap_array_owned(CUarray arr, ContextHandle h_context) { - auto box = std::shared_ptr( - new ArrayBox{arr, {}, std::move(h_context)}, - [](const ArrayBox* b) { - GILReleaseGuard gil; - pw_cuArrayDestroy(b->resource); - delete b; - } - ); - return OpaqueArrayHandle(box, &box->resource); -} - -} // namespace - -OpaqueArrayHandle create_array_handle(const ContextHandle& h_context, const CUDA_ARRAY3D_DESCRIPTOR& desc) { - GILReleaseGuard gil; - CUarray arr = nullptr; - err = invoke_in_context_or_undo( - h_context, - [&]() noexcept { return p_cuArray3DCreate(&arr, &desc); }, - [&]() noexcept { pw_cuArrayDestroy(arr); }, - /*undo_requires_target_context=*/false); - if (err != CUDA_SUCCESS) { - return {}; - } - return wrap_array_owned(arr, h_context); -} - -OpaqueArrayHandle create_array_handle_ref(CUarray arr) { - if (!arr) { - return {}; - } - auto box = std::make_shared(ArrayBox{arr, {}, {}}); - return OpaqueArrayHandle(box, &box->resource); -} - -OpaqueArrayHandle create_array_handle_owning(CUarray arr) { - if (!arr) { - return {}; - } - return wrap_array_owned(arr, {}); -} - -// Return the context retained by an array handle. -ContextHandle get_array_context(const OpaqueArrayHandle& h) noexcept { - return h ? get_box(h)->h_context : ContextHandle{}; -} - -OpaqueArrayHandle create_array_level_handle(const MipmappedArrayHandle& h_mip, unsigned int level) { - GILReleaseGuard gil; - CUarray arr; - ContextHandle h_context = h_mip ? get_box(h_mip)->h_context : ContextHandle{}; - if (CUDA_SUCCESS != (err = p_cuMipmappedArrayGetLevel(&arr, as_cu(h_mip), level))) { - return {}; - } - // Non-owning level view: storage belongs to the mipmap. Embed the mipmap - // handle so the parent outlives this level; the deleter does not destroy. - auto box = std::shared_ptr( - new ArrayBox{arr, h_mip, h_context}, - [](const ArrayBox* b) { delete b; } - ); - return OpaqueArrayHandle(box, &box->resource); -} - -MipmappedArrayHandle create_mipmapped_array_handle(const ContextHandle& h_context, - const CUDA_ARRAY3D_DESCRIPTOR& desc, - unsigned int num_levels) { - GILReleaseGuard gil; - CUmipmappedArray mip = nullptr; - err = invoke_in_context_or_undo( - h_context, - [&]() noexcept { return p_cuMipmappedArrayCreate(&mip, &desc, num_levels); }, - [&]() noexcept { pw_cuMipmappedArrayDestroy(mip); }, - /*undo_requires_target_context=*/false); - if (err != CUDA_SUCCESS) { - return {}; - } - auto box = std::shared_ptr( - new MipmappedArrayBox{mip, h_context}, - [](const MipmappedArrayBox* b) { - GILReleaseGuard gil; - pw_cuMipmappedArrayDestroy(b->resource); - delete b; - } - ); - return MipmappedArrayHandle(box, &box->resource); -} - -// Return the context retained by a mipmapped array handle. -ContextHandle get_mipmapped_array_context(const MipmappedArrayHandle& h) noexcept { - return h ? get_box(h)->h_context : ContextHandle{}; -} - -namespace { -TexObjectHandle make_tex_object_handle(const CUDA_RESOURCE_DESC& res, - const CUDA_TEXTURE_DESC& tex, - std::shared_ptr h_backing, - const ContextHandle& h_context) { - GILReleaseGuard gil; - CUtexObject obj = 0; - err = invoke_in_context_or_undo( - h_context, - [&]() noexcept { return p_cuTexObjectCreate(&obj, &res, &tex, nullptr); }, - [&]() noexcept { pw_cuTexObjectDestroy(obj); }, - /*undo_requires_target_context=*/true); - if (err != CUDA_SUCCESS) { - return {}; - } - auto box = std::shared_ptr( - new TexObjectBox{TexObjectValue{obj}, std::move(h_backing), h_context}, - [](const TexObjectBox* b) { - GILReleaseGuard gil; - cleanup_in_context(b->h_context, "cuTexObjectDestroy", [&]() noexcept { - return p_cuTexObjectDestroy(b->resource.raw); - }); - delete b; - } - ); - return TexObjectHandle(box, &box->resource); -} -} // namespace - -TexObjectHandle create_tex_object_handle_array(const ContextHandle& h_context, - const CUDA_RESOURCE_DESC& res, - const CUDA_TEXTURE_DESC& tex, - const OpaqueArrayHandle& h_backing) { - return make_tex_object_handle(res, tex, h_backing, h_context); -} - -TexObjectHandle create_tex_object_handle_mipmap(const ContextHandle& h_context, - const CUDA_RESOURCE_DESC& res, - const CUDA_TEXTURE_DESC& tex, - const MipmappedArrayHandle& h_backing) { - return make_tex_object_handle(res, tex, h_backing, h_context); -} - -TexObjectHandle create_tex_object_handle_linear(const ContextHandle& h_context, - const CUDA_RESOURCE_DESC& res, - const CUDA_TEXTURE_DESC& tex, - const DevicePtrHandle& h_backing) { - return make_tex_object_handle(res, tex, h_backing, h_context); -} - -SurfObjectHandle create_surf_object_handle(const ContextHandle& h_context, - const CUDA_RESOURCE_DESC& res, - const OpaqueArrayHandle& h_backing) { - GILReleaseGuard gil; - CUsurfObject obj = 0; - err = invoke_in_context_or_undo( - h_context, - [&]() noexcept { return p_cuSurfObjectCreate(&obj, &res); }, - [&]() noexcept { pw_cuSurfObjectDestroy(obj); }, - /*undo_requires_target_context=*/true); - if (err != CUDA_SUCCESS) { - return {}; - } - auto box = std::shared_ptr( - new SurfObjectBox{SurfObjectValue{obj}, h_backing, h_context}, - [](const SurfObjectBox* b) { - GILReleaseGuard gil; - cleanup_in_context(b->h_context, "cuSurfObjectDestroy", [&]() noexcept { - return p_cuSurfObjectDestroy(b->resource.raw); - }); - delete b; - } - ); - return SurfObjectHandle(box, &box->resource); -} - -// ============================================================================ -// SM resource split wrapper -// ============================================================================ - -CUresult sm_resource_split(CUdevResource* result, unsigned int nbGroups, - const CUdevResource* input, CUdevResource* remainder, - unsigned int flags, void* groupParams) { -#if CUDA_VERSION >= 13010 - if (!p_cuDevSmResourceSplit) { - return CUDA_ERROR_NOT_SUPPORTED; - } - return p_cuDevSmResourceSplit( - result, nbGroups, input, remainder, flags, - static_cast(groupParams)); -#else - return CUDA_ERROR_NOT_SUPPORTED; -#endif -} - -bool has_sm_resource_split() noexcept { - return p_cuDevSmResourceSplit != nullptr; -} - -// ============================================================================ -// cuMemcpyWithAttributesAsync wrapper -// ============================================================================ - -CUresult memcpy_with_attributes_async(CUdeviceptr dst, CUdeviceptr src, size_t size, - void* attr, CUstream hStream) { -#if CUDA_VERSION >= 13020 - if (!p_cuMemcpyWithAttributesAsync) { - return CUDA_ERROR_NOT_SUPPORTED; - } - return p_cuMemcpyWithAttributesAsync( - dst, src, size, static_cast(attr), hStream); -#else - return CUDA_ERROR_NOT_SUPPORTED; -#endif -} - -bool has_memcpy_with_attributes_async() noexcept { - return p_cuMemcpyWithAttributesAsync != nullptr; -} - -} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/stream.cpp b/cuda_core/cuda/core/_cpp/rt/stream.cpp new file mode 100644 index 00000000000..eb17b428f11 --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/stream.cpp @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "context_scope.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +namespace { +// Return whether a stream handle needs a current context to resolve it. +bool is_default_stream(CUstream stream) noexcept { + return stream == nullptr || stream == CU_STREAM_LEGACY || stream == CU_STREAM_PER_THREAD; +} +} // namespace + +namespace detail { +// Return the context a deallocation-stream token must run under. Real streams +// resolve their own context; default-stream tokens use the context bound at +// allocation time. Warn when PTDS deallocation crosses host threads. +ContextHandle deallocation_context(const DeallocationStream& stream) noexcept { + if (!is_default_stream(as_cu(stream.h_stream))) { + return {}; + } + if (stream.ptds_tid != std::thread::id{} + && stream.ptds_tid != std::this_thread::get_id()) { + report_message( + "Buffer deallocation for a per-thread default stream " + "is running on a different host thread than the one that recorded " + "the deallocation stream; ordering relative to the allocating " + "thread's PTDS is not preserved"); + } + return get_stream_context(stream.h_stream); +} +} // namespace detail + +// ============================================================================ +// Stream Handles +// ============================================================================ + +namespace { +struct StreamBox { + CUstream resource; + ContextHandle h_context; +}; + +static const StreamBox* get_box(const StreamHandle& h) noexcept { + const CUstream* p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) - offsetof(StreamBox, resource) + ); +} + +// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) +static HandleRegistry stream_registry; +} // namespace + +StreamHandle create_stream_handle(const ContextHandle& h_ctx, unsigned int flags, int priority) { + GILReleaseGuard gil; + CUstream stream = nullptr; + GreenCtxHandle h_green = get_context_green_ctx(h_ctx); + if (h_green) { + err = p_cuGreenCtxStreamCreate + ? p_cuGreenCtxStreamCreate(&stream, as_cu(h_green), flags, priority) + : CUDA_ERROR_NOT_SUPPORTED; + } else { + err = invoke_in_context_or_undo( + h_ctx, + [&]() noexcept { return p_cuStreamCreateWithPriority(&stream, flags, priority); }, + [&]() noexcept { pw_cuStreamDestroy(stream); }, + /*undo_requires_target_context=*/false); + } + if (err != CUDA_SUCCESS) { + return {}; + } + + auto box = std::shared_ptr( + new StreamBox{stream, h_ctx}, + [](const StreamBox* b) { + stream_registry.unregister_handle(b->resource); + GILReleaseGuard gil; + pw_cuStreamDestroy(b->resource); + delete b; + } + ); + StreamHandle h(box, &box->resource); + stream_registry.register_handle(stream, h); + return h; +} + +StreamHandle create_stream_handle_ref(CUstream stream) { + if (auto h = stream_registry.lookup(stream)) { + return h; + } + auto box = std::shared_ptr( + new StreamBox{stream, {}}, + [](const StreamBox* b) { + stream_registry.unregister_handle(b->resource); + delete b; + } + ); + StreamHandle h(box, &box->resource); + stream_registry.register_handle(stream, h); + return h; +} + +StreamHandle create_stream_handle_with_owner(CUstream stream, PyObject* owner) { + if (auto h = stream_registry.lookup(stream)) { + // Reuse handles that already carry structural context metadata, e.g. + // cuda-core-owned streams. + if (get_box(h)->h_context) { + return h; + } + } + if (!owner) { + return create_stream_handle_ref(stream); + } + // GIL required when owner is provided + GILAcquireGuard gil; + if (!gil.acquired()) { + // Python finalizing - fall back to ref version (no owner tracking) + return create_stream_handle_ref(stream); + } + Py_INCREF(owner); + // Owner-backed handles are NOT registered in the stream registry to avoid + // corruption when multiple owners wrap the same CUstream (each stacks its + // own Py_INCREF/Py_DECREF independently). + auto box = std::shared_ptr( + new StreamBox{stream, {}}, + [owner](const StreamBox* b) { + GILAcquireGuard gil; + if (gil.acquired()) { + Py_DECREF(owner); + } + delete b; + } + ); + return StreamHandle(box, &box->resource); +} + +// Return the context retained by a stream handle. +ContextHandle get_stream_context(const StreamHandle& h) noexcept { + return h ? get_box(h)->h_context : ContextHandle{}; +} + +StreamHandle get_legacy_stream() { + static StreamHandle handle = create_stream_handle_ref(CU_STREAM_LEGACY); + return handle; +} + +StreamHandle get_per_thread_stream() { + static StreamHandle handle = create_stream_handle_ref(CU_STREAM_PER_THREAD); + return handle; +} + +StreamHandle create_context_bound_legacy_stream(const ContextHandle& h_context) { + if (!h_context) { + return {}; + } + // Default deleter: this handle never owns CU_STREAM_LEGACY, so nothing + // needs to run when the last reference is released. + auto box = std::make_shared(StreamBox{CU_STREAM_LEGACY, h_context}); + return StreamHandle(box, &box->resource); +} + +// ============================================================================ +// Deallocation streams +// +// A DeallocationStream is a StreamHandle used for ordering frees. It differs +// from an ordinary StreamHandle only for default-stream tokens, for which it +// stores the (de)allocation context. Ordinarily, the LEGACY and PER_THREAD +// default streams resolve to whichever context is active at the time they are +// used, but for storing deallocation recipes we need to pin the context. With +// the PER_THREAD token, it is not possible to restore the original stream when +// deallocation runs on a different thread. Therefore, in that case the +// allocating host thread id is also stored so that cross-thread frees can be +// detected and warnings can be issued. +// ============================================================================ + +namespace detail { +// Real streams are copied unchanged. Default-stream tokens without an embedded +// context are bound to the current context. Returns false (and sets err) when a +// default-stream token cannot be bound because no context is current. +bool make_deallocation_stream( + const StreamHandle& h, DeallocationStream& out) noexcept { + out = {}; + if (!h) { + return true; + } + + const CUstream stream = as_cu(h); + if (!is_default_stream(stream)) { + out = DeallocationStream{h, {}}; + return true; + } + + StreamHandle h_bound = h; + if (!get_stream_context(h)) { + ContextHandle h_ctx = get_current_context(); + if (!h_ctx) { + if (err == CUDA_SUCCESS) { + err = CUDA_ERROR_INVALID_CONTEXT; + } + return false; + } + // Do not register in stream_registry: the token value alone is not + // a unique stream identity (context is part of the meaning). + auto box = std::shared_ptr( + new StreamBox{stream, h_ctx}); + h_bound = StreamHandle(box, &box->resource); + } + + std::thread::id ptds_tid{}; + if (stream == CU_STREAM_PER_THREAD) { + ptds_tid = std::this_thread::get_id(); + } + out = DeallocationStream{std::move(h_bound), ptds_tid}; + return true; +} +} // namespace detail + +} // namespace cuda_core::rt diff --git a/cuda_core/cuda/core/_cpp/rt/texture.cpp b/cuda_core/cuda/core/_cpp/rt/texture.cpp new file mode 100644 index 00000000000..11680a7e6bc --- /dev/null +++ b/cuda_core/cuda/core/_cpp/rt/texture.cpp @@ -0,0 +1,265 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +#include "py.hpp" +#include "api.hpp" +#include "context_scope.hpp" +#include "driver_api.hpp" +#include "error.hpp" +#include "internal.hpp" +#include +#include +#include + +namespace cuda_core::rt { + +using namespace detail; + +// ============================================================================ +// Graphics Resource Handles +// ============================================================================ + +namespace { +struct GraphicsResourceBox { + CUgraphicsResource resource; +}; +} // namespace + +GraphicsResourceHandle create_graphics_resource_handle(CUgraphicsResource resource) { + auto box = std::shared_ptr( + new GraphicsResourceBox{resource}, + [](const GraphicsResourceBox* b) { + GILReleaseGuard gil; + pw_cuGraphicsUnregisterResource(b->resource); + delete b; + } + ); + return GraphicsResourceHandle(box, &box->resource); +} + +// ============================================================================ +// Array / mipmapped-array / texture / surface handles (PR #467) +// ============================================================================ + +namespace { +struct ArrayBox { + CUarray resource; + // Non-null only for a mipmap-level view: keeps the parent mipmap (the real + // owner of the level's storage) alive for as long as the level is held. + MipmappedArrayHandle h_parent; + ContextHandle h_context; +}; + +struct MipmappedArrayBox { + CUmipmappedArray resource; + ContextHandle h_context; +}; + +// Texture and surface objects are per-context pool indices. Destroying one +// with the wrong context current can silently succeed without freeing it or +// can free an unrelated object, so destruction must enter the creating +// context. Handle-based resources resolve their own context and must not. +struct TexObjectBox { + // Tagged so TexObjectHandle is a distinct C++ type from DevicePtrHandle / + // SurfObjectHandle (all wrap `unsigned long long`). + TexObjectValue resource; + // Type-erased backing dependency (OpaqueArrayHandle / MipmappedArrayHandle / + // DevicePtrHandle). The texture's resource is a union; we only need to keep + // whichever backing it was built from alive, never to dereference it. + std::shared_ptr h_backing; + ContextHandle h_context; +}; + +struct SurfObjectBox { + SurfObjectValue resource; + OpaqueArrayHandle h_array; // surfaces are always array-backed + ContextHandle h_context; +}; + +// Recover an array's owning box from its aliased resource pointer. +const ArrayBox* get_box(const OpaqueArrayHandle& h) noexcept { + const CUarray* p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) - offsetof(ArrayBox, resource)); +} + +// Recover a mipmapped array's owning box from its aliased resource pointer. +const MipmappedArrayBox* get_box(const MipmappedArrayHandle& h) noexcept { + const CUmipmappedArray* p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) + - offsetof(MipmappedArrayBox, resource)); +} + +// Wrap an array with shared owning-destruction behavior. +static OpaqueArrayHandle wrap_array_owned(CUarray arr, ContextHandle h_context) { + auto box = std::shared_ptr( + new ArrayBox{arr, {}, std::move(h_context)}, + [](const ArrayBox* b) { + GILReleaseGuard gil; + pw_cuArrayDestroy(b->resource); + delete b; + } + ); + return OpaqueArrayHandle(box, &box->resource); +} + +} // namespace + +OpaqueArrayHandle create_array_handle(const ContextHandle& h_context, const CUDA_ARRAY3D_DESCRIPTOR& desc) { + GILReleaseGuard gil; + CUarray arr = nullptr; + err = invoke_in_context_or_undo( + h_context, + [&]() noexcept { return p_cuArray3DCreate(&arr, &desc); }, + [&]() noexcept { pw_cuArrayDestroy(arr); }, + /*undo_requires_target_context=*/false); + if (err != CUDA_SUCCESS) { + return {}; + } + return wrap_array_owned(arr, h_context); +} + +OpaqueArrayHandle create_array_handle_ref(CUarray arr) { + if (!arr) { + return {}; + } + auto box = std::make_shared(ArrayBox{arr, {}, {}}); + return OpaqueArrayHandle(box, &box->resource); +} + +OpaqueArrayHandle create_array_handle_owning(CUarray arr) { + if (!arr) { + return {}; + } + return wrap_array_owned(arr, {}); +} + +// Return the context retained by an array handle. +ContextHandle get_array_context(const OpaqueArrayHandle& h) noexcept { + return h ? get_box(h)->h_context : ContextHandle{}; +} + +OpaqueArrayHandle create_array_level_handle(const MipmappedArrayHandle& h_mip, unsigned int level) { + GILReleaseGuard gil; + CUarray arr; + ContextHandle h_context = h_mip ? get_box(h_mip)->h_context : ContextHandle{}; + if (CUDA_SUCCESS != (err = p_cuMipmappedArrayGetLevel(&arr, as_cu(h_mip), level))) { + return {}; + } + // Non-owning level view: storage belongs to the mipmap. Embed the mipmap + // handle so the parent outlives this level; the deleter does not destroy. + auto box = std::shared_ptr( + new ArrayBox{arr, h_mip, h_context}, + [](const ArrayBox* b) { delete b; } + ); + return OpaqueArrayHandle(box, &box->resource); +} + +MipmappedArrayHandle create_mipmapped_array_handle(const ContextHandle& h_context, + const CUDA_ARRAY3D_DESCRIPTOR& desc, + unsigned int num_levels) { + GILReleaseGuard gil; + CUmipmappedArray mip = nullptr; + err = invoke_in_context_or_undo( + h_context, + [&]() noexcept { return p_cuMipmappedArrayCreate(&mip, &desc, num_levels); }, + [&]() noexcept { pw_cuMipmappedArrayDestroy(mip); }, + /*undo_requires_target_context=*/false); + if (err != CUDA_SUCCESS) { + return {}; + } + auto box = std::shared_ptr( + new MipmappedArrayBox{mip, h_context}, + [](const MipmappedArrayBox* b) { + GILReleaseGuard gil; + pw_cuMipmappedArrayDestroy(b->resource); + delete b; + } + ); + return MipmappedArrayHandle(box, &box->resource); +} + +// Return the context retained by a mipmapped array handle. +ContextHandle get_mipmapped_array_context(const MipmappedArrayHandle& h) noexcept { + return h ? get_box(h)->h_context : ContextHandle{}; +} + +namespace { +TexObjectHandle make_tex_object_handle(const CUDA_RESOURCE_DESC& res, + const CUDA_TEXTURE_DESC& tex, + std::shared_ptr h_backing, + const ContextHandle& h_context) { + GILReleaseGuard gil; + CUtexObject obj = 0; + err = invoke_in_context_or_undo( + h_context, + [&]() noexcept { return p_cuTexObjectCreate(&obj, &res, &tex, nullptr); }, + [&]() noexcept { pw_cuTexObjectDestroy(obj); }, + /*undo_requires_target_context=*/true); + if (err != CUDA_SUCCESS) { + return {}; + } + auto box = std::shared_ptr( + new TexObjectBox{TexObjectValue{obj}, std::move(h_backing), h_context}, + [](const TexObjectBox* b) { + GILReleaseGuard gil; + cleanup_in_context(b->h_context, "cuTexObjectDestroy", [&]() noexcept { + return p_cuTexObjectDestroy(b->resource.raw); + }); + delete b; + } + ); + return TexObjectHandle(box, &box->resource); +} +} // namespace + +TexObjectHandle create_tex_object_handle_array(const ContextHandle& h_context, + const CUDA_RESOURCE_DESC& res, + const CUDA_TEXTURE_DESC& tex, + const OpaqueArrayHandle& h_backing) { + return make_tex_object_handle(res, tex, h_backing, h_context); +} + +TexObjectHandle create_tex_object_handle_mipmap(const ContextHandle& h_context, + const CUDA_RESOURCE_DESC& res, + const CUDA_TEXTURE_DESC& tex, + const MipmappedArrayHandle& h_backing) { + return make_tex_object_handle(res, tex, h_backing, h_context); +} + +TexObjectHandle create_tex_object_handle_linear(const ContextHandle& h_context, + const CUDA_RESOURCE_DESC& res, + const CUDA_TEXTURE_DESC& tex, + const DevicePtrHandle& h_backing) { + return make_tex_object_handle(res, tex, h_backing, h_context); +} + +SurfObjectHandle create_surf_object_handle(const ContextHandle& h_context, + const CUDA_RESOURCE_DESC& res, + const OpaqueArrayHandle& h_backing) { + GILReleaseGuard gil; + CUsurfObject obj = 0; + err = invoke_in_context_or_undo( + h_context, + [&]() noexcept { return p_cuSurfObjectCreate(&obj, &res); }, + [&]() noexcept { pw_cuSurfObjectDestroy(obj); }, + /*undo_requires_target_context=*/true); + if (err != CUDA_SUCCESS) { + return {}; + } + auto box = std::shared_ptr( + new SurfObjectBox{SurfObjectValue{obj}, h_backing, h_context}, + [](const SurfObjectBox* b) { + GILReleaseGuard gil; + cleanup_in_context(b->h_context, "cuSurfObjectDestroy", [&]() noexcept { + return p_cuSurfObjectDestroy(b->resource.raw); + }); + delete b; + } + ); + return SurfObjectHandle(box, &box->resource); +} + +} // namespace cuda_core::rt diff --git a/cuda_core/tests/test_rt_layout.py b/cuda_core/tests/test_rt_layout.py new file mode 100644 index 00000000000..53158370968 --- /dev/null +++ b/cuda_core/tests/test_rt_layout.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Layout rules of cuda/core/_cpp/rt/, the C++ behind cuda.core._rt. + +Source-tree properties only: no compiler, no GPU. Run with --noconftest, since +conftest.py imports the compiled package: + + pytest tests/test_rt_layout.py -v --noconftest + +Why the rules exist: _rt.pxd names handles.hpp, and Cython compiles every +cimporting extension against a copy of that header placed in its build +directory, next to copies of the extension's `depends`. So handles.hpp may pull +in only what thirty-odd consumer extensions can safely compile: types, +templates and inline functions, through file-relative includes. Everything with +storage or a body lives behind rt.hpp, which only _rt.pyx names. +""" + +import re +from pathlib import Path + +import pytest + +CORE = Path(__file__).resolve().parent.parent / "cuda" / "core" +RT = CORE / "_cpp" / "rt" +HEADERS = sorted(RT.glob("*.hpp")) +SOURCES = sorted(RT.glob("*.cpp")) +UMBRELLAS = {"rt.hpp", "handles.hpp"} +PYTHON_TOKEN = re.compile(r"\bPy[A-Z_]\w*|\bPyObject\b|Python\.h") +QUOTED_INCLUDE = re.compile(r'^\s*#\s*include\s+"([^"]+)"', re.M) + + +def read(path): + return path.read_text(encoding="utf-8") + + +def quoted_includes(path): + return QUOTED_INCLUDE.findall(read(path)) + + +def include_closure(path): + seen = [] + todo = [path] + while todo: + current = todo.pop() + if current in seen: + continue + seen.append(current) + todo.extend(current.parent / name for name in quoted_includes(current)) + return sorted(seen) + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_python_h_is_spelled_only_in_py_hpp(): + spellers = [p.name for p in HEADERS + SOURCES if "" in read(p)] + assert spellers == ["py.hpp"] + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_neutral_files_do_not_name_python(): + """Every header except the seam and the umbrellas, and every source that does + not include py.hpp, compiles without a Python include path.""" + neutral_sources = [p for p in SOURCES if "py.hpp" not in quoted_includes(p)] + assert {p.name for p in neutral_sources} == {"driver_api.cpp", "error.cpp"} + neutral = [p for p in HEADERS if p.name not in UMBRELLAS | {"py.hpp"}] + neutral_sources + offenders = {p.name: PYTHON_TOKEN.findall(read(p)) for p in neutral} + assert {name: hits for name, hits in offenders.items() if hits} == {} + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_in_tree_includes_are_bare_sibling_names_that_exist(): + for path in HEADERS + SOURCES: + for name in quoted_includes(path): + assert "/" not in name, f"{path.name} includes {name!r}; in-tree includes are bare sibling names" + assert (RT / name).is_file(), f"{path.name} includes {name!r}, which does not exist" + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_umbrellas_are_named_only_by_their_cython_file(): + for path in HEADERS + SOURCES: + assert UMBRELLAS.isdisjoint(quoted_includes(path)), f"{path.name} includes an umbrella" + named = {} + for path in sorted(list(CORE.rglob("*.pyx")) + list(CORE.rglob("*.pxd"))): + for header in re.findall(r'cdef extern from "(_cpp/rt/[^"]+)"', read(path)): + named.setdefault(header, set()).add(path.name) + assert named == {"_cpp/rt/rt.hpp": {"_rt.pyx"}, "_cpp/rt/handles.hpp": {"_rt.pxd"}} + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_consumer_closure_is_types_and_the_python_seam(): + closure = {p.name for p in include_closure(RT / "handles.hpp")} + assert closure == {"handles.hpp", "py.hpp", "types.hpp"} + # Consumers are RTLD_LOCAL extensions that cannot link to _rt: nothing with storage. + for name in sorted(closure): + text = read(RT / name) + assert not re.search(r'^extern (?!"C")', text, re.M), f"{name} declares an extern variable" + assert not re.search(r"^(static|thread_local)\b", text, re.M), f"{name} defines storage" + + +@pytest.mark.agent_authored(model="claude-fable-5-1") +def test_pxd_functions_are_not_called_by_name_inside_the_module(): + """Cython emits a static prototype for each cdef function the .pxd declares, so + calling one by that name from _rt.pyx clashes with the extern C++ declaration. + The module calls through an alias with a different Cython name instead.""" + names = re.findall(r"^cdef\s+(?:[\w:.*& \[\]]+?\s)?(\w+)\(", read(CORE / "_rt.pxd"), re.M) + assert len(names) > 90 + pyx = re.sub(r'""".*?"""', "", read(CORE / "_rt.pyx"), flags=re.S) + pyx = re.sub(r"#[^\n]*", "", pyx) + code = "\n".join(line for line in pyx.split("\n") if '"cuda_core::rt::' not in line) + assert {name for name in names if re.search(rf"\b{name}\(", code)} == set() From 35ecf47f798e63dd6c421e8f1e2219ad35bbdc16 Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Fri, 11 Sep 2026 10:59:47 -0700 Subject: [PATCH 13/14] cuda.core: update the design notes and agent guide for _rt DESIGN.md, GRAPH_ATTACHMENTS.md and REGISTRY_DESIGN.md moved with the code; this keeps them from misleading: the module name, the file layout, the cdef extern examples, and the heading that still named the deleted _CUDA_DRIVER_API_V1 capsule. AGENTS.md learns the directory form of _cpp// and the new paths. One release note for the renamed private module and its shipped .pxd. --- ci/tools/tests/test_compute_ci_plan.py | 2 +- cuda_core/AGENTS.md | 11 ++-- cuda_core/cuda/core/_cpp/rt/DESIGN.md | 52 +++++++++++-------- .../cuda/core/_cpp/rt/REGISTRY_DESIGN.md | 2 +- 4 files changed, 39 insertions(+), 28 deletions(-) diff --git a/ci/tools/tests/test_compute_ci_plan.py b/ci/tools/tests/test_compute_ci_plan.py index 79a83394dfa..40d1ebd9431 100644 --- a/ci/tools/tests/test_compute_ci_plan.py +++ b/ci/tools/tests/test_compute_ci_plan.py @@ -113,7 +113,7 @@ def test_ignored_paths_select_no_work(self) -> None: "cuda_core/tests/fixtures/pixi.toml", "benchmarks/cuda_bindings/pixi.toml", "benchmarks/cuda_bindings/AGENTS.md", - "cuda_core/cuda/core/_cpp/DESIGN.md", + "cuda_core/cuda/core/_cpp/rt/DESIGN.md", "cuda_bindings/README.md", "cuda_core/README.md", "new-area/pixi.toml", diff --git a/cuda_core/AGENTS.md b/cuda_core/AGENTS.md index fa1b2ce5c51..cec6daf78a7 100644 --- a/cuda_core/AGENTS.md +++ b/cuda_core/AGENTS.md @@ -19,8 +19,9 @@ This file describes `cuda_core`, the high-level Pythonic CUDA subpackage in the - system-level APIs: `cuda/core/system/` - compile/link path: `_program.pyx`, `_linker.pyx`, `_module.pyx` - execution path: `_launcher.pyx`, `_launch_config.pyx`, `_stream.pyx` -- **C++ helpers**: module-specific C++ implementations live under - `cuda/core/_cpp/`. +- **C++ helpers**: module-specific C++ lives under `cuda/core/_cpp/`, either as + one `_cpp/.cpp` or as a directory `_cpp//` whose sources all + compile into the `_` extension (`_cpp/rt/` for `_rt`). - **Build backend**: `build_hooks.py` handles Cython extension setup and build dependency wiring. @@ -90,7 +91,7 @@ and agents should flag violations. objects that are not meant to be shared (e.g., the thread-local `Device`) do not need such guards (see #2321). Reference-count integrity is guaranteed; cache value-identity/idempotency is not. -- **Entry points assume the GIL is held**: the helpers in `_cpp/resource_handles.*` +- **Entry points assume the GIL is held**: the helpers in `_cpp/rt/` are called from Cython with the GIL held and do not re-acquire it. Driver and destructor callbacks run at arbitrary times, so they take the GIL (`with gil`) and probe for interpreter shutdown before touching Python objects. @@ -141,7 +142,7 @@ below are for contributors. Reviewers and agents should flag violations. `KeyboardInterrupt`. - **Finalization**: once `py_is_finalizing()` is true, do no Python work from destructors or callbacks and accept the leak (see - `_cpp/resource_handles.hpp` and `_cpp/GRAPH_ATTACHMENTS.md`). + `_cpp/rt/py.hpp` and `_cpp/rt/GRAPH_ATTACHMENTS.md`). - **Aborting**: `std::abort` (or any process termination) is reserved for an internal invariant violation where continuing could corrupt memory or produce silently wrong results *and* no leak-based fallback exists. A failed CUDA @@ -157,7 +158,7 @@ below are for contributors. Reviewers and agents should flag violations. an allocation inside `noexcept` code) is a bug (#1489, #2417), not a policy choice: `noexcept` helpers must not allocate, or must catch what they call. - **Testing**: inject restoration failures with - `cuda.core._resource_handles._set_context_restore_fault_for_testing`; assert + `cuda.core._rt._set_context_restore_fault_for_testing`; assert reports with `pytest.warns(CUDAWarning)` or `warnings.catch_warnings`, never by matching stderr text. diff --git a/cuda_core/cuda/core/_cpp/rt/DESIGN.md b/cuda_core/cuda/core/_cpp/rt/DESIGN.md index d0f2fdcc799..46290d625de 100644 --- a/cuda_core/cuda/core/_cpp/rt/DESIGN.md +++ b/cuda_core/cuda/core/_cpp/rt/DESIGN.md @@ -106,37 +106,47 @@ return as_py(h_stream) # cuda.bindings.driver.CUstream ``` cuda/core/ -├── _resource_handles.pyx # Cython module (compiles resource_handles.cpp) -├── _resource_handles.pxd # Cython declarations for consumer modules -└── _cpp/ - ├── resource_handles.hpp # C++ API declarations - └── resource_handles.cpp # C++ implementation +├── _rt.pyx # Cython module (compiles everything under _cpp/rt/) +├── _rt.pxd # Cython declarations for consumer modules +└── _cpp/rt/ + ├── rt.hpp # Module umbrella, named only by _rt.pyx + ├── handles.hpp # Consumer umbrella, named only by _rt.pxd + ├── types.hpp # Handle aliases, tagged values, inline accessors + ├── api.hpp # Prototypes of the handle factories and accessors + ├── driver_api.hpp/.cpp # Driver function-pointer table, version-gated shims + ├── error.hpp/.cpp # Thread-local error state, non-propagating reporting + ├── py.hpp # The one header that includes + ├── context_scope.hpp # Scoped-context helpers (namespace detail) + ├── internal.hpp # Registry, cleanup wrappers, deferred-cleanup item (namespace detail) + ├── py_report.cpp, py_deferred_cleanup.cpp # Python-coupled bodies + └── context.cpp, stream.cpp, event.cpp, memory.cpp, program.cpp, + graph.cpp, graph_exec.cpp, texture.cpp # One resource family per file ``` ### Build Implications -The `_cpp/` subdirectory contains C++ source files that are compiled into the -`_resource_handles` extension module. Other Cython modules in cuda.core do **not** -link against this code directly—they `cimport` functions from -`_resource_handles.pxd`, and calls go through `_resource_handles.so` at runtime. +Every `.cpp` under `_cpp/rt/` is compiled into the one `_rt` extension module. +Other Cython modules in cuda.core do **not** link against this code +directly—they `cimport` functions from `_rt.pxd`, and calls go through +`_rt.so` at runtime. ## Cross-Module Function Sharing **Problem**: Cython extension modules compile independently. If multiple modules -(`_memory.pyx`, `_ipc.pyx`, etc.) each linked `resource_handles.cpp`, they would +(`_memory.pyx`, `_ipc.pyx`, etc.) each linked the C++ under `_cpp/rt/`, they would each have their own copies of: - Static driver function pointers - Thread-local error state - Other static data, including global caches -**Solution**: Only `_resource_handles.so` links the C++ code. The `.pyx` file +**Solution**: Only `_rt.so` links the C++ code. The `.pyx` file uses `cdef extern from` to declare C++ functions with Cython-accessible names: ```cython -# In _resource_handles.pyx -cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": - StreamHandle create_stream_handle "cuda_core::create_stream_handle" ( +# In _rt.pyx +cdef extern from "_cpp/rt/rt.hpp" namespace "cuda_core::rt": + StreamHandle create_stream_handle "cuda_core::rt::create_stream_handle" ( ContextHandle h_ctx, unsigned int flags, int priority) nogil # ... other functions ``` @@ -144,18 +154,18 @@ cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": The `.pxd` file declares these same functions so other modules can `cimport` them: ```cython -# In _resource_handles.pxd +# In _rt.pxd cdef StreamHandle create_stream_handle( ContextHandle h_ctx, unsigned int flags, int priority) noexcept nogil ``` The `cdef extern from` declaration in the `.pyx` satisfies the `.pxd` declaration directly—no wrapper functions are needed. When consumer modules `cimport` these -functions, Cython generates calls through `_resource_handles.so` at runtime. +functions, Cython generates calls through `_rt.so` at runtime. This ensures all static and thread-local state lives in a single shared library, avoiding the duplicate state problem. -## CUDA Driver API Capsule (`_CUDA_DRIVER_API_V1`) +## CUDA driver function pointers via cuda-bindings' `__pyx_capi__` **Problem**: cuda.core cannot directly call CUDA driver functions because: @@ -165,13 +175,13 @@ avoiding the duplicate state problem. **Solution**: The C++ code declares extern function pointer variables: ```cpp -// resource_handles.hpp +// driver_api.hpp extern decltype(&cuStreamCreateWithPriority) p_cuStreamCreateWithPriority; extern decltype(&cuMemPoolCreate) p_cuMemPoolCreate; // ... etc ``` -At module import time, `_resource_handles.pyx` populates these pointers by +At module import time, `_rt.pyx` populates these pointers by extracting them from `cuda.bindings.cydriver.__pyx_capi__`: ```cython @@ -324,7 +334,7 @@ only when there is no such exception or notes are unavailable. ## Usage from Cython ```cython -from cuda.core._resource_handles cimport ( +from cuda.core._rt cimport ( StreamHandle, create_stream_handle, as_cu, @@ -353,7 +363,7 @@ The resource handle design: 2. **Encodes lifetimes structurally** via embedded handle dependencies. 3. **Uses Cython's `cimport` mechanism** to share C++ code across modules without duplicate static/thread-local state. -4. **Uses a capsule** to resolve CUDA driver symbols dynamically through cuda-bindings. +4. **Resolves CUDA driver symbols** dynamically through cuda-bindings' `__pyx_capi__` capsules. 5. **Provides overloaded accessors** (`as_cu`, `as_intptr`, `as_py`) since handles cannot have attributes without unnecessary Python object wrappers. diff --git a/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md b/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md index 42311eddb48..55d3fb043a9 100644 --- a/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md +++ b/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md @@ -19,7 +19,7 @@ expires. ## Level 1: Driver Handle -> Resource Handle (C++) -`HandleRegistry` in `resource_handles.cpp` maps a raw CUDA handle +`HandleRegistry` in `_cpp/rt/internal.hpp` maps a raw CUDA handle (e.g., `CUevent`, `CUkernel`, `CUgraph`) to a `weak_ptr` for its owning resource handle. When a `_ref` constructor receives a raw handle, it checks the registry first. If found, it returns the existing From 2161c0191aa78173dcc0d205a9989c9acc784903 Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Fri, 11 Sep 2026 14:45:00 -0700 Subject: [PATCH 14/14] cuda.core build: parallelize only compilers that use CCompiler.compile The base CCompiler defines a placeholder _compile(), so testing for that attribute matched MSVC as well. MSVCCompiler overrides compile() wholesale and never calls _compile(), so the override produced no objects on Windows and every Windows build failed at link time. Gate on the compile() method itself: apply the thread pool only when the compiler still uses CCompiler.compile(), which is what drives _compile() (the Unix family). --- cuda_core/setup.py | 8 +++++--- cuda_core/tests/test_build_hooks.py | 19 +++++++++++++------ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/cuda_core/setup.py b/cuda_core/setup.py index 19dd2cd7d23..b19a99c8d2c 100644 --- a/cuda_core/setup.py +++ b/cuda_core/setup.py @@ -5,6 +5,7 @@ import contextlib import os from concurrent.futures import ThreadPoolExecutor +from distutils.ccompiler import CCompiler from pathlib import Path import build_hooks # our build backend @@ -88,11 +89,12 @@ def _parallel_source_compilation(self): cuda.core._rt (a dozen .cpp files) becomes the critical path. This mirrors CCompiler.compile() and fans its per-object _compile() calls out to a pool shared by all extensions, so at most `nthreads` compiler - processes run at once. MSVC's compiler class has no _compile(); it keeps - the stock path. + processes run at once. It applies only to compilers that still use + CCompiler.compile(), which drives the per-object _compile() hook (the + Unix family); MSVC overrides compile() wholesale and keeps the stock path. """ compiler = self.compiler - if nthreads <= 1 or not hasattr(compiler, "_compile"): + if nthreads <= 1 or type(compiler).compile is not CCompiler.compile: yield return stock_compile = compiler.compile diff --git a/cuda_core/tests/test_build_hooks.py b/cuda_core/tests/test_build_hooks.py index 8807a4fb05d..27c057e0297 100644 --- a/cuda_core/tests/test_build_hooks.py +++ b/cuda_core/tests/test_build_hooks.py @@ -22,7 +22,7 @@ import sys import tempfile import threading -import types +from distutils.ccompiler import CCompiler from pathlib import Path from unittest import mock @@ -386,8 +386,13 @@ def test_headers_under_module_directories_only(self, tmp_path, monkeypatch): class TestParallelSourceCompilation: """setup.py compiles an extension's sources through one shared thread pool.""" - class FakeCompiler: + class FakeCompiler(CCompiler): + """Uses the stock CCompiler.compile(), like the Unix compilers.""" + + executables = {} + def __init__(self, fail_on=None): + super().__init__() self.compiled = [] self.fail_on = fail_on self.lock = threading.Lock() @@ -406,6 +411,9 @@ def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): with self.lock: self.compiled.append((obj, src, ext, tuple(cc_args), tuple(extra_postargs), tuple(pp_opts))) + class MsvcLikeCompiler(FakeCompiler): + """Overrides compile() wholesale, like MSVCCompiler.""" + def compile(self, *args, **kwargs): return "stock" @@ -427,7 +435,7 @@ def test_every_source_compiles_once_and_the_object_order_is_kept(self, monkeypat assert objects == [source + ".o" for source in sources] assert sorted(entry[0] for entry in cmd.compiler.compiled) == sorted(objects) assert {entry[2:] for entry in cmd.compiler.compiled} == {(".cpp", ("-c", "-Dpp"), ("-O2",), ("-Dpp",))} - assert cmd.compiler.compile(sources) == "stock" # restored on exit + assert cmd.compiler.compile.__func__ is CCompiler.compile # restored on exit @pytest.mark.agent_authored(model="claude-fable-5-1") def test_a_failing_source_fails_the_extension(self, monkeypatch): @@ -439,8 +447,7 @@ def test_a_failing_source_fails_the_extension(self, monkeypatch): def test_serial_builds_and_compilers_without_the_hook_keep_the_stock_path(self, monkeypatch): cmd = self._build_ext(monkeypatch, 1, self.FakeCompiler()) with cmd._parallel_source_compilation(): - assert cmd.compiler.compile(["a.cpp"]) == "stock" - msvc_like = types.SimpleNamespace(compile=self.FakeCompiler().compile) # no _compile() - cmd = self._build_ext(monkeypatch, 4, msvc_like) + assert cmd.compiler.compile.__func__ is CCompiler.compile + cmd = self._build_ext(monkeypatch, 4, self.MsvcLikeCompiler()) with cmd._parallel_source_compilation(): assert cmd.compiler.compile(["a.cpp"]) == "stock"