Skip to content

Commit e251232

Browse files
committed
fix(cuda.core): bypass access revocation when destroying host pools
Skip peer-access revocation for locally created HOST and HOST_NUMA pools to investigate pinned-pool teardown OOMs and crashes. Continue destroying the pools and preserve cleanup for device, managed, and imported pools. This is a diagnostic workaround pending Windows CI validation. Related to #3426.
1 parent bcfd4c4 commit e251232

1 file changed

Lines changed: 12 additions & 5 deletions

File tree

‎cuda_core/cuda/core/_cpp/rt/memory.cpp‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ clear_mempool_peer_access(CUmemoryPool pool, int owner_device) noexcept
7676
// owner_device is the ordinal of a device-located pool, or -1 when there is no
7777
// device to skip (host pools) or the location is unknown (imported pools).
7878
static MemoryPoolHandle
79-
wrap_mempool_owned(CUmemoryPool pool, int owner_device)
79+
wrap_mempool_owned(CUmemoryPool pool, int owner_device, bool revoke_peer_access)
8080
{
8181
auto *raw = new MemoryPoolBox{pool};
8282
// Count before constructing the shared_ptr: if its control-block allocation
@@ -85,9 +85,11 @@ wrap_mempool_owned(CUmemoryPool pool, int owner_device)
8585
// clang-format off
8686
auto box = std::shared_ptr<const MemoryPoolBox>(
8787
raw,
88-
[owner_device](const MemoryPoolBox *b) {
88+
[owner_device, revoke_peer_access](const MemoryPoolBox *b) {
8989
GILReleaseGuard gil;
90-
clear_mempool_peer_access(b->resource, owner_device);
90+
if (revoke_peer_access) {
91+
clear_mempool_peer_access(b->resource, owner_device);
92+
}
9193
pw_cuMemPoolDestroy(b->resource);
9294
g_live_owned_mempools.fetch_sub(1, std::memory_order_relaxed);
9395
delete b;
@@ -113,7 +115,11 @@ create_mempool_handle(const CUmemPoolProps &props)
113115
return {};
114116
}
115117
int owner_device = props.location.type == CU_MEM_LOCATION_TYPE_DEVICE ? props.location.id : -1;
116-
return wrap_mempool_owned(pool, owner_device);
118+
// Diagnostic for gh-3426: avoid revoking GPU access to host pools
119+
// before destruction. Preserve the workaround for other pool locations.
120+
const bool is_host_pool =
121+
props.location.type == CU_MEM_LOCATION_TYPE_HOST || props.location.type == CU_MEM_LOCATION_TYPE_HOST_NUMA;
122+
return wrap_mempool_owned(pool, owner_device, !is_host_pool);
117123
}
118124

119125
MemoryPoolHandle
@@ -143,7 +149,8 @@ create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type)
143149
if (CUDA_SUCCESS != (err = DRIVER_CALL(cuMemPoolImportFromShareableHandle, &pool, handle_ptr, handle_type, 0))) {
144150
return {};
145151
}
146-
return wrap_mempool_owned(pool, -1);
152+
// Imported pools have no location metadata here; preserve their cleanup.
153+
return wrap_mempool_owned(pool, -1, true);
147154
}
148155

149156
// ============================================================================

0 commit comments

Comments
 (0)