Repository navigation
Expand file tree
/
Copy pathCUDACachingAllocator.cpp
More file actions
6069 lines (5501 loc) · 221 KB
/
Copy pathCUDACachingAllocator.cpp
File metadata and controls
6069 lines (5501 loc) · 221 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#include <c10/cuda/CUDACachingAllocator.h>
#include <c10/core/RingBuffer.h>
#include <c10/core/impl/GPUTrace.h>
#include <c10/cuda/CUDAAllocatorConfig.h>
#include <c10/cuda/CUDAException.h>
#include <c10/cuda/CUDAFunctions.h>
#include <c10/cuda/CUDAGraphsC10Utils.h>
#include <c10/cuda/CUDAGuard.h>
#include <c10/cuda/PeerToPeerAccess.h>
#include <c10/cuda/impl/CUDAGraphMemory.h>
#include <c10/util/Gauge.h>
#include <c10/util/Logging.h>
#include <c10/util/ScopeExit.h>
#include <c10/util/SmallVector.h>
#include <c10/util/UniqueVoidPtr.h>
#include <c10/util/env.h>
#include <c10/util/error.h>
#include <c10/util/flat_hash_map.h>
#include <c10/util/hash.h>
#include <c10/util/static_tracepoint.h>
#if defined(PYTORCH_C10_DRIVER_API_SUPPORTED) || defined(USE_ROCM)
#if defined(PYTORCH_C10_DRIVER_API_SUPPORTED)
#include <c10/cuda/driver_api.h>
#endif
#ifndef _WIN32
#include <sys/syscall.h>
#include <sys/types.h>
#include <unistd.h>
#else
#include <process.h>
#endif
#endif
#include <c10/util/Exception.h>
#include <c10/util/irange.h>
#include <cuda_runtime_api.h>
#include <algorithm>
#include <array>
#include <atomic>
#include <bit>
#include <cmath>
#include <cstddef>
#include <cstdint>
#include <deque>
#include <exception>
#include <functional>
#include <memory>
#include <mutex>
#include <new>
#include <ranges>
#include <set>
#include <stack>
#include <thread>
#include <type_traits>
#include <utility>
#include <vector>
TORCH_SDT_DEFINE_SEMAPHORE(malloc)
TORCH_SDT_DEFINE_SEMAPHORE(free)
// add these definitions so that we can compile with CUDA < 12.3
// borrowed from
// https://github.com/NVIDIA/nccl/blob/3ea7eedf3b9b94f1d9f99f4e55536dfcbd23c1ca/src/include/p2p.h#L20
#if CUDA_VERSION < 12030
#define CU_MEM_HANDLE_TYPE_FABRIC ((CUmemAllocationHandleType)0x8ULL)
#define CU_IPC_HANDLE_SIZE 64
typedef struct CUmemFabricHandle_st {
// NOLINTNEXTLINE(*-avoid-c-arrays)
unsigned char data[CU_IPC_HANDLE_SIZE];
} CUmemFabricHandle_v1;
typedef CUmemFabricHandle_v1 CUmemFabricHandle;
#endif
namespace c10 {
// NOLINTNEXTLINE(misc-use-internal-linkage)
C10_DEFINE_REGISTRY(FreeCudaMemoryCallbacksRegistry, FreeMemoryCallback)
namespace cuda::CUDACachingAllocator {
using namespace c10::CachingAllocator;
using namespace c10::CachingDeviceAllocator;
namespace {
// Process-wide expandable-segment virtual-address gauges. Bumped in the
// ExpandableSegment ctor (after a successful reservation) and dtor; read by
// getExpandableSegmentsReservedBytes()/getExpandableSegmentsCount() for ODS and
// out-of-VM diagnostics. Relaxed atomics: plain counters, not used for
// synchronization.
std::atomic<size_t> g_expandable_segments_reserved_bytes{0};
std::atomic<size_t> g_expandable_segments_count{0};
} // namespace
#if defined(PYTORCH_C10_DRIVER_API_SUPPORTED) || defined(USE_ROCM)
static int get_self_pid() {
#ifdef _WIN32
return _getpid();
#else
return getpid();
#endif // _WIN32
}
#endif // defined(PYTORCH_C10_DRIVER_API_SUPPORTED) || defined(USE_ROCM)
namespace Native {
//
// Yet another caching allocator for CUDA device allocations.
//
// - Allocations are associated with a stream. Once freed, blocks can be
// re-allocated on the same stream, but not on any other stream.
// - The allocator attempts to find the smallest cached block that will fit the
// requested size. If the block is larger than the requested size, it may be
// split. If no block is found, the allocator will delegate to cudaMalloc.
// - If the cudaMalloc fails, the allocator will attempt to free one cached
// block of sufficient size that is not split and retry the allocation.
// If this also fails, the allocator will attempt to free all cached blocks
// that are not split and retry the allocation.
// - Large (>1MB) and small allocations are stored in separate pools.
// Small requests are packed into 2MB buffers. Large requests will use the
// smallest available free block or allocate a new block using cudaMalloc.
// - To reduce fragmentation, requests between 1MB and 10MB will allocate and
// split a 20MB block, if no free block of sufficient size is available.
// - To further reduce fragmentation, blocks >= max_split_size are not allowed
// to be split. These oversize cached blocks will still satisfy requests
// within 1MB of the oversize cached block size.
//
// With this allocator, allocations and frees should logically be considered
// "usages" of the memory segment associated with streams, just like kernel
// launches. The programmer must insert the proper synchronization if memory
// segments are used from multiple streams.
//
// The library provides a recordStream() function to help insert the correct
// synchronization when allocations are used on multiple streams. This will
// ensure that the block is not reused before each recorded stream completes
// work.
//
/**
* Note [Interaction with CUDA graph capture]
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* Graph capture performs a dry run of a region of execution, freezing all CUDA
* work (and virtual addresses used during that work) into a "graph." The graph
* may be "replayed" like a single giant kernel, with greatly reduced CPU
* overhead as well as modestly improved GPU performance.
*
* Because capture bakes in memory addresses, the memory used during capture
* must be available for the graph to use during replay. DeviceCachingAllocator
* assigns and frees memory eagerly and dynamically, so if we're not careful
* about managing graphs' memory, at replay time those memory addresses could be
* used by other tensors.
*
* To guarantee a graph's baked in addresses are safe to reuse in replay,
* DeviceAllocator satisfies allocations from a graph-private memory pool during
* capture, and doesn't begin cudaFreeing those addresses until the graph is
* destroyed.
*
* Within the private pool, allocations are freed and reassigned as usual during
* capture. Memory regions will be used in a consistent order during replay. So
* a private pool doesn't use memory more wastefully than the default pools
* during capture, but it does reserve its high-water mark of used memory away
* from the default pools as long as the capture(s) it served survive
* (regardless whether those captures are idle or replaying).
*
* CUDAGraph's requests for private pools are mediated by
* DeviceAllocator::notifyCaptureBegin,
* notifyCaptureAboutToEnd,
* notifyCaptureEnded,
* notifyCaptureDestroy.
*/
// counter to track order for Mempool Registration
static std::atomic<int32_t> registration_counter_global{-1};
static char SHAREABLE_HANDLE_VERSION = 3;
enum ShareableHandleType : char {
SHAREABLE_CUDA_MALLOC = 'c',
SHAREABLE_CUDA_EXPANDABLE_SEGMENT = 'e'
};
namespace {
using stream_set = ska::flat_hash_set<cuda::CUDAStream>;
void decrease_stat_array(
StatArray& stat_array,
size_t amount,
const StatTypes& stat_types) {
for_each_selected_stat_type(
stat_types, [&stat_array, amount](size_t stat_type) {
stat_array[stat_type].decrease(amount);
});
}
struct Block;
struct PrivatePool;
struct BlockComparatorSizeCounterAddress {
bool operator()(const Block* a, const Block* b) const;
};
struct BlockComparatorAddress {
bool operator()(const Block* a, const Block* b) const;
};
struct BlockPool {
BlockPool(bool small, PrivatePool* private_pool = nullptr)
: is_small(small), owner_PrivatePool(private_pool) {}
// Do not insert or erase a Block from blocks/blocks_by_addr directly; use
// insert_into_blocks()/erase_from_blocks() instead.
std::set<Block*, BlockComparatorSizeCounterAddress> blocks;
std::set<Block*, BlockComparatorAddress> blocks_by_addr;
std::set<Block*, BlockComparatorAddress> unmapped;
// NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members)
const bool is_small;
PrivatePool* owner_PrivatePool;
int64_t get_free_blocks_call_count{0};
// Add a Block into blocks set with updating gc counter.
std::pair<decltype(blocks)::iterator, bool> insert_into_blocks(Block* block);
size_t erase_from_blocks(Block* block);
MempoolId_t owner_MempoolId() const;
};
struct ExpandableSegment;
struct Block {
c10::DeviceIndex device; // gpu
cudaStream_t stream; // allocation stream
CaptureId_t capture_id{0}; // capture in which this allocation was made
stream_set stream_uses; // streams on which the block was used
int32_t registration_counter{-1};
size_t size; // block size in bytes
size_t requested_size; // memory originally requested
BlockPool* pool{nullptr}; // owning memory pool
void* ptr{nullptr}; // memory address
bool allocated{false}; // in-use flag
bool mapped{true}; // is the virtual address range this Block references
// backed by physical pages. Always true when
// expandable_segment_ is null. When false
// This Block will be aligned to the segment size
// of its expandable_segment_.
Block* prev{nullptr}; // prev block if split from a larger allocation
Block* next{nullptr}; // next block if split from a larger allocation
int event_count{0}; // number of outstanding CUDA events
int64_t gc_count_base{0}; // get_free_blocks_call_count when Block is inserted
std::shared_ptr<GatheredContext> context_when_allocated;
// only set for the first block in the segment (when prev == null)
// this records the frame information when cudaMalloc was called
// whereas context_when_allocated records the last time we handed this
// memory out from our cache.
std::shared_ptr<GatheredContext> context_when_segment_allocated;
ExpandableSegment* expandable_segment_{nullptr};
Block(
c10::DeviceIndex device,
cudaStream_t stream,
size_t size,
BlockPool* pool,
void* ptr)
: device(device),
stream(stream),
size(size),
requested_size(0),
pool(pool),
ptr(ptr) {}
// constructor for search key
// Use the default value for registration_counter and not modify
// registration_counter_global, because the search key is just a
// dummy placeholder.
Block(c10::DeviceIndex device, cudaStream_t stream, size_t size)
: device(device), stream(stream), size(size), requested_size(0) {}
size_t gc_count() {
TORCH_INTERNAL_ASSERT(pool);
return static_cast<int>(pool->get_free_blocks_call_count - gc_count_base);
}
bool is_split() const {
return (prev != nullptr) || (next != nullptr);
}
void splice(Block* before, Block* after) {
if (before) {
TORCH_INTERNAL_ASSERT(before->next == after);
before->next = this;
}
prev = before;
if (after) {
TORCH_INTERNAL_ASSERT(after->prev == before);
after->prev = this;
}
next = after;
}
};
std::pair<decltype(BlockPool::blocks)::iterator, bool> BlockPool::
insert_into_blocks(Block* block) {
block->gc_count_base = get_free_blocks_call_count;
auto inserted = blocks.insert(block);
auto inserted_by_addr = blocks_by_addr.insert(block);
TORCH_INTERNAL_ASSERT(inserted.second == inserted_by_addr.second);
return inserted;
}
size_t BlockPool::erase_from_blocks(Block* block) {
auto erased = blocks.erase(block);
auto erased_by_addr = blocks_by_addr.erase(block);
TORCH_INTERNAL_ASSERT(erased == erased_by_addr);
return erased;
}
struct SegmentRange {
char* ptr;
size_t size;
SegmentRange(void* p, size_t s) : ptr(static_cast<char*>(p)), size(s) {}
};
// Expandable segments have no ROCm fabric handle path, so ROCm keeps POSIX fds
// even on devices where get_fabric_access() is true for symmetric memory.
bool use_fabric_handles(c10::DeviceIndex device) {
#ifdef USE_ROCM
(void)device;
return false;
#else
return cuda::get_fabric_access(device);
#endif
}
#if !defined(USE_ROCM) && defined(PYTORCH_C10_DRIVER_API_SUPPORTED) || \
defined(USE_ROCM)
/*
Note [Expandable Segments]
Rationale
For large (>2MB) allocations, the allocator calls cudaMalloc to get allocations
that are the same size as what the user requests. In the future, parts of these
allocations can be reused for other requests if they are free. This works well
when the program makes many requests of exactly the same size or of sizes that
even multiples of that size. Many deep learning models follow this behavior.
However, one common exception is when the batch size changes slightly from one
iteration to the next, e.g. in batched inference. When the program runs
initially with batch size N, it will make allocations appropriate for that size.
If in the future, it runs at size N - 1, the existing allocations will still be
big enough. However, if it runs at size N + 1, then it will have to make new
allocations that are slightly larger. Not all the tensors are the same size.
Some might be (N + 1)*A and others (N + 1)*A*B where A and B are some non-batch
dimensions in the model. Because the allocator reuses existing allocations when
they are big enough, some number of (N + 1)*A allocations will actually fit in
the already existing N*B*A segments, though not perfectly. As the model runs it
will partially fill up all of these segments leaving unusable free slices of
memory at the end of these segments. The allocator at some point will need to
cudaMalloc a new (N + 1)*A*B segment. If there is not enough memory, there is
now no way to recover the slices of memory that are free at the end of existing
segments. With models 50+ layers deep, this pattern might repeat 50+ times
creating many slivers.
Approach
Expandable segments allows the allocator to create a segment initially and then
expand its size later when more memory is needed. Instead of making one segment
per allocation, it tries to make one segment (per stream) that grows as
necessary. Now when the N + 1 case runs, the allocations will tile nicely into
the one large segment until it fills up. Then more memory is requested and
appended to the end of the segment. This process does not create as many slivers
of unusable memory, so it is more likely to succeed at finding this memory.
Implementation
The expandable_segments:True option is used to enable/disable this behavior. We
use cuda's low-level memory APIs, which are similar to mmap, to extend the
memory segments. These APIs separate the allocation of physical memory
(cuMemCreate) from the allocation of virtual address space (cuMemAddressReserve)
and the associate between them cuMemMap/cuMemSetAccess.
When we allocate a new segment, we allocate enough address space to map
basically the entire physical memory of the GPU (there is 256TiB of address
space), but we only map enough physical memory to handle the current amount of
memory needed by the program. As more is requested, we add more physical memory
to the segment. This can work at the granularity of GPU pages which are 2MiB
currently.
If we end up out of memory, we can unmap all the memory in our segment
corresponding to empty physical pages, and return it to CUDA for use at another
address in the segment or in a segment for a different stream.
A current limitation of CUDA's API is that physical memory
(CUmemGenericAllocationHandle) cannot be split up after it is mapped even if the
handle holds multiple GPU pages. The cost to map/unmap memory is proportional to
the number of physical memory chunks that were allocated (mapping 10 separately
allocated 2MiB pages takes 10x time compared to mapping one 20MiB physical
allocation of 10 pages). Changing memory mappings also appears to involve at
least some synchronous actions with the GPU and so should be considered an
expensive operation. To limit overhead, we use 2MiB pages for our small pool and
20MiB pages for our large pool. Initially allocation using expandable_blocks
will be slower than cudaMalloc, though still in the milliseconds range for
mapping the entire memory.
When mapping new memory to expand the segment, we look for the lowest address at
which we can fit a new allocation by adding new pages. Normally this will be at
the end of the block. But if have previously unmapped blocks earlier in the
segment during an OOM, it will first try to fill in those gaps to keep the
segment as a single block. By allocating at the lowest address we encourage
the split up parts of the block to merge into a single block again, reducing
fragmentation potential.
Allocation of blocks in the segment uses the same best-fit heuristics of the
rest of the allocator.
Expandable blocks can be enabled/disabled throughout the run of a program. When
disabled, the allocator will not put new allocations in an expandable block.
Limitations
* Slightly slower initial memory allocation speed.
* IPC of cuda tensors (e.g. for multiprocess dataloaders) is not supported.
However, it is possible to temporarily disable (expandable_segments:False) the
bevhavior for allocator tensors that need to be used cross-process.
* CUDA runtime APIs related to sharing memory across process
(cudaDeviceEnablePeerAccess) do not work for memory allocated with cuMemMap.
Instead these mapping have to be done manually. The allocator now has an
`enablePeerAccess` method to do this.
*/
// Address space to reserve for a segment that may still grow: 1 1/8 of device
// memory, less any downsizing configured for the stream's reserve class.
static size_t growableReserveBytes(
c10::DeviceIndex device,
std::optional<cudaStream_t> stream) {
cudaDeviceProp prop{};
C10_CUDA_CHECK(cudaGetDeviceProperties(&prop, device));
// we allocate enough address space for 1 1/8 the total memory on the GPU.
// This allows for some cases where we have to unmap pages earlier in the
// segment to put them at the end.
const size_t full_reserve = prop.totalGlobalMem + prop.totalGlobalMem / 8;
// Serving streams may be tagged with a reserve class to downsize their VA
// reservation (see PYTORCH_CUDA_ALLOC_CONF expandable_segments_reserve*).
// Untagged streams keep the full reserve, so this path is a byte-for-byte
// no-op unless reserve config is set. A single allocation cannot span
// segments, so the reserve is raised to the floor, and the result is capped
// at the historical full reserve (see clamp_reserve_bytes).
if (!stream.has_value()) {
return full_reserve;
}
const std::string reserve_class =
getExpandableSegmentReserveClassForStream(*stream);
// Single locked snapshot so a concurrent setAllocatorSettings() re-parse
// cannot compose an inconsistent (reserve, class_known, floor) view.
const auto decision =
CUDAAllocatorConfig::expandable_segments_reserve_decision(
reserve_class, prop.totalGlobalMem);
if (decision.reserve_bytes.has_value() && !reserve_class.empty() &&
!decision.class_known) {
static std::mutex warn_mutex;
static ska::flat_hash_set<std::string> warned;
std::lock_guard<std::mutex> lock(warn_mutex);
if (warned.insert(reserve_class).second) {
TORCH_WARN(
"expandable_segments reserve class '",
reserve_class,
"' has no configured reserve in "
"expandable_segments_reserve_by_class; using the default reserve.");
}
}
return CUDAAllocatorConfig::clamp_reserve_bytes(decision, full_reserve);
}
/*
Note [Expandable Segment Reserved Address]
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A segment normally lets the driver pick where to reserve its address range. A
caller rebuilding state recorded by an earlier process needs the *same* address
instead: a CUDA graph's kernel arguments embed device pointers verbatim,
sometimes inside opaque packed structs, so the memory has to come back where it
was rather than be relocated.
`requested_addr` asks for one. The address only has to be *recorded*, not
predictable in advance, so the normal allocation path is unchanged; restoring
fails if the range is no longer free in the new process.
The trap: cuMemAddressReserve returns CUDA_SUCCESS while silently ignoring a
hint it cannot honor, so compare the returned pointer instead of trusting the
status.
*/
struct ExpandableSegment {
ExpandableSegment(
c10::DeviceIndex device,
std::optional<cudaStream_t> stream,
size_t segment_size,
std::vector<c10::DeviceIndex> peers,
// If set, reserve exactly max_handles * segment_size bytes of virtual
// address space. Otherwise, derive the reservation size from device
// memory and allocator settings.
//
// IPC imports pass the number of imported handles because they cannot
// grow. Restoration passes the saved reservation's capacity to reproduce
// its size. These slots need not all have physical memory mapped.
std::optional<size_t> max_handles = std::nullopt,
// When set, reserve at this exact address rather than letting the driver
// choose, and fail rather than fall back. See
// Note [Expandable Segment Reserved Address].
std::optional<size_t> requested_addr = std::nullopt)
: device_(device),
stream_(stream),
// 2MB for small pool, 20MB for large pool
segment_size_(segment_size),
peers_(std::move(peers)) {
mapped_size_ = 0;
max_handles_ = max_handles.has_value()
? *max_handles
: numSegments(growableReserveBytes(device_, stream));
const size_t reserve_bytes = segment_size_ * max_handles_;
// Log expandable-segment VA context immediately BEFORE the (unchanged)
// driver check throws, so an out-of-virtual-memory crash is self-explaining
// in task logs. Built only on the failure branch -> perf-neutral on
// success.
auto logVaExhaustion = [&](const char* err_str) {
LOG(ERROR)
<< "[ExpandableSegmentVA] Failed to reserve " << reserve_bytes
<< " bytes (~" << (reserve_bytes >> 30)
<< " GiB) of virtual address space for an expandable CUDA segment on device "
<< static_cast<int>(device_) << " (driver reported: " << err_str
<< "). This process already holds " << getExpandableSegmentsCount()
<< " live expandable segment(s) reserving "
<< getExpandableSegmentsReservedBytes()
<< " bytes of virtual address space. This is virtual-address-space "
"(VSZ) exhaustion from expandable segments, not physical GPU OOM. "
"Reduce it via PYTORCH_CUDA_ALLOC_CONF expandable_segments_reserve / "
"expandable_segments_reserve_by_class, or by using fewer CUDA streams.";
};
#ifdef USE_ROCM
TORCH_CHECK(
!requested_addr.has_value(),
"reserving an expandable segment at a requested address is not supported on ROCm");
hipError_t reserve_err =
hipMemAddressReserve(&ptr_, reserve_bytes, 0ULL, 0, 0ULL);
if (reserve_err != hipSuccess) [[unlikely]] {
const char* err_str = hipGetErrorString(reserve_err);
logVaExhaustion(err_str ? err_str : "unknown error");
}
C10_CUDA_CHECK(reserve_err);
#else
CUresult reserve_err = DriverAPI::get()->cuMemAddressReserve_(
&ptr_, reserve_bytes, 0ULL, requested_addr.value_or(0), 0ULL);
if (reserve_err != CUDA_SUCCESS) [[unlikely]] {
const char* err_str = nullptr;
DriverAPI::get()->cuGetErrorString_(reserve_err, &err_str);
logVaExhaustion(err_str ? err_str : "unknown error");
}
C10_CUDA_DRIVER_CHECK(reserve_err);
// A hint is advisory: the driver reports success having placed the
// reservation elsewhere, so the caller only got its address if this
// matches. Throwing skips the destructor, so free the stray range here.
if (requested_addr.has_value() && ptr_ != *requested_addr) {
const CUdeviceptr placed = ptr_;
C10_CUDA_DRIVER_CHECK(
DriverAPI::get()->cuMemAddressFree_(placed, reserve_bytes));
TORCH_CHECK(
false,
"could not reserve the recorded expandable segment address ",
*requested_addr,
"; the driver placed the reservation at ",
placed,
" instead");
}
#endif
// Reservation succeeded (a failure would have thrown above): publish
// gauges.
g_expandable_segments_reserved_bytes.fetch_add(
reserve_bytes, std::memory_order_relaxed);
g_expandable_segments_count.fetch_add(1, std::memory_order_relaxed);
}
ExpandableSegment(const ExpandableSegment&) = delete;
ExpandableSegment(ExpandableSegment&&) = delete;
ExpandableSegment operator=(const ExpandableSegment&) = delete;
ExpandableSegment operator=(ExpandableSegment&&) = delete;
// begin must be aligned to segment_size_.
// returns the actual range mapped, which may be
// greater than requested if size is not aligned to segment_size_.
// return size of 0 indicates OOM
SegmentRange map(SegmentRange range) {
auto begin = segmentLeft(range.ptr);
auto end = segmentRight(range.ptr + range.size);
TORCH_INTERNAL_ASSERT(ptr() + begin * segment_size_ == range.ptr);
if (begin == end) {
return rangeFromHandles(begin, end);
}
if (end > handles_.size()) {
handles_.resize(end);
}
for (auto i : c10::irange(begin, end)) {
TORCH_INTERNAL_ASSERT(!handles_.at(i));
CUmemGenericAllocationHandle handle = 0;
CUmemAllocationProp prop = {};
prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
if (ipcEnabled()) {
if (use_fabric_handles(device_)) {
#ifndef USE_ROCM
prop.requestedHandleTypes = CU_MEM_HANDLE_TYPE_FABRIC;
#endif
} else {
#ifdef USE_ROCM
prop.requestedHandleType = hipMemHandleTypePosixFileDescriptor;
#else
prop.requestedHandleTypes = CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR;
#endif
}
}
int flag = 0;
#ifndef USE_ROCM
C10_CUDA_DRIVER_CHECK(DriverAPI::get()->cuDeviceGetAttribute_(
&flag,
CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_WITH_CUDA_VMM_SUPPORTED,
device_));
#endif
if (flag)
prop.allocFlags.gpuDirectRDMACapable = 1;
prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
// NOLINTNEXTLINE(bugprone-signed-char-misuse)
prop.location.id = static_cast<int>(device_);
#ifdef USE_ROCM
auto status = hipMemCreate(&handle, segment_size_, &prop, 0);
#else
auto status =
DriverAPI::get()->cuMemCreate_(&handle, segment_size_, &prop, 0);
#endif
if (status != CUDA_SUCCESS) {
if (status == CUDA_ERROR_OUT_OF_MEMORY) {
{
size_t device_free = 0;
size_t device_total = 0;
(void)cudaMemGetInfo(&device_free, &device_total);
LOG(WARNING)
<< "expandable_segments: memory mapping failed with OOM on device "
<< static_cast<int>(device_) << " while trying to map "
<< segment_size_ << " bytes (free: " << device_free
<< ", total: " << device_total << ").";
}
#ifdef USE_ROCM
// hipMemCreate above returned hipErrorOutOfMemory and treated it
// like a sticky runtime error. Which means we need to clear it.
// Unlike the corresponding CUDA Driver API.
(void)hipGetLastError();
#endif
for (auto j : c10::irange(begin, i)) {
auto& maybe_handle = handles_.at(j);
TORCH_INTERNAL_ASSERT(maybe_handle.has_value());
auto h = *maybe_handle;
maybe_handle = std::nullopt;
#ifdef USE_ROCM
C10_CUDA_CHECK(hipMemRelease(h.handle));
#else
C10_CUDA_DRIVER_CHECK(DriverAPI::get()->cuMemRelease_(h.handle));
#endif
}
trimHandles();
return rangeFromHandles(begin, begin);
}
#ifdef USE_ROCM
C10_CUDA_CHECK(status);
#else
C10_CUDA_DRIVER_CHECK(status);
#endif
}
handles_.at(i) = Handle{.handle = handle};
}
mapAndSetAccess(begin, end);
return rangeFromHandles(begin, end);
}
// unmaps all the completely empty segment_size_ segments between
// [begin, begin + size), returns the offset where the range begin,
// and the actual size unmapped (multiple of segment_size_)
SegmentRange unmap(SegmentRange range) {
auto begin = segmentRight(range.ptr);
auto end = segmentLeft(range.ptr + range.size);
if (begin >= end) {
return SegmentRange{range.ptr, 0};
}
unmapHandles(begin, end);
mapped_size_ -= (end - begin) * segment_size_;
return rangeFromHandles(begin, end);
}
// Setup IPC sharing for range.
// Returns the (larger) range that was actually shared.
// Serializes data to std::ostream that can be passed to the
// other process, and then restored as an exapandable segment
// via ExpandableSegment::fromShared(istream);
SegmentRange share(SegmentRange range, std::ostream& buf) {
TORCH_CHECK(
ipcEnabled(),
"Cannot share an expandable_segments allocation: IPC handles are ",
"disabled. Set TORCH_CUDA_EXPANDABLE_SEGMENTS_IPC=1 or disable ",
"expandable_segments for cross-process tensors.");
auto begin = segmentLeft(range.ptr);
auto end = segmentRight(range.ptr + range.size);
ShareHeader header{};
header.pid = get_self_pid();
header.segment_size = segment_size_;
header.num_handles = end - begin;
header.handle_type = use_fabric_handles(device_)
? Expandable_Segments_Handle_Type::FABRIC_HANDLE
: Expandable_Segments_Handle_Type::POSIX_FD;
buf.write(reinterpret_cast<const char*>(&header), sizeof(ShareHeader));
for (auto i : c10::irange(begin, end)) {
auto& maybe_handle = handles_.at(i);
TORCH_INTERNAL_ASSERT(maybe_handle.has_value());
auto& handle = *maybe_handle;
if (header.handle_type !=
Expandable_Segments_Handle_Type::FABRIC_HANDLE) {
if (!handle.shareable_handle) {
int fd = 0;
#ifdef USE_ROCM
C10_CUDA_CHECK(hipMemExportToShareableHandle(
&fd, handle.handle, hipMemHandleTypePosixFileDescriptor, 0));
#else
C10_CUDA_DRIVER_CHECK(DriverAPI::get()->cuMemExportToShareableHandle_(
&fd, handle.handle, CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR, 0));
#endif
handle.shareable_handle = fd;
LOG(INFO) << "use posix fd to share expandable segments.";
}
const int fd = std::get<int>(*handle.shareable_handle);
buf.write(reinterpret_cast<const char*>(&fd), sizeof(fd));
} else {
#ifdef USE_ROCM
TORCH_INTERNAL_ASSERT(
false, "expandable segment with fabric handle not supported");
#else
if (!handle.shareable_handle) {
CUmemFabricHandle fabric_handle;
C10_CUDA_DRIVER_CHECK(DriverAPI::get()->cuMemExportToShareableHandle_(
&fabric_handle, handle.handle, CU_MEM_HANDLE_TYPE_FABRIC, 0));
handle.shareable_handle = fabric_handle;
LOG(INFO) << "use fabric handle to share expandable segments.";
}
const auto& exported =
std::get<CUmemFabricHandle>(*handle.shareable_handle);
buf.write(reinterpret_cast<const char*>(&exported), sizeof(exported));
#endif
}
}
return rangeFromHandles(begin, end);
}
static std::unique_ptr<ExpandableSegment> fromShared(
c10::DeviceIndex device,
std::vector<c10::DeviceIndex> peers,
std::istream& buf) {
ShareHeader header{};
buf.read(reinterpret_cast<char*>(&header), sizeof(ShareHeader));
TORCH_CHECK(
header.handle_type == Expandable_Segments_Handle_Type::POSIX_FD ||
(header.handle_type ==
Expandable_Segments_Handle_Type::FABRIC_HANDLE &&
use_fabric_handles(device)),
"Unsupported expandable_segments IPC handle type ",
static_cast<int>(header.handle_type),
" on device ",
static_cast<int>(device));
TORCH_CHECK(
header.num_handles > 0,
"IPC share header describes an empty expandable segment");
auto segment = std::make_unique<ExpandableSegment>(
device,
std::nullopt,
header.segment_size,
std::move(peers),
header.num_handles);
// older build setups (e.g. multiwheels) do not have this syscall, added 2020
// but the kernel on the system might still support it.
#ifndef _WIN32
#ifndef SYS_pidfd_open
#define SYS_pidfd_open 434
#endif
#ifndef SYS_pidfd_getfd
#define SYS_pidfd_getfd 438
#endif
#endif // !_WIN32
if (header.handle_type != Expandable_Segments_Handle_Type::FABRIC_HANDLE) {
#ifdef _WIN32
TORCH_CHECK(
false, "IPC expandable segments are not supported on Windows");
#else
auto pidfd = syscall(SYS_pidfd_open, header.pid, 0);
TORCH_CHECK(
pidfd != -1 || errno != ENOSYS,
"The kernel on this machine does not support the pidfd_open syscall needed to use IPC for CUDA tensors when expandable_segments:True is set. "
"Consider using expandable_segments:False via torch.cuda.memory._set_allocator_settings('expandable_segments:False') for this allocation.");
TORCH_CHECK(pidfd != -1, "pidfd_open:", c10::utils::str_error(errno));
auto close_pidfd =
c10::make_scope_exit([&pidfd]() { close(static_cast<int>(pidfd)); });
for (auto i : c10::irange(header.num_handles)) {
(void)i;
int fd = 0;
buf.read(reinterpret_cast<char*>(&fd), sizeof(int));
auto myfd = syscall(SYS_pidfd_getfd, pidfd, fd, 0);
if (myfd == -1) {
auto err = errno;
TORCH_CHECK(
err != ENOSYS,
"The kernel on this machine does not support the pidfd_getfd syscall needed to use IPC for CUDA tensors when expandable_segments:True is set. "
"Consider using expandable_segments:False via torch.cuda.memory._set_allocator_settings('expandable_segments:False') for this allocation.");
TORCH_CHECK(false, "pidfd_getfd: ", c10::utils::str_error(err));
}
// close myfd on every path, including the import throwing below
auto close_myfd =
c10::make_scope_exit([&myfd]() { close(static_cast<int>(myfd)); });
CUmemGenericAllocationHandle handle = 0;
#ifdef USE_ROCM
#if ROCM_VERSION >= 70100
void* myfd_handle =
reinterpret_cast<void*>(static_cast<uintptr_t>(myfd));
#else
void* myfd_handle = (void*)(uintptr_t)&myfd;
#endif
C10_CUDA_CHECK(hipMemImportFromShareableHandle(
&handle, myfd_handle, hipMemHandleTypePosixFileDescriptor));
#else
C10_CUDA_DRIVER_CHECK_MSG(
DriverAPI::get()->cuMemImportFromShareableHandle_(
&handle,
// NOLINTNEXTLINE(performance-no-int-to-ptr)
(void*)(uintptr_t)myfd,
CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR),
" fabric_info: {",
get_nvml_fabric_info(device),
"}");
#endif
LOG(INFO) << "use posix fd to import expandable segments.";
segment->handles_.emplace_back(handle);
}
#endif // !_WIN32
} else {
#ifdef USE_ROCM
TORCH_INTERNAL_ASSERT(
false, "expandable segment with fabric handle not supported");
#elif defined(_WIN32)
TORCH_CHECK(
false, "IPC expandable segments are not supported on Windows");
#else
for (auto i : c10::irange(header.num_handles)) {
(void)i;
CUmemFabricHandle fabric_handle;
buf.read(
reinterpret_cast<char*>(&fabric_handle), sizeof(CUmemFabricHandle));
CUmemGenericAllocationHandle handle = 0;
C10_CUDA_DRIVER_CHECK_MSG(
DriverAPI::get()->cuMemImportFromShareableHandle_(
&handle,
// NOLINTNEXTLINE(performance-no-int-to-ptr)
(void*)&fabric_handle,
CU_MEM_HANDLE_TYPE_FABRIC),
" fabric_info: {",
get_nvml_fabric_info(device),
"}");
LOG(INFO) << "use fabric handle to import expandable segments.";
segment->handles_.emplace_back(handle);
}
#endif
}
segment->mapAndSetAccess(0, header.num_handles);
return segment;
}
char* ptr() const {
// NOLINTNEXTLINE(performance-no-int-to-ptr)
return reinterpret_cast<char*>(ptr_);
}
size_t size() const {
return max_handles_ * segment_size_;
}
cudaStream_t getStream() {
return stream_.value_or(nullptr);
}
size_t getMappedSize() const {
return mapped_size_;
}
size_t getSegmentSize() const {
return segment_size_;
}
// Mirrors the handle type map() requests and share() reports.
Expandable_Segments_Handle_Type getHandleType() const {
if (!ipcEnabled()) {
return Expandable_Segments_Handle_Type::UNSPECIFIED;
}
return use_fabric_handles(device_)
? Expandable_Segments_Handle_Type::FABRIC_HANDLE
: Expandable_Segments_Handle_Type::POSIX_FD;
}
static bool ipcEnabled() {
#ifdef _WIN32
return false;
#else
// In fbcode, IPC handle types for expandable segments are disabled by
// default because some jobs were failing (see
// https://github.com/pytorch/pytorch/pull/132890), but can be explicitly
// enabled via environment variable when IPC functionality is required
// (e.g., for multi-process communication with CTran). In non-fbcode
// builds, IPC handle types are enabled by default.
#ifdef FBCODE_CAFFE2
constexpr bool default_enable_ipc = false;
#else
constexpr bool default_enable_ipc = true;
#endif
static const bool enabled =
c10::utils::check_env("TORCH_CUDA_EXPANDABLE_SEGMENTS_IPC")
.value_or(default_enable_ipc);
return enabled;
#endif // _WIN32
}
void addPeer(c10::DeviceIndex device) {
peers_.push_back(device);
forEachAllocatedRange(
[&](size_t begin, size_t end) { setAccess(device, begin, end); });
}
~ExpandableSegment() {
forEachAllocatedRange(
[&](size_t begin, size_t end) { unmapHandles(begin, end); });
#ifdef USE_ROCM
C10_CUDA_CHECK(hipMemAddressFree(ptr_, segment_size_ * max_handles_));
#else
C10_CUDA_DRIVER_CHECK(DriverAPI::get()->cuMemAddressFree_(
ptr_, segment_size_ * max_handles_));
#endif
// Decrement only after a successful free (the checks above throw on
// failure), so the gauges never under-report VA that is still reserved by
// the driver.
g_expandable_segments_reserved_bytes.fetch_sub(
segment_size_ * max_handles_, std::memory_order_relaxed);
g_expandable_segments_count.fetch_sub(1, std::memory_order_relaxed);
}
private:
void setAccess(c10::DeviceIndex device, size_t begin, size_t end) {
#if defined(USE_ROCM) && (ROCM_VERSION >= 70200)
constexpr int num_desc = 2;
std::array<CUmemAccessDesc, num_desc> desc{};
desc[1].location.type = CU_MEM_LOCATION_TYPE_HOST;
desc[1].location.id = 0; // ignored
desc[1].flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
#else
constexpr int num_desc = 1;
std::array<CUmemAccessDesc, num_desc> desc{};
#endif
desc[0].location.type = CU_MEM_LOCATION_TYPE_DEVICE;
// NOLINTNEXTLINE(bugprone-signed-char-misuse)
desc[0].location.id = static_cast<int>(device);
desc[0].flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
#ifdef USE_ROCM
C10_CUDA_CHECK(hipMemSetAccess(
ptr() + begin * segment_size_,
(end - begin) * segment_size_,
&desc[0],
num_desc));
#else
C10_CUDA_DRIVER_CHECK(DriverAPI::get()->cuMemSetAccess_(
ptr_ + begin * segment_size_,
(end - begin) * segment_size_,
&desc[0],
num_desc));
#endif
}
void mapAndSetAccess(size_t begin, size_t end) {
for (auto i : c10::irange(begin, end)) {
auto& maybe_handle = handles_.at(i);
TORCH_INTERNAL_ASSERT(maybe_handle.has_value());
#ifdef USE_ROCM
C10_CUDA_CHECK(hipMemMap(
ptr() + i * segment_size_,
segment_size_,
0,
maybe_handle->handle,
0ULL));
#else
C10_CUDA_DRIVER_CHECK(DriverAPI::get()->cuMemMap_(
ptr_ + i * segment_size_,
segment_size_,
0,
maybe_handle->handle,