Repository navigation
Expand file tree
/
Copy pathos.c
More file actions
1097 lines (965 loc) · 47.4 KB
/
Copy pathos.c
File metadata and controls
1097 lines (965 loc) · 47.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/* ----------------------------------------------------------------------------
Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
-----------------------------------------------------------------------------*/
#include "mimalloc.h"
#include "mimalloc/prim/prim.h"
#include "mimalloc/track.h"
#include "mimalloc/prim/prim-tls.h" // _mi_theap_default for random
#include "mimalloc/util/libc.h"
#include "mimalloc/util/options.h"
#include "mimalloc/util/stats.h"
#include "mimalloc/util/memid.h"
#include "mimalloc/prim/os.h"
#include "mimalloc/util/random.h"
#include "mimalloc/prim/prim-pac.h"
#include "mimalloc/arena/arena.h" // mi_memid_arena
#if !defined(_WIN32) && MI_HAS_UNISTDH
#include <unistd.h> // issetugid, getuid, geteuid, getgid, getegid
#endif
/* -----------------------------------------------------------
Initialization.
----------------------------------------------------------- */
#ifndef MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB
#if MI_SIZE_SIZE < 8
#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 4*MI_MiB // 4 GiB
#else
#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 32*MI_MiB // 32 GiB
#endif
#endif
// Default OS memory configuration (used until `_mi_os_init`)
static const mi_os_mem_config_t mi_os_mem_config_default = {
4096, // page size
0, // large page size (usually 2MiB)
4096, // allocation granularity
MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB,
MI_MAX_VABITS, // in `arch.h`
0, // memory tag check mode
0, // memory tag line size
true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems)
false, // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span)
true, // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory)
false, // has aligned allocation? (if true the OS can directly allocate at a requested alignment)
false, // has transparent huge pages? (if true we purge in (aligned) large page size chunks only to not fragment such pages)
false, // is elevated? (set in `_mi_os_init`)
false, // has pointer authentication? (only detected with `MI_PAC=ON`)
false // has memory tagging? (if true, all non-large OS memory supports memory tagging)
};
// The OS memory configuration in the sealed process data (zero until `_mi_os_init`)
// OS statistics are recorded in the given sub-process, or in the bootstrap statistics if `subproc` is `NULL`.
// A `NULL` sub-process is only used to allocate the process meta data (which contains the main sub-process),
// and the bootstrap statistics are merged into the main sub-process at its initialization (see `_mi_os_stats_bootstrap_merge`).
static mi_stats_t mi_os_stats_bootstrap;
static inline mi_stats_t* mi_os_stats(mi_subproc_t* subproc) {
return (subproc == NULL ? &mi_os_stats_bootstrap : &subproc->stats);
}
#define mi_os_stat_counter_increase(subproc,stat,amount) __mi_stat_counter_increase_mt( &mi_os_stats(subproc)->stat, amount)
#define mi_os_stat_increase(subproc,stat,amount) __mi_stat_increase_mt( &mi_os_stats(subproc)->stat, amount)
#define mi_os_stat_decrease(subproc,stat,amount) __mi_stat_decrease_mt( &mi_os_stats(subproc)->stat, amount)
#define mi_os_stat_adjust_decrease(subproc,stat,amount) __mi_stat_adjust_decrease_mt( &mi_os_stats(subproc)->stat, amount)
void _mi_os_stats_bootstrap_merge(mi_subproc_t* subproc) {
_mi_stats_merge_into(&subproc->stats, &mi_os_stats_bootstrap);
}
static inline const mi_os_mem_config_t* mi_os_mem_config(void) {
const mi_os_mem_config_t* const config = &mi_process_get_sealed_once()->os_mem_config;
return (config->page_size != 0 ? config : &mi_os_mem_config_default);
}
bool _mi_os_has_overcommit(void) {
return mi_os_mem_config()->has_overcommit;
}
bool _mi_os_has_virtual_reserve(void) {
return mi_os_mem_config()->has_virtual_reserve;
}
bool _mi_os_has_thp(void) {
return mi_os_mem_config()->has_transparent_huge_pages;
}
bool _mi_os_has_memtag(void) {
return mi_os_mem_config()->has_memtag;
}
size_t _mi_os_memtag_line_size(void) {
return mi_os_mem_config()->memtag_line_size;
}
// Are we running with elevated privileges (e.g. a setuid/setgid executable)?
static bool mi_os_detect_elevated(void) {
#if defined(_WIN32)
return false; // no direct setuid/setgid equivalent; a UAC elevation does not inherit the parent's environment
#elif MI_HAS_UNISTDH && (defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__DragonFly__))
return (issetugid() != 0);
#elif MI_HAS_UNISTDH
return (getuid() != geteuid() || getgid() != getegid());
#else
return false; // no portable way to check without `unistd.h`; assume not elevated
#endif
}
// Are we running with elevated privileges? In that case the environment is untrusted (and we ignore
// all `MIMALLOC_` environment options and don't show addresses in messages).
// Before `_mi_os_init` has published the OS configuration (e.g. when reading options during the
// initialization in `_mi_prim_mem_init`) we use a direct (system call) check.
bool _mi_os_is_elevated(void) {
const mi_os_mem_config_t* const config = &mi_process_get_sealed_once()->os_mem_config;
return (config->page_size != 0 ? config->is_elevated : mi_os_detect_elevated());
}
// OS (small) page size
size_t _mi_os_page_size(void) {
return mi_os_mem_config()->page_size;
}
// OS allocation granularity (usually the page size, but 64KiB on Windows)
size_t _mi_os_alloc_granularity(void) {
return mi_os_mem_config()->alloc_granularity;
}
// if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB)
size_t _mi_os_large_page_size(void) {
const size_t lpsize = mi_os_mem_config()->large_page_size;
return (lpsize != 0 ? lpsize : _mi_os_page_size());
}
// minimal purge size. Can be larger than the page size if transparent huge pages are enabled.
size_t _mi_os_minimal_purge_size(void) {
size_t minsize = mi_option_get_size(mi_option_minimal_purge_size);
if (minsize != 0) {
// set by user
return mi_align_up(minsize, _mi_os_page_size());
}
else if (mi_os_mem_config()->has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) {
// don't break up THP pages;
// we don't do this by default as that can lead to increased memory usage (see issue #1282).
// on the other hand, breaking up THP pages can lead to performance degradation and perhaps
// allow_thp should be 2 by default (and the user can set the OS THP setting to [never]).
return _mi_os_large_page_size();
}
else {
// OS page size by default
return _mi_os_page_size();
}
}
size_t _mi_os_guard_page_size(void) {
const size_t gsize = _mi_os_page_size();
mi_assert(gsize <= (MI_ARENA_SLICE_SIZE/4)); // issue #1166
return gsize;
}
size_t _mi_os_virtual_address_bits(void) {
const size_t vbits = mi_os_mem_config()->virtual_address_bits;
mi_assert(vbits <= MI_MAX_VABITS);
return vbits;
}
bool _mi_os_canuse_large_page(size_t size, size_t alignment) {
// if we have access, check the size and alignment requirements
const size_t lpsize = mi_os_mem_config()->large_page_size;
if (lpsize == 0) return false;
return ((size % lpsize) == 0 && (alignment % lpsize) == 0);
}
// round to a good OS allocation size (bounded by max 12.5% waste)
size_t _mi_os_good_alloc_size(size_t size) {
size_t align_size;
if (size < 512*MI_KiB) align_size = _mi_os_page_size();
else if (size < 2*MI_MiB) align_size = 64*MI_KiB;
else if (size < 8*MI_MiB) align_size = 256*MI_KiB;
else if (size < 32*MI_MiB) align_size = 1*MI_MiB;
else align_size = 4*MI_MiB;
if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; // possible overflow?
return mi_align_up(size, align_size);
}
// Create a fresh random process key (for encoded pointers and heap cookies)
static mi_addr_t mi_random_key_new(void) {
mi_random_ctx_t ctx;
_mi_random_init(&ctx);
const mi_addr_t key = (mi_addr_t)_mi_random_next(&ctx);
mi_memzero(&ctx, sizeof(ctx));
return key;
}
void _mi_os_init(void) {
mi_os_mem_config_t config = mi_os_mem_config_default;
_mi_prim_mem_init(&config);
mi_assert(config.page_size != 0);
config.is_elevated = mi_os_detect_elevated();
if (config.has_memtag) {
const size_t line_size = mi_memtag_cpu_line_size();
if (line_size < MI_MEMTAG_LINE_MIN || line_size > MI_MEMTAG_LINE_MAX || !mi_is_power_of_two(line_size)) {
_mi_warning_message("memory tagging is disabled as the CPU tag cache line size (%zu bytes) is not supported\n", line_size);
config.has_memtag = false;
}
else {
config.memtag_line_size = line_size;
}
}
#if MI_MTE >= 2
if (!config.has_memtag) {
_mi_fatal_error("built with MI_MTE=ON but memory tagging is not enabled at runtime\n"
" (on macOS the program needs the MTE entitlement, and the `mte` option must not be disabled)\n");
}
#endif
#if MI_PAC >= 2
if (!config.has_pac) {
_mi_fatal_error("built with MI_PAC=ON but the CPU (or OS) does not support pointer authentication (PAC)\n");
}
#endif
mi_process_sealed_once_t* const once = _mi_process_unseal_once();
once->os_mem_config = config; // publish at once
if (once->process_key == 0) { // `_mi_os_init` may be called twice; the key never changes
#if MI_PAC == 1
once->process_key = mi_random_key_new() | MI_PAC_KEY_ALWAYS;
#else
once->process_key = mi_random_key_new() | 1; // never 0
#endif
}
_mi_process_seal_once();
if (_mi_os_page_size() > MI_ARENA_SLICE_SIZE) {
_mi_error_message(ENOTSUP, "the OS page size (%zu KiB) is larger than the arena slice size (%zu KiB); increase MI_ARENA_SLICE_SHIFT\n", _mi_os_page_size()/MI_KiB, (size_t)(MI_ARENA_SLICE_SIZE/MI_KiB));
}
if (config.has_memtag) { _mi_verbose_message("memory tagging is enabled (for blocks up to %zu KiB)\n", (size_t)(MI_LARGE_MAX_OBJ_SIZE / MI_KiB)); }
#if MI_PAC
_mi_verbose_message("signing is enabled (%s)\n", (config.has_pac && MI_PAC>=2 ? "hardware supported" : "weak, using a process key"));
#endif
}
/* -----------------------------------------------------------
Util
-------------------------------------------------------------- */
bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size);
bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero);
// On systems with enough virtual address bits, we can do efficient aligned allocation by using
// the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address
// space (64TiB) we use this technique. (but see issue #939)
#if (MI_SIZE_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID)
// Return a `try_alignment` aligned address that is probably available.
// If this returns NULL, the OS will determine the address but on some OS's that may not be
// properly aligned which can be more costly as it needs to be adjusted afterwards.
// In secure mode, for a size > 16GiB this always returns NULL in order to guarantee good ASLR randomization;
// (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses
// in the middle of the 2TiB - 6TiB address range (see issue #372))
#define MI_HINT_BASE (MI_ADDR(2) << 40) // 2TiB start
#define MI_HINT_AREA (MI_ADDR(4) << 40) // upto (2+4) 6TiB (since before win8 there is "only" 8TiB available to processes)
#define MI_HINT_MAX (MI_ADDR(30) << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages)
void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze)
{
// todo: perhaps only do alignment hints if THP is enabled?
if (try_alignment <= mi_os_mem_config()->alloc_granularity || try_alignment > 16*MI_GiB) return NULL;
if (mi_os_mem_config()->virtual_address_bits < 46) return NULL; // < 64TiB virtual address space
size_t req_size = sze + _mi_os_page_size(); // always reserve a bit more to create virtual gaps between hinted blocks.
req_size += (try_alignment - 1); // ensure we can align in the requested size
req_size = mi_align_up(req_size, _mi_os_large_page_size());
#if MI_SECURE
if (req_size > 32*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<34) = 1/256
#endif
// initialize (or wrap) the base before advancing it so an early return leaves it unchanged
mi_addr_t base = mi_atomic_load_relaxed(&_mi_process.os.aligned_base);
if (base == 0 || base > MI_HINT_MAX) {
mi_addr_t init = MI_HINT_BASE;
#if (MI_SECURE || !MI_DEBUG) // security: randomize start of aligned allocations unless in debug mode
mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)
if (!mi_theap_is_initialized(theap)) return NULL; // no hint as we lack randomness at this point
const size_t r = _mi_theap_random_next(theap);
init = init + ((MI_ADDR(MI_MiB) * ((r>>17) & 0x3FFFFF)) % MI_HINT_AREA); // (randomly 22 bits)* 1MiB == 0 to 4TiB
#endif
mi_atomic_cas_strong_relaxed(&_mi_process.os.aligned_base, &base, init); // if this fails, another thread initialized it
}
const mi_addr_t hint = mi_atomic_add_relaxed(&_mi_process.os.aligned_base, req_size);
if (hint == 0 || hint > MI_HINT_MAX) return NULL; // (rare) wrapped concurrently: the next call reinitializes
const mi_addr_t hint_align = mi_addr_align_up(hint,try_alignment);
mi_assert_internal(hint_align + sze < hint + req_size);
return mi_ptr_from_addr(hint_align);
}
#else
void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
MI_UNUSED(try_alignment); MI_UNUSED(size);
return NULL;
}
#endif
/* -----------------------------------------------------------
Guard page allocation
----------------------------------------------------------- */
// Return the size of a guard page, (or 0 if security is disabled or memory cannot be protected)
size_t _mi_os_secure_guard_page_size(void) {
#if MI_SECURE && MI_OS_HAS_GUARD_PAGES // no memory protection on WebAssembly
return _mi_os_guard_page_size();
#else
return 0;
#endif
}
// Set or reset a guard page. The memid `initially_committed` is the accounted commit state
// of the guard page which must not change between a set and reset (see `_mi_prim_guard`).
static bool mi_os_secure_guard_page(void* addr, mi_memid_t memid, bool set) {
const size_t size = _mi_os_secure_guard_page_size();
if (addr == NULL || size == 0 || memid.is_pinned) return true;
mi_assert_internal(mi_is_aligned(addr, _mi_os_page_size()));
bool ok;
mi_arena_t* const arena = mi_memid_arena(memid);
mi_commit_fun_t* const commit_fun = (arena != NULL ? mi_fun_auth_at(mi_commit_fun_t, &arena->commit_fun) : NULL);
if (commit_fun != NULL) {
// decommit on a set, and only recommit on a reset if accounted as committed
ok = (set || memid.initially_committed ? (*commit_fun)(!set /* commit */, addr, size, NULL, arena->commit_fun_arg) : true);
}
else {
ok = (set ? _mi_os_guard(addr, size, memid.initially_committed) : _mi_os_unguard(addr, size, memid.initially_committed));
}
if (!ok) {
_mi_error_message(EINVAL, "secure level %d, but failed to %s guard page (at %p of size %zu)\n", MI_SECURE, (set ? "set" : "reset"), addr, size);
}
return ok;
}
// Make an area inaccessible and output a warning if this fails.
bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {
MI_UNUSED(subproc);
return mi_os_secure_guard_page(addr, memid, true);
}
// Make an area inaccessible and output a warning if this fails.
bool _mi_os_secure_guard_page_set_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {
return _mi_os_secure_guard_page_set_at(subproc, (uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);
}
// Set guard pages right before `addr` and right after `addr + size` (rounded up to the OS page size).
bool _mi_os_secure_guard_pages_set_around(mi_subproc_t* subproc, void* addr, size_t size, mi_memid_t memid) {
const bool ok_before = _mi_os_secure_guard_page_set_before(subproc, addr, memid);
const bool ok_after = _mi_os_secure_guard_page_set_at(subproc, mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()), memid);
return (ok_before && ok_after);
}
// Make an area accessible again (if it is accounted as committed)
bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {
MI_UNUSED(subproc);
return mi_os_secure_guard_page(addr, memid, false);
}
// Set guard pages right before `area + offset` and right after `area + offset + size` (rounded to OS pages)
// within a padded static `area`. Outputs a warning if the guard pages do not fit (or cannot be set).
bool _mi_os_secure_guard_pages_set_within(void* area, size_t area_size, size_t offset, size_t size) {
const size_t gsize = _mi_os_secure_guard_page_size();
if (gsize == 0) return true;
const mi_addr_t start = mi_addr(area);
const mi_addr_t before = mi_addr_align_down(start + offset, _mi_os_page_size()) - gsize;
const mi_addr_t after = mi_addr_align_up(start + offset + size, _mi_os_page_size());
if (before < start || after + gsize > start + area_size) {
_mi_warning_message("secure level %d, but unable to set guard pages around static data at %p as the OS page size (%zu KiB) is too large\n", MI_SECURE, area, gsize / MI_KiB);
return false;
}
mi_memid_t memid = _mi_memid_create(MI_MEM_STATIC); // not pinned so it can be protected
memid.initially_committed = true;
const bool ok_before = mi_os_secure_guard_page(mi_ptr_with_addr(area, before), memid, true);
const bool ok_after = mi_os_secure_guard_page(mi_ptr_with_addr(area, after), memid, true);
return (ok_before && ok_after);
}
/* -----------------------------------------------------------
Free memory
-------------------------------------------------------------- */
static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size);
static void mi_os_prim_free(mi_subproc_t* subproc, void* addr, size_t size, size_t commit_size, bool adjust) {
mi_assert_internal((size % _mi_os_page_size()) == 0);
if (addr == NULL) return; // || _mi_os_is_huge_reserved(addr)
int err = _mi_prim_free(addr, size); // allow size==0 (issue #1041)
if (err != 0) {
_mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
}
if (adjust) {
if (commit_size>0) { mi_os_stat_adjust_decrease(subproc, committed, commit_size); }
mi_os_stat_adjust_decrease(subproc, reserved, size);
}
else {
if (commit_size>0) { mi_os_stat_decrease(subproc, committed, commit_size); }
mi_os_stat_decrease(subproc, reserved, size);
}
}
void _mi_os_free_ex(mi_subproc_t* subproc, void* addr, size_t size, bool still_committed, mi_memid_t memid) {
MI_UNUSED(addr); MI_UNUSED(size);
if (mi_memkind_is_os(memid.memkind)) {
void* const base = memid.mem.os.base;
const size_t full_size = mi_memid_os_size(memid);
mi_assert_internal((uint8_t*)base <= (uint8_t*)addr && (uint8_t*)addr + size <= (uint8_t*)base + full_size);
if (memid.memkind == MI_MEM_OS_HUGE) {
mi_assert(memid.is_pinned);
mi_os_free_huge_os_pages(subproc, base, full_size);
}
else {
mi_os_prim_free(subproc, base, full_size, (still_committed ? mi_memid_os_committed(memid) : 0), false);
}
}
else {
// nothing to do
mi_assert(memid.memkind < MI_MEM_OS);
}
}
void _mi_os_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid) {
_mi_os_free_ex(subproc, p, size, true, memid);
}
/* -----------------------------------------------------------
Primitive allocation from the OS.
-------------------------------------------------------------- */
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
// Also `hint_addr` is a hint and may be ignored.
static void* mi_os_prim_alloc_at(mi_subproc_t* subproc, void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
mi_assert_internal(is_zero != NULL);
mi_assert_internal(is_large != NULL);
if (size == 0) return NULL;
if (!mi_memid_os_size_fits(size)) {
_mi_warning_message("unable to allocate OS memory as the size is too large (size: 0x%zx bytes)\n", size);
return NULL;
}
if (!commit) { allow_large = false; }
if (try_alignment == 0) { try_alignment = 1; } // avoid 0 to ensure there will be no divide by zero when aligning
// try to align along large OS page size for larger allocations
const size_t large_page_size = mi_os_mem_config()->large_page_size;
if (large_page_size > 0 && hint_addr == NULL && size >= 8*large_page_size && mi_is_power_of_two(try_alignment) && try_alignment < large_page_size) {
try_alignment = large_page_size;
}
*is_zero = false;
void* p = NULL;
int err = _mi_prim_alloc(hint_addr, size, try_alignment, commit, allow_large, is_large, is_zero, &p);
if (err != 0) {
_mi_warning_message("unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, hint_addr, size, try_alignment, commit, allow_large);
}
mi_os_stat_counter_increase(subproc, mmap_calls, 1);
if (p != NULL) {
mi_os_stat_increase(subproc, reserved, size);
if (commit) {
mi_os_stat_increase(subproc, committed, size);
// seems needed for asan (or `mimalloc-test-api` fails)
#ifdef MI_TRACK_ASAN
if (*is_zero) { mi_track_mem_defined(p,size); }
else { mi_track_mem_undefined(p,size); }
#endif
}
}
return p;
}
static void* mi_os_prim_alloc(mi_subproc_t* subproc, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
return mi_os_prim_alloc_at(subproc, NULL /* hint addr */, size, try_alignment, commit, allow_large, is_large, is_zero);
}
// Primitive aligned allocation from the OS.
// This function guarantees the allocated memory is aligned.
static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) {
mi_assert_internal(memid!=NULL);
mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
*memid = _mi_memid_none();
if (!commit) allow_large = false;
if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
size = mi_align_up(size, _mi_os_page_size());
#if MI_SIZE_SIZE >= 8
const bool try_direct_alloc = true;
#else
// try a direct allocation if the OS supports aligned allocation, the alignment is below the default, or less than or equal to 1/4 fraction of the size.
const bool try_direct_alloc = (mi_os_mem_config()->has_aligned_alloc || alignment <= mi_os_mem_config()->alloc_granularity || alignment <= size/4);
#endif
bool os_is_large = false;
bool os_is_zero = false;
void* os_base = NULL;
size_t os_size = size;
void* p = NULL;
if (try_direct_alloc) {
p = mi_os_prim_alloc(subproc, size, alignment, commit, allow_large, &os_is_large, &os_is_zero);
}
// aligned already?
if (p != NULL && mi_is_aligned(p,alignment)) {
os_base = p;
}
else {
// if not aligned, free it, overallocate, and unmap around it
// (define MI_NO_ALIGN_FALLBACK_WARNING to silence this benign warning, e.g. when
// stderr output is checked by tests or the address space layout is unusual)
#if !MI_TRACK_ASAN && !defined(MI_NO_ALIGN_FALLBACK_WARNING)
if (try_direct_alloc) {
_mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit);
}
#endif
if (p != NULL) { mi_os_prim_free(subproc, p, size, (commit ? size : 0), true /* adjust so we "forget" the previous reservation */); }
if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
const size_t over_size = size + alignment;
if (!mi_os_mem_config()->has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block
// over-allocate uncommitted (virtual) memory
p = mi_os_prim_alloc(subproc, over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, &os_is_large, &os_is_zero);
if (p == NULL) return NULL;
// set p to the aligned part in the full region
// note: Windows VirtualFree needs the actual base pointer
// this is handled though by having the `base` field in the memid
os_base = p; // remember the base
os_size = over_size;
p = mi_align_up_ptr(p, alignment);
// explicitly commit only the aligned part
if (commit) {
if (!_mi_os_commit(subproc, p, size, NULL)) {
mi_os_prim_free(subproc, os_base, over_size, 0, true);
return NULL;
}
}
}
else { // mmap can free inside an allocation
// overallocate...
// note: we keep `allow_large` but this only works if we can partially free any large OS pages later on. This seems to be the case though.
p = mi_os_prim_alloc(subproc, over_size, 1, commit, allow_large /* or false? */, &os_is_large, &os_is_zero);
if (p == NULL) return NULL;
// and selectively unmap parts around the over-allocated area.
void* const aligned_p = mi_align_up_ptr(p, alignment);
const size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
const size_t mid_size = mi_align_up(size, _mi_os_page_size());
const size_t post_size = over_size - pre_size - mid_size;
mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);
if (pre_size > 0) { mi_os_prim_free(subproc, p, pre_size, (commit ? pre_size : 0), true /* adjust */); }
if (post_size > 0) { mi_os_prim_free(subproc, (uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), true /* adjust */); }
// we can return the aligned pointer on `mmap` systems
p = aligned_p;
os_base = aligned_p; // since we freed the pre part, `*base == p`.
os_size = mid_size;
}
}
mi_assert_internal(p != NULL && os_base != NULL && mi_is_aligned(p,alignment));
mi_assert_internal(os_base <= p && size <= os_size);
*memid = _mi_memid_create_os(os_base, os_size, (commit ? size : 0), os_is_zero, os_is_large);
return p;
}
/* -----------------------------------------------------------
OS API: alloc and alloc_aligned
----------------------------------------------------------- */
void* _mi_os_alloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid) {
*memid = _mi_memid_none();
if (size == 0) return NULL;
size = _mi_os_good_alloc_size(size);
bool os_is_large = false;
bool os_is_zero = false;
void* p = mi_os_prim_alloc(subproc, size, 0, true, false, &os_is_large, &os_is_zero);
if (p == NULL) return NULL;
*memid = _mi_memid_create_os(p, size, size, os_is_zero, os_is_large);
mi_assert_internal(mi_memid_os_size(*memid) >= size);
mi_assert_internal(memid->initially_committed);
return p;
}
void* _mi_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid)
{
MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings
*memid = _mi_memid_none();
if (size == 0) return NULL;
size = _mi_os_good_alloc_size(size);
alignment = mi_align_up(alignment, _mi_os_page_size());
void* p = mi_os_prim_alloc_aligned(subproc, size, alignment, commit, allow_large, memid );
if (p == NULL) return NULL;
mi_assert_internal(mi_memid_os_size(*memid) >= size);
mi_assert_internal(mi_is_aligned(p,alignment));
if (commit) { mi_assert_internal(memid->initially_committed); }
return p;
}
mi_decl_nodiscard static void* mi_os_ensure_zero(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t* memid) {
if (p==NULL || size==0) return p;
// ensure committed
if (!memid->initially_committed) {
const size_t commit_size = mi_memid_os_size(*memid);
bool is_zero = false;
if (!_mi_os_commit(subproc, memid->mem.os.base, commit_size, &is_zero)) {
_mi_os_free(subproc, p, size, *memid);
return NULL;
}
mi_memid_os_set_committed(memid, commit_size);
memid->initially_committed = true;
}
// ensure zero'd
if (memid->initially_zero) return p;
mi_memzero_aligned(p,size);
memid->initially_zero = true;
return p;
}
void* _mi_os_zalloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid) {
void* p = _mi_os_alloc(subproc, size,memid);
return mi_os_ensure_zero(subproc, p, size, memid);
}
// Allocate OS memory with (in secure mode) a guard page right before and after it.
// The memory can be freed using `_mi_os_free_with_guards` with the returned `memid`.
// The `subproc` can be `NULL` for statistics (see `mi_os_stats`).
void* _mi_os_alloc_with_guards(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, mi_memid_t* memid) {
const size_t guard_size = _mi_os_secure_guard_page_size();
size = mi_align_up(size, _mi_os_page_size());
uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(subproc, size + 2*guard_size, 1, commit, allow_large, memid);
if (base == NULL) return NULL;
uint8_t* const p = base + guard_size;
if (memid->initially_committed) { // (if not committed, the guard pages are never committed)
_mi_os_secure_guard_pages_set_around(subproc, p, size, *memid);
}
return p;
}
// Free memory allocated by `_mi_os_alloc_with_guards` (including the guard pages).
void _mi_os_free_with_guards(mi_subproc_t* subproc, mi_memid_t memid) {
_mi_os_free(subproc, memid.mem.os.base, mi_memid_os_size(memid), memid);
}
/* -----------------------------------------------------------
OS aligned allocation with an offset. This is used
for large alignments > MI_BLOCK_ALIGNMENT_MAX. We use a large mimalloc
page where the object can be aligned at an offset from the start of the segment.
As we may need to overallocate, we need to free such pointers using `mi_free_aligned`
to use the actual start of the memory region.
----------------------------------------------------------- */
void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) {
mi_assert(offset <= size);
mi_assert((alignment % _mi_os_page_size()) == 0);
*memid = _mi_memid_none();
if (offset > size) return NULL;
if (offset == 0) {
// regular aligned allocation
return _mi_os_alloc_aligned(subproc, size, alignment, commit, allow_large, memid);
}
else {
// overallocate to align at an offset
const size_t extra = mi_align_up(offset, alignment) - offset;
if (size >= SIZE_MAX - extra) return NULL; // too large
const size_t oversize = size + extra;
void* const start = _mi_os_alloc_aligned(subproc, oversize, alignment, commit, allow_large, memid);
if (start == NULL) return NULL;
void* const p = (uint8_t*)start + extra;
mi_assert(mi_is_aligned((uint8_t*)p + offset, alignment));
// decommit the overallocation at the start
if (commit && extra >= _mi_os_page_size()) {
_mi_os_decommit(subproc, start, extra);
mi_memid_os_set_committed(memid, mi_memid_os_committed(*memid) - extra);
}
return p;
}
}
/* -----------------------------------------------------------
OS memory API: reset, commit, decommit, protect, unprotect.
----------------------------------------------------------- */
// OS page align within a given area, either conservative (pages inside the area only),
// or not (straddling pages outside the area is possible)
static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) {
mi_assert(addr != NULL && size > 0);
if (newsize != NULL) *newsize = 0;
if (size == 0 || addr == NULL) return NULL;
// page align conservatively within the range, or liberally straddling pages outside the range
void* start = (conservative ? mi_align_up_ptr(addr, _mi_os_page_size())
: mi_align_down_ptr(addr, _mi_os_page_size()));
void* end = (conservative ? mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size())
: mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()));
ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start;
if (diff <= 0) return NULL;
mi_assert_internal((conservative && (size_t)diff <= size) || (!conservative && (size_t)diff >= size));
if (newsize != NULL) *newsize = (size_t)diff;
return start;
}
static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) {
return mi_os_page_align_areax(true, addr, size, newsize);
}
bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed) {
mi_assert_internal(size >= stat_already_committed);
if (is_zero != NULL) { *is_zero = false; }
mi_os_stat_counter_increase(subproc, commit_calls, 1);
// page align range
size_t csize;
void* start = mi_os_page_align_areax(false /* conservative? */, addr, size, &csize);
if (csize == 0) return true;
// commit
bool os_is_zero = false;
int err = _mi_prim_commit(start, csize, &os_is_zero);
if (err != 0) {
_mi_warning_message("cannot commit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
return false;
}
if (os_is_zero && is_zero != NULL) {
*is_zero = true;
mi_assert_expensive(mi_mem_is_zero(start, csize));
}
// note: the following seems required for asan (otherwise `mimalloc-test-stress` fails)
#ifdef MI_TRACK_ASAN
if (os_is_zero) { mi_track_mem_defined(start,csize); }
else { mi_track_mem_undefined(start,csize); }
#endif
mi_os_stat_increase(subproc, committed, size - stat_already_committed); // for precise commit vs. decommit
return true;
}
bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero) {
return _mi_os_commit_ex(subproc, addr, size, is_zero, 0);
}
static bool mi_os_decommit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* needs_recommit, size_t stat_size) {
mi_assert_internal(needs_recommit!=NULL);
// page align
size_t csize;
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return true;
// decommit
*needs_recommit = true;
int err = _mi_prim_decommit(start,csize,needs_recommit);
if (err != 0) {
_mi_warning_message("cannot decommit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
}
else if (*needs_recommit) {
mi_os_stat_decrease(subproc, committed, stat_size);
}
mi_assert_internal(err == 0);
return (err == 0);
}
bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size) {
bool needs_recommit;
return mi_os_decommit_ex(subproc, addr, size, &needs_recommit, size);
}
// Signal to the OS that the address range is no longer in use
// but may be used later again. This will release physical memory
// pages and reduce swapping while keeping the memory committed.
// We page align to a conservative area inside the range to reset.
bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size) {
// page align conservatively within the range
size_t csize;
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr)
mi_os_stat_counter_increase(subproc, reset, csize);
mi_os_stat_counter_increase(subproc, reset_calls, 1);
#if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN
memset(start, 0, csize); // pretend it is eagerly reset
#endif
int err = _mi_prim_reset(start, csize);
if (err != 0) {
_mi_warning_message("cannot reset OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
}
return (err == 0);
}
void _mi_os_reuse( mi_subproc_t* subproc, void* addr, size_t size ) {
MI_UNUSED(subproc);
// page align conservatively within the range
size_t csize = 0;
void* const start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return;
const int err = _mi_prim_reuse(start, csize);
if (err != 0) {
_mi_warning_message("cannot reuse OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
}
}
// either resets or decommits memory, returns true if the memory needs
// to be recommitted if it is to be re-used later on.
bool _mi_os_purge_ex(mi_subproc_t* subproc, void* p, size_t size, bool allow_reset, size_t stat_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg)
{
if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed?
mi_os_stat_counter_increase(subproc, purge_calls, 1);
mi_os_stat_counter_increase(subproc, purged, size);
if (commit_fun != NULL) {
bool decommitted = (*commit_fun)(false, p, size, NULL, commit_fun_arg);
return decommitted; // needs_recommit?
}
else if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit?
!_mi_preloading()) // don't decommit during preloading (unsafe)
{
bool needs_recommit = true;
mi_os_decommit_ex(subproc, p, size, &needs_recommit, stat_size);
return needs_recommit;
}
else {
if (allow_reset) { // this can sometimes be not allowed if the range is not fully committed (on Windows, we cannot reset uncommitted memory)
_mi_os_reset(subproc, p, size);
}
return false; // needs no recommit
}
}
// either resets or decommits memory, returns true if the memory needs
// to be recommitted if it is to be re-used later on.
bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size) {
return _mi_os_purge_ex(subproc, p, size, true, size, NULL, NULL);
}
// Set (or reset) a guard range (see `_mi_prim_guard` for the meaning of `is_committed`).
static bool mi_os_guardx(void* addr, size_t size, bool set, bool is_committed) {
// page align conservatively within the range
size_t csize = 0;
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return false;
int err = _mi_prim_guard(start, csize, set, is_committed);
if (err != 0) {
_mi_warning_message("cannot %s guard OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", (set ? "set" : "reset"), err, err, start, csize);
}
return (err == 0);
}
// Make a range inaccessible
bool _mi_os_guard(void* addr, size_t size, bool is_committed) {
return mi_os_guardx(addr, size, true, is_committed);
}
// Make a range accessible again (if it is accounted as committed)
bool _mi_os_unguard(void* addr, size_t size, bool is_committed) {
return mi_os_guardx(addr, size, false, is_committed);
}
/* ----------------------------------------------------------------------------
Support for allocating huge OS pages (1Gib) that are reserved up-front
and possibly associated with a specific NUMA node. (use `numa_node>=0`)
-----------------------------------------------------------------------------*/
#define MI_HUGE_OS_PAGE_SIZE (MI_GiB)
#if (MI_SIZE_SIZE >= 8) && !MI_CHERI // on CHERI the separately mapped huge pages cannot be covered by one capability
// Claim an aligned address range for huge pages
static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
if (total_size != NULL) *total_size = 0;
size_t size = 0;
if (mi_mul_overflow(pages,MI_HUGE_OS_PAGE_SIZE,&size) || !mi_memid_os_size_fits(size)) {
_mi_warning_message("too many huge pages requested: %zu\n", pages);
return NULL;
}
mi_addr_t start = 0;
mi_addr_t end = 0;
mi_addr_t huge_start = mi_atomic_load_relaxed(&_mi_process.os.huge_start);
do {
start = huge_start;
if (start == 0) {
// Initialize the start address after the 32TiB area
start = (MI_ADDR(32) << 40); // 32TiB virtual start address (after addresses returned by _mi_os_get_aligned_hint)
#if (MI_SECURE || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode
mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)
if (mi_theap_is_initialized(theap)) { // todo: or no hint at all if we lack randomness?
const size_t r = _mi_theap_random_next(theap);
start = start + (MI_ADDR(MI_HUGE_OS_PAGE_SIZE) * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB
}
else {
_mi_warning_message("failed to randomize the start address of huge pages allocation (%zu bytes at %p)", size, mi_ptr_from_addr(start));
}
#endif
}
end = start + size;
} while (!mi_atomic_cas_weak_relaxed(&_mi_process.os.huge_start, &huge_start, end));
if (total_size != NULL) *total_size = size;
return (uint8_t*)mi_ptr_from_addr(start);
}
#else
static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
MI_UNUSED(pages);
if (total_size != NULL) *total_size = 0;
return NULL;
}
#endif
// Allocate MI_ARENA_SLICE_ALIGN aligned huge pages
void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) {
*memid = _mi_memid_none();
if (psize != NULL) *psize = 0;
if (pages_reserved != NULL) *pages_reserved = 0;
size_t size = 0;
uint8_t* const start = mi_os_claim_huge_pages(pages, &size);
if (start == NULL) return NULL; // or 32-bit systems
// Allocate one page at the time but try to place them contiguously
// We allocate one page at the time to be able to abort if it takes too long
// or to at least allocate as many as available on the system.
mi_msecs_t start_t = _mi_clock_start();
size_t page = 0;
bool all_zero = true;
while (page < pages) {
// allocate a page
bool is_zero = false;
void* addr = start + (page * MI_HUGE_OS_PAGE_SIZE);
void* p = NULL;
int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &is_zero, &p);
if (!is_zero) { all_zero = false; }
if (err != 0) {
_mi_warning_message("unable to allocate huge OS page (error: %d (0x%x), address: %p, size: %zx bytes)\n", err, err, addr, MI_HUGE_OS_PAGE_SIZE);
break;
}
// Did we succeed at a contiguous address?
if (p != addr) {
// no success, issue a warning and break
if (p != NULL) {
_mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr);
mi_os_prim_free(subproc, p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, true /* adjust */);
}
break;
}
// success, record it
page++; // increase before timeout check (see issue #711)
mi_os_stat_increase(subproc, committed, MI_HUGE_OS_PAGE_SIZE);
mi_os_stat_increase(subproc, reserved, MI_HUGE_OS_PAGE_SIZE);
// check for timeout
if (max_msecs > 0) {
mi_msecs_t elapsed = _mi_clock_end(start_t);
if (page >= 1) {
mi_msecs_t estimate = ((elapsed / (page==0 ? 1 : page)) * pages);
if (estimate > 2*max_msecs) { // seems like we are going to timeout, break
elapsed = max_msecs + 1;
}
}
if (elapsed > max_msecs) {
_mi_warning_message("huge OS page allocation timed out (after allocating %zu page(s))\n", page);
break;
}
}
}
const size_t allocated = page * MI_HUGE_OS_PAGE_SIZE;
mi_assert_internal(allocated <= size);
if (pages_reserved != NULL) { *pages_reserved = page; }
if (psize != NULL) { *psize = allocated; }
if (page != 0) {
mi_assert(start != NULL);
*memid = _mi_memid_create_os(start, allocated, allocated, all_zero, true /* is_large */);
memid->memkind = MI_MEM_OS_HUGE;
mi_assert(memid->is_pinned);
#ifdef MI_TRACK_ASAN
if (all_zero) { mi_track_mem_defined(start,allocated); }
#endif
}
return (page == 0 ? NULL : start);
}
// free every huge page in a range individually (as we allocated per page)
// note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems.
static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size) {