Skip to content

Commit 6a7e041

Browse files
committed
with no-abandon heaps (page_full_retain=-1) ensure huge pages are more aggressively collected (see also python/cpython#159718)
1 parent 36e29b8 commit 6a7e041

5 files changed

Lines changed: 25 additions & 11 deletions

File tree

‎include/mimalloc/alloc/page.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page);
3030
void _mi_page_update_stats(mi_page_t* page);
3131

3232
void _mi_theap_collect_retired(mi_theap_t* theap, bool force);
33+
void _mi_theap_collect_huge_pages(mi_theap_t* theap);
3334
bool _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page); // true if an empty page was freed
3435
#if MI_DEBUG>1
3536
bool _mi_page_is_valid(mi_page_t* page);

‎src/alloc/alloc-generic.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,7 @@ static inline mi_page_t* mi_page_queue_find_free(mi_theap_t* theap, mi_page_queu
247247
// so their size is not always `> MI_LARGE_OBJ_SIZE_MAX`.
248248
static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page_alignment, mi_page_queue_t* pq) {
249249
_mi_theap_qsbr_collect_delayed(theap);
250+
_mi_theap_collect_huge_pages(theap);
250251
const size_t block_size = mi_bounds_size(_mi_os_good_alloc_size(size));
251252
// on CHERI, very large blocks need a large alignment for exact bounds: use an OS aligned page then (see `mi_page_meta_in_front_block_start`)
252253
const size_t bounds_align = mi_bounds_align(block_size);

‎src/alloc/page-queue.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -405,9 +405,9 @@ void mi_page_queue_move_to_back(mi_theap_t* theap, mi_page_queue_t* queue, mi_pa
405405
mi_assert_internal(queue->last == page);
406406
}
407407

408-
// Rotate the full queue such that `page` becomes the first page (and the pages before it move to the back)
409-
void mi_page_queue_full_rotate_to(mi_page_queue_t* queue, mi_page_t* page) {
410-
mi_assert_internal(mi_page_queue_is_full(queue));
408+
// Rotate the full or huge queue such that `page` becomes the first page (and the pages before it move to the back)
409+
void mi_page_queue_rotate_to(mi_page_queue_t* queue, mi_page_t* page) {
410+
mi_assert_internal(mi_page_queue_is_special(queue));
411411
mi_assert_expensive(mi_page_queue_contains(queue, page));
412412
if (queue->first == page) return;
413413
queue->last->next = queue->first;

‎src/alloc/page-queue.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_p
2929
void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page);
3030
void mi_page_queue_move_to_front(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page);
3131
void mi_page_queue_move_to_back(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page);
32-
void mi_page_queue_full_rotate_to(mi_page_queue_t* queue, mi_page_t* page);
32+
void mi_page_queue_rotate_to(mi_page_queue_t* queue, mi_page_t* page);
3333

3434

3535
static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) {

‎src/alloc/page.c‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -634,6 +634,10 @@ void _mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) {
634634
// abandon full pages (this is the usual case in order to allow for sharing of memory between theaps)
635635
_mi_page_abandon(page, pq);
636636
}
637+
else if (mi_page_queue_is_huge(pq)) {
638+
// keep huge pages in the huge queue: it is never searched for free blocks (and is collected before each huge allocation)
639+
_mi_page_update_stats(page);
640+
}
637641
else if (!mi_page_is_in_full(page)) {
638642
// put full pages in a theap local queue (this is for theaps that cannot abandon, for example, if the theap can be destroyed)
639643
mi_page_queue_enqueue_from(&mi_page_theap(page)->pages[MI_BIN_FULL], pq, page);
@@ -734,14 +738,15 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept {
734738
}
735739

736740

737-
#define MI_FULL_COLLECT_MAX_PAGES (4) /* visit at most N full pages per collect (unless forced) */
741+
#define MI_FULL_COLLECT_MAX_PAGES (8) /* visit at most N full (or huge) pages per collect (unless forced); only for theaps that cannot abandon */
738742

739-
// Collect pages in the full queue that had blocks freed by other threads.
743+
// Collect pages in the full or huge queue that had blocks freed by other threads.
740744
// Visits at most MI_FULL_COLLECT_MAX_PAGES pages (unless forced) and rotates the queue so the next call continues after them.
741-
// note: normally full pages get immediately abandoned and the full queue is always empty;
745+
// With N live pages in the queue, a page freed by another thread is thus revisited after about N/MI_FULL_COLLECT_MAX_PAGES calls.
746+
// note: normally full and huge pages get immediately abandoned and these queues stay (almost) empty;
742747
// this is only used if abandoning is disabled for the theap (see `theap.c:mi_theap_options_init`).
743-
static void mi_theap_collect_full_pages(mi_theap_t* theap, bool force) {
744-
mi_page_queue_t* pq = &theap->pages[MI_BIN_FULL];
748+
static void mi_theap_collect_special_pages(mi_page_queue_t* pq, bool force) {
749+
mi_assert_internal(mi_page_queue_is_special(pq));
745750
mi_page_t* page = pq->first;
746751
for (size_t count = 0; page != NULL && (force || count < MI_FULL_COLLECT_MAX_PAGES); count++) {
747752
mi_page_t* next = page->next; // get next in case we free the page
@@ -752,12 +757,19 @@ static void mi_theap_collect_full_pages(mi_theap_t* theap, bool force) {
752757
_mi_page_free(page, pq);
753758
}
754759
else {
760+
mi_assert_internal(mi_page_queue_is_full(pq)); // huge pages contain a single block
755761
_mi_page_unfull(page);
756762
}
757763
}
758764
page = next;
759765
}
760-
if (page != NULL) { mi_page_queue_full_rotate_to(pq, page); }
766+
if (page != NULL) { mi_page_queue_rotate_to(pq, page); }
767+
}
768+
769+
// Before allocating a fresh huge page, free huge pages whose block was freed by another thread.
770+
void _mi_theap_collect_huge_pages(mi_theap_t* theap) {
771+
if (theap->allow_page_abandon) return; // huge pages were abandoned and are freed by the thread that frees their block
772+
mi_theap_collect_special_pages(&theap->pages[MI_BIN_HUGE], false);
761773
}
762774

763775
static void mi_page_try_retire(mi_page_queue_t* pq, mi_page_t* page, size_t bin, bool force, size_t* min, size_t* max) {
@@ -795,7 +807,7 @@ void _mi_theap_collect_retired(mi_theap_t* theap, bool force) {
795807
theap->page_retired_min = min;
796808
theap->page_retired_max = max;
797809
if (!theap->allow_page_abandon) {
798-
mi_theap_collect_full_pages(theap, force);
810+
mi_theap_collect_special_pages(&theap->pages[MI_BIN_FULL], force);
799811
}
800812
_mi_theap_qsbr_collect_delayed(theap);
801813
}

0 commit comments

Comments
 (0)