@@ -634,6 +634,10 @@ void _mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) {
634634 // abandon full pages (this is the usual case in order to allow for sharing of memory between theaps)
635635 _mi_page_abandon(page, pq);
636636 }
637+ else if (mi_page_queue_is_huge(pq)) {
638+ // keep huge pages in the huge queue: it is never searched for free blocks (and is collected before each huge allocation)
639+ _mi_page_update_stats(page);
640+ }
637641 else if (!mi_page_is_in_full(page)) {
638642 // put full pages in a theap local queue (this is for theaps that cannot abandon, for example, if the theap can be destroyed)
639643 mi_page_queue_enqueue_from(&mi_page_theap(page)->pages[MI_BIN_FULL], pq, page);
@@ -734,14 +738,15 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept {
734738}
735739
736740
737- #define MI_FULL_COLLECT_MAX_PAGES (4 ) /* visit at most N full pages per collect (unless forced) */
741+ #define MI_FULL_COLLECT_MAX_PAGES (8 ) /* visit at most N full (or huge) pages per collect (unless forced); only for theaps that cannot abandon */
738742
739- // Collect pages in the full queue that had blocks freed by other threads.
743+ // Collect pages in the full or huge queue that had blocks freed by other threads.
740744// Visits at most MI_FULL_COLLECT_MAX_PAGES pages (unless forced) and rotates the queue so the next call continues after them.
741- // note: normally full pages get immediately abandoned and the full queue is always empty;
745+ // With N live pages in the queue, a page freed by another thread is thus revisited after about N/MI_FULL_COLLECT_MAX_PAGES calls.
746+ // note: normally full and huge pages get immediately abandoned and these queues stay (almost) empty;
742747// this is only used if abandoning is disabled for the theap (see `theap.c:mi_theap_options_init`).
743- static void mi_theap_collect_full_pages(mi_theap_t* theap , bool force) {
744- mi_page_queue_t* pq = &theap->pages[MI_BIN_FULL] ;
748+ static void mi_theap_collect_special_pages(mi_page_queue_t* pq , bool force) {
749+ mi_assert_internal(mi_page_queue_is_special(pq)) ;
745750 mi_page_t* page = pq->first;
746751 for (size_t count = 0; page != NULL && (force || count < MI_FULL_COLLECT_MAX_PAGES); count++) {
747752 mi_page_t* next = page->next; // get next in case we free the page
@@ -752,12 +757,19 @@ static void mi_theap_collect_full_pages(mi_theap_t* theap, bool force) {
752757 _mi_page_free(page, pq);
753758 }
754759 else {
760+ mi_assert_internal(mi_page_queue_is_full(pq)); // huge pages contain a single block
755761 _mi_page_unfull(page);
756762 }
757763 }
758764 page = next;
759765 }
760- if (page != NULL) { mi_page_queue_full_rotate_to(pq, page); }
766+ if (page != NULL) { mi_page_queue_rotate_to(pq, page); }
767+ }
768+
769+ // Before allocating a fresh huge page, free huge pages whose block was freed by another thread.
770+ void _mi_theap_collect_huge_pages(mi_theap_t* theap) {
771+ if (theap->allow_page_abandon) return; // huge pages were abandoned and are freed by the thread that frees their block
772+ mi_theap_collect_special_pages(&theap->pages[MI_BIN_HUGE], false);
761773}
762774
763775static void mi_page_try_retire(mi_page_queue_t* pq, mi_page_t* page, size_t bin, bool force, size_t* min, size_t* max) {
@@ -795,7 +807,7 @@ void _mi_theap_collect_retired(mi_theap_t* theap, bool force) {
795807 theap->page_retired_min = min;
796808 theap->page_retired_max = max;
797809 if (!theap->allow_page_abandon) {
798- mi_theap_collect_full_pages( theap, force);
810+ mi_theap_collect_special_pages(& theap->pages[MI_BIN_FULL] , force);
799811 }
800812 _mi_theap_qsbr_collect_delayed(theap);
801813}
0 commit comments