Skip to content

Commit 583668d

Browse files
peterzhu2118peterzhu2118
committed
Remove gc_bump_pointer_heap
It was leftover and we don't actually need it.
1 parent f1ce5fc commit 583668d

6 files changed

Lines changed: 54 additions & 112 deletions

File tree

‎gc/default/default.c‎

Lines changed: 49 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -241,15 +241,17 @@ static RB_THREAD_LOCAL_SPECIFIER int malloc_increase_local;
241241
#endif
242242

243243
typedef struct ractor_newobj_heap_cache {
244+
uintptr_t cursor;
245+
uintptr_t cursor_end;
244246
struct free_region *next_region;
245247
struct heap_page *using_page;
246248
uintptr_t region_end;
249+
size_t allocated_objects_count;
247250
} rb_ractor_newobj_heap_cache_t;
248251

249252
typedef struct ractor_newobj_cache {
250253
size_t incremental_mark_step_allocated_slots;
251254
rb_ractor_newobj_heap_cache_t heap_caches[HEAP_COUNT];
252-
struct gc_bump_pointer_heap bump_heaps[];
253255
} rb_ractor_newobj_cache_t;
254256

255257
typedef struct {
@@ -1764,31 +1766,11 @@ calloc1(size_t n)
17641766
return calloc(1, n);
17651767
}
17661768

1767-
static void
1768-
gc_ractor_jit_cursor_sync(void *c, void *data)
1769-
{
1770-
rb_ractor_newobj_cache_t *gc_cache = c;
1771-
bool tracing = (bool)(uintptr_t)data;
1772-
1773-
for (size_t heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) {
1774-
struct gc_bump_pointer_heap *bump = &gc_cache->bump_heaps[heap_idx];
1775-
bump->jit_cursor_end = tracing ? bump->cursor : bump->cursor_end;
1776-
}
1777-
}
1778-
17791769
void
17801770
rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event)
17811771
{
17821772
rb_objspace_t *objspace = objspace_ptr;
1783-
rb_event_flag_t old_events = objspace->hook_events;
17841773
objspace->hook_events = event & RUBY_INTERNAL_EVENT_OBJSPACE_MASK;
1785-
1786-
bool was_tracing = old_events & RUBY_INTERNAL_EVENT_NEWOBJ;
1787-
bool now_tracing = objspace->hook_events & RUBY_INTERNAL_EVENT_NEWOBJ;
1788-
if (was_tracing != now_tracing) {
1789-
rb_gc_ractor_newobj_cache_foreach(gc_ractor_jit_cursor_sync,
1790-
(void *)(uintptr_t)now_tracing);
1791-
}
17921774
}
17931775

17941776
unsigned long long
@@ -2498,46 +2480,41 @@ rb_gc_impl_size_allocatable_p(size_t size)
24982480
}
24992481

25002482
static inline void
2501-
gc_bump_flush_alloc_count(struct gc_bump_pointer_heap *bump, rb_heap_t *heap)
2483+
gc_bump_flush_alloc_count(rb_ractor_newobj_heap_cache_t *heap_cache, rb_heap_t *heap)
25022484
{
2503-
if (bump->slot_size == 0) return;
2504-
2505-
size_t n = (bump->cursor - bump->region_start) / bump->slot_size;
2506-
if (n > 0) {
2507-
RUBY_ATOMIC_SIZE_ADD(heap->total_allocated_objects, n);
2508-
bump->region_start = bump->cursor;
2485+
if (heap_cache->allocated_objects_count > 0) {
2486+
RUBY_ATOMIC_SIZE_ADD(heap->total_allocated_objects, heap_cache->allocated_objects_count);
2487+
heap_cache->allocated_objects_count = 0;
25092488
}
25102489
}
25112490

25122491
static void
25132492
ractor_cache_flush_count(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *gc_cache)
25142493
{
25152494
for (int heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) {
2516-
gc_bump_flush_alloc_count(&gc_cache->bump_heaps[heap_idx], &heaps[heap_idx]);
2495+
gc_bump_flush_alloc_count(&gc_cache->heap_caches[heap_idx], &heaps[heap_idx]);
25172496
}
25182497
}
25192498

25202499
static inline void
2521-
ractor_cache_open_window(rb_objspace_t *objspace, struct gc_bump_pointer_heap *bump,
2522-
rb_ractor_newobj_heap_cache_t *heap_cache)
2500+
ractor_cache_open_window(rb_objspace_t *objspace, rb_ractor_newobj_heap_cache_t *heap_cache,
2501+
size_t heap_idx)
25232502
{
25242503
uintptr_t end = heap_cache->region_end;
25252504

25262505
if (RB_UNLIKELY(is_incremental_marking(objspace))) {
2527-
uintptr_t window_end = bump->cursor + INCREMENTAL_MARK_STEP_ALLOCATIONS * bump->slot_size;
2506+
uintptr_t window_end = heap_cache->cursor + INCREMENTAL_MARK_STEP_ALLOCATIONS * pool_slot_sizes[heap_idx];
25282507
if (window_end < end) end = window_end;
25292508
}
25302509

2531-
bump->cursor_end = end;
2532-
bump->jit_cursor_end =
2533-
RB_UNLIKELY(objspace->hook_events & RUBY_INTERNAL_EVENT_NEWOBJ) ? bump->cursor : end;
2510+
heap_cache->cursor_end = end;
25342511
}
25352512

25362513
static inline bool
2537-
ractor_cache_advance_region(rb_objspace_t *objspace, struct gc_bump_pointer_heap *bump,
2538-
rb_ractor_newobj_heap_cache_t *heap_cache, rb_heap_t *heap)
2514+
ractor_cache_advance_region(rb_objspace_t *objspace, rb_ractor_newobj_heap_cache_t *heap_cache,
2515+
size_t heap_idx)
25392516
{
2540-
gc_bump_flush_alloc_count(bump, heap);
2517+
gc_bump_flush_alloc_count(heap_cache, &heaps[heap_idx]);
25412518

25422519
struct free_region *region = heap_cache->next_region;
25432520
if (region == NULL) {
@@ -2546,13 +2523,12 @@ ractor_cache_advance_region(rb_objspace_t *objspace, struct gc_bump_pointer_heap
25462523

25472524
rb_asan_unpoison_object((VALUE)region, false);
25482525
GC_ASSERT(RB_TYPE_P((VALUE)region, T_NONE));
2549-
bump->cursor = (uintptr_t)region;
2550-
bump->region_start = (uintptr_t)region;
2526+
heap_cache->cursor = (uintptr_t)region;
25512527
heap_cache->region_end = region->end;
25522528
heap_cache->next_region = region->next;
25532529
rb_asan_poison_object((VALUE)region);
25542530

2555-
ractor_cache_open_window(objspace, bump, heap_cache);
2531+
ractor_cache_open_window(objspace, heap_cache, heap_idx);
25562532

25572533
return true;
25582534
}
@@ -2561,24 +2537,25 @@ static inline VALUE
25612537
ractor_cache_allocate_slot(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *gc_cache,
25622538
size_t heap_idx)
25632539
{
2564-
struct gc_bump_pointer_heap *bump = &gc_cache->bump_heaps[heap_idx];
2540+
rb_ractor_newobj_heap_cache_t *heap_cache = &gc_cache->heap_caches[heap_idx];
2541+
size_t slot_size = pool_slot_sizes[heap_idx];
25652542

2566-
uintptr_t cursor = bump->cursor;
2567-
if (RB_UNLIKELY(cursor + bump->slot_size > bump->cursor_end)) {
2543+
uintptr_t cursor = heap_cache->cursor;
2544+
if (RB_UNLIKELY(cursor + slot_size > heap_cache->cursor_end)) {
25682545
if (RB_UNLIKELY(is_incremental_marking(objspace))) {
25692546
return Qfalse;
25702547
}
25712548

2572-
rb_ractor_newobj_heap_cache_t *heap_cache = &gc_cache->heap_caches[heap_idx];
2573-
if (!ractor_cache_advance_region(objspace, bump, heap_cache, &heaps[heap_idx])) {
2549+
if (!ractor_cache_advance_region(objspace, heap_cache, heap_idx)) {
25742550
return Qfalse;
25752551
}
2576-
cursor = bump->cursor;
2552+
cursor = heap_cache->cursor;
25772553
}
25782554

25792555
VALUE obj = (VALUE)cursor;
25802556
rb_asan_unpoison_object(obj, true);
2581-
bump->cursor = cursor + bump->slot_size;
2557+
heap_cache->cursor = cursor + slot_size;
2558+
heap_cache->allocated_objects_count++;
25822559

25832560
#if RGENGC_CHECK_MODE
25842561
GC_ASSERT(rb_gc_impl_obj_slot_size(obj) == heap_slot_size(heap_idx));
@@ -2612,10 +2589,9 @@ ractor_cache_set_page(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *gc_cach
26122589
{
26132590
gc_report(3, objspace, "ractor_set_cache: Using page %p\n", (void *)page->body);
26142591

2615-
struct gc_bump_pointer_heap *bump = &gc_cache->bump_heaps[heap_idx];
26162592
rb_ractor_newobj_heap_cache_t *heap_cache = &gc_cache->heap_caches[heap_idx];
26172593

2618-
GC_ASSERT(bump->cursor + bump->slot_size > bump->cursor_end);
2594+
GC_ASSERT(heap_cache->cursor + pool_slot_sizes[heap_idx] > heap_cache->cursor_end);
26192595
GC_ASSERT(heap_cache->next_region == NULL);
26202596
GC_ASSERT(page->free_slots != 0);
26212597
GC_ASSERT(page->free_region != NULL);
@@ -2625,13 +2601,12 @@ ractor_cache_set_page(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *gc_cach
26252601
struct free_region *region = page->free_region;
26262602
rb_asan_unpoison_object((VALUE)region, false);
26272603
GC_ASSERT(RB_TYPE_P((VALUE)region, T_NONE));
2628-
bump->cursor = (uintptr_t)region;
2629-
bump->region_start = (uintptr_t)region;
2604+
heap_cache->cursor = (uintptr_t)region;
26302605
heap_cache->region_end = region->end;
26312606
heap_cache->next_region = region->next;
26322607
rb_asan_poison_object((VALUE)region);
26332608

2634-
ractor_cache_open_window(objspace, bump, heap_cache);
2609+
ractor_cache_open_window(objspace, heap_cache, heap_idx);
26352610

26362611
page->free_slots = 0;
26372612
page->free_region = NULL;
@@ -2684,12 +2659,12 @@ rb_gc_impl_zjit_new_obj_fastpath(void *objspace_ptr, size_t alloc_size, VALUE fl
26842659
}
26852660
if (slot_size == 0) return false;
26862661

2687-
size_t bump_base = offsetof(rb_ractor_newobj_cache_t, bump_heaps) +
2688-
heap_idx * sizeof(struct gc_bump_pointer_heap);
2662+
size_t base = offsetof(rb_ractor_newobj_cache_t, heap_caches) +
2663+
heap_idx * sizeof(rb_ractor_newobj_heap_cache_t);
26892664

26902665
struct rb_gc_zjit_default_new_obj_fastpath default_fastpath = {
2691-
bump_base + offsetof(struct gc_bump_pointer_heap, cursor),
2692-
bump_base + offsetof(struct gc_bump_pointer_heap, jit_cursor_end),
2666+
base + offsetof(rb_ractor_newobj_heap_cache_t, cursor),
2667+
base + offsetof(rb_ractor_newobj_heap_cache_t, cursor_end),
26932668
slot_size,
26942669
flags,
26952670
klass
@@ -2710,7 +2685,6 @@ NOINLINE(static VALUE newobj_bump_pointer_miss(rb_objspace_t *objspace, rb_racto
27102685
static VALUE
27112686
newobj_bump_pointer_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *gc_cache, size_t heap_idx, bool vm_locked)
27122687
{
2713-
struct gc_bump_pointer_heap *bump = &gc_cache->bump_heaps[heap_idx];
27142688
rb_ractor_newobj_cache_t *cache = gc_cache;
27152689
rb_ractor_newobj_heap_cache_t *heap_cache = &cache->heap_caches[heap_idx];
27162690
rb_heap_t *heap = &heaps[heap_idx];
@@ -2743,25 +2717,22 @@ newobj_bump_pointer_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *gc_c
27432717
}
27442718

27452719
if (is_incremental_marking(objspace)) {
2746-
if (bump->slot_size > 0) {
2747-
cache->incremental_mark_step_allocated_slots +=
2748-
(bump->cursor - bump->region_start) / bump->slot_size;
2749-
}
2750-
gc_bump_flush_alloc_count(bump, heap);
2720+
cache->incremental_mark_step_allocated_slots += heap_cache->allocated_objects_count;
2721+
gc_bump_flush_alloc_count(heap_cache, heap);
27512722

27522723
if (cache->incremental_mark_step_allocated_slots >= INCREMENTAL_MARK_STEP_ALLOCATIONS) {
27532724
gc_continue(objspace, heap);
27542725
cache->incremental_mark_step_allocated_slots = 0;
27552726
}
27562727

2757-
if (bump->cursor + bump->slot_size <= heap_cache->region_end) {
2758-
ractor_cache_open_window(objspace, bump, heap_cache);
2728+
if (heap_cache->cursor + pool_slot_sizes[heap_idx] <= heap_cache->region_end) {
2729+
ractor_cache_open_window(objspace, heap_cache, heap_idx);
27592730
obj = ractor_cache_allocate_slot(objspace, gc_cache, heap_idx);
27602731
}
27612732
}
27622733

27632734
if (obj == Qfalse) {
2764-
if (ractor_cache_advance_region(objspace, bump, heap_cache, heap)) {
2735+
if (ractor_cache_advance_region(objspace, heap_cache, heap_idx)) {
27652736
obj = ractor_cache_allocate_slot(objspace, gc_cache, heap_idx);
27662737
}
27672738
}
@@ -2774,8 +2745,7 @@ newobj_bump_pointer_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *gc_c
27742745
}
27752746

27762747
if (RB_UNLIKELY(ruby_gc_stressful)) {
2777-
bump->cursor_end = bump->cursor;
2778-
bump->jit_cursor_end = bump->cursor;
2748+
heap_cache->cursor_end = heap_cache->cursor;
27792749
}
27802750
}
27812751

@@ -4112,13 +4082,12 @@ gc_mode_transition(rb_objspace_t *objspace, enum gc_mode mode)
41124082
}
41134083

41144084
static void
4115-
heap_page_flush_cache_regions(struct heap_page *page, struct gc_bump_pointer_heap *bump,
4116-
rb_ractor_newobj_heap_cache_t *heap_cache)
4085+
heap_page_flush_cache_regions(struct heap_page *page, rb_ractor_newobj_heap_cache_t *heap_cache)
41174086
{
41184087
struct free_region *chain = heap_cache->next_region;
41194088

4120-
if (bump->cursor < heap_cache->region_end) {
4121-
VALUE start = (VALUE)bump->cursor;
4089+
if (heap_cache->cursor < heap_cache->region_end) {
4090+
VALUE start = (VALUE)heap_cache->cursor;
41224091
rb_asan_unpoison_object(start, false);
41234092
struct free_region *remnant = (struct free_region *)start;
41244093
remnant->flags = 0;
@@ -4186,26 +4155,23 @@ gc_ractor_newobj_cache_clear(void *c, void *data)
41864155
newobj_cache->incremental_mark_step_allocated_slots = 0;
41874156

41884157
for (size_t heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) {
4189-
struct gc_bump_pointer_heap *bump = &gc_cache->bump_heaps[heap_idx];
41904158
rb_ractor_newobj_heap_cache_t *cache = &newobj_cache->heap_caches[heap_idx];
41914159

41924160
rb_heap_t *heap = &heaps[heap_idx];
4193-
gc_bump_flush_alloc_count(bump, heap);
4161+
gc_bump_flush_alloc_count(cache, heap);
41944162

41954163
struct heap_page *page = cache->using_page;
4196-
RUBY_DEBUG_LOG("ractor using_page:%p cursor:%p", (void *)page, (void *)bump->cursor);
4164+
RUBY_DEBUG_LOG("ractor using_page:%p cursor:%p", (void *)page, (void *)cache->cursor);
41974165

41984166
if (page) {
4199-
heap_page_flush_cache_regions(page, bump, cache);
4167+
heap_page_flush_cache_regions(page, cache);
42004168
}
42014169

42024170
cache->using_page = NULL;
42034171
cache->next_region = NULL;
42044172
cache->region_end = 0;
4205-
bump->cursor = 0;
4206-
bump->cursor_end = 0;
4207-
bump->jit_cursor_end = 0;
4208-
bump->region_start = 0;
4173+
cache->cursor = 0;
4174+
cache->cursor_end = 0;
42094175
}
42104176
}
42114177

@@ -4215,9 +4181,8 @@ gc_ractor_newobj_cache_exhaust(void *c, void *data)
42154181
rb_ractor_newobj_cache_t *gc_cache = c;
42164182

42174183
for (size_t heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) {
4218-
struct gc_bump_pointer_heap *bump = &gc_cache->bump_heaps[heap_idx];
4219-
bump->cursor_end = bump->cursor;
4220-
bump->jit_cursor_end = bump->cursor;
4184+
rb_ractor_newobj_heap_cache_t *heap_cache = &gc_cache->heap_caches[heap_idx];
4185+
heap_cache->cursor_end = heap_cache->cursor;
42214186
}
42224187
}
42234188

@@ -6826,13 +6791,8 @@ rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor)
68266791

68276792
objspace->live_ractor_cache_count++;
68286793

6829-
rb_ractor_newobj_cache_t *gc_cache =
6830-
calloc1(sizeof(rb_ractor_newobj_cache_t) +
6831-
sizeof(struct gc_bump_pointer_heap) * (HEAP_COUNT + 1));
6794+
rb_ractor_newobj_cache_t *gc_cache = calloc1(sizeof(rb_ractor_newobj_cache_t));
68326795

6833-
for (size_t i = 0; i < HEAP_COUNT; i++) {
6834-
gc_cache->bump_heaps[i].slot_size = pool_slot_sizes[i];
6835-
}
68366796
return gc_cache;
68376797
}
68386798

‎gc/default/zjit_fastpath.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
struct rb_gc_zjit_default_new_obj_fastpath {
1111
size_t cursor_offset;
12-
size_t jit_cursor_end_offset;
12+
size_t cursor_end_offset;
1313
size_t slot_size;
1414
VALUE flags;
1515
VALUE klass;

‎gc/gc_impl.h‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,6 @@
1313
#include <stddef.h>
1414
#include <stdint.h>
1515

16-
struct gc_bump_pointer_heap {
17-
uintptr_t cursor;
18-
uintptr_t cursor_end;
19-
uintptr_t jit_cursor_end;
20-
uintptr_t region_start;
21-
size_t slot_size;
22-
};
23-
2416
enum rb_gc_zjit_fastpath_kind {
2517
RB_GC_ZJIT_FASTPATH_DEFAULT = 1,
2618
RB_GC_ZJIT_FASTPATH_MMTK = 2,

‎zjit/bindgen/src/main.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,6 @@ fn main() {
9191
.allowlist_type("RBasic")
9292

9393
.allowlist_type("RArray")
94-
.allowlist_type("gc_bump_pointer_heap")
9594
.allowlist_type("rb_gc_zjit_fastpath_kind")
9695
.allowlist_type("rb_gc_zjit_fastpath")
9796
.allowlist_type("rb_gc_zjit_fastpath_data")

‎zjit/src/codegen/gc_fastpath.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ use super::JITState;
1212
#[derive(Clone, Copy)]
1313
struct RbGcZjitDefaultNewObjFastpath {
1414
cursor_offset: usize,
15-
jit_cursor_end_offset: usize,
15+
cursor_end_offset: usize,
1616
slot_size: usize,
1717
flags: VALUE,
1818
klass: VALUE,
@@ -173,7 +173,7 @@ fn emit_default_new_obj_fastpath(
173173
miss: &Target,
174174
) -> Option<Opnd> {
175175
let cursor_offset: i32 = fastpath.cursor_offset.try_into().ok()?;
176-
let jit_cursor_end_offset: i32 = fastpath.jit_cursor_end_offset.try_into().ok()?;
176+
let cursor_end_offset: i32 = fastpath.cursor_end_offset.try_into().ok()?;
177177
let slot_size: u64 = fastpath.slot_size.try_into().ok()?;
178178

179179
let thread = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_THREAD_PTR as i32));
@@ -184,7 +184,7 @@ fn emit_default_new_obj_fastpath(
184184
let gc_cache = asm.load(Opnd::mem(64, ractor, ractor_newobj_cache_offset));
185185

186186
let cursor = asm.load(Opnd::mem(64, gc_cache, cursor_offset));
187-
let cursor_end = asm.load(Opnd::mem(64, gc_cache, jit_cursor_end_offset));
187+
let cursor_end = asm.load(Opnd::mem(64, gc_cache, cursor_end_offset));
188188

189189
let new_cursor = asm.add(cursor, Opnd::UImm(slot_size));
190190
asm.cmp(cursor_end, new_cursor);

0 commit comments

Comments
 (0)