Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
5b8d63e
fix GroupValues*::size()
kosiew Sep 11, 2026
b453da8
fix: subtract inline RowConverter/Rows descriptors from nested .size(…
kosiew Sep 11, 2026
0eca435
test: add independent scratch-capacity/reuse test for multi_group_by
kosiew Sep 11, 2026
ac44e83
feat(groupcolumn): charge missing GroupColumn owner descriptors, dedu…
kosiew Sep 11, 2026
4d8f7de
feat: add row.rs emit/reuse accounting tests
kosiew Sep 11, 2026
b3d2b21
refactor: move comment to correct emit test
kosiew Sep 11, 2026
617b0ed
feat: improve SLT peak measurement and spill test assertions
kosiew Sep 11, 2026
a57639a
fix: Disable partial‑agg skip only under memory limits and shrink DIS…
kosiew Sep 12, 2026
3388a4d
fix: increase DISTINCT spill test budget to stabilize flaky nested_nu…
kosiew Sep 12, 2026
069dd3f
fix: stabilize DISTINCT spill case in nested_nullability.rs
kosiew Sep 12, 2026
ec95c89
fix(memory): release accumulator capacity before reservation reconcil…
kosiew Sep 12, 2026
66d3441
refactor(nested_nullability): wrap FairSpillPool in TrackConsumersPool
kosiew Sep 12, 2026
8a2095b
fix: disable single_distinct_aggregation_to_group_by for DISTINCT spi…
kosiew Sep 12, 2026
fe8f8ed
fix(partial_oom_drain): emit partial OOM drain states via EmitTo::Fir…
kosiew Sep 12, 2026
dde38d4
fix(datafusion/core/tests/memory_limit/mod.rs): correct expected erro…
kosiew Sep 12, 2026
43f4724
feat(test): add memory limit and target partitions for case G in aggr…
kosiew Sep 13, 2026
95d7ce6
fix: safely reset capacities in GroupValuesColumn::clear_shrink and a…
kosiew Sep 13, 2026
fe3d616
fix: refine cached_values to `Option<Weak<dyn Array>>` and adjust cac…
kosiew Sep 13, 2026
cf77086
perf(datafusion/physical-plan/src/aggregates/group_values/multi_group…
kosiew Sep 13, 2026
3bfe7f6
Merge branch 'main' into memcalc-11-23393
kosiew Sep 16, 2026
88acc2b
Merge branch 'main' into memcalc-11-23393
kosiew Sep 19, 2026
ca93e84
Fix descriptor-Aware Legacy Spill Budget and Test Updates
kosiew Sep 21, 2026
2034050
Global FairSpillPool Cap Across Siblings
kosiew Sep 21, 2026
d7bca4a
Merge legacy state/replay under shared pool
kosiew Sep 21, 2026
df2278a
Merge branch 'main' into memcalc-11-23393
kosiew Sep 21, 2026
8b69b81
Restore and Optimize Take State and Materialized Output
kosiew Sep 24, 2026
da6e482
Remove mirrored expected_size tests
kosiew Sep 24, 2026
f432ac1
Refactor: Remove broad post-state accumulator rebuilds
kosiew Sep 24, 2026
b57b794
Restore expectations in memory_limit test
kosiew Sep 25, 2026
8ec5f3e
Update multi_group_by/mo: Replace finish/append/truncate with Boolean…
kosiew Sep 25, 2026
8a477d6
Update dictionary cache with weak pointers and improved checks
kosiew Sep 25, 2026
bea282b
Fix datafusion/physical-plan/src/aggregates/mod.rs: Remove test re-ex…
kosiew Sep 25, 2026
4995794
Fix memory limit in datafusion/sqllogictest/test_files/aggregate_memo…
kosiew Sep 25, 2026
2127ab1
Merge branch 'main' into memcalc-11-23393
kosiew Oct 5, 2026
6be2499
fix: use is_cached for Weak dictionary cache after merge
kosiew Oct 5, 2026
252777f
fix: adjust holder baseline and initial-table pool handling
kosiew Oct 5, 2026
232dd97
Update hash stream memory pool handling for unbounded limits
kosiew Oct 7, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update dictionary cache with weak pointers and improved checks
- Added is_cached(&self, values) method.
- Used std::ptr::addr_eq on Weak::as_ptr / Arc::as_ptr.
- Replaced cache checks in sync_value_cache + tests.
- Kept upgrade() only for weak-liveness test.
  • Loading branch information
kosiew committed Sep 25, 2026
commit 8a477d64e529db201cbbba30e996c5f2560d931e
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,12 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> DictionaryGroupValuesColumn<K> {
.unwrap();
}

fn is_cached(&self, values: &ArrayRef) -> bool {
self.cached_values
.as_ref()
.is_some_and(|cached| std::ptr::addr_eq(cached.as_ptr(), Arc::as_ptr(values)))
}

/// Makes `val_hashes` and `val_to_inner` correspond to `dict_values`,
/// reusing them when they were already built for this same values array.
///
Expand All @@ -190,11 +196,7 @@ impl<K: ArrowDictionaryKeyType + Send + Sync> DictionaryGroupValuesColumn<K> {
/// rather than O(rows). `take_n` remaps `inner` and clears
/// `cached_values`, forcing a rebuild.
fn sync_value_cache(&mut self, dict_values: &ArrayRef) {
if self.cached_values.as_ref().is_some_and(|cached| {
cached
.upgrade()
.is_some_and(|values| Arc::ptr_eq(&values, dict_values))
}) {
if self.is_cached(dict_values) {
return;
}
self.hash_values(dict_values);
Expand Down Expand Up @@ -985,11 +987,7 @@ mod tests {
usize::MAX,
"unreferenced value resolved"
);
assert!(col.cached_values.as_ref().is_some_and(|cached| {
cached
.upgrade()
.is_some_and(|value| Arc::ptr_eq(&value, &values))
}));
assert!(col.is_cached(&values));
assert_eq!(col.inner.len(), 2);
}

Expand Down Expand Up @@ -1025,11 +1023,7 @@ mod tests {
col.vectorized_append(&second, &[0]).unwrap();

assert!(
col.cached_values.as_ref().is_some_and(|cached| {
cached.upgrade().is_some_and(|value| {
Arc::ptr_eq(&value, second.as_dictionary::<Int32Type>().values())
})
}),
col.is_cached(second.as_dictionary::<Int32Type>().values()),
"cache should track the most recent values array"
);
assert_eq!(col.inner.len(), 1, "'a' must dedup across both arrays");
Expand Down Expand Up @@ -1074,11 +1068,7 @@ mod tests {
let mut col = utf8_col();

col.append_val(&batch, 0).unwrap();
assert!(col.cached_values.as_ref().is_some_and(|cached| {
cached
.upgrade()
.is_some_and(|value| Arc::ptr_eq(&value, &values))
}));
assert!(col.is_cached(&values));

col.vectorized_append(&batch, &[0, 1]).unwrap();
assert_eq!(col.inner.len(), 2, "'a' must not be duplicated");
Expand Down