Skip to content

Commit 06fb9f8

Browse files
committed
Added new Unrecorded node to the StatsNode. This is utilized for
children of a nullable struct as the stats are incorrect for these stats see vortex-data#9205 Signed-off-by: Thor <[email protected]>
1 parent 3e3c385 commit 06fb9f8

5 files changed

Lines changed: 152 additions & 33 deletions

File tree

‎vortex-duckdb/src/copy.rs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -444,8 +444,6 @@ mod tests {
444444

445445
#[test]
446446
fn column_stats_reports_nested_leaves_by_quoted_path() -> VortexResult<()> {
447-
// A nullable struct `s` contributes `s.b` and a trailing own entry to the post-order stats
448-
// layout; only the leaves are statistics columns, keyed by their full quoted path.
449447
let summary = RUNTIME.block_on(async {
450448
let b = buffer![10i32, 20, 30].into_array();
451449
let inner = StructArray::new(["b"].into(), [b], 3, Validity::AllValid).into_array();
@@ -470,7 +468,8 @@ mod tests {
470468
assert_eq!(leaf_stats_columns(file_stats).len(), 2);
471469
let nested = column_stats(&summary, 0)?;
472470
assert_eq!(nested.column_key, "\"s\".\"b\"");
473-
assert!(nested.min.is_some());
471+
assert!(nested.min.is_none());
472+
assert!(nested.null_count.is_none());
474473
assert!(nested.column_size_bytes.is_some_and(|size| size > 0));
475474

476475
let top_level = column_stats(&summary, 1)?;

‎vortex-file/src/tests.rs‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1491,8 +1491,6 @@ async fn write_nullable_top_level_struct() -> VortexResult<()> {
14911491
.write(&mut buf, array.to_array_stream())
14921492
.await?;
14931493

1494-
// The root struct is nullable and has 2 null rows, so its own null-count entry (keyed by the
1495-
// root field path) should reflect that, in addition to the leaf `age` field's stats.
14961494
let stats = summary
14971495
.footer()
14981496
.statistics()
@@ -1504,6 +1502,15 @@ async fn write_nullable_top_level_struct() -> VortexResult<()> {
15041502
stat_value(root_stats, Stat::NullCount),
15051503
Precision::exact(ScalarValue::from(2u64))
15061504
);
1505+
let (age, _) = stats.get_by_path(&field_path!(age)).expect("age entry");
1506+
assert!(age.is_empty());
1507+
1508+
let file = SESSION.open_options().open_buffer(ByteBuffer::from(buf))?;
1509+
let stats = file
1510+
.file_stats()
1511+
.expect("file statistics should be present");
1512+
let (age, _) = stats.get_by_path(&field_path!(age)).expect("age entry");
1513+
assert!(age.is_empty());
15071514

15081515
Ok(())
15091516
}
@@ -2251,18 +2258,18 @@ async fn test_writer_with_statistics() -> VortexResult<()> {
22512258
let summary = writer.finish().await?;
22522259

22532260
let statistics = summary.footer().statistics().expect("requested statistics");
2254-
assert_eq!(statistics.stats_sets().len(), 1);
2255-
let stats = &statistics.stats_sets()[0];
2261+
assert_eq!(statistics.aggregates().len(), 1);
2262+
let (stats, _) = statistics.get(0);
22562263
assert_eq!(
2257-
stats.get(Stat::Min),
2264+
stat_value(stats, Stat::Min),
22582265
Precision::exact(ScalarValue::from(1u32))
22592266
);
22602267
assert_eq!(
2261-
stats.get(Stat::Max),
2268+
stat_value(stats, Stat::Max),
22622269
Precision::exact(ScalarValue::from(5u32))
22632270
);
22642271
assert_eq!(
2265-
stats.get(Stat::NullCount),
2272+
stat_value(stats, Stat::NullCount),
22662273
Precision::exact(ScalarValue::from(0u64))
22672274
);
22682275
assert_eq!(summary.row_count(), 5);

‎vortex-layout/src/layouts/file_stats.rs‎

Lines changed: 131 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use std::future;
5+
use std::iter;
56
use std::sync::Arc;
67

78
use futures::StreamExt;
@@ -204,11 +205,16 @@ struct StatsAccumulator {
204205
}
205206

206207
impl StatsAccumulator {
208+
/// An accumulator that records nothing.
209+
fn empty() -> Self {
210+
Self {
211+
aggregates: Vec::new(),
212+
}
213+
}
214+
207215
fn new(dtype: &DType, stats: Option<&[AggregateFnRef]>) -> Self {
208216
if !supports_file_stats(dtype) {
209-
return Self {
210-
aggregates: Vec::new(),
211-
};
217+
return Self::empty();
212218
}
213219

214220
// When no explicit list is requested, each leaf picks its own default the way
@@ -334,6 +340,9 @@ fn default_pruning_aggregate_fns(dtype: &DType) -> Vec<AggregateFnRef> {
334340
/// [`DType::Variant`], are skipped entirely: no entry is emitted for them or anything beneath
335341
/// them.
336342
///
343+
/// Entries beneath a nullable struct are still emitted, but the writer records no aggregates for
344+
/// them (see [`StatsNode::Unrecorded`]).
345+
///
337346
/// This function is the single source of truth for the number and order of stats entries, used
338347
/// both when accumulating stats at write time and when reconstructing them from the footer at
339348
/// read time, so no paths or dtypes need to be persisted in the flatbuffer itself.
@@ -364,6 +373,18 @@ enum StatsNode {
364373
Leaf(StatsAccumulator),
365374
/// A dtype that does not support file stats (e.g. [`DType::Variant`]); contributes no entries.
366375
Skipped,
376+
/// A subtree beneath a nullable struct, which contributes `entries` entries that record no
377+
/// aggregates.
378+
///
379+
/// Struct fields are stored unmasked, so a row where the struct is null holds an unspecified
380+
/// value in each child that the child's own validity may report as valid. Stats over those
381+
/// children would count that value, e.g. undercount nulls, which can prune a file that
382+
/// `is_null(s.a)` matches. Until nulls are propagated from structs into their children, these
383+
/// entries are left empty. See <https://github.com/vortex-data/vortex/issues/9205>.
384+
///
385+
/// The entries are kept rather than dropped so the footer layout from
386+
/// [`postorder_stats_layout`] doesn't depend on whether they're recorded.
387+
Unrecorded { entries: usize },
367388
/// A dtype with addressable children. Currently only built for `DType::Struct`.
368389
Container {
369390
/// One child per addressable sub-dtype, in declaration order (one per struct field,
@@ -382,15 +403,25 @@ impl StatsNode {
382403
fn build(dtype: &DType, stats: Option<&[AggregateFnRef]>) -> Self {
383404
match dtype.as_struct_fields_opt() {
384405
Some(struct_fields) => {
406+
let nullable = dtype.nullability() == Nullability::Nullable;
385407
let children = struct_fields
386408
.names()
387409
.iter()
388410
.zip(struct_fields.fields())
389411
.map(|(name, field_dtype)| {
390-
(Field::Name(name.clone()), Self::build(&field_dtype, stats))
412+
let child = if nullable {
413+
Self::Unrecorded {
414+
entries: postorder_stats_layout(&field_dtype).len(),
415+
}
416+
} else {
417+
Self::build(&field_dtype, stats)
418+
};
419+
(Field::Name(name.clone()), child)
391420
})
392421
.collect();
393-
let own = if dtype.nullability() == Nullability::Nullable {
422+
// Only reached through non-nullable ancestors, so the struct's own validity is
423+
// its full logical validity.
424+
let own = if nullable {
394425
Self::Leaf(StatsAccumulator::new(dtype, stats))
395426
} else {
396427
Self::Skipped
@@ -407,7 +438,7 @@ impl StatsNode {
407438

408439
fn push_chunk(&mut self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()> {
409440
match self {
410-
Self::Skipped => Ok(()),
441+
Self::Skipped | Self::Unrecorded { .. } => Ok(()),
411442
Self::Leaf(acc) => acc.push_chunk(array, ctx),
412443
Self::Container { .. } => {
413444
let struct_array = array.clone().execute::<StructArray>(ctx)?;
@@ -460,6 +491,10 @@ impl StatsNode {
460491
) -> VortexResult<()> {
461492
match self {
462493
Self::Skipped => Ok(()),
494+
Self::Unrecorded { entries } => {
495+
out.extend(iter::repeat_n(AggregateStats::default(), *entries));
496+
Ok(())
497+
}
463498
Self::Leaf(acc) => {
464499
out.push(acc.aggregate_stats(allowed)?);
465500
Ok(())
@@ -509,6 +544,13 @@ impl FileStatsAccumulator {
509544

510545
let legacy = Arc::new(Mutex::new(if write_legacy_stats {
511546
match dtype.as_struct_fields_opt() {
547+
// The fields of a nullable root are recorded empty, like `StatsNode::Unrecorded`.
548+
Some(struct_fields) if dtype.nullability() == Nullability::Nullable => {
549+
struct_fields
550+
.fields()
551+
.map(|_| StatsAccumulator::empty())
552+
.collect()
553+
}
512554
Some(struct_fields) => struct_fields
513555
.fields()
514556
.map(|field_dtype| StatsAccumulator::new(&field_dtype, stats.as_deref()))
@@ -855,10 +897,11 @@ mod tests {
855897
}
856898

857899
#[test]
858-
fn three_level_nested_struct_accumulates_stats_at_each_level() -> VortexResult<()> {
900+
fn three_level_nested_struct_records_only_entries_without_nullable_ancestors()
901+
-> VortexResult<()> {
859902
// Same shape as `postorder_layout_three_level_nesting_with_mixed_nullability`, but
860-
// exercises actual accumulation: each nullable level along the path should independently
861-
// contribute its own null-count entry, in post-order.
903+
// exercises actual accumulation. The root is nullable, so every entry beneath it is left
904+
// empty; only the root's own null-count entry is recorded.
862905
let mut ctx = array_session().create_execution_ctx();
863906

864907
let leaf_c = buffer![10i32, 20, 30].into_array();
@@ -882,28 +925,69 @@ mod tests {
882925

883926
let stats_sets = node_stats_sets(&node)?;
884927

885-
// `a.b.c`'s own stats come first (post-order), then `a.b`'s null-count entry, then the
886-
// root's own null-count entry.
928+
// Post-order: `a.b.c`, then `a.b`'s null-count entry, then the root's.
887929
assert_eq!(stats_sets.len(), 3);
930+
assert!(stats_sets[0].is_empty());
931+
assert!(stats_sets[1].is_empty());
888932
assert_eq!(
889-
stats_sets[0].get(Stat::Min).as_exact(),
890-
Some(ScalarValue::from(10i32))
933+
stats_sets[2].get(Stat::NullCount).as_exact(),
934+
Some(ScalarValue::from(1u64))
891935
);
936+
Ok(())
937+
}
938+
939+
#[test]
940+
fn children_of_nullable_struct_are_unrecorded() -> VortexResult<()> {
941+
// https://github.com/vortex-data/vortex/issues/9205
942+
let mut ctx = array_session().create_execution_ctx();
943+
944+
let a = buffer![1i32, 999, 3].into_array();
945+
let s_validity = Validity::Array(BoolArray::from_iter([true, false, true]).into_array());
946+
let s = StructArray::new(FieldNames::from(["a"]), [a], 3, s_validity).into_array();
947+
let root =
948+
StructArray::new(FieldNames::from(["s"]), [s], 3, Validity::NonNullable).into_array();
949+
950+
let requested = [agg(Stat::NullCount), agg(Stat::Min), agg(Stat::Max)];
951+
let mut node = StatsNode::build(root.dtype(), Some(&requested));
952+
node.push_chunk(&root, &mut ctx)?;
953+
954+
let mut aggregates = Vec::new();
955+
node.collect_aggregate_stats(&|_| true, &mut aggregates)?;
956+
957+
// `s.a`, then `s`'s own null-count entry.
958+
assert_eq!(aggregates.len(), 2);
959+
assert!(aggregates[0].is_empty());
892960
assert_eq!(
893-
stats_sets[0].get(Stat::Max).as_exact(),
894-
Some(ScalarValue::from(30i32))
961+
aggregates[1].get(&agg(Stat::NullCount)),
962+
Precision::exact(Scalar::from(1u64))
895963
);
964+
Ok(())
965+
}
966+
967+
#[test]
968+
fn nullable_children_of_non_nullable_struct_are_recorded() -> VortexResult<()> {
969+
// `s` is non-nullable, so `s.a`'s own validity is its full logical validity.
970+
let mut ctx = array_session().create_execution_ctx();
971+
972+
let a = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]).into_array();
973+
let s =
974+
StructArray::new(FieldNames::from(["a"]), [a], 3, Validity::NonNullable).into_array();
975+
let root =
976+
StructArray::new(FieldNames::from(["s"]), [s], 3, Validity::NonNullable).into_array();
977+
978+
let requested = [agg(Stat::NullCount), agg(Stat::Max)];
979+
let mut node = StatsNode::build(root.dtype(), Some(&requested));
980+
node.push_chunk(&root, &mut ctx)?;
981+
982+
let stats_sets = node_stats_sets(&node)?;
983+
assert_eq!(stats_sets.len(), 1);
896984
assert_eq!(
897985
stats_sets[0].get(Stat::NullCount).as_exact(),
898-
Some(ScalarValue::from(0u64))
899-
);
900-
assert_eq!(
901-
stats_sets[1].get(Stat::NullCount).as_exact(),
902986
Some(ScalarValue::from(1u64))
903987
);
904988
assert_eq!(
905-
stats_sets[2].get(Stat::NullCount).as_exact(),
906-
Some(ScalarValue::from(1u64))
989+
stats_sets[0].get(Stat::Max).as_exact(),
990+
Some(ScalarValue::from(3i32))
907991
);
908992
Ok(())
909993
}
@@ -975,6 +1059,32 @@ mod tests {
9751059
Ok(())
9761060
}
9771061

1062+
#[test]
1063+
fn legacy_aggregate_stats_of_nullable_root_are_unrecorded() -> VortexResult<()> {
1064+
let session = array_session();
1065+
let dtype = DType::struct_([("a", i32_dtype())], Nullability::Nullable);
1066+
1067+
let acc = FileStatsAccumulator::try_new(
1068+
&dtype,
1069+
Some(Arc::from([agg(Stat::NullCount), agg(Stat::Max)])),
1070+
&session,
1071+
true,
1072+
&LayoutWriterContext::new(ArrayContext::empty()),
1073+
)?;
1074+
1075+
let a = buffer![1i32, 999, 3].into_array();
1076+
let validity = Validity::Array(BoolArray::from_iter([true, false, true]).into_array());
1077+
let root = StructArray::new(FieldNames::from(["a"]), [a], 3, validity).into_array();
1078+
1079+
let (mut ptr, _eof) = SequenceId::root().split();
1080+
acc.process(Ok((ptr.advance(), root)))?;
1081+
1082+
let legacy = acc.legacy_aggregate_stats()?;
1083+
assert_eq!(legacy.len(), 1);
1084+
assert!(legacy[0].is_empty());
1085+
Ok(())
1086+
}
1087+
9781088
#[rstest]
9791089
#[case(DType::struct_([("a", i32_dtype())], Nullability::NonNullable), StructArray::new(FieldNames::from(["a"]), [buffer![1i32, 2, 3].into_array()], 3, Validity::NonNullable).into_array())]
9801090
#[case(i32_dtype(), buffer![1i32, 2, 3].into_array())]

‎vortex-layout/src/layouts/zoned/writer.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,8 @@ fn default_zoned_aggregate_fns(dtype: &DType, session: &VortexSession) -> Arc<[A
233233
// Sum is deliberately absent: zone maps exist to prune, and a zone sum prunes nothing.
234234
// Its semantics are also unsettled - null-on-empty was changed in #9113 and reverted in
235235
// #9324 - so it is not a stat to record in every zone of every file, let alone freeze
236-
// into an edition. File-level statistics still record `Stat::Sum` via `PRUNING_STATS`.
236+
// into an edition. File-level statistics still record a sum by default (see
237+
// `file_stats::default_pruning_aggregate_fns`).
237238
let mut aggregate_fns = vec![
238239
max,
239240
min,

‎vortex-sqllogictest/slt/duckdb/copy_return_stats.slt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,10 @@ TO '${WORK_DIR}/copy_return_stats.vortex' (FORMAT vortex, RETURN_STATS);
1111

1212
# As in parquet, statistics are reported per leaf column: struct fields are keyed by their full
1313
# quoted path and the struct itself gets no entry. Lists are reported as a single column.
14+
# DuckDB structs are nullable, so their fields report no min, max or null count until struct
15+
# nulls are propagated into children (https://github.com/vortex-data/vortex/issues/9205).
1416
query TIII???
1517
COPY (SELECT {'a': 1, 'b': 2} AS st, [1, 2, 3] AS lst, 7 AS i)
1618
TO '${WORK_DIR}/copy_return_stats_nested.vortex' (FORMAT vortex, RETURN_STATS);
1719
----
18-
<REGEX>:\$\{WORK_DIR\}/copy_return_stats_nested\.vortex 1 \d+ 0 \{'"i"'=\{column_size_bytes=\d+, max=7, min=7, null_count=0, num_values=1\}, '"lst"'=\{column_size_bytes=\d+, null_count=0, num_values=1\}, '"st"\."a"'=\{column_size_bytes=\d+, max=1, min=1, null_count=0, num_values=1\}, '"st"\."b"'=\{column_size_bytes=\d+, max=2, min=2, null_count=0, num_values=1\}\} NULL \{\}
20+
<REGEX>:\$\{WORK_DIR\}/copy_return_stats_nested\.vortex 1 \d+ 0 \{'"i"'=\{column_size_bytes=\d+, max=7, min=7, null_count=0, num_values=1\}, '"lst"'=\{column_size_bytes=\d+, null_count=0, num_values=1\}, '"st"\."a"'=\{column_size_bytes=\d+, num_values=1\}, '"st"\."b"'=\{column_size_bytes=\d+, num_values=1\}\} NULL \{\}

0 commit comments

Comments
 (0)