Skip to content

Commit 47e17ed

Browse files
committed
Simplify file sum regression coverage
Signed-off-by: Connor Tsui <[email protected]>
1 parent d1a277f commit 47e17ed

2 files changed

Lines changed: 82 additions & 173 deletions

File tree

‎vortex-file/src/tests.rs‎

Lines changed: 7 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -2128,17 +2128,8 @@ async fn test_writer_with_statistics() -> VortexResult<()> {
21282128
Ok(())
21292129
}
21302130

2131-
#[rstest]
2132-
#[case::overflow_then_positive(vec![i64::MAX, 1], vec![2], None)]
2133-
#[case::overflow_then_zero(vec![i64::MAX, 1], vec![0], None)]
2134-
#[case::all_chunks_overflow(vec![i64::MAX, 1], vec![i64::MAX, 1], None)]
2135-
#[case::valid_sum(vec![3, 4], vec![5], Some(12))]
21362131
#[tokio::test]
2137-
async fn file_sum_preserves_overflow_across_chunks(
2138-
#[case] first: Vec<i64>,
2139-
#[case] second: Vec<i64>,
2140-
#[case] expected: Option<i64>,
2141-
) -> VortexResult<()> {
2132+
async fn file_sum_preserves_overflow_across_chunks() -> VortexResult<()> {
21422133
let dtype = DType::Struct(
21432134
StructFields::from_iter([("numbers", DType::from(PType::I64))]),
21442135
Nullability::NonNullable,
@@ -2148,63 +2139,28 @@ async fn file_sum_preserves_overflow_across_chunks(
21482139
.write_options()
21492140
.with_file_statistics(vec![Stat::Sum])
21502141
.writer(&mut buf, dtype);
2151-
for values in [first, second] {
2152-
let array = StructArray::from_fields(&[(
2153-
"numbers",
2154-
PrimitiveArray::from_iter(values).into_array(),
2155-
)])?
2156-
.into_array();
2157-
writer.push(array).await?;
2158-
}
2159-
let summary = writer.finish().await?;
2160-
let expected = expected.map_or(Precision::Absent, Precision::exact);
2161-
assert_eq!(
2162-
summary.footer().statistics().unwrap().stats_sets()[0].get(Stat::Sum),
2163-
expected
2164-
);
2165-
let file = SESSION.open_options().open_buffer(ByteBuffer::from(buf))?;
2166-
assert_eq!(
2167-
file.file_stats().unwrap().stats_sets()[0].get(Stat::Sum),
2168-
expected
2169-
);
2170-
Ok(())
2171-
}
21722142

2173-
#[rstest]
2174-
#[case::arithmetic_nan_then_finite(vec![1.0], 1.0)]
2175-
#[case::all_nan_finals(vec![f64::INFINITY, f64::NEG_INFINITY], 0.0)]
2176-
#[tokio::test]
2177-
async fn file_sum_skips_nan_chunk_finals(
2178-
#[case] second: Vec<f64>,
2179-
#[case] expected: f64,
2180-
) -> VortexResult<()> {
2181-
let dtype = DType::Struct(
2182-
StructFields::from_iter([("numbers", DType::from(PType::F64))]),
2183-
Nullability::NonNullable,
2184-
);
2185-
let mut buf = ByteBufferMut::empty();
2186-
let mut writer = SESSION
2187-
.write_options()
2188-
.with_file_statistics(vec![Stat::Sum])
2189-
.writer(&mut buf, dtype);
2190-
for values in [vec![f64::INFINITY, f64::NEG_INFINITY], second] {
2143+
for values in [vec![i64::MAX, 1], vec![2]] {
21912144
let array = StructArray::from_fields(&[(
21922145
"numbers",
21932146
PrimitiveArray::from_iter(values).into_array(),
21942147
)])?
21952148
.into_array();
21962149
writer.push(array).await?;
21972150
}
2151+
21982152
let summary = writer.finish().await?;
21992153
assert_eq!(
22002154
summary.footer().statistics().unwrap().stats_sets()[0].get(Stat::Sum),
2201-
Precision::exact(expected)
2155+
Precision::Absent
22022156
);
2157+
22032158
let file = SESSION.open_options().open_buffer(ByteBuffer::from(buf))?;
22042159
assert_eq!(
22052160
file.file_stats().unwrap().stats_sets()[0].get(Stat::Sum),
2206-
Precision::exact(expected)
2161+
Precision::Absent
22072162
);
2163+
22082164
Ok(())
22092165
}
22102166

‎vortex-layout/src/layouts/file_stats.rs‎

Lines changed: 75 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4+
//! Collects chunk statistics and combines them into file statistics.
5+
//!
6+
//! Sums retain their accumulator state so overflow survives across chunks. Other statistics use
7+
//! intermediate columns, including truncation markers for variable-length bounds.
8+
49
use std::future;
510
use std::marker::PhantomData;
611
use std::sync::Arc;
@@ -72,7 +77,7 @@ pub fn accumulate_stats(
7277
/// Accumulates write-time statistics for a single file column.
7378
struct StatsAccumulator {
7479
builders: Vec<Box<dyn StatsArrayBuilder>>,
75-
/// Keep overflow in the partial state instead of treating its null final as an invalid row.
80+
/// Retains overflow that a nullable column of finalized chunk sums would discard.
7681
sum: Option<Accumulator<Sum>>,
7782
length: usize,
7883
}
@@ -89,8 +94,9 @@ impl StatsAccumulator {
8994

9095
let sum = (stats.contains(&Stat::Sum) && Stat::Sum.dtype(dtype).is_some()).then(|| {
9196
Accumulator::try_new(Sum, NumericalAggregateOpts::skip_nans(), dtype.clone())
92-
.vortex_expect("supported Sum dtype must have an accumulator")
97+
.vortex_expect("StatsAccumulator::new checked that the input dtype supports Sum")
9398
});
99+
94100
let builders = stats
95101
.iter()
96102
.filter(|&&stat| stat != Stat::Sum)
@@ -121,26 +127,31 @@ impl StatsAccumulator {
121127
array.dtype().clone(),
122128
)?;
123129
chunk_sum.accumulate(array, ctx)?;
130+
124131
let result = chunk_sum.final_scalar()?;
125132
let is_nan = result
126133
.as_primitive_opt()
127134
.is_some_and(|value| value.is_nan());
128135
if let Some(value) = result.into_value() {
129136
array.statistics().set(Stat::Sum, Precision::Exact(value));
130137
}
138+
131139
// The file writer's NaN-skipping policy also applies to chunk finals.
132140
if !is_nan {
133141
sum.merge_from(&mut chunk_sum)?;
134142
}
135143
}
144+
136145
for builder in &mut self.builders {
137146
if let Some(value) = array.statistics().compute_stat(builder.stat(), ctx)? {
138147
builder.append_scalar(value.cast(&value.dtype().as_nullable())?)?;
139148
} else {
140149
builder.append_null();
141150
}
142151
}
152+
143153
self.length += 1;
154+
144155
Ok(())
145156
}
146157

@@ -175,6 +186,7 @@ impl StatsAccumulator {
175186
/// Returns an aggregated stats set for the table.
176187
fn as_stats_set(&mut self, stats: &[Stat], ctx: &mut ExecutionCtx) -> VortexResult<StatsSet> {
177188
let mut stats_set = StatsSet::default();
189+
178190
if self.length != 0
179191
&& stats.contains(&Stat::Sum)
180192
&& let Some(sum) = &self.sum
@@ -183,6 +195,7 @@ impl StatsAccumulator {
183195
// The legacy stats set cannot store a null, so an overflowed sum remains absent.
184196
stats_set.set(Stat::Sum, Precision::Exact(value));
185197
}
198+
186199
let Some(stats_table) = self.as_array(ctx)? else {
187200
return Ok(stats_set);
188201
};
@@ -578,6 +591,7 @@ mod tests {
578591
use vortex_array::builders::VarBinViewBuilder;
579592
#[cfg(target_pointer_width = "64")]
580593
use vortex_array::dtype::DecimalDType;
594+
#[cfg(target_pointer_width = "64")]
581595
use vortex_array::expr::stats::StatsProvider;
582596
#[cfg(target_pointer_width = "64")]
583597
use vortex_array::scalar::DecimalValue;
@@ -586,6 +600,17 @@ mod tests {
586600

587601
use super::*;
588602

603+
fn accumulate_sum_stats(dtype: &DType, chunks: &[ArrayRef]) -> VortexResult<StatsSet> {
604+
let mut ctx = array_session().create_execution_ctx();
605+
let mut acc = StatsAccumulator::new(dtype, &[Stat::Sum], 64);
606+
607+
for chunk in chunks {
608+
acc.push_chunk(chunk, &mut ctx)?;
609+
}
610+
611+
acc.as_stats_set(&[Stat::Sum], &mut ctx)
612+
}
613+
589614
#[rstest]
590615
#[case(DType::Utf8(Nullability::NonNullable))]
591616
#[case(DType::Binary(Nullability::NonNullable))]
@@ -686,27 +711,26 @@ mod tests {
686711
}
687712

688713
#[rstest]
689-
#[case::overflow_then_positive(vec![i64::MAX, 1], vec![2], None)]
690-
#[case::overflow_then_zero(vec![i64::MAX, 1], vec![0], None)]
691-
#[case::all_chunks_overflow(vec![i64::MAX, 1], vec![i64::MAX, 1], None)]
692-
#[case::overflow_across_chunks(vec![i64::MAX], vec![1], None)]
693-
#[case::valid_sum(vec![3, 4], vec![5], Some(12))]
694-
fn sum_keeps_overflow_across_chunks(
695-
#[case] first: Vec<i64>,
696-
#[case] second: Vec<i64>,
714+
#[case::overflow_across_chunks(vec![vec![Some(i64::MAX)], vec![Some(1)]], None)]
715+
#[case::nullable_chunks(vec![vec![None, Some(3)], vec![Some(4), None]], Some(7))]
716+
#[case::no_chunks(vec![], None)]
717+
#[case::empty_chunk(vec![vec![]], Some(0))]
718+
fn sum_combines_chunk_states(
719+
#[case] chunks: Vec<Vec<Option<i64>>>,
697720
#[case] expected: Option<i64>,
698721
) -> VortexResult<()> {
699-
let first = PrimitiveArray::from_iter(first).into_array();
700-
let second = PrimitiveArray::from_iter(second).into_array();
701-
let mut ctx = array_session().create_execution_ctx();
702-
let mut acc = StatsAccumulator::new(first.dtype(), &[Stat::Sum], 64);
703-
acc.push_chunk(&first, &mut ctx)?;
704-
acc.push_chunk(&second, &mut ctx)?;
705-
let stats = acc.as_stats_set(&[Stat::Sum], &mut ctx)?;
722+
let dtype = DType::Primitive(PType::I64, Nullability::Nullable);
723+
let chunks = chunks
724+
.into_iter()
725+
.map(|values| PrimitiveArray::from_option_iter(values).into_array())
726+
.collect::<Vec<_>>();
727+
728+
let stats = accumulate_sum_stats(&dtype, &chunks)?;
706729
assert_eq!(
707730
stats.get(Stat::Sum),
708731
expected.map_or(Precision::Absent, Precision::exact)
709732
);
733+
710734
Ok(())
711735
}
712736

@@ -718,12 +742,15 @@ mod tests {
718742
DecimalDType::new(1, 0),
719743
Nullability::NonNullable,
720744
);
721-
let first = ConstantArray::new(value.clone(), 10_000_000_000).into_array();
722-
let second = ConstantArray::new(value, 10_000_000_000).into_array();
723-
let mut ctx = array_session().create_execution_ctx();
724-
let mut acc = StatsAccumulator::new(first.dtype(), &[Stat::Sum], 64);
725-
for chunk in [&first, &second] {
726-
acc.push_chunk(chunk, &mut ctx)?;
745+
let chunks = [
746+
ConstantArray::new(value.clone(), 10_000_000_000).into_array(),
747+
ConstantArray::new(value, 10_000_000_000).into_array(),
748+
];
749+
750+
let stats = accumulate_sum_stats(chunks[0].dtype(), &chunks)?;
751+
assert_eq!(stats.get(Stat::Sum), Precision::Absent);
752+
753+
for chunk in &chunks {
727754
assert_eq!(
728755
chunk.statistics().get(Stat::Sum),
729756
Precision::Exact(Scalar::decimal(
@@ -733,115 +760,41 @@ mod tests {
733760
))
734761
);
735762
}
736-
let stats = acc.as_stats_set(&[Stat::Sum], &mut ctx)?;
737-
assert_eq!(stats.get(Stat::Sum), Precision::Absent);
738-
Ok(())
739-
}
740763

741-
#[rstest]
742-
#[case::non_null(vec![3, 4], Some(7))]
743-
#[case::overflow(vec![i64::MAX, 1], None)]
744-
fn sum_preserves_chunk_cache_publication(
745-
#[case] values: Vec<i64>,
746-
#[case] expected: Option<i64>,
747-
) -> VortexResult<()> {
748-
let array = PrimitiveArray::from_iter(values).into_array();
749-
let mut ctx = array_session().create_execution_ctx();
750-
let mut acc = StatsAccumulator::new(array.dtype(), &[Stat::Sum], 64);
751-
acc.push_chunk(&array, &mut ctx)?;
752-
assert_eq!(
753-
array.statistics().get(Stat::Sum),
754-
expected.map_or(Precision::Absent, |value| {
755-
Precision::Exact(Scalar::primitive(value, Nullability::Nullable))
756-
})
757-
);
758764
Ok(())
759765
}
760766

761767
#[rstest]
762-
#[case::arithmetic_nan_then_finite(vec![1.0], 1.0)]
763-
#[case::all_nan_finals(vec![f64::INFINITY, f64::NEG_INFINITY], 0.0)]
764-
fn sum_skips_nan_chunk_finals(
768+
#[case::nan_chunk_then_finite(vec![f64::INFINITY, f64::NEG_INFINITY], vec![1.0], 1.0)]
769+
#[case::all_nan_chunks(
770+
vec![f64::INFINITY, f64::NEG_INFINITY],
771+
vec![f64::INFINITY, f64::NEG_INFINITY],
772+
0.0
773+
)]
774+
#[case::nan_from_merge(vec![f64::INFINITY], vec![f64::NEG_INFINITY], f64::NAN)]
775+
#[case::chunk_grouping(vec![1e16], vec![-1e16, 1.0], 0.0)]
776+
fn sum_preserves_float_chunk_semantics(
777+
#[case] first: Vec<f64>,
765778
#[case] second: Vec<f64>,
766779
#[case] expected: f64,
767-
#[values(false, true)] cached: bool,
768780
) -> VortexResult<()> {
769-
let first = PrimitiveArray::from_iter([f64::INFINITY, f64::NEG_INFINITY]).into_array();
770-
let second = PrimitiveArray::from_iter(second).into_array();
771-
let mut ctx = array_session().create_execution_ctx();
772-
if cached {
773-
sum(&first, &mut ctx)?;
774-
sum(&second, &mut ctx)?;
781+
let chunks = [
782+
PrimitiveArray::from_iter(first).into_array(),
783+
PrimitiveArray::from_iter(second).into_array(),
784+
];
785+
786+
let stats = accumulate_sum_stats(chunks[0].dtype(), &chunks)?;
787+
if expected.is_nan() {
788+
assert!(
789+
stats
790+
.get(Stat::Sum)
791+
.as_exact()
792+
.is_some_and(|value| value.as_primitive().is_nan())
793+
);
794+
} else {
795+
assert_eq!(stats.get(Stat::Sum), Precision::exact(expected));
775796
}
776-
let mut acc = StatsAccumulator::new(first.dtype(), &[Stat::Sum], 64);
777-
acc.push_chunk(&first, &mut ctx)?;
778-
acc.push_chunk(&second, &mut ctx)?;
779-
assert!(
780-
first
781-
.statistics()
782-
.get(Stat::Sum)
783-
.as_exact()
784-
.is_some_and(|value| value.as_primitive().is_nan())
785-
);
786-
let stats = acc.as_stats_set(&[Stat::Sum], &mut ctx)?;
787-
assert_eq!(stats.get(Stat::Sum), Precision::exact(expected));
788-
Ok(())
789-
}
790797

791-
#[test]
792-
fn sum_keeps_nan_from_merging_chunk_finals() -> VortexResult<()> {
793-
let first = PrimitiveArray::from_iter([f64::INFINITY]).into_array();
794-
let second = PrimitiveArray::from_iter([f64::NEG_INFINITY]).into_array();
795-
let mut ctx = array_session().create_execution_ctx();
796-
let mut acc = StatsAccumulator::new(first.dtype(), &[Stat::Sum], 64);
797-
acc.push_chunk(&first, &mut ctx)?;
798-
acc.push_chunk(&second, &mut ctx)?;
799-
let stats = acc.as_stats_set(&[Stat::Sum], &mut ctx)?;
800-
assert!(
801-
stats
802-
.get(Stat::Sum)
803-
.as_exact()
804-
.is_some_and(|value| value.as_primitive().is_nan())
805-
);
806-
Ok(())
807-
}
808-
809-
#[test]
810-
fn sum_keeps_chunk_float_grouping() -> VortexResult<()> {
811-
let first = PrimitiveArray::from_iter([1e16f64]).into_array();
812-
let second = PrimitiveArray::from_iter([-1e16f64, 1.0]).into_array();
813-
let mut ctx = array_session().create_execution_ctx();
814-
let mut acc = StatsAccumulator::new(first.dtype(), &[Stat::Sum], 64);
815-
acc.push_chunk(&first, &mut ctx)?;
816-
acc.push_chunk(&second, &mut ctx)?;
817-
let stats = acc.as_stats_set(&[Stat::Sum], &mut ctx)?;
818-
assert_eq!(stats.get(Stat::Sum), Precision::exact(0.0f64));
819-
Ok(())
820-
}
821-
822-
#[rstest]
823-
#[case::no_chunks(vec![], None)]
824-
#[case::empty_chunk(vec![vec![]], Some(0))]
825-
#[case::all_null_chunk(vec![vec![None, None]], Some(0))]
826-
#[case::nullable_chunks(vec![vec![None, Some(3)], vec![Some(4), None]], Some(7))]
827-
fn sum_distinguishes_missing_input_from_zero(
828-
#[case] chunks: Vec<Vec<Option<i64>>>,
829-
#[case] expected: Option<i64>,
830-
) -> VortexResult<()> {
831-
let mut ctx = array_session().create_execution_ctx();
832-
let dtype = DType::Primitive(PType::I64, Nullability::Nullable);
833-
let mut acc = StatsAccumulator::new(&dtype, &[Stat::Sum], 64);
834-
for chunk in chunks {
835-
acc.push_chunk(
836-
&PrimitiveArray::from_option_iter(chunk).into_array(),
837-
&mut ctx,
838-
)?;
839-
}
840-
let stats = acc.as_stats_set(&[Stat::Sum], &mut ctx)?;
841-
assert_eq!(
842-
stats.get(Stat::Sum),
843-
expected.map_or(Precision::Absent, Precision::exact)
844-
);
845798
Ok(())
846799
}
847800
}

0 commit comments

Comments
 (0)