|
| 1 | +// SPDX-License-Identifier: Apache-2.0 |
| 2 | +// SPDX-FileCopyrightText: Copyright the Vortex contributors |
| 3 | + |
| 4 | +use flatbuffers::FlatBufferBuilder; |
| 5 | +use flatbuffers::WIPOffset; |
| 6 | +use vortex_error::VortexResult; |
| 7 | +use vortex_error::vortex_bail; |
| 8 | +use vortex_error::vortex_ensure; |
| 9 | +use vortex_error::vortex_err; |
| 10 | +use vortex_session::VortexSession; |
| 11 | + |
| 12 | +use crate::dtype::DType; |
| 13 | +use crate::dtype::PType; |
| 14 | +use crate::expr::stats::Precision; |
| 15 | +use crate::expr::stats::Stat; |
| 16 | +use crate::flatbuffers::array as fba; |
| 17 | +use crate::scalar::Scalar; |
| 18 | +use crate::scalar::ScalarValue; |
| 19 | +use crate::stats::AggregateResults; |
| 20 | +use crate::stats::StatsSet; |
| 21 | + |
| 22 | +/// Convert legacy writer output without reading array values or computing aggregates. |
| 23 | +/// |
| 24 | +/// Only values present in the legacy set are supplied. The legacy set cannot contain nulls. |
| 25 | +pub fn legacy_stats_to_results( |
| 26 | + input_dtype: &DType, |
| 27 | + stats: &StatsSet, |
| 28 | +) -> VortexResult<AggregateResults> { |
| 29 | + let mut entries = Vec::new(); |
| 30 | + for stat in Stat::all() { |
| 31 | + let value = stats.get(stat); |
| 32 | + if value.is_absent() { |
| 33 | + continue; |
| 34 | + } |
| 35 | + let dtype = historical_dtype(stat, input_dtype) |
| 36 | + .ok_or_else(|| vortex_err!("File statistic {stat} does not support {input_dtype}"))?; |
| 37 | + let value = value |
| 38 | + .map(|value| Scalar::try_new(dtype, Some(value))) |
| 39 | + .transpose()?; |
| 40 | + entries.push((stat.finalized_aggregate_fn().clone(), value)); |
| 41 | + } |
| 42 | + Ok(AggregateResults::from_validated(entries)) |
| 43 | +} |
| 44 | + |
| 45 | +/// Decode finalized results using historical field types. |
| 46 | +/// |
| 47 | +/// Present nullable nulls retain their precision. Missing fields are absent. Nulls in non-nullable |
| 48 | +/// fields and invalid precision tags are rejected. Numerical keys use NaN-skipping options. |
| 49 | +pub fn read_summary( |
| 50 | + fb: &fba::ArrayStats<'_>, |
| 51 | + input_dtype: &DType, |
| 52 | + session: &VortexSession, |
| 53 | +) -> VortexResult<AggregateResults> { |
| 54 | + let mut entries = Vec::new(); |
| 55 | + for stat in Stat::all() { |
| 56 | + let Some(dtype) = historical_dtype(stat, input_dtype) else { |
| 57 | + continue; |
| 58 | + }; |
| 59 | + let value = match stat { |
| 60 | + Stat::Min | Stat::Max | Stat::Sum => { |
| 61 | + let bytes = match stat { |
| 62 | + Stat::Min => fb.min(), |
| 63 | + Stat::Max => fb.max(), |
| 64 | + _ => fb.sum(), |
| 65 | + }; |
| 66 | + let Some(bytes) = bytes else { continue }; |
| 67 | + let value = ScalarValue::from_proto_bytes(bytes.bytes(), &dtype, session)?; |
| 68 | + let value = Scalar::try_new(dtype, value)?; |
| 69 | + let precision = match stat { |
| 70 | + Stat::Min => fb.min_precision(), |
| 71 | + Stat::Max => fb.max_precision(), |
| 72 | + _ => fba::Precision::Exact, |
| 73 | + }; |
| 74 | + match precision { |
| 75 | + fba::Precision::Exact => Precision::Exact(value), |
| 76 | + fba::Precision::Inexact => Precision::Inexact(value), |
| 77 | + other => vortex_bail!("Corrupted {stat} precision: {other:?}"), |
| 78 | + } |
| 79 | + } |
| 80 | + Stat::IsConstant | Stat::IsSorted | Stat::IsStrictSorted => { |
| 81 | + let value = match stat { |
| 82 | + Stat::IsConstant => fb.is_constant(), |
| 83 | + Stat::IsSorted => fb.is_sorted(), |
| 84 | + _ => fb.is_strict_sorted(), |
| 85 | + }; |
| 86 | + let Some(value) = value else { continue }; |
| 87 | + Precision::Exact(Scalar::try_new(dtype, Some(value.into()))?) |
| 88 | + } |
| 89 | + Stat::NullCount | Stat::NaNCount | Stat::UncompressedSizeInBytes => { |
| 90 | + let value = match stat { |
| 91 | + Stat::NullCount => fb.null_count(), |
| 92 | + Stat::NaNCount => fb.nan_count(), |
| 93 | + _ => fb.uncompressed_size_in_bytes(), |
| 94 | + }; |
| 95 | + let Some(value) = value else { continue }; |
| 96 | + Precision::Exact(Scalar::try_new(dtype, Some(value.into()))?) |
| 97 | + } |
| 98 | + }; |
| 99 | + entries.push((stat.finalized_aggregate_fn().clone(), value)); |
| 100 | + } |
| 101 | + Ok(AggregateResults::from_validated(entries)) |
| 102 | +} |
| 103 | + |
| 104 | +/// Write finalized results into the existing footer fields. |
| 105 | +/// |
| 106 | +/// Only fixed historical functions and options are supported. Only extrema can be inexact. |
| 107 | +/// Scalar types must match the historical field, allowing a change to root nullability. Nulls |
| 108 | +/// require a nullable historical field. Legacy `StatsSet` decoding omits those null results. |
| 109 | +pub fn write_summary<'fb>( |
| 110 | + results: &AggregateResults, |
| 111 | + input_dtype: &DType, |
| 112 | + fbb: &mut FlatBufferBuilder<'fb>, |
| 113 | +) -> VortexResult<WIPOffset<fba::ArrayStats<'fb>>> { |
| 114 | + for (aggregate, _) in results.iter() { |
| 115 | + vortex_ensure!( |
| 116 | + Stat::all().any(|stat| stat.finalized_aggregate_fn() == aggregate), |
| 117 | + "File footer cannot represent aggregate {aggregate}" |
| 118 | + ); |
| 119 | + } |
| 120 | + |
| 121 | + let mut args = fba::ArrayStatsArgs::default(); |
| 122 | + // Preserve the legacy serializer's scalar-vector order so unchanged writer output has the same |
| 123 | + // bytes. The table's optional numeric fields do not allocate vectors. |
| 124 | + for stat in [ |
| 125 | + Stat::Min, |
| 126 | + Stat::Max, |
| 127 | + Stat::Sum, |
| 128 | + Stat::IsConstant, |
| 129 | + Stat::IsSorted, |
| 130 | + Stat::IsStrictSorted, |
| 131 | + Stat::NullCount, |
| 132 | + Stat::UncompressedSizeInBytes, |
| 133 | + Stat::NaNCount, |
| 134 | + ] { |
| 135 | + let value = results.get_result(stat.finalized_aggregate_fn()); |
| 136 | + let Some(scalar) = value.as_ref().into_inner() else { |
| 137 | + continue; |
| 138 | + }; |
| 139 | + let dtype = historical_dtype(stat, input_dtype) |
| 140 | + .ok_or_else(|| vortex_err!("File statistic {stat} does not support {input_dtype}"))?; |
| 141 | + vortex_ensure!( |
| 142 | + scalar.dtype().as_nullable() == dtype.as_nullable(), |
| 143 | + "File statistic {stat} requires result dtype {dtype}, got {}", |
| 144 | + scalar.dtype() |
| 145 | + ); |
| 146 | + let scalar = Scalar::try_new(dtype, scalar.value().cloned())?; |
| 147 | + vortex_ensure!( |
| 148 | + value.is_exact() || matches!(stat, Stat::Min | Stat::Max), |
| 149 | + "File footer cannot represent an inexact {stat} result" |
| 150 | + ); |
| 151 | + match stat { |
| 152 | + Stat::Min | Stat::Max | Stat::Sum => { |
| 153 | + let bytes = ScalarValue::to_proto_bytes::<Vec<u8>>(scalar.value()); |
| 154 | + let bytes = Some(fbb.create_vector(&bytes)); |
| 155 | + let precision = if value.is_exact() { |
| 156 | + fba::Precision::Exact |
| 157 | + } else { |
| 158 | + fba::Precision::Inexact |
| 159 | + }; |
| 160 | + match stat { |
| 161 | + Stat::Min => { |
| 162 | + args.min = bytes; |
| 163 | + args.min_precision = precision; |
| 164 | + } |
| 165 | + Stat::Max => { |
| 166 | + args.max = bytes; |
| 167 | + args.max_precision = precision; |
| 168 | + } |
| 169 | + _ => args.sum = bytes, |
| 170 | + } |
| 171 | + } |
| 172 | + Stat::IsConstant => args.is_constant = Some(bool::try_from(&scalar)?), |
| 173 | + Stat::IsSorted => args.is_sorted = Some(bool::try_from(&scalar)?), |
| 174 | + Stat::IsStrictSorted => args.is_strict_sorted = Some(bool::try_from(&scalar)?), |
| 175 | + Stat::NullCount => args.null_count = Some(u64::try_from(&scalar)?), |
| 176 | + Stat::NaNCount => args.nan_count = Some(u64::try_from(&scalar)?), |
| 177 | + Stat::UncompressedSizeInBytes => { |
| 178 | + args.uncompressed_size_in_bytes = Some(u64::try_from(&scalar)?); |
| 179 | + } |
| 180 | + } |
| 181 | + } |
| 182 | + Ok(fba::ArrayStats::create(fbb, &args)) |
| 183 | +} |
| 184 | + |
| 185 | +fn historical_dtype(stat: Stat, input_dtype: &DType) -> Option<DType> { |
| 186 | + match stat { |
| 187 | + // These wire fields exist for every input type, even where a current kernel declines. |
| 188 | + Stat::NullCount | Stat::NaNCount | Stat::UncompressedSizeInBytes => Some(PType::U64.into()), |
| 189 | + Stat::Sum => stat.dtype(input_dtype).or_else(|| { |
| 190 | + // Older writers also summed extension storage values. |
| 191 | + if let DType::Extension(ext) = input_dtype { |
| 192 | + historical_dtype(stat, ext.storage_dtype()) |
| 193 | + } else { |
| 194 | + None |
| 195 | + } |
| 196 | + }), |
| 197 | + _ => stat.dtype(input_dtype), |
| 198 | + } |
| 199 | +} |
0 commit comments