Skip to content

Commit 54c2a59

Browse files
test(querier): pin that a nan in a promoted float column keeps its value (#1278)
Parquet float min/max statistics leave NaN out, so a column chunk of one value plus NaNs looks constant. With collect_statistics on, DataFusion substitutes the statistic for the data and the NaN rows read back as that value (apache/datafusion#26490). The querier keeps statistics off; this test pins the behaviour itself, so a DataFusion upgrade or a session change that brings the substitution back fails. With collect_statistics switched on locally the test fails (`attr.cost_usd < 2.0` counts 3 rows instead of 2) on DataFusion 55.2.0. The session() comment now names both upstream bugs that keep statistics off. Claude-Session: https://claude.ai/code/session_0125HXBoGeZ4wPFs9hDidYSx Signed-off-by: Jens Holdgaard Pedersen <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 2ee9351 commit 54c2a59

3 files changed

Lines changed: 98 additions & 3 deletions

File tree

‎crates/ourios-querier/src/exec.rs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,15 @@ pub(crate) enum SchemaMode<'a> {
4949
/// fast. Collection also costs a per-file footer read at plan time,
5050
/// which a many-file log store pays on every query.
5151
///
52-
/// Remove this override once the upstream fix ships in a release we
53-
/// depend on; the RFC0044.7/.8 and RFC0007.1 pruning tests are the
54-
/// gate that will catch it either way.
52+
/// The same constant substitution also returns wrong values: Parquet
53+
/// float min/max leave NaN out, so a float column of one value plus NaNs
54+
/// reads back as that value (apache/datafusion#26490), which would break
55+
/// OTLP fidelity for promoted `Float64` attributes. The
56+
/// `nan_promoted_float` test pins that.
57+
///
58+
/// Keep this override until both upstream bugs are fixed in a release we
59+
/// depend on (#24769's remaining case is apache/datafusion#27415). The
60+
/// RFC0044.7/.8 and RFC0007.1 pruning tests and the NaN test are the gate.
5561
///
5662
/// Each session also carries a [`crate::fetch_count::FetchCounter`]: the local
5763
/// filesystem it reads through is wrapped to count fetched bytes, as is

‎crates/ourios-querier/tests/it/main.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ mod drift;
1818
mod execution;
1919
mod forward_compat;
2020
mod manifest;
21+
mod nan_promoted_float;
2122
mod query_read_accounting;
2223
mod rfc0001_otlp_fidelity;
2324
mod rfc0001_query_semantics;
Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
//! OTLP fidelity for NaN in a promoted `Float64` attribute column.
2+
//!
3+
//! Parquet float min/max statistics leave NaN out, so a column chunk of
4+
//! one finite value plus NaNs has `min == max` and no nulls. With
5+
//! `execution.collect_statistics` on, `DataFusion` treats that column as
6+
//! constant and returns the statistic for the NaN rows
7+
//! (apache/datafusion#26490). The querier keeps statistics collection off;
8+
//! this test pins the behaviour itself, so a `DataFusion` upgrade or a
9+
//! session change that brings the substitution back fails here.
10+
11+
use ourios_parquet::{PromotedAttributes, PromotedClass, PromotedKey};
12+
13+
use crate::common::{DEFAULT_WINDOW_NS, NOW, TS0, kv, rec_with_attrs, write_all_with_promoted};
14+
use ourios_core::otlp::any_value::Value as AvValue;
15+
use ourios_core::otlp::{AnyValue, KeyValue};
16+
17+
fn cost(value: f64) -> KeyValue {
18+
KeyValue {
19+
key: "cost_usd".to_string(),
20+
value: Some(AnyValue {
21+
value: Some(AvValue::DoubleValue(value)),
22+
}),
23+
..Default::default()
24+
}
25+
}
26+
27+
#[tokio::test]
28+
async fn a_nan_in_a_promoted_float_column_is_never_read_as_the_statistics_constant() {
29+
let bucket = tempfile::TempDir::new().expect("temp");
30+
let records: Vec<_> = [1.5, f64::NAN, 1.5]
31+
.into_iter()
32+
.enumerate()
33+
.map(|(i, value)| {
34+
rec_with_attrs(
35+
"a",
36+
TS0 + i as u64 * 1_000,
37+
vec![kv("service.name", "agent")],
38+
vec![cost(value)],
39+
)
40+
})
41+
.collect();
42+
let promoted = PromotedAttributes::new_typed(
43+
[],
44+
[PromotedKey {
45+
key: "cost_usd".into(),
46+
class: PromotedClass::F64,
47+
}],
48+
);
49+
write_all_with_promoted(bucket.path(), &records, &promoted);
50+
51+
let querier = ourios_querier::Querier::new(bucket.path()).with_promoted_attributes(promoted);
52+
let run = |dsl: &'static str| {
53+
let querier = &querier;
54+
async move {
55+
let query = ourios_querier::dsl::parse(dsl).expect("parse DSL");
56+
querier
57+
.run_query(
58+
&query,
59+
&ourios_core::tenant::TenantId::new("a"),
60+
NOW,
61+
DEFAULT_WINDOW_NS,
62+
Some(&crate::common::no_aliases()),
63+
)
64+
.await
65+
.expect("query")
66+
}
67+
};
68+
69+
// Arrow orders NaN above every number, so `< 2.0` matches the two
70+
// finite rows. Read as the statistic, the NaN row would match too.
71+
let below = run("attr.cost_usd < 2.0 | count").await;
72+
assert_eq!(
73+
below.rows, 2,
74+
"the NaN row must not compare as 1.5; 3 means the column was read as the \
75+
min == max statistic (apache/datafusion#26490)",
76+
);
77+
78+
// A non-finite aggregate is reported as NULL (JSON has no NaN). Read as
79+
// the statistic, the sum would be a finite 4.5.
80+
let sum = run("template_id == 1 | sum(attr.cost_usd)").await;
81+
let group = &sum.aggregate.as_ref().expect("aggregate map")[0];
82+
assert_eq!(group.count, 3);
83+
assert_eq!(
84+
group.value.flatten(),
85+
None,
86+
"the NaN row makes the sum non-finite, reported as NULL",
87+
);
88+
}

0 commit comments

Comments
 (0)