Skip to content

Commit 48320dd

Browse files
fix: attach query context to regr_r2's divide-by-zero error
Move ZeroDivisor into welford.rs so regr_r2 shares it with corr, and pass the aggregate's expr_id and the query context registry to Regr. Both ANSI tests now compare the error's query-context fragment with Spark's.
1 parent 3cd480a commit 48320dd

6 files changed

Lines changed: 147 additions & 95 deletions

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3418,6 +3418,8 @@ impl PhysicalPlanner {
34183418
expr.filter_var_by_pair_nulls,
34193419
expr.r2_constant_dependent_is_perfect_fit,
34203420
from_protobuf_eval_mode(expr.eval_mode)?,
3421+
spark_expr.expr_id,
3422+
Arc::clone(&self.query_context_registry),
34213423
));
34223424
Self::create_aggr_func_expr(name, schema, vec![child1, child2], func)
34233425
}

‎native/spark-expr/benches/aggregate_misc.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,8 @@ fn criterion_benchmark(c: &mut Criterion) {
179179
true,
180180
false,
181181
EvalMode::Legacy,
182+
None,
183+
datafusion_comet_spark_expr::create_query_context_map(),
182184
))),
183185
),
184186
];

‎native/spark-expr/src/agg_funcs/correlation.rs‎

Lines changed: 9 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,10 @@ use std::sync::Arc;
2323
use crate::agg_funcs::covariance::{CovarianceAccumulator, CovarianceGroupsAccumulator};
2424
use crate::agg_funcs::stddev::StddevAccumulator;
2525
use crate::agg_funcs::variance::VarianceGroupsAccumulator;
26-
use crate::{divide_by_zero_error, EvalMode, QueryContextMap, SparkErrorWithContext};
26+
use crate::agg_funcs::welford::ZeroDivisor;
27+
use crate::{EvalMode, QueryContextMap};
2728
use arrow::compute::filter;
28-
use datafusion::common::{not_impl_err, DataFusionError, Result, ScalarValue};
29+
use datafusion::common::{not_impl_err, Result, ScalarValue};
2930
use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs};
3031
use datafusion::logical_expr::{
3132
Accumulator, AggregateUDFImpl, EmitTo, GroupsAccumulator, Signature, Volatility,
@@ -38,37 +39,14 @@ use datafusion::physical_expr::expressions::StatsType;
3839
/// we have our own implementation is that DataFusion has UInt64 for state_field `count`,
3940
/// while Spark has Double for count. Also we have added `null_on_divide_by_zero`
4041
/// and `eval_mode` to be consistent with Spark's implementation.
41-
#[derive(Debug)]
42+
#[derive(Debug, PartialEq, Eq, Hash)]
4243
pub struct Correlation {
4344
name: String,
4445
signature: Signature,
4546
null_on_divide_by_zero: bool,
4647
zero_divisor: ZeroDivisor,
4748
}
4849

49-
// Manually implement PartialEq, Eq, and Hash excluding the registry field
50-
impl PartialEq for Correlation {
51-
fn eq(&self, other: &Self) -> bool {
52-
self.name == other.name
53-
&& self.signature == other.signature
54-
&& self.null_on_divide_by_zero == other.null_on_divide_by_zero
55-
&& self.zero_divisor.eval_mode == other.zero_divisor.eval_mode
56-
&& self.zero_divisor.expr_id == other.zero_divisor.expr_id
57-
}
58-
}
59-
60-
impl Eq for Correlation {}
61-
62-
impl std::hash::Hash for Correlation {
63-
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
64-
self.name.hash(state);
65-
self.signature.hash(state);
66-
self.null_on_divide_by_zero.hash(state);
67-
self.zero_divisor.eval_mode.hash(state);
68-
self.zero_divisor.expr_id.hash(state);
69-
}
70-
}
71-
7250
impl Correlation {
7351
pub fn new(
7452
name: impl Into<String>,
@@ -87,38 +65,8 @@ impl Correlation {
8765
Volatility::Immutable,
8866
),
8967
null_on_divide_by_zero,
90-
zero_divisor: ZeroDivisor {
91-
eval_mode,
92-
expr_id,
93-
registry,
94-
},
95-
}
96-
}
97-
}
98-
99-
/// How the `Divide` in Spark's `ck / sqrt(xMk * yMk)` treats a zero denominator.
100-
/// It returns null in LEGACY mode and raises DIVIDE_BY_ZERO, carrying the `corr`
101-
/// call's query context, in ANSI mode.
102-
#[derive(Debug, Clone)]
103-
pub(crate) struct ZeroDivisor {
104-
eval_mode: EvalMode,
105-
expr_id: Option<u64>,
106-
registry: Arc<QueryContextMap>,
107-
}
108-
109-
impl ZeroDivisor {
110-
/// The error to raise for a zero denominator, or `None` to return null.
111-
fn error(&self) -> Option<DataFusionError> {
112-
if self.eval_mode != EvalMode::Ansi {
113-
return None;
68+
zero_divisor: ZeroDivisor::new(eval_mode, expr_id, registry),
11469
}
115-
let error = divide_by_zero_error();
116-
Some(match self.expr_id.and_then(|id| self.registry.get(id)) {
117-
Some(ctx) => {
118-
DataFusionError::External(Box::new(SparkErrorWithContext::with_context(error, ctx)))
119-
}
120-
None => error.into(),
121-
})
12270
}
12371
}
12472

@@ -534,17 +482,14 @@ impl GroupsAccumulator for CorrelationGroupsAccumulator {
534482
#[cfg(test)]
535483
mod groups_tests {
536484
use super::*;
537-
use crate::{create_query_context_map, QueryContext};
485+
use crate::{create_query_context_map, QueryContext, SparkErrorWithContext};
538486
use arrow::array::AsArray;
539487
use arrow::datatypes::Float64Type;
488+
use datafusion::common::DataFusionError;
540489
use datafusion_comet_common::SparkError;
541490

542491
fn zero_divisor(eval_mode: EvalMode) -> ZeroDivisor {
543-
ZeroDivisor {
544-
eval_mode,
545-
expr_id: None,
546-
registry: create_query_context_map(),
547-
}
492+
ZeroDivisor::new(eval_mode, None, create_query_context_map())
548493
}
549494

550495
fn acc(legacy: bool) -> CorrelationGroupsAccumulator {
@@ -793,11 +738,7 @@ mod groups_tests {
793738
7,
794739
QueryContext::new(sql.to_string(), 7, 16, None, None, 1, 7),
795740
);
796-
let zero_divisor = ZeroDivisor {
797-
eval_mode: EvalMode::Ansi,
798-
expr_id: Some(7),
799-
registry,
800-
};
741+
let zero_divisor = ZeroDivisor::new(EvalMode::Ansi, Some(7), registry);
801742
let input: Vec<ArrayRef> = vec![
802743
Arc::new(Float64Array::from(vec![0.0, 1.0, 2.0])),
803744
Arc::new(Float64Array::from(vec![0.1, 0.1, 0.1])),

‎native/spark-expr/src/agg_funcs/regr.rs‎

Lines changed: 61 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ use std::sync::Arc;
4545

4646
use crate::agg_funcs::covariance::CovarianceAccumulator;
4747
use crate::agg_funcs::variance::VarianceAccumulator;
48-
use crate::{divide_by_zero_error, EvalMode};
48+
use crate::agg_funcs::welford::ZeroDivisor;
49+
use crate::{EvalMode, QueryContextMap};
4950

5051
/// The kind of linear-regression statistic to compute.
5152
///
@@ -82,7 +83,7 @@ pub struct Regr {
8283
/// independent variable to `null`. When `false`, those two cases are reversed.
8384
r2_constant_dependent_is_perfect_fit: bool,
8485
/// Only consulted for `R2` when the raw moment product underflows to zero.
85-
eval_mode: EvalMode,
86+
zero_divisor: ZeroDivisor,
8687
}
8788

8889
impl Regr {
@@ -92,6 +93,8 @@ impl Regr {
9293
filter_var_by_pair_nulls: bool,
9394
r2_constant_dependent_is_perfect_fit: bool,
9495
eval_mode: EvalMode,
96+
expr_id: Option<u64>,
97+
registry: Arc<QueryContextMap>,
9598
) -> Self {
9699
Self {
97100
name: name.into(),
@@ -102,7 +105,7 @@ impl Regr {
102105
regr_type,
103106
filter_var_by_pair_nulls,
104107
r2_constant_dependent_is_perfect_fit,
105-
eval_mode,
108+
zero_divisor: ZeroDivisor::new(eval_mode, expr_id, registry),
106109
}
107110
}
108111

@@ -145,7 +148,7 @@ impl AggregateUDFImpl for Regr {
145148
RegrType::SXY => Box::new(RegrCovAccumulator::try_new()?),
146149
RegrType::R2 => Box::new(RegrR2Accumulator::try_new(
147150
self.r2_constant_dependent_is_perfect_fit,
148-
self.eval_mode,
151+
self.zero_divisor.clone(),
149152
)?),
150153
RegrType::Slope => Box::new(RegrLineAccumulator::try_new(
151154
false,
@@ -306,19 +309,19 @@ struct RegrR2Accumulator {
306309
/// When `true` (after SPARK-55969), a constant dependent variable yields
307310
/// `1.0` and a constant independent variable yields `null`; reversed otherwise.
308311
constant_dependent_is_perfect_fit: bool,
309-
eval_mode: EvalMode,
312+
zero_divisor: ZeroDivisor,
310313
}
311314

312315
impl RegrR2Accumulator {
313-
fn try_new(constant_dependent_is_perfect_fit: bool, eval_mode: EvalMode) -> Result<Self> {
316+
fn try_new(constant_dependent_is_perfect_fit: bool, zero_divisor: ZeroDivisor) -> Result<Self> {
314317
Ok(Self {
315318
covar: CovarianceAccumulator::try_new(StatsType::Population, false)?,
316319
var_y: VarianceAccumulator::try_new(StatsType::Population, false)?
317320
.with_pearson_update(),
318321
var_x: VarianceAccumulator::try_new(StatsType::Population, false)?
319322
.with_pearson_update(),
320323
constant_dependent_is_perfect_fit,
321-
eval_mode,
324+
zero_divisor,
322325
})
323326
}
324327
}
@@ -393,10 +396,9 @@ impl Accumulator for RegrR2Accumulator {
393396
// Spark's Divide handles this only after the constant-input guards.
394397
let product = m2_y * m2_x;
395398
if product == 0.0 {
396-
return if self.eval_mode == EvalMode::Ansi {
397-
Err(divide_by_zero_error().into())
398-
} else {
399-
Ok(ScalarValue::Float64(None))
399+
return match self.zero_divisor.error() {
400+
Some(error) => Err(error),
401+
None => Ok(ScalarValue::Float64(None)),
400402
};
401403
}
402404
// Mirror Spark's exact evaluation order (corr = ck / sqrt(m2_y * m2_x);
@@ -516,16 +518,23 @@ impl Accumulator for RegrLineAccumulator {
516518
#[cfg(test)]
517519
mod tests {
518520
use super::*;
521+
use crate::{create_query_context_map, QueryContext, SparkErrorWithContext};
519522
use arrow::array::Float64Array;
520523
use datafusion::common::DataFusionError;
521524
use datafusion_comet_common::SparkError;
522525

526+
fn zero_divisor(eval_mode: EvalMode) -> ZeroDivisor {
527+
ZeroDivisor::new(eval_mode, None, create_query_context_map())
528+
}
529+
523530
fn acc(regr_type: RegrType) -> Box<dyn Accumulator> {
524531
match regr_type {
525532
RegrType::SXX | RegrType::SYY => Box::new(RegrMomentAccumulator::try_new().unwrap()),
526533
RegrType::SXY => Box::new(RegrCovAccumulator::try_new().unwrap()),
527534
// Default to the post-SPARK-55969 degenerate-case semantics.
528-
RegrType::R2 => Box::new(RegrR2Accumulator::try_new(true, EvalMode::Legacy).unwrap()),
535+
RegrType::R2 => {
536+
Box::new(RegrR2Accumulator::try_new(true, zero_divisor(EvalMode::Legacy)).unwrap())
537+
}
529538
// Existing tests exercise the Spark 3.5+ both-non-null semantics.
530539
RegrType::Slope => Box::new(RegrLineAccumulator::try_new(false, true).unwrap()),
531540
RegrType::Intercept => Box::new(RegrLineAccumulator::try_new(true, true).unwrap()),
@@ -636,20 +645,50 @@ mod tests {
636645
}
637646
}
638647

648+
#[test]
649+
fn r2_ansi_error_carries_query_context() {
650+
let sql = "SELECT regr_r2(v, v) FROM t";
651+
let registry = create_query_context_map();
652+
registry.register(
653+
7,
654+
QueryContext::new(sql.to_string(), 7, 19, None, None, 1, 7),
655+
);
656+
for perfect_dep in [false, true] {
657+
let mut a = RegrR2Accumulator::try_new(
658+
perfect_dep,
659+
ZeroDivisor::new(EvalMode::Ansi, Some(7), Arc::clone(&registry)),
660+
)
661+
.unwrap();
662+
let v = vec![Some(1e-100), Some(2e-100)];
663+
a.update_batch(&cols(v.clone(), v)).unwrap();
664+
let DataFusionError::External(error) = a.evaluate().unwrap_err() else {
665+
panic!("expected a Spark error with query context");
666+
};
667+
let error = error.downcast_ref::<SparkErrorWithContext>().unwrap();
668+
assert!(matches!(error.error, SparkError::DivideByZero));
669+
assert_eq!(error.context.as_ref().unwrap().sql_text.as_str(), sql);
670+
}
671+
}
672+
639673
#[test]
640674
fn r2_moment_product_underflow() {
641675
for eval_mode in [EvalMode::Legacy, EvalMode::Ansi] {
642676
for perfect_dep in [false, true] {
643677
for sign in [-1.0, 1.0] {
644678
for merge in [false, true] {
645-
let mut a = RegrR2Accumulator::try_new(perfect_dep, eval_mode).unwrap();
679+
let mut a =
680+
RegrR2Accumulator::try_new(perfect_dep, zero_divisor(eval_mode))
681+
.unwrap();
646682
let y = vec![Some(1e-100), None, Some(2e-100)];
647683
let x = vec![Some(sign * 1e-100), Some(1e200), Some(sign * 2e-100)];
648684
if merge {
649685
// Merge two single-row partials, including an unpaired null.
650686
for range in [0..2, 2..3] {
651-
let mut partial =
652-
RegrR2Accumulator::try_new(perfect_dep, eval_mode).unwrap();
687+
let mut partial = RegrR2Accumulator::try_new(
688+
perfect_dep,
689+
zero_divisor(eval_mode),
690+
)
691+
.unwrap();
653692
partial
654693
.update_batch(&cols(
655694
y[range.clone()].to_vec(),
@@ -678,7 +717,8 @@ mod tests {
678717
fn r2_zero_covariance_with_underflowed_product() {
679718
for eval_mode in [EvalMode::Legacy, EvalMode::Ansi] {
680719
for perfect_dep in [false, true] {
681-
let mut a = RegrR2Accumulator::try_new(perfect_dep, eval_mode).unwrap();
720+
let mut a =
721+
RegrR2Accumulator::try_new(perfect_dep, zero_divisor(eval_mode)).unwrap();
682722
// A zero co-moment still divides by zero, rather than returning NaN.
683723
let state = [4.0, 0.0, 0.0, 0.0, 4e-200, 4e-200]
684724
.into_iter()
@@ -697,7 +737,9 @@ mod tests {
697737
for sign in [-1.0, 1.0] {
698738
for (values, nan) in [(vec![1e100, 2e100], false), (vec![1e200, -1e200], true)]
699739
{
700-
let mut a = RegrR2Accumulator::try_new(perfect_dep, eval_mode).unwrap();
740+
let mut a =
741+
RegrR2Accumulator::try_new(perfect_dep, zero_divisor(eval_mode))
742+
.unwrap();
701743
a.update_batch(&cols(
702744
values.iter().copied().map(Some).collect(),
703745
values.iter().map(|v| Some(sign * v)).collect(),
@@ -739,7 +781,8 @@ mod tests {
739781

740782
for eval_mode in [EvalMode::Legacy, EvalMode::Ansi] {
741783
let r2 = |perfect_dep: bool, y: Vec<Option<f64>>, x: Vec<Option<f64>>| {
742-
let mut a = RegrR2Accumulator::try_new(perfect_dep, eval_mode).unwrap();
784+
let mut a =
785+
RegrR2Accumulator::try_new(perfect_dep, zero_divisor(eval_mode)).unwrap();
743786
a.update_batch(&cols(y, x)).unwrap();
744787
match a.evaluate().unwrap() {
745788
ScalarValue::Float64(v) => v,

0 commit comments

Comments
 (0)