Skip to content

Commit 548730b

Browse files
Toby1009Omega359
authored andcommitted
test: strengthen scalar UDF type recovery regression coverage (apache#25967)
## Which issue does this PR close? - Closes apache#25925. ## Rationale for this change The time arithmetic SQL tests added in apache#25668 retain their sorts even when scalar UDF type recovery is removed, because ordinary arithmetic already treats unbounded results as potentially overflowing. Checked arithmetic needs the recovered time type to detect wrapping across midnight, so it provides a regression test for that interaction. ## What changes are included in this PR? - Add a checked `date_trunc('hour', t) + INTERVAL '2 hours'` property test over ordered Time64 input. Assert that truncation preserves ordering before checking that time arithmetic drops it. - Cover an override returning unbounded Null bounds, and separate bounds-preservation tests from error propagation. - Clarify which intervals are replaced or preserved and that direct `PhysicalExpr::evaluate_bounds` calls bypass the fallback. ## What is the testing strategy for this PR? The new checked arithmetic test passes with type recovery and fails with `Ordered` instead of `Unordered` when the fallback is removed. Existing SQL tests are retained. Validation: `cargo clippy --all-targets --all-features -- -D warnings`, `./dev/rust_lint.sh`, the extended workspace test suite, and all 8 scalar-function unit tests. Ablation checks confirm that removing type recovery fails the ordering assertion, and making the truncated child unordered fails the new precondition assertion. ## Are there any user-facing changes? UDF bounds documentation is more precise. Execution behavior and public APIs are unchanged.
1 parent 21c0459 commit 548730b

2 files changed

Lines changed: 125 additions & 57 deletions

File tree

‎datafusion/expr/src/udf.rs‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -910,13 +910,16 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any {
910910
/// The default implementation returns an unbounded [`DataType::Null`]
911911
/// interval because the output type cannot be inferred from input intervals
912912
/// alone. During physical property analysis, `ScalarFunctionExpr` uses its
913-
/// resolved return type to replace this default with a typed unbounded
914-
/// interval when supported. UDFs need not override this method solely to
915-
/// preserve their output type for that analysis.
916-
///
917-
/// Override this method to provide more precise bounds. Explicit bounds
918-
/// are preserved. The fallback does not apply to direct bounds evaluation
919-
/// by the constraint solver.
913+
/// resolved return type to replace any unbounded [`DataType::Null`]
914+
/// interval returned here with a typed unbounded interval when supported,
915+
/// including intervals returned by overrides. If the resolved type is not
916+
/// supported by [`Interval::make_unbounded`], the original interval is kept.
917+
/// UDFs need not override this method solely to preserve their output type
918+
/// for that analysis.
919+
///
920+
/// Override this method to provide more precise bounds. All intervals other
921+
/// than an unbounded [`DataType::Null`] interval are preserved. The fallback
922+
/// does not apply to direct calls to `PhysicalExpr::evaluate_bounds`.
920923
fn evaluate_bounds(&self, _input: &[&Interval]) -> Result<Interval> {
921924
// We cannot assume the input datatype is the same of output type.
922925
Interval::make_unbounded(&DataType::Null)

‎datafusion/physical-expr/src/scalar_function.rs‎

Lines changed: 115 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -322,8 +322,8 @@ impl PhysicalExpr for ScalarFunctionExpr {
322322
.map(|props| &props.range)
323323
.collect::<Vec<_>>();
324324
let range = self.fun().evaluate_bounds(&children_range)?;
325-
// The default UDF bounds carry no type information. Recover the resolved
326-
// output type for property inference without discarding explicit bounds.
325+
// Unbounded Null bounds carry no type information, even from an override.
326+
// Recover the resolved output type while preserving all other intervals.
327327
let range = if range.data_type() == DataType::Null && range.is_unbounded() {
328328
Interval::make_unbounded(self.return_type()).unwrap_or(range)
329329
} else {
@@ -433,7 +433,7 @@ mod tests {
433433
assert_eq!(properties.sort_properties, SortProperties::Unordered);
434434
assert!(!properties.preserves_lex_ordering);
435435
assert!(!properties.strictly_order_preserving);
436-
// The constraint solver calls this method directly, bypassing the fallback.
436+
// Direct bounds evaluation does not use the property-analysis fallback.
437437
assert_eq!(
438438
expr.evaluate_bounds(&[]).unwrap(),
439439
Interval::make_unbounded(&DataType::Null).unwrap()
@@ -527,6 +527,55 @@ mod tests {
527527
}
528528
}
529529

530+
#[test]
531+
fn checked_time_arithmetic_uses_recovered_function_type() -> Result<()> {
532+
use crate::expressions::BinaryExpr;
533+
use arrow::compute::SortOptions;
534+
use datafusion_expr::Operator;
535+
use datafusion_functions::datetime::date_trunc::DateTruncFunc;
536+
537+
let time_type = DataType::Time64(TimeUnit::Nanosecond);
538+
let precision = Arc::new(Literal::new(ScalarValue::from("hour")));
539+
let precision_properties = precision.get_properties(&[])?;
540+
let date_trunc = ScalarFunctionExpr::try_new(
541+
Arc::new(ScalarUDF::from(DateTruncFunc::new())),
542+
vec![precision, Arc::new(Column::new("t", 0))],
543+
&Schema::new(vec![Field::new("t", time_type.clone(), true)]),
544+
Arc::new(ConfigOptions::default()),
545+
)?;
546+
let time_properties = ExprProperties::new_unknown()
547+
.with_order(SortProperties::Ordered(SortOptions::default()))
548+
.with_range(Interval::make_unbounded(&time_type)?);
549+
let truncated_properties =
550+
date_trunc.get_properties(&[precision_properties, time_properties])?;
551+
assert_eq!(
552+
truncated_properties.range,
553+
Interval::make_unbounded(&time_type)?
554+
);
555+
assert_eq!(
556+
truncated_properties.sort_properties,
557+
SortProperties::Ordered(SortOptions::default())
558+
);
559+
let interval = Arc::new(Literal::new(ScalarValue::new_interval_mdn(
560+
0,
561+
0,
562+
7_200_000_000_000,
563+
)));
564+
let interval_properties = interval.get_properties(&[])?;
565+
let shifted = BinaryExpr::new(Arc::new(date_trunc), Operator::Plus, interval)
566+
.with_fail_on_overflow(true);
567+
568+
// Checked arithmetic still wraps time-of-day across midnight. Recovering
569+
// the function's type is necessary to recognize this non-monotonic case.
570+
assert_eq!(
571+
shifted
572+
.get_properties(&[truncated_properties, interval_properties])?
573+
.sort_properties,
574+
SortProperties::Unordered
575+
);
576+
Ok(())
577+
}
578+
530579
#[derive(Debug, PartialEq, Eq, Hash)]
531580
struct BoundsUDF {
532581
inner: MockScalarUDF,
@@ -558,57 +607,73 @@ mod tests {
558607
}
559608
}
560609

610+
fn bounds_expr(fail: bool) -> ScalarFunctionExpr {
611+
ScalarFunctionExpr::try_new(
612+
Arc::new(ScalarUDF::from(BoundsUDF {
613+
inner: MockScalarUDF {
614+
signature: Signature::exact(
615+
vec![DataType::Int32],
616+
Volatility::Immutable,
617+
),
618+
return_type: DataType::Int32,
619+
},
620+
fail,
621+
})),
622+
vec![Arc::new(Column::new("a", 0))],
623+
&Schema::new(vec![Field::new("a", DataType::Int32, true)]),
624+
Arc::new(ConfigOptions::default()),
625+
)
626+
.unwrap()
627+
}
628+
561629
#[test]
562-
fn properties_preserve_udf_bounds_and_errors() {
563-
for fail in [false, true] {
564-
let expr = ScalarFunctionExpr::try_new(
565-
Arc::new(ScalarUDF::from(BoundsUDF {
566-
inner: MockScalarUDF {
567-
signature: Signature::exact(
568-
vec![DataType::Int32],
569-
Volatility::Immutable,
570-
),
571-
return_type: DataType::Int32,
572-
},
573-
fail,
574-
})),
575-
vec![Arc::new(Column::new("a", 0))],
576-
&Schema::new(vec![Field::new("a", DataType::Int32, true)]),
577-
Arc::new(ConfigOptions::default()),
578-
)
579-
.unwrap();
580-
for (lower, upper) in [
581-
(Some(0), Some(100)),
582-
(Some(1), Some(1)),
583-
(None, Some(100)),
584-
(Some(0), None),
585-
(None, None),
586-
] {
587-
let bounds = Interval::make::<i32>(lower, upper).unwrap();
588-
let child = ExprProperties::new_unknown().with_range(bounds.clone());
589-
let properties = expr.get_properties(&[child]);
590-
let direct = expr.evaluate_bounds(&[&bounds]);
591-
if fail {
592-
assert!(
593-
properties
594-
.unwrap_err()
595-
.to_string()
596-
.contains("bounds evaluation failed")
597-
);
598-
assert!(
599-
direct
600-
.unwrap_err()
601-
.to_string()
602-
.contains("bounds evaluation failed")
603-
);
604-
} else {
605-
assert_eq!(properties.unwrap().range, bounds);
606-
assert_eq!(direct.unwrap(), bounds);
607-
}
608-
}
630+
fn properties_preserve_udf_bounds() {
631+
let expr = bounds_expr(false);
632+
for (lower, upper) in [
633+
(Some(0), Some(100)),
634+
(Some(1), Some(1)),
635+
(None, Some(100)),
636+
(Some(0), None),
637+
(None, None),
638+
] {
639+
let bounds = Interval::make::<i32>(lower, upper).unwrap();
640+
let child = ExprProperties::new_unknown().with_range(bounds.clone());
641+
assert_eq!(expr.get_properties(&[child]).unwrap().range, bounds);
642+
assert_eq!(expr.evaluate_bounds(&[&bounds]).unwrap(), bounds);
609643
}
610644
}
611645

646+
#[test]
647+
fn properties_recover_type_from_explicit_unknown_bounds() {
648+
let expr = bounds_expr(false);
649+
let bounds = Interval::make_unbounded(&DataType::Null).unwrap();
650+
let child = ExprProperties::new_unknown().with_range(bounds.clone());
651+
assert_eq!(
652+
expr.get_properties(&[child]).unwrap().range,
653+
Interval::make_unbounded(&DataType::Int32).unwrap()
654+
);
655+
assert_eq!(expr.evaluate_bounds(&[&bounds]).unwrap(), bounds);
656+
}
657+
658+
#[test]
659+
fn properties_propagate_udf_bounds_errors() {
660+
let expr = bounds_expr(true);
661+
let bounds = Interval::make::<i32>(Some(0), Some(100)).unwrap();
662+
let child = ExprProperties::new_unknown().with_range(bounds.clone());
663+
assert!(
664+
expr.get_properties(&[child])
665+
.unwrap_err()
666+
.to_string()
667+
.contains("bounds evaluation failed")
668+
);
669+
assert!(
670+
expr.evaluate_bounds(&[&bounds])
671+
.unwrap_err()
672+
.to_string()
673+
.contains("bounds evaluation failed")
674+
);
675+
}
676+
612677
#[test]
613678
fn test_scalar_function_volatile_node() {
614679
// Create a volatile UDF

0 commit comments

Comments
 (0)