Skip to content

Commit ab8fbb9

Browse files
authored
Merge branch 'main' into qizhu/enable-pushdown-filters-by-default
2 parents c61e14b + 7ff7278 commit ab8fbb9

20 files changed

Lines changed: 1085 additions & 83 deletions

File tree

‎Cargo.lock‎

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/catalog-listing/src/table.rs‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -148,15 +148,15 @@ pub struct ListFilesResult {
148148
/// # use datafusion_datasource_parquet::file_format::ParquetFormat;/// #
149149
/// # use datafusion_catalog::Session;
150150
/// async fn get_listing_table(session: &dyn Session) -> Result<Arc<dyn TableProvider>> {
151-
/// let table_path = "/path/to/parquet";
151+
/// let table_path = "/path/to/parquet";
152152
///
153-
/// // Parse the path
154-
/// let table_path = ListingTableUrl::parse(table_path)?;
153+
/// // Parse the path
154+
/// let table_path = ListingTableUrl::parse(table_path)?;
155155
///
156-
/// // Create default parquet options
157-
/// let file_format = ParquetFormat::new();
158-
/// let listing_options = ListingOptions::new(Arc::new(file_format))
159-
/// .with_file_extension(".parquet");
156+
/// // Create default parquet options
157+
/// let file_format = ParquetFormat::new();
158+
/// let listing_options = ListingOptions::new(Arc::new(file_format))
159+
/// .with_file_extension(".parquet");
160160
///
161161
/// // Resolve the schema
162162
/// let resolved_schema = listing_options
@@ -170,8 +170,8 @@ pub struct ListFilesResult {
170170
/// // Create a new TableProvider
171171
/// let provider = Arc::new(ListingTable::try_new(config)?);
172172
///
173-
/// # Ok(provider)
174-
/// # }
173+
/// Ok(provider)
174+
/// }
175175
/// ```
176176
#[derive(Debug, Clone)]
177177
pub struct ListingTable {

‎datafusion/core/src/dataframe/mod.rs‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2439,7 +2439,7 @@ impl DataFrame {
24392439
}
24402440

24412441
/// Fill null values in specified columns with a given value
2442-
/// If no columns are specified (empty vector), applies to all columns
2442+
/// If no columns are specified (empty slice), applies to all columns
24432443
/// Only fills if the value can be cast to the column's type
24442444
///
24452445
/// # Arguments
@@ -2458,19 +2458,14 @@ impl DataFrame {
24582458
/// .read_csv("tests/data/example.csv", CsvReadOptions::new())
24592459
/// .await?;
24602460
/// // Fill nulls in only columns "a" and "c":
2461-
/// let df = df.fill_null(ScalarValue::from(0), vec!["a".to_owned(), "c".to_owned()])?;
2461+
/// let df = df.fill_null(&ScalarValue::from(0), &["a", "c"])?;
24622462
/// // Fill nulls across all columns:
2463-
/// let df = df.fill_null(ScalarValue::from(0), vec![])?;
2463+
/// let df = df.fill_null(&ScalarValue::from(0), &[])?;
24642464
/// # Ok(())
24652465
/// # }
24662466
/// ```
2467-
#[expect(clippy::needless_pass_by_value)]
2468-
pub fn fill_null(
2469-
&self,
2470-
value: ScalarValue,
2471-
columns: Vec<String>,
2472-
) -> Result<DataFrame> {
2473-
self.fill_columns(&value, &columns, &coalesce(), |_| true)
2467+
pub fn fill_null(&self, value: &ScalarValue, columns: &[&str]) -> Result<DataFrame> {
2468+
self.fill_columns(value, columns, &coalesce(), |_| true)
24742469
}
24752470

24762471
// Helper to find columns from names

‎datafusion/core/src/execution/context/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ where
167167
/// * Create a [`DataFrame`] from a CSV or Parquet data source.
168168
/// * Register a CSV or Parquet data source as a table that can be referenced from a SQL query.
169169
/// * Register a custom data source that can be referenced from a SQL query.
170-
/// * Execution a SQL query
170+
/// * Execute a SQL query
171171
///
172172
/// # Example: DataFrame API
173173
///

‎datafusion/core/tests/dataframe/mod.rs‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6472,11 +6472,8 @@ async fn test_fill_null() -> Result<()> {
64726472

64736473
// Use fill_null to replace nulls on each column.
64746474
let df_filled = df
6475-
.fill_null(ScalarValue::Int32(Some(0)), vec!["a".to_string()])?
6476-
.fill_null(
6477-
ScalarValue::Utf8(Some("default".to_string())),
6478-
vec!["b".to_string()],
6479-
)?;
6475+
.fill_null(&ScalarValue::Int32(Some(0)), &["a"])?
6476+
.fill_null(&ScalarValue::Utf8(Some("default".to_string())), &["b"])?;
64806477

64816478
let results = df_filled.collect().await?;
64826479
assert_snapshot!(
@@ -6502,8 +6499,7 @@ async fn test_fill_null_all_columns() -> Result<()> {
65026499
// Use fill_null to replace nulls on all columns.
65036500
// Only column "b" will be replaced since ScalarValue::Utf8(Some("default".to_string()))
65046501
// can be cast to Utf8.
6505-
let df_filled =
6506-
df.fill_null(ScalarValue::Utf8(Some("default".to_string())), vec![])?;
6502+
let df_filled = df.fill_null(&ScalarValue::Utf8(Some("default".to_string())), &[])?;
65076503

65086504
let results = df_filled.clone().collect().await?;
65096505

@@ -6521,7 +6517,7 @@ async fn test_fill_null_all_columns() -> Result<()> {
65216517
);
65226518

65236519
// Fill column "a" null values with a value that cannot be cast to Int32.
6524-
let df_filled = df_filled.fill_null(ScalarValue::Int32(Some(0)), vec![])?;
6520+
let df_filled = df_filled.fill_null(&ScalarValue::Int32(Some(0)), &[])?;
65256521

65266522
let results = df_filled.collect().await?;
65276523
assert_snapshot!(

‎datafusion/expr/src/predicate_bounds.rs‎

Lines changed: 77 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,10 @@ impl PredicateBoundsEvaluator<'_> {
183183
Expr::BinaryExpr(BinaryExpr { op, .. }) if op.returns_null_on_null() => {
184184
self.is_null_if_any_child_null(expr)
185185
}
186+
// Strict scalar functions return NULL when any argument is NULL.
187+
Expr::ScalarFunction(func) if func.func.is_strict() => {
188+
self.is_null_if_any_child_null(expr)
189+
}
186190
Expr::Alias(_)
187191
| Expr::Cast(_)
188192
| Expr::Like(_)
@@ -235,8 +239,9 @@ mod tests {
235239
use crate::expr::ScalarFunction;
236240
use crate::predicate_bounds::evaluate_bounds;
237241
use crate::{
238-
Expr, binary_expr, col, create_udf, is_false, is_not_false, is_not_null,
239-
is_not_true, is_not_unknown, is_null, is_true, is_unknown, lit, not,
242+
Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, binary_expr, col,
243+
create_udf, is_false, is_not_false, is_not_null, is_not_true, is_not_unknown,
244+
is_null, is_true, is_unknown, lit, not,
240245
};
241246
use arrow::datatypes::{DataType, Field, Schema};
242247
use datafusion_common::{DFSchema, Result, ScalarValue};
@@ -666,6 +671,29 @@ mod tests {
666671
}
667672
}
668673

674+
#[test]
675+
fn evaluate_bounds_strict_udf_is_null_when_child_null() {
676+
let col = col("col");
677+
let strict_func = make_test_udf_expr("strict_test", true, vec![col.clone()]);
678+
let non_strict_func =
679+
make_test_udf_expr("non_strict_test", false, vec![col.clone()]);
680+
let schema = DFSchema::try_from(Schema::new(vec![Field::new(
681+
"col",
682+
DataType::UInt8,
683+
true,
684+
)]))
685+
.unwrap();
686+
687+
assert_eq!(
688+
evaluate_bounds(&is_not_null(strict_func), Some(&col), &schema).unwrap(),
689+
NullableInterval::FALSE,
690+
);
691+
assert_eq!(
692+
evaluate_bounds(&is_not_null(non_strict_func), Some(&col), &schema).unwrap(),
693+
NullableInterval::TRUE_OR_FALSE,
694+
);
695+
}
696+
669697
fn make_scalar_func_expr() -> Expr {
670698
let scalar_func_impl =
671699
|_: &[ColumnarValue]| Ok(ColumnarValue::Scalar(ScalarValue::Null));
@@ -678,4 +706,51 @@ mod tests {
678706
);
679707
Expr::ScalarFunction(ScalarFunction::new_udf(Arc::new(udf), vec![]))
680708
}
709+
710+
#[derive(Debug, PartialEq, Eq, Hash)]
711+
struct TestUdf {
712+
name: &'static str,
713+
signature: Signature,
714+
strict: bool,
715+
}
716+
717+
impl TestUdf {
718+
fn new(name: &'static str, strict: bool) -> Self {
719+
Self {
720+
name,
721+
signature: Signature::uniform(
722+
1,
723+
vec![DataType::UInt8],
724+
Volatility::Immutable,
725+
),
726+
strict,
727+
}
728+
}
729+
}
730+
731+
impl ScalarUDFImpl for TestUdf {
732+
fn name(&self) -> &str {
733+
self.name
734+
}
735+
736+
fn signature(&self) -> &Signature {
737+
&self.signature
738+
}
739+
740+
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
741+
Ok(DataType::UInt8)
742+
}
743+
744+
fn is_strict(&self) -> bool {
745+
self.strict
746+
}
747+
748+
fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
749+
unimplemented!()
750+
}
751+
}
752+
753+
fn make_test_udf_expr(name: &'static str, strict: bool, args: Vec<Expr>) -> Expr {
754+
ScalarUDF::from(TestUdf::new(name, strict)).call(args)
755+
}
681756
}

‎datafusion/expr/src/udf.rs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,14 @@ impl ScalarUDF {
209209
self.inner.aliases()
210210
}
211211

212+
/// Returns true if this function always returns NULL when any argument is
213+
/// NULL.
214+
///
215+
/// See [`ScalarUDFImpl::is_strict`] for more details.
216+
pub fn is_strict(&self) -> bool {
217+
self.inner.is_strict()
218+
}
219+
212220
/// Returns this function's [`Signature`] (what input types are accepted).
213221
///
214222
/// See [`ScalarUDFImpl::signature`] for more details.
@@ -693,6 +701,20 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any {
693701
true
694702
}
695703

704+
/// Returns true if this function always returns NULL when any argument is
705+
/// NULL.
706+
///
707+
/// Strict functions are NULL-propagating: if any argument evaluates to
708+
/// NULL, the function result is guaranteed to be NULL. Optimizer rules can
709+
/// use this property when reasoning about expression nullability and
710+
/// null-rejecting filters.
711+
///
712+
/// Defaults to `false` because user-defined functions may choose to accept
713+
/// NULL inputs and produce non-NULL results.
714+
fn is_strict(&self) -> bool {
715+
false
716+
}
717+
696718
/// Invoke the function returning the appropriate result.
697719
///
698720
/// # Performance
@@ -1103,6 +1125,10 @@ impl ScalarUDFImpl for AliasedScalarUDFImpl {
11031125
self.inner.is_nullable(args, schema)
11041126
}
11051127

1128+
fn is_strict(&self) -> bool {
1129+
self.inner.is_strict()
1130+
}
1131+
11061132
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
11071133
self.inner.invoke_with_args(args)
11081134
}

‎datafusion/functions/src/math/abs.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,10 @@ impl ScalarUDFImpl for AbsFunc {
158158
Ok(arg_types[0].clone())
159159
}
160160

161+
fn is_strict(&self) -> bool {
162+
true
163+
}
164+
161165
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
162166
let args = ColumnarValue::values_to_arrays(&args.args)?;
163167
let [input] = take_function_args(self.name(), args)?;

0 commit comments

Comments
 (0)