Skip to content

Commit 3f12ae4

Browse files
authored
Preserve extension literals in DataFusion comparisons (#11067)
## Summary - Tracking Issue: #7683. Depends on #11016. DataFusion literals can hold an extension column's Arrow storage value without its extension metadata. Uses the other operand's field metadata to reconstruct Vortex extension scalars, so UUID and JSON filters can still be pushed down with matching dtypes. ## Changes Uses one binary operand conversion path, with field context for comparison literals. Comparison and `IN`-list literals use the registered Arrow importer, preserving explicit literal metadata and timestamp units. Inherited extension names and payloads come from the same field, including when the payload is absent, and import errors retain their original cause. Tests cover literal metadata, including stale payloads, static filters, and dynamic filter updates with residual evaluation. Formatting passes with CI's pinned `nightly-2026-09-10` for `vortex-datafusion`. Tests, Clippy, and benchmarks were not run locally for this layer. ## API Changes Exporting a non-temporal extension scalar as a bare DataFusion `ScalarValue` now returns an error instead of discarding its extension type. Scalar statistics that cannot preserve the extension type are omitted. --------- Signed-off-by: Connor Tsui <[email protected]>
1 parent dafc832 commit 3f12ae4

7 files changed

Lines changed: 519 additions & 46 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vortex-datafusion/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ tempfile = { workspace = true }
4848
tokio = { workspace = true, features = ["test-util", "rt-multi-thread", "fs"] }
4949
url = { workspace = true }
5050
vortex-array = { workspace = true, features = ["_test-harness"] }
51+
vortex-json = { workspace = true }
5152

5253
[lints]
5354
workspace = true

‎vortex-datafusion/src/convert/exprs/mod.rs‎

Lines changed: 113 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,19 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4+
//! Converts DataFusion physical expressions for native Vortex evaluation.
5+
//!
6+
//! Unsupported expressions stay in DataFusion. Comparison and membership literals retain their own
7+
//! extension metadata, or inherit missing metadata from the operand they are compared with. This
8+
//! supplies the logical dtype that Vortex requires without adding implicit casts.
9+
410
use std::sync::Arc;
511

612
use arrow_schema::DataType;
713
use arrow_schema::Field;
814
use arrow_schema::Schema;
15+
use arrow_schema::extension::EXTENSION_TYPE_METADATA_KEY;
16+
use arrow_schema::extension::EXTENSION_TYPE_NAME_KEY;
917
use datafusion_common::DataFusionError;
1018
use datafusion_common::Result as DFResult;
1119
use datafusion_common::ScalarValue;
@@ -39,18 +47,15 @@ use vortex::expr::nested_case_when;
3947
use vortex::expr::not;
4048
use vortex::expr::pack;
4149
use vortex::expr::root;
50+
use vortex::scalar::Scalar;
4251
use vortex::scalar_fn::ScalarFnVTableExt;
4352
use vortex::scalar_fn::fns::binary::Binary;
4453
use vortex::scalar_fn::fns::like::Like;
4554
use vortex::scalar_fn::fns::like::LikeOptions;
46-
use vortex::scalar_fn::fns::literal::Literal;
4755
use vortex::scalar_fn::fns::operators::Operator;
4856
use vortex::session::VortexSession;
4957
use vortex_arrow::ArrowSessionExt;
5058

51-
#[cfg(test)]
52-
mod tests;
53-
5459
/// Result of splitting a projection into Vortex expressions and leftover DataFusion projections.
5560
pub struct ProcessedProjection {
5661
/// Projection evaluated by the Vortex scan.
@@ -309,8 +314,21 @@ impl DefaultExpressionConverter {
309314
} else if !supported_data_types(&left_type) || !supported_data_types(&right_type) {
310315
return Err(Unconverted::Unsupported);
311316
}
312-
let left = self.convert_expr(binary_expr.left(), schema, input_dtype)?;
313-
let right = self.convert_expr(binary_expr.right(), schema, input_dtype)?;
317+
let left = self.convert_binary_operand(
318+
binary_expr.left(),
319+
binary_expr.right(),
320+
operator,
321+
schema,
322+
input_dtype,
323+
)?;
324+
let right = self.convert_binary_operand(
325+
binary_expr.right(),
326+
binary_expr.left(),
327+
operator,
328+
schema,
329+
input_dtype,
330+
)?;
331+
314332
// DataFusion may evaluate the RHS only on rows selected by the LHS.
315333
if boolean_operator && !(is_infallible(&left) && is_infallible(&right)) {
316334
return Err(Unconverted::Unsupported);
@@ -329,23 +347,8 @@ impl DefaultExpressionConverter {
329347

330348
if let Some(literal) = expr.downcast_ref::<df_expr::Literal>() {
331349
let field = literal.return_field(schema)?;
332-
let array = literal.value().to_array()?;
333-
if array.len() != 1 {
334-
return Err(exec_datafusion_err!(
335-
"Literal must contain exactly one value, found {}",
336-
array.len()
337-
)
338-
.into());
339-
}
340-
// Literals of unknown Arrow types stay in DataFusion; conversion failures are errors.
341-
self.arrow_dtype(&field)?;
342-
let scalar = self
343-
.session
344-
.arrow()
345-
.from_arrow_array(array, &field)
346-
.and_then(|array| array.execute_scalar(0, &mut self.session.create_execution_ctx()))
347-
.map_err(|e| exec_datafusion_err!("Failed to convert literal {expr}: {e}"))?;
348-
return Ok(lit(scalar));
350+
351+
return Ok(lit(self.import_literal(literal, &field)?));
349352
}
350353

351354
if let Some(cast_expr) = expr.downcast_ref::<df_expr::CastExpr>() {
@@ -427,6 +430,82 @@ impl DefaultExpressionConverter {
427430
Err(Unconverted::Unsupported)
428431
}
429432

433+
/// Converts a binary operand, using the other operand's field to type comparison literals.
434+
///
435+
/// Arithmetic and Boolean operands convert independently. They do not accept extension dtypes,
436+
/// so the other operand cannot supply an expected extension type for their literals.
437+
fn convert_binary_operand(
438+
&self,
439+
expr: &Arc<dyn PhysicalExpr>,
440+
other: &Arc<dyn PhysicalExpr>,
441+
operator: Operator,
442+
schema: &Schema,
443+
input_dtype: &DType,
444+
) -> Conversion<Expression> {
445+
if operator.is_comparison()
446+
&& let Some(literal) = expr.downcast_ref::<df_expr::Literal>()
447+
{
448+
let field = other.return_field(schema)?;
449+
let scalar = self.import_literal_for_operand(literal, &field, schema)?;
450+
451+
return Ok(lit(scalar));
452+
}
453+
454+
self.convert_expr(expr, schema, input_dtype)
455+
}
456+
457+
/// Imports a literal, using `operand_field` only when the literal has no extension metadata.
458+
///
459+
/// The literal keeps its Arrow dtype, so different storage types or timestamp units still fail
460+
/// the comparison's dtype check. Explicit metadata, including an unknown extension, is preserved.
461+
/// The inherited extension name and payload both come from `operand_field`, including an absent
462+
/// payload, so the import cannot combine metadata from two different types.
463+
fn import_literal_for_operand(
464+
&self,
465+
literal: &df_expr::Literal,
466+
operand_field: &Field,
467+
schema: &Schema,
468+
) -> Conversion<Scalar> {
469+
let field = literal.return_field(schema)?;
470+
if field.extension_type_name().is_some() || operand_field.extension_type_name().is_none() {
471+
return self.import_literal(literal, &field);
472+
}
473+
474+
let mut field = field.as_ref().clone();
475+
for key in [EXTENSION_TYPE_NAME_KEY, EXTENSION_TYPE_METADATA_KEY] {
476+
field.metadata_mut().remove(key);
477+
if let Some(value) = operand_field.metadata().get(key) {
478+
field.metadata_mut().insert(key.into(), value.clone());
479+
}
480+
}
481+
482+
self.import_literal(literal, &field)
483+
}
484+
485+
/// Imports one scalar through the session's Arrow plugins, preserving `field`'s metadata.
486+
fn import_literal(&self, literal: &df_expr::Literal, field: &Field) -> Conversion<Scalar> {
487+
let array = literal.value().to_array()?;
488+
if array.len() != 1 {
489+
return Err(exec_datafusion_err!(
490+
"Literal must contain exactly one value, got {}",
491+
array.len()
492+
)
493+
.into());
494+
}
495+
496+
self.arrow_dtype(field)?;
497+
498+
self.session
499+
.arrow()
500+
.from_arrow_array(array, field)
501+
.and_then(|array| array.execute_scalar(0, &mut self.session.create_execution_ctx()))
502+
.map_err(|error| {
503+
Unconverted::Failed(DataFusionError::External(Box::new(
504+
error.with_context(format!("Failed to convert literal {literal}")),
505+
)))
506+
})
507+
}
508+
430509
fn convert_in_list(
431510
&self,
432511
in_list: &df_expr::InListExpr,
@@ -448,23 +527,20 @@ impl DefaultExpressionConverter {
448527
return Err(Unconverted::Unsupported);
449528
}
450529
let value_dtype = converted_dtype(&value, input_dtype)?;
530+
let value_field = in_list.expr().return_field(schema)?;
451531

452532
// The list of a hash join filter holds every build-side key, which repeat. Each element
453-
// converts through Arrow, so equal elements are converted once.
533+
// converts through Arrow, so equal values with equal metadata are converted once.
454534
let elements = in_list
455535
.list()
456536
.iter()
457-
.unique_by(|element| {
458-
element
459-
.downcast_ref::<df_expr::Literal>()
460-
.map(|literal| literal.value().clone())
461-
})
537+
.unique_by(|element| element.downcast_ref::<df_expr::Literal>().cloned())
462538
.map(|element| {
463-
let element = self.convert_expr(element, schema, input_dtype)?;
464-
element
465-
.as_opt::<Literal>()
466-
.cloned()
467-
.ok_or(Unconverted::Unsupported)
539+
let literal = element
540+
.downcast_ref::<df_expr::Literal>()
541+
.ok_or(Unconverted::Unsupported)?;
542+
543+
self.import_literal_for_operand(literal, &value_field, schema)
468544
})
469545
.collect::<Conversion<Vec<_>>>()?;
470546

@@ -747,3 +823,6 @@ fn array_length_input(scalar_fn: &ScalarFunctionExpr) -> DFResult<Option<&Arc<dy
747823
)),
748824
}
749825
}
826+
827+
#[cfg(test)]
828+
mod tests;

‎vortex-datafusion/src/convert/exprs/tests.rs‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ use arrow_schema::Field;
1919
use arrow_schema::IntervalUnit;
2020
use arrow_schema::Schema;
2121
use arrow_schema::TimeUnit as ArrowTimeUnit;
22+
use arrow_schema::extension::EXTENSION_TYPE_METADATA_KEY;
23+
use arrow_schema::extension::EXTENSION_TYPE_NAME_KEY;
2224
use arrow_schema::extension::Uuid as ArrowUuid;
2325
use datafusion::common::DFSchema;
2426
use datafusion::execution::context::ExecutionProps;
@@ -739,6 +741,96 @@ fn test_literal_preserves_extension_metadata(
739741
Ok(())
740742
}
741743

744+
#[rstest]
745+
#[case::comparison(false)]
746+
#[case::membership(true)]
747+
fn test_comparison_literal_metadata_is_not_replaced(
748+
#[case] membership: bool,
749+
) -> anyhow::Result<()> {
750+
let mut field = Field::new("id", DataType::FixedSizeBinary(16), true);
751+
field.try_with_extension_type(ArrowUuid)?;
752+
let schema = Schema::new(vec![field.clone()]);
753+
let value = ScalarValue::FixedSizeBinary(16, Some(b"0123456789abcdef".to_vec()));
754+
let valid = Arc::new(df_expr::Literal::new_with_metadata(
755+
value.clone(),
756+
Some(FieldMetadata::new_from_field(&field)),
757+
)) as Arc<dyn PhysicalExpr>;
758+
field.metadata_mut().insert(
759+
"ARROW:extension:name".into(),
760+
"unregistered.extension".into(),
761+
);
762+
let invalid = Arc::new(df_expr::Literal::new_with_metadata(
763+
value,
764+
Some(FieldMetadata::new_from_field(&field)),
765+
)) as Arc<dyn PhysicalExpr>;
766+
let column = Arc::new(df_expr::Column::new("id", 0)) as Arc<dyn PhysicalExpr>;
767+
let predicate = if membership {
768+
Arc::new(df_expr::InListExpr::try_new(
769+
column,
770+
vec![valid, invalid],
771+
false,
772+
&schema,
773+
)?) as Arc<dyn PhysicalExpr>
774+
} else {
775+
Arc::new(df_expr::BinaryExpr::new(column, DFOperator::Eq, invalid))
776+
};
777+
778+
assert!(!converts(&predicate, &schema)?);
779+
780+
Ok(())
781+
}
782+
783+
#[test]
784+
fn test_comparison_inherits_complete_extension_metadata() -> anyhow::Result<()> {
785+
let mut field = Field::new("id", DataType::FixedSizeBinary(16), false);
786+
field.try_with_extension_type(ArrowUuid)?;
787+
let bytes = *b"0123456789abcdef";
788+
let values = FixedSizeBinaryArray::try_from_iter([bytes].into_iter())?;
789+
let batch = RecordBatch::try_new(
790+
Arc::new(Schema::new(vec![field.clone()])),
791+
vec![Arc::new(values)],
792+
)?;
793+
794+
let mut literal_field = field;
795+
literal_field.metadata_mut().remove(EXTENSION_TYPE_NAME_KEY);
796+
literal_field.metadata_mut().insert(
797+
EXTENSION_TYPE_METADATA_KEY.into(),
798+
"unrelated payload".into(),
799+
);
800+
let literal = df_expr::Literal::new_with_metadata(
801+
ScalarValue::FixedSizeBinary(16, Some(bytes.to_vec())),
802+
Some(FieldMetadata::new_from_field(&literal_field)),
803+
);
804+
let predicate = Arc::new(df_expr::BinaryExpr::new(
805+
Arc::new(df_expr::Column::new("id", 0)),
806+
DFOperator::Eq,
807+
Arc::new(literal),
808+
));
809+
810+
assert_native_matches(predicate, batch)
811+
}
812+
813+
#[test]
814+
fn test_comparison_literal_keeps_timestamp_units() -> DFResult<()> {
815+
let schema = Schema::new(vec![Field::new(
816+
"ts",
817+
DataType::Timestamp(ArrowTimeUnit::Millisecond, None),
818+
true,
819+
)]);
820+
let predicate = Arc::new(df_expr::BinaryExpr::new(
821+
Arc::new(df_expr::Column::new("ts", 0)),
822+
DFOperator::Eq,
823+
Arc::new(df_expr::Literal::new(ScalarValue::TimestampNanosecond(
824+
Some(1_000_000),
825+
None,
826+
))),
827+
)) as Arc<dyn PhysicalExpr>;
828+
829+
assert!(!converts(&predicate, &schema)?);
830+
831+
Ok(())
832+
}
833+
742834
#[test]
743835
fn test_expr_from_df_binary() -> DFResult<()> {
744836
let left = Arc::new(df_expr::Column::new("left", 0)) as Arc<dyn PhysicalExpr>;

0 commit comments

Comments
 (0)