Skip to content

Commit 9ab88ff

Browse files
adivaishCopilot
andcommitted
feat(delta): data-filter pushdown for native Delta scan
Pushes the supported subset of a Delta scan's data filters into delta-kernel-rs for best-effort file-level data skipping, and lets filtered queries run natively: - New native delta::predicate module translates Comet's serialized Spark filter exprs (eq/neq/gt/gt_eq/lt/lt_eq, and/or/not, is null / is not null over a column and a typed literal) into a kernel Predicate. Translation is exact and conservative: any unsupported expr or type-mismatched literal is dropped (never mistranslated), so a Filter above the scan always enforces correctness — only skipping is affected. - DeltaScan proto carries data_filters; CometScanRule serializes scanExec.dataFilters as unbound (by-name) references (binding=false, like Iceberg) and no longer bails on data filters. It still bails on partition filters (consumed by pruning with no Filter above). - DeltaScanExec translates the filters to a kernel predicate and passes it to ScanBuilder.with_predicate. Tests: 6 native unit tests for the translator (comparisons, and, is null/not null, dropping unsupported filters, rejecting type-mismatched literals); 2 e2e tests — a multi-file table with a selective filter (a skipping bug would drop rows and fail checkSparkAnswer) and a projection+filter where the filter column is outside the projection. Validated end to end on Spark 4.1 + Delta 4.1.0 (7/7 suite tests pass; plan shows CometFilterExec -> CometDeltaNativeScanExec); native delta 13/13. Part of apache#174. Co-authored-by: Copilot <[email protected]>
1 parent dc35e97 commit 9ab88ff

7 files changed

Lines changed: 355 additions & 33 deletions

File tree

‎native/core/src/delta/mod.rs‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ use delta_kernel::{DeltaResult, Error, Snapshot, SnapshotRef};
4646
use itertools::Itertools;
4747
use url::Url;
4848

49+
mod predicate;
4950
mod scan_exec;
5051
pub use scan_exec::DeltaScanExec;
5152

@@ -132,18 +133,21 @@ fn projected_read_schema(
132133

133134
/// Read the latest snapshot of the Delta table at `table_uri` into Arrow `RecordBatch`es. When
134135
/// `columns` is `Some`, only those columns are read, in the given order (column pruning); `None`
135-
/// reads the full schema. The kernel applies deletion vectors and column-mapping transforms and
136-
/// injects partition values, so the returned batches are logically correct, Spark-compatible Arrow
137-
/// data.
136+
/// reads the full schema. When `predicate` is `Some`, it is pushed into the kernel for best-effort
137+
/// file-level data skipping (a Filter above the scan still enforces exact correctness). The kernel
138+
/// applies deletion vectors and column-mapping transforms and injects partition values, so the
139+
/// returned batches are logically correct, Spark-compatible Arrow data.
138140
pub fn scan_to_batches(
139141
table_uri: &str,
140142
columns: Option<&[String]>,
143+
predicate: Option<delta_kernel::expressions::PredicateRef>,
141144
) -> DeltaResult<Vec<RecordBatch>> {
142145
let (snapshot, engine) = open_snapshot(table_uri)?;
143146
let read_schema = projected_read_schema(&snapshot, columns)?;
144147
let scan = snapshot
145148
.scan_builder()
146149
.with_schema_opt(read_schema)
150+
.with_predicate(predicate)
147151
.build()?;
148152
let batches: Vec<RecordBatch> = scan
149153
.execute(Arc::new(engine))?
@@ -270,7 +274,7 @@ mod tests {
270274
fn opening_a_missing_table_returns_err() {
271275
assert!(snapshot_summary("file:///definitely/not/a/delta/table").is_err());
272276
assert!(list_scan_files("file:///definitely/not/a/delta/table").is_err());
273-
assert!(scan_to_batches("file:///definitely/not/a/delta/table", None).is_err());
277+
assert!(scan_to_batches("file:///definitely/not/a/delta/table", None, None).is_err());
274278
}
275279

276280
#[test]
@@ -295,7 +299,7 @@ mod tests {
295299
"expected two files"
296300
);
297301

298-
let batches = scan_to_batches(&uri, None).unwrap();
302+
let batches = scan_to_batches(&uri, None, None).unwrap();
299303
assert_eq!(total_rows(&batches), 8);
300304

301305
// verify the actual values round-trip
@@ -322,7 +326,7 @@ mod tests {
322326

323327
// No data files, no rows, but the schema is readable.
324328
assert_eq!(list_scan_files(&uri).unwrap().len(), 0);
325-
assert_eq!(total_rows(&scan_to_batches(&uri, None).unwrap()), 0);
329+
assert_eq!(total_rows(&scan_to_batches(&uri, None, None).unwrap()), 0);
326330
let summary = snapshot_summary(&uri).unwrap();
327331
assert!(summary.contains("version=0"), "summary was: {summary}");
328332
assert!(summary.contains("id") && summary.contains("score"));
@@ -339,14 +343,14 @@ mod tests {
339343

340344
// Project a single column.
341345
let only_score = vec!["score".to_string()];
342-
let batches = scan_to_batches(&uri, Some(&only_score)).unwrap();
346+
let batches = scan_to_batches(&uri, Some(&only_score), None).unwrap();
343347
assert_eq!(total_rows(&batches), 4);
344348
assert_eq!(batches[0].num_columns(), 1);
345349
assert_eq!(batches[0].schema().field(0).name(), "score");
346350

347351
// Project both columns in reverse order; the output must follow the requested order.
348352
let reordered = vec!["score".to_string(), "id".to_string()];
349-
let batches = scan_to_batches(&uri, Some(&reordered)).unwrap();
353+
let batches = scan_to_batches(&uri, Some(&reordered), None).unwrap();
350354
assert_eq!(batches[0].num_columns(), 2);
351355
assert_eq!(batches[0].schema().field(0).name(), "score");
352356
assert_eq!(batches[0].schema().field(1).name(), "id");
@@ -358,7 +362,7 @@ mod tests {
358362

359363
// An unknown column is an error.
360364
let missing = vec!["does_not_exist".to_string()];
361-
assert!(scan_to_batches(&uri, Some(&missing)).is_err());
365+
assert!(scan_to_batches(&uri, Some(&missing), None).is_err());
362366
}
363367

364368
#[test]
@@ -378,16 +382,16 @@ mod tests {
378382
let write_ms = write_start.elapsed().as_secs_f64() * 1000.0;
379383

380384
// warm up, then measure the full read
381-
let _ = scan_to_batches(&uri, None).unwrap();
385+
let _ = scan_to_batches(&uri, None, None).unwrap();
382386
let read_start = Instant::now();
383-
let batches = scan_to_batches(&uri, None).unwrap();
387+
let batches = scan_to_batches(&uri, None, None).unwrap();
384388
let read_secs = read_start.elapsed().as_secs_f64();
385389

386390
// measure a projected read of a single column (id) to show column-pruning speedup
387391
let one_col = vec!["id".to_string()];
388-
let _ = scan_to_batches(&uri, Some(&one_col)).unwrap();
392+
let _ = scan_to_batches(&uri, Some(&one_col), None).unwrap();
389393
let proj_start = Instant::now();
390-
let proj_batches = scan_to_batches(&uri, Some(&one_col)).unwrap();
394+
let proj_batches = scan_to_batches(&uri, Some(&one_col), None).unwrap();
391395
let proj_secs = proj_start.elapsed().as_secs_f64();
392396
assert_eq!(proj_batches[0].num_columns(), 1);
393397

‎native/core/src/delta/predicate.rs‎

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! Translate Comet's serialized Spark data filters into delta-kernel predicates for file-level
19+
//! data skipping.
20+
//!
21+
//! Translation is deliberately conservative: only an exact, well-understood subset is converted,
22+
//! and anything else yields `None` and is simply not pushed. Data skipping is best-effort and a
23+
//! `Filter` above the scan still enforces correctness, so dropping a filter only costs skipping —
24+
//! never correctness. A *wrong* predicate, by contrast, could skip files that hold matching rows,
25+
//! so each mapping below is exact (operator and literal type both checked).
26+
27+
use std::sync::Arc;
28+
29+
use datafusion_comet_proto::spark_expression::{self, expr::ExprStruct, literal::Value};
30+
use delta_kernel::expressions::{Expression, Predicate, PredicateRef, Scalar};
31+
32+
/// Spark `DataTypeId` ids (see `types.proto`) used to validate that a literal's declared type
33+
/// matches the value we read, so e.g. a date-as-int is never treated as a plain integer.
34+
mod type_id {
35+
pub const BOOL: i32 = 0;
36+
pub const INT8: i32 = 1;
37+
pub const INT16: i32 = 2;
38+
pub const INT32: i32 = 3;
39+
pub const INT64: i32 = 4;
40+
pub const FLOAT: i32 = 5;
41+
pub const DOUBLE: i32 = 6;
42+
pub const STRING: i32 = 7;
43+
pub const DATE: i32 = 12;
44+
}
45+
46+
/// Translate a list of AND-ed Spark data filters into a single kernel predicate, dropping any
47+
/// conjunct that is not in the supported subset. Returns `None` when nothing is translatable.
48+
pub(crate) fn translate_filters(filters: &[spark_expression::Expr]) -> Option<PredicateRef> {
49+
let preds: Vec<Predicate> = filters.iter().filter_map(expr_to_predicate).collect();
50+
if preds.is_empty() {
51+
None
52+
} else {
53+
Some(Arc::new(Predicate::and_from(preds)))
54+
}
55+
}
56+
57+
fn binary_operands(b: &spark_expression::BinaryExpr) -> Option<(Expression, Expression)> {
58+
let left = expr_to_expression(b.left.as_deref()?)?;
59+
let right = expr_to_expression(b.right.as_deref()?)?;
60+
Some((left, right))
61+
}
62+
63+
fn expr_to_predicate(expr: &spark_expression::Expr) -> Option<Predicate> {
64+
match expr.expr_struct.as_ref()? {
65+
ExprStruct::Eq(b) => binary_operands(b).map(|(l, r)| Predicate::eq(l, r)),
66+
ExprStruct::Neq(b) => binary_operands(b).map(|(l, r)| Predicate::ne(l, r)),
67+
ExprStruct::Gt(b) => binary_operands(b).map(|(l, r)| Predicate::gt(l, r)),
68+
ExprStruct::GtEq(b) => binary_operands(b).map(|(l, r)| Predicate::ge(l, r)),
69+
ExprStruct::Lt(b) => binary_operands(b).map(|(l, r)| Predicate::lt(l, r)),
70+
ExprStruct::LtEq(b) => binary_operands(b).map(|(l, r)| Predicate::le(l, r)),
71+
ExprStruct::And(b) => {
72+
let left = expr_to_predicate(b.left.as_deref()?)?;
73+
let right = expr_to_predicate(b.right.as_deref()?)?;
74+
Some(Predicate::and(left, right))
75+
}
76+
ExprStruct::Or(b) => {
77+
let left = expr_to_predicate(b.left.as_deref()?)?;
78+
let right = expr_to_predicate(b.right.as_deref()?)?;
79+
Some(Predicate::or(left, right))
80+
}
81+
ExprStruct::Not(u) => Some(Predicate::not(expr_to_predicate(u.child.as_deref()?)?)),
82+
ExprStruct::IsNull(u) => Some(Predicate::is_null(expr_to_expression(u.child.as_deref()?)?)),
83+
ExprStruct::IsNotNull(u) => Some(Predicate::is_not_null(expr_to_expression(
84+
u.child.as_deref()?,
85+
)?)),
86+
_ => None,
87+
}
88+
}
89+
90+
fn expr_to_expression(expr: &spark_expression::Expr) -> Option<Expression> {
91+
match expr.expr_struct.as_ref()? {
92+
// Comet serializes scan filters with `binding = false`, so columns arrive as unbound
93+
// (name) references.
94+
ExprStruct::Unbound(unbound) => Some(Expression::column([unbound.name.clone()])),
95+
ExprStruct::Literal(literal) => literal_to_scalar(literal).map(Expression::literal),
96+
_ => None,
97+
}
98+
}
99+
100+
fn literal_to_scalar(literal: &spark_expression::Literal) -> Option<Scalar> {
101+
if literal.is_null {
102+
return None;
103+
}
104+
let declared = literal.datatype.as_ref().map(|d| d.type_id)?;
105+
match (literal.value.as_ref()?, declared) {
106+
(Value::BoolVal(v), type_id::BOOL) => Some(Scalar::Boolean(*v)),
107+
(Value::ByteVal(v), type_id::INT8) => Some(Scalar::Byte(*v as i8)),
108+
(Value::ShortVal(v), type_id::INT16) => Some(Scalar::Short(*v as i16)),
109+
(Value::IntVal(v), type_id::INT32) => Some(Scalar::Integer(*v)),
110+
(Value::IntVal(v), type_id::DATE) => Some(Scalar::Date(*v)),
111+
(Value::LongVal(v), type_id::INT64) => Some(Scalar::Long(*v)),
112+
(Value::FloatVal(v), type_id::FLOAT) => Some(Scalar::Float(*v)),
113+
(Value::DoubleVal(v), type_id::DOUBLE) => Some(Scalar::Double(*v)),
114+
(Value::StringVal(v), type_id::STRING) => Some(Scalar::String(v.clone())),
115+
_ => None,
116+
}
117+
}
118+
119+
#[cfg(test)]
120+
mod tests {
121+
use super::*;
122+
use datafusion_comet_proto::spark_expression::{
123+
BinaryExpr, DataType, Expr, Literal, UnaryExpr, UnboundReference,
124+
};
125+
126+
fn expr(s: ExprStruct) -> Expr {
127+
Expr {
128+
expr_struct: Some(s),
129+
..Default::default()
130+
}
131+
}
132+
133+
fn col(name: &str) -> Expr {
134+
expr(ExprStruct::Unbound(UnboundReference {
135+
name: name.to_string(),
136+
datatype: None,
137+
}))
138+
}
139+
140+
fn lit_int(v: i32) -> Expr {
141+
lit_int_typed(v, type_id::INT32)
142+
}
143+
144+
fn lit_int_typed(v: i32, type_id: i32) -> Expr {
145+
expr(ExprStruct::Literal(Literal {
146+
value: Some(Value::IntVal(v)),
147+
datatype: Some(DataType {
148+
type_id,
149+
type_info: None,
150+
}),
151+
is_null: false,
152+
}))
153+
}
154+
155+
fn binary(left: Expr, right: Expr) -> Box<BinaryExpr> {
156+
Box::new(BinaryExpr {
157+
left: Some(Box::new(left)),
158+
right: Some(Box::new(right)),
159+
})
160+
}
161+
162+
fn unary(child: Expr) -> Box<UnaryExpr> {
163+
Box::new(UnaryExpr {
164+
child: Some(Box::new(child)),
165+
})
166+
}
167+
168+
#[test]
169+
fn translates_simple_comparison() {
170+
let gt = expr(ExprStruct::Gt(binary(col("id"), lit_int(10))));
171+
let pred = translate_filters(&[gt]).expect("should translate");
172+
// id > 10
173+
assert!(matches!(pred.as_ref(), Predicate::Binary(_)));
174+
}
175+
176+
#[test]
177+
fn translates_and_of_two_comparisons() {
178+
let lt = expr(ExprStruct::Lt(binary(col("id"), lit_int(100))));
179+
let gt = expr(ExprStruct::Gt(binary(col("id"), lit_int(0))));
180+
let and = expr(ExprStruct::And(binary(lt, gt)));
181+
assert!(translate_filters(&[and]).is_some());
182+
}
183+
184+
#[test]
185+
fn translates_is_null_and_is_not_null() {
186+
let isnull = expr(ExprStruct::IsNull(unary(col("id"))));
187+
let isnotnull = expr(ExprStruct::IsNotNull(unary(col("score"))));
188+
assert!(translate_filters(&[isnull, isnotnull]).is_some());
189+
}
190+
191+
#[test]
192+
fn drops_unsupported_filter_but_keeps_supported_ones() {
193+
// A `like` filter (unsupported) plus a supported comparison: only the comparison is pushed.
194+
let like = expr(ExprStruct::Like(binary(col("name"), lit_int(1))));
195+
let gt = expr(ExprStruct::Gt(binary(col("id"), lit_int(5))));
196+
assert!(translate_filters(&[like.clone(), gt]).is_some());
197+
// The unsupported filter on its own yields nothing.
198+
assert!(translate_filters(&[like]).is_none());
199+
}
200+
201+
#[test]
202+
fn rejects_literal_whose_declared_type_mismatches_the_value() {
203+
// An int value declared as TIMESTAMP (id 9) must not be treated as a plain integer.
204+
let bad = expr(ExprStruct::Gt(binary(col("ts"), lit_int_typed(10, 9))));
205+
assert!(translate_filters(&[bad]).is_none());
206+
// A date-typed int, however, maps to Scalar::Date.
207+
let date = expr(ExprStruct::Gt(binary(
208+
col("d"),
209+
lit_int_typed(18262, type_id::DATE),
210+
)));
211+
assert!(translate_filters(&[date]).is_some());
212+
}
213+
214+
#[test]
215+
fn empty_filters_translate_to_none() {
216+
assert!(translate_filters(&[]).is_none());
217+
}
218+
}

0 commit comments

Comments
 (0)