Skip to content

Commit dbb7d1d

Browse files
adivaishCopilot
andcommitted
feat(delta): native DeltaScanExec + proto DeltaScan + planner dispatch
Wires the native Delta read engine into Comet's plan model: - proto DeltaScan { table_uri, required_schema } added to the Operator oneof. - DeltaScanExec: a DataFusion ExecutionPlan (leaf) that reads a Delta table via the kernel and streams Arrow batches; output schema is read from the Delta log. - planner dispatch for OpStruct::DeltaScan (feature-gated; returns a clear error when Comet is built without the delta feature). Validated by a native test that executes DeltaScanExec through DataFusion collect() and verifies row count. clippy-clean on both default and --features delta builds. Part of apache#174. Co-authored-by: Copilot <[email protected]>
1 parent 5e02ca6 commit dbb7d1d

6 files changed

Lines changed: 231 additions & 1 deletion

File tree

‎native/core/src/delta/mod.rs‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@ use std::collections::HashMap;
3131
use std::sync::Arc;
3232

3333
use delta_kernel::arrow::array::RecordBatch;
34+
use delta_kernel::arrow::datatypes::SchemaRef as ArrowSchemaRef;
3435
use delta_kernel::committer::FileSystemCommitter;
36+
use delta_kernel::engine::arrow_conversion::TryIntoArrow;
3537
use delta_kernel::engine::arrow_data::{ArrowEngineData, EngineDataArrowExt};
3638
use delta_kernel::engine::default::executor::tokio::TokioBackgroundExecutor;
3739
use delta_kernel::engine::default::storage::store_from_url_opts;
@@ -44,6 +46,9 @@ use delta_kernel::{DeltaResult, Error, Snapshot, SnapshotRef};
4446
use itertools::Itertools;
4547
use url::Url;
4648

49+
mod scan_exec;
50+
pub use scan_exec::DeltaScanExec;
51+
4752
/// The kernel default engine specialized for Comet (tokio-backed object-store IO).
4853
type KernelEngine = DefaultEngine<TokioBackgroundExecutor>;
4954

@@ -117,6 +122,15 @@ pub fn scan_to_batches(table_uri: &str) -> DeltaResult<Vec<RecordBatch>> {
117122
Ok(batches)
118123
}
119124

125+
/// Read the latest snapshot's logical schema as an Arrow schema. Used by `DeltaScanExec` to
126+
/// report its output schema without reading any data.
127+
pub fn snapshot_arrow_schema(table_uri: &str) -> DeltaResult<ArrowSchemaRef> {
128+
let (snapshot, _engine) = open_snapshot(table_uri)?;
129+
let arrow_schema: delta_kernel::arrow::datatypes::Schema =
130+
snapshot.schema().as_ref().try_into_arrow()?;
131+
Ok(Arc::new(arrow_schema))
132+
}
133+
120134
/// Create an empty Delta table with the given logical schema at `table_uri`. Errors if the table
121135
/// already exists.
122136
pub fn create_table(table_uri: &str, schema: SchemaRef) -> DeltaResult<()> {
@@ -174,7 +188,6 @@ mod tests {
174188
use super::*;
175189
use delta_kernel::arrow::array::{Float64Array, Int64Array};
176190
use delta_kernel::arrow::datatypes::Schema as ArrowSchema;
177-
use delta_kernel::engine::arrow_conversion::TryIntoArrow;
178191
use delta_kernel::schema::{DataType, StructField, StructType};
179192
use std::time::Instant;
180193
use tempfile::TempDir;

‎native/core/src/delta/scan_exec.rs‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! DataFusion `ExecutionPlan` that reads a Delta table natively via delta-kernel-rs.
19+
20+
use std::any::Any;
21+
use std::fmt;
22+
use std::sync::Arc;
23+
24+
use datafusion::arrow::datatypes::SchemaRef;
25+
use datafusion::common::{DataFusionError, Result as DFResult};
26+
use datafusion::execution::{SendableRecordBatchStream, TaskContext};
27+
use datafusion::physical_expr::EquivalenceProperties;
28+
use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
29+
use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet};
30+
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
31+
use datafusion::physical_plan::{
32+
DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
33+
};
34+
use futures::TryStreamExt;
35+
36+
use super::{scan_to_batches, snapshot_arrow_schema};
37+
38+
/// Leaf operator that reads a whole Delta table through the kernel default engine and emits the
39+
/// resulting Arrow batches. The kernel applies deletion vectors, column mapping, and partition
40+
/// values, so the output is Spark-correct. This v1 reads the table as a single partition; per-file
41+
/// split parallelism is a follow-up.
42+
#[derive(Debug)]
43+
pub struct DeltaScanExec {
44+
table_uri: String,
45+
output_schema: SchemaRef,
46+
plan_properties: Arc<PlanProperties>,
47+
metrics: ExecutionPlanMetricsSet,
48+
}
49+
50+
impl DeltaScanExec {
51+
/// Build the operator, reading the table's logical schema from the Delta log.
52+
pub fn try_new(table_uri: String) -> DFResult<Self> {
53+
let output_schema = snapshot_arrow_schema(&table_uri).map_err(|e| {
54+
DataFusionError::Execution(format!("delta: failed to read schema: {e}"))
55+
})?;
56+
let plan_properties = Arc::new(PlanProperties::new(
57+
EquivalenceProperties::new(Arc::clone(&output_schema)),
58+
Partitioning::UnknownPartitioning(1),
59+
EmissionType::Incremental,
60+
Boundedness::Bounded,
61+
));
62+
Ok(Self {
63+
table_uri,
64+
output_schema,
65+
plan_properties,
66+
metrics: ExecutionPlanMetricsSet::new(),
67+
})
68+
}
69+
}
70+
71+
impl DisplayAs for DeltaScanExec {
72+
fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
73+
write!(f, "DeltaScanExec: table_uri={}", self.table_uri)
74+
}
75+
}
76+
77+
impl ExecutionPlan for DeltaScanExec {
78+
fn name(&self) -> &str {
79+
"DeltaScanExec"
80+
}
81+
82+
fn as_any(&self) -> &dyn Any {
83+
self
84+
}
85+
86+
fn schema(&self) -> SchemaRef {
87+
Arc::clone(&self.output_schema)
88+
}
89+
90+
fn properties(&self) -> &Arc<PlanProperties> {
91+
&self.plan_properties
92+
}
93+
94+
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
95+
vec![]
96+
}
97+
98+
fn with_new_children(
99+
self: Arc<Self>,
100+
_children: Vec<Arc<dyn ExecutionPlan>>,
101+
) -> DFResult<Arc<dyn ExecutionPlan>> {
102+
Ok(self)
103+
}
104+
105+
fn execute(
106+
&self,
107+
_partition: usize,
108+
_context: Arc<TaskContext>,
109+
) -> DFResult<SendableRecordBatchStream> {
110+
let table_uri = self.table_uri.clone();
111+
let schema = Arc::clone(&self.output_schema);
112+
113+
// The kernel read is synchronous and blocking; run it on a blocking worker so it does not
114+
// stall the tokio reactor, then stream the resulting batches.
115+
let fut = async move {
116+
let batches = tokio::task::spawn_blocking(move || scan_to_batches(&table_uri))
117+
.await
118+
.map_err(|e| DataFusionError::Execution(format!("delta: scan task failed: {e}")))?
119+
.map_err(|e| DataFusionError::Execution(format!("delta: scan failed: {e}")))?;
120+
Ok::<_, DataFusionError>(futures::stream::iter(batches.into_iter().map(Ok)))
121+
};
122+
let stream = futures::stream::once(fut).try_flatten();
123+
Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream)))
124+
}
125+
126+
fn metrics(&self) -> Option<MetricsSet> {
127+
Some(self.metrics.clone_inner())
128+
}
129+
}
130+
131+
#[cfg(test)]
132+
mod tests {
133+
use super::*;
134+
use datafusion::physical_plan::collect;
135+
use datafusion::prelude::SessionContext;
136+
use delta_kernel::arrow::array::{Float64Array, Int64Array, RecordBatch};
137+
use delta_kernel::arrow::datatypes::Schema as ArrowSchema;
138+
use delta_kernel::engine::arrow_conversion::TryIntoArrow;
139+
use delta_kernel::schema::{DataType, SchemaRef as KernelSchemaRef, StructField, StructType};
140+
use tempfile::TempDir;
141+
142+
#[test]
143+
fn delta_scan_exec_reads_table_through_datafusion() {
144+
let dir = TempDir::new().unwrap();
145+
let uri = url::Url::from_directory_path(dir.path())
146+
.unwrap()
147+
.to_string();
148+
149+
let kschema: KernelSchemaRef = Arc::new(
150+
StructType::try_new(vec![
151+
StructField::nullable("id", DataType::LONG),
152+
StructField::nullable("score", DataType::DOUBLE),
153+
])
154+
.unwrap(),
155+
);
156+
super::super::create_table(&uri, Arc::clone(&kschema)).unwrap();
157+
158+
let arrow_schema: ArrowSchema = kschema.as_ref().try_into_arrow().unwrap();
159+
let batch = RecordBatch::try_new(
160+
Arc::new(arrow_schema),
161+
vec![
162+
Arc::new(Int64Array::from(vec![0i64, 1, 2, 3])),
163+
Arc::new(Float64Array::from(vec![0.0f64, 1.5, 3.0, 4.5])),
164+
],
165+
)
166+
.unwrap();
167+
168+
let rt = tokio::runtime::Runtime::new().unwrap();
169+
rt.block_on(super::super::append(&uri, batch)).unwrap();
170+
171+
let exec: Arc<dyn ExecutionPlan> = Arc::new(DeltaScanExec::try_new(uri).unwrap());
172+
assert_eq!(exec.schema().fields().len(), 2);
173+
174+
let ctx = SessionContext::new();
175+
let batches = rt.block_on(collect(exec, ctx.task_ctx())).unwrap();
176+
let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
177+
assert_eq!(rows, 4);
178+
}
179+
}

‎native/core/src/execution/jni_api.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,7 @@ fn op_name(op: &OpStruct) -> &'static str {
240240
OpStruct::CsvScan(_) => "CsvScan",
241241
OpStruct::ShuffleScan(_) => "ShuffleScan",
242242
OpStruct::BroadcastNestedLoopJoin(_) => "BroadcastNestedLoopJoin",
243+
OpStruct::DeltaScan(_) => "DeltaScan",
243244
}
244245
}
245246

‎native/core/src/execution/planner.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1597,6 +1597,31 @@ impl PhysicalPlanner {
15971597
)),
15981598
))
15991599
}
1600+
OpStruct::DeltaScan(scan) => {
1601+
#[cfg(feature = "delta")]
1602+
{
1603+
let delta_scan = crate::delta::DeltaScanExec::try_new(scan.table_uri.clone())
1604+
.map_err(|e| {
1605+
GeneralError(format!("Failed to create DeltaScanExec: {e}"))
1606+
})?;
1607+
Ok((
1608+
vec![],
1609+
vec![],
1610+
Arc::new(SparkPlan::new(
1611+
spark_plan.plan_id,
1612+
Arc::new(delta_scan),
1613+
vec![],
1614+
)),
1615+
))
1616+
}
1617+
#[cfg(not(feature = "delta"))]
1618+
{
1619+
let _ = scan;
1620+
Err(GeneralError(
1621+
"DeltaScan requires Comet to be built with the `delta` feature".into(),
1622+
))
1623+
}
1624+
}
16001625
OpStruct::ShuffleWriter(writer) => {
16011626
assert_eq!(children.len(), 1);
16021627
let (scans, shuffle_scans, child) =

‎native/core/src/execution/planner/operator_registry.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,5 +152,6 @@ fn get_operator_type(spark_operator: &Operator) -> Option<OperatorType> {
152152
OpStruct::CsvScan(_) => Some(OperatorType::CsvScan),
153153
OpStruct::ShuffleScan(_) => None, // Not yet in OperatorType enum
154154
OpStruct::BroadcastNestedLoopJoin(_) => None,
155+
OpStruct::DeltaScan(_) => None, // Handled directly in planner.rs (behind `delta` feature)
155156
}
156157
}

‎native/proto/src/proto/operator.proto‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ message Operator {
5454
CsvScan csv_scan = 115;
5555
ShuffleScan shuffle_scan = 116;
5656
BroadcastNestedLoopJoin broadcast_nested_loop_join = 117;
57+
DeltaScan delta_scan = 118;
5758
}
5859
}
5960

@@ -94,6 +95,16 @@ message ShuffleScan {
9495
string source = 2;
9596
}
9697

98+
// Native Delta Lake scan executed via delta-kernel-rs. Only present when Comet is built with the
99+
// `delta` cargo feature; the kernel performs log replay, applies deletion vectors and column
100+
// mapping, and injects partition values, so the native operator emits Spark-correct Arrow batches.
101+
message DeltaScan {
102+
// Table root URI, e.g. file:///path or s3://bucket/path.
103+
string table_uri = 1;
104+
// Required output schema (logical column order and types).
105+
repeated SparkStructField required_schema = 2;
106+
}
107+
97108
// Common data shared by all partitions in split mode (sent once at planning)
98109
message NativeScanCommon {
99110
repeated SparkStructField required_schema = 1;

0 commit comments

Comments
 (0)