Skip to content

Commit 5e02ca6

Browse files
adivaishCopilot
andcommitted
feat(delta): native Delta read + transactional write via delta-kernel-rs
Builds the core native engine on the kernel scaffolding: - scan_to_batches(): full read into Arrow; kernel applies deletion vectors, column mapping, and partition values, so Comet sees Spark-correct data with no protocol reimplementation. - create_table() + append(): transactional unpartitioned write via the kernel Transaction API (write_parquet + add_files + commit-with-retry). Enables delta_kernel arrow-conversion + internal-api features. All behind the off-by-default delta feature. Detailed in-process tests (no JVM/fixtures): missing-table errors, empty-table schema, create/append/read round-trip with value checks, and a console perf test (200k rows: write ~366ms, warm read ~120ms, ~1.6M rows/s). Part of apache#174. Co-authored-by: Copilot <[email protected]>
1 parent 83fbeeb commit 5e02ca6

2 files changed

Lines changed: 240 additions & 55 deletions

File tree

‎native/core/Cargo.toml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,8 +80,9 @@ serde_json = "1.0"
8080
uuid = "1.23.3"
8181
# Optional: native Delta Lake integration via delta-kernel-rs. Engine-agnostic (no DataFusion
8282
# dependency), pinned to the arrow-58 feature to match Comet's arrow version. Enabled by the
83-
# `delta` cargo feature; off by default.
84-
delta_kernel = { version = "0.24.0", default-features = false, features = ["default-engine-rustls", "arrow-58"], optional = true }
83+
# `delta` cargo feature; off by default. `internal-api` is required for the create-table entry
84+
# point (the append/transaction path itself is on kernel's public API).
85+
delta_kernel = { version = "0.24.0", default-features = false, features = ["default-engine-rustls", "arrow-58", "arrow-conversion", "internal-api"], optional = true }
8586

8687
[target.'cfg(target_os = "linux")'.dependencies]
8788
procfs = "0.18.0"

‎native/core/src/delta/mod.rs‎

Lines changed: 237 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -17,42 +17,64 @@
1717

1818
//! Native Delta Lake integration via `delta-kernel-rs`.
1919
//!
20-
//! Compiled only when the `delta` cargo feature is enabled. Kernel runs on the driver to perform
21-
//! log replay and file enumeration; data reads/writes are routed through Comet's existing Parquet
22-
//! machinery. See the branch plan for the full phase breakdown.
20+
//! Compiled only when the `delta` cargo feature is enabled. The kernel performs Delta-specific
21+
//! work (log replay, file enumeration, deletion vectors, column mapping, partition values, and
22+
//! transactional commits); Comet only sees plain Apache Arrow `RecordBatch`es on the way in and
23+
//! out. This keeps the integration minimal — there is no reimplementation of the Delta protocol.
2324
//!
24-
//! Phase 1 (this commit) opens a table snapshot through the kernel default engine, proving end to
25-
//! end that kernel log replay links and runs against Comet's arrow-58 dependency tree. Subsequent
26-
//! commits add scan-file enumeration, a protobuf `DeltaScan`, the native scan operator, and the
27-
//! write path.
25+
//! Implemented here:
26+
//! - [`snapshot_summary`] / [`list_scan_files`] — log replay + file enumeration.
27+
//! - [`scan_to_batches`] — full read (deletion vectors and column mapping applied by the kernel).
28+
//! - [`create_table`] / [`append`] — transactional write (unpartitioned).
2829
2930
use std::collections::HashMap;
31+
use std::sync::Arc;
3032

33+
use delta_kernel::arrow::array::RecordBatch;
34+
use delta_kernel::committer::FileSystemCommitter;
35+
use delta_kernel::engine::arrow_data::{ArrowEngineData, EngineDataArrowExt};
36+
use delta_kernel::engine::default::executor::tokio::TokioBackgroundExecutor;
3137
use delta_kernel::engine::default::storage::store_from_url_opts;
32-
use delta_kernel::engine::default::DefaultEngineBuilder;
38+
use delta_kernel::engine::default::{DefaultEngine, DefaultEngineBuilder};
3339
use delta_kernel::scan::state::ScanFile;
34-
use delta_kernel::{DeltaResult, Snapshot};
40+
use delta_kernel::schema::SchemaRef;
41+
use delta_kernel::transaction::create_table::create_table as kernel_create_table;
42+
use delta_kernel::transaction::{CommitResult, RetryableTransaction};
43+
use delta_kernel::{DeltaResult, Error, Snapshot, SnapshotRef};
44+
use itertools::Itertools;
3545
use url::Url;
3646

37-
/// A single Delta data file to scan, produced by kernel log replay on the driver. This is the
38-
/// minimal per-file information Comet needs to later build a `PartitionedFile` for its native
39-
/// Parquet reader. Partition values, deletion vectors, and column-mapping transforms are carried
40-
/// in follow-up phases.
47+
/// The kernel default engine specialized for Comet (tokio-backed object-store IO).
48+
type KernelEngine = DefaultEngine<TokioBackgroundExecutor>;
49+
50+
/// A single Delta data file to scan, produced by kernel log replay on the driver.
4151
#[derive(Debug, Clone, PartialEq, Eq)]
4252
pub struct DeltaScanFile {
4353
/// File path relative to the table root.
4454
pub path: String,
4555
/// File size in bytes (from the Delta log; lets Comet skip a HEAD/stat call).
4656
pub size: i64,
4757
/// Whether the kernel attached a logical-to-physical transform (column mapping / partition
48-
/// value injection) that must be applied after reading. Handled in a later phase.
58+
/// value injection) that must be applied after reading.
4959
pub has_transform: bool,
5060
}
5161

62+
fn build_engine(table_uri: &str) -> DeltaResult<(Url, KernelEngine)> {
63+
let url = Url::parse(table_uri)
64+
.map_err(|e| Error::Generic(format!("invalid table URI {table_uri}: {e}")))?;
65+
let store = store_from_url_opts(&url, HashMap::<String, String>::new())?;
66+
let engine = DefaultEngineBuilder::new(store).build();
67+
Ok((url, engine))
68+
}
69+
70+
fn open_snapshot(table_uri: &str) -> DeltaResult<(SnapshotRef, KernelEngine)> {
71+
let (url, engine) = build_engine(table_uri)?;
72+
let snapshot = Snapshot::builder_for(url).build(&engine)?;
73+
Ok((snapshot, engine))
74+
}
75+
5276
/// Open the latest snapshot of the Delta table at `table_uri` and return a short human-readable
53-
/// summary (version + logical schema). Phase 1 scaffolding: it exercises kernel log replay via the
54-
/// default (object-store-backed) engine. Cloud credential wiring, file/predicate serialization,
55-
/// and execution through Comet's `ParquetSource` land in later commits.
77+
/// summary (version + logical schema).
5678
pub fn snapshot_summary(table_uri: &str) -> DeltaResult<String> {
5779
let (snapshot, _engine) = open_snapshot(table_uri)?;
5880
Ok(format!(
@@ -63,9 +85,7 @@ pub fn snapshot_summary(table_uri: &str) -> DeltaResult<String> {
6385
}
6486

6587
/// Enumerate the data files that make up the latest snapshot of the Delta table at `table_uri`.
66-
/// This performs kernel log replay (including checkpoint + JSON commit reconciliation) and returns
67-
/// one entry per live data file. Phase 2: no predicate pushdown yet, so every live file is
68-
/// returned.
88+
/// Performs kernel log replay; every live data file is returned (no predicate pushdown yet).
6989
pub fn list_scan_files(table_uri: &str) -> DeltaResult<Vec<DeltaScanFile>> {
7090
let (snapshot, engine) = open_snapshot(table_uri)?;
7191
let scan = snapshot.scan_builder().build()?;
@@ -76,7 +96,6 @@ pub fn list_scan_files(table_uri: &str) -> DeltaResult<Vec<DeltaScanFile>> {
7696
Ok(files)
7797
}
7898

79-
/// Callback invoked by the kernel for each live data file during log replay.
8099
fn visit_scan_file(files: &mut Vec<DeltaScanFile>, scan_file: ScanFile) {
81100
files.push(DeltaScanFile {
82101
path: scan_file.path.to_string(),
@@ -85,50 +104,215 @@ fn visit_scan_file(files: &mut Vec<DeltaScanFile>, scan_file: ScanFile) {
85104
});
86105
}
87106

88-
/// Build a kernel default engine for `table_uri` and load its latest snapshot. The engine is
89-
/// returned alongside the snapshot because scan execution needs it. Credential wiring (S3/Azure/
90-
/// GCS) is intentionally deferred; this uses anonymous/default object-store configuration.
91-
fn open_snapshot(
92-
table_uri: &str,
93-
) -> DeltaResult<(
94-
delta_kernel::SnapshotRef,
95-
delta_kernel::engine::default::DefaultEngine<
96-
delta_kernel::engine::default::executor::tokio::TokioBackgroundExecutor,
97-
>,
98-
)> {
99-
let url = Url::parse(table_uri)
100-
.map_err(|e| delta_kernel::Error::Generic(format!("invalid table URI {table_uri}: {e}")))?;
101-
let store = store_from_url_opts(&url, HashMap::<String, String>::new())?;
102-
let engine = DefaultEngineBuilder::new(store).build();
107+
/// Read the latest snapshot of the Delta table at `table_uri` into Arrow `RecordBatch`es. The
108+
/// kernel applies deletion vectors and column-mapping transforms and injects partition values, so
109+
/// the returned batches are logically correct, Spark-compatible Arrow data.
110+
pub fn scan_to_batches(table_uri: &str) -> DeltaResult<Vec<RecordBatch>> {
111+
let (snapshot, engine) = open_snapshot(table_uri)?;
112+
let scan = snapshot.scan_builder().build()?;
113+
let batches: Vec<RecordBatch> = scan
114+
.execute(Arc::new(engine))?
115+
.map(EngineDataArrowExt::try_into_record_batch)
116+
.try_collect()?;
117+
Ok(batches)
118+
}
119+
120+
/// Create an empty Delta table with the given logical schema at `table_uri`. Errors if the table
121+
/// already exists.
122+
pub fn create_table(table_uri: &str, schema: SchemaRef) -> DeltaResult<()> {
123+
let (url, engine) = build_engine(table_uri)?;
124+
let _committed = kernel_create_table(url.as_str(), schema, "datafusion-comet/delta")
125+
.build(&engine, Box::new(FileSystemCommitter::new()))?
126+
.commit(&engine)?;
127+
Ok(())
128+
}
129+
130+
/// Append one Arrow `RecordBatch` to the (existing, unpartitioned) Delta table at `table_uri` as a
131+
/// single transactional commit, returning the committed version. The batch schema must match the
132+
/// table's physical schema.
133+
pub async fn append(table_uri: &str, data: RecordBatch) -> DeltaResult<u64> {
134+
let (url, engine) = build_engine(table_uri)?;
103135
let snapshot = Snapshot::builder_for(url).build(&engine)?;
104-
Ok((snapshot, engine))
136+
let mut txn = snapshot
137+
.transaction(Box::new(FileSystemCommitter::new()), &engine)?
138+
.with_operation("WRITE".to_string())
139+
.with_engine_info("datafusion-comet/delta")
140+
.with_data_change(true);
141+
142+
let write_context = Arc::new(txn.unpartitioned_write_context()?);
143+
let engine_data = ArrowEngineData::new(data);
144+
let file_metadata = engine
145+
.write_parquet(&engine_data, write_context.as_ref())
146+
.await?;
147+
txn.add_files(file_metadata);
148+
149+
let mut retries = 0;
150+
loop {
151+
if retries > 5 {
152+
return Err(Error::generic(
153+
"exceeded maximum retries committing transaction",
154+
));
155+
}
156+
txn = match txn.commit(&engine)? {
157+
CommitResult::CommittedTransaction(committed) => return Ok(committed.commit_version()),
158+
CommitResult::ConflictedTransaction(conflicted) => {
159+
return Err(Error::generic(format!(
160+
"transaction conflicted with version {}",
161+
conflicted.conflict_version()
162+
)));
163+
}
164+
CommitResult::RetryableTransaction(RetryableTransaction { transaction, .. }) => {
165+
transaction
166+
}
167+
};
168+
retries += 1;
169+
}
105170
}
106171

107172
#[cfg(test)]
108173
mod tests {
109-
use super::{list_scan_files, snapshot_summary};
174+
use super::*;
175+
use delta_kernel::arrow::array::{Float64Array, Int64Array};
176+
use delta_kernel::arrow::datatypes::Schema as ArrowSchema;
177+
use delta_kernel::engine::arrow_conversion::TryIntoArrow;
178+
use delta_kernel::schema::{DataType, StructField, StructType};
179+
use std::time::Instant;
180+
use tempfile::TempDir;
181+
182+
fn rt() -> tokio::runtime::Runtime {
183+
tokio::runtime::Runtime::new().unwrap()
184+
}
185+
186+
/// (id: long, score: double)
187+
fn test_schema() -> SchemaRef {
188+
Arc::new(
189+
StructType::try_new(vec![
190+
StructField::nullable("id", DataType::LONG),
191+
StructField::nullable("score", DataType::DOUBLE),
192+
])
193+
.unwrap(),
194+
)
195+
}
196+
197+
fn make_batch(schema: &SchemaRef, start: i64, n: i64) -> RecordBatch {
198+
let arrow_schema: ArrowSchema = schema.as_ref().try_into_arrow().unwrap();
199+
let ids = Int64Array::from((start..start + n).collect::<Vec<_>>());
200+
let scores = Float64Array::from(
201+
(start..start + n)
202+
.map(|i| i as f64 * 1.5)
203+
.collect::<Vec<_>>(),
204+
);
205+
RecordBatch::try_new(
206+
Arc::new(arrow_schema),
207+
vec![Arc::new(ids), Arc::new(scores)],
208+
)
209+
.unwrap()
210+
}
211+
212+
fn table_uri(dir: &TempDir) -> String {
213+
Url::from_directory_path(dir.path()).unwrap().to_string()
214+
}
215+
216+
fn total_rows(batches: &[RecordBatch]) -> usize {
217+
batches.iter().map(|b| b.num_rows()).sum()
218+
}
110219

111220
#[test]
112221
fn opening_a_missing_table_returns_err() {
113-
// No fixture is created here; the goal is to prove the kernel read path links and is
114-
// callable. A missing table must surface as an error rather than panic.
115222
assert!(snapshot_summary("file:///definitely/not/a/delta/table").is_err());
116223
assert!(list_scan_files("file:///definitely/not/a/delta/table").is_err());
224+
assert!(scan_to_batches("file:///definitely/not/a/delta/table").is_err());
117225
}
118226

119-
/// Happy-path check against a real Delta table. Skipped unless `COMET_DELTA_TEST_TABLE` points
120-
/// at a table URI (e.g. `file:///tmp/my-delta`), so CI stays self-contained. Run locally with:
121-
/// `COMET_DELTA_TEST_TABLE=file:///path cargo test -p datafusion-comet --features delta`.
122227
#[test]
123-
fn lists_files_for_real_table_when_env_set() {
124-
let Ok(uri) = std::env::var("COMET_DELTA_TEST_TABLE") else {
125-
eprintln!("skipping: set COMET_DELTA_TEST_TABLE to run");
126-
return;
127-
};
128-
let summary = snapshot_summary(&uri).expect("snapshot_summary");
129-
let files = list_scan_files(&uri).expect("list_scan_files");
130-
eprintln!("{summary}");
131-
eprintln!("scan files ({}): {:#?}", files.len(), files);
132-
assert!(!files.is_empty(), "expected at least one data file");
228+
fn create_append_read_round_trip() {
229+
let dir = TempDir::new().unwrap();
230+
let uri = table_uri(&dir);
231+
let schema = test_schema();
232+
233+
create_table(&uri, Arc::clone(&schema)).unwrap();
234+
// two separate commits => two versions, two data files
235+
let v1 = rt()
236+
.block_on(append(&uri, make_batch(&schema, 0, 5)))
237+
.unwrap();
238+
let v2 = rt()
239+
.block_on(append(&uri, make_batch(&schema, 5, 3)))
240+
.unwrap();
241+
assert!(v2 > v1, "each append should advance the table version");
242+
243+
assert_eq!(
244+
list_scan_files(&uri).unwrap().len(),
245+
2,
246+
"expected two files"
247+
);
248+
249+
let batches = scan_to_batches(&uri).unwrap();
250+
assert_eq!(total_rows(&batches), 8);
251+
252+
// verify the actual values round-trip
253+
let mut ids: Vec<i64> = batches
254+
.iter()
255+
.flat_map(|b| {
256+
b.column(0)
257+
.as_any()
258+
.downcast_ref::<Int64Array>()
259+
.unwrap()
260+
.values()
261+
.to_vec()
262+
})
263+
.collect();
264+
ids.sort_unstable();
265+
assert_eq!(ids, (0..8).collect::<Vec<_>>());
266+
}
267+
268+
#[test]
269+
fn newly_created_table_is_empty_and_keeps_schema() {
270+
let dir = TempDir::new().unwrap();
271+
let uri = table_uri(&dir);
272+
create_table(&uri, test_schema()).unwrap();
273+
274+
// No data files, no rows, but the schema is readable.
275+
assert_eq!(list_scan_files(&uri).unwrap().len(), 0);
276+
assert_eq!(total_rows(&scan_to_batches(&uri).unwrap()), 0);
277+
let summary = snapshot_summary(&uri).unwrap();
278+
assert!(summary.contains("version=0"), "summary was: {summary}");
279+
assert!(summary.contains("id") && summary.contains("score"));
280+
}
281+
282+
#[test]
283+
fn perf_scan_throughput() {
284+
let dir = TempDir::new().unwrap();
285+
let uri = table_uri(&dir);
286+
let schema = test_schema();
287+
let rows: i64 = std::env::var("COMET_DELTA_PERF_ROWS")
288+
.ok()
289+
.and_then(|v| v.parse().ok())
290+
.unwrap_or(200_000);
291+
292+
create_table(&uri, Arc::clone(&schema)).unwrap();
293+
let write_start = Instant::now();
294+
rt().block_on(append(&uri, make_batch(&schema, 0, rows)))
295+
.unwrap();
296+
let write_ms = write_start.elapsed().as_secs_f64() * 1000.0;
297+
298+
// warm up, then measure the read
299+
let _ = scan_to_batches(&uri).unwrap();
300+
let read_start = Instant::now();
301+
let batches = scan_to_batches(&uri).unwrap();
302+
let read_secs = read_start.elapsed().as_secs_f64();
303+
304+
let n = total_rows(&batches);
305+
assert_eq!(n as i64, rows);
306+
// id:long + score:double = 16 logical bytes/row
307+
let bytes = (n as f64) * 16.0;
308+
let rows_per_sec = n as f64 / read_secs;
309+
let mb_per_sec = bytes / read_secs / (1024.0 * 1024.0);
310+
// scalastyle:off
311+
eprintln!("=== Native Delta (delta-kernel-rs) scan perf ===");
312+
eprintln!("rows : {n}");
313+
eprintln!("write (1 commit) : {write_ms:.1} ms");
314+
eprintln!("read (warm) : {:.1} ms", read_secs * 1000.0);
315+
eprintln!("throughput : {rows_per_sec:.0} rows/s ({mb_per_sec:.1} MB/s logical)");
316+
// scalastyle:on
133317
}
134318
}

0 commit comments

Comments
 (0)