1717
1818//! Native Delta Lake integration via `delta-kernel-rs`.
1919//!
20- //! Compiled only when the `delta` cargo feature is enabled. Kernel runs on the driver to perform
21- //! log replay and file enumeration; data reads/writes are routed through Comet's existing Parquet
22- //! machinery. See the branch plan for the full phase breakdown.
20+ //! Compiled only when the `delta` cargo feature is enabled. The kernel performs Delta-specific
21+ //! work (log replay, file enumeration, deletion vectors, column mapping, partition values, and
22+ //! transactional commits); Comet only sees plain Apache Arrow `RecordBatch`es on the way in and
23+ //! out. This keeps the integration minimal — there is no reimplementation of the Delta protocol.
2324//!
24- //! Phase 1 (this commit) opens a table snapshot through the kernel default engine, proving end to
25- //! end that kernel log replay links and runs against Comet's arrow-58 dependency tree. Subsequent
26- //! commits add scan-file enumeration, a protobuf `DeltaScan`, the native scan operator, and the
27- //! write path .
25+ //! Implemented here:
26+ //! - [`snapshot_summary`] / [`list_scan_files`] — log replay + file enumeration.
27+ //! - [`scan_to_batches`] — full read (deletion vectors and column mapping applied by the kernel).
28+ //! - [`create_table`] / [`append`] — transactional write (unpartitioned) .
2829
2930use std:: collections:: HashMap ;
31+ use std:: sync:: Arc ;
3032
33+ use delta_kernel:: arrow:: array:: RecordBatch ;
34+ use delta_kernel:: committer:: FileSystemCommitter ;
35+ use delta_kernel:: engine:: arrow_data:: { ArrowEngineData , EngineDataArrowExt } ;
36+ use delta_kernel:: engine:: default:: executor:: tokio:: TokioBackgroundExecutor ;
3137use delta_kernel:: engine:: default:: storage:: store_from_url_opts;
32- use delta_kernel:: engine:: default:: DefaultEngineBuilder ;
38+ use delta_kernel:: engine:: default:: { DefaultEngine , DefaultEngineBuilder } ;
3339use delta_kernel:: scan:: state:: ScanFile ;
34- use delta_kernel:: { DeltaResult , Snapshot } ;
40+ use delta_kernel:: schema:: SchemaRef ;
41+ use delta_kernel:: transaction:: create_table:: create_table as kernel_create_table;
42+ use delta_kernel:: transaction:: { CommitResult , RetryableTransaction } ;
43+ use delta_kernel:: { DeltaResult , Error , Snapshot , SnapshotRef } ;
44+ use itertools:: Itertools ;
3545use url:: Url ;
3646
37- /// A single Delta data file to scan, produced by kernel log replay on the driver. This is the
38- /// minimal per-file information Comet needs to later build a `PartitionedFile` for its native
39- /// Parquet reader. Partition values, deletion vectors, and column-mapping transforms are carried
40- /// in follow-up phases .
47+ /// The kernel default engine specialized for Comet (tokio-backed object-store IO).
48+ type KernelEngine = DefaultEngine < TokioBackgroundExecutor > ;
49+
50+ /// A single Delta data file to scan, produced by kernel log replay on the driver .
4151#[ derive( Debug , Clone , PartialEq , Eq ) ]
4252pub struct DeltaScanFile {
4353 /// File path relative to the table root.
4454 pub path : String ,
4555 /// File size in bytes (from the Delta log; lets Comet skip a HEAD/stat call).
4656 pub size : i64 ,
4757 /// Whether the kernel attached a logical-to-physical transform (column mapping / partition
48- /// value injection) that must be applied after reading. Handled in a later phase.
58+ /// value injection) that must be applied after reading.
4959 pub has_transform : bool ,
5060}
5161
62+ fn build_engine ( table_uri : & str ) -> DeltaResult < ( Url , KernelEngine ) > {
63+ let url = Url :: parse ( table_uri)
64+ . map_err ( |e| Error :: Generic ( format ! ( "invalid table URI {table_uri}: {e}" ) ) ) ?;
65+ let store = store_from_url_opts ( & url, HashMap :: < String , String > :: new ( ) ) ?;
66+ let engine = DefaultEngineBuilder :: new ( store) . build ( ) ;
67+ Ok ( ( url, engine) )
68+ }
69+
70+ fn open_snapshot ( table_uri : & str ) -> DeltaResult < ( SnapshotRef , KernelEngine ) > {
71+ let ( url, engine) = build_engine ( table_uri) ?;
72+ let snapshot = Snapshot :: builder_for ( url) . build ( & engine) ?;
73+ Ok ( ( snapshot, engine) )
74+ }
75+
5276/// Open the latest snapshot of the Delta table at `table_uri` and return a short human-readable
53- /// summary (version + logical schema). Phase 1 scaffolding: it exercises kernel log replay via the
54- /// default (object-store-backed) engine. Cloud credential wiring, file/predicate serialization,
55- /// and execution through Comet's `ParquetSource` land in later commits.
77+ /// summary (version + logical schema).
5678pub fn snapshot_summary ( table_uri : & str ) -> DeltaResult < String > {
5779 let ( snapshot, _engine) = open_snapshot ( table_uri) ?;
5880 Ok ( format ! (
@@ -63,9 +85,7 @@ pub fn snapshot_summary(table_uri: &str) -> DeltaResult<String> {
6385}
6486
6587/// Enumerate the data files that make up the latest snapshot of the Delta table at `table_uri`.
66- /// This performs kernel log replay (including checkpoint + JSON commit reconciliation) and returns
67- /// one entry per live data file. Phase 2: no predicate pushdown yet, so every live file is
68- /// returned.
88+ /// Performs kernel log replay; every live data file is returned (no predicate pushdown yet).
6989pub fn list_scan_files ( table_uri : & str ) -> DeltaResult < Vec < DeltaScanFile > > {
7090 let ( snapshot, engine) = open_snapshot ( table_uri) ?;
7191 let scan = snapshot. scan_builder ( ) . build ( ) ?;
@@ -76,7 +96,6 @@ pub fn list_scan_files(table_uri: &str) -> DeltaResult<Vec<DeltaScanFile>> {
7696 Ok ( files)
7797}
7898
79- /// Callback invoked by the kernel for each live data file during log replay.
8099fn visit_scan_file ( files : & mut Vec < DeltaScanFile > , scan_file : ScanFile ) {
81100 files. push ( DeltaScanFile {
82101 path : scan_file. path . to_string ( ) ,
@@ -85,50 +104,215 @@ fn visit_scan_file(files: &mut Vec<DeltaScanFile>, scan_file: ScanFile) {
85104 } ) ;
86105}
87106
88- /// Build a kernel default engine for `table_uri` and load its latest snapshot. The engine is
89- /// returned alongside the snapshot because scan execution needs it. Credential wiring (S3/Azure/
90- /// GCS) is intentionally deferred; this uses anonymous/default object-store configuration.
91- fn open_snapshot (
92- table_uri : & str ,
93- ) -> DeltaResult < (
94- delta_kernel:: SnapshotRef ,
95- delta_kernel:: engine:: default:: DefaultEngine <
96- delta_kernel:: engine:: default:: executor:: tokio:: TokioBackgroundExecutor ,
97- > ,
98- ) > {
99- let url = Url :: parse ( table_uri)
100- . map_err ( |e| delta_kernel:: Error :: Generic ( format ! ( "invalid table URI {table_uri}: {e}" ) ) ) ?;
101- let store = store_from_url_opts ( & url, HashMap :: < String , String > :: new ( ) ) ?;
102- let engine = DefaultEngineBuilder :: new ( store) . build ( ) ;
107+ /// Read the latest snapshot of the Delta table at `table_uri` into Arrow `RecordBatch`es. The
108+ /// kernel applies deletion vectors and column-mapping transforms and injects partition values, so
109+ /// the returned batches are logically correct, Spark-compatible Arrow data.
110+ pub fn scan_to_batches ( table_uri : & str ) -> DeltaResult < Vec < RecordBatch > > {
111+ let ( snapshot, engine) = open_snapshot ( table_uri) ?;
112+ let scan = snapshot. scan_builder ( ) . build ( ) ?;
113+ let batches: Vec < RecordBatch > = scan
114+ . execute ( Arc :: new ( engine) ) ?
115+ . map ( EngineDataArrowExt :: try_into_record_batch)
116+ . try_collect ( ) ?;
117+ Ok ( batches)
118+ }
119+
120+ /// Create an empty Delta table with the given logical schema at `table_uri`. Errors if the table
121+ /// already exists.
122+ pub fn create_table ( table_uri : & str , schema : SchemaRef ) -> DeltaResult < ( ) > {
123+ let ( url, engine) = build_engine ( table_uri) ?;
124+ let _committed = kernel_create_table ( url. as_str ( ) , schema, "datafusion-comet/delta" )
125+ . build ( & engine, Box :: new ( FileSystemCommitter :: new ( ) ) ) ?
126+ . commit ( & engine) ?;
127+ Ok ( ( ) )
128+ }
129+
130+ /// Append one Arrow `RecordBatch` to the (existing, unpartitioned) Delta table at `table_uri` as a
131+ /// single transactional commit, returning the committed version. The batch schema must match the
132+ /// table's physical schema.
133+ pub async fn append ( table_uri : & str , data : RecordBatch ) -> DeltaResult < u64 > {
134+ let ( url, engine) = build_engine ( table_uri) ?;
103135 let snapshot = Snapshot :: builder_for ( url) . build ( & engine) ?;
104- Ok ( ( snapshot, engine) )
136+ let mut txn = snapshot
137+ . transaction ( Box :: new ( FileSystemCommitter :: new ( ) ) , & engine) ?
138+ . with_operation ( "WRITE" . to_string ( ) )
139+ . with_engine_info ( "datafusion-comet/delta" )
140+ . with_data_change ( true ) ;
141+
142+ let write_context = Arc :: new ( txn. unpartitioned_write_context ( ) ?) ;
143+ let engine_data = ArrowEngineData :: new ( data) ;
144+ let file_metadata = engine
145+ . write_parquet ( & engine_data, write_context. as_ref ( ) )
146+ . await ?;
147+ txn. add_files ( file_metadata) ;
148+
149+ let mut retries = 0 ;
150+ loop {
151+ if retries > 5 {
152+ return Err ( Error :: generic (
153+ "exceeded maximum retries committing transaction" ,
154+ ) ) ;
155+ }
156+ txn = match txn. commit ( & engine) ? {
157+ CommitResult :: CommittedTransaction ( committed) => return Ok ( committed. commit_version ( ) ) ,
158+ CommitResult :: ConflictedTransaction ( conflicted) => {
159+ return Err ( Error :: generic ( format ! (
160+ "transaction conflicted with version {}" ,
161+ conflicted. conflict_version( )
162+ ) ) ) ;
163+ }
164+ CommitResult :: RetryableTransaction ( RetryableTransaction { transaction, .. } ) => {
165+ transaction
166+ }
167+ } ;
168+ retries += 1 ;
169+ }
105170}
106171
107172#[ cfg( test) ]
108173mod tests {
109- use super :: { list_scan_files, snapshot_summary} ;
174+ use super :: * ;
175+ use delta_kernel:: arrow:: array:: { Float64Array , Int64Array } ;
176+ use delta_kernel:: arrow:: datatypes:: Schema as ArrowSchema ;
177+ use delta_kernel:: engine:: arrow_conversion:: TryIntoArrow ;
178+ use delta_kernel:: schema:: { DataType , StructField , StructType } ;
179+ use std:: time:: Instant ;
180+ use tempfile:: TempDir ;
181+
182+ fn rt ( ) -> tokio:: runtime:: Runtime {
183+ tokio:: runtime:: Runtime :: new ( ) . unwrap ( )
184+ }
185+
186+ /// (id: long, score: double)
187+ fn test_schema ( ) -> SchemaRef {
188+ Arc :: new (
189+ StructType :: try_new ( vec ! [
190+ StructField :: nullable( "id" , DataType :: LONG ) ,
191+ StructField :: nullable( "score" , DataType :: DOUBLE ) ,
192+ ] )
193+ . unwrap ( ) ,
194+ )
195+ }
196+
197+ fn make_batch ( schema : & SchemaRef , start : i64 , n : i64 ) -> RecordBatch {
198+ let arrow_schema: ArrowSchema = schema. as_ref ( ) . try_into_arrow ( ) . unwrap ( ) ;
199+ let ids = Int64Array :: from ( ( start..start + n) . collect :: < Vec < _ > > ( ) ) ;
200+ let scores = Float64Array :: from (
201+ ( start..start + n)
202+ . map ( |i| i as f64 * 1.5 )
203+ . collect :: < Vec < _ > > ( ) ,
204+ ) ;
205+ RecordBatch :: try_new (
206+ Arc :: new ( arrow_schema) ,
207+ vec ! [ Arc :: new( ids) , Arc :: new( scores) ] ,
208+ )
209+ . unwrap ( )
210+ }
211+
212+ fn table_uri ( dir : & TempDir ) -> String {
213+ Url :: from_directory_path ( dir. path ( ) ) . unwrap ( ) . to_string ( )
214+ }
215+
216+ fn total_rows ( batches : & [ RecordBatch ] ) -> usize {
217+ batches. iter ( ) . map ( |b| b. num_rows ( ) ) . sum ( )
218+ }
110219
111220 #[ test]
112221 fn opening_a_missing_table_returns_err ( ) {
113- // No fixture is created here; the goal is to prove the kernel read path links and is
114- // callable. A missing table must surface as an error rather than panic.
115222 assert ! ( snapshot_summary( "file:///definitely/not/a/delta/table" ) . is_err( ) ) ;
116223 assert ! ( list_scan_files( "file:///definitely/not/a/delta/table" ) . is_err( ) ) ;
224+ assert ! ( scan_to_batches( "file:///definitely/not/a/delta/table" ) . is_err( ) ) ;
117225 }
118226
119- /// Happy-path check against a real Delta table. Skipped unless `COMET_DELTA_TEST_TABLE` points
120- /// at a table URI (e.g. `file:///tmp/my-delta`), so CI stays self-contained. Run locally with:
121- /// `COMET_DELTA_TEST_TABLE=file:///path cargo test -p datafusion-comet --features delta`.
122227 #[ test]
123- fn lists_files_for_real_table_when_env_set ( ) {
124- let Ok ( uri) = std:: env:: var ( "COMET_DELTA_TEST_TABLE" ) else {
125- eprintln ! ( "skipping: set COMET_DELTA_TEST_TABLE to run" ) ;
126- return ;
127- } ;
128- let summary = snapshot_summary ( & uri) . expect ( "snapshot_summary" ) ;
129- let files = list_scan_files ( & uri) . expect ( "list_scan_files" ) ;
130- eprintln ! ( "{summary}" ) ;
131- eprintln ! ( "scan files ({}): {:#?}" , files. len( ) , files) ;
132- assert ! ( !files. is_empty( ) , "expected at least one data file" ) ;
228+ fn create_append_read_round_trip ( ) {
229+ let dir = TempDir :: new ( ) . unwrap ( ) ;
230+ let uri = table_uri ( & dir) ;
231+ let schema = test_schema ( ) ;
232+
233+ create_table ( & uri, Arc :: clone ( & schema) ) . unwrap ( ) ;
234+ // two separate commits => two versions, two data files
235+ let v1 = rt ( )
236+ . block_on ( append ( & uri, make_batch ( & schema, 0 , 5 ) ) )
237+ . unwrap ( ) ;
238+ let v2 = rt ( )
239+ . block_on ( append ( & uri, make_batch ( & schema, 5 , 3 ) ) )
240+ . unwrap ( ) ;
241+ assert ! ( v2 > v1, "each append should advance the table version" ) ;
242+
243+ assert_eq ! (
244+ list_scan_files( & uri) . unwrap( ) . len( ) ,
245+ 2 ,
246+ "expected two files"
247+ ) ;
248+
249+ let batches = scan_to_batches ( & uri) . unwrap ( ) ;
250+ assert_eq ! ( total_rows( & batches) , 8 ) ;
251+
252+ // verify the actual values round-trip
253+ let mut ids: Vec < i64 > = batches
254+ . iter ( )
255+ . flat_map ( |b| {
256+ b. column ( 0 )
257+ . as_any ( )
258+ . downcast_ref :: < Int64Array > ( )
259+ . unwrap ( )
260+ . values ( )
261+ . to_vec ( )
262+ } )
263+ . collect ( ) ;
264+ ids. sort_unstable ( ) ;
265+ assert_eq ! ( ids, ( 0 ..8 ) . collect:: <Vec <_>>( ) ) ;
266+ }
267+
268+ #[ test]
269+ fn newly_created_table_is_empty_and_keeps_schema ( ) {
270+ let dir = TempDir :: new ( ) . unwrap ( ) ;
271+ let uri = table_uri ( & dir) ;
272+ create_table ( & uri, test_schema ( ) ) . unwrap ( ) ;
273+
274+ // No data files, no rows, but the schema is readable.
275+ assert_eq ! ( list_scan_files( & uri) . unwrap( ) . len( ) , 0 ) ;
276+ assert_eq ! ( total_rows( & scan_to_batches( & uri) . unwrap( ) ) , 0 ) ;
277+ let summary = snapshot_summary ( & uri) . unwrap ( ) ;
278+ assert ! ( summary. contains( "version=0" ) , "summary was: {summary}" ) ;
279+ assert ! ( summary. contains( "id" ) && summary. contains( "score" ) ) ;
280+ }
281+
282+ #[ test]
283+ fn perf_scan_throughput ( ) {
284+ let dir = TempDir :: new ( ) . unwrap ( ) ;
285+ let uri = table_uri ( & dir) ;
286+ let schema = test_schema ( ) ;
287+ let rows: i64 = std:: env:: var ( "COMET_DELTA_PERF_ROWS" )
288+ . ok ( )
289+ . and_then ( |v| v. parse ( ) . ok ( ) )
290+ . unwrap_or ( 200_000 ) ;
291+
292+ create_table ( & uri, Arc :: clone ( & schema) ) . unwrap ( ) ;
293+ let write_start = Instant :: now ( ) ;
294+ rt ( ) . block_on ( append ( & uri, make_batch ( & schema, 0 , rows) ) )
295+ . unwrap ( ) ;
296+ let write_ms = write_start. elapsed ( ) . as_secs_f64 ( ) * 1000.0 ;
297+
298+ // warm up, then measure the read
299+ let _ = scan_to_batches ( & uri) . unwrap ( ) ;
300+ let read_start = Instant :: now ( ) ;
301+ let batches = scan_to_batches ( & uri) . unwrap ( ) ;
302+ let read_secs = read_start. elapsed ( ) . as_secs_f64 ( ) ;
303+
304+ let n = total_rows ( & batches) ;
305+ assert_eq ! ( n as i64 , rows) ;
306+ // id:long + score:double = 16 logical bytes/row
307+ let bytes = ( n as f64 ) * 16.0 ;
308+ let rows_per_sec = n as f64 / read_secs;
309+ let mb_per_sec = bytes / read_secs / ( 1024.0 * 1024.0 ) ;
310+ // scalastyle:off
311+ eprintln ! ( "=== Native Delta (delta-kernel-rs) scan perf ===" ) ;
312+ eprintln ! ( "rows : {n}" ) ;
313+ eprintln ! ( "write (1 commit) : {write_ms:.1} ms" ) ;
314+ eprintln ! ( "read (warm) : {:.1} ms" , read_secs * 1000.0 ) ;
315+ eprintln ! ( "throughput : {rows_per_sec:.0} rows/s ({mb_per_sec:.1} MB/s logical)" ) ;
316+ // scalastyle:on
133317 }
134318}
0 commit comments