@@ -30,35 +30,105 @@ use std::collections::HashMap;
3030
3131use delta_kernel:: engine:: default:: storage:: store_from_url_opts;
3232use delta_kernel:: engine:: default:: DefaultEngineBuilder ;
33+ use delta_kernel:: scan:: state:: ScanFile ;
3334use delta_kernel:: { DeltaResult , Snapshot } ;
3435use url:: Url ;
3536
37+ /// A single Delta data file to scan, produced by kernel log replay on the driver. This is the
38+ /// minimal per-file information Comet needs to later build a `PartitionedFile` for its native
39+ /// Parquet reader. Partition values, deletion vectors, and column-mapping transforms are carried
40+ /// in follow-up phases.
41+ #[ derive( Debug , Clone , PartialEq , Eq ) ]
42+ pub struct DeltaScanFile {
43+ /// File path relative to the table root.
44+ pub path : String ,
45+ /// File size in bytes (from the Delta log; lets Comet skip a HEAD/stat call).
46+ pub size : i64 ,
47+ /// Whether the kernel attached a logical-to-physical transform (column mapping / partition
48+ /// value injection) that must be applied after reading. Handled in a later phase.
49+ pub has_transform : bool ,
50+ }
51+
3652/// Open the latest snapshot of the Delta table at `table_uri` and return a short human-readable
3753/// summary (version + logical schema). Phase 1 scaffolding: it exercises kernel log replay via the
3854/// default (object-store-backed) engine. Cloud credential wiring, file/predicate serialization,
3955/// and execution through Comet's `ParquetSource` land in later commits.
4056pub fn snapshot_summary ( table_uri : & str ) -> DeltaResult < String > {
41- let url = Url :: parse ( table_uri)
42- . map_err ( |e| delta_kernel:: Error :: Generic ( format ! ( "invalid table URI {table_uri}: {e}" ) ) ) ?;
43- let store = store_from_url_opts ( & url, HashMap :: < String , String > :: new ( ) ) ?;
44- let engine = DefaultEngineBuilder :: new ( store) . build ( ) ;
45- let snapshot = Snapshot :: builder_for ( url) . build ( & engine) ?;
57+ let ( snapshot, _engine) = open_snapshot ( table_uri) ?;
4658 Ok ( format ! (
4759 "version={:?} schema={:?}" ,
4860 snapshot. version( ) ,
4961 snapshot. schema( )
5062 ) )
5163}
5264
65+ /// Enumerate the data files that make up the latest snapshot of the Delta table at `table_uri`.
66+ /// This performs kernel log replay (including checkpoint + JSON commit reconciliation) and returns
67+ /// one entry per live data file. Phase 2: no predicate pushdown yet, so every live file is
68+ /// returned.
69+ pub fn list_scan_files ( table_uri : & str ) -> DeltaResult < Vec < DeltaScanFile > > {
70+ let ( snapshot, engine) = open_snapshot ( table_uri) ?;
71+ let scan = snapshot. scan_builder ( ) . build ( ) ?;
72+ let mut files: Vec < DeltaScanFile > = Vec :: new ( ) ;
73+ for batch in scan. scan_metadata ( & engine) ? {
74+ files = batch?. visit_scan_files ( files, visit_scan_file) ?;
75+ }
76+ Ok ( files)
77+ }
78+
79+ /// Callback invoked by the kernel for each live data file during log replay.
80+ fn visit_scan_file ( files : & mut Vec < DeltaScanFile > , scan_file : ScanFile ) {
81+ files. push ( DeltaScanFile {
82+ path : scan_file. path . to_string ( ) ,
83+ size : scan_file. size ,
84+ has_transform : scan_file. transform . is_some ( ) ,
85+ } ) ;
86+ }
87+
88+ /// Build a kernel default engine for `table_uri` and load its latest snapshot. The engine is
89+ /// returned alongside the snapshot because scan execution needs it. Credential wiring (S3/Azure/
90+ /// GCS) is intentionally deferred; this uses anonymous/default object-store configuration.
91+ fn open_snapshot (
92+ table_uri : & str ,
93+ ) -> DeltaResult < (
94+ delta_kernel:: SnapshotRef ,
95+ delta_kernel:: engine:: default:: DefaultEngine <
96+ delta_kernel:: engine:: default:: executor:: tokio:: TokioBackgroundExecutor ,
97+ > ,
98+ ) > {
99+ let url = Url :: parse ( table_uri)
100+ . map_err ( |e| delta_kernel:: Error :: Generic ( format ! ( "invalid table URI {table_uri}: {e}" ) ) ) ?;
101+ let store = store_from_url_opts ( & url, HashMap :: < String , String > :: new ( ) ) ?;
102+ let engine = DefaultEngineBuilder :: new ( store) . build ( ) ;
103+ let snapshot = Snapshot :: builder_for ( url) . build ( & engine) ?;
104+ Ok ( ( snapshot, engine) )
105+ }
106+
53107#[ cfg( test) ]
54108mod tests {
55- use super :: snapshot_summary;
109+ use super :: { list_scan_files , snapshot_summary} ;
56110
57111 #[ test]
58112 fn opening_a_missing_table_returns_err ( ) {
59113 // No fixture is created here; the goal is to prove the kernel read path links and is
60114 // callable. A missing table must surface as an error rather than panic.
61- let result = snapshot_summary ( "file:///definitely/not/a/delta/table" ) ;
62- assert ! ( result. is_err( ) , "expected an error for a missing table" ) ;
115+ assert ! ( snapshot_summary( "file:///definitely/not/a/delta/table" ) . is_err( ) ) ;
116+ assert ! ( list_scan_files( "file:///definitely/not/a/delta/table" ) . is_err( ) ) ;
117+ }
118+
119+ /// Happy-path check against a real Delta table. Skipped unless `COMET_DELTA_TEST_TABLE` points
120+ /// at a table URI (e.g. `file:///tmp/my-delta`), so CI stays self-contained. Run locally with:
121+ /// `COMET_DELTA_TEST_TABLE=file:///path cargo test -p datafusion-comet --features delta`.
122+ #[ test]
123+ fn lists_files_for_real_table_when_env_set ( ) {
124+ let Ok ( uri) = std:: env:: var ( "COMET_DELTA_TEST_TABLE" ) else {
125+ eprintln ! ( "skipping: set COMET_DELTA_TEST_TABLE to run" ) ;
126+ return ;
127+ } ;
128+ let summary = snapshot_summary ( & uri) . expect ( "snapshot_summary" ) ;
129+ let files = list_scan_files ( & uri) . expect ( "list_scan_files" ) ;
130+ eprintln ! ( "{summary}" ) ;
131+ eprintln ! ( "scan files ({}): {:#?}" , files. len( ) , files) ;
132+ assert ! ( !files. is_empty( ) , "expected at least one data file" ) ;
63133 }
64134}
0 commit comments