Skip to content

Commit 83fbeeb

Browse files
adivaishCopilot
andcommitted
feat(delta): enumerate scan files via delta-kernel-rs (phase 2)
Adds list_scan_files(), which performs kernel log replay and returns one DeltaScanFile {path, size, has_transform} per live data file via scan_metadata/visit_scan_files. Validated against a real Spark-written Delta table: version + schema read correctly and all 5 data files enumerated with correct paths and sizes. Adds an env-gated (COMET_DELTA_TEST_TABLE) happy-path test that skips in CI. Part of apache#174. Co-authored-by: Copilot <[email protected]>
1 parent db03b6a commit 83fbeeb

1 file changed

Lines changed: 78 additions & 8 deletions

File tree

‎native/core/src/delta/mod.rs‎

Lines changed: 78 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,35 +30,105 @@ use std::collections::HashMap;
3030

3131
use delta_kernel::engine::default::storage::store_from_url_opts;
3232
use delta_kernel::engine::default::DefaultEngineBuilder;
33+
use delta_kernel::scan::state::ScanFile;
3334
use delta_kernel::{DeltaResult, Snapshot};
3435
use url::Url;
3536

37+
/// A single Delta data file to scan, produced by kernel log replay on the driver. This is the
38+
/// minimal per-file information Comet needs to later build a `PartitionedFile` for its native
39+
/// Parquet reader. Partition values, deletion vectors, and column-mapping transforms are carried
40+
/// in follow-up phases.
41+
#[derive(Debug, Clone, PartialEq, Eq)]
42+
pub struct DeltaScanFile {
43+
/// File path relative to the table root.
44+
pub path: String,
45+
/// File size in bytes (from the Delta log; lets Comet skip a HEAD/stat call).
46+
pub size: i64,
47+
/// Whether the kernel attached a logical-to-physical transform (column mapping / partition
48+
/// value injection) that must be applied after reading. Handled in a later phase.
49+
pub has_transform: bool,
50+
}
51+
3652
/// Open the latest snapshot of the Delta table at `table_uri` and return a short human-readable
3753
/// summary (version + logical schema). Phase 1 scaffolding: it exercises kernel log replay via the
3854
/// default (object-store-backed) engine. Cloud credential wiring, file/predicate serialization,
3955
/// and execution through Comet's `ParquetSource` land in later commits.
4056
pub fn snapshot_summary(table_uri: &str) -> DeltaResult<String> {
41-
let url = Url::parse(table_uri)
42-
.map_err(|e| delta_kernel::Error::Generic(format!("invalid table URI {table_uri}: {e}")))?;
43-
let store = store_from_url_opts(&url, HashMap::<String, String>::new())?;
44-
let engine = DefaultEngineBuilder::new(store).build();
45-
let snapshot = Snapshot::builder_for(url).build(&engine)?;
57+
let (snapshot, _engine) = open_snapshot(table_uri)?;
4658
Ok(format!(
4759
"version={:?} schema={:?}",
4860
snapshot.version(),
4961
snapshot.schema()
5062
))
5163
}
5264

65+
/// Enumerate the data files that make up the latest snapshot of the Delta table at `table_uri`.
66+
/// This performs kernel log replay (including checkpoint + JSON commit reconciliation) and returns
67+
/// one entry per live data file. Phase 2: no predicate pushdown yet, so every live file is
68+
/// returned.
69+
pub fn list_scan_files(table_uri: &str) -> DeltaResult<Vec<DeltaScanFile>> {
70+
let (snapshot, engine) = open_snapshot(table_uri)?;
71+
let scan = snapshot.scan_builder().build()?;
72+
let mut files: Vec<DeltaScanFile> = Vec::new();
73+
for batch in scan.scan_metadata(&engine)? {
74+
files = batch?.visit_scan_files(files, visit_scan_file)?;
75+
}
76+
Ok(files)
77+
}
78+
79+
/// Callback invoked by the kernel for each live data file during log replay.
80+
fn visit_scan_file(files: &mut Vec<DeltaScanFile>, scan_file: ScanFile) {
81+
files.push(DeltaScanFile {
82+
path: scan_file.path.to_string(),
83+
size: scan_file.size,
84+
has_transform: scan_file.transform.is_some(),
85+
});
86+
}
87+
88+
/// Build a kernel default engine for `table_uri` and load its latest snapshot. The engine is
89+
/// returned alongside the snapshot because scan execution needs it. Credential wiring (S3/Azure/
90+
/// GCS) is intentionally deferred; this uses anonymous/default object-store configuration.
91+
fn open_snapshot(
92+
table_uri: &str,
93+
) -> DeltaResult<(
94+
delta_kernel::SnapshotRef,
95+
delta_kernel::engine::default::DefaultEngine<
96+
delta_kernel::engine::default::executor::tokio::TokioBackgroundExecutor,
97+
>,
98+
)> {
99+
let url = Url::parse(table_uri)
100+
.map_err(|e| delta_kernel::Error::Generic(format!("invalid table URI {table_uri}: {e}")))?;
101+
let store = store_from_url_opts(&url, HashMap::<String, String>::new())?;
102+
let engine = DefaultEngineBuilder::new(store).build();
103+
let snapshot = Snapshot::builder_for(url).build(&engine)?;
104+
Ok((snapshot, engine))
105+
}
106+
53107
#[cfg(test)]
54108
mod tests {
55-
use super::snapshot_summary;
109+
use super::{list_scan_files, snapshot_summary};
56110

57111
#[test]
58112
fn opening_a_missing_table_returns_err() {
59113
// No fixture is created here; the goal is to prove the kernel read path links and is
60114
// callable. A missing table must surface as an error rather than panic.
61-
let result = snapshot_summary("file:///definitely/not/a/delta/table");
62-
assert!(result.is_err(), "expected an error for a missing table");
115+
assert!(snapshot_summary("file:///definitely/not/a/delta/table").is_err());
116+
assert!(list_scan_files("file:///definitely/not/a/delta/table").is_err());
117+
}
118+
119+
/// Happy-path check against a real Delta table. Skipped unless `COMET_DELTA_TEST_TABLE` points
120+
/// at a table URI (e.g. `file:///tmp/my-delta`), so CI stays self-contained. Run locally with:
121+
/// `COMET_DELTA_TEST_TABLE=file:///path cargo test -p datafusion-comet --features delta`.
122+
#[test]
123+
fn lists_files_for_real_table_when_env_set() {
124+
let Ok(uri) = std::env::var("COMET_DELTA_TEST_TABLE") else {
125+
eprintln!("skipping: set COMET_DELTA_TEST_TABLE to run");
126+
return;
127+
};
128+
let summary = snapshot_summary(&uri).expect("snapshot_summary");
129+
let files = list_scan_files(&uri).expect("list_scan_files");
130+
eprintln!("{summary}");
131+
eprintln!("scan files ({}): {:#?}", files.len(), files);
132+
assert!(!files.is_empty(), "expected at least one data file");
63133
}
64134
}

0 commit comments

Comments
 (0)