|
30 | 30 | use std::collections::HashMap; |
31 | 31 | use std::sync::Arc; |
32 | 32 |
|
33 | | -use delta_kernel::arrow::array::RecordBatch; |
| 33 | +use delta_kernel::arrow::array::{BooleanArray, RecordBatch}; |
| 34 | +use delta_kernel::arrow::compute::filter_record_batch; |
34 | 35 | use delta_kernel::arrow::datatypes::SchemaRef as ArrowSchemaRef; |
35 | 36 | use delta_kernel::committer::FileSystemCommitter; |
36 | 37 | use delta_kernel::engine::arrow_conversion::{TryFromArrow, TryIntoArrow}; |
37 | 38 | use delta_kernel::engine::arrow_data::{ArrowEngineData, EngineDataArrowExt}; |
38 | 39 | use delta_kernel::engine::default::executor::tokio::TokioBackgroundExecutor; |
39 | 40 | use delta_kernel::engine::default::storage::store_from_url_opts; |
40 | 41 | use delta_kernel::engine::default::{DefaultEngine, DefaultEngineBuilder}; |
41 | | -use delta_kernel::scan::state::ScanFile; |
| 42 | +use delta_kernel::scan::state::{transform_to_logical, ScanFile}; |
42 | 43 | use delta_kernel::schema::{SchemaRef, StructType}; |
43 | 44 | use delta_kernel::transaction::create_table::create_table as kernel_create_table; |
44 | 45 | use delta_kernel::transaction::{CommitResult, RetryableTransaction}; |
45 | | -use delta_kernel::{DeltaResult, Error, Snapshot, SnapshotRef}; |
| 46 | +use delta_kernel::{DeltaResult, Engine, Error, FileMeta, Snapshot, SnapshotRef}; |
46 | 47 | use itertools::Itertools; |
47 | 48 | use url::Url; |
48 | 49 |
|
@@ -79,6 +80,35 @@ fn open_snapshot(table_uri: &str) -> DeltaResult<(SnapshotRef, KernelEngine)> { |
79 | 80 | Ok((snapshot, engine)) |
80 | 81 | } |
81 | 82 |
|
| 83 | +/// Open a snapshot of the Delta table at `table_uri`, pinned to `version` when `Some` (else |
| 84 | +/// latest). Used so every split-read partition reads the exact same snapshot. |
| 85 | +fn open_snapshot_at( |
| 86 | + table_uri: &str, |
| 87 | + version: Option<u64>, |
| 88 | +) -> DeltaResult<(SnapshotRef, KernelEngine)> { |
| 89 | + let (url, engine) = build_engine(table_uri)?; |
| 90 | + let mut builder = Snapshot::builder_for(url); |
| 91 | + if let Some(v) = version { |
| 92 | + builder = builder.at_version(v); |
| 93 | + } |
| 94 | + let snapshot = builder.build(&engine)?; |
| 95 | + Ok((snapshot, engine)) |
| 96 | +} |
| 97 | + |
| 98 | +/// Return the latest snapshot's version and live data-file count for the table at `table_uri`. The |
| 99 | +/// driver uses this to pin the version and choose a partition count for split-parallel reads. |
| 100 | +pub fn snapshot_info(table_uri: &str) -> DeltaResult<(u64, u64)> { |
| 101 | + let (snapshot, engine) = open_snapshot(table_uri)?; |
| 102 | + let version = snapshot.version(); |
| 103 | + let scan = snapshot.scan_builder().build()?; |
| 104 | + let mut count: u64 = 0; |
| 105 | + for batch in scan.scan_metadata(&engine)? { |
| 106 | + let files = batch?.visit_scan_files(Vec::<()>::new(), |acc, _f| acc.push(()))?; |
| 107 | + count += files.len() as u64; |
| 108 | + } |
| 109 | + Ok((version, count)) |
| 110 | +} |
| 111 | + |
82 | 112 | /// Open the latest snapshot of the Delta table at `table_uri` and return a short human-readable |
83 | 113 | /// summary (version + logical schema). |
84 | 114 | pub fn snapshot_summary(table_uri: &str) -> DeltaResult<String> { |
@@ -156,6 +186,98 @@ pub fn scan_to_batches( |
156 | 186 | Ok(batches) |
157 | 187 | } |
158 | 188 |
|
| 189 | +/// Split-parallel read: read only the data files assigned to partition `partition_index` of |
| 190 | +/// `num_partitions`, from the snapshot pinned at `version`. Files are enumerated, sorted by path |
| 191 | +/// for a deterministic cross-partition assignment, then partitioned round-robin by index. This |
| 192 | +/// replicates the kernel's per-file read loop (deletion vectors via a keep-mask filter, plus the |
| 193 | +/// logical transform for column mapping / partition values) using only public kernel APIs, so the |
| 194 | +/// output is identical to `scan_to_batches` for the assigned subset. Used by `DeltaScanExec` when |
| 195 | +/// the JVM schedules more than one partition, so the read parallelizes across Spark cores. |
| 196 | +pub fn scan_to_batches_split( |
| 197 | + table_uri: &str, |
| 198 | + columns: Option<&[String]>, |
| 199 | + predicate: Option<delta_kernel::expressions::PredicateRef>, |
| 200 | + version: Option<u64>, |
| 201 | + partition_index: usize, |
| 202 | + num_partitions: usize, |
| 203 | +) -> DeltaResult<Vec<RecordBatch>> { |
| 204 | + let num_partitions = num_partitions.max(1); |
| 205 | + let (snapshot, engine) = open_snapshot_at(table_uri, version)?; |
| 206 | + let read_schema = projected_read_schema(&snapshot, columns)?; |
| 207 | + let scan = snapshot |
| 208 | + .scan_builder() |
| 209 | + .with_schema_opt(read_schema) |
| 210 | + .with_predicate(predicate) |
| 211 | + .build()?; |
| 212 | + |
| 213 | + // Enumerate all live scan files, then sort by path so every partition derives the identical |
| 214 | + // ordering independently (the round-robin assignment below depends on a stable order). |
| 215 | + let mut scan_files: Vec<ScanFile> = Vec::new(); |
| 216 | + for batch in scan.scan_metadata(&engine)? { |
| 217 | + scan_files = batch?.visit_scan_files(scan_files, |acc, f| acc.push(f))?; |
| 218 | + } |
| 219 | + scan_files.sort_by(|a, b| a.path.cmp(&b.path)); |
| 220 | + |
| 221 | + let table_root = scan.snapshot().table_root().clone(); |
| 222 | + let physical_schema = Arc::clone(scan.physical_schema()); |
| 223 | + let logical_schema = Arc::clone(scan.logical_schema()); |
| 224 | + |
| 225 | + let mut batches: Vec<RecordBatch> = Vec::new(); |
| 226 | + for scan_file in scan_files |
| 227 | + .into_iter() |
| 228 | + .enumerate() |
| 229 | + .filter(|(i, _)| i % num_partitions == partition_index) |
| 230 | + .map(|(_, f)| f) |
| 231 | + { |
| 232 | + let file_path = table_root.join(&scan_file.path)?; |
| 233 | + // Deletion-vector keep-mask for the whole file (true = keep). None when the file has no DV. |
| 234 | + let mut keep_mask: Option<Vec<bool>> = scan_file |
| 235 | + .dv_info |
| 236 | + .get_selection_vector(&engine, &table_root)?; |
| 237 | + let meta = FileMeta { |
| 238 | + last_modified: 0, |
| 239 | + size: scan_file.size as u64, |
| 240 | + location: file_path, |
| 241 | + }; |
| 242 | + let read_iter = engine.parquet_handler().read_parquet_files( |
| 243 | + &[meta], |
| 244 | + Arc::clone(&physical_schema), |
| 245 | + None, |
| 246 | + )?; |
| 247 | + for read_result in read_iter { |
| 248 | + let logical = transform_to_logical( |
| 249 | + &engine, |
| 250 | + read_result?, |
| 251 | + &physical_schema, |
| 252 | + &logical_schema, |
| 253 | + scan_file.transform.clone(), |
| 254 | + )?; |
| 255 | + let batch = logical.try_into_record_batch()?; |
| 256 | + let n = batch.num_rows(); |
| 257 | + // Consume the leading `n` entries of the keep mask for this batch; the remainder |
| 258 | + // applies to subsequent batches of the same file. Rows past the mask are kept. |
| 259 | + let masked = match keep_mask.as_mut() { |
| 260 | + Some(mask) => { |
| 261 | + let rest = if n < mask.len() { |
| 262 | + Some(mask.split_off(n)) |
| 263 | + } else { |
| 264 | + None |
| 265 | + }; |
| 266 | + let mut this = std::mem::take(mask); |
| 267 | + keep_mask = rest; |
| 268 | + if this.len() < n { |
| 269 | + this.resize(n, true); |
| 270 | + } |
| 271 | + Some(filter_record_batch(&batch, &BooleanArray::from(this))?) |
| 272 | + } |
| 273 | + None => None, |
| 274 | + }; |
| 275 | + batches.push(masked.unwrap_or(batch)); |
| 276 | + } |
| 277 | + } |
| 278 | + Ok(batches) |
| 279 | +} |
| 280 | + |
159 | 281 | /// Read the latest snapshot's logical schema as an Arrow schema, projected to `columns` when |
160 | 282 | /// `Some`. Used by `DeltaScanExec` to report its output schema without reading any data. |
161 | 283 | pub fn snapshot_arrow_schema( |
@@ -335,6 +457,77 @@ mod tests { |
335 | 457 | batches.iter().map(|b| b.num_rows()).sum() |
336 | 458 | } |
337 | 459 |
|
| 460 | + fn sorted_ids(batches: &[RecordBatch]) -> Vec<i64> { |
| 461 | + let mut ids: Vec<i64> = batches |
| 462 | + .iter() |
| 463 | + .flat_map(|b| { |
| 464 | + b.column(0) |
| 465 | + .as_any() |
| 466 | + .downcast_ref::<Int64Array>() |
| 467 | + .unwrap() |
| 468 | + .values() |
| 469 | + .to_vec() |
| 470 | + }) |
| 471 | + .collect(); |
| 472 | + ids.sort_unstable(); |
| 473 | + ids |
| 474 | + } |
| 475 | + |
| 476 | + #[test] |
| 477 | + fn split_read_covers_all_rows_exactly_once() { |
| 478 | + let dir = TempDir::new().unwrap(); |
| 479 | + let uri = table_uri(&dir); |
| 480 | + let schema = test_schema(); |
| 481 | + create_table(&uri, Arc::clone(&schema)).unwrap(); |
| 482 | + // five commits => five data files, so a 3-way split has uneven file counts per partition. |
| 483 | + for i in 0..5 { |
| 484 | + rt().block_on(append(&uri, make_batch(&schema, i * 10, 10))) |
| 485 | + .unwrap(); |
| 486 | + } |
| 487 | + |
| 488 | + let (version, num_files) = snapshot_info(&uri).unwrap(); |
| 489 | + assert_eq!(num_files, 5); |
| 490 | + |
| 491 | + let full = sorted_ids(&scan_to_batches(&uri, None, None).unwrap()); |
| 492 | + assert_eq!(full.len(), 50); |
| 493 | + |
| 494 | + let num_partitions = 3; |
| 495 | + let mut combined: Vec<i64> = Vec::new(); |
| 496 | + for p in 0..num_partitions { |
| 497 | + let part = |
| 498 | + scan_to_batches_split(&uri, None, None, Some(version), p, num_partitions).unwrap(); |
| 499 | + combined.extend(sorted_ids(&part)); |
| 500 | + } |
| 501 | + combined.sort_unstable(); |
| 502 | + // Every row is read exactly once across the partitions, matching the full read. |
| 503 | + assert_eq!(combined, full); |
| 504 | + } |
| 505 | + |
| 506 | + #[test] |
| 507 | + fn split_read_with_projection_matches_full() { |
| 508 | + let dir = TempDir::new().unwrap(); |
| 509 | + let uri = table_uri(&dir); |
| 510 | + let schema = test_schema(); |
| 511 | + create_table(&uri, Arc::clone(&schema)).unwrap(); |
| 512 | + for i in 0..4 { |
| 513 | + rt().block_on(append(&uri, make_batch(&schema, i * 8, 8))) |
| 514 | + .unwrap(); |
| 515 | + } |
| 516 | + let (version, _) = snapshot_info(&uri).unwrap(); |
| 517 | + let cols = vec!["id".to_string()]; |
| 518 | + |
| 519 | + let mut total = 0usize; |
| 520 | + for p in 0..2 { |
| 521 | + let part = scan_to_batches_split(&uri, Some(&cols), None, Some(version), p, 2).unwrap(); |
| 522 | + for b in &part { |
| 523 | + assert_eq!(b.num_columns(), 1); |
| 524 | + assert_eq!(b.schema().field(0).name(), "id"); |
| 525 | + } |
| 526 | + total += total_rows(&part); |
| 527 | + } |
| 528 | + assert_eq!(total, 32); |
| 529 | + } |
| 530 | + |
338 | 531 | #[test] |
339 | 532 | fn opening_a_missing_table_returns_err() { |
340 | 533 | assert!(snapshot_summary("file:///definitely/not/a/delta/table").is_err()); |
|
0 commit comments