Skip to content

Commit 0228cd4

Browse files
committed
refactor(scan): encapsulate delete file metadata
1 parent e499648 commit 0228cd4

7 files changed

Lines changed: 165 additions & 94 deletions

File tree

‎crates/iceberg/public-api.txt‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1310,17 +1310,18 @@ pub fn iceberg::scan::FileScanTask::serialize<__S>(&self, __serializer: __S) ->
13101310
impl<'de> serde_core::de::Deserialize<'de> for iceberg::scan::FileScanTask
13111311
pub fn iceberg::scan::FileScanTask::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de>
13121312
pub struct iceberg::scan::FileScanTaskDeleteFile
1313-
pub iceberg::scan::FileScanTaskDeleteFile::content_offset: core::option::Option<i64>
1314-
pub iceberg::scan::FileScanTaskDeleteFile::content_size_in_bytes: core::option::Option<i64>
1315-
pub iceberg::scan::FileScanTaskDeleteFile::equality_ids: core::option::Option<alloc::vec::Vec<i32>>
1316-
pub iceberg::scan::FileScanTaskDeleteFile::file_format: iceberg::spec::DataFileFormat
1317-
pub iceberg::scan::FileScanTaskDeleteFile::file_path: alloc::string::String
1318-
pub iceberg::scan::FileScanTaskDeleteFile::file_size_in_bytes: u64
1319-
pub iceberg::scan::FileScanTaskDeleteFile::file_type: iceberg::spec::DataContentType
1320-
pub iceberg::scan::FileScanTaskDeleteFile::key_metadata: core::option::Option<alloc::boxed::Box<[u8]>>
1321-
pub iceberg::scan::FileScanTaskDeleteFile::partition_spec_id: i32
1322-
pub iceberg::scan::FileScanTaskDeleteFile::record_count: core::option::Option<u64>
1323-
pub iceberg::scan::FileScanTaskDeleteFile::referenced_data_file: core::option::Option<alloc::string::String>
1313+
impl iceberg::scan::FileScanTaskDeleteFile
1314+
pub fn iceberg::scan::FileScanTaskDeleteFile::content_offset(&self) -> core::option::Option<i64>
1315+
pub fn iceberg::scan::FileScanTaskDeleteFile::content_size_in_bytes(&self) -> core::option::Option<i64>
1316+
pub fn iceberg::scan::FileScanTaskDeleteFile::equality_ids(&self) -> core::option::Option<&[i32]>
1317+
pub fn iceberg::scan::FileScanTaskDeleteFile::file_format(&self) -> iceberg::spec::DataFileFormat
1318+
pub fn iceberg::scan::FileScanTaskDeleteFile::file_path(&self) -> &str
1319+
pub fn iceberg::scan::FileScanTaskDeleteFile::file_size_in_bytes(&self) -> u64
1320+
pub fn iceberg::scan::FileScanTaskDeleteFile::file_type(&self) -> iceberg::spec::DataContentType
1321+
pub fn iceberg::scan::FileScanTaskDeleteFile::key_metadata(&self) -> core::option::Option<&[u8]>
1322+
pub fn iceberg::scan::FileScanTaskDeleteFile::partition_spec_id(&self) -> i32
1323+
pub fn iceberg::scan::FileScanTaskDeleteFile::record_count(&self) -> core::option::Option<u64>
1324+
pub fn iceberg::scan::FileScanTaskDeleteFile::referenced_data_file(&self) -> core::option::Option<&str>
13241325
impl core::clone::Clone for iceberg::scan::FileScanTaskDeleteFile
13251326
pub fn iceberg::scan::FileScanTaskDeleteFile::clone(&self) -> iceberg::scan::FileScanTaskDeleteFile
13261327
impl core::cmp::PartialEq for iceberg::scan::FileScanTaskDeleteFile

‎crates/iceberg/src/arrow/caching_delete_file_loader.rs‎

Lines changed: 29 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -262,15 +262,15 @@ impl CachingDeleteFileLoader {
262262
del_filter: DeleteFilter,
263263
schema: SchemaRef,
264264
) -> Result<DeleteFileContext> {
265-
match task.file_type {
265+
match task.file_type() {
266266
DataContentType::PositionDeletes => {
267267
// A V3 deletion vector arrives as a PositionDeletes entry whose deletes live in
268268
// a Puffin blob, not in a positional-delete parquet file.
269-
if task.file_format == DataFileFormat::Puffin {
269+
if task.file_format() == DataFileFormat::Puffin {
270270
return Self::load_deletion_vector(task, basic_delete_file_loader).await;
271271
}
272272

273-
match del_filter.try_start_pos_del_load(&task.file_path) {
273+
match del_filter.try_start_pos_del_load(task.file_path()) {
274274
PosDelLoadAction::AlreadyLoaded => Ok(DeleteFileContext::ExistingPosDel),
275275
PosDelLoadAction::WaitFor(notified) => {
276276
// Positional deletes are accessed synchronously by ArrowReader.
@@ -280,35 +280,35 @@ impl CachingDeleteFileLoader {
280280
Ok(DeleteFileContext::ExistingPosDel)
281281
}
282282
PosDelLoadAction::Load => Ok(DeleteFileContext::PosDels {
283-
file_path: task.file_path.clone(),
283+
file_path: task.file_path().to_string(),
284284
stream: basic_delete_file_loader
285285
.parquet_to_batch_stream(
286-
&task.file_path,
287-
task.file_size_in_bytes,
288-
task.key_metadata.as_deref(),
286+
task.file_path(),
287+
task.file_size_in_bytes(),
288+
task.key_metadata(),
289289
)
290290
.await?,
291291
}),
292292
}
293293
}
294294

295295
DataContentType::EqualityDeletes => {
296-
let Some(notify) = del_filter.try_start_eq_del_load(&task.file_path) else {
296+
let Some(notify) = del_filter.try_start_eq_del_load(task.file_path()) else {
297297
return Ok(DeleteFileContext::ExistingEqDel);
298298
};
299299

300300
let (sender, receiver) = channel();
301-
del_filter.insert_equality_delete(&task.file_path, receiver);
301+
del_filter.insert_equality_delete(task.file_path(), receiver);
302302

303303
// Per the Iceberg spec, evolve schema for equality deletes but only for the
304304
// equality_ids columns, not all table columns.
305-
let equality_ids_vec = task.equality_ids.clone().unwrap();
305+
let equality_ids_vec = task.equality_ids().unwrap().to_vec();
306306
let evolved_stream = BasicDeleteFileLoader::evolve_schema(
307307
basic_delete_file_loader
308308
.parquet_to_batch_stream(
309-
&task.file_path,
310-
task.file_size_in_bytes,
311-
task.key_metadata.as_deref(),
309+
task.file_path(),
310+
task.file_size_in_bytes(),
311+
task.key_metadata(),
312312
)
313313
.await?,
314314
schema,
@@ -347,25 +347,25 @@ impl CachingDeleteFileLoader {
347347
task: &FileScanTaskDeleteFile,
348348
) -> Result<(u64, u64, String, u64)> {
349349
let content_offset = task
350-
.content_offset
350+
.content_offset()
351351
.expect("validated deletion vector must have content_offset");
352352
let content_size = task
353-
.content_size_in_bytes
353+
.content_size_in_bytes()
354354
.expect("validated deletion vector must have content_size_in_bytes");
355355
let data_file_path = task
356-
.referenced_data_file
357-
.clone()
356+
.referenced_data_file()
357+
.map(ToOwned::to_owned)
358358
.expect("validated deletion vector must have referenced_data_file");
359359
let record_count = task
360-
.record_count
360+
.record_count()
361361
.expect("validated deletion vector must have record_count");
362362

363363
let start = u64::try_from(content_offset).map_err(|_| {
364364
Error::new(
365365
ErrorKind::DataInvalid,
366366
format!(
367367
"deletion vector {} has negative content_offset {content_offset}",
368-
task.file_path
368+
task.file_path()
369369
),
370370
)
371371
})?;
@@ -374,7 +374,7 @@ impl CachingDeleteFileLoader {
374374
ErrorKind::DataInvalid,
375375
format!(
376376
"deletion vector {} has negative content_size_in_bytes {content_size}",
377-
task.file_path
377+
task.file_path()
378378
),
379379
)
380380
})?;
@@ -419,8 +419,8 @@ impl CachingDeleteFileLoader {
419419

420420
let input_file = basic_delete_file_loader
421421
.file_io()
422-
.new_input(&task.file_path)?;
423-
let blob = match task.key_metadata.as_deref() {
422+
.new_input(task.file_path())?;
423+
let blob = match task.key_metadata() {
424424
Some(key_metadata) => {
425425
let key_metadata = StandardKeyMetadata::decode(key_metadata)?;
426426
EncryptedInputFile::new(input_file, key_metadata)
@@ -436,7 +436,7 @@ impl CachingDeleteFileLoader {
436436
data_file_path,
437437
blob,
438438
record_count,
439-
dv_path: task.file_path.clone(),
439+
dv_path: task.file_path().to_string(),
440440
})
441441
}
442442

@@ -1729,8 +1729,9 @@ mod tests {
17291729

17301730
#[test]
17311731
fn test_validate_deletion_vector_task_rejects_negative_content_offset() {
1732-
let mut task = valid_dv_task();
1733-
task.content_offset = Some(-1);
1732+
let mut task = serde_json::to_value(valid_dv_task()).unwrap();
1733+
task["content_offset"] = serde_json::json!(-1);
1734+
let task = serde_json::from_value(task).unwrap();
17341735

17351736
let err = CachingDeleteFileLoader::validate_deletion_vector_task(&task).unwrap_err();
17361737
assert_eq!(err.kind(), ErrorKind::DataInvalid);
@@ -1739,8 +1740,9 @@ mod tests {
17391740

17401741
#[test]
17411742
fn test_validate_deletion_vector_task_rejects_negative_content_size() {
1742-
let mut task = valid_dv_task();
1743-
task.content_size_in_bytes = Some(-1);
1743+
let mut task = serde_json::to_value(valid_dv_task()).unwrap();
1744+
task["content_size_in_bytes"] = serde_json::json!(-1);
1745+
let task = serde_json::from_value(task).unwrap();
17441746

17451747
let err = CachingDeleteFileLoader::validate_deletion_vector_task(&task).unwrap_err();
17461748
assert_eq!(err.kind(), ErrorKind::DataInvalid);

‎crates/iceberg/src/arrow/delete_file_loader.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -124,16 +124,16 @@ impl DeleteFileLoader for BasicDeleteFileLoader {
124124
) -> Result<ArrowRecordBatchStream> {
125125
let raw_batch_stream = self
126126
.parquet_to_batch_stream(
127-
&task.file_path,
128-
task.file_size_in_bytes,
129-
task.key_metadata.as_deref(),
127+
task.file_path(),
128+
task.file_size_in_bytes(),
129+
task.key_metadata(),
130130
)
131131
.await?;
132132

133133
// For equality deletes, only evolve the equality_ids columns.
134134
// For positional deletes (equality_ids is None), use all field IDs.
135-
let field_ids = match &task.equality_ids {
136-
Some(ids) => ids.clone(),
135+
let field_ids = match task.equality_ids() {
136+
Some(ids) => ids.to_vec(),
137137
None => schema.field_id_to_name_map().keys().cloned().collect(),
138138
};
139139

‎crates/iceberg/src/arrow/delete_filter.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -203,14 +203,14 @@ impl DeleteFilter {
203203
}
204204

205205
let Some(predicate) = self
206-
.get_equality_delete_predicate_for_delete_file_path(&delete.file_path)
206+
.get_equality_delete_predicate_for_delete_file_path(delete.file_path())
207207
.await
208208
else {
209209
return Err(Error::new(
210210
ErrorKind::Unexpected,
211211
format!(
212212
"Missing predicate for equality delete file '{}'",
213-
delete.file_path
213+
delete.file_path()
214214
),
215215
));
216216
};
@@ -274,7 +274,7 @@ impl DeleteFilter {
274274
}
275275

276276
pub(crate) fn is_equality_delete(f: &FileScanTaskDeleteFile) -> bool {
277-
matches!(f.file_type, DataContentType::EqualityDeletes)
277+
matches!(f.file_type(), DataContentType::EqualityDeletes)
278278
}
279279

280280
#[cfg(test)]

‎crates/iceberg/src/delete_file_index.rs‎

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -433,7 +433,7 @@ mod tests {
433433
.unwrap();
434434
let actual_paths_to_apply_for_seq_4: Vec<String> = delete_files_to_apply_for_seq_4
435435
.into_iter()
436-
.map(|file| file.file_path)
436+
.map(|file| file.file_path().to_string())
437437
.collect();
438438

439439
assert_eq!(
@@ -447,7 +447,7 @@ mod tests {
447447
.unwrap();
448448
let actual_paths_to_apply_for_seq_5: Vec<String> = delete_files_to_apply_for_seq_5
449449
.into_iter()
450-
.map(|file| file.file_path)
450+
.map(|file| file.file_path().to_string())
451451
.collect();
452452
assert_eq!(
453453
actual_paths_to_apply_for_seq_5,
@@ -460,7 +460,7 @@ mod tests {
460460
.unwrap();
461461
let actual_paths_to_apply_for_seq_6: Vec<String> = delete_files_to_apply_for_seq_6
462462
.into_iter()
463-
.map(|file| file.file_path)
463+
.map(|file| file.file_path().to_string())
464464
.collect();
465465
assert_eq!(
466466
actual_paths_to_apply_for_seq_6,
@@ -477,7 +477,7 @@ mod tests {
477477
let actual_paths_to_apply_for_partitioned_file: Vec<String> =
478478
delete_files_to_apply_for_partitioned_file
479479
.into_iter()
480-
.map(|file| file.file_path)
480+
.map(|file| file.file_path().to_string())
481481
.collect();
482482
assert_eq!(
483483
actual_paths_to_apply_for_partitioned_file,
@@ -532,7 +532,7 @@ mod tests {
532532
.unwrap();
533533
let actual_paths_to_apply_for_seq_4: Vec<String> = delete_files_to_apply_for_seq_4
534534
.into_iter()
535-
.map(|file| file.file_path)
535+
.map(|file| file.file_path().to_string())
536536
.collect();
537537

538538
assert_eq!(
@@ -546,7 +546,7 @@ mod tests {
546546
.unwrap();
547547
let actual_paths_to_apply_for_seq_5: Vec<String> = delete_files_to_apply_for_seq_5
548548
.into_iter()
549-
.map(|file| file.file_path)
549+
.map(|file| file.file_path().to_string())
550550
.collect();
551551
assert_eq!(
552552
actual_paths_to_apply_for_seq_5,
@@ -559,7 +559,7 @@ mod tests {
559559
.unwrap();
560560
let actual_paths_to_apply_for_seq_6: Vec<String> = delete_files_to_apply_for_seq_6
561561
.into_iter()
562-
.map(|file| file.file_path)
562+
.map(|file| file.file_path().to_string())
563563
.collect();
564564
assert_eq!(
565565
actual_paths_to_apply_for_seq_6,
@@ -575,7 +575,7 @@ mod tests {
575575
let actual_paths_to_apply_for_different_partition: Vec<String> =
576576
delete_files_to_apply_for_different_partition
577577
.into_iter()
578-
.map(|file| file.file_path)
578+
.map(|file| file.file_path().to_string())
579579
.collect();
580580
assert!(actual_paths_to_apply_for_different_partition.is_empty());
581581

@@ -587,7 +587,7 @@ mod tests {
587587
let actual_paths_to_apply_for_different_spec: Vec<String> =
588588
delete_files_to_apply_for_different_spec
589589
.into_iter()
590-
.map(|file| file.file_path)
590+
.map(|file| file.file_path().to_string())
591591
.collect();
592592
assert!(actual_paths_to_apply_for_different_spec.is_empty());
593593
}
@@ -608,7 +608,7 @@ mod tests {
608608
.get_deletes_for_data_file(&data_file_a, Some(0))
609609
.unwrap();
610610
assert_eq!(deletes_for_a.len(), 1);
611-
assert_eq!(deletes_for_a[0].file_path, pos_delete.file_path());
611+
assert_eq!(deletes_for_a[0].file_path(), pos_delete.file_path());
612612

613613
// The delete references data file A, so it must not apply to data file B
614614
// even though B shares A's partition.
@@ -776,7 +776,7 @@ mod tests {
776776
.get_deletes_for_data_file(&data_file, Some(0))
777777
.unwrap()
778778
.into_iter()
779-
.map(|delete| delete.file_path)
779+
.map(|delete| delete.file_path().to_string())
780780
.collect();
781781
actual_paths.sort();
782782

@@ -813,7 +813,7 @@ mod tests {
813813
.get_deletes_for_data_file(&data_file, Some(0))
814814
.unwrap()
815815
.into_iter()
816-
.map(|delete| delete.file_path)
816+
.map(|delete| delete.file_path().to_string())
817817
.collect();
818818
actual_paths.sort();
819819

@@ -852,7 +852,10 @@ mod tests {
852852
.get_deletes_for_data_file(&data_file, Some(0))
853853
.unwrap();
854854
assert_eq!(deletes_for_referenced.len(), 1);
855-
assert_eq!(deletes_for_referenced[0].file_path, pos_delete.file_path());
855+
assert_eq!(
856+
deletes_for_referenced[0].file_path(),
857+
pos_delete.file_path()
858+
);
856859

857860
assert!(
858861
index
@@ -956,12 +959,12 @@ mod tests {
956959
};
957960

958961
let task: FileScanTaskDeleteFile = (&ctx).into();
959-
assert_eq!(task.file_type, DataContentType::PositionDeletes);
960-
assert_eq!(task.content_offset, Some(4));
961-
assert_eq!(task.content_size_in_bytes, Some(40));
962-
assert_eq!(task.record_count, Some(3));
962+
assert_eq!(task.file_type(), DataContentType::PositionDeletes);
963+
assert_eq!(task.content_offset(), Some(4));
964+
assert_eq!(task.content_size_in_bytes(), Some(40));
965+
assert_eq!(task.record_count(), Some(3));
963966
assert_eq!(
964-
task.referenced_data_file.as_deref(),
967+
task.referenced_data_file(),
965968
Some("s3://bucket/data/part-0.parquet")
966969
);
967970
}
@@ -992,7 +995,7 @@ mod tests {
992995
.get_deletes_for_data_file(&data_file, Some(0))
993996
.unwrap()
994997
.into_iter()
995-
.map(|f| f.file_path)
998+
.map(|f| f.file_path().to_string())
996999
.collect();
9971000

9981001
// Only the deletion vector applies; the partition-scoped position delete file, which
@@ -1107,7 +1110,7 @@ mod tests {
11071110
.get_deletes_for_data_file(&data_file, Some(0))
11081111
.unwrap()
11091112
.into_iter()
1110-
.map(|f| f.file_path)
1113+
.map(|f| f.file_path().to_string())
11111114
.collect();
11121115

11131116
// Only the deletion vector applies; the path-scoped position delete file, which would
@@ -1142,7 +1145,7 @@ mod tests {
11421145
.get_deletes_for_data_file(&data_file, Some(0))
11431146
.unwrap()
11441147
.into_iter()
1145-
.map(|f| f.file_path)
1148+
.map(|f| f.file_path().to_string())
11461149
.collect();
11471150
applied.sort();
11481151

0 commit comments

Comments
 (0)