Skip to content

Commit b2981e2

Browse files
committed
fix(reader): rely on Parquet's footer check for delete-file sizes
Parquet already rejects a size below its footer, so drop the separate 8-byte check and assert Parquet's error in the test instead. Also document leaving the size unset when the recorded one can't be trusted (apache/iceberg#12554), add a serde round trip for a delete file without a size, and tidy the metadata-counting test storage.
1 parent 25a34eb commit b2981e2

3 files changed

Lines changed: 68 additions & 62 deletions

File tree

‎crates/iceberg/src/arrow/caching_delete_file_loader.rs‎

Lines changed: 61 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -823,23 +823,48 @@ mod tests {
823823
use std::collections::HashMap;
824824
use std::fs::File;
825825
use std::sync::Arc;
826+
use std::sync::atomic::{AtomicUsize, Ordering};
826827

827828
use arrow_array::cast::AsArray;
828829
use arrow_array::{
829830
ArrayRef, BinaryArray, Int32Array, Int64Array, RecordBatch, StringArray, StructArray,
830831
};
831832
use arrow_schema::{DataType, Field, Fields};
833+
use futures::stream::BoxStream;
832834
use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY};
833835
use parquet::basic::Compression;
834836
use parquet::file::properties::WriterProperties;
835837
use tempfile::TempDir;
836838

837839
use super::*;
838840
use crate::arrow::delete_filter::tests::setup;
841+
use crate::io::{
842+
FileIOBuilder, FileMetadata, FileRead, FileWrite, InputFile, LocalFsStorage, OutputFile,
843+
Storage, StorageConfig, StorageFactory,
844+
};
839845
use crate::scan::FileScanTaskDeleteFile;
840846
use crate::spec::{DataContentType, DataFileFormat, Schema};
841847
use crate::test_utils::encode_dv_blob;
842848

849+
#[test]
850+
fn test_delete_file_without_size_round_trips_through_serde() {
851+
let task = FileScanTaskDeleteFile::builder()
852+
.with_file_path("pos-del.parquet".to_string())
853+
.with_file_type(DataContentType::PositionDeletes)
854+
.with_file_format(DataFileFormat::Parquet)
855+
.with_partition_spec_id(0)
856+
.build()
857+
.unwrap();
858+
assert_eq!(task.file_size_in_bytes(), None);
859+
860+
// The unknown size is left out when serializing, and a task without the
861+
// field deserializes to `None`.
862+
let json = serde_json::to_value(&task).unwrap();
863+
assert!(json.get("file_size_in_bytes").is_none(), "{json}");
864+
let round_tripped: FileScanTaskDeleteFile = serde_json::from_value(json).unwrap();
865+
assert_eq!(round_tripped, task);
866+
}
867+
843868
#[test]
844869
fn test_deserialized_deletion_vector_rejects_missing_fields() {
845870
// A deletion vector that omits its required coordinates fails validation
@@ -1133,88 +1158,82 @@ mod tests {
11331158
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
11341159
struct MetadataCountingStorage {
11351160
#[serde(skip)]
1136-
calls: Arc<std::sync::atomic::AtomicUsize>,
1161+
inner: LocalFsStorage,
1162+
#[serde(skip)]
1163+
calls: Arc<AtomicUsize>,
11371164
/// Fail every metadata request with a retryable error.
11381165
#[serde(skip)]
11391166
fail_metadata: bool,
11401167
}
11411168

11421169
#[async_trait::async_trait]
11431170
#[typetag::serde(name = "test_metadata_counting_storage")]
1144-
impl crate::io::Storage for MetadataCountingStorage {
1145-
async fn exists(&self, path: &str) -> Result<bool> {
1146-
crate::io::LocalFsStorage::new().exists(path).await
1147-
}
1148-
1149-
async fn metadata(&self, path: &str) -> Result<crate::io::FileMetadata> {
1150-
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1171+
impl Storage for MetadataCountingStorage {
1172+
async fn metadata(&self, path: &str) -> Result<FileMetadata> {
1173+
self.calls.fetch_add(1, Ordering::SeqCst);
11511174
if self.fail_metadata {
11521175
return Err(
11531176
Error::new(ErrorKind::Unexpected, "metadata temporarily unavailable")
11541177
.with_retryable(true),
11551178
);
11561179
}
1157-
crate::io::LocalFsStorage::new().metadata(path).await
1180+
self.inner.metadata(path).await
11581181
}
11591182

1160-
async fn read(&self, path: &str) -> Result<Bytes> {
1161-
crate::io::LocalFsStorage::new().read(path).await
1183+
fn new_input(&self, path: &str) -> Result<InputFile> {
1184+
Ok(InputFile::new(Arc::new(self.clone()), path.to_string()))
11621185
}
11631186

1164-
async fn reader(&self, path: &str) -> Result<Box<dyn crate::io::FileRead>> {
1165-
crate::io::LocalFsStorage::new().reader(path).await
1187+
fn new_output(&self, path: &str) -> Result<OutputFile> {
1188+
Ok(OutputFile::new(Arc::new(self.clone()), path.to_string()))
11661189
}
11671190

1168-
async fn write(&self, path: &str, bs: Bytes) -> Result<()> {
1169-
crate::io::LocalFsStorage::new().write(path, bs).await
1191+
async fn exists(&self, path: &str) -> Result<bool> {
1192+
self.inner.exists(path).await
11701193
}
11711194

1172-
async fn writer(&self, path: &str) -> Result<Box<dyn crate::io::FileWrite>> {
1173-
crate::io::LocalFsStorage::new().writer(path).await
1195+
async fn read(&self, path: &str) -> Result<Bytes> {
1196+
self.inner.read(path).await
11741197
}
11751198

1176-
async fn delete(&self, path: &str) -> Result<()> {
1177-
crate::io::LocalFsStorage::new().delete(path).await
1199+
async fn reader(&self, path: &str) -> Result<Box<dyn FileRead>> {
1200+
self.inner.reader(path).await
11781201
}
11791202

1180-
async fn delete_prefix(&self, path: &str) -> Result<()> {
1181-
crate::io::LocalFsStorage::new().delete_prefix(path).await
1203+
async fn write(&self, path: &str, bs: Bytes) -> Result<()> {
1204+
self.inner.write(path, bs).await
11821205
}
11831206

1184-
async fn delete_stream(
1185-
&self,
1186-
paths: futures::stream::BoxStream<'static, String>,
1187-
) -> Result<()> {
1188-
crate::io::LocalFsStorage::new().delete_stream(paths).await
1207+
async fn writer(&self, path: &str) -> Result<Box<dyn FileWrite>> {
1208+
self.inner.writer(path).await
11891209
}
11901210

1191-
fn new_input(&self, path: &str) -> Result<crate::io::InputFile> {
1192-
Ok(crate::io::InputFile::new(
1193-
Arc::new(self.clone()),
1194-
path.to_string(),
1195-
))
1211+
async fn delete(&self, path: &str) -> Result<()> {
1212+
self.inner.delete(path).await
11961213
}
11971214

1198-
fn new_output(&self, path: &str) -> Result<crate::io::OutputFile> {
1199-
Ok(crate::io::OutputFile::new(
1200-
Arc::new(self.clone()),
1201-
path.to_string(),
1202-
))
1215+
async fn delete_prefix(&self, path: &str) -> Result<()> {
1216+
self.inner.delete_prefix(path).await
1217+
}
1218+
1219+
async fn delete_stream(&self, paths: BoxStream<'static, String>) -> Result<()> {
1220+
self.inner.delete_stream(paths).await
12031221
}
12041222
}
12051223

12061224
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
12071225
struct MetadataCountingStorageFactory {
12081226
#[serde(skip)]
1209-
calls: Arc<std::sync::atomic::AtomicUsize>,
1227+
calls: Arc<AtomicUsize>,
12101228
#[serde(skip)]
12111229
fail_metadata: bool,
12121230
}
12131231

12141232
#[typetag::serde(name = "test_metadata_counting_storage_factory")]
1215-
impl crate::io::StorageFactory for MetadataCountingStorageFactory {
1216-
fn build(&self, _config: &crate::io::StorageConfig) -> Result<Arc<dyn crate::io::Storage>> {
1233+
impl StorageFactory for MetadataCountingStorageFactory {
1234+
fn build(&self, _config: &StorageConfig) -> Result<Arc<dyn Storage>> {
12171235
Ok(Arc::new(MetadataCountingStorage {
1236+
inner: LocalFsStorage,
12181237
calls: Arc::clone(&self.calls),
12191238
fail_metadata: self.fail_metadata,
12201239
}))
@@ -1240,13 +1259,11 @@ mod tests {
12401259

12411260
#[tokio::test]
12421261
async fn test_unknown_delete_sizes_are_resolved_once_per_file() {
1243-
use std::sync::atomic::Ordering;
1244-
12451262
let tmp_dir = TempDir::new().unwrap();
12461263
let file_scan_tasks = setup(tmp_dir.path());
12471264
let factory = Arc::new(MetadataCountingStorageFactory::default());
12481265
let calls = Arc::clone(&factory.calls);
1249-
let file_io = crate::io::FileIOBuilder::new(factory).build();
1266+
let file_io = FileIOBuilder::new(factory).build();
12501267
let task = &file_scan_tasks[0];
12511268
let unknown: Vec<_> = task
12521269
.deletes()
@@ -1295,8 +1312,6 @@ mod tests {
12951312

12961313
#[tokio::test]
12971314
async fn test_unknown_size_equality_deletes_are_resolved_once_and_cached() {
1298-
use std::sync::atomic::Ordering;
1299-
13001315
use crate::scan::FileScanTask;
13011316
use crate::spec::DataFileFormat;
13021317

@@ -1341,7 +1356,7 @@ mod tests {
13411356

13421357
let factory = Arc::new(MetadataCountingStorageFactory::default());
13431358
let calls = Arc::clone(&factory.calls);
1344-
let file_io = crate::io::FileIOBuilder::new(factory).build();
1359+
let file_io = FileIOBuilder::new(factory).build();
13451360

13461361
// Known size: no metadata request.
13471362
let known_task = data_task("known.parquet", equality_delete(Some(known_size)));
@@ -1402,7 +1417,7 @@ mod tests {
14021417
fail_metadata: true,
14031418
..Default::default()
14041419
});
1405-
let file_io = crate::io::FileIOBuilder::new(factory).build();
1420+
let file_io = FileIOBuilder::new(factory).build();
14061421
let present = unknown_size(task.deletes()[0].file_path());
14071422
let error = CachingDeleteFileLoader::new(file_io, 10, Runtime::current())
14081423
.load_deletes(&[present], task.schema_ref())

‎crates/iceberg/src/arrow/delete_file_loader.rs‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -94,17 +94,6 @@ impl BasicDeleteFileLoader {
9494
.size
9595
}
9696
};
97-
// Reading a Parquet footer needs at least its last eight bytes. Reject
98-
// a smaller size rather than read the file as if it had no deletes.
99-
if file_size_in_bytes < 8 {
100-
return Err(Error::new(
101-
ErrorKind::DataInvalid,
102-
format!(
103-
"Delete file '{data_file_path}' has a size of {file_size_in_bytes} bytes, \
104-
below the Parquet footer minimum"
105-
),
106-
));
107-
}
10897
let parquet_read_options = ParquetReadOptions::builder().build();
10998

11099
let (parquet_file_reader, arrow_metadata) = ArrowReader::open_parquet_file(
@@ -386,7 +375,7 @@ mod tests {
386375
assert_eq!(error.kind(), storage_error.kind());
387376
assert_eq!(error.retryable(), storage_error.retryable());
388377

389-
// A size below the Parquet footer minimum is rejected whether it was
378+
// A size too small for a Parquet footer fails the read, whether it was
390379
// resolved from storage or supplied by the task.
391380
let empty = temp.path().join("empty.parquet");
392381
std::fs::write(&empty, []).unwrap();
@@ -396,8 +385,7 @@ mod tests {
396385
.await
397386
.err()
398387
.unwrap();
399-
assert_eq!(error.kind(), ErrorKind::DataInvalid);
400-
assert!(error.to_string().contains("footer minimum"), "{error}");
388+
assert!(error.to_string().contains("is less than footer"), "{error}");
401389
}
402390

403391
// A real file with an unknown size is sized lazily and read in full.

‎crates/iceberg/src/scan/task.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -368,8 +368,11 @@ pub struct FileScanTaskDeleteFile {
368368

369369
/// The total size of the delete file in bytes. Tasks planned from manifest entries carry
370370
/// the recorded size. An externally constructed task may leave it unset when the size is
371-
/// unavailable; the Parquet delete-file loader then resolves it with one metadata request
372-
/// when the file is first loaded. Deletion vectors do not use it.
371+
/// unavailable or the recorded size can't be trusted, such as position delete files
372+
/// rewritten by `rewrite_table_path` before Iceberg Java 1.12.0
373+
/// (<https://github.com/apache/iceberg/issues/12554>). The Parquet delete-file loader then
374+
/// resolves it with one metadata request when the file is first loaded. Deletion vectors
375+
/// do not use it.
373376
#[serde(default)]
374377
#[serde(skip_serializing_if = "Option::is_none")]
375378
#[builder(default, setter(strip_option(fallback = with_file_size_in_bytes_opt)))]

0 commit comments

Comments
 (0)