Skip to content

Commit 632340c

Browse files
committed
fix(arrow): pass the whole load error to waiters
1 parent 5fa6051 commit 632340c

2 files changed

Lines changed: 45 additions & 63 deletions

File tree

‎crates/iceberg/src/arrow/caching_delete_file_loader.rs‎

Lines changed: 38 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1938,7 +1938,7 @@ mod tests {
19381938
.err()
19391939
.unwrap();
19401940
assert_eq!(err.kind(), ErrorKind::Unexpected);
1941-
assert_eq!(err.message(), "load failed");
1941+
assert!(err.message().contains("load failed"), "{err}");
19421942
}
19431943
}
19441944

@@ -2067,43 +2067,49 @@ mod tests {
20672067
}
20682068
}
20692069

2070+
/// Writes a position delete file at `path` for `data.parquet` with `positions`.
2071+
fn write_pos_del_file(path: &str, positions: Vec<i64>) -> FileScanTaskDeleteFile {
2072+
let batch = RecordBatch::try_new(
2073+
crate::arrow::delete_filter::tests::create_pos_del_schema(),
2074+
vec![
2075+
Arc::new(StringArray::from_iter_values(vec![
2076+
"data.parquet";
2077+
positions.len()
2078+
])),
2079+
Arc::new(Int64Array::from_iter_values(positions)),
2080+
],
2081+
)
2082+
.unwrap();
2083+
let mut writer =
2084+
ArrowWriter::try_new(File::create(path).unwrap(), batch.schema(), None).unwrap();
2085+
writer.write(&batch).unwrap();
2086+
writer.close().unwrap();
2087+
FileScanTaskDeleteFile::builder()
2088+
.with_file_path(path.to_string())
2089+
.with_file_size_in_bytes(std::fs::metadata(path).unwrap().len())
2090+
.with_file_type(DataContentType::PositionDeletes)
2091+
.with_file_format(DataFileFormat::Parquet)
2092+
.with_partition_spec_id(0)
2093+
.build()
2094+
.unwrap()
2095+
}
2096+
20702097
/// A failing positional delete file ends `load_deletes` early while the load of the
20712098
/// other file in the same call is still in flight, dropping its guard. That file
20722099
/// still loads later.
20732100
#[tokio::test]
20742101
async fn test_pos_del_load_dropped_by_a_failing_sibling_can_be_loaded_later() {
20752102
let tmp_dir = TempDir::new().unwrap();
20762103
let schema = Arc::new(Schema::builder().build().unwrap());
2077-
let write = |name: &str, positions: Vec<i64>| {
2078-
let path = tmp_dir.path().join(name);
2079-
let path = path.to_str().unwrap().to_string();
2080-
let batch = RecordBatch::try_new(
2081-
crate::arrow::delete_filter::tests::create_pos_del_schema(),
2082-
vec![
2083-
Arc::new(StringArray::from_iter_values(vec![
2084-
"data.parquet";
2085-
positions.len()
2086-
])),
2087-
Arc::new(Int64Array::from_iter_values(positions)),
2088-
],
2089-
)
2090-
.unwrap();
2091-
let mut writer =
2092-
ArrowWriter::try_new(File::create(&path).unwrap(), batch.schema(), None).unwrap();
2093-
writer.write(&batch).unwrap();
2094-
writer.close().unwrap();
2095-
FileScanTaskDeleteFile::builder()
2096-
.with_file_path(path.clone())
2097-
.with_file_size_in_bytes(std::fs::metadata(&path).unwrap().len())
2098-
.with_file_type(DataContentType::PositionDeletes)
2099-
.with_file_format(DataFileFormat::Parquet)
2100-
.with_partition_spec_id(0)
2101-
.build()
2102-
.unwrap()
2103-
};
2104-
let invalid = write("invalid.parquet", vec![-1]);
2104+
let invalid = write_pos_del_file(
2105+
tmp_dir.path().join("invalid.parquet").to_str().unwrap(),
2106+
vec![-1],
2107+
);
21052108
// Its read waits for the gate, so its load is in flight when `invalid` fails.
2106-
let valid = write("gated.parquet", vec![1, 3]);
2109+
let valid = write_pos_del_file(
2110+
tmp_dir.path().join("gated.parquet").to_str().unwrap(),
2111+
vec![1, 3],
2112+
);
21072113
let factory = Arc::new(gated_storage::GatedStorageFactory::default());
21082114
let gate = Arc::clone(&factory.open);
21092115
let file_io = crate::io::FileIOBuilder::new(factory).build();
@@ -2142,35 +2148,10 @@ mod tests {
21422148
let path = tmp_dir.path().join("pos-del.parquet");
21432149
let path = path.to_str().unwrap();
21442150
let schema = Arc::new(Schema::builder().build().unwrap());
2145-
let write = |positions: Vec<i64>| {
2146-
let batch = RecordBatch::try_new(
2147-
crate::arrow::delete_filter::tests::create_pos_del_schema(),
2148-
vec![
2149-
Arc::new(StringArray::from_iter_values(vec![
2150-
"data.parquet";
2151-
positions.len()
2152-
])),
2153-
Arc::new(Int64Array::from_iter_values(positions)),
2154-
],
2155-
)
2156-
.unwrap();
2157-
let mut writer =
2158-
ArrowWriter::try_new(File::create(path).unwrap(), batch.schema(), None).unwrap();
2159-
writer.write(&batch).unwrap();
2160-
writer.close().unwrap();
2161-
FileScanTaskDeleteFile::builder()
2162-
.with_file_path(path.to_string())
2163-
.with_file_size_in_bytes(std::fs::metadata(path).unwrap().len())
2164-
.with_file_type(DataContentType::PositionDeletes)
2165-
.with_file_format(DataFileFormat::Parquet)
2166-
.with_partition_spec_id(0)
2167-
.build()
2168-
.unwrap()
2169-
};
21702151
let loader = CachingDeleteFileLoader::new(FileIO::new_with_fs(), 10, Runtime::current());
21712152

21722153
// The file opens but fails to parse, so the load is abandoned after the open.
2173-
let invalid = write(vec![-1]);
2154+
let invalid = write_pos_del_file(path, vec![-1]);
21742155
let err = loader
21752156
.load_deletes(&[invalid], Arc::clone(&schema))
21762157
.await
@@ -2180,7 +2161,7 @@ mod tests {
21802161
assert!(err.message().contains("negative position"), "{err}");
21812162

21822163
// Once the file is valid, the same loader loads it instead of waiting.
2183-
let valid = write(vec![1, 3]);
2164+
let valid = write_pos_del_file(path, vec![1, 3]);
21842165
let filter = tokio::time::timeout(
21852166
std::time::Duration::from_secs(5),
21862167
loader.load_deletes(&[valid], schema),

‎crates/iceberg/src/arrow/delete_filter.rs‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -128,10 +128,7 @@ impl PosDelLoadGuard {
128128
/// Ends the load with `error`, which the tasks waiting on it return. Later calls to
129129
/// [`DeleteFilter::try_start_pos_del_load`] start a new attempt.
130130
pub(crate) fn fail(self, error: &Error) {
131-
let _ = self
132-
.attempt
133-
.failure
134-
.set((error.kind(), error.message().to_string()));
131+
let _ = self.attempt.failure.set((error.kind(), error.to_string()));
135132
}
136133
}
137134

@@ -473,15 +470,19 @@ pub(crate) mod tests {
473470
})
474471
.collect();
475472

476-
guard.fail(&Error::new(ErrorKind::DataInvalid, "bad file"));
473+
guard.fail(
474+
&Error::new(ErrorKind::DataInvalid, "bad file")
475+
.with_source(std::io::Error::other("403 Forbidden")),
476+
);
477477

478478
for wait in waits {
479479
let err = tokio::time::timeout(std::time::Duration::from_secs(5), wait.wait())
480480
.await
481481
.expect("WaitFor future must resolve when the load fails")
482482
.unwrap_err();
483483
assert_eq!(err.kind(), ErrorKind::DataInvalid);
484-
assert_eq!(err.message(), "bad file");
484+
assert!(err.message().contains("bad file"), "{err}");
485+
assert!(err.message().contains("403 Forbidden"), "{err}");
485486
}
486487

487488
// Only the waiters of that attempt get the error; a later call loads again.

0 commit comments

Comments
 (0)