Skip to content

Commit d51812c

Browse files
committed
fix(arrow): return a failed positional delete load's error to its waiters
Waiters on a failed load used to retry it one after another, so a missing file shared by many tasks was opened once per task. The load now records its error and the tasks waiting on that attempt return it. A load dropped without an error, when a sibling's error stops load_deletes, still lets its waiters retry, and later calls start a new attempt either way. Abandoning a load also tolerates a poisoned lock, since it runs from Drop.
1 parent 339fff4 commit d51812c

2 files changed

Lines changed: 259 additions & 73 deletions

File tree

‎crates/iceberg/src/arrow/caching_delete_file_loader.rs‎

Lines changed: 152 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -276,23 +276,29 @@ impl CachingDeleteFileLoader {
276276
PosDelLoadAction::AlreadyLoaded => {
277277
return Ok(DeleteFileContext::ExistingPosDel);
278278
}
279-
PosDelLoadAction::WaitFor(notified) => {
279+
PosDelLoadAction::WaitFor(wait) => {
280280
// Positional deletes are accessed synchronously by ArrowReader.
281281
// We must wait here to ensure the data is ready before returning,
282282
// otherwise ArrowReader might get an empty/partial result.
283-
// The wait also ends if that load fails, so check again.
284-
notified.await;
283+
// A failed load returns its error here. A load dropped without
284+
// one (cancelled) ends the wait too, so check again.
285+
wait.wait().await?;
285286
}
286287
PosDelLoadAction::Load(guard) => {
287-
// On error the guard is dropped, which releases the file for a
288-
// later attempt.
289-
let stream = basic_delete_file_loader
288+
let stream = match basic_delete_file_loader
290289
.parquet_to_batch_stream(
291290
task.file_path(),
292291
task.file_size_in_bytes(),
293292
task.key_metadata(),
294293
)
295-
.await?;
294+
.await
295+
{
296+
Ok(stream) => stream,
297+
Err(err) => {
298+
guard.fail(&err);
299+
return Err(err);
300+
}
301+
};
296302
return Ok(DeleteFileContext::PosDels { guard, stream });
297303
}
298304
}
@@ -442,11 +448,16 @@ impl CachingDeleteFileLoader {
442448
DeleteFileContext::ExistingEqDel => Ok(ParsedDeleteFileContext::EqDel),
443449
DeleteFileContext::ExistingPosDel => Ok(ParsedDeleteFileContext::ExistingPosDel),
444450
DeleteFileContext::PosDels { guard, stream } => {
445-
let del_vecs = Self::parse_positional_deletes_record_batch_stream(stream).await?;
446-
Ok(ParsedDeleteFileContext::DelVecs {
447-
guard,
448-
results: del_vecs,
449-
})
451+
match Self::parse_positional_deletes_record_batch_stream(stream).await {
452+
Ok(del_vecs) => Ok(ParsedDeleteFileContext::DelVecs {
453+
guard,
454+
results: del_vecs,
455+
}),
456+
Err(err) => {
457+
guard.fail(&err);
458+
Err(err)
459+
}
460+
}
450461
}
451462
DeleteFileContext::DelVec {
452463
data_file_path,
@@ -1849,15 +1860,17 @@ mod tests {
18491860
assert_eq!(positions(&retried), positions(&fresh));
18501861
}
18511862

1863+
/// The load of a positional delete file taken by the test, and tasks polled until
1864+
/// they wait on it.
1865+
struct PendingPosDelLoad {
1866+
guard: PosDelLoadGuard,
1867+
waiters: Vec<futures::future::BoxFuture<'static, Result<DeleteFileContext>>>,
1868+
path: String,
1869+
}
1870+
18521871
/// Takes the load of the first test task's positional delete file, then polls
1853-
/// `load_file_for_task` for the same file until it waits on that load.
1854-
async fn wait_on_pos_del_load(
1855-
tmp_dir: &TempDir,
1856-
) -> (
1857-
PosDelLoadGuard,
1858-
std::pin::Pin<Box<impl Future<Output = Result<DeleteFileContext>>>>,
1859-
String,
1860-
) {
1872+
/// `load_file_for_task` for the same file from `waiters` tasks until each waits on it.
1873+
async fn wait_on_pos_del_load(tmp_dir: &TempDir, waiters: usize) -> PendingPosDelLoad {
18611874
let task = setup(tmp_dir.path()).remove(0);
18621875
let pos_del = task
18631876
.deletes()
@@ -1873,47 +1886,146 @@ mod tests {
18731886
else {
18741887
panic!("expected Load for a new file");
18751888
};
1876-
let path = pos_del.file_path().to_string();
1877-
let mut waiter = Box::pin(async move {
1878-
CachingDeleteFileLoader::load_file_for_task(
1879-
&pos_del,
1889+
let mut pending = Vec::with_capacity(waiters);
1890+
for _ in 0..waiters {
1891+
let (pos_del, schema) = (pos_del.clone(), task.schema_ref());
1892+
let (basic, filter) = (
18801893
loader.basic_delete_file_loader.clone(),
18811894
loader.delete_filter.clone(),
1882-
task.schema_ref(),
1883-
)
1884-
.await
1885-
});
1886-
assert!(futures::poll!(&mut waiter).is_pending());
1887-
(guard, waiter, path)
1895+
);
1896+
let mut waiter: futures::future::BoxFuture<'static, _> = Box::pin(async move {
1897+
CachingDeleteFileLoader::load_file_for_task(&pos_del, basic, filter, schema).await
1898+
});
1899+
assert!(futures::poll!(&mut waiter).is_pending());
1900+
pending.push(waiter);
1901+
}
1902+
PendingPosDelLoad {
1903+
guard,
1904+
waiters: pending,
1905+
path: pos_del.file_path().to_string(),
1906+
}
18881907
}
18891908

18901909
#[tokio::test]
1891-
async fn test_pos_del_waiter_loads_file_after_abandoned_load() {
1910+
async fn test_pos_del_waiter_loads_file_after_cancelled_load() {
18921911
let tmp_dir = TempDir::new().unwrap();
1893-
let (guard, waiter, _) = wait_on_pos_del_load(&tmp_dir).await;
1912+
let PendingPosDelLoad {
1913+
guard, mut waiters, ..
1914+
} = wait_on_pos_del_load(&tmp_dir, 1).await;
18941915

1916+
// Dropped without an error, as when a sibling's error stops `load_deletes`.
18951917
drop(guard);
18961918

1897-
let ctx = tokio::time::timeout(std::time::Duration::from_secs(5), waiter)
1919+
let ctx = tokio::time::timeout(std::time::Duration::from_secs(5), waiters.remove(0))
18981920
.await
1899-
.expect("waiter did not wake after the load was abandoned")
1921+
.expect("waiter did not wake after the load was cancelled")
19001922
.unwrap();
19011923
assert!(matches!(ctx, DeleteFileContext::PosDels { .. }));
19021924
}
19031925

19041926
#[tokio::test]
1905-
async fn test_pos_del_waiter_returns_open_error_after_abandoned_load() {
1927+
async fn test_pos_del_waiters_return_the_error_of_a_failed_load() {
1928+
let tmp_dir = TempDir::new().unwrap();
1929+
let PendingPosDelLoad { guard, waiters, .. } = wait_on_pos_del_load(&tmp_dir, 2).await;
1930+
1931+
// The file is still there, so a waiter that opened it again would succeed.
1932+
guard.fail(&Error::new(ErrorKind::Unexpected, "load failed"));
1933+
1934+
for waiter in waiters {
1935+
let err = tokio::time::timeout(std::time::Duration::from_secs(5), waiter)
1936+
.await
1937+
.expect("waiter did not wake after the load failed")
1938+
.err()
1939+
.unwrap();
1940+
assert_eq!(err.kind(), ErrorKind::Unexpected);
1941+
assert_eq!(err.message(), "load failed");
1942+
}
1943+
}
1944+
1945+
#[tokio::test]
1946+
async fn test_pos_del_waiters_retry_after_cancelled_load() {
19061947
let tmp_dir = TempDir::new().unwrap();
1907-
let (guard, waiter, path) = wait_on_pos_del_load(&tmp_dir).await;
1948+
let PendingPosDelLoad {
1949+
guard,
1950+
waiters,
1951+
path,
1952+
} = wait_on_pos_del_load(&tmp_dir, 2).await;
19081953

1954+
// Each waiter loads the file itself after a cancelled load, and gets its own error.
19091955
std::fs::remove_file(&path).unwrap();
19101956
drop(guard);
19111957

1912-
let result = tokio::time::timeout(std::time::Duration::from_secs(5), waiter)
1958+
for waiter in waiters {
1959+
let err = tokio::time::timeout(std::time::Duration::from_secs(5), waiter)
1960+
.await
1961+
.expect("waiter did not wake after the load was cancelled")
1962+
.err()
1963+
.unwrap();
1964+
assert!(err.to_string().contains("Failed to open file"), "{err}");
1965+
}
1966+
}
1967+
1968+
/// A failing positional delete file ends `load_deletes` early, dropping the load of
1969+
/// the other file in the same call. The other file still loads later.
1970+
#[tokio::test]
1971+
async fn test_pos_del_load_dropped_by_a_failing_sibling_can_be_loaded_later() {
1972+
let tmp_dir = TempDir::new().unwrap();
1973+
let schema = Arc::new(Schema::builder().build().unwrap());
1974+
let write = |name: &str, positions: Vec<i64>| {
1975+
let path = tmp_dir.path().join(name);
1976+
let path = path.to_str().unwrap().to_string();
1977+
let batch = RecordBatch::try_new(
1978+
crate::arrow::delete_filter::tests::create_pos_del_schema(),
1979+
vec![
1980+
Arc::new(StringArray::from_iter_values(vec![
1981+
"data.parquet";
1982+
positions.len()
1983+
])),
1984+
Arc::new(Int64Array::from_iter_values(positions)),
1985+
],
1986+
)
1987+
.unwrap();
1988+
let mut writer =
1989+
ArrowWriter::try_new(File::create(&path).unwrap(), batch.schema(), None).unwrap();
1990+
writer.write(&batch).unwrap();
1991+
writer.close().unwrap();
1992+
FileScanTaskDeleteFile::builder()
1993+
.with_file_path(path.clone())
1994+
.with_file_size_in_bytes(std::fs::metadata(&path).unwrap().len())
1995+
.with_file_type(DataContentType::PositionDeletes)
1996+
.with_file_format(DataFileFormat::Parquet)
1997+
.with_partition_spec_id(0)
1998+
.build()
1999+
.unwrap()
2000+
};
2001+
let invalid = write("invalid.parquet", vec![-1]);
2002+
let valid = write("valid.parquet", vec![1, 3]);
2003+
let loader = CachingDeleteFileLoader::new(FileIO::new_with_fs(), 10, Runtime::current());
2004+
2005+
let err = loader
2006+
.load_deletes(&[invalid, valid.clone()], Arc::clone(&schema))
19132007
.await
1914-
.expect("waiter did not wake after the load was abandoned");
1915-
let err = result.err().unwrap();
1916-
assert!(err.to_string().contains("Failed to open file"), "{err}");
2008+
.unwrap()
2009+
.err()
2010+
.unwrap();
2011+
assert!(err.message().contains("negative position"), "{err}");
2012+
2013+
let filter = tokio::time::timeout(
2014+
std::time::Duration::from_secs(5),
2015+
loader.load_deletes(&[valid], schema),
2016+
)
2017+
.await
2018+
.expect("load waited on the dropped load")
2019+
.unwrap()
2020+
.unwrap();
2021+
let positions: Vec<u64> = filter
2022+
.get_delete_vector_for_path("data.parquet")
2023+
.unwrap()
2024+
.lock()
2025+
.unwrap()
2026+
.iter()
2027+
.collect();
2028+
assert_eq!(positions, vec![1, 3]);
19172029
}
19182030

19192031
#[tokio::test]

0 commit comments

Comments
 (0)