Skip to content

Commit deff41b

Browse files
committed
fix(arrow): release a positional delete file whose load fails
try_start_pos_del_load marks a positional delete file Loading, and only the success path marked it Loaded. If opening or parsing the file failed, or the load was dropped because another delete file failed, the entry stayed Loading: a retry on the same loader, a clone of the same ArrowReader, or a concurrent load of the same file waited for a notification that never came. The Load action now carries a PosDelLoadGuard. finish() marks the file loaded; dropping the guard without finishing removes the Loading marker and wakes the waiters. A woken waiter checks the state again and loads the file itself if it isn't loaded, instead of assuming it is. Closes apache#3434.
1 parent b154603 commit deff41b

2 files changed

Lines changed: 178 additions & 34 deletions

File tree

‎crates/iceberg/src/arrow/caching_delete_file_loader.rs‎

Lines changed: 82 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ use bytes::Bytes;
2323
use futures::{StreamExt, TryStreamExt};
2424
use tokio::sync::oneshot::{Receiver, channel};
2525

26-
use super::delete_filter::{DeleteFilter, PosDelLoadAction};
26+
use super::delete_filter::{DeleteFilter, PosDelLoadAction, PosDelLoadGuard};
2727
use crate::arrow::delete_file_loader::BasicDeleteFileLoader;
2828
use crate::arrow::scan_metrics::ScanMetrics;
2929
use crate::arrow::{arrow_primitive_to_literal, arrow_schema_to_schema};
@@ -57,7 +57,7 @@ enum DeleteFileContext {
5757
ExistingEqDel,
5858
ExistingPosDel,
5959
PosDels {
60-
file_path: String,
60+
guard: PosDelLoadGuard,
6161
stream: ArrowRecordBatchStream,
6262
},
6363
FreshEqDel {
@@ -80,7 +80,7 @@ enum DeleteFileContext {
8080
// results are fully merged into the DeleteFileManager's state
8181
enum ParsedDeleteFileContext {
8282
DelVecs {
83-
file_path: String,
83+
guard: PosDelLoadGuard,
8484
results: HashMap<String, DeleteVector>,
8585
},
8686
// A single deletion vector decoded from a Puffin blob, keyed by the data file it applies to.
@@ -229,12 +229,12 @@ impl CachingDeleteFileLoader {
229229

230230
while let Some(item) = results_stream.next().await {
231231
match item? {
232-
ParsedDeleteFileContext::DelVecs { file_path, results } => {
232+
ParsedDeleteFileContext::DelVecs { guard, results } => {
233233
for (data_file_path, delete_vector) in results.into_iter() {
234234
del_filter.upsert_delete_vector(data_file_path, delete_vector);
235235
}
236236
// Mark the positional delete file as fully loaded so waiters can proceed
237-
del_filter.finish_pos_del_load(&file_path);
237+
guard.finish();
238238
}
239239
ParsedDeleteFileContext::DelVec {
240240
data_file_path,
@@ -271,25 +271,31 @@ impl CachingDeleteFileLoader {
271271
return Self::load_deletion_vector(task, basic_delete_file_loader).await;
272272
}
273273

274-
match del_filter.try_start_pos_del_load(task.file_path()) {
275-
PosDelLoadAction::AlreadyLoaded => Ok(DeleteFileContext::ExistingPosDel),
276-
PosDelLoadAction::WaitFor(notified) => {
277-
// Positional deletes are accessed synchronously by ArrowReader.
278-
// We must wait here to ensure the data is ready before returning,
279-
// otherwise ArrowReader might get an empty/partial result.
280-
notified.await;
281-
Ok(DeleteFileContext::ExistingPosDel)
274+
loop {
275+
match del_filter.try_start_pos_del_load(task.file_path()) {
276+
PosDelLoadAction::AlreadyLoaded => {
277+
return Ok(DeleteFileContext::ExistingPosDel);
278+
}
279+
PosDelLoadAction::WaitFor(notified) => {
280+
// Positional deletes are accessed synchronously by ArrowReader.
281+
// We must wait here to ensure the data is ready before returning,
282+
// otherwise ArrowReader might get an empty/partial result.
283+
// The wait also ends if that load fails, so check again.
284+
notified.await;
285+
}
286+
PosDelLoadAction::Load(guard) => {
287+
// On error the guard is dropped, which releases the file for a
288+
// later attempt.
289+
let stream = basic_delete_file_loader
290+
.parquet_to_batch_stream(
291+
task.file_path(),
292+
task.file_size_in_bytes(),
293+
task.key_metadata(),
294+
)
295+
.await?;
296+
return Ok(DeleteFileContext::PosDels { guard, stream });
297+
}
282298
}
283-
PosDelLoadAction::Load => Ok(DeleteFileContext::PosDels {
284-
file_path: task.file_path().to_string(),
285-
stream: basic_delete_file_loader
286-
.parquet_to_batch_stream(
287-
task.file_path(),
288-
task.file_size_in_bytes(),
289-
task.key_metadata(),
290-
)
291-
.await?,
292-
}),
293299
}
294300
}
295301

@@ -435,10 +441,10 @@ impl CachingDeleteFileLoader {
435441
match ctx {
436442
DeleteFileContext::ExistingEqDel => Ok(ParsedDeleteFileContext::EqDel),
437443
DeleteFileContext::ExistingPosDel => Ok(ParsedDeleteFileContext::ExistingPosDel),
438-
DeleteFileContext::PosDels { file_path, stream } => {
444+
DeleteFileContext::PosDels { guard, stream } => {
439445
let del_vecs = Self::parse_positional_deletes_record_batch_stream(stream).await?;
440446
Ok(ParsedDeleteFileContext::DelVecs {
441-
file_path,
447+
guard,
442448
results: del_vecs,
443449
})
444450
}
@@ -1791,4 +1797,55 @@ mod tests {
17911797
assert_eq!(err.kind(), ErrorKind::DataInvalid);
17921798
assert!(err.message().contains("expected 2 from record_count"));
17931799
}
1800+
1801+
#[tokio::test]
1802+
async fn test_pos_del_load_can_be_retried_after_failure() {
1803+
let tmp_dir = TempDir::new().unwrap();
1804+
let file_scan_tasks = setup(tmp_dir.path());
1805+
let task = &file_scan_tasks[0];
1806+
let pos_del = task
1807+
.deletes()
1808+
.iter()
1809+
.find(|delete| delete.file_type() == DataContentType::PositionDeletes)
1810+
.unwrap()
1811+
.clone();
1812+
let path = pos_del.file_path().to_string();
1813+
let hidden = format!("{path}.hidden");
1814+
let load = |loader: &CachingDeleteFileLoader| {
1815+
loader.load_deletes(std::slice::from_ref(&pos_del), task.schema_ref())
1816+
};
1817+
1818+
// The first load fails because the file is not there yet.
1819+
std::fs::rename(&path, &hidden).unwrap();
1820+
let loader = CachingDeleteFileLoader::new(FileIO::new_with_fs(), 10, Runtime::current());
1821+
assert!(load(&loader).await.unwrap().is_err());
1822+
1823+
// Once it is, the same loader loads it instead of waiting on the failed load.
1824+
std::fs::rename(&hidden, &path).unwrap();
1825+
let retried = tokio::time::timeout(std::time::Duration::from_secs(5), load(&loader))
1826+
.await
1827+
.expect("retry waited on the failed load")
1828+
.unwrap()
1829+
.unwrap();
1830+
let fresh = load(&CachingDeleteFileLoader::new(
1831+
FileIO::new_with_fs(),
1832+
10,
1833+
Runtime::current(),
1834+
))
1835+
.await
1836+
.unwrap()
1837+
.unwrap();
1838+
1839+
let positions = |filter: &DeleteFilter| -> Vec<u64> {
1840+
filter
1841+
.get_delete_vector(task)
1842+
.unwrap()
1843+
.lock()
1844+
.unwrap()
1845+
.iter()
1846+
.collect()
1847+
};
1848+
assert!(!positions(&fresh).is_empty());
1849+
assert_eq!(positions(&retried), positions(&fresh));
1850+
}
17941851
}

‎crates/iceberg/src/arrow/delete_filter.rs‎

Lines changed: 96 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -63,18 +63,51 @@ pub(crate) struct DeleteFilter {
6363

6464
/// Action to take when trying to start loading a positional delete file
6565
pub(crate) enum PosDelLoadAction {
66-
/// The file is not loaded, the caller should load it.
67-
Load,
66+
/// The file is not loaded, the caller should load it and call
67+
/// [`PosDelLoadGuard::finish`] once its delete vectors are merged.
68+
Load(PosDelLoadGuard),
6869
/// The file is already loaded, nothing to do.
6970
AlreadyLoaded,
7071
/// The file is currently being loaded by another task.
7172
/// The caller *must* await this future to ensure data availability before
7273
/// returning, as subsequent access (get_delete_vector) is synchronous. The
7374
/// future is created under the state lock so it cannot miss the loader's
7475
/// `notify_waiters()` (which stores no permit).
76+
///
77+
/// The future also resolves when that load fails or is dropped, so the caller must
78+
/// call [`DeleteFilter::try_start_pos_del_load`] again rather than assume the file
79+
/// is loaded.
7580
WaitFor(OwnedNotified),
7681
}
7782

83+
/// Ownership of an in-progress positional delete load, returned by
84+
/// [`DeleteFilter::try_start_pos_del_load`].
85+
///
86+
/// [`finish`](Self::finish) marks the file loaded. Dropping the guard without finishing,
87+
/// because the load failed or was cancelled, removes the `Loading` marker and wakes any
88+
/// waiters, so a later attempt can load the file instead of waiting forever.
89+
pub(crate) struct PosDelLoadGuard {
90+
filter: DeleteFilter,
91+
file_path: String,
92+
finished: bool,
93+
}
94+
95+
impl PosDelLoadGuard {
96+
/// Marks the file as loaded and wakes any waiters.
97+
pub(crate) fn finish(mut self) {
98+
self.finished = true;
99+
self.filter.finish_pos_del_load(&self.file_path);
100+
}
101+
}
102+
103+
impl Drop for PosDelLoadGuard {
104+
fn drop(&mut self) {
105+
if !self.finished {
106+
self.filter.abandon_pos_del_load(&self.file_path);
107+
}
108+
}
109+
}
110+
78111
impl DeleteFilter {
79112
/// Create a new DeleteFilter with the given runtime.
80113
pub(crate) fn new(runtime: Runtime) -> Self {
@@ -141,11 +174,15 @@ impl DeleteFilter {
141174
.positional_deletes
142175
.insert(file_path.to_string(), PosDelState::Loading(notifier));
143176

144-
PosDelLoadAction::Load
177+
PosDelLoadAction::Load(PosDelLoadGuard {
178+
filter: self.clone(),
179+
file_path: file_path.to_string(),
180+
finished: false,
181+
})
145182
}
146183

147184
/// Marks a positional delete file as successfully loaded and notifies any waiting tasks.
148-
pub(crate) fn finish_pos_del_load(&self, file_path: &str) {
185+
fn finish_pos_del_load(&self, file_path: &str) {
149186
let notify = {
150187
let mut state = self.state.write().unwrap();
151188
if let Some(PosDelState::Loading(notify)) = state
@@ -163,6 +200,29 @@ impl DeleteFilter {
163200
}
164201
}
165202

203+
/// Removes the `Loading` marker of a positional delete file whose load failed or was
204+
/// dropped, and wakes any waiting tasks so they can try the load again.
205+
fn abandon_pos_del_load(&self, file_path: &str) {
206+
let notify = {
207+
let mut state = self.state.write().unwrap();
208+
match state.positional_deletes.remove(file_path) {
209+
Some(PosDelState::Loading(notify)) => Some(notify),
210+
// Not ours to remove: keep whatever is there.
211+
Some(other) => {
212+
state
213+
.positional_deletes
214+
.insert(file_path.to_string(), other);
215+
None
216+
}
217+
None => None,
218+
}
219+
};
220+
221+
if let Some(notify) = notify {
222+
notify.notify_waiters();
223+
}
224+
}
225+
166226
/// Retrieve the equality delete predicate for a given eq delete file path
167227
pub(crate) async fn get_equality_delete_predicate_for_delete_file_path(
168228
&self,
@@ -319,17 +379,16 @@ pub(crate) mod tests {
319379
let filter = DeleteFilter::new(Runtime::current());
320380
let path = "s3://bucket/pos-delete.parquet";
321381

322-
assert!(matches!(
323-
filter.try_start_pos_del_load(path),
324-
PosDelLoadAction::Load
325-
));
382+
let PosDelLoadAction::Load(guard) = filter.try_start_pos_del_load(path) else {
383+
panic!("expected Load for a new file");
384+
};
326385

327386
let PosDelLoadAction::WaitFor(notified) = filter.try_start_pos_del_load(path) else {
328387
panic!("expected WaitFor for an in-progress load");
329388
};
330389

331390
// Loader completes and signals before the waiter awaits.
332-
filter.finish_pos_del_load(path);
391+
guard.finish();
333392

334393
let waited = tokio::time::timeout(std::time::Duration::from_secs(5), notified).await;
335394
assert!(
@@ -338,6 +397,34 @@ pub(crate) mod tests {
338397
);
339398
}
340399

400+
#[tokio::test]
401+
async fn test_abandoned_pos_del_load_wakes_waiters_and_can_be_retried() {
402+
let filter = DeleteFilter::new(Runtime::current());
403+
let path = "s3://bucket/pos-delete.parquet";
404+
405+
let PosDelLoadAction::Load(guard) = filter.try_start_pos_del_load(path) else {
406+
panic!("expected Load for a new file");
407+
};
408+
let PosDelLoadAction::WaitFor(notified) = filter.try_start_pos_del_load(path) else {
409+
panic!("expected WaitFor for an in-progress load");
410+
};
411+
412+
// The load fails: its guard is dropped without `finish`.
413+
drop(guard);
414+
415+
let waited = tokio::time::timeout(std::time::Duration::from_secs(5), notified).await;
416+
assert!(
417+
waited.is_ok(),
418+
"WaitFor future must resolve when the load is dropped"
419+
);
420+
421+
// The file is not loaded, so the next caller is told to load it.
422+
assert!(matches!(
423+
filter.try_start_pos_del_load(path),
424+
PosDelLoadAction::Load(_)
425+
));
426+
}
427+
341428
#[tokio::test]
342429
async fn test_delete_file_filter_load_deletes() {
343430
let tmp_dir = TempDir::new().unwrap();

0 commit comments

Comments
 (0)