@@ -276,23 +276,29 @@ impl CachingDeleteFileLoader {
276276 PosDelLoadAction :: AlreadyLoaded => {
277277 return Ok ( DeleteFileContext :: ExistingPosDel ) ;
278278 }
279- PosDelLoadAction :: WaitFor ( notified ) => {
279+ PosDelLoadAction :: WaitFor ( wait ) => {
280280 // Positional deletes are accessed synchronously by ArrowReader.
281281 // We must wait here to ensure the data is ready before returning,
282282 // otherwise ArrowReader might get an empty/partial result.
283- // The wait also ends if that load fails, so check again.
284- notified. await ;
283+ // A failed load returns its error here. A load dropped without
284+ // one (cancelled) ends the wait too, so check again.
285+ wait. wait ( ) . await ?;
285286 }
286287 PosDelLoadAction :: Load ( guard) => {
287- // On error the guard is dropped, which releases the file for a
288- // later attempt.
289- let stream = basic_delete_file_loader
288+ let stream = match basic_delete_file_loader
290289 . parquet_to_batch_stream (
291290 task. file_path ( ) ,
292291 task. file_size_in_bytes ( ) ,
293292 task. key_metadata ( ) ,
294293 )
295- . await ?;
294+ . await
295+ {
296+ Ok ( stream) => stream,
297+ Err ( err) => {
298+ guard. fail ( & err) ;
299+ return Err ( err) ;
300+ }
301+ } ;
296302 return Ok ( DeleteFileContext :: PosDels { guard, stream } ) ;
297303 }
298304 }
@@ -442,11 +448,16 @@ impl CachingDeleteFileLoader {
442448 DeleteFileContext :: ExistingEqDel => Ok ( ParsedDeleteFileContext :: EqDel ) ,
443449 DeleteFileContext :: ExistingPosDel => Ok ( ParsedDeleteFileContext :: ExistingPosDel ) ,
444450 DeleteFileContext :: PosDels { guard, stream } => {
445- let del_vecs = Self :: parse_positional_deletes_record_batch_stream ( stream) . await ?;
446- Ok ( ParsedDeleteFileContext :: DelVecs {
447- guard,
448- results : del_vecs,
449- } )
451+ match Self :: parse_positional_deletes_record_batch_stream ( stream) . await {
452+ Ok ( del_vecs) => Ok ( ParsedDeleteFileContext :: DelVecs {
453+ guard,
454+ results : del_vecs,
455+ } ) ,
456+ Err ( err) => {
457+ guard. fail ( & err) ;
458+ Err ( err)
459+ }
460+ }
450461 }
451462 DeleteFileContext :: DelVec {
452463 data_file_path,
@@ -1849,15 +1860,17 @@ mod tests {
18491860 assert_eq ! ( positions( & retried) , positions( & fresh) ) ;
18501861 }
18511862
1863+ /// The load of a positional delete file taken by the test, and tasks polled until
1864+ /// they wait on it.
1865+ struct PendingPosDelLoad {
1866+ guard : PosDelLoadGuard ,
1867+ waiters : Vec < futures:: future:: BoxFuture < ' static , Result < DeleteFileContext > > > ,
1868+ path : String ,
1869+ }
1870+
18521871 /// Takes the load of the first test task's positional delete file, then polls
1853- /// `load_file_for_task` for the same file until it waits on that load.
1854- async fn wait_on_pos_del_load (
1855- tmp_dir : & TempDir ,
1856- ) -> (
1857- PosDelLoadGuard ,
1858- std:: pin:: Pin < Box < impl Future < Output = Result < DeleteFileContext > > > > ,
1859- String ,
1860- ) {
1872+ /// `load_file_for_task` for the same file from `waiters` tasks until each waits on it.
1873+ async fn wait_on_pos_del_load ( tmp_dir : & TempDir , waiters : usize ) -> PendingPosDelLoad {
18611874 let task = setup ( tmp_dir. path ( ) ) . remove ( 0 ) ;
18621875 let pos_del = task
18631876 . deletes ( )
@@ -1873,47 +1886,146 @@ mod tests {
18731886 else {
18741887 panic ! ( "expected Load for a new file" ) ;
18751888 } ;
1876- let path = pos_del . file_path ( ) . to_string ( ) ;
1877- let mut waiter = Box :: pin ( async move {
1878- CachingDeleteFileLoader :: load_file_for_task (
1879- & pos_del ,
1889+ let mut pending = Vec :: with_capacity ( waiters ) ;
1890+ for _ in 0 ..waiters {
1891+ let ( pos_del , schema ) = ( pos_del . clone ( ) , task . schema_ref ( ) ) ;
1892+ let ( basic , filter ) = (
18801893 loader. basic_delete_file_loader . clone ( ) ,
18811894 loader. delete_filter . clone ( ) ,
1882- task. schema_ref ( ) ,
1883- )
1884- . await
1885- } ) ;
1886- assert ! ( futures:: poll!( & mut waiter) . is_pending( ) ) ;
1887- ( guard, waiter, path)
1895+ ) ;
1896+ let mut waiter: futures:: future:: BoxFuture < ' static , _ > = Box :: pin ( async move {
1897+ CachingDeleteFileLoader :: load_file_for_task ( & pos_del, basic, filter, schema) . await
1898+ } ) ;
1899+ assert ! ( futures:: poll!( & mut waiter) . is_pending( ) ) ;
1900+ pending. push ( waiter) ;
1901+ }
1902+ PendingPosDelLoad {
1903+ guard,
1904+ waiters : pending,
1905+ path : pos_del. file_path ( ) . to_string ( ) ,
1906+ }
18881907 }
18891908
18901909 #[ tokio:: test]
1891- async fn test_pos_del_waiter_loads_file_after_abandoned_load ( ) {
1910+ async fn test_pos_del_waiter_loads_file_after_cancelled_load ( ) {
18921911 let tmp_dir = TempDir :: new ( ) . unwrap ( ) ;
1893- let ( guard, waiter, _) = wait_on_pos_del_load ( & tmp_dir) . await ;
1912+ let PendingPosDelLoad {
1913+ guard, mut waiters, ..
1914+ } = wait_on_pos_del_load ( & tmp_dir, 1 ) . await ;
18941915
1916+ // Dropped without an error, as when a sibling's error stops `load_deletes`.
18951917 drop ( guard) ;
18961918
1897- let ctx = tokio:: time:: timeout ( std:: time:: Duration :: from_secs ( 5 ) , waiter )
1919+ let ctx = tokio:: time:: timeout ( std:: time:: Duration :: from_secs ( 5 ) , waiters . remove ( 0 ) )
18981920 . await
1899- . expect ( "waiter did not wake after the load was abandoned " )
1921+ . expect ( "waiter did not wake after the load was cancelled " )
19001922 . unwrap ( ) ;
19011923 assert ! ( matches!( ctx, DeleteFileContext :: PosDels { .. } ) ) ;
19021924 }
19031925
19041926 #[ tokio:: test]
1905- async fn test_pos_del_waiter_returns_open_error_after_abandoned_load ( ) {
1927+ async fn test_pos_del_waiters_return_the_error_of_a_failed_load ( ) {
1928+ let tmp_dir = TempDir :: new ( ) . unwrap ( ) ;
1929+ let PendingPosDelLoad { guard, waiters, .. } = wait_on_pos_del_load ( & tmp_dir, 2 ) . await ;
1930+
1931+ // The file is still there, so a waiter that opened it again would succeed.
1932+ guard. fail ( & Error :: new ( ErrorKind :: Unexpected , "load failed" ) ) ;
1933+
1934+ for waiter in waiters {
1935+ let err = tokio:: time:: timeout ( std:: time:: Duration :: from_secs ( 5 ) , waiter)
1936+ . await
1937+ . expect ( "waiter did not wake after the load failed" )
1938+ . err ( )
1939+ . unwrap ( ) ;
1940+ assert_eq ! ( err. kind( ) , ErrorKind :: Unexpected ) ;
1941+ assert_eq ! ( err. message( ) , "load failed" ) ;
1942+ }
1943+ }
1944+
1945+ #[ tokio:: test]
1946+ async fn test_pos_del_waiters_retry_after_cancelled_load ( ) {
19061947 let tmp_dir = TempDir :: new ( ) . unwrap ( ) ;
1907- let ( guard, waiter, path) = wait_on_pos_del_load ( & tmp_dir) . await ;
1948+ let PendingPosDelLoad {
1949+ guard,
1950+ waiters,
1951+ path,
1952+ } = wait_on_pos_del_load ( & tmp_dir, 2 ) . await ;
19081953
1954+ // Each waiter loads the file itself after a cancelled load, and gets its own error.
19091955 std:: fs:: remove_file ( & path) . unwrap ( ) ;
19101956 drop ( guard) ;
19111957
1912- let result = tokio:: time:: timeout ( std:: time:: Duration :: from_secs ( 5 ) , waiter)
1958+ for waiter in waiters {
1959+ let err = tokio:: time:: timeout ( std:: time:: Duration :: from_secs ( 5 ) , waiter)
1960+ . await
1961+ . expect ( "waiter did not wake after the load was cancelled" )
1962+ . err ( )
1963+ . unwrap ( ) ;
1964+ assert ! ( err. to_string( ) . contains( "Failed to open file" ) , "{err}" ) ;
1965+ }
1966+ }
1967+
1968+ /// A failing positional delete file ends `load_deletes` early, dropping the load of
1969+ /// the other file in the same call. The other file still loads later.
1970+ #[ tokio:: test]
1971+ async fn test_pos_del_load_dropped_by_a_failing_sibling_can_be_loaded_later ( ) {
1972+ let tmp_dir = TempDir :: new ( ) . unwrap ( ) ;
1973+ let schema = Arc :: new ( Schema :: builder ( ) . build ( ) . unwrap ( ) ) ;
1974+ let write = |name : & str , positions : Vec < i64 > | {
1975+ let path = tmp_dir. path ( ) . join ( name) ;
1976+ let path = path. to_str ( ) . unwrap ( ) . to_string ( ) ;
1977+ let batch = RecordBatch :: try_new (
1978+ crate :: arrow:: delete_filter:: tests:: create_pos_del_schema ( ) ,
1979+ vec ! [
1980+ Arc :: new( StringArray :: from_iter_values( vec![
1981+ "data.parquet" ;
1982+ positions. len( )
1983+ ] ) ) ,
1984+ Arc :: new( Int64Array :: from_iter_values( positions) ) ,
1985+ ] ,
1986+ )
1987+ . unwrap ( ) ;
1988+ let mut writer =
1989+ ArrowWriter :: try_new ( File :: create ( & path) . unwrap ( ) , batch. schema ( ) , None ) . unwrap ( ) ;
1990+ writer. write ( & batch) . unwrap ( ) ;
1991+ writer. close ( ) . unwrap ( ) ;
1992+ FileScanTaskDeleteFile :: builder ( )
1993+ . with_file_path ( path. clone ( ) )
1994+ . with_file_size_in_bytes ( std:: fs:: metadata ( & path) . unwrap ( ) . len ( ) )
1995+ . with_file_type ( DataContentType :: PositionDeletes )
1996+ . with_file_format ( DataFileFormat :: Parquet )
1997+ . with_partition_spec_id ( 0 )
1998+ . build ( )
1999+ . unwrap ( )
2000+ } ;
2001+ let invalid = write ( "invalid.parquet" , vec ! [ -1 ] ) ;
2002+ let valid = write ( "valid.parquet" , vec ! [ 1 , 3 ] ) ;
2003+ let loader = CachingDeleteFileLoader :: new ( FileIO :: new_with_fs ( ) , 10 , Runtime :: current ( ) ) ;
2004+
2005+ let err = loader
2006+ . load_deletes ( & [ invalid, valid. clone ( ) ] , Arc :: clone ( & schema) )
19132007 . await
1914- . expect ( "waiter did not wake after the load was abandoned" ) ;
1915- let err = result. err ( ) . unwrap ( ) ;
1916- assert ! ( err. to_string( ) . contains( "Failed to open file" ) , "{err}" ) ;
2008+ . unwrap ( )
2009+ . err ( )
2010+ . unwrap ( ) ;
2011+ assert ! ( err. message( ) . contains( "negative position" ) , "{err}" ) ;
2012+
2013+ let filter = tokio:: time:: timeout (
2014+ std:: time:: Duration :: from_secs ( 5 ) ,
2015+ loader. load_deletes ( & [ valid] , schema) ,
2016+ )
2017+ . await
2018+ . expect ( "load waited on the dropped load" )
2019+ . unwrap ( )
2020+ . unwrap ( ) ;
2021+ let positions: Vec < u64 > = filter
2022+ . get_delete_vector_for_path ( "data.parquet" )
2023+ . unwrap ( )
2024+ . lock ( )
2025+ . unwrap ( )
2026+ . iter ( )
2027+ . collect ( ) ;
2028+ assert_eq ! ( positions, vec![ 1 , 3 ] ) ;
19172029 }
19182030
19192031 #[ tokio:: test]
0 commit comments