@@ -1938,7 +1938,7 @@ mod tests {
19381938 . err ( )
19391939 . unwrap ( ) ;
19401940 assert_eq ! ( err. kind( ) , ErrorKind :: Unexpected ) ;
1941- assert_eq ! ( err. message( ) , "load failed" ) ;
1941+ assert ! ( err. message( ) . contains ( "load failed" ) , "{err} ") ;
19421942 }
19431943 }
19441944
@@ -2067,43 +2067,49 @@ mod tests {
20672067 }
20682068 }
20692069
2070+ /// Writes a position delete file at `path` for `data.parquet` with `positions`.
2071+ fn write_pos_del_file ( path : & str , positions : Vec < i64 > ) -> FileScanTaskDeleteFile {
2072+ let batch = RecordBatch :: try_new (
2073+ crate :: arrow:: delete_filter:: tests:: create_pos_del_schema ( ) ,
2074+ vec ! [
2075+ Arc :: new( StringArray :: from_iter_values( vec![
2076+ "data.parquet" ;
2077+ positions. len( )
2078+ ] ) ) ,
2079+ Arc :: new( Int64Array :: from_iter_values( positions) ) ,
2080+ ] ,
2081+ )
2082+ . unwrap ( ) ;
2083+ let mut writer =
2084+ ArrowWriter :: try_new ( File :: create ( path) . unwrap ( ) , batch. schema ( ) , None ) . unwrap ( ) ;
2085+ writer. write ( & batch) . unwrap ( ) ;
2086+ writer. close ( ) . unwrap ( ) ;
2087+ FileScanTaskDeleteFile :: builder ( )
2088+ . with_file_path ( path. to_string ( ) )
2089+ . with_file_size_in_bytes ( std:: fs:: metadata ( path) . unwrap ( ) . len ( ) )
2090+ . with_file_type ( DataContentType :: PositionDeletes )
2091+ . with_file_format ( DataFileFormat :: Parquet )
2092+ . with_partition_spec_id ( 0 )
2093+ . build ( )
2094+ . unwrap ( )
2095+ }
2096+
20702097 /// A failing positional delete file ends `load_deletes` early while the load of the
20712098 /// other file in the same call is still in flight, dropping its guard. That file
20722099 /// still loads later.
20732100 #[ tokio:: test]
20742101 async fn test_pos_del_load_dropped_by_a_failing_sibling_can_be_loaded_later ( ) {
20752102 let tmp_dir = TempDir :: new ( ) . unwrap ( ) ;
20762103 let schema = Arc :: new ( Schema :: builder ( ) . build ( ) . unwrap ( ) ) ;
2077- let write = |name : & str , positions : Vec < i64 > | {
2078- let path = tmp_dir. path ( ) . join ( name) ;
2079- let path = path. to_str ( ) . unwrap ( ) . to_string ( ) ;
2080- let batch = RecordBatch :: try_new (
2081- crate :: arrow:: delete_filter:: tests:: create_pos_del_schema ( ) ,
2082- vec ! [
2083- Arc :: new( StringArray :: from_iter_values( vec![
2084- "data.parquet" ;
2085- positions. len( )
2086- ] ) ) ,
2087- Arc :: new( Int64Array :: from_iter_values( positions) ) ,
2088- ] ,
2089- )
2090- . unwrap ( ) ;
2091- let mut writer =
2092- ArrowWriter :: try_new ( File :: create ( & path) . unwrap ( ) , batch. schema ( ) , None ) . unwrap ( ) ;
2093- writer. write ( & batch) . unwrap ( ) ;
2094- writer. close ( ) . unwrap ( ) ;
2095- FileScanTaskDeleteFile :: builder ( )
2096- . with_file_path ( path. clone ( ) )
2097- . with_file_size_in_bytes ( std:: fs:: metadata ( & path) . unwrap ( ) . len ( ) )
2098- . with_file_type ( DataContentType :: PositionDeletes )
2099- . with_file_format ( DataFileFormat :: Parquet )
2100- . with_partition_spec_id ( 0 )
2101- . build ( )
2102- . unwrap ( )
2103- } ;
2104- let invalid = write ( "invalid.parquet" , vec ! [ -1 ] ) ;
2104+ let invalid = write_pos_del_file (
2105+ tmp_dir. path ( ) . join ( "invalid.parquet" ) . to_str ( ) . unwrap ( ) ,
2106+ vec ! [ -1 ] ,
2107+ ) ;
21052108 // Its read waits for the gate, so its load is in flight when `invalid` fails.
2106- let valid = write ( "gated.parquet" , vec ! [ 1 , 3 ] ) ;
2109+ let valid = write_pos_del_file (
2110+ tmp_dir. path ( ) . join ( "gated.parquet" ) . to_str ( ) . unwrap ( ) ,
2111+ vec ! [ 1 , 3 ] ,
2112+ ) ;
21072113 let factory = Arc :: new ( gated_storage:: GatedStorageFactory :: default ( ) ) ;
21082114 let gate = Arc :: clone ( & factory. open ) ;
21092115 let file_io = crate :: io:: FileIOBuilder :: new ( factory) . build ( ) ;
@@ -2142,35 +2148,10 @@ mod tests {
21422148 let path = tmp_dir. path ( ) . join ( "pos-del.parquet" ) ;
21432149 let path = path. to_str ( ) . unwrap ( ) ;
21442150 let schema = Arc :: new ( Schema :: builder ( ) . build ( ) . unwrap ( ) ) ;
2145- let write = |positions : Vec < i64 > | {
2146- let batch = RecordBatch :: try_new (
2147- crate :: arrow:: delete_filter:: tests:: create_pos_del_schema ( ) ,
2148- vec ! [
2149- Arc :: new( StringArray :: from_iter_values( vec![
2150- "data.parquet" ;
2151- positions. len( )
2152- ] ) ) ,
2153- Arc :: new( Int64Array :: from_iter_values( positions) ) ,
2154- ] ,
2155- )
2156- . unwrap ( ) ;
2157- let mut writer =
2158- ArrowWriter :: try_new ( File :: create ( path) . unwrap ( ) , batch. schema ( ) , None ) . unwrap ( ) ;
2159- writer. write ( & batch) . unwrap ( ) ;
2160- writer. close ( ) . unwrap ( ) ;
2161- FileScanTaskDeleteFile :: builder ( )
2162- . with_file_path ( path. to_string ( ) )
2163- . with_file_size_in_bytes ( std:: fs:: metadata ( path) . unwrap ( ) . len ( ) )
2164- . with_file_type ( DataContentType :: PositionDeletes )
2165- . with_file_format ( DataFileFormat :: Parquet )
2166- . with_partition_spec_id ( 0 )
2167- . build ( )
2168- . unwrap ( )
2169- } ;
21702151 let loader = CachingDeleteFileLoader :: new ( FileIO :: new_with_fs ( ) , 10 , Runtime :: current ( ) ) ;
21712152
21722153 // The file opens but fails to parse, so the load is abandoned after the open.
2173- let invalid = write ( vec ! [ -1 ] ) ;
2154+ let invalid = write_pos_del_file ( path , vec ! [ -1 ] ) ;
21742155 let err = loader
21752156 . load_deletes ( & [ invalid] , Arc :: clone ( & schema) )
21762157 . await
@@ -2180,7 +2161,7 @@ mod tests {
21802161 assert ! ( err. message( ) . contains( "negative position" ) , "{err}" ) ;
21812162
21822163 // Once the file is valid, the same loader loads it instead of waiting.
2183- let valid = write ( vec ! [ 1 , 3 ] ) ;
2164+ let valid = write_pos_del_file ( path , vec ! [ 1 , 3 ] ) ;
21842165 let filter = tokio:: time:: timeout (
21852166 std:: time:: Duration :: from_secs ( 5 ) ,
21862167 loader. load_deletes ( & [ valid] , schema) ,
0 commit comments