@@ -823,23 +823,48 @@ mod tests {
823823 use std:: collections:: HashMap ;
824824 use std:: fs:: File ;
825825 use std:: sync:: Arc ;
826+ use std:: sync:: atomic:: { AtomicUsize , Ordering } ;
826827
827828 use arrow_array:: cast:: AsArray ;
828829 use arrow_array:: {
829830 ArrayRef , BinaryArray , Int32Array , Int64Array , RecordBatch , StringArray , StructArray ,
830831 } ;
831832 use arrow_schema:: { DataType , Field , Fields } ;
833+ use futures:: stream:: BoxStream ;
832834 use parquet:: arrow:: { ArrowWriter , PARQUET_FIELD_ID_META_KEY } ;
833835 use parquet:: basic:: Compression ;
834836 use parquet:: file:: properties:: WriterProperties ;
835837 use tempfile:: TempDir ;
836838
837839 use super :: * ;
838840 use crate :: arrow:: delete_filter:: tests:: setup;
841+ use crate :: io:: {
842+ FileIOBuilder , FileMetadata , FileRead , FileWrite , InputFile , LocalFsStorage , OutputFile ,
843+ Storage , StorageConfig , StorageFactory ,
844+ } ;
839845 use crate :: scan:: FileScanTaskDeleteFile ;
840846 use crate :: spec:: { DataContentType , DataFileFormat , Schema } ;
841847 use crate :: test_utils:: encode_dv_blob;
842848
849+ #[ test]
850+ fn test_delete_file_without_size_round_trips_through_serde ( ) {
851+ let task = FileScanTaskDeleteFile :: builder ( )
852+ . with_file_path ( "pos-del.parquet" . to_string ( ) )
853+ . with_file_type ( DataContentType :: PositionDeletes )
854+ . with_file_format ( DataFileFormat :: Parquet )
855+ . with_partition_spec_id ( 0 )
856+ . build ( )
857+ . unwrap ( ) ;
858+ assert_eq ! ( task. file_size_in_bytes( ) , None ) ;
859+
860+ // The unknown size is left out when serializing, and a task without the
861+ // field deserializes to `None`.
862+ let json = serde_json:: to_value ( & task) . unwrap ( ) ;
863+ assert ! ( json. get( "file_size_in_bytes" ) . is_none( ) , "{json}" ) ;
864+ let round_tripped: FileScanTaskDeleteFile = serde_json:: from_value ( json) . unwrap ( ) ;
865+ assert_eq ! ( round_tripped, task) ;
866+ }
867+
843868 #[ test]
844869 fn test_deserialized_deletion_vector_rejects_missing_fields ( ) {
845870 // A deletion vector that omits its required coordinates fails validation
@@ -1133,88 +1158,82 @@ mod tests {
11331158 #[ derive( Debug , Clone , Default , serde:: Serialize , serde:: Deserialize ) ]
11341159 struct MetadataCountingStorage {
11351160 #[ serde( skip) ]
1136- calls : Arc < std:: sync:: atomic:: AtomicUsize > ,
1161+ inner : LocalFsStorage ,
1162+ #[ serde( skip) ]
1163+ calls : Arc < AtomicUsize > ,
11371164 /// Fail every metadata request with a retryable error.
11381165 #[ serde( skip) ]
11391166 fail_metadata : bool ,
11401167 }
11411168
11421169 #[ async_trait:: async_trait]
11431170 #[ typetag:: serde( name = "test_metadata_counting_storage" ) ]
1144- impl crate :: io:: Storage for MetadataCountingStorage {
1145- async fn exists ( & self , path : & str ) -> Result < bool > {
1146- crate :: io:: LocalFsStorage :: new ( ) . exists ( path) . await
1147- }
1148-
1149- async fn metadata ( & self , path : & str ) -> Result < crate :: io:: FileMetadata > {
1150- self . calls . fetch_add ( 1 , std:: sync:: atomic:: Ordering :: SeqCst ) ;
1171+ impl Storage for MetadataCountingStorage {
1172+ async fn metadata ( & self , path : & str ) -> Result < FileMetadata > {
1173+ self . calls . fetch_add ( 1 , Ordering :: SeqCst ) ;
11511174 if self . fail_metadata {
11521175 return Err (
11531176 Error :: new ( ErrorKind :: Unexpected , "metadata temporarily unavailable" )
11541177 . with_retryable ( true ) ,
11551178 ) ;
11561179 }
1157- crate :: io :: LocalFsStorage :: new ( ) . metadata ( path) . await
1180+ self . inner . metadata ( path) . await
11581181 }
11591182
1160- async fn read ( & self , path : & str ) -> Result < Bytes > {
1161- crate :: io :: LocalFsStorage :: new ( ) . read ( path ) . await
1183+ fn new_input ( & self , path : & str ) -> Result < InputFile > {
1184+ Ok ( InputFile :: new ( Arc :: new ( self . clone ( ) ) , path . to_string ( ) ) )
11621185 }
11631186
1164- async fn reader ( & self , path : & str ) -> Result < Box < dyn crate :: io :: FileRead > > {
1165- crate :: io :: LocalFsStorage :: new ( ) . reader ( path ) . await
1187+ fn new_output ( & self , path : & str ) -> Result < OutputFile > {
1188+ Ok ( OutputFile :: new ( Arc :: new ( self . clone ( ) ) , path . to_string ( ) ) )
11661189 }
11671190
1168- async fn write ( & self , path : & str , bs : Bytes ) -> Result < ( ) > {
1169- crate :: io :: LocalFsStorage :: new ( ) . write ( path, bs ) . await
1191+ async fn exists ( & self , path : & str ) -> Result < bool > {
1192+ self . inner . exists ( path) . await
11701193 }
11711194
1172- async fn writer ( & self , path : & str ) -> Result < Box < dyn crate :: io :: FileWrite > > {
1173- crate :: io :: LocalFsStorage :: new ( ) . writer ( path) . await
1195+ async fn read ( & self , path : & str ) -> Result < Bytes > {
1196+ self . inner . read ( path) . await
11741197 }
11751198
1176- async fn delete ( & self , path : & str ) -> Result < ( ) > {
1177- crate :: io :: LocalFsStorage :: new ( ) . delete ( path) . await
1199+ async fn reader ( & self , path : & str ) -> Result < Box < dyn FileRead > > {
1200+ self . inner . reader ( path) . await
11781201 }
11791202
1180- async fn delete_prefix ( & self , path : & str ) -> Result < ( ) > {
1181- crate :: io :: LocalFsStorage :: new ( ) . delete_prefix ( path) . await
1203+ async fn write ( & self , path : & str , bs : Bytes ) -> Result < ( ) > {
1204+ self . inner . write ( path, bs ) . await
11821205 }
11831206
1184- async fn delete_stream (
1185- & self ,
1186- paths : futures:: stream:: BoxStream < ' static , String > ,
1187- ) -> Result < ( ) > {
1188- crate :: io:: LocalFsStorage :: new ( ) . delete_stream ( paths) . await
1207+ async fn writer ( & self , path : & str ) -> Result < Box < dyn FileWrite > > {
1208+ self . inner . writer ( path) . await
11891209 }
11901210
1191- fn new_input ( & self , path : & str ) -> Result < crate :: io:: InputFile > {
1192- Ok ( crate :: io:: InputFile :: new (
1193- Arc :: new ( self . clone ( ) ) ,
1194- path. to_string ( ) ,
1195- ) )
1211+ async fn delete ( & self , path : & str ) -> Result < ( ) > {
1212+ self . inner . delete ( path) . await
11961213 }
11971214
1198- fn new_output ( & self , path : & str ) -> Result < crate :: io:: OutputFile > {
1199- Ok ( crate :: io:: OutputFile :: new (
1200- Arc :: new ( self . clone ( ) ) ,
1201- path. to_string ( ) ,
1202- ) )
1215+ async fn delete_prefix ( & self , path : & str ) -> Result < ( ) > {
1216+ self . inner . delete_prefix ( path) . await
1217+ }
1218+
1219+ async fn delete_stream ( & self , paths : BoxStream < ' static , String > ) -> Result < ( ) > {
1220+ self . inner . delete_stream ( paths) . await
12031221 }
12041222 }
12051223
12061224 #[ derive( Debug , Clone , Default , serde:: Serialize , serde:: Deserialize ) ]
12071225 struct MetadataCountingStorageFactory {
12081226 #[ serde( skip) ]
1209- calls : Arc < std :: sync :: atomic :: AtomicUsize > ,
1227+ calls : Arc < AtomicUsize > ,
12101228 #[ serde( skip) ]
12111229 fail_metadata : bool ,
12121230 }
12131231
12141232 #[ typetag:: serde( name = "test_metadata_counting_storage_factory" ) ]
1215- impl crate :: io :: StorageFactory for MetadataCountingStorageFactory {
1216- fn build ( & self , _config : & crate :: io :: StorageConfig ) -> Result < Arc < dyn crate :: io :: Storage > > {
1233+ impl StorageFactory for MetadataCountingStorageFactory {
1234+ fn build ( & self , _config : & StorageConfig ) -> Result < Arc < dyn Storage > > {
12171235 Ok ( Arc :: new ( MetadataCountingStorage {
1236+ inner : LocalFsStorage ,
12181237 calls : Arc :: clone ( & self . calls ) ,
12191238 fail_metadata : self . fail_metadata ,
12201239 } ) )
@@ -1240,13 +1259,11 @@ mod tests {
12401259
12411260 #[ tokio:: test]
12421261 async fn test_unknown_delete_sizes_are_resolved_once_per_file ( ) {
1243- use std:: sync:: atomic:: Ordering ;
1244-
12451262 let tmp_dir = TempDir :: new ( ) . unwrap ( ) ;
12461263 let file_scan_tasks = setup ( tmp_dir. path ( ) ) ;
12471264 let factory = Arc :: new ( MetadataCountingStorageFactory :: default ( ) ) ;
12481265 let calls = Arc :: clone ( & factory. calls ) ;
1249- let file_io = crate :: io :: FileIOBuilder :: new ( factory) . build ( ) ;
1266+ let file_io = FileIOBuilder :: new ( factory) . build ( ) ;
12501267 let task = & file_scan_tasks[ 0 ] ;
12511268 let unknown: Vec < _ > = task
12521269 . deletes ( )
@@ -1295,8 +1312,6 @@ mod tests {
12951312
12961313 #[ tokio:: test]
12971314 async fn test_unknown_size_equality_deletes_are_resolved_once_and_cached ( ) {
1298- use std:: sync:: atomic:: Ordering ;
1299-
13001315 use crate :: scan:: FileScanTask ;
13011316 use crate :: spec:: DataFileFormat ;
13021317
@@ -1341,7 +1356,7 @@ mod tests {
13411356
13421357 let factory = Arc :: new ( MetadataCountingStorageFactory :: default ( ) ) ;
13431358 let calls = Arc :: clone ( & factory. calls ) ;
1344- let file_io = crate :: io :: FileIOBuilder :: new ( factory) . build ( ) ;
1359+ let file_io = FileIOBuilder :: new ( factory) . build ( ) ;
13451360
13461361 // Known size: no metadata request.
13471362 let known_task = data_task ( "known.parquet" , equality_delete ( Some ( known_size) ) ) ;
@@ -1402,7 +1417,7 @@ mod tests {
14021417 fail_metadata : true ,
14031418 ..Default :: default ( )
14041419 } ) ;
1405- let file_io = crate :: io :: FileIOBuilder :: new ( factory) . build ( ) ;
1420+ let file_io = FileIOBuilder :: new ( factory) . build ( ) ;
14061421 let present = unknown_size ( task. deletes ( ) [ 0 ] . file_path ( ) ) ;
14071422 let error = CachingDeleteFileLoader :: new ( file_io, 10 , Runtime :: current ( ) )
14081423 . load_deletes ( & [ present] , task. schema_ref ( ) )
0 commit comments