@@ -23,7 +23,7 @@ use store_api::ManifestVersion;
2323use crate :: error:: { OpenDalSnafu , Result } ;
2424use crate :: manifest:: storage:: delta:: DeltaStorage ;
2525use crate :: manifest:: storage:: size_tracker:: NoopTracker ;
26- use crate :: manifest:: storage:: utils:: sort_manifests;
26+ use crate :: manifest:: storage:: utils:: { sort_manifests, write_and_put_cache } ;
2727use crate :: manifest:: storage:: { file_version, is_delta_file} ;
2828
2929/// A simple blob storage for arbitrary binary data in the staging directory.
@@ -79,10 +79,7 @@ impl StagingBlobStorage {
7979 self . object_store. info( ) . root( ) ,
8080 bytes. len( )
8181 ) ;
82- self . object_store
83- . write ( & path, bytes)
84- . await
85- . context ( OpenDalSnafu ) ?;
82+ write_and_put_cache ( & self . object_store , None , & path, bytes) . await ?;
8683 Ok ( ( ) )
8784 }
8885
@@ -222,13 +219,75 @@ impl StagingStorage {
222219
223220#[ cfg( test) ]
224221mod tests {
222+ use std:: sync:: { Arc , Mutex } ;
223+
225224 use common_datasource:: compression:: CompressionType ;
226225 use common_test_util:: temp_dir:: create_temp_dir;
227226 use futures:: TryStreamExt ;
227+ use object_store:: layers:: mock:: { MockLayerBuilder , ObservedWriter , WriterObservations , oio} ;
228228 use object_store:: services:: Fs ;
229229 use object_store:: { ErrorKind , ObjectStore } ;
230230
231- use super :: { StagingStorage , staging_blob_path, staging_manifest_path} ;
231+ use super :: { StagingBlobStorage , StagingStorage , staging_blob_path, staging_manifest_path} ;
232+
233+ // A test-only per-write limit, not a storage provider's default limit.
234+ const WRITE_LIMIT : usize = 64 * 1024 * 1024 ;
235+
236+ struct LimitedWriter {
237+ inner : oio:: Writer ,
238+ }
239+
240+ impl oio:: Write for LimitedWriter {
241+ async fn write ( & mut self , bytes : object_store:: Buffer ) -> object_store:: Result < ( ) > {
242+ if bytes. len ( ) > WRITE_LIMIT {
243+ return Err ( object_store:: Error :: new (
244+ ErrorKind :: Unexpected ,
245+ "test write size limit exceeded" ,
246+ ) ) ;
247+ }
248+ self . inner . write ( bytes) . await
249+ }
250+
251+ async fn close ( & mut self ) -> object_store:: Result < object_store:: layers:: mock:: Metadata > {
252+ self . inner . close ( ) . await
253+ }
254+
255+ async fn abort ( & mut self ) -> object_store:: Result < ( ) > {
256+ self . inner . abort ( ) . await
257+ }
258+ }
259+
260+ #[ tokio:: test]
261+ async fn test_staging_blob_put_chunked ( ) {
262+ let tmp_dir = create_temp_dir ( "test_staging_blob_put_chunked" ) ;
263+ let observations = Arc :: new ( Mutex :: new ( WriterObservations :: default ( ) ) ) ;
264+ let captured = observations. clone ( ) ;
265+ let layer = MockLayerBuilder :: default ( )
266+ . writer_factory ( Arc :: new ( move |_, _, inner| {
267+ Box :: new ( ObservedWriter :: new (
268+ Box :: new ( LimitedWriter { inner } ) ,
269+ captured. clone ( ) ,
270+ ) )
271+ } ) )
272+ . build ( )
273+ . unwrap ( ) ;
274+ let store = ObjectStore :: new ( Fs :: default ( ) . root ( & tmp_dir. path ( ) . to_string_lossy ( ) ) )
275+ . unwrap ( )
276+ . layer ( layer) ;
277+ let storage = StagingBlobStorage :: new ( "staging/blob/" . to_string ( ) , store) ;
278+
279+ for size in [ 17 , WRITE_LIMIT , WRITE_LIMIT + 17 ] {
280+ let bytes = ( 0 ..size) . map ( |i| ( i % 251 ) as u8 ) . collect :: < Vec < _ > > ( ) ;
281+ * observations. lock ( ) . unwrap ( ) = WriterObservations :: default ( ) ;
282+ storage. put ( "manifest" , bytes. clone ( ) ) . await . unwrap ( ) ;
283+ assert_eq ! ( storage. get( "manifest" ) . await . unwrap( ) , bytes) ;
284+ let observed = observations. lock ( ) . unwrap ( ) ;
285+ assert_eq ! ( observed. write_sizes. iter( ) . sum:: <usize >( ) , size) ;
286+ assert ! ( observed. write_sizes. iter( ) . all( |size| * size <= WRITE_LIMIT ) ) ;
287+ assert_eq ! ( observed. write_sizes. len( ) , size. div_ceil( WRITE_LIMIT ) ) ;
288+ assert_eq ! ( observed. close_calls, 1 ) ;
289+ }
290+ }
232291
233292 #[ test]
234293 fn test_staging_path ( ) {
0 commit comments