Skip to content

Commit f2be11a

Browse files
authored
fix: upload staging manifests in chunks (#9919)
Signed-off-by: WenyXu <[email protected]>
1 parent 1cb01d0 commit f2be11a

1 file changed

Lines changed: 65 additions & 6 deletions

File tree

‎src/mito2/src/manifest/storage/staging.rs‎

Lines changed: 65 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ use store_api::ManifestVersion;
2323
use crate::error::{OpenDalSnafu, Result};
2424
use crate::manifest::storage::delta::DeltaStorage;
2525
use crate::manifest::storage::size_tracker::NoopTracker;
26-
use crate::manifest::storage::utils::sort_manifests;
26+
use crate::manifest::storage::utils::{sort_manifests, write_and_put_cache};
2727
use crate::manifest::storage::{file_version, is_delta_file};
2828

2929
/// A simple blob storage for arbitrary binary data in the staging directory.
@@ -79,10 +79,7 @@ impl StagingBlobStorage {
7979
self.object_store.info().root(),
8080
bytes.len()
8181
);
82-
self.object_store
83-
.write(&path, bytes)
84-
.await
85-
.context(OpenDalSnafu)?;
82+
write_and_put_cache(&self.object_store, None, &path, bytes).await?;
8683
Ok(())
8784
}
8885

@@ -222,13 +219,75 @@ impl StagingStorage {
222219

223220
#[cfg(test)]
224221
mod tests {
222+
use std::sync::{Arc, Mutex};
223+
225224
use common_datasource::compression::CompressionType;
226225
use common_test_util::temp_dir::create_temp_dir;
227226
use futures::TryStreamExt;
227+
use object_store::layers::mock::{MockLayerBuilder, ObservedWriter, WriterObservations, oio};
228228
use object_store::services::Fs;
229229
use object_store::{ErrorKind, ObjectStore};
230230

231-
use super::{StagingStorage, staging_blob_path, staging_manifest_path};
231+
use super::{StagingBlobStorage, StagingStorage, staging_blob_path, staging_manifest_path};
232+
233+
// A test-only per-write limit, not a storage provider's default limit.
234+
const WRITE_LIMIT: usize = 64 * 1024 * 1024;
235+
236+
struct LimitedWriter {
237+
inner: oio::Writer,
238+
}
239+
240+
impl oio::Write for LimitedWriter {
241+
async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> {
242+
if bytes.len() > WRITE_LIMIT {
243+
return Err(object_store::Error::new(
244+
ErrorKind::Unexpected,
245+
"test write size limit exceeded",
246+
));
247+
}
248+
self.inner.write(bytes).await
249+
}
250+
251+
async fn close(&mut self) -> object_store::Result<object_store::layers::mock::Metadata> {
252+
self.inner.close().await
253+
}
254+
255+
async fn abort(&mut self) -> object_store::Result<()> {
256+
self.inner.abort().await
257+
}
258+
}
259+
260+
#[tokio::test]
261+
async fn test_staging_blob_put_chunked() {
262+
let tmp_dir = create_temp_dir("test_staging_blob_put_chunked");
263+
let observations = Arc::new(Mutex::new(WriterObservations::default()));
264+
let captured = observations.clone();
265+
let layer = MockLayerBuilder::default()
266+
.writer_factory(Arc::new(move |_, _, inner| {
267+
Box::new(ObservedWriter::new(
268+
Box::new(LimitedWriter { inner }),
269+
captured.clone(),
270+
))
271+
}))
272+
.build()
273+
.unwrap();
274+
let store = ObjectStore::new(Fs::default().root(&tmp_dir.path().to_string_lossy()))
275+
.unwrap()
276+
.layer(layer);
277+
let storage = StagingBlobStorage::new("staging/blob/".to_string(), store);
278+
279+
for size in [17, WRITE_LIMIT, WRITE_LIMIT + 17] {
280+
let bytes = (0..size).map(|i| (i % 251) as u8).collect::<Vec<_>>();
281+
*observations.lock().unwrap() = WriterObservations::default();
282+
storage.put("manifest", bytes.clone()).await.unwrap();
283+
assert_eq!(storage.get("manifest").await.unwrap(), bytes);
284+
let observed = observations.lock().unwrap();
285+
assert_eq!(observed.write_sizes.iter().sum::<usize>(), size);
286+
assert!(observed.write_sizes.iter().all(|size| *size <= WRITE_LIMIT));
287+
assert_eq!(observed.write_sizes.len(), size.div_ceil(WRITE_LIMIT));
288+
assert_eq!(observed.close_calls, 1);
289+
}
290+
}
232291

233292
#[test]
234293
fn test_staging_path() {

0 commit comments

Comments
 (0)