Skip to content

Commit a35661d

Browse files
authored
fix(hdfs): clean up temporary file when an atomic write fails (#9483)
* fix(hdfs): clean up temporary file when an atomic write fails Signed-off-by: fengyue-xve <[email protected]> * style: apply rustfmt to hdfs layer test Signed-off-by: fengyue-xve <[email protected]> * fix(hdfs): report earlier write failures and log cleanup failures - Later `write`/`close` calls on a writer whose failed write was cleaned up now mention the original failure instead of the bare "unavailable after close or abort" message. - `abort_and_delete` logs abort and deletion failures with the temporary path instead of discarding them; the original write error is still returned to the caller. - The temporary-file cleanup test now lists the directory holding the temporary file, so it fails if the cleanup is reverted. Signed-off-by: 风月 <[email protected]> Signed-off-by: fengyue-xve <[email protected]> --------- Signed-off-by: fengyue-xve <[email protected]> Signed-off-by: 风月 <[email protected]>
1 parent 36423a9 commit a35661d

1 file changed

Lines changed: 126 additions & 12 deletions

File tree

‎src/object-store/src/layers/hdfs.rs‎

Lines changed: 126 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
use std::fmt::{self, Debug};
1616
use std::sync::Arc;
1717

18+
use common_telemetry::warn;
1819
use hdfs_native::{Client, ClientBuilder};
1920
use opendal::raw::oio::{Delete as _, Read as _, ReadStream as _, Write as _};
2021
use opendal::raw::{
@@ -162,19 +163,39 @@ enum HdfsWriterInner {
162163
writer: Option<oio::Writer>,
163164
temporary_path: String,
164165
target_path: String,
166+
/// Message of the write that failed and removed the writer, so later
167+
/// `write`/`close` calls can explain why the writer is gone.
168+
failure: Option<String>,
165169
},
166170
}
167171

168172
impl oio::Write for HdfsWriter {
169173
async fn write(&mut self, buffer: Buffer) -> Result<()> {
170174
match &mut self.0 {
171175
HdfsWriterInner::Direct(writer) => writer.write(buffer).await,
172-
HdfsWriterInner::Atomic { writer, .. } => {
173-
writer
176+
HdfsWriterInner::Atomic {
177+
inner,
178+
context,
179+
writer,
180+
temporary_path,
181+
failure,
182+
..
183+
} => {
184+
let result = writer
174185
.as_mut()
175-
.ok_or_else(writer_unavailable)?
186+
.ok_or_else(|| writer_unavailable(failure.as_deref()))?
176187
.write(buffer)
177-
.await
188+
.await;
189+
if let Err(error) = result {
190+
// A failed write leaves the temporary file behind, so abort
191+
// it before returning the original error to the caller.
192+
if let Some(writer) = writer.take() {
193+
abort_and_delete(inner, context, writer, temporary_path).await;
194+
}
195+
*failure = Some(error.to_string());
196+
return Err(error);
197+
}
198+
Ok(())
178199
}
179200
}
180201
}
@@ -189,8 +210,11 @@ impl oio::Write for HdfsWriter {
189210
writer,
190211
temporary_path,
191212
target_path,
213+
failure,
192214
} => {
193-
let mut writer = writer.take().ok_or_else(writer_unavailable)?;
215+
let mut writer = writer
216+
.take()
217+
.ok_or_else(|| writer_unavailable(failure.as_deref()))?;
194218
let metadata = match writer.close().await {
195219
Ok(metadata) => metadata,
196220
Err(error) => {
@@ -242,11 +266,17 @@ impl oio::Write for HdfsWriter {
242266
}
243267
}
244268

245-
fn writer_unavailable() -> opendal::Error {
246-
opendal::Error::new(
247-
ErrorKind::Unexpected,
248-
"HDFS writer is unavailable after close or abort",
249-
)
269+
fn writer_unavailable(failure: Option<&str>) -> opendal::Error {
270+
match failure {
271+
Some(failure) => opendal::Error::new(
272+
ErrorKind::Unexpected,
273+
format!("HDFS writer is unavailable: a previous write failed: {failure}"),
274+
),
275+
None => opendal::Error::new(
276+
ErrorKind::Unexpected,
277+
"HDFS writer is unavailable after close or abort",
278+
),
279+
}
250280
}
251281

252282
impl Service for HdfsCompatibilityService {
@@ -301,6 +331,7 @@ impl Service for HdfsCompatibilityService {
301331
writer: Some(writer),
302332
temporary_path,
303333
target_path: path.to_string(),
334+
failure: None,
304335
}))
305336
}
306337

@@ -416,9 +447,13 @@ async fn abort_and_delete(
416447
mut writer: oio::Writer,
417448
path: &str,
418449
) {
419-
let _ = writer.abort().await;
450+
if let Err(error) = writer.abort().await {
451+
warn!(error; "Failed to abort HDFS writer for temporary file {path}");
452+
}
420453
drop(writer);
421-
let _ = delete_path(inner, context, path).await;
454+
if let Err(error) = delete_path(inner, context, path).await {
455+
warn!(error; "Failed to delete temporary file {path} after a failed HDFS write");
456+
}
422457
}
423458

424459
async fn delete_path(inner: &Servicer, context: &OperationContext, path: &str) -> Result<()> {
@@ -443,6 +478,7 @@ mod tests {
443478
use tempfile::TempDir;
444479

445480
use super::*;
481+
use crate::layers::mock::{MockLayerBuilder, MockWriterFactory};
446482

447483
fn test_store() -> (TempDir, Operator) {
448484
let directory = tempfile::tempdir().unwrap();
@@ -548,4 +584,82 @@ mod tests {
548584
.as_ref()
549585
);
550586
}
587+
588+
/// A writer that starts failing once more than `fail_after` bytes have been
589+
/// accepted, to simulate a failure in the middle of an atomic write.
590+
struct FailingWriter {
591+
inner: oio::Writer,
592+
written: usize,
593+
fail_after: usize,
594+
}
595+
596+
impl oio::Write for FailingWriter {
597+
async fn write(&mut self, buffer: Buffer) -> Result<()> {
598+
if self.written + buffer.len() > self.fail_after {
599+
return Err(opendal::Error::new(
600+
ErrorKind::Unexpected,
601+
"injected write failure",
602+
));
603+
}
604+
self.written += buffer.len();
605+
self.inner.write(buffer).await
606+
}
607+
608+
async fn close(&mut self) -> Result<Metadata> {
609+
self.inner.close().await
610+
}
611+
612+
async fn abort(&mut self) -> Result<()> {
613+
self.inner.abort().await
614+
}
615+
}
616+
617+
#[tokio::test]
618+
async fn test_atomic_write_cleans_up_temporary_file_after_write_error() {
619+
let directory = tempfile::tempdir().unwrap();
620+
let factory: MockWriterFactory = Arc::new(|path, _args, inner| {
621+
if path.contains(".greptime-") {
622+
Box::new(FailingWriter {
623+
inner,
624+
written: 0,
625+
fail_after: 4,
626+
})
627+
} else {
628+
inner
629+
}
630+
});
631+
let mock_layer = MockLayerBuilder::default()
632+
.writer_factory(factory)
633+
.build()
634+
.unwrap();
635+
let store = Operator::new(Fs::default().root(directory.path().to_str().unwrap()))
636+
.unwrap()
637+
.layer(mock_layer)
638+
.layer(HdfsCompatibilityLayer::new_for_test());
639+
640+
store.write("data/target", "old").await.unwrap();
641+
642+
let mut writer: Writer = store.writer("data/target").await.unwrap();
643+
writer.write("part").await.unwrap();
644+
645+
let error = writer.write("ial").await.unwrap_err();
646+
assert_eq!(ErrorKind::Unexpected, error.kind());
647+
assert!(error.to_string().contains("injected write failure"));
648+
649+
// The destination keeps its previous contents...
650+
assert_eq!(
651+
b"old",
652+
store.read("data/target").await.unwrap().to_bytes().as_ref()
653+
);
654+
// ...and the temporary file is removed even though the writer was not
655+
// closed or aborted by the caller.
656+
assert!(
657+
store
658+
.list("data/")
659+
.await
660+
.unwrap()
661+
.iter()
662+
.all(|entry| !entry.path().contains(".greptime-"))
663+
);
664+
}
551665
}

0 commit comments

Comments
 (0)