1515use std:: fmt:: { self , Debug } ;
1616use std:: sync:: Arc ;
1717
18+ use common_telemetry:: warn;
1819use hdfs_native:: { Client , ClientBuilder } ;
1920use opendal:: raw:: oio:: { Delete as _, Read as _, ReadStream as _, Write as _} ;
2021use opendal:: raw:: {
@@ -162,19 +163,39 @@ enum HdfsWriterInner {
162163 writer : Option < oio:: Writer > ,
163164 temporary_path : String ,
164165 target_path : String ,
166+ /// Message of the write that failed and removed the writer, so later
167+ /// `write`/`close` calls can explain why the writer is gone.
168+ failure : Option < String > ,
165169 } ,
166170}
167171
168172impl oio:: Write for HdfsWriter {
169173 async fn write ( & mut self , buffer : Buffer ) -> Result < ( ) > {
170174 match & mut self . 0 {
171175 HdfsWriterInner :: Direct ( writer) => writer. write ( buffer) . await ,
172- HdfsWriterInner :: Atomic { writer, .. } => {
173- writer
176+ HdfsWriterInner :: Atomic {
177+ inner,
178+ context,
179+ writer,
180+ temporary_path,
181+ failure,
182+ ..
183+ } => {
184+ let result = writer
174185 . as_mut ( )
175- . ok_or_else ( writer_unavailable) ?
186+ . ok_or_else ( || writer_unavailable ( failure . as_deref ( ) ) ) ?
176187 . write ( buffer)
177- . await
188+ . await ;
189+ if let Err ( error) = result {
190+ // A failed write leaves the temporary file behind, so abort
191+ // it before returning the original error to the caller.
192+ if let Some ( writer) = writer. take ( ) {
193+ abort_and_delete ( inner, context, writer, temporary_path) . await ;
194+ }
195+ * failure = Some ( error. to_string ( ) ) ;
196+ return Err ( error) ;
197+ }
198+ Ok ( ( ) )
178199 }
179200 }
180201 }
@@ -189,8 +210,11 @@ impl oio::Write for HdfsWriter {
189210 writer,
190211 temporary_path,
191212 target_path,
213+ failure,
192214 } => {
193- let mut writer = writer. take ( ) . ok_or_else ( writer_unavailable) ?;
215+ let mut writer = writer
216+ . take ( )
217+ . ok_or_else ( || writer_unavailable ( failure. as_deref ( ) ) ) ?;
194218 let metadata = match writer. close ( ) . await {
195219 Ok ( metadata) => metadata,
196220 Err ( error) => {
@@ -242,11 +266,17 @@ impl oio::Write for HdfsWriter {
242266 }
243267}
244268
245- fn writer_unavailable ( ) -> opendal:: Error {
246- opendal:: Error :: new (
247- ErrorKind :: Unexpected ,
248- "HDFS writer is unavailable after close or abort" ,
249- )
269+ fn writer_unavailable ( failure : Option < & str > ) -> opendal:: Error {
270+ match failure {
271+ Some ( failure) => opendal:: Error :: new (
272+ ErrorKind :: Unexpected ,
273+ format ! ( "HDFS writer is unavailable: a previous write failed: {failure}" ) ,
274+ ) ,
275+ None => opendal:: Error :: new (
276+ ErrorKind :: Unexpected ,
277+ "HDFS writer is unavailable after close or abort" ,
278+ ) ,
279+ }
250280}
251281
252282impl Service for HdfsCompatibilityService {
@@ -301,6 +331,7 @@ impl Service for HdfsCompatibilityService {
301331 writer : Some ( writer) ,
302332 temporary_path,
303333 target_path : path. to_string ( ) ,
334+ failure : None ,
304335 } ) )
305336 }
306337
@@ -416,9 +447,13 @@ async fn abort_and_delete(
416447 mut writer : oio:: Writer ,
417448 path : & str ,
418449) {
419- let _ = writer. abort ( ) . await ;
450+ if let Err ( error) = writer. abort ( ) . await {
451+ warn ! ( error; "Failed to abort HDFS writer for temporary file {path}" ) ;
452+ }
420453 drop ( writer) ;
421- let _ = delete_path ( inner, context, path) . await ;
454+ if let Err ( error) = delete_path ( inner, context, path) . await {
455+ warn ! ( error; "Failed to delete temporary file {path} after a failed HDFS write" ) ;
456+ }
422457}
423458
424459async fn delete_path ( inner : & Servicer , context : & OperationContext , path : & str ) -> Result < ( ) > {
@@ -443,6 +478,7 @@ mod tests {
443478 use tempfile:: TempDir ;
444479
445480 use super :: * ;
481+ use crate :: layers:: mock:: { MockLayerBuilder , MockWriterFactory } ;
446482
447483 fn test_store ( ) -> ( TempDir , Operator ) {
448484 let directory = tempfile:: tempdir ( ) . unwrap ( ) ;
@@ -548,4 +584,82 @@ mod tests {
548584 . as_ref( )
549585 ) ;
550586 }
587+
588+ /// A writer that starts failing once more than `fail_after` bytes have been
589+ /// accepted, to simulate a failure in the middle of an atomic write.
590+ struct FailingWriter {
591+ inner : oio:: Writer ,
592+ written : usize ,
593+ fail_after : usize ,
594+ }
595+
596+ impl oio:: Write for FailingWriter {
597+ async fn write ( & mut self , buffer : Buffer ) -> Result < ( ) > {
598+ if self . written + buffer. len ( ) > self . fail_after {
599+ return Err ( opendal:: Error :: new (
600+ ErrorKind :: Unexpected ,
601+ "injected write failure" ,
602+ ) ) ;
603+ }
604+ self . written += buffer. len ( ) ;
605+ self . inner . write ( buffer) . await
606+ }
607+
608+ async fn close ( & mut self ) -> Result < Metadata > {
609+ self . inner . close ( ) . await
610+ }
611+
612+ async fn abort ( & mut self ) -> Result < ( ) > {
613+ self . inner . abort ( ) . await
614+ }
615+ }
616+
617+ #[ tokio:: test]
618+ async fn test_atomic_write_cleans_up_temporary_file_after_write_error ( ) {
619+ let directory = tempfile:: tempdir ( ) . unwrap ( ) ;
620+ let factory: MockWriterFactory = Arc :: new ( |path, _args, inner| {
621+ if path. contains ( ".greptime-" ) {
622+ Box :: new ( FailingWriter {
623+ inner,
624+ written : 0 ,
625+ fail_after : 4 ,
626+ } )
627+ } else {
628+ inner
629+ }
630+ } ) ;
631+ let mock_layer = MockLayerBuilder :: default ( )
632+ . writer_factory ( factory)
633+ . build ( )
634+ . unwrap ( ) ;
635+ let store = Operator :: new ( Fs :: default ( ) . root ( directory. path ( ) . to_str ( ) . unwrap ( ) ) )
636+ . unwrap ( )
637+ . layer ( mock_layer)
638+ . layer ( HdfsCompatibilityLayer :: new_for_test ( ) ) ;
639+
640+ store. write ( "data/target" , "old" ) . await . unwrap ( ) ;
641+
642+ let mut writer: Writer = store. writer ( "data/target" ) . await . unwrap ( ) ;
643+ writer. write ( "part" ) . await . unwrap ( ) ;
644+
645+ let error = writer. write ( "ial" ) . await . unwrap_err ( ) ;
646+ assert_eq ! ( ErrorKind :: Unexpected , error. kind( ) ) ;
647+ assert ! ( error. to_string( ) . contains( "injected write failure" ) ) ;
648+
649+ // The destination keeps its previous contents...
650+ assert_eq ! (
651+ b"old" ,
652+ store. read( "data/target" ) . await . unwrap( ) . to_bytes( ) . as_ref( )
653+ ) ;
654+ // ...and the temporary file is removed even though the writer was not
655+ // closed or aborted by the caller.
656+ assert ! (
657+ store
658+ . list( "data/" )
659+ . await
660+ . unwrap( )
661+ . iter( )
662+ . all( |entry| !entry. path( ) . contains( ".greptime-" ) )
663+ ) ;
664+ }
551665}
0 commit comments