@@ -117,9 +117,12 @@ impl ParquetWriter {
117117 // Write batch to in-memory buffer
118118 arrow_writer. write ( batch) ?;
119119
120- // Flush and take ownership of the buffered bytes so the next batch encodes
121- // into a fresh empty Vec (no clone, no explicit clear).
120+ // `flush()` closes the in-progress row group but leaves bytes in the internal
121+ // `BufWriter`. `sync()` pushes those bytes down into our cursor so the upload
122+ // is genuinely incremental. Then take ownership of the cursor's buffer and
123+ // reset it to empty for the next batch (no clone, no explicit clear).
122124 arrow_writer. flush ( ) ?;
125+ arrow_writer. sync ( ) ?;
123126 let cursor = arrow_writer. inner_mut ( ) ;
124127 let current_data = std:: mem:: take ( cursor. get_mut ( ) ) ;
125128 cursor. set_position ( 0 ) ;
@@ -169,10 +172,14 @@ impl ParquetWriter {
169172 } => {
170173 // Finalize the Parquet footer into the in-memory cursor. `bytes_written()`
171174 // reports the authoritative file size once `finish()` has flushed the footer.
175+ // We cannot call `into_inner()` after `finish()`: `finish()` marks the
176+ // underlying `SerializedFileWriter` as finished, and `into_inner()` then fails
177+ // with `SerializedFileWriter already finished`. Pull the bytes out through
178+ // `inner_mut()` instead - `finish()` has already flushed the buffered writer
179+ // into the cursor.
172180 arrow_writer. finish ( ) ?;
173181 let total_bytes = arrow_writer. bytes_written ( ) as u64 ;
174- let cursor = arrow_writer. into_inner ( ) ?;
175- let final_data = cursor. into_inner ( ) ;
182+ let final_data = std:: mem:: take ( arrow_writer. inner_mut ( ) . get_mut ( ) ) ;
176183
177184 if !final_data. is_empty ( ) {
178185 let mut w = match hdfs_writer. take ( ) {
@@ -520,13 +527,14 @@ impl ExecutionPlan for ParquetWriterExec {
520527
521528 // Write batches
522529 let write_task = async move {
523- let _timer = elapsed_compute. timer ( ) ;
524530 let mut stream = input;
525531 let mut total_rows = 0i64 ;
526532
527533 while let Some ( batch_result) = stream. try_next ( ) . await . transpose ( ) {
528534 let batch = batch_result?;
529535
536+ let mut timer = elapsed_compute. timer ( ) ;
537+
530538 // Track row count
531539 total_rows += batch. num_rows ( ) as i64 ;
532540
@@ -546,12 +554,16 @@ impl ExecutionPlan for ParquetWriterExec {
546554 writer. write ( & renamed_batch) . await . map_err ( |e| {
547555 DataFusionError :: Execution ( format ! ( "Failed to write batch: {}" , e) )
548556 } ) ?;
557+
558+ timer. stop ( ) ;
549559 }
550560
561+ let mut timer = elapsed_compute. timer ( ) ;
551562 let file_size =
552563 writer. close ( ) . await . map_err ( |e| {
553564 DataFusionError :: Execution ( format ! ( "Failed to close writer: {}" , e) )
554565 } ) ? as i64 ;
566+ timer. stop ( ) ;
555567
556568 // Update metrics with write statistics
557569 files_written. add ( 1 ) ;
@@ -609,6 +621,80 @@ mod tests {
609621 ) ;
610622 }
611623
624+ /// Exercise the `ParquetWriter::Remote` write/close path against an in-memory
625+ /// opendal `Operator`, so the remote path has real automated coverage without
626+ /// requiring an HDFS cluster. Writes a handful of batches, reads the uploaded
627+ /// bytes back with `ParquetRecordBatchReaderBuilder`, and asserts the returned
628+ /// row count and the reported `bytes_written` both match the upload.
629+ #[ tokio:: test]
630+ #[ cfg( feature = "hdfs-opendal" ) ]
631+ async fn test_parquet_writer_remote_memory_backend ( ) -> Result < ( ) > {
632+ use opendal:: services:: Memory ;
633+ use opendal:: Operator ;
634+ use parquet:: arrow:: arrow_reader:: ParquetRecordBatchReaderBuilder ;
635+
636+ let op = Operator :: new ( Memory :: default ( ) )
637+ . map_err ( |e| {
638+ DataFusionError :: Execution ( format ! ( "Failed to create memory operator: {}" , e) )
639+ } ) ?
640+ . finish ( ) ;
641+ let output_path = "test/data.parquet" . to_string ( ) ;
642+
643+ let schema = create_test_record_batch ( 1 ) ?. schema ( ) ;
644+ let props = WriterProperties :: builder ( )
645+ . set_compression ( Compression :: UNCOMPRESSED )
646+ . build ( ) ;
647+ let buffer = Vec :: new ( ) ;
648+ let cursor = Cursor :: new ( buffer) ;
649+ let arrow_writer = ArrowWriter :: try_new ( cursor, Arc :: clone ( & schema) , Some ( props) )
650+ . map_err ( |e| DataFusionError :: Execution ( format ! ( "try_new failed: {}" , e) ) ) ?;
651+
652+ let mut writer = ParquetWriter :: Remote {
653+ arrow_writer,
654+ hdfs_writer : None ,
655+ op : op. clone ( ) ,
656+ output_path : output_path. clone ( ) ,
657+ } ;
658+
659+ let mut expected_rows: i64 = 0 ;
660+ for i in 1 ..=3 {
661+ let batch = create_test_record_batch ( i) ?;
662+ expected_rows += batch. num_rows ( ) as i64 ;
663+ writer. write ( & batch) . await . map_err ( |e| {
664+ DataFusionError :: Execution ( format ! ( "Failed to write batch {}: {}" , i, e) )
665+ } ) ?;
666+ }
667+
668+ let reported_bytes = writer
669+ . close ( )
670+ . await
671+ . map_err ( |e| DataFusionError :: Execution ( format ! ( "Failed to close writer: {}" , e) ) ) ?;
672+
673+ let uploaded = op. read ( & output_path) . await . map_err ( |e| {
674+ DataFusionError :: Execution ( format ! ( "Failed to read uploaded object: {}" , e) )
675+ } ) ?;
676+ let uploaded_bytes = uploaded. to_vec ( ) ;
677+ assert_eq ! (
678+ reported_bytes as usize ,
679+ uploaded_bytes. len( ) ,
680+ "bytes_written must match uploaded object length"
681+ ) ;
682+
683+ let reader = ParquetRecordBatchReaderBuilder :: try_new ( bytes:: Bytes :: from ( uploaded_bytes) )
684+ . map_err ( |e| DataFusionError :: Execution ( format ! ( "Reader builder failed: {}" , e) ) ) ?
685+ . build ( )
686+ . map_err ( |e| DataFusionError :: Execution ( format ! ( "Reader build failed: {}" , e) ) ) ?;
687+ let mut actual_rows: i64 = 0 ;
688+ for batch in reader {
689+ let batch =
690+ batch. map_err ( |e| DataFusionError :: Execution ( format ! ( "Read error: {}" , e) ) ) ?;
691+ actual_rows += batch. num_rows ( ) as i64 ;
692+ }
693+ assert_eq ! ( actual_rows, expected_rows) ;
694+
695+ Ok ( ( ) )
696+ }
697+
612698 /// Helper function to create a test RecordBatch with 1000 rows of (int, string) data
613699 /// Example batch_id 1 -> 0..1000, 2 -> 1001..2000
614700 #[ allow( dead_code) ]
0 commit comments