Skip to content

Commit 8937de0

Browse files
committed
fmt
1 parent 1a889bd commit 8937de0

2 files changed

Lines changed: 92 additions & 5 deletions

File tree

‎native/core/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ lazy_static = "1.4"
9292
assertables = "10"
9393
hex = "0.4.3"
9494
datafusion-functions-nested = { version = "54.1.0" }
95+
opendal = { version = "0.57.0", features = ["services-memory"] }
9596

9697
[features]
9798
backtrace = ["datafusion/backtrace"]

‎native/core/src/execution/operators/parquet_writer.rs‎

Lines changed: 91 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,12 @@ impl ParquetWriter {
117117
// Write batch to in-memory buffer
118118
arrow_writer.write(batch)?;
119119

120-
// Flush and take ownership of the buffered bytes so the next batch encodes
121-
// into a fresh empty Vec (no clone, no explicit clear).
120+
// `flush()` closes the in-progress row group but leaves bytes in the internal
121+
// `BufWriter`. `sync()` pushes those bytes down into our cursor so the upload
122+
// is genuinely incremental. Then take ownership of the cursor's buffer and
123+
// reset it to empty for the next batch (no clone, no explicit clear).
122124
arrow_writer.flush()?;
125+
arrow_writer.sync()?;
123126
let cursor = arrow_writer.inner_mut();
124127
let current_data = std::mem::take(cursor.get_mut());
125128
cursor.set_position(0);
@@ -169,10 +172,14 @@ impl ParquetWriter {
169172
} => {
170173
// Finalize the Parquet footer into the in-memory cursor. `bytes_written()`
171174
// reports the authoritative file size once `finish()` has flushed the footer.
175+
// We cannot call `into_inner()` after `finish()`: `finish()` marks the
176+
// underlying `SerializedFileWriter` as finished, and `into_inner()` then fails
177+
// with `SerializedFileWriter already finished`. Pull the bytes out through
178+
// `inner_mut()` instead - `finish()` has already flushed the buffered writer
179+
// into the cursor.
172180
arrow_writer.finish()?;
173181
let total_bytes = arrow_writer.bytes_written() as u64;
174-
let cursor = arrow_writer.into_inner()?;
175-
let final_data = cursor.into_inner();
182+
let final_data = std::mem::take(arrow_writer.inner_mut().get_mut());
176183

177184
if !final_data.is_empty() {
178185
let mut w = match hdfs_writer.take() {
@@ -520,13 +527,14 @@ impl ExecutionPlan for ParquetWriterExec {
520527

521528
// Write batches
522529
let write_task = async move {
523-
let _timer = elapsed_compute.timer();
524530
let mut stream = input;
525531
let mut total_rows = 0i64;
526532

527533
while let Some(batch_result) = stream.try_next().await.transpose() {
528534
let batch = batch_result?;
529535

536+
let mut timer = elapsed_compute.timer();
537+
530538
// Track row count
531539
total_rows += batch.num_rows() as i64;
532540

@@ -546,12 +554,16 @@ impl ExecutionPlan for ParquetWriterExec {
546554
writer.write(&renamed_batch).await.map_err(|e| {
547555
DataFusionError::Execution(format!("Failed to write batch: {}", e))
548556
})?;
557+
558+
timer.stop();
549559
}
550560

561+
let mut timer = elapsed_compute.timer();
551562
let file_size =
552563
writer.close().await.map_err(|e| {
553564
DataFusionError::Execution(format!("Failed to close writer: {}", e))
554565
})? as i64;
566+
timer.stop();
555567

556568
// Update metrics with write statistics
557569
files_written.add(1);
@@ -609,6 +621,80 @@ mod tests {
609621
);
610622
}
611623

624+
/// Exercise the `ParquetWriter::Remote` write/close path against an in-memory
625+
/// opendal `Operator`, so the remote path has real automated coverage without
626+
/// requiring an HDFS cluster. Writes a handful of batches, reads the uploaded
627+
/// bytes back with `ParquetRecordBatchReaderBuilder`, and asserts the returned
628+
/// row count and the reported `bytes_written` both match the upload.
629+
#[tokio::test]
630+
#[cfg(feature = "hdfs-opendal")]
631+
async fn test_parquet_writer_remote_memory_backend() -> Result<()> {
632+
use opendal::services::Memory;
633+
use opendal::Operator;
634+
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
635+
636+
let op = Operator::new(Memory::default())
637+
.map_err(|e| {
638+
DataFusionError::Execution(format!("Failed to create memory operator: {}", e))
639+
})?
640+
.finish();
641+
let output_path = "test/data.parquet".to_string();
642+
643+
let schema = create_test_record_batch(1)?.schema();
644+
let props = WriterProperties::builder()
645+
.set_compression(Compression::UNCOMPRESSED)
646+
.build();
647+
let buffer = Vec::new();
648+
let cursor = Cursor::new(buffer);
649+
let arrow_writer = ArrowWriter::try_new(cursor, Arc::clone(&schema), Some(props))
650+
.map_err(|e| DataFusionError::Execution(format!("try_new failed: {}", e)))?;
651+
652+
let mut writer = ParquetWriter::Remote {
653+
arrow_writer,
654+
hdfs_writer: None,
655+
op: op.clone(),
656+
output_path: output_path.clone(),
657+
};
658+
659+
let mut expected_rows: i64 = 0;
660+
for i in 1..=3 {
661+
let batch = create_test_record_batch(i)?;
662+
expected_rows += batch.num_rows() as i64;
663+
writer.write(&batch).await.map_err(|e| {
664+
DataFusionError::Execution(format!("Failed to write batch {}: {}", i, e))
665+
})?;
666+
}
667+
668+
let reported_bytes = writer
669+
.close()
670+
.await
671+
.map_err(|e| DataFusionError::Execution(format!("Failed to close writer: {}", e)))?;
672+
673+
let uploaded = op.read(&output_path).await.map_err(|e| {
674+
DataFusionError::Execution(format!("Failed to read uploaded object: {}", e))
675+
})?;
676+
let uploaded_bytes = uploaded.to_vec();
677+
assert_eq!(
678+
reported_bytes as usize,
679+
uploaded_bytes.len(),
680+
"bytes_written must match uploaded object length"
681+
);
682+
683+
let reader = ParquetRecordBatchReaderBuilder::try_new(bytes::Bytes::from(uploaded_bytes))
684+
.map_err(|e| DataFusionError::Execution(format!("Reader builder failed: {}", e)))?
685+
.build()
686+
.map_err(|e| DataFusionError::Execution(format!("Reader build failed: {}", e)))?;
687+
let mut actual_rows: i64 = 0;
688+
for batch in reader {
689+
let batch =
690+
batch.map_err(|e| DataFusionError::Execution(format!("Read error: {}", e)))?;
691+
actual_rows += batch.num_rows() as i64;
692+
}
693+
assert_eq!(actual_rows, expected_rows);
694+
695+
Ok(())
696+
}
697+
612698
/// Helper function to create a test RecordBatch with 1000 rows of (int, string) data
613699
/// Example batch_id 1 -> 0..1000, 2 -> 1001..2000
614700
#[allow(dead_code)]

0 commit comments

Comments
 (0)