Skip to content

Commit 62dc7ac

Browse files
authored
Support INSERT OVERWRITE for MemTable (apache#24969)
## Which issue does this PR close? - Part of apache#19617. ## Rationale for this change DataFusion parses and plans `INSERT OVERWRITE`, but `MemTable` rejects every insert operation except append. Users therefore cannot replace the contents of an in-memory table with the standard overwrite operation. ## What changes are included in this PR? - allow `InsertOp::Overwrite` in `MemTable::insert_into` - let `MemSink` replace each target partition after the input stream completes successfully - continue rejecting the distinct `InsertOp::Replace` operation ## What is the testing strategy for this PR? Three unit tests verify that overwrite replaces existing rows, distributes multiple replacement batches across multiple target partitions, and clears the table for empty input. A separate test confirms that `InsertOp::Replace` remains unsupported. Validated with: - `cargo test -p datafusion --lib test_insert_overwrite -- --nocapture` - `cargo clippy -p datafusion -p datafusion-datasource -p datafusion-catalog --all-targets --all-features -- -D warnings` - `cargo fmt --all -- --check` ## Are there any user-facing changes? Yes. `INSERT OVERWRITE` now replaces all existing data in a `MemTable`. There are no breaking public API changes.
1 parent 18cbb87 commit 62dc7ac

3 files changed

Lines changed: 151 additions & 8 deletions

File tree

‎datafusion/catalog/src/memory/table.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -349,10 +349,11 @@ impl MemTable {
349349
self.schema()
350350
.logically_equivalent_names_and_types(&input.schema())?;
351351

352-
if insert_op != InsertOp::Append {
352+
if insert_op == InsertOp::Replace {
353353
return not_impl_err!("{insert_op} not implemented for MemoryTable yet");
354354
}
355-
let sink = MemSink::try_new(self.batches.clone(), Arc::clone(&self.schema))?;
355+
let sink = MemSink::try_new(self.batches.clone(), Arc::clone(&self.schema))?
356+
.with_overwrite(insert_op == InsertOp::Overwrite);
356357
Ok(Arc::new(DataSinkExec::new(input, Arc::new(sink), None)))
357358
}
358359

‎datafusion/core/src/datasource/memory_test.rs‎

Lines changed: 128 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ mod tests {
2323
use crate::physical_plan::collect;
2424
use crate::prelude::SessionContext;
2525
use arrow::array::{AsArray, Int32Array};
26-
use arrow::datatypes::{DataType, Field, Schema, UInt64Type};
26+
use arrow::datatypes::{DataType, Field, Int32Type, Schema, UInt64Type};
2727
use arrow::error::ArrowError;
2828
use arrow::record_batch::RecordBatch;
2929
use arrow_schema::SchemaRef;
@@ -318,6 +318,16 @@ mod tests {
318318
schema: SchemaRef,
319319
initial_data: Vec<Vec<RecordBatch>>,
320320
inserted_data: Vec<Vec<RecordBatch>>,
321+
) -> Result<Vec<Vec<RecordBatch>>> {
322+
experiment_with_insert_op(schema, initial_data, inserted_data, InsertOp::Append)
323+
.await
324+
}
325+
326+
async fn experiment_with_insert_op(
327+
schema: SchemaRef,
328+
initial_data: Vec<Vec<RecordBatch>>,
329+
inserted_data: Vec<Vec<RecordBatch>>,
330+
insert_op: InsertOp,
321331
) -> Result<Vec<Vec<RecordBatch>>> {
322332
let expected_count: u64 = inserted_data
323333
.iter()
@@ -339,7 +349,7 @@ mod tests {
339349
let scan_plan = LogicalPlanBuilder::scan("source", source, None)?.build()?;
340350
// Create an insert plan to insert the source data into the initial table
341351
let insert_into_table =
342-
LogicalPlanBuilder::insert_into(scan_plan, "t", target, InsertOp::Append)?
352+
LogicalPlanBuilder::insert_into(scan_plan, "t", target, insert_op)?
343353
.build()?;
344354
// Create a physical plan from the insert plan
345355
let plan = session_ctx
@@ -479,6 +489,122 @@ mod tests {
479489
Ok(())
480490
}
481491

492+
#[tokio::test]
493+
async fn test_insert_overwrite_replaces_existing_data() -> Result<()> {
494+
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
495+
let initial_batch = RecordBatch::try_new(
496+
Arc::clone(&schema),
497+
vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
498+
)?;
499+
let replacement_batch = RecordBatch::try_new(
500+
Arc::clone(&schema),
501+
vec![Arc::new(Int32Array::from(vec![4, 5]))],
502+
)?;
503+
504+
let resulting_data = experiment_with_insert_op(
505+
schema,
506+
vec![vec![initial_batch]],
507+
vec![vec![replacement_batch]],
508+
InsertOp::Overwrite,
509+
)
510+
.await?;
511+
512+
assert_eq!(resulting_data[0].len(), 1);
513+
assert_eq!(
514+
resulting_data[0][0]
515+
.column(0)
516+
.as_primitive::<Int32Type>()
517+
.values(),
518+
&[4, 5]
519+
);
520+
Ok(())
521+
}
522+
523+
#[tokio::test]
524+
async fn test_insert_overwrite_replaces_multiple_partitions() -> Result<()> {
525+
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
526+
let batch = |values| {
527+
RecordBatch::try_new(
528+
Arc::clone(&schema),
529+
vec![Arc::new(Int32Array::from(values))],
530+
)
531+
};
532+
533+
let resulting_data = experiment_with_insert_op(
534+
Arc::clone(&schema),
535+
vec![vec![batch(vec![1])?], vec![batch(vec![2])?]],
536+
vec![vec![
537+
batch(vec![10])?,
538+
batch(vec![20])?,
539+
batch(vec![30])?,
540+
batch(vec![40])?,
541+
]],
542+
InsertOp::Overwrite,
543+
)
544+
.await?;
545+
546+
assert_eq!(resulting_data.len(), 2);
547+
for (partition, expected) in resulting_data.iter().zip([[10, 30], [20, 40]]) {
548+
let actual = partition
549+
.iter()
550+
.flat_map(|batch| {
551+
batch
552+
.column(0)
553+
.as_primitive::<Int32Type>()
554+
.values()
555+
.iter()
556+
.copied()
557+
})
558+
.collect::<Vec<_>>();
559+
assert_eq!(actual, expected);
560+
}
561+
Ok(())
562+
}
563+
564+
#[tokio::test]
565+
async fn test_insert_overwrite_with_empty_input_clears_table() -> Result<()> {
566+
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
567+
let initial_batch = RecordBatch::try_new(
568+
Arc::clone(&schema),
569+
vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
570+
)?;
571+
572+
let resulting_data = experiment_with_insert_op(
573+
schema,
574+
vec![vec![initial_batch]],
575+
vec![vec![]],
576+
InsertOp::Overwrite,
577+
)
578+
.await?;
579+
580+
assert!(resulting_data[0].is_empty());
581+
Ok(())
582+
}
583+
584+
#[tokio::test]
585+
async fn test_insert_replace_remains_unsupported() -> Result<()> {
586+
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
587+
let batch = RecordBatch::try_new(
588+
Arc::clone(&schema),
589+
vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
590+
)?;
591+
592+
let error = experiment_with_insert_op(
593+
schema,
594+
vec![vec![batch.clone()]],
595+
vec![vec![batch]],
596+
InsertOp::Replace,
597+
)
598+
.await
599+
.unwrap_err();
600+
601+
assert_eq!(
602+
error.strip_backtrace(),
603+
"This feature is not implemented: Replace Into not implemented for MemoryTable yet"
604+
);
605+
Ok(())
606+
}
607+
482608
// Test inserting a batch into a MemTable without any partitions
483609
#[tokio::test]
484610
async fn test_insert_into_zero_partition() -> Result<()> {

‎datafusion/datasource/src/memory.rs‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -920,6 +920,7 @@ pub struct MemSink {
920920
/// Target locations for writing data
921921
batches: Vec<PartitionData>,
922922
schema: SchemaRef,
923+
overwrite: bool,
923924
}
924925

925926
impl Debug for MemSink {
@@ -953,7 +954,17 @@ impl MemSink {
953954
if batches.is_empty() {
954955
return plan_err!("Cannot insert into MemTable with zero partitions");
955956
}
956-
Ok(Self { batches, schema })
957+
Ok(Self {
958+
batches,
959+
schema,
960+
overwrite: false,
961+
})
962+
}
963+
964+
/// Configures whether writes replace the existing data instead of appending to it.
965+
pub fn with_overwrite(mut self, overwrite: bool) -> Self {
966+
self.overwrite = overwrite;
967+
self
957968
}
958969
}
959970

@@ -981,10 +992,15 @@ impl DataSink for MemSink {
981992
i = (i + 1) % num_partitions;
982993
}
983994

984-
// write the outputs into the batches
995+
// Modify the table only after the input stream has completed successfully.
985996
for (target, mut batches) in self.batches.iter().zip(new_batches) {
986-
// Append all the new batches in one go to minimize locking overhead
987-
target.write().await.append(&mut batches);
997+
let mut target = target.write().await;
998+
if self.overwrite {
999+
*target = batches;
1000+
} else {
1001+
// Append all the new batches in one go to minimize locking overhead
1002+
target.append(&mut batches);
1003+
}
9881004
}
9891005

9901006
Ok(row_count as u64)

0 commit comments

Comments
 (0)