Skip to content

Commit 59d74ca

Browse files
authored
test: cover unsorted contiguous groups in one partition (apache#24737)
## Which issue does this PR relate to? - Part of apache#24438. ## Rationale for this change A source can concatenate several sorted logical runs into one DataFusion output partition. The resulting stream may be globally unsorted while every distinct `(key, time_bin)` tuple still occupies one contiguous range. This PR records how aggregate planning handles that layout before grouped input properties are available. It provides the behavioral baseline for the remaining PRs in the stack. ## What changes are included in this PR? - Add a single-partition `TestMemoryExec` fixture containing two sorted logical runs whose `(key, time_bin)` order resets at the record-batch boundary. - Aggregate by the complete `(key, time_bin)` tuple and verify the result. - Assert that planning selects `InputOrderMode::Linear`, `EmissionType::Final`, and `SingleHashAggregateStream`. ## Stack 1. [apache#24737 — test: cover unsorted contiguous groups in one partition](apache#24737) ← **this PR** 2. [apache#24697 — refactor: separate aggregate group completion from input ordering](apache#24697) 3. [apache#24698 — feat: add grouped equivalence properties](apache#24698) 4. [apache#24497 — feat: stream aggregates over grouped input](apache#24497) ## Are these changes tested? The new aggregate test executes the single-partition input and snapshot-checks all four grouped sums. ## Are there any user-facing changes? No. ## Review this layer [View only this PR layer](apache@36969e7)
1 parent 1bd2ad3 commit 59d74ca

1 file changed

Lines changed: 77 additions & 0 deletions

File tree

  • datafusion/physical-plan/src/aggregates

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5530,6 +5530,83 @@ mod tests {
55305530
Ok(())
55315531
}
55325532

5533+
#[tokio::test]
5534+
async fn unsorted_contiguous_groups_use_final_emission() -> Result<()> {
5535+
let schema = Arc::new(Schema::new(vec![
5536+
Field::new("key", DataType::Int32, false),
5537+
Field::new("time_bin", DataType::Int64, false),
5538+
Field::new("value", DataType::Int64, false),
5539+
]));
5540+
// Two sorted logical runs are emitted as batches in one DataFusion
5541+
// partition. Every distinct grouping tuple occupies one contiguous range,
5542+
// but tuple order resets at the batch boundary, so (key, time_bin) is not
5543+
// globally sorted.
5544+
let input_batches = vec![
5545+
RecordBatch::try_new(
5546+
Arc::clone(&schema),
5547+
vec![
5548+
Arc::new(Int32Array::from(vec![1, 1, 2, 2])),
5549+
Arc::new(Int64Array::from(vec![20, 20, 20, 20])),
5550+
Arc::new(Int64Array::from(vec![10, 20, 30, 40])),
5551+
],
5552+
)?,
5553+
RecordBatch::try_new(
5554+
Arc::clone(&schema),
5555+
vec![
5556+
Arc::new(Int32Array::from(vec![1, 1, 2, 2])),
5557+
Arc::new(Int64Array::from(vec![0, 0, 0, 0])),
5558+
Arc::new(Int64Array::from(vec![50, 60, 70, 80])),
5559+
],
5560+
)?,
5561+
];
5562+
let group_by = PhysicalGroupBy::new_single(vec![
5563+
(col("key", &schema)?, "key".to_string()),
5564+
(col("time_bin", &schema)?, "time_bin".to_string()),
5565+
]);
5566+
let aggr_expr = Arc::new(
5567+
AggregateExprBuilder::new(sum_udaf(), vec![col("value", &schema)?])
5568+
.schema(Arc::clone(&schema))
5569+
.alias("SUM(value)")
5570+
.build()?,
5571+
);
5572+
let input: Arc<dyn ExecutionPlan> =
5573+
TestMemoryExec::try_new_exec(&[input_batches], Arc::clone(&schema), None)?;
5574+
assert_eq!(input.output_partitioning().partition_count(), 1);
5575+
5576+
let aggregate = AggregateExec::try_new(
5577+
AggregateMode::Single,
5578+
group_by,
5579+
vec![aggr_expr],
5580+
vec![None],
5581+
input,
5582+
schema,
5583+
)?;
5584+
5585+
assert_eq!(aggregate.input_order_mode(), &InputOrderMode::Linear);
5586+
// This captures the behavior before #24438. When the source can declare
5587+
// `(key, time_bin)` group-contiguous, the corresponding case can use
5588+
// `EmissionType::Incremental`.
5589+
assert_eq!(aggregate.cache().emission_type, EmissionType::Final);
5590+
5591+
let task_ctx = new_migrated_hash_ctx(1024);
5592+
let stream = aggregate.execute_typed(0, &task_ctx)?;
5593+
assert!(matches!(stream, StreamType::SingleHash(_)));
5594+
let stream: SendableRecordBatchStream = stream.into();
5595+
let output = collect(stream).await?;
5596+
assert_snapshot!(batches_to_sort_string(&output), @r"
5597+
+-----+----------+------------+
5598+
| key | time_bin | SUM(value) |
5599+
+-----+----------+------------+
5600+
| 1 | 0 | 110 |
5601+
| 1 | 20 | 30 |
5602+
| 2 | 0 | 150 |
5603+
| 2 | 20 | 70 |
5604+
+-----+----------+------------+
5605+
");
5606+
5607+
Ok(())
5608+
}
5609+
55335610
/// Ensures for ordered input, `OrderedPartialAggregateStream` is used.
55345611
#[tokio::test]
55355612
async fn ordered_partial_aggregate_planning() -> Result<()> {

0 commit comments

Comments
 (0)