Skip to content

Commit 6c0dda4

Browse files
committed
Support pushing down empty projections into joins (apache#20191)
- Closes apache#20190. We should push down empty projections into HashJoinExec 1. try_embed_projection should embed empty projections 2. build_batch_empty_build_side should support empty schemas Yes No
1 parent 614f0bc commit 6c0dda4

6 files changed

Lines changed: 251 additions & 64 deletions

File tree

‎datafusion/core/tests/physical_optimizer/projection_pushdown.rs‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1778,3 +1778,46 @@ fn test_cooperative_exec_after_projection() -> Result<()> {
17781778

17791779
Ok(())
17801780
}
1781+
1782+
#[test]
1783+
fn test_hash_join_empty_projection_embeds() -> Result<()> {
1784+
let left_csv = create_simple_csv_exec();
1785+
let right_csv = create_simple_csv_exec();
1786+
1787+
let join = Arc::new(HashJoinExec::try_new(
1788+
left_csv,
1789+
right_csv,
1790+
vec![(Arc::new(Column::new("a", 0)), Arc::new(Column::new("a", 0)))],
1791+
None,
1792+
&JoinType::Right,
1793+
None,
1794+
PartitionMode::CollectLeft,
1795+
NullEquality::NullEqualsNothing,
1796+
)?);
1797+
1798+
// Empty projection: no columns needed from the join output
1799+
let projection: Arc<dyn ExecutionPlan> = Arc::new(ProjectionExec::try_new(
1800+
vec![] as Vec<ProjectionExpr>,
1801+
join,
1802+
)?);
1803+
1804+
let after_optimize =
1805+
ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
1806+
let after_optimize_string = displayable(after_optimize.as_ref())
1807+
.indent(true)
1808+
.to_string();
1809+
let actual = after_optimize_string.trim();
1810+
1811+
// The empty projection should be embedded into the HashJoinExec,
1812+
// resulting in projection=[] on the join and no ProjectionExec wrapper.
1813+
assert_snapshot!(
1814+
actual,
1815+
@r"
1816+
HashJoinExec: mode=CollectLeft, join_type=Right, on=[(a@0, a@0)], projection=[]
1817+
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
1818+
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
1819+
"
1820+
);
1821+
1822+
Ok(())
1823+
}

‎datafusion/physical-plan/src/joins/utils.rs‎

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -977,6 +977,17 @@ pub(crate) fn apply_join_filter_to_indices(
977977
))
978978
}
979979

980+
/// Creates a [RecordBatch] with zero columns but the given row count.
981+
/// Used when a join has an empty projection (e.g. `SELECT count(1) ...`).
982+
fn new_empty_schema_batch(schema: &Schema, row_count: usize) -> Result<RecordBatch> {
983+
let options = RecordBatchOptions::new().with_row_count(Some(row_count));
984+
Ok(RecordBatch::try_new_with_options(
985+
Arc::new(schema.clone()),
986+
vec![],
987+
&options,
988+
)?)
989+
}
990+
980991
/// Returns a new [RecordBatch] by combining the `left` and `right` according to `indices`.
981992
/// The resulting batch has [Schema] `schema`.
982993
pub(crate) fn build_batch_from_indices(
@@ -989,15 +1000,7 @@ pub(crate) fn build_batch_from_indices(
9891000
build_side: JoinSide,
9901001
) -> Result<RecordBatch> {
9911002
if schema.fields().is_empty() {
992-
let options = RecordBatchOptions::new()
993-
.with_match_field_names(true)
994-
.with_row_count(Some(build_indices.len()));
995-
996-
return Ok(RecordBatch::try_new_with_options(
997-
Arc::new(schema.clone()),
998-
vec![],
999-
&options,
1000-
)?);
1003+
return new_empty_schema_batch(schema, build_indices.len());
10011004
}
10021005

10031006
// build the columns of the new [RecordBatch]:
@@ -1057,6 +1060,9 @@ pub(crate) fn build_batch_empty_build_side(
10571060
// the remaining joins will return data for the right columns and null for the left ones
10581061
JoinType::Right | JoinType::Full | JoinType::RightAnti | JoinType::RightMark => {
10591062
let num_rows = probe_batch.num_rows();
1063+
if schema.fields().is_empty() {
1064+
return new_empty_schema_batch(schema, num_rows);
1065+
}
10601066
let mut columns: Vec<Arc<dyn Array>> =
10611067
Vec::with_capacity(schema.fields().len());
10621068

@@ -2889,4 +2895,35 @@ mod tests {
28892895

28902896
Ok(())
28912897
}
2898+
2899+
#[test]
2900+
fn test_build_batch_empty_build_side_empty_schema() -> Result<()> {
2901+
// When the output schema has no fields (empty projection pushed into
2902+
// the join), build_batch_empty_build_side should return a RecordBatch
2903+
// with the correct row count but no columns.
2904+
let empty_schema = Schema::empty();
2905+
2906+
let build_batch = RecordBatch::try_new(
2907+
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])),
2908+
vec![Arc::new(arrow::array::Int32Array::from(vec![1, 2, 3]))],
2909+
)?;
2910+
2911+
let probe_batch = RecordBatch::try_new(
2912+
Arc::new(Schema::new(vec![Field::new("b", DataType::Int32, true)])),
2913+
vec![Arc::new(arrow::array::Int32Array::from(vec![4, 5, 6, 7]))],
2914+
)?;
2915+
2916+
let result = build_batch_empty_build_side(
2917+
&empty_schema,
2918+
&build_batch,
2919+
&probe_batch,
2920+
&[], // no column indices with empty projection
2921+
JoinType::Right,
2922+
)?;
2923+
2924+
assert_eq!(result.num_rows(), 4);
2925+
assert_eq!(result.num_columns(), 0);
2926+
2927+
Ok(())
2928+
}
28922929
}

‎datafusion/physical-plan/src/projection.rs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -485,6 +485,15 @@ impl RecordBatchStream for ProjectionStream {
485485
}
486486
}
487487

488+
/// Trait for execution plans that can embed a projection, avoiding a separate
489+
/// [`ProjectionExec`] wrapper.
490+
///
491+
/// # Empty projections
492+
///
493+
/// `Some(vec![])` is a valid projection that produces zero output columns while
494+
/// preserving the correct row count. Implementors must ensure that runtime batch
495+
/// construction still returns batches with the right number of rows even when no
496+
/// columns are selected (e.g. for `SELECT count(1) … JOIN …`).
488497
pub trait EmbeddedProjection: ExecutionPlan + Sized {
489498
fn with_projection(&self, projection: Option<Vec<usize>>) -> Result<Self>;
490499
}
@@ -495,6 +504,15 @@ pub fn try_embed_projection<Exec: EmbeddedProjection + 'static>(
495504
projection: &ProjectionExec,
496505
execution_plan: &Exec,
497506
) -> Result<Option<Arc<dyn ExecutionPlan>>> {
507+
// If the projection has no expressions at all (e.g., ProjectionExec: expr=[]),
508+
// embed an empty projection into the execution plan so it outputs zero columns.
509+
// This avoids allocating throwaway null arrays for build-side columns
510+
// when no output columns are actually needed (e.g., count(1) over a right join).
511+
if projection.expr().is_empty() {
512+
let new_execution_plan = Arc::new(execution_plan.with_projection(Some(vec![]))?);
513+
return Ok(Some(new_execution_plan));
514+
}
515+
498516
// Collect all column indices from the given projection expressions.
499517
let projection_index = collect_column_indices(projection.expr());
500518

‎datafusion/sqllogictest/test_files/array.slt‎

Lines changed: 18 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6457,10 +6457,9 @@ physical_plan
64576457
02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]
64586458
03)----CoalescePartitionsExec
64596459
04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]
6460-
05)--------ProjectionExec: expr=[]
6461-
06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c])
6462-
07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
6463-
08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
6460+
05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c]), projection=[]
6461+
06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
6462+
07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
64646463

64656464
query I
64666465
with test AS (SELECT substr(md5(i::text)::text, 1, 32) as needle FROM generate_series(1, 100000) t(i))
@@ -6485,10 +6484,9 @@ physical_plan
64856484
02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]
64866485
03)----CoalescePartitionsExec
64876486
04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]
6488-
05)--------ProjectionExec: expr=[]
6489-
06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c])
6490-
07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
6491-
08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
6487+
05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c]), projection=[]
6488+
06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
6489+
07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
64926490

64936491
query I
64946492
with test AS (SELECT substr(md5(i::text)::text, 1, 32) as needle FROM generate_series(1, 100000) t(i))
@@ -6513,10 +6511,9 @@ physical_plan
65136511
02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]
65146512
03)----CoalescePartitionsExec
65156513
04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]
6516-
05)--------ProjectionExec: expr=[]
6517-
06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c])
6518-
07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
6519-
08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
6514+
05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c]), projection=[]
6515+
06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
6516+
07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
65206517

65216518
query I
65226519
with test AS (SELECT substr(md5(i::text)::text, 1, 32) as needle FROM generate_series(1, 100000) t(i))
@@ -6541,10 +6538,9 @@ physical_plan
65416538
02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]
65426539
03)----CoalescePartitionsExec
65436540
04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]
6544-
05)--------ProjectionExec: expr=[]
6545-
06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c])
6546-
07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
6547-
08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
6541+
05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c]), projection=[]
6542+
06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
6543+
07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
65486544

65496545
query I
65506546
with test AS (SELECT substr(md5(i::text)::text, 1, 32) as needle FROM generate_series(1, 100000) t(i))
@@ -6569,10 +6565,9 @@ physical_plan
65696565
02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]
65706566
03)----CoalescePartitionsExec
65716567
04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]
6572-
05)--------ProjectionExec: expr=[]
6573-
06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c])
6574-
07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
6575-
08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
6568+
05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c]), projection=[]
6569+
06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
6570+
07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
65766571

65776572
query I
65786573
with test AS (SELECT substr(md5(i::text)::text, 1, 32) as needle FROM generate_series(1, 100000) t(i))
@@ -6599,10 +6594,9 @@ physical_plan
65996594
02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]
66006595
03)----CoalescePartitionsExec
66016596
04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]
6602-
05)--------ProjectionExec: expr=[]
6603-
06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IS NOT NULL OR NULL
6604-
07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
6605-
08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
6597+
05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IS NOT NULL OR NULL, projection=[]
6598+
06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
6599+
07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
66066600

66076601
# any operator
66086602
query ?

0 commit comments

Comments
 (0)