Skip to content

Commit 0e7d37b

Browse files
committed
fix: preserve native scan output schema for empty buckets
1 parent 4c491b3 commit 0e7d37b

3 files changed

Lines changed: 235 additions & 1 deletion

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ use datafusion_comet_spark_expr::{
9494
EvalMode, SparkArraysZipFunc, SparkBloomFilterVersion, SparkListAgg, SparkPercentile,
9595
SumInteger, ToCsv,
9696
};
97+
use datafusion_datasource::TableSchema;
9798
use iceberg::expr::Bind;
9899

99100
use crate::execution::operators::ExecutionError::GeneralError;
@@ -1622,7 +1623,22 @@ impl PhysicalPlanner {
16221623

16231624
// Check if this partition has any files (bucketed scan with bucket pruning may have empty partitions)
16241625
if partition_files.partitioned_file.is_empty() {
1625-
let empty_exec = Arc::new(EmptyExec::new(required_schema));
1626+
// Match the nonempty scan's projected table schema, including partition
1627+
// and constant metadata columns. Fused operators bind against this schema
1628+
// even when the bucket has no rows.
1629+
let partition_fields: Vec<FieldRef> = partition_schema
1630+
.fields()
1631+
.iter()
1632+
.map(|f| {
1633+
Arc::new(Field::new(f.name(), f.data_type().clone(), f.is_nullable()))
1634+
})
1635+
.collect();
1636+
let table_schema = TableSchema::builder(data_schema)
1637+
.with_table_partition_cols(partition_fields)
1638+
.build();
1639+
let output_schema =
1640+
Arc::new(table_schema.table_schema().project(&projection_vector)?);
1641+
let empty_exec = Arc::new(EmptyExec::new(output_schema));
16261642
return Ok((
16271643
vec![],
16281644
vec![],
@@ -5097,6 +5113,8 @@ fn needs_fields_coercion(sig: &TypeSignature) -> bool {
50975113

50985114
#[cfg(test)]
50995115
mod tests {
5116+
mod empty_native_scan;
5117+
51005118
use futures::{poll, StreamExt};
51015119
use std::{
51025120
sync::{
Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use std::collections::HashMap;
19+
20+
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
21+
use datafusion_comet_proto::spark_expression::{
22+
data_type::DataTypeId, expr::ExprStruct, literal::Value, DataType as SparkDataType, Expr,
23+
Literal,
24+
};
25+
use datafusion_comet_proto::spark_operator::{
26+
operator::OpStruct, NativeScan, NativeScanCommon, Operator, SparkFilePartition,
27+
SparkPartitionedFile, SparkStructField,
28+
};
29+
30+
use crate::execution::planner::PhysicalPlanner;
31+
32+
fn spark_field(
33+
name: &str,
34+
data_type: DataTypeId,
35+
nullable: bool,
36+
field_id: &str,
37+
) -> SparkStructField {
38+
SparkStructField {
39+
name: name.to_string(),
40+
data_type: Some(SparkDataType {
41+
type_id: data_type as i32,
42+
type_info: None,
43+
}),
44+
nullable,
45+
metadata: HashMap::from([("parquet.field.id".to_string(), field_id.to_string())]),
46+
}
47+
}
48+
49+
fn common(required_data_indices: &[usize], projection: Vec<i64>) -> NativeScanCommon {
50+
let data_schema = vec![
51+
spark_field("id", DataTypeId::Int32, false, "1"),
52+
spark_field("unused", DataTypeId::String, true, "2"),
53+
spark_field("payload", DataTypeId::Int64, true, "3"),
54+
];
55+
NativeScanCommon {
56+
required_schema: required_data_indices
57+
.iter()
58+
.map(|&index| data_schema[index].clone())
59+
.collect(),
60+
data_schema,
61+
// Constant file metadata is appended alongside partition values by the JVM scan.
62+
partition_schema: vec![
63+
spark_field("part", DataTypeId::Int32, false, "4"),
64+
spark_field("_metadata.file_size", DataTypeId::Int64, false, "5"),
65+
],
66+
projection_vector: projection,
67+
session_timezone: "UTC".to_string(),
68+
..Default::default()
69+
}
70+
}
71+
72+
fn literal(data_type: DataTypeId, value: Value) -> Expr {
73+
Expr {
74+
expr_struct: Some(ExprStruct::Literal(Literal {
75+
datatype: Some(SparkDataType {
76+
type_id: data_type as i32,
77+
type_info: None,
78+
}),
79+
value: Some(value),
80+
is_null: false,
81+
})),
82+
..Default::default()
83+
}
84+
}
85+
86+
fn plan_schema(common: &NativeScanCommon, empty: bool) -> SchemaRef {
87+
let files = if empty {
88+
vec![]
89+
} else {
90+
// Planning does not read the file. Only its presence selects the nonempty path.
91+
vec![SparkPartitionedFile {
92+
file_path: "file:///tmp/comet-empty-native-scan-schema.parquet".to_string(),
93+
start: 0,
94+
length: 16,
95+
file_size: 16,
96+
partition_values: vec![
97+
literal(DataTypeId::Int32, Value::IntVal(7)),
98+
literal(DataTypeId::Int64, Value::LongVal(16)),
99+
],
100+
}]
101+
};
102+
let operator = Operator {
103+
op_struct: Some(OpStruct::NativeScan(NativeScan {
104+
common: Some(common.clone()),
105+
file_partition: Some(SparkFilePartition {
106+
partitioned_file: files,
107+
}),
108+
..Default::default()
109+
})),
110+
..Default::default()
111+
};
112+
let (_, _, plan) = PhysicalPlanner::default()
113+
.create_plan(&operator, &mut vec![], 1)
114+
.unwrap();
115+
plan.schema()
116+
}
117+
118+
fn assert_schema_matches_nonempty(common: NativeScanCommon, expected: Vec<Field>) {
119+
let nonempty_schema = plan_schema(&common, false);
120+
let empty_schema = plan_schema(&common, true);
121+
assert_eq!(nonempty_schema.as_ref(), &Schema::new(expected));
122+
assert_eq!(empty_schema, nonempty_schema);
123+
}
124+
125+
fn id_field() -> Field {
126+
Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
127+
"parquet.field.id".to_string(),
128+
"1".to_string(),
129+
)]))
130+
}
131+
132+
fn payload_field() -> Field {
133+
Field::new("payload", DataType::Int64, true).with_metadata(HashMap::from([(
134+
"parquet.field.id".to_string(),
135+
"3".to_string(),
136+
)]))
137+
}
138+
139+
#[test]
140+
fn empty_scan_preserves_reordered_and_pruned_data_columns() {
141+
assert_schema_matches_nonempty(
142+
common(&[0, 2], vec![2, 0]),
143+
vec![payload_field(), id_field()],
144+
);
145+
}
146+
147+
#[test]
148+
fn empty_scan_preserves_partition_and_constant_metadata_projection() {
149+
// Match the nonempty path: data-field metadata is preserved, while partition
150+
// and constant metadata fields retain their names, types and nullability only.
151+
assert_schema_matches_nonempty(
152+
common(&[0, 2], vec![3, 2, 4, 0]),
153+
vec![
154+
Field::new("part", DataType::Int32, false),
155+
payload_field(),
156+
Field::new("_metadata.file_size", DataType::Int64, false),
157+
id_field(),
158+
],
159+
);
160+
}
161+
162+
#[test]
163+
fn empty_scan_preserves_partition_only_projection() {
164+
assert_schema_matches_nonempty(
165+
common(&[], vec![3]),
166+
vec![Field::new("part", DataType::Int32, false)],
167+
);
168+
}
169+
170+
#[test]
171+
fn empty_scan_preserves_zero_column_projection() {
172+
assert_schema_matches_nonempty(common(&[], vec![]), vec![]);
173+
}

‎spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,4 +240,47 @@ class CometTopKSuite extends CometTestBase {
240240
}
241241
}
242242
}
243+
244+
test("local TopK preserves partition columns in empty buckets") {
245+
val session = spark
246+
import session.implicits._
247+
withSQLConf(
248+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
249+
SQLConf.BUCKETING_ENABLED.key -> "true",
250+
"spark.sql.sources.bucketing.autoBucketedScan.enabled" -> "false") {
251+
withTable("topk_bucketed_partitioned") {
252+
// Two rows cannot fill eight buckets, so the native plan must also handle empty scans.
253+
Seq((1, 10), (2, 20))
254+
.toDF("id", "p")
255+
.coalesce(1)
256+
.write
257+
.format("parquet")
258+
.partitionBy("p")
259+
.bucketBy(8, "id")
260+
.saveAsTable("topk_bucketed_partitioned")
261+
for (fusion <- Seq(false, true)) {
262+
withSQLConf(CometConf.COMET_EXEC_TOPK_FUSION_ENABLED.key -> fusion.toString) {
263+
for ((columns, expected) <- Seq(
264+
"id, p" -> Seq(Row(1, 10), Row(2, 20)),
265+
"p, id" -> Seq(Row(10, 1), Row(20, 2)))) {
266+
val query =
267+
sql(s"SELECT $columns FROM topk_bucketed_partitioned ORDER BY p LIMIT 5")
268+
val plan = query.queryExecution.executedPlan
269+
val scans = collect(plan) { case scan: CometNativeScanExec => scan }
270+
assert(scans.size == 1, s"Expected one native scan:\n$plan")
271+
assert(scans.head.bucketedScan)
272+
assert(scans.head.perPartitionData.length == 8)
273+
assert(scans.head.perPartitionFilePaths.count(_.isEmpty) >= 6)
274+
val topKs = collect(plan) { case topK: CometTakeOrderedAndProjectExec => topK }
275+
assert(topKs.size == 1, s"Expected one native TopK:\n$plan")
276+
assert(topKs.head.child.executeColumnar().getNumPartitions == 8)
277+
val locals = collect(plan) { case local: CometLocalTopKExec => local }
278+
assert(locals.size == (if (fusion) 1 else 0), s"fusion=$fusion:\n$plan")
279+
assert(query.collect().toSeq == expected)
280+
}
281+
}
282+
}
283+
}
284+
}
285+
}
243286
}

0 commit comments

Comments
 (0)