Skip to content

Commit b696cc3

Browse files
committed
fix: scope duplicate checks to projected roots
1 parent fc6b3d9 commit b696cc3

4 files changed

Lines changed: 215 additions & 48 deletions

File tree

‎docs/source/user-guide/latest/compatibility/scans.md‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -62,11 +62,13 @@ The following limitation may produce incorrect results without falling back to S
6262

6363
The following limitations raise an error at scan time rather than falling back to Spark:
6464

65-
- Byte-identical sibling field names, including inside structs, arrays, and maps. Comet rejects
66-
the entire file before decoding, even when the duplicate fields are not projected or the read
67-
schema uses field IDs. This prevents row multiplication and decoder synchronization errors.
68-
The check applies in both case-sensitivity modes; names in separate groups do not collide.
69-
Disable Comet for the query to use Spark's duplicate-name resolution with an explicit read schema.
65+
- Byte-identical sibling field names in selected top-level columns, including inside structs,
66+
arrays, and maps. Comet rejects these before decoding to prevent row multiplication and decoder
67+
synchronization errors. Unselected top-level columns are skipped, except for field-ID reads,
68+
which validate the entire file schema. The check applies in both case-sensitivity modes;
69+
names in separate groups do not collide. Disable Comet for the query to use Spark's duplicate-name
70+
resolution with an explicit read schema. Spark-compatible resolution is tracked in
71+
[#5884](https://github.com/apache/datafusion-comet/issues/5884).
7072
- Invalid UTF-8 bytes in `STRING` columns. Spark permits arbitrary byte sequences in a `STRING`
7173
column (for example from `CAST(X'C1' AS STRING)`), but Comet's native execution path is built on
7274
Arrow, whose string type is strictly UTF-8. Reading a Parquet file whose `STRING` column contains

‎native/core/src/parquet/eager_page_index_reader_factory.rs‎

Lines changed: 125 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,8 @@
4747
//! page-index load back into `FileMetadataCache` instead of bypassing it. Preserve the
4848
//! duplicate-field validation when replacing this factory.
4949
50-
use arrow::datatypes::{DataType, FieldRef, Schema};
50+
use crate::parquet::name_fold::{fold_name, fold_schema_names};
51+
use arrow::datatypes::{DataType, FieldRef, Schema, SchemaRef};
5152
use async_trait::async_trait;
5253
use bytes::Bytes;
5354
use datafusion::common::Result as DFResult;
@@ -165,6 +166,8 @@ pub struct EagerPageIndexReaderFactory {
165166
// Enable the footer workaround only for scans that project Variant.
166167
// https://github.com/apache/datafusion-comet/issues/5477
167168
spark_variant_schema: bool,
169+
projected_fields: Option<Arc<HashSet<String>>>,
170+
case_sensitive: bool,
168171
}
169172

170173
impl EagerPageIndexReaderFactory {
@@ -193,9 +196,29 @@ impl EagerPageIndexReaderFactory {
193196
metadata_cache,
194197
scan_io_metrics,
195198
spark_variant_schema: false,
199+
projected_fields: None,
200+
case_sensitive: true,
196201
}
197202
}
198203

204+
pub(crate) fn with_required_schema(
205+
mut self,
206+
schema: &SchemaRef,
207+
case_sensitive: bool,
208+
use_field_id: bool,
209+
) -> Self {
210+
// Field-ID projections can rename columns, so names cannot safely restrict the walk.
211+
self.projected_fields = (!use_field_id).then(|| {
212+
Arc::new(
213+
fold_schema_names(schema, case_sensitive)
214+
.into_iter()
215+
.collect(),
216+
)
217+
});
218+
self.case_sensitive = case_sensitive;
219+
self
220+
}
221+
199222
pub fn with_spark_variant_schema(mut self, enabled: bool) -> Self {
200223
self.spark_variant_schema = enabled;
201224
self
@@ -227,6 +250,8 @@ impl ParquetFileReaderFactory for EagerPageIndexReaderFactory {
227250
metadata_cache: Arc::clone(&self.metadata_cache),
228251
metadata_size_hint,
229252
spark_variant_schema: self.spark_variant_schema,
253+
projected_fields: self.projected_fields.clone(),
254+
case_sensitive: self.case_sensitive,
230255
}))
231256
}
232257
}
@@ -242,6 +267,8 @@ struct EagerPageIndexReader {
242267
metadata_cache: Arc<FileMetadataCache>,
243268
metadata_size_hint: Option<usize>,
244269
spark_variant_schema: bool,
270+
projected_fields: Option<Arc<HashSet<String>>>,
271+
case_sensitive: bool,
245272
}
246273

247274
// Arrow infers ENUM as Binary, losing the distinction from raw binary that Spark needs.
@@ -376,20 +403,29 @@ fn with_spark_arrow_schema(metadata: Arc<ParquetMetaData>) -> ParquetResult<Arc<
376403

377404
// Duplicate sibling names can make the decoder combine distinct leaves into one column,
378405
// multiplying rows before schema adaptation can reject or resolve the duplicate (#5783).
379-
// Reject the entire file, including unprojected fields, until the decoder can safely
380-
// resolve duplicate siblings. Names in separate groups do not collide.
381-
fn validate_field_names(schema: &Type) -> parquet::errors::Result<()> {
406+
// Only selected top-level subtrees can reach the decoder. Recurse fully within each selected
407+
// subtree because nested projection does not safely separate duplicate leaves (#5884).
408+
fn validate_field_names(
409+
schema: &Type,
410+
projected_fields: Option<&HashSet<String>>,
411+
case_sensitive: bool,
412+
) -> parquet::errors::Result<()> {
382413
if let Type::GroupType { fields, .. } = schema {
383414
let mut names = HashSet::with_capacity(fields.len());
384415
for field in fields {
416+
if projected_fields.is_some_and(|projected| {
417+
!projected.contains(&fold_name(field.name(), case_sensitive))
418+
}) {
419+
continue;
420+
}
385421
if !names.insert(field.name()) {
386422
return Err(ParquetError::General(format!(
387423
"Comet native scan does not support duplicate Parquet field name '{}' in group '{}'",
388424
field.name(),
389425
schema.name()
390426
)));
391427
}
392-
validate_field_names(field)?;
428+
validate_field_names(field, None, case_sensitive)?;
393429
}
394430
}
395431
Ok(())
@@ -462,6 +498,8 @@ impl AsyncFileReader for EagerPageIndexReader {
462498
let metadata_size_hint = self.metadata_size_hint;
463499
let scan_io_metrics = Arc::clone(&self.scan_io_metrics);
464500
let spark_variant_schema = self.spark_variant_schema;
501+
let projected_fields = self.projected_fields.clone();
502+
let case_sensitive = self.case_sensitive;
465503
async move {
466504
let file_decryption_properties = options
467505
.and_then(|o| o.file_decryption_properties())
@@ -522,7 +560,11 @@ impl AsyncFileReader for EagerPageIndexReader {
522560

523561
let metadata = metadata?;
524562
// Validate cache hits too, before Arrow constructs a decoder for any projection.
525-
validate_field_names(metadata.file_metadata().schema_descr().root_schema())?;
563+
validate_field_names(
564+
metadata.file_metadata().schema_descr().root_schema(),
565+
projected_fields.as_deref(),
566+
case_sensitive,
567+
)?;
526568
if spark_variant_schema {
527569
with_spark_arrow_schema(metadata)
528570
} else {
@@ -866,6 +908,83 @@ mod tests {
866908
},
867909
};
868910

911+
#[test]
912+
fn projected_fields_skip_unselected_roots() {
913+
let schema = parquet::schema::parser::parse_message_type(
914+
"message root { optional int64 a; optional int64 a; optional int64 b; }",
915+
)
916+
.unwrap();
917+
let selected = HashSet::from(["b".to_string()]);
918+
validate_field_names(&schema, Some(&selected), true).unwrap();
919+
validate_field_names(&schema, Some(&HashSet::new()), true).unwrap();
920+
let selected = HashSet::from(["a".to_string()]);
921+
assert!(validate_field_names(&schema, Some(&selected), true).is_err());
922+
}
923+
924+
#[test]
925+
fn projected_fields_match_case_insensitively_and_recurse_fully() {
926+
let schema = parquet::schema::parser::parse_message_type(
927+
"message root { optional group Selected {
928+
optional int64 valid; optional int64 dup; optional int64 dup;
929+
} optional int64 unrelated; }",
930+
)
931+
.unwrap();
932+
let selected = HashSet::from(["selected".to_string()]);
933+
assert!(validate_field_names(&schema, Some(&selected), false).is_err());
934+
let selected = HashSet::from(["unrelated".to_string()]);
935+
validate_field_names(&schema, Some(&selected), false).unwrap();
936+
}
937+
938+
#[test]
939+
fn duplicate_names_in_list_element_are_rejected() {
940+
let schema = parquet::schema::parser::parse_message_type(
941+
"message root { optional group a (LIST) { repeated group list {
942+
optional group element { optional int64 dup; optional int64 dup; }
943+
} } }",
944+
)
945+
.unwrap();
946+
assert!(validate_field_names(&schema, None, true)
947+
.unwrap_err()
948+
.to_string()
949+
.contains("group 'element'"));
950+
}
951+
952+
#[test]
953+
fn duplicate_names_in_map_key_value_are_rejected() {
954+
let schema = parquet::schema::parser::parse_message_type(
955+
"message root { optional group a (MAP) { repeated group key_value {
956+
required binary key (UTF8); optional int64 value; optional int64 value;
957+
} } }",
958+
)
959+
.unwrap();
960+
assert!(validate_field_names(&schema, None, true)
961+
.unwrap_err()
962+
.to_string()
963+
.contains("group 'key_value'"));
964+
}
965+
966+
#[test]
967+
fn repeated_names_in_separate_groups_are_valid() {
968+
let schema = parquet::schema::parser::parse_message_type(
969+
"message root { optional group a { optional int64 same; }
970+
optional group b { optional int64 same; } }",
971+
)
972+
.unwrap();
973+
validate_field_names(&schema, None, true).unwrap();
974+
}
975+
976+
#[test]
977+
fn duplicate_root_names_are_rejected() {
978+
let schema = parquet::schema::parser::parse_message_type(
979+
"message root { optional int64 a; optional int64 a; optional int64 b; }",
980+
)
981+
.unwrap();
982+
assert!(validate_field_names(&schema, None, true)
983+
.unwrap_err()
984+
.to_string()
985+
.contains("group 'root'"));
986+
}
987+
869988
#[derive(Debug)]
870989
struct RecordingRangeStore {
871990
inner: InMemory,

‎native/core/src/parquet/parquet_exec.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,8 @@ pub(crate) fn init_datasource_exec(
194194
scan_io_source,
195195
parquet_source.metrics(),
196196
)
197-
.with_spark_variant_schema(projects_variant),
197+
.with_spark_variant_schema(projects_variant)
198+
.with_required_schema(&required_schema, case_sensitive, use_field_id),
198199
);
199200
parquet_source = parquet_source.with_parquet_file_reader_factory(reader_factory);
200201

‎spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala‎

Lines changed: 81 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -72,35 +72,29 @@ class CometNativeReaderSuite extends CometTestBase with AdaptiveSparkPlanHelper
7272
"map value",
7373
"map('key', named_struct('dup', id, 'dup', id + 100))",
7474
"map<string, struct<dup: bigint>>")).foreach { case (shape, expression, readType) =>
75-
Seq(1, 4096).foreach { batchSize =>
76-
test(s"duplicate Parquet field names fail before decoding - $shape - batch $batchSize") {
77-
withSQLConf(
78-
SQLConf.CASE_SENSITIVE.key -> "true",
79-
CometConf.COMET_BATCH_SIZE.key -> batchSize.toString) {
80-
withTempPath { path =>
81-
withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
82-
// Keep all rows in one file so batch size 1 exercises a multi-batch read.
83-
spark
84-
.range(3)
85-
.coalesce(1)
86-
.selectExpr(s"$expression as s")
87-
.write
88-
.parquet(path.toString)
89-
// The file is readable by Spark with an explicit schema.
90-
assert(
91-
spark.read.schema(s"s $readType").parquet(path.toString).collect().length == 3)
92-
}
93-
val df = spark.read.schema(s"s $readType").parquet(path.toString)
94-
assert(
95-
find(df.queryExecution.executedPlan)(_.isInstanceOf[CometNativeScanExec]).isDefined)
96-
val error = intercept[Exception](df.collect())
97-
val messages = Iterator
98-
.iterate[Throwable](error)(_.getCause)
99-
.takeWhile(_ != null)
100-
.map(_.getMessage)
101-
.mkString("\n")
102-
assert(messages.contains("duplicate Parquet field name 'dup'"), messages)
75+
test(s"duplicate Parquet field names fail before decoding - $shape") {
76+
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
77+
withTempPath { path =>
78+
withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
79+
spark
80+
.range(3)
81+
.coalesce(1)
82+
.selectExpr(s"$expression as s")
83+
.write
84+
.parquet(path.toString)
85+
// The file is readable by Spark with an explicit schema.
86+
assert(spark.read.schema(s"s $readType").parquet(path.toString).collect().length == 3)
10387
}
88+
val df = spark.read.schema(s"s $readType").parquet(path.toString)
89+
assert(
90+
find(df.queryExecution.executedPlan)(_.isInstanceOf[CometNativeScanExec]).isDefined)
91+
val error = intercept[Exception](df.collect())
92+
val messages = Iterator
93+
.iterate[Throwable](error)(_.getCause)
94+
.takeWhile(_ != null)
95+
.map(_.getMessage)
96+
.mkString("\n")
97+
assert(messages.contains("duplicate Parquet field name 'dup'"), messages)
10498
}
10599
}
106100
}
@@ -117,23 +111,74 @@ class CometNativeReaderSuite extends CometTestBase with AdaptiveSparkPlanHelper
117111
}
118112
Seq(true, false).foreach { caseSensitive =>
119113
withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
120-
val df = spark.read.schema("id bigint").parquet(path.toString)
114+
val name = if (caseSensitive) "id" else "ID"
115+
val df = spark.read.schema(s"$name bigint").parquet(path.toString)
121116
assert(
122117
find(df.queryExecution.executedPlan)(_.isInstanceOf[CometNativeScanExec]).isDefined)
123118
(1 to 2).foreach { _ =>
124-
val error = intercept[Exception](df.collect())
125-
val messages = Iterator
126-
.iterate[Throwable](error)(_.getCause)
127-
.takeWhile(_ != null)
128-
.map(_.getMessage)
129-
.mkString("\n")
130-
assert(messages.contains("duplicate Parquet field name 'dup'"), messages)
119+
checkAnswer(df, Seq(Row(0L), Row(1L), Row(2L)))
120+
checkAnswer(df.where("id > 1000"), Seq.empty)
121+
checkAnswer(df.selectExpr("count(*)"), Seq(Row(3L)))
131122
}
132123
}
133124
}
134125
}
135126
}
136127

128+
test("duplicate Parquet field names - root group and unprojected root duplicates") {
129+
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
130+
withTempPath { path =>
131+
writeDirect(
132+
path.toString,
133+
"message spark_schema { optional int64 a = 1; optional int64 a = 2; optional int64 b = 3; }",
134+
{ rc =>
135+
rc.startMessage()
136+
Seq(("a", 0, 1L), ("a", 1, 2L), ("b", 2, 3L)).foreach { case (name, index, value) =>
137+
rc.startField(name, index)
138+
rc.addLong(value)
139+
rc.endField(name, index)
140+
}
141+
rc.endMessage()
142+
})
143+
withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
144+
checkAnswer(spark.read.schema("a bigint").parquet(path.toString), Seq(Row(1L)))
145+
}
146+
val selected = spark.read.schema("a bigint").parquet(path.toString)
147+
assert(
148+
find(selected.queryExecution.executedPlan)(
149+
_.isInstanceOf[CometNativeScanExec]).isDefined)
150+
val error = intercept[Exception](selected.collect())
151+
val messages = Iterator
152+
.iterate[Throwable](error)(_.getCause)
153+
.takeWhile(_ != null)
154+
.map(_.getMessage)
155+
.mkString("\n")
156+
assert(messages.contains("duplicate Parquet field name 'a'"), messages)
157+
val valid = spark.read.schema("b bigint").parquet(path.toString)
158+
assert(
159+
find(valid.queryExecution.executedPlan)(_.isInstanceOf[CometNativeScanExec]).isDefined)
160+
checkAnswer(valid, Seq(Row(3L)))
161+
withSQLConf(SQLConf.PARQUET_FIELD_ID_READ_ENABLED.key -> "true") {
162+
val schema = new StructType().add(
163+
"renamed_b",
164+
LongType,
165+
nullable = true,
166+
new MetadataBuilder().putLong("parquet.field.id", 3L).build())
167+
val byId = spark.read.schema(schema).parquet(path.toString)
168+
assert(
169+
find(byId.queryExecution.executedPlan)(_.isInstanceOf[CometNativeScanExec]).isDefined)
170+
val fieldIdError = intercept[Exception](byId.collect())
171+
val fieldIdMessages = Iterator
172+
.iterate[Throwable](fieldIdError)(_.getCause)
173+
.takeWhile(_ != null)
174+
.map(_.getMessage)
175+
.mkString("\n")
176+
assert(fieldIdMessages.contains("duplicate Parquet field name 'a'"), fieldIdMessages)
177+
}
178+
}
179+
}
180+
}
181+
137182
test(
138183
"duplicate Parquet field names - distinct siblings and repeated names in separate groups") {
139184
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {

0 commit comments

Comments
 (0)