Skip to content

Commit bd8abfc

Browse files
committed
docs: describe list and map nulling for all-missing nested leaves
1 parent 0186736 commit bd8abfc

3 files changed

Lines changed: 20 additions & 18 deletions

File tree

‎native/core/src/parquet/parquet_support.rs‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -93,10 +93,11 @@ pub struct SparkParquetOptions {
9393
pub use_legacy_date_timestamp_or_ntz: bool,
9494
// Whether schema field names are case sensitive
9595
pub case_sensitive: bool,
96-
/// SPARK-53535 (Spark 4.1+): when reading a struct whose requested fields are all
97-
/// missing in the Parquet file, true returns the entire struct as null (pre-4.1
98-
/// legacy behavior); false preserves the parent struct's nullness from the file
99-
/// so non-null parents return a struct of all-null fields.
96+
/// SPARK-53535 (Spark 4.1+): when none of the requested leaves of a struct, list or
97+
/// map exist in the Parquet file, true returns the entire value as null (pre-4.1
98+
/// legacy behavior); false preserves the parent's nullness from the file, so
99+
/// non-null parents keep their structure with all-null leaves. Comet sets it to true
100+
/// on Spark 3.4 through 4.0, which always use the legacy behavior.
100101
pub return_null_struct_if_all_fields_missing: bool,
101102
/// When true, resolve fields by parquet.field.id metadata instead of name
102103
/// (mirrors Spark's `spark.sql.parquet.fieldId.read.enabled`). Only takes effect
@@ -570,12 +571,13 @@ fn parquet_convert_struct_to_struct(
570571
}
571572
}
572573

573-
// When the file's struct contains none of the requested fields, the
574-
// returned validity buffer depends on Spark's
575-
// `spark.sql.legacy.parquet.returnNullStructIfAllFieldsMissing` (SPARK-53535,
576-
// Spark 4.1+). Legacy mode marks the whole column null; the new default
577-
// preserves the file's parent-row nullness so non-null parents materialize
578-
// as a struct of all-null fields.
574+
// When no requested leaf exists in the file, directly or through nested
575+
// struct, list or map children, the returned validity buffer depends on
576+
// Spark's `spark.sql.legacy.parquet.returnNullStructIfAllFieldsMissing`
577+
// (SPARK-53535, Spark 4.1+). Legacy mode marks the whole column null; the new
578+
// default preserves the file's parent-row nullness so non-null parents keep
579+
// their nested structure with all-null leaves. The list and map arms apply the
580+
// same rule.
579581
let nulls =
580582
if !field_overlap && parquet_options.return_null_struct_if_all_fields_missing {
581583
Some(NullBuffer::new_null(array.len()))

‎native/proto/src/proto/operator.proto‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -156,10 +156,10 @@ message NativeScanCommon {
156156
bool encryption_enabled = 11;
157157
string source = 12;
158158
repeated spark.spark_expression.DataType fields = 13;
159-
// SPARK-53535 (Spark 4.1+): when reading a struct whose requested fields are all
160-
// missing in the Parquet file, true returns the entire struct as null (legacy
161-
// pre-4.1 behavior); false preserves the parent struct's nullness from the file
162-
// so non-null parents return a struct of all-null fields.
159+
// SPARK-53535 (Spark 4.1+): when none of the requested leaves of a struct, list or
160+
// map exist in the Parquet file, true returns the entire value as null (legacy
161+
// pre-4.1 behavior); false preserves the parent's nullness from the file, so
162+
// non-null parents keep their structure with all-null leaves.
163163
bool return_null_struct_if_all_fields_missing = 14;
164164
// True when spark.sql.parquet.fieldId.read.enabled is set and the requested
165165
// schema actually carries parquet.field.id metadata. When false the native

‎spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -422,10 +422,10 @@ object CometNativeScan extends CometOperatorSerde[CometScanExec] with CometTypeS
422422
CometTimeZone.nativeId(Some(sessionTimeZone)).getOrElse(sessionTimeZone))
423423
commonBuilder.setCaseSensitive(conf.getConf[Boolean](SQLConf.CASE_SENSITIVE))
424424

425-
// SPARK-53535 (Spark 4.1+): when reading a struct whose requested fields are all
426-
// missing in the Parquet file, the new default preserves the parent struct's
427-
// nullness from the file (so non-null parents materialize as a struct of all-null
428-
// fields). Older Spark versions always use legacy behavior and ignore this key.
425+
// SPARK-53535 (Spark 4.1+): when none of the requested leaves of a struct, list or
426+
// map exist in the Parquet file, the new default preserves the parent's nullness
427+
// from the file (so non-null parents keep their structure with all-null leaves).
428+
// Older Spark versions always use legacy behavior and ignore this key.
429429
val returnNullStructConfKey =
430430
"spark.sql.legacy.parquet.returnNullStructIfAllFieldsMissing"
431431
commonBuilder.setReturnNullStructIfAllFieldsMissing(

0 commit comments

Comments
 (0)