@@ -93,10 +93,11 @@ pub struct SparkParquetOptions {
9393 pub use_legacy_date_timestamp_or_ntz : bool ,
9494 // Whether schema field names are case sensitive
9595 pub case_sensitive : bool ,
96- /// SPARK-53535 (Spark 4.1+): when reading a struct whose requested fields are all
97- /// missing in the Parquet file, true returns the entire struct as null (pre-4.1
98- /// legacy behavior); false preserves the parent struct's nullness from the file
99- /// so non-null parents return a struct of all-null fields.
96+ /// SPARK-53535 (Spark 4.1+): when none of the requested leaves of a struct, list or
97+ /// map exist in the Parquet file, true returns the entire value as null (pre-4.1
98+ /// legacy behavior); false preserves the parent's nullness from the file, so
99+ /// non-null parents keep their structure with all-null leaves. Comet sets it to true
100+ /// on Spark 3.4 through 4.0, which always use the legacy behavior.
100101 pub return_null_struct_if_all_fields_missing : bool ,
101102 /// When true, resolve fields by parquet.field.id metadata instead of name
102103 /// (mirrors Spark's `spark.sql.parquet.fieldId.read.enabled`). Only takes effect
@@ -570,12 +571,13 @@ fn parquet_convert_struct_to_struct(
570571 }
571572 }
572573
573- // When the file's struct contains none of the requested fields, the
574- // returned validity buffer depends on Spark's
575- // `spark.sql.legacy.parquet.returnNullStructIfAllFieldsMissing` (SPARK-53535,
576- // Spark 4.1+). Legacy mode marks the whole column null; the new default
577- // preserves the file's parent-row nullness so non-null parents materialize
578- // as a struct of all-null fields.
574+ // When no requested leaf exists in the file, directly or through nested
575+ // struct, list or map children, the returned validity buffer depends on
576+ // Spark's `spark.sql.legacy.parquet.returnNullStructIfAllFieldsMissing`
577+ // (SPARK-53535, Spark 4.1+). Legacy mode marks the whole column null; the new
578+ // default preserves the file's parent-row nullness so non-null parents keep
579+ // their nested structure with all-null leaves. The list and map arms apply the
580+ // same rule.
579581 let nulls =
580582 if !field_overlap && parquet_options. return_null_struct_if_all_fields_missing {
581583 Some ( NullBuffer :: new_null ( array. len ( ) ) )
0 commit comments