Skip to content

Commit a9b61ab

Browse files
authored
fix(parquet): remap sorting columns for partitioned writes (#24211)
## Which issue does this PR close? - Closes #24210. ## Rationale for this change When Hive partition columns are not kept in Parquet files, `FileSinkConfig::output_schema()` still describes the sink input while `get_writer_schema(&conf)` removes those partition columns. `ParquetFormat::create_writer_physical_plan` currently converts the input ordering directly to Parquet `sorting_columns`, so its column indices can reference removed columns or the wrong positions in the written schema. Reading that metadata can then panic in `SchemaDescriptor::column` with an out-of-bounds index. The execution ordering and the Parquet footer ordering have different schema domains: `DataSinkExec` must retain the original input ordering, while `ParquetSink` metadata must use indices from the actual writer schema. ## What changes are included in this PR? - Derive Parquet `sorting_columns` from both the input schema and `get_writer_schema(&conf)`. - Omit ordering keys that are removed from the written file, such as Hive partition columns. - Remap retained ordering keys to their writer-schema indices while preserving sort direction and null ordering. - Keep the original `order_requirements` unchanged for `DataSinkExec`. - Avoid writing an empty `sorting_columns` list when every ordering key is removed. - Add unit and end-to-end regression coverage for partitioned Parquet writes. ## Are these changes tested? Yes. The new tests cover the writer-schema remapping directly and verify the resulting Parquet footer after a partitioned write. ## Are there any user-facing changes? There are no public API changes. Newly written partitioned Parquet files now contain valid `sorting_columns` metadata based on their physical file schema. Existing files with invalid metadata are unchanged and must be rewritten separately.
1 parent d5bd10d commit a9b61ab

3 files changed

Lines changed: 177 additions & 10 deletions

File tree

‎datafusion/core/tests/parquet/ordering.rs‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,3 +101,83 @@ async fn test_create_table_with_order_writes_sorting_columns() -> Result<()> {
101101

102102
Ok(())
103103
}
104+
105+
/// Test that partition columns are removed and remaining column indices are
106+
/// remapped when writing sorting_columns to Parquet metadata.
107+
#[tokio::test]
108+
async fn test_partitioned_table_remaps_sorting_columns() -> Result<()> {
109+
use parquet::file::reader::FileReader;
110+
use parquet::file::serialized_reader::SerializedFileReader;
111+
use std::fs::File;
112+
113+
let ctx = SessionContext::new();
114+
let tmp_dir = tempdir()?;
115+
let table_path = tmp_dir.path().join("sorted_partitioned_table");
116+
std::fs::create_dir_all(&table_path)?;
117+
118+
let create_table_sql = format!(
119+
"CREATE EXTERNAL TABLE sorted_partitioned_data (a INT, b VARCHAR, part VARCHAR) \
120+
STORED AS PARQUET \
121+
LOCATION '{}' \
122+
PARTITIONED BY (part) \
123+
WITH ORDER (part ASC NULLS FIRST, a ASC NULLS FIRST, b DESC NULLS LAST)",
124+
table_path.display()
125+
);
126+
ctx.sql(&create_table_sql).await?;
127+
128+
ctx.sql(
129+
"INSERT INTO sorted_partitioned_data VALUES \
130+
(2, 'c', 'x'), (1, 'a', 'x'), (1, 'b', 'x')",
131+
)
132+
.await?
133+
.collect()
134+
.await?;
135+
136+
let parquet_file = find_parquet_file(&table_path)?
137+
.expect("expected a Parquet file in the partition directory");
138+
139+
let file = File::open(parquet_file)?;
140+
let reader = SerializedFileReader::new(file)?;
141+
let metadata = reader.metadata();
142+
let parquet_schema = metadata.file_metadata().schema_descr();
143+
assert_eq!(parquet_schema.num_columns(), 2);
144+
assert_eq!(parquet_schema.column(0).name(), "a");
145+
assert_eq!(parquet_schema.column(1).name(), "b");
146+
147+
let sorting = metadata
148+
.row_group(0)
149+
.sorting_columns()
150+
.expect("expected sorting_columns in row group metadata");
151+
assert_eq!(sorting.len(), 2);
152+
153+
assert_eq!(sorting[0].column_idx, 0);
154+
assert!(!sorting[0].descending);
155+
assert!(sorting[0].nulls_first);
156+
157+
assert_eq!(sorting[1].column_idx, 1);
158+
assert!(sorting[1].descending);
159+
assert!(!sorting[1].nulls_first);
160+
161+
Ok(())
162+
}
163+
164+
fn find_parquet_file(
165+
path: &std::path::Path,
166+
) -> std::io::Result<Option<std::path::PathBuf>> {
167+
for entry in std::fs::read_dir(path)? {
168+
let entry = entry?;
169+
let path = entry.path();
170+
if path.is_dir() {
171+
if let Some(path) = find_parquet_file(&path)? {
172+
return Ok(Some(path));
173+
}
174+
} else if path
175+
.extension()
176+
.is_some_and(|extension| extension == "parquet")
177+
{
178+
return Ok(Some(path));
179+
}
180+
}
181+
182+
Ok(None)
183+
}

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ use datafusion_common::{
5050
use datafusion_datasource::file::FileSource;
5151
use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
5252
use datafusion_datasource::sink::DataSinkExec;
53+
use datafusion_datasource::write::get_writer_schema;
5354
use datafusion_expr::dml::InsertOp;
5455
use datafusion_physical_expr_common::sort_expr::{LexOrdering, LexRequirement};
5556
use datafusion_physical_plan::ExecutionPlan;
@@ -534,11 +535,18 @@ impl FileFormat for ParquetFormat {
534535
// Convert ordering requirements to Parquet SortingColumns for file metadata
535536
let sorting_columns = if let Some(ref requirements) = order_requirements {
536537
let ordering: LexOrdering = requirements.clone().into();
538+
let writer_schema = get_writer_schema(&conf);
537539
// In cases like `COPY (... ORDER BY ...) TO ...` the ORDER BY clause
538540
// may not be compatible with Parquet sorting columns (e.g. ordering on `random()`).
539541
// So if we cannot create a Parquet sorting column from the ordering requirement,
540542
// we skip setting sorting columns on the Parquet sink.
541-
lex_ordering_to_sorting_columns(&ordering).ok()
543+
lex_ordering_to_sorting_columns(
544+
&ordering,
545+
conf.output_schema(),
546+
&writer_schema,
547+
)
548+
.ok()
549+
.filter(|columns| !columns.is_empty())
542550
} else {
543551
None
544552
};

‎datafusion/datasource-parquet/src/metadata.rs‎

Lines changed: 88 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ use datafusion_common::encryption::FileDecryptionProperties;
2828
use datafusion_common::stats::Precision;
2929
use datafusion_common::{
3030
ColumnStatistics, DataFusionError, HashMap, Result, ScalarValue, Statistics,
31+
internal_datafusion_err,
3132
};
3233
use datafusion_execution::cache::cache_manager::{
3334
CachedFileMetadataEntry, FileMetadata, FileMetadataCache,
@@ -952,10 +953,15 @@ impl FileMetadata for CachedParquetMetaData {
952953

953954
/// Convert a [`PhysicalSortExpr`] to a Parquet [`SortingColumn`].
954955
///
955-
/// Returns `Err` if the expression is not a simple column reference.
956+
/// Returns `Ok(None)` if the referenced column is not in `writer_schema`, such as a
957+
/// hive partition column that is removed before writing the Parquet file. Returns
958+
/// `Err` if the expression is not a simple column reference or references a column
959+
/// outside `input_schema`.
956960
pub(crate) fn sort_expr_to_sorting_column(
957961
sort_expr: &PhysicalSortExpr,
958-
) -> Result<SortingColumn> {
962+
input_schema: &Schema,
963+
writer_schema: &Schema,
964+
) -> Result<Option<SortingColumn>> {
959965
let column = sort_expr.expr.downcast_ref::<Column>().ok_or_else(|| {
960966
DataFusionError::Plan(format!(
961967
"Parquet sorting_columns only supports simple column references, \
@@ -964,27 +970,49 @@ pub(crate) fn sort_expr_to_sorting_column(
964970
))
965971
})?;
966972

967-
let column_idx: i32 = column.index().try_into().map_err(|_| {
973+
let input_field = input_schema.fields().get(column.index()).ok_or_else(|| {
974+
internal_datafusion_err!(
975+
"Parquet sorting column '{}' references index {} but the input schema has {} columns",
976+
column.name(),
977+
column.index(),
978+
input_schema.fields().len()
979+
)
980+
})?;
981+
let Some((writer_index, _)) = writer_schema.column_with_name(input_field.name())
982+
else {
983+
return Ok(None);
984+
};
985+
986+
let column_idx: i32 = writer_index.try_into().map_err(|_| {
968987
DataFusionError::Plan(format!(
969-
"Column index {} is too large to be represented as i32",
970-
column.index()
988+
"Column index {writer_index} is too large to be represented as i32"
971989
))
972990
})?;
973991

974-
Ok(SortingColumn {
992+
Ok(Some(SortingColumn {
975993
column_idx,
976994
descending: sort_expr.options.descending,
977995
nulls_first: sort_expr.options.nulls_first,
978-
})
996+
}))
979997
}
980998

981999
/// Convert a [`LexOrdering`] to `Vec<SortingColumn>` for Parquet.
9821000
///
983-
/// Returns `Err` if any expression is not a simple column reference.
1001+
/// Columns that are not present in `writer_schema` are omitted from the resulting
1002+
/// metadata. Returns `Err` if any expression is not a simple column reference or
1003+
/// references a column outside `input_schema`.
9841004
pub(crate) fn lex_ordering_to_sorting_columns(
9851005
ordering: &LexOrdering,
1006+
input_schema: &Schema,
1007+
writer_schema: &Schema,
9861008
) -> Result<Vec<SortingColumn>> {
987-
ordering.iter().map(sort_expr_to_sorting_column).collect()
1009+
ordering
1010+
.iter()
1011+
.filter_map(|sort_expr| {
1012+
sort_expr_to_sorting_column(sort_expr, input_schema, writer_schema)
1013+
.transpose()
1014+
})
1015+
.collect()
9881016
}
9891017

9901018
/// Extracts ordering information from Parquet metadata.
@@ -1058,6 +1086,57 @@ fn sorting_columns_to_physical_exprs(
10581086
mod tests {
10591087
use super::*;
10601088
use arrow::array::Int32Array;
1089+
use arrow::compute::SortOptions;
1090+
use arrow::datatypes::Field;
1091+
1092+
#[test]
1093+
fn test_lex_ordering_to_sorting_columns_uses_writer_schema() -> Result<()> {
1094+
let input_schema = Schema::new(vec![
1095+
Field::new("a", DataType::Int32, true),
1096+
Field::new("part", DataType::Utf8, true),
1097+
Field::new("b", DataType::Utf8, true),
1098+
]);
1099+
let writer_schema = Schema::new(vec![
1100+
Field::new("a", DataType::Int32, true),
1101+
Field::new("b", DataType::Utf8, true),
1102+
]);
1103+
let ordering = LexOrdering::new(vec![
1104+
PhysicalSortExpr::new(
1105+
Arc::new(Column::new("part", 1)),
1106+
SortOptions::default(),
1107+
),
1108+
PhysicalSortExpr::new(Arc::new(Column::new("a", 0)), SortOptions::default()),
1109+
PhysicalSortExpr::new(
1110+
Arc::new(Column::new("b", 2)),
1111+
SortOptions {
1112+
descending: true,
1113+
nulls_first: false,
1114+
},
1115+
),
1116+
])
1117+
.unwrap();
1118+
1119+
let sorting_columns =
1120+
lex_ordering_to_sorting_columns(&ordering, &input_schema, &writer_schema)?;
1121+
1122+
assert_eq!(
1123+
sorting_columns,
1124+
vec![
1125+
SortingColumn {
1126+
column_idx: 0,
1127+
descending: false,
1128+
nulls_first: true,
1129+
},
1130+
SortingColumn {
1131+
column_idx: 1,
1132+
descending: true,
1133+
nulls_first: false,
1134+
},
1135+
]
1136+
);
1137+
1138+
Ok(())
1139+
}
10611140

10621141
#[test]
10631142
fn test_has_any_exact_match() {

0 commit comments

Comments
 (0)