Skip to content

Commit 96eb0df

Browse files
rxdrccqalamb
andauthored
fix: validate parquet encoding when configured (apache#25784)
## Which issue does this PR close? Part of apache#17498. This PR validates the global Parquet encoding setting; the issue also tracks other configuration values. ## Rationale for this change `SET datafusion.execution.parquet.encoding = 'invalid'` currently succeeds. The error appears only when a Parquet write starts, so the invalid setting is harder to diagnose. ## What changes are included in this PR? - Add `DFParquetEncoding` to validate supported names when the global setting is assigned, following the approach used for compression in apache#25626. - Preserve the previous value after an invalid `SET`, and display valid values in canonical lowercase form. - Use the validated value when building Parquet writer properties while retaining column-specific encoding behavior. - Validate the global setting in both Proto conversion paths and add regression tests. ## What is the testing strategy for this PR? Added unit tests for encoding parsing and conversion and Proto conversion, plus SQL logic coverage for `SET`, `SHOW`, invalid `SET`, and `RESET`. Validation with Rust 1.99.0: - Before removing the optimizer lint annotations, the extended workspace test command in `AGENTS.md` passed: 12,678 Rust tests, 8 ignored, and all 534 SQL logic test files. - `cargo fmt --all --check` - `./dev/rust_lint.sh` (before removing the optimizer lint annotations) - After removing the annotations: `cargo fmt --all` and `./ci/scripts/rust_clippy.sh` passed. There is no relevant local benchmark for configuration parsing or writer-property setup. ## Are there any user-facing changes? Invalid global Parquet encodings now fail at `SET` time instead of at write time. `SHOW` displays valid encodings in lowercase. The public Rust field `ParquetOptions.encoding` changes from `Option<String>` to `Option<DFParquetEncoding>`. Please apply the `api change` label. --------- Co-authored-by: Andrew Lamb <[email protected]>
1 parent 63af08d commit 96eb0df

8 files changed

Lines changed: 279 additions & 41 deletions

File tree

‎datafusion/common/src/config.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ use crate::encryption::{FileDecryptionProperties, FileEncryptionProperties};
2424
use crate::error::{_config_datafusion_err, _config_err};
2525
use crate::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType};
2626
use crate::parquet_config::{
27-
DFParquetCompression, DFParquetStatistics, DFParquetWriterVersion, DFTimeUnit,
28-
RowGroupRangeAssignment,
27+
DFParquetCompression, DFParquetEncoding, DFParquetStatistics, DFParquetWriterVersion,
28+
DFTimeUnit, RowGroupRangeAssignment,
2929
};
3030
use crate::parsers::{CompressionTypeVariant, CsvQuoteStyle};
3131
use crate::utils::get_available_parallelism;
@@ -1544,7 +1544,7 @@ config_namespace! {
15441544
/// delta_byte_array, rle_dictionary, and byte_stream_split.
15451545
/// These values are not case sensitive. If NULL, uses
15461546
/// default parquet writer setting
1547-
pub encoding: Option<String>, transform = str::to_lowercase, default = None
1547+
pub encoding: Option<DFParquetEncoding>, default = None
15481548

15491549
/// (writing) Write bloom filters for all columns when creating parquet files
15501550
pub bloom_filter_on_write: bool, default = false

‎datafusion/common/src/file_options/parquet_writer.rs‎

Lines changed: 11 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ use std::sync::Arc;
2222
use crate::{
2323
_internal_datafusion_err, DataFusionError, Result,
2424
config::{ParquetCdcOptions, ParquetOptions, TableParquetOptions},
25-
parquet_config::DFParquetCompression,
25+
parquet_config::{DFParquetCompression, DFParquetEncoding},
2626
};
2727

2828
use arrow::datatypes::Schema;
@@ -291,39 +291,19 @@ impl ParquetOptions {
291291
builder = builder.set_compression((*compression).into());
292292
}
293293
if let Some(encoding) = encoding {
294-
builder = builder.set_encoding(parse_encoding_string(encoding)?);
294+
builder = builder.set_encoding((*encoding).into());
295295
}
296296
builder = builder.set_content_defined_chunking(content_defined_chunking.into());
297297

298298
Ok(builder)
299299
}
300300
}
301301

302-
/// Parses datafusion.execution.parquet.encoding String to a parquet::basic::Encoding
302+
/// Parses a column-specific parquet encoding string.
303303
pub(crate) fn parse_encoding_string(
304304
str_setting: &str,
305305
) -> Result<parquet::basic::Encoding> {
306-
let str_setting_lower: &str = &str_setting.to_lowercase();
307-
match str_setting_lower {
308-
"plain" => Ok(parquet::basic::Encoding::PLAIN),
309-
"plain_dictionary" => Ok(parquet::basic::Encoding::PLAIN_DICTIONARY),
310-
"rle" => Ok(parquet::basic::Encoding::RLE),
311-
#[expect(deprecated)]
312-
"bit_packed" => Ok(parquet::basic::Encoding::BIT_PACKED),
313-
"delta_binary_packed" => Ok(parquet::basic::Encoding::DELTA_BINARY_PACKED),
314-
"delta_length_byte_array" => {
315-
Ok(parquet::basic::Encoding::DELTA_LENGTH_BYTE_ARRAY)
316-
}
317-
"delta_byte_array" => Ok(parquet::basic::Encoding::DELTA_BYTE_ARRAY),
318-
"rle_dictionary" => Ok(parquet::basic::Encoding::RLE_DICTIONARY),
319-
"byte_stream_split" => Ok(parquet::basic::Encoding::BYTE_STREAM_SPLIT),
320-
_ => Err(DataFusionError::Configuration(format!(
321-
"Unknown or unsupported parquet encoding: \
322-
{str_setting}. Valid values are: plain, plain_dictionary, rle, \
323-
bit_packed, delta_binary_packed, delta_length_byte_array, \
324-
delta_byte_array, rle_dictionary, and byte_stream_split."
325-
))),
326-
}
306+
str_setting.parse::<DFParquetEncoding>().map(Into::into)
327307
}
328308

329309
/// Parses datafusion.execution.parquet.compression String to a parquet::basic::Compression
@@ -357,7 +337,8 @@ mod tests {
357337
ParquetEncryptionOptions, ParquetOptions,
358338
};
359339
use crate::parquet_config::{
360-
DFParquetCompression, DFParquetStatistics, DFParquetWriterVersion,
340+
DFParquetCompression, DFParquetEncoding, DFParquetStatistics,
341+
DFParquetWriterVersion,
361342
};
362343
use parquet::basic::Compression;
363344
use parquet::file::properties::{
@@ -406,7 +387,7 @@ mod tests {
406387
column_index_truncate_length: Some(42),
407388
statistics_truncate_length: Some(42),
408389
data_page_row_count_limit: 42,
409-
encoding: Some("BYTE_STREAM_SPLIT".into()),
390+
encoding: Some(DFParquetEncoding::ByteStreamSplit),
410391
bloom_filter_on_write: !defaults.bloom_filter_on_write,
411392
bloom_filter_fpp: Some(0.42),
412393
bloom_filter_ndv: Some(42),
@@ -525,7 +506,10 @@ mod tests {
525506
data_page_row_count_limit: props.data_page_row_count_limit(),
526507

527508
// global options which set the default column props
528-
encoding: default_col_props.encoding,
509+
encoding: default_col_props
510+
.encoding
511+
.as_deref()
512+
.map(|encoding| encoding.parse().expect("valid parquet encoding")),
529513
compression: match props.compression(&default_col) {
530514
Compression::ZSTD(level) => {
531515
Some(DFParquetCompression::Zstd(level.compression_level() as u32))

‎datafusion/common/src/parquet_config.rs‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,123 @@ impl From<parquet::file::properties::EnabledStatistics> for DFParquetStatistics
274274
}
275275
}
276276

277+
/// Parquet encodings accepted by the writer.
278+
///
279+
/// Parsing the encoding when it is configured rejects unsupported values before
280+
/// a Parquet write begins.
281+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
282+
pub enum DFParquetEncoding {
283+
/// Plain encoding
284+
Plain,
285+
/// Deprecated plain dictionary encoding
286+
PlainDictionary,
287+
/// Run-length encoding
288+
Rle,
289+
/// Deprecated bit-packed encoding
290+
BitPacked,
291+
/// Delta encoding for integers
292+
DeltaBinaryPacked,
293+
/// Delta length encoding for byte arrays
294+
DeltaLengthByteArray,
295+
/// Delta encoding for byte arrays
296+
DeltaByteArray,
297+
/// Run-length dictionary encoding
298+
RleDictionary,
299+
/// Byte stream split encoding
300+
ByteStreamSplit,
301+
}
302+
303+
impl FromStr for DFParquetEncoding {
304+
type Err = DataFusionError;
305+
306+
fn from_str(s: &str) -> Result<Self, Self::Err> {
307+
match s.to_ascii_lowercase().as_str() {
308+
"plain" => Ok(Self::Plain),
309+
"plain_dictionary" => Ok(Self::PlainDictionary),
310+
"rle" => Ok(Self::Rle),
311+
"bit_packed" => Ok(Self::BitPacked),
312+
"delta_binary_packed" => Ok(Self::DeltaBinaryPacked),
313+
"delta_length_byte_array" => Ok(Self::DeltaLengthByteArray),
314+
"delta_byte_array" => Ok(Self::DeltaByteArray),
315+
"rle_dictionary" => Ok(Self::RleDictionary),
316+
"byte_stream_split" => Ok(Self::ByteStreamSplit),
317+
_ => Err(DataFusionError::Configuration(format!(
318+
"Unknown or unsupported parquet encoding: \
319+
{s}. Valid values are: plain, plain_dictionary, rle, \
320+
bit_packed, delta_binary_packed, delta_length_byte_array, \
321+
delta_byte_array, rle_dictionary, and byte_stream_split."
322+
))),
323+
}
324+
}
325+
}
326+
327+
impl Display for DFParquetEncoding {
328+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
329+
f.write_str(match self {
330+
Self::Plain => "plain",
331+
Self::PlainDictionary => "plain_dictionary",
332+
Self::Rle => "rle",
333+
Self::BitPacked => "bit_packed",
334+
Self::DeltaBinaryPacked => "delta_binary_packed",
335+
Self::DeltaLengthByteArray => "delta_length_byte_array",
336+
Self::DeltaByteArray => "delta_byte_array",
337+
Self::RleDictionary => "rle_dictionary",
338+
Self::ByteStreamSplit => "byte_stream_split",
339+
})
340+
}
341+
}
342+
343+
impl ConfigField for Option<DFParquetEncoding> {
344+
fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {
345+
match self {
346+
Some(encoding) => v.some(key, encoding, description),
347+
None => v.none(key, description),
348+
}
349+
}
350+
351+
fn set(&mut self, key: &str, value: &str) -> Result<()> {
352+
if !key.is_empty() {
353+
return crate::error::_config_err!(
354+
"Config field parquet.encoding is a scalar Option<DFParquetEncoding> and does not have nested field \"{}\"",
355+
key
356+
);
357+
}
358+
359+
*self = Some(value.parse()?);
360+
Ok(())
361+
}
362+
363+
fn reset(&mut self, key: &str) -> Result<()> {
364+
if key.is_empty() {
365+
*self = None;
366+
Ok(())
367+
} else {
368+
crate::error::_config_err!(
369+
"Config field parquet.encoding is a scalar Option<DFParquetEncoding> and does not have nested field \"{}\"",
370+
key
371+
)
372+
}
373+
}
374+
}
375+
376+
#[cfg(feature = "parquet")]
377+
impl From<DFParquetEncoding> for parquet::basic::Encoding {
378+
fn from(value: DFParquetEncoding) -> Self {
379+
match value {
380+
DFParquetEncoding::Plain => Self::PLAIN,
381+
DFParquetEncoding::PlainDictionary => Self::PLAIN_DICTIONARY,
382+
DFParquetEncoding::Rle => Self::RLE,
383+
#[expect(deprecated)]
384+
DFParquetEncoding::BitPacked => Self::BIT_PACKED,
385+
DFParquetEncoding::DeltaBinaryPacked => Self::DELTA_BINARY_PACKED,
386+
DFParquetEncoding::DeltaLengthByteArray => Self::DELTA_LENGTH_BYTE_ARRAY,
387+
DFParquetEncoding::DeltaByteArray => Self::DELTA_BYTE_ARRAY,
388+
DFParquetEncoding::RleDictionary => Self::RLE_DICTIONARY,
389+
DFParquetEncoding::ByteStreamSplit => Self::BYTE_STREAM_SPLIT,
390+
}
391+
}
392+
}
393+
277394
/// Parquet compression codec, with a level for the codecs that take one
278395
///
279396
/// This enum validates compression settings at configuration time. An unknown
@@ -589,6 +706,67 @@ impl From<DFTimeUnit> for arrow::datatypes::TimeUnit {
589706
mod tests {
590707
use super::*;
591708

709+
#[cfg(feature = "parquet")]
710+
#[test]
711+
fn parquet_encoding_parses_displays_and_converts() {
712+
use parquet::basic::Encoding;
713+
714+
for (input, expected, parquet_encoding) in [
715+
("PLAIN", DFParquetEncoding::Plain, Encoding::PLAIN),
716+
(
717+
"plain_dictionary",
718+
DFParquetEncoding::PlainDictionary,
719+
Encoding::PLAIN_DICTIONARY,
720+
),
721+
("rle", DFParquetEncoding::Rle, Encoding::RLE),
722+
#[expect(deprecated)]
723+
(
724+
"bit_packed",
725+
DFParquetEncoding::BitPacked,
726+
Encoding::BIT_PACKED,
727+
),
728+
(
729+
"delta_binary_packed",
730+
DFParquetEncoding::DeltaBinaryPacked,
731+
Encoding::DELTA_BINARY_PACKED,
732+
),
733+
(
734+
"delta_length_byte_array",
735+
DFParquetEncoding::DeltaLengthByteArray,
736+
Encoding::DELTA_LENGTH_BYTE_ARRAY,
737+
),
738+
(
739+
"delta_byte_array",
740+
DFParquetEncoding::DeltaByteArray,
741+
Encoding::DELTA_BYTE_ARRAY,
742+
),
743+
(
744+
"rle_dictionary",
745+
DFParquetEncoding::RleDictionary,
746+
Encoding::RLE_DICTIONARY,
747+
),
748+
(
749+
"byte_stream_split",
750+
DFParquetEncoding::ByteStreamSplit,
751+
Encoding::BYTE_STREAM_SPLIT,
752+
),
753+
] {
754+
let parsed: DFParquetEncoding = input.parse().unwrap();
755+
assert_eq!(parsed, expected, "input: {input}");
756+
assert_eq!(parsed.to_string(), input.to_ascii_lowercase());
757+
assert_eq!(Encoding::from(parsed), parquet_encoding);
758+
}
759+
760+
for input in ["", "invalid", "zstd"] {
761+
let err = input.parse::<DFParquetEncoding>().unwrap_err();
762+
assert!(
763+
err.to_string()
764+
.contains("Unknown or unsupported parquet encoding"),
765+
"input {input:?}: {err}"
766+
);
767+
}
768+
}
769+
592770
#[test]
593771
fn parquet_compression_parses_and_displays() {
594772
for (input, expected, displayed) in [

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -739,7 +739,7 @@ impl From<&ParquetFormatFactory> for protobuf::TableParquetOptions {
739739
}),
740740
data_page_row_count_limit: global_options.global.data_page_row_count_limit as u64,
741741
encoding_opt: global_options.global.encoding.map(|encoding| {
742-
parquet_options::EncodingOpt::Encoding(encoding)
742+
parquet_options::EncodingOpt::Encoding(encoding.to_string())
743743
}),
744744
bloom_filter_on_read: global_options.global.bloom_filter_on_read,
745745
bloom_filter_on_write: global_options.global.bloom_filter_on_write,

‎datafusion/proto-common/src/from_proto/mod.rs‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1253,11 +1253,12 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
12531253
"data_page_row_count_limit",
12541254
)?,
12551255
encoding: value
1256-
.encoding_opt.clone()
1256+
.encoding_opt
1257+
.as_ref()
12571258
.map(|opt| match opt {
1258-
protobuf::parquet_options::EncodingOpt::Encoding(v) => Some(v),
1259+
protobuf::parquet_options::EncodingOpt::Encoding(v) => v.parse(),
12591260
})
1260-
.unwrap_or(None),
1261+
.transpose()?,
12611262
bloom_filter_on_read: value.bloom_filter_on_read,
12621263
bloom_filter_on_write: value.bloom_filter_on_write,
12631264
bloom_filter_fpp: value.clone()
@@ -1454,7 +1455,8 @@ mod tests {
14541455
MaxRowGroupBytes, ParquetCdcOptions, ParquetOptions, TableParquetOptions,
14551456
};
14561457
use datafusion_common::parquet_config::{
1457-
DFParquetCompression, DFParquetStatistics, DFTimeUnit, RowGroupRangeAssignment,
1458+
DFParquetCompression, DFParquetEncoding, DFParquetStatistics, DFTimeUnit,
1459+
RowGroupRangeAssignment,
14581460
};
14591461

14601462
#[test]
@@ -1721,6 +1723,37 @@ mod tests {
17211723
assert_eq!(recovered.compression, Some(DFParquetCompression::Gzip(6)));
17221724
}
17231725

1726+
#[test]
1727+
fn test_parquet_encoding_round_trip() {
1728+
for encoding in [None, Some(DFParquetEncoding::ByteStreamSplit)] {
1729+
let opts = ParquetOptions {
1730+
encoding,
1731+
..ParquetOptions::default()
1732+
};
1733+
let recovered = parquet_options_proto_round_trip(opts);
1734+
assert_eq!(recovered.encoding, encoding);
1735+
}
1736+
}
1737+
1738+
#[test]
1739+
fn test_invalid_parquet_encoding_rejected_from_proto() {
1740+
let opts = ParquetOptions::default();
1741+
let mut proto: crate::protobuf_common::ParquetOptions =
1742+
(&opts).try_into().expect("to_proto");
1743+
proto.encoding_opt = Some(
1744+
crate::protobuf_common::parquet_options::EncodingOpt::Encoding(
1745+
"invalid".to_string(),
1746+
),
1747+
);
1748+
1749+
let err = ParquetOptions::try_from(&proto).unwrap_err();
1750+
assert!(
1751+
err.to_string()
1752+
.contains("Unknown or unsupported parquet encoding: invalid"),
1753+
"unexpected error: {err}"
1754+
);
1755+
}
1756+
17241757
#[test]
17251758
fn test_invalid_parquet_compression_rejected_from_proto() {
17261759
let opts = ParquetOptions::default();

‎datafusion/proto-common/src/to_proto/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -955,7 +955,7 @@ impl TryFrom<&ParquetOptions> for protobuf::ParquetOptions {
955955
column_index_truncate_length_opt: value.column_index_truncate_length.map(|v| protobuf::parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(v as u64)),
956956
statistics_truncate_length_opt: value.statistics_truncate_length.map(|v| protobuf::parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(v as u64)),
957957
data_page_row_count_limit: value.data_page_row_count_limit as u64,
958-
encoding_opt: value.encoding.clone().map(protobuf::parquet_options::EncodingOpt::Encoding),
958+
encoding_opt: value.encoding.map(|encoding| protobuf::parquet_options::EncodingOpt::Encoding(encoding.to_string())),
959959
bloom_filter_on_read: value.bloom_filter_on_read,
960960
bloom_filter_on_write: value.bloom_filter_on_write,
961961
bloom_filter_fpp_opt: value.bloom_filter_fpp.map(protobuf::parquet_options::BloomFilterFppOpt::BloomFilterFpp),

0 commit comments

Comments
 (0)