Skip to content

Commit 87bb5b8

Browse files
adriangbclaude
andcommitted
refactor(proto): restore format-factory conversions as From in the format crates
`CsvFormatFactory`, `JsonFormatFactory` and `ParquetFormatFactory` are owned by `datafusion-datasource-{csv,json,parquet}`, so their options-encoding conversions become plain `From` impls next to the types, behind each crate's existing `proto` feature. This is the last `FromProto` / `TryFromProto` impl in the tree: `convert.rs` and `convert_required_proto!` now have no implementors, clearing the way for the final cleanup in apache#24019. Part of apache#24019. Co-Authored-By: Claude Opus 5 <[email protected]>
1 parent 87c6303 commit 87bb5b8

4 files changed

Lines changed: 200 additions & 185 deletions

File tree

‎datafusion/datasource-csv/src/file_format.rs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -926,6 +926,54 @@ impl CsvSink {
926926
}
927927
}
928928

929+
/// Encode a [`CsvFormatFactory`]'s options as their protobuf form.
930+
///
931+
/// The reverse direction is `From<&protobuf::CsvOptions> for CsvOptions` in
932+
/// `datafusion-proto-models`: `CsvOptions` is a `datafusion-common` type, so
933+
/// that half cannot live here.
934+
#[cfg(feature = "proto")]
935+
impl From<&CsvFormatFactory> for datafusion_proto_models::protobuf::CsvOptions {
936+
fn from(factory: &CsvFormatFactory) -> Self {
937+
if let Some(options) = &factory.options {
938+
datafusion_proto_models::protobuf::CsvOptions {
939+
has_header: options.has_header.map_or(vec![], |v| vec![v as u8]),
940+
delimiter: vec![options.delimiter],
941+
quote: vec![options.quote],
942+
terminator: options.terminator.map_or(vec![], |v| vec![v]),
943+
escape: options.escape.map_or(vec![], |v| vec![v]),
944+
double_quote: options.double_quote.map_or(vec![], |v| vec![v as u8]),
945+
compression: options.compression as i32,
946+
schema_infer_max_rec: options.schema_infer_max_rec.map(|v| v as u64),
947+
date_format: options.date_format.clone().unwrap_or_default(),
948+
datetime_format: options.datetime_format.clone().unwrap_or_default(),
949+
timestamp_format: options.timestamp_format.clone().unwrap_or_default(),
950+
timestamp_tz_format: options
951+
.timestamp_tz_format
952+
.clone()
953+
.unwrap_or_default(),
954+
time_format: options.time_format.clone().unwrap_or_default(),
955+
null_value: options.null_value.clone().unwrap_or_default(),
956+
null_regex: options.null_regex.clone().unwrap_or_default(),
957+
comment: options.comment.map_or(vec![], |v| vec![v]),
958+
newlines_in_values: options
959+
.newlines_in_values
960+
.map_or(vec![], |v| vec![v as u8]),
961+
truncated_rows: options.truncated_rows.map_or(vec![], |v| vec![v as u8]),
962+
compression_level: options.compression_level,
963+
quote_style: options.quote_style as i32,
964+
ignore_leading_whitespace: options
965+
.ignore_leading_whitespace
966+
.map_or(vec![], |v| vec![v as u8]),
967+
ignore_trailing_whitespace: options
968+
.ignore_trailing_whitespace
969+
.map_or(vec![], |v| vec![v as u8]),
970+
}
971+
} else {
972+
datafusion_proto_models::protobuf::CsvOptions::default()
973+
}
974+
}
975+
}
976+
929977
#[cfg(test)]
930978
mod tests {
931979
use super::build_schema_helper;

‎datafusion/datasource-json/src/file_format.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -615,3 +615,24 @@ impl Decoder for JsonDecoder {
615615
false
616616
}
617617
}
618+
619+
/// Encode a [`JsonFormatFactory`]'s options as their protobuf form.
620+
///
621+
/// The reverse direction is `From<&protobuf::JsonOptions> for JsonOptions` in
622+
/// `datafusion-proto-models`: `JsonOptions` is a `datafusion-common` type, so
623+
/// that half cannot live here.
624+
#[cfg(feature = "proto")]
625+
impl From<&JsonFormatFactory> for datafusion_proto_models::protobuf::JsonOptions {
626+
fn from(factory: &JsonFormatFactory) -> Self {
627+
if let Some(options) = &factory.options {
628+
datafusion_proto_models::protobuf::JsonOptions {
629+
compression: options.compression as i32,
630+
schema_infer_max_rec: options.schema_infer_max_rec.map(|v| v as u64),
631+
compression_level: options.compression_level,
632+
newline_delimited: Some(options.newline_delimited),
633+
}
634+
} else {
635+
datafusion_proto_models::protobuf::JsonOptions::default()
636+
}
637+
}
638+
}

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -680,3 +680,129 @@ pub fn statistics_from_parquet_meta_calc(
680680
) -> Result<Statistics> {
681681
DFParquetMetadata::statistics_from_parquet_metadata(metadata, &table_schema)
682682
}
683+
684+
#[cfg(feature = "proto")]
685+
use datafusion_proto_models::protobuf::{self, parquet_column_options, parquet_options};
686+
687+
/// Encode a [`ParquetFormatFactory`]'s options as their protobuf form.
688+
///
689+
/// The reverse direction is `TryFrom<&protobuf::TableParquetOptions> for
690+
/// TableParquetOptions` in `datafusion-proto-models`: `TableParquetOptions` is
691+
/// a `datafusion-common` type, so that half cannot live here.
692+
#[cfg(feature = "proto")]
693+
impl From<&ParquetFormatFactory> for protobuf::TableParquetOptions {
694+
fn from(factory: &ParquetFormatFactory) -> Self {
695+
let global_options = if let Some(ref options) = factory.options {
696+
options.clone()
697+
} else {
698+
return protobuf::TableParquetOptions::default();
699+
};
700+
701+
let column_specific_options = global_options.column_specific_options;
702+
protobuf::TableParquetOptions {
703+
global: Some(protobuf::ParquetOptions {
704+
enable_page_index: global_options.global.enable_page_index,
705+
pruning: global_options.global.pruning,
706+
skip_metadata: global_options.global.skip_metadata,
707+
metadata_size_hint_opt: global_options.global.metadata_size_hint.map(|size| {
708+
parquet_options::MetadataSizeHintOpt::MetadataSizeHint(size as u64)
709+
}),
710+
pushdown_filters: global_options.global.pushdown_filters,
711+
reorder_filters: global_options.global.reorder_filters,
712+
force_filter_selections: global_options.global.force_filter_selections,
713+
data_pagesize_limit: global_options.global.data_pagesize_limit as u64,
714+
write_batch_size: global_options.global.write_batch_size as u64,
715+
writer_version: global_options.global.writer_version.to_string(),
716+
compression_opt: global_options.global.compression.map(|compression| {
717+
parquet_options::CompressionOpt::Compression(compression)
718+
}),
719+
dictionary_enabled_opt: global_options.global.dictionary_enabled.map(|enabled| {
720+
parquet_options::DictionaryEnabledOpt::DictionaryEnabled(enabled)
721+
}),
722+
dictionary_page_size_limit: global_options.global.dictionary_page_size_limit as u64,
723+
statistics_enabled_opt: global_options.global.statistics_enabled.map(|enabled| {
724+
parquet_options::StatisticsEnabledOpt::StatisticsEnabled(enabled)
725+
}),
726+
max_row_group_size: global_options.global.max_row_group_size as u64,
727+
max_in_list_size: global_options.global.max_in_list_size as u64,
728+
created_by: global_options.global.created_by.clone(),
729+
column_index_truncate_length_opt: global_options.global.column_index_truncate_length.map(|length| {
730+
parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(length as u64)
731+
}),
732+
statistics_truncate_length_opt: global_options.global.statistics_truncate_length.map(|length| {
733+
parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(length as u64)
734+
}),
735+
data_page_row_count_limit: global_options.global.data_page_row_count_limit as u64,
736+
encoding_opt: global_options.global.encoding.map(|encoding| {
737+
parquet_options::EncodingOpt::Encoding(encoding)
738+
}),
739+
bloom_filter_on_read: global_options.global.bloom_filter_on_read,
740+
bloom_filter_on_write: global_options.global.bloom_filter_on_write,
741+
bloom_filter_fpp_opt: global_options.global.bloom_filter_fpp.map(|fpp| {
742+
parquet_options::BloomFilterFppOpt::BloomFilterFpp(fpp)
743+
}),
744+
bloom_filter_ndv_opt: global_options.global.bloom_filter_ndv.map(|ndv| {
745+
parquet_options::BloomFilterNdvOpt::BloomFilterNdv(ndv)
746+
}),
747+
allow_single_file_parallelism: global_options.global.allow_single_file_parallelism,
748+
maximum_parallel_row_group_writers: global_options.global.maximum_parallel_row_group_writers as u64,
749+
maximum_buffered_record_batches_per_stream: global_options.global.maximum_buffered_record_batches_per_stream as u64,
750+
schema_force_view_types: global_options.global.schema_force_view_types,
751+
binary_as_string: global_options.global.binary_as_string,
752+
skip_arrow_metadata: global_options.global.skip_arrow_metadata,
753+
coerce_int96_opt: global_options.global.coerce_int96.map(|compression| {
754+
parquet_options::CoerceInt96Opt::CoerceInt96(compression)
755+
}),
756+
coerce_int96_tz_opt: global_options.global.coerce_int96_tz.map(|tz| {
757+
parquet_options::CoerceInt96TzOpt::CoerceInt96Tz(tz)
758+
}),
759+
max_predicate_cache_size_opt: global_options.global.max_predicate_cache_size.map(|size| {
760+
parquet_options::MaxPredicateCacheSizeOpt::MaxPredicateCacheSize(size as u64)
761+
}),
762+
max_row_group_bytes_opt: global_options.global.max_row_group_bytes.map(|size| {
763+
parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(size.get() as u64)
764+
}),
765+
content_defined_chunking: Some(protobuf::ParquetCdcOptions {
766+
enabled: global_options.global.content_defined_chunking.enabled,
767+
min_chunk_size: global_options.global.content_defined_chunking.min_chunk_size as u64,
768+
max_chunk_size: global_options.global.content_defined_chunking.max_chunk_size as u64,
769+
norm_level: global_options.global.content_defined_chunking.norm_level,
770+
}),
771+
}),
772+
column_specific_options: column_specific_options.into_iter().map(|(column_name, options)| {
773+
protobuf::ParquetColumnSpecificOptions {
774+
column_name,
775+
options: Some(protobuf::ParquetColumnOptions {
776+
bloom_filter_enabled_opt: options.bloom_filter_enabled.map(|enabled| {
777+
parquet_column_options::BloomFilterEnabledOpt::BloomFilterEnabled(enabled)
778+
}),
779+
encoding_opt: options.encoding.map(|encoding| {
780+
parquet_column_options::EncodingOpt::Encoding(encoding)
781+
}),
782+
dictionary_enabled_opt: options.dictionary_enabled.map(|enabled| {
783+
parquet_column_options::DictionaryEnabledOpt::DictionaryEnabled(enabled)
784+
}),
785+
compression_opt: options.compression.map(|compression| {
786+
parquet_column_options::CompressionOpt::Compression(compression)
787+
}),
788+
statistics_enabled_opt: options.statistics_enabled.map(|enabled| {
789+
parquet_column_options::StatisticsEnabledOpt::StatisticsEnabled(enabled)
790+
}),
791+
bloom_filter_fpp_opt: options.bloom_filter_fpp.map(|fpp| {
792+
parquet_column_options::BloomFilterFppOpt::BloomFilterFpp(fpp)
793+
}),
794+
bloom_filter_ndv_opt: options.bloom_filter_ndv.map(|ndv| {
795+
parquet_column_options::BloomFilterNdvOpt::BloomFilterNdv(ndv)
796+
}),
797+
})
798+
}
799+
}).collect(),
800+
key_value_metadata: global_options.key_value_metadata
801+
.iter()
802+
.filter_map(|(key, value)| {
803+
value.as_ref().map(|v| (key.clone(), v.clone()))
804+
})
805+
.collect(),
806+
}
807+
}
808+
}

0 commit comments

Comments
 (0)