Skip to content

Commit 7940058

Browse files
adriangbclaude
andcommitted
refactor(proto): restore format-factory conversions as From in the format crates
`CsvFormatFactory`, `JsonFormatFactory` and `ParquetFormatFactory` are owned by `datafusion-datasource-{csv,json,parquet}`, so their options-encoding conversions become plain `From` impls next to the types, behind each crate's existing `proto` feature. This is the last `FromProto` / `TryFromProto` impl in the tree: `convert.rs` and `convert_required_proto!` now have no implementors, clearing the way for the final cleanup in apache#24019. Part of apache#24019. Co-Authored-By: Claude Opus 5 <[email protected]>
1 parent a31d4ad commit 7940058

4 files changed

Lines changed: 200 additions & 185 deletions

File tree

‎datafusion/datasource-csv/src/file_format.rs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -926,6 +926,54 @@ impl CsvSink {
926926
}
927927
}
928928

929+
/// Encode a [`CsvFormatFactory`]'s options as their protobuf form.
930+
///
931+
/// The reverse direction is `From<&protobuf::CsvOptions> for CsvOptions` in
932+
/// `datafusion-proto-models`: `CsvOptions` is a `datafusion-common` type, so
933+
/// that half cannot live here.
934+
#[cfg(feature = "proto")]
935+
impl From<&CsvFormatFactory> for datafusion_proto_models::protobuf::CsvOptions {
936+
fn from(factory: &CsvFormatFactory) -> Self {
937+
if let Some(options) = &factory.options {
938+
datafusion_proto_models::protobuf::CsvOptions {
939+
has_header: options.has_header.map_or(vec![], |v| vec![v as u8]),
940+
delimiter: vec![options.delimiter],
941+
quote: vec![options.quote],
942+
terminator: options.terminator.map_or(vec![], |v| vec![v]),
943+
escape: options.escape.map_or(vec![], |v| vec![v]),
944+
double_quote: options.double_quote.map_or(vec![], |v| vec![v as u8]),
945+
compression: options.compression as i32,
946+
schema_infer_max_rec: options.schema_infer_max_rec.map(|v| v as u64),
947+
date_format: options.date_format.clone().unwrap_or_default(),
948+
datetime_format: options.datetime_format.clone().unwrap_or_default(),
949+
timestamp_format: options.timestamp_format.clone().unwrap_or_default(),
950+
timestamp_tz_format: options
951+
.timestamp_tz_format
952+
.clone()
953+
.unwrap_or_default(),
954+
time_format: options.time_format.clone().unwrap_or_default(),
955+
null_value: options.null_value.clone().unwrap_or_default(),
956+
null_regex: options.null_regex.clone().unwrap_or_default(),
957+
comment: options.comment.map_or(vec![], |v| vec![v]),
958+
newlines_in_values: options
959+
.newlines_in_values
960+
.map_or(vec![], |v| vec![v as u8]),
961+
truncated_rows: options.truncated_rows.map_or(vec![], |v| vec![v as u8]),
962+
compression_level: options.compression_level,
963+
quote_style: options.quote_style as i32,
964+
ignore_leading_whitespace: options
965+
.ignore_leading_whitespace
966+
.map_or(vec![], |v| vec![v as u8]),
967+
ignore_trailing_whitespace: options
968+
.ignore_trailing_whitespace
969+
.map_or(vec![], |v| vec![v as u8]),
970+
}
971+
} else {
972+
datafusion_proto_models::protobuf::CsvOptions::default()
973+
}
974+
}
975+
}
976+
929977
#[cfg(test)]
930978
mod tests {
931979
use super::build_schema_helper;

‎datafusion/datasource-json/src/file_format.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -615,3 +615,24 @@ impl Decoder for JsonDecoder {
615615
false
616616
}
617617
}
618+
619+
/// Encode a [`JsonFormatFactory`]'s options as their protobuf form.
620+
///
621+
/// The reverse direction is `From<&protobuf::JsonOptions> for JsonOptions` in
622+
/// `datafusion-proto-models`: `JsonOptions` is a `datafusion-common` type, so
623+
/// that half cannot live here.
624+
#[cfg(feature = "proto")]
625+
impl From<&JsonFormatFactory> for datafusion_proto_models::protobuf::JsonOptions {
626+
fn from(factory: &JsonFormatFactory) -> Self {
627+
if let Some(options) = &factory.options {
628+
datafusion_proto_models::protobuf::JsonOptions {
629+
compression: options.compression as i32,
630+
schema_infer_max_rec: options.schema_infer_max_rec.map(|v| v as u64),
631+
compression_level: options.compression_level,
632+
newline_delimited: Some(options.newline_delimited),
633+
}
634+
} else {
635+
datafusion_proto_models::protobuf::JsonOptions::default()
636+
}
637+
}
638+
}

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -688,3 +688,129 @@ pub fn statistics_from_parquet_meta_calc(
688688
) -> Result<Statistics> {
689689
DFParquetMetadata::statistics_from_parquet_metadata(metadata, &table_schema)
690690
}
691+
692+
#[cfg(feature = "proto")]
693+
use datafusion_proto_models::protobuf::{self, parquet_column_options, parquet_options};
694+
695+
/// Encode a [`ParquetFormatFactory`]'s options as their protobuf form.
696+
///
697+
/// The reverse direction is `TryFrom<&protobuf::TableParquetOptions> for
698+
/// TableParquetOptions` in `datafusion-proto-models`: `TableParquetOptions` is
699+
/// a `datafusion-common` type, so that half cannot live here.
700+
#[cfg(feature = "proto")]
701+
impl From<&ParquetFormatFactory> for protobuf::TableParquetOptions {
702+
fn from(factory: &ParquetFormatFactory) -> Self {
703+
let global_options = if let Some(ref options) = factory.options {
704+
options.clone()
705+
} else {
706+
return protobuf::TableParquetOptions::default();
707+
};
708+
709+
let column_specific_options = global_options.column_specific_options;
710+
protobuf::TableParquetOptions {
711+
global: Some(protobuf::ParquetOptions {
712+
enable_page_index: global_options.global.enable_page_index,
713+
pruning: global_options.global.pruning,
714+
skip_metadata: global_options.global.skip_metadata,
715+
metadata_size_hint_opt: global_options.global.metadata_size_hint.map(|size| {
716+
parquet_options::MetadataSizeHintOpt::MetadataSizeHint(size as u64)
717+
}),
718+
pushdown_filters: global_options.global.pushdown_filters,
719+
reorder_filters: global_options.global.reorder_filters,
720+
force_filter_selections: global_options.global.force_filter_selections,
721+
data_pagesize_limit: global_options.global.data_pagesize_limit as u64,
722+
write_batch_size: global_options.global.write_batch_size as u64,
723+
writer_version: global_options.global.writer_version.to_string(),
724+
compression_opt: global_options.global.compression.map(|compression| {
725+
parquet_options::CompressionOpt::Compression(compression)
726+
}),
727+
dictionary_enabled_opt: global_options.global.dictionary_enabled.map(|enabled| {
728+
parquet_options::DictionaryEnabledOpt::DictionaryEnabled(enabled)
729+
}),
730+
dictionary_page_size_limit: global_options.global.dictionary_page_size_limit as u64,
731+
statistics_enabled_opt: global_options.global.statistics_enabled.map(|enabled| {
732+
parquet_options::StatisticsEnabledOpt::StatisticsEnabled(enabled)
733+
}),
734+
max_row_group_size: global_options.global.max_row_group_size as u64,
735+
max_in_list_size: global_options.global.max_in_list_size as u64,
736+
created_by: global_options.global.created_by.clone(),
737+
column_index_truncate_length_opt: global_options.global.column_index_truncate_length.map(|length| {
738+
parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(length as u64)
739+
}),
740+
statistics_truncate_length_opt: global_options.global.statistics_truncate_length.map(|length| {
741+
parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(length as u64)
742+
}),
743+
data_page_row_count_limit: global_options.global.data_page_row_count_limit as u64,
744+
encoding_opt: global_options.global.encoding.map(|encoding| {
745+
parquet_options::EncodingOpt::Encoding(encoding)
746+
}),
747+
bloom_filter_on_read: global_options.global.bloom_filter_on_read,
748+
bloom_filter_on_write: global_options.global.bloom_filter_on_write,
749+
bloom_filter_fpp_opt: global_options.global.bloom_filter_fpp.map(|fpp| {
750+
parquet_options::BloomFilterFppOpt::BloomFilterFpp(fpp)
751+
}),
752+
bloom_filter_ndv_opt: global_options.global.bloom_filter_ndv.map(|ndv| {
753+
parquet_options::BloomFilterNdvOpt::BloomFilterNdv(ndv)
754+
}),
755+
allow_single_file_parallelism: global_options.global.allow_single_file_parallelism,
756+
maximum_parallel_row_group_writers: global_options.global.maximum_parallel_row_group_writers as u64,
757+
maximum_buffered_record_batches_per_stream: global_options.global.maximum_buffered_record_batches_per_stream as u64,
758+
schema_force_view_types: global_options.global.schema_force_view_types,
759+
binary_as_string: global_options.global.binary_as_string,
760+
skip_arrow_metadata: global_options.global.skip_arrow_metadata,
761+
coerce_int96_opt: global_options.global.coerce_int96.map(|compression| {
762+
parquet_options::CoerceInt96Opt::CoerceInt96(compression)
763+
}),
764+
coerce_int96_tz_opt: global_options.global.coerce_int96_tz.map(|tz| {
765+
parquet_options::CoerceInt96TzOpt::CoerceInt96Tz(tz)
766+
}),
767+
max_predicate_cache_size_opt: global_options.global.max_predicate_cache_size.map(|size| {
768+
parquet_options::MaxPredicateCacheSizeOpt::MaxPredicateCacheSize(size as u64)
769+
}),
770+
max_row_group_bytes_opt: global_options.global.max_row_group_bytes.map(|size| {
771+
parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(size.get() as u64)
772+
}),
773+
content_defined_chunking: Some(protobuf::ParquetCdcOptions {
774+
enabled: global_options.global.content_defined_chunking.enabled,
775+
min_chunk_size: global_options.global.content_defined_chunking.min_chunk_size as u64,
776+
max_chunk_size: global_options.global.content_defined_chunking.max_chunk_size as u64,
777+
norm_level: global_options.global.content_defined_chunking.norm_level,
778+
}),
779+
}),
780+
column_specific_options: column_specific_options.into_iter().map(|(column_name, options)| {
781+
protobuf::ParquetColumnSpecificOptions {
782+
column_name,
783+
options: Some(protobuf::ParquetColumnOptions {
784+
bloom_filter_enabled_opt: options.bloom_filter_enabled.map(|enabled| {
785+
parquet_column_options::BloomFilterEnabledOpt::BloomFilterEnabled(enabled)
786+
}),
787+
encoding_opt: options.encoding.map(|encoding| {
788+
parquet_column_options::EncodingOpt::Encoding(encoding)
789+
}),
790+
dictionary_enabled_opt: options.dictionary_enabled.map(|enabled| {
791+
parquet_column_options::DictionaryEnabledOpt::DictionaryEnabled(enabled)
792+
}),
793+
compression_opt: options.compression.map(|compression| {
794+
parquet_column_options::CompressionOpt::Compression(compression)
795+
}),
796+
statistics_enabled_opt: options.statistics_enabled.map(|enabled| {
797+
parquet_column_options::StatisticsEnabledOpt::StatisticsEnabled(enabled)
798+
}),
799+
bloom_filter_fpp_opt: options.bloom_filter_fpp.map(|fpp| {
800+
parquet_column_options::BloomFilterFppOpt::BloomFilterFpp(fpp)
801+
}),
802+
bloom_filter_ndv_opt: options.bloom_filter_ndv.map(|ndv| {
803+
parquet_column_options::BloomFilterNdvOpt::BloomFilterNdv(ndv)
804+
}),
805+
})
806+
}
807+
}).collect(),
808+
key_value_metadata: global_options.key_value_metadata
809+
.iter()
810+
.filter_map(|(key, value)| {
811+
value.as_ref().map(|v| (key.clone(), v.clone()))
812+
})
813+
.collect(),
814+
}
815+
}
816+
}

0 commit comments

Comments
 (0)