Skip to content

Commit b616514

Browse files
adriangbclaude
andcommitted
refactor(proto): drop TryFromProto shims for datasource and sink types
The real `TryFrom` impls for `PartitionedFile`, `FileRange`, `FileGroup` (apache#24006) and for `JsonSink` / `CsvSink` / `ParquetSink` / `FileSinkConfig` (apache#23781) now live next to the types, so the `TryFromProto` copies in `datafusion-proto` were pure delegation. `TryFrom<&[PartitionedFile]> for protobuf::FileGroup` goes away here and comes back in the `datafusion-proto-models` commit, which is the one crate that can express it. The `PartitionedFile` tests move to `datafusion-datasource` alongside the logic they cover; the two that duplicated existing coverage there are dropped. Part of apache#24019. Co-Authored-By: Claude Opus 5 <[email protected]>
1 parent 047f531 commit b616514

5 files changed

Lines changed: 54 additions & 250 deletions

File tree

‎Cargo.lock‎

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/datasource/src/proto.rs‎

Lines changed: 50 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,9 @@
1818
//! Protobuf conversions for the file-scan leaf types owned by this crate:
1919
//! [`FileRange`], [`PartitionedFile`] and [`FileGroup`].
2020
//!
21-
//! These are the single copy of that wire logic. `datafusion-proto`'s
22-
//! `TryFromProto` implementations for the same types are thin shims that
23-
//! delegate here, so the format cannot drift between the central serializer and
24-
//! the per-source `try_to_proto` hooks.
21+
//! These are the single copy of that wire logic, used both by the central
22+
//! serializer in `datafusion-proto` and by the per-source `try_to_proto` hooks,
23+
//! so the format cannot drift between them.
2524
//!
2625
//! None of these conversions need a codec or an encode/decode context: every
2726
//! field is plain data or goes through `datafusion-proto-common`. That is why
@@ -204,6 +203,53 @@ mod tests {
204203
Ok(())
205204
}
206205

206+
#[test]
207+
fn partitioned_file_path_roundtrip_percent_encoded() -> Result<()> {
208+
// The wire format carries the *encoded* path, so a location that already
209+
// contains percent escapes must survive without a second round of
210+
// encoding or decoding.
211+
let path_str = "foo/foo%2Fbar/baz%252Fqux";
212+
let pf = PartitionedFile::new_from_meta(ObjectMeta {
213+
location: Path::parse(path_str)?,
214+
last_modified: Utc.timestamp_nanos(1_000),
215+
size: 42,
216+
e_tag: None,
217+
version: None,
218+
});
219+
220+
let encoded = protobuf::PartitionedFile::try_from(&pf)?;
221+
assert_eq!(encoded.path, path_str);
222+
223+
let decoded = PartitionedFile::try_from(&encoded)?;
224+
assert_eq!(decoded.object_meta.location.as_ref(), path_str);
225+
assert_eq!(decoded.object_meta.location, pf.object_meta.location);
226+
Ok(())
227+
}
228+
229+
#[test]
230+
fn partitioned_file_arrow_schema_roundtrip_preserves_metadata() -> Result<()> {
231+
use std::collections::HashMap;
232+
233+
let arrow_schema = Arc::new(Schema::new_with_metadata(
234+
vec![
235+
Field::new("id", DataType::Int64, false),
236+
Field::new("value", DataType::Utf8, true).with_metadata(HashMap::from([
237+
("field_meta".to_string(), "field_value".to_string()),
238+
])),
239+
],
240+
HashMap::from([("schema_meta".to_string(), "schema_value".to_string())]),
241+
));
242+
let pf = PartitionedFile::new("foo/bar.parquet", 10)
243+
.with_arrow_schema(Arc::clone(&arrow_schema));
244+
245+
let encoded = protobuf::PartitionedFile::try_from(&pf)?;
246+
assert!(encoded.arrow_schema.is_some());
247+
248+
let decoded = PartitionedFile::try_from(&encoded)?;
249+
assert_eq!(decoded.arrow_schema.as_deref(), Some(arrow_schema.as_ref()));
250+
Ok(())
251+
}
252+
207253
#[test]
208254
fn partitioned_file_from_proto_rejects_invalid_path() {
209255
let proto = protobuf::PartitionedFile {

‎datafusion/proto/Cargo.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ avro = ["datafusion-datasource-avro"]
5757

5858
[dependencies]
5959
arrow = { workspace = true }
60-
chrono = { workspace = true }
6160
datafusion-catalog = { workspace = true }
6261
datafusion-catalog-listing = { workspace = true }
6362
datafusion-common = { workspace = true }

‎datafusion/proto/src/physical_plan/from_proto.rs‎

Lines changed: 2 additions & 163 deletions
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,10 @@ use arrow::array::RecordBatch;
2323
use arrow::compute::SortOptions;
2424
use arrow::datatypes::{Field, Schema};
2525
use arrow::ipc::reader::StreamReader;
26-
use datafusion_common::{DataFusionError, Result, internal_datafusion_err, not_impl_err};
26+
use datafusion_common::{Result, internal_datafusion_err, not_impl_err};
27+
use datafusion_datasource::TableSchema;
2728
use datafusion_datasource::file::FileSource;
28-
use datafusion_datasource::file_groups::FileGroup;
2929
use datafusion_datasource::file_scan_config::FileScanConfig;
30-
use datafusion_datasource::file_sink_config::FileSinkConfig;
31-
use datafusion_datasource::{FileRange, PartitionedFile, TableSchema};
32-
use datafusion_datasource_csv::file_format::CsvSink;
33-
use datafusion_datasource_json::file_format::JsonSink;
34-
#[cfg(feature = "parquet")]
35-
use datafusion_datasource_parquet::file_format::ParquetSink;
3630
use datafusion_execution::{FunctionRegistry, TaskContext};
3731
use datafusion_expr::WindowFunctionDefinition;
3832
use datafusion_physical_expr::expressions::{LambdaExpr, LambdaVariable};
@@ -475,66 +469,6 @@ pub fn parse_record_batches(buf: &[u8]) -> Result<Vec<RecordBatch>> {
475469
Ok(batches)
476470
}
477471

478-
/// Thin shim over `TryFrom<&protobuf::PartitionedFile>`, which owns the wire logic.
479-
impl TryFromProto<&protobuf::PartitionedFile> for PartitionedFile {
480-
type Error = DataFusionError;
481-
482-
fn try_from_proto(val: &protobuf::PartitionedFile) -> Result<Self, Self::Error> {
483-
PartitionedFile::try_from(val)
484-
}
485-
}
486-
487-
/// Thin shim over `TryFrom<&protobuf::FileRange>`, which owns the wire logic.
488-
impl TryFromProto<&protobuf::FileRange> for FileRange {
489-
type Error = DataFusionError;
490-
491-
fn try_from_proto(value: &protobuf::FileRange) -> Result<Self, Self::Error> {
492-
FileRange::try_from(value)
493-
}
494-
}
495-
496-
/// Thin shim over `TryFrom<&protobuf::FileGroup>`, which owns the wire logic.
497-
impl TryFromProto<&protobuf::FileGroup> for FileGroup {
498-
type Error = DataFusionError;
499-
500-
fn try_from_proto(val: &protobuf::FileGroup) -> Result<Self, Self::Error> {
501-
FileGroup::try_from(val)
502-
}
503-
}
504-
505-
impl TryFromProto<&protobuf::JsonSink> for JsonSink {
506-
type Error = DataFusionError;
507-
508-
fn try_from_proto(value: &protobuf::JsonSink) -> Result<Self, Self::Error> {
509-
Self::try_from(value)
510-
}
511-
}
512-
513-
#[cfg(feature = "parquet")]
514-
impl TryFromProto<&protobuf::ParquetSink> for ParquetSink {
515-
type Error = DataFusionError;
516-
517-
fn try_from_proto(value: &protobuf::ParquetSink) -> Result<Self, Self::Error> {
518-
Self::try_from(value)
519-
}
520-
}
521-
522-
impl TryFromProto<&protobuf::CsvSink> for CsvSink {
523-
type Error = DataFusionError;
524-
525-
fn try_from_proto(value: &protobuf::CsvSink) -> Result<Self, Self::Error> {
526-
Self::try_from(value)
527-
}
528-
}
529-
530-
impl TryFromProto<&protobuf::FileSinkConfig> for FileSinkConfig {
531-
type Error = DataFusionError;
532-
533-
fn try_from_proto(conf: &protobuf::FileSinkConfig) -> Result<Self, Self::Error> {
534-
conf.try_into()
535-
}
536-
}
537-
538472
/// Concrete [`PhysicalExprDecode`] driver that backs
539473
/// [`PhysicalExprDecodeCtx`] inside `parse_physical_expr_with_converter`.
540474
///
@@ -561,98 +495,3 @@ impl datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprD
561495
.proto_to_physical_expr(node, schema, self.ctx)
562496
}
563497
}
564-
565-
#[cfg(test)]
566-
mod tests {
567-
use super::*;
568-
use arrow::datatypes::{DataType, Field, Schema};
569-
use chrono::{TimeZone, Utc};
570-
use datafusion_common::ScalarValue;
571-
use object_store::ObjectMeta;
572-
use object_store::path::Path;
573-
574-
#[test]
575-
fn partitioned_file_path_roundtrip_percent_encoded() {
576-
let path_str = "foo/foo%2Fbar/baz%252Fqux";
577-
let pf = PartitionedFile::new_from_meta(ObjectMeta {
578-
location: Path::parse(path_str).unwrap(),
579-
last_modified: Utc.timestamp_nanos(1_000),
580-
size: 42,
581-
e_tag: None,
582-
version: None,
583-
});
584-
585-
let proto = protobuf::PartitionedFile::try_from_proto(&pf).unwrap();
586-
assert_eq!(proto.path, path_str);
587-
588-
let pf2 = PartitionedFile::try_from_proto(&proto).unwrap();
589-
assert_eq!(pf2.object_meta.location.as_ref(), path_str);
590-
assert_eq!(pf2.object_meta.location, pf.object_meta.location);
591-
assert_eq!(pf2.object_meta.size, pf.object_meta.size);
592-
assert_eq!(pf2.object_meta.last_modified, pf.object_meta.last_modified);
593-
}
594-
595-
#[test]
596-
fn partitioned_file_arrow_schema_roundtrip() {
597-
use std::collections::HashMap;
598-
599-
let arrow_schema = Arc::new(Schema::new_with_metadata(
600-
vec![
601-
Field::new("id", DataType::Int64, false),
602-
Field::new("value", DataType::Utf8, true).with_metadata(HashMap::from([
603-
("field_meta".to_string(), "field_value".to_string()),
604-
])),
605-
],
606-
HashMap::from([("schema_meta".to_string(), "schema_value".to_string())]),
607-
));
608-
let pf = PartitionedFile::new("foo/bar.parquet", 10)
609-
.with_arrow_schema(Arc::clone(&arrow_schema));
610-
611-
let proto = protobuf::PartitionedFile::try_from_proto(&pf).unwrap();
612-
assert!(proto.arrow_schema.is_some());
613-
614-
let decoded = PartitionedFile::try_from_proto(&proto).unwrap();
615-
assert_eq!(
616-
decoded.arrow_schema.as_ref().map(|s| s.as_ref()),
617-
Some(arrow_schema.as_ref())
618-
);
619-
}
620-
621-
#[test]
622-
fn partitioned_file_statistics_roundtrip_with_partition_values() {
623-
use datafusion_common::Statistics;
624-
let file_schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);
625-
let pf = PartitionedFile::new("foo/bar.parquet", 1234)
626-
.with_partition_values(vec![ScalarValue::from("2024-01-01")])
627-
.with_statistics(Arc::new(Statistics::new_unknown(&file_schema)));
628-
629-
// `statistics` covers the full table schema: file columns followed by one
630-
// entry per partition column.
631-
let expected_len = file_schema.fields().len() + pf.partition_values.len();
632-
assert_eq!(
633-
pf.statistics.as_ref().unwrap().column_statistics.len(),
634-
expected_len
635-
);
636-
637-
let proto = protobuf::PartitionedFile::try_from_proto(&pf).unwrap();
638-
let decoded = PartitionedFile::try_from_proto(&proto).unwrap();
639-
640-
assert_eq!(decoded.statistics, pf.statistics);
641-
}
642-
643-
#[test]
644-
fn partitioned_file_from_proto_invalid_path() {
645-
let proto = protobuf::PartitionedFile {
646-
arrow_schema: None,
647-
path: "foo//bar".to_string(),
648-
size: 1,
649-
last_modified_ns: 0,
650-
partition_values: vec![],
651-
range: None,
652-
statistics: None,
653-
};
654-
655-
let err = PartitionedFile::try_from_proto(&proto).unwrap_err();
656-
assert!(err.to_string().contains("Invalid object_store path"));
657-
}
658-
}

‎datafusion/proto/src/physical_plan/to_proto.rs‎

Lines changed: 2 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,8 @@ use std::sync::Arc;
2020
use arrow::array::RecordBatch;
2121
use arrow::datatypes::Schema;
2222
use arrow::ipc::writer::StreamWriter;
23-
use datafusion_common::{
24-
DataFusionError, Result, internal_datafusion_err, internal_err, not_impl_err,
25-
};
23+
use datafusion_common::{Result, internal_datafusion_err, internal_err, not_impl_err};
2624
use datafusion_datasource::file_scan_config::FileScanConfig;
27-
use datafusion_datasource::file_sink_config::FileSinkConfig;
28-
use datafusion_datasource::{FileRange, PartitionedFile};
29-
use datafusion_datasource_csv::file_format::CsvSink;
30-
use datafusion_datasource_json::file_format::JsonSink;
31-
#[cfg(feature = "parquet")]
32-
use datafusion_datasource_parquet::file_format::ParquetSink;
3325
use datafusion_expr::WindowFrame;
3426
use datafusion_physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};
3527
use datafusion_physical_expr::{HigherOrderFunctionExpr, ScalarFunctionExpr};
@@ -43,7 +35,6 @@ use super::{
4335
ConverterPlanEncoder, DefaultPhysicalProtoConverter, PhysicalExtensionCodec,
4436
PhysicalProtoConverterExtension, encode_human_display_alias,
4537
};
46-
use crate::convert::TryFromProto;
4738
use crate::protobuf::{
4839
self, PhysicalSortExprNode, physical_aggregate_expr_node, physical_window_expr_node,
4940
};
@@ -173,7 +164,7 @@ pub fn serialize_physical_window_expr(
173164
codec,
174165
proto_converter,
175166
)?;
176-
let window_frame = protobuf::WindowFrame::try_from_proto(window_frame.as_ref())
167+
let window_frame = protobuf::WindowFrame::try_from(window_frame.as_ref())
177168
.map_err(|e| internal_datafusion_err!("{e}"))?;
178169

179170
Ok(protobuf::PhysicalWindowExprNode {
@@ -366,43 +357,6 @@ pub fn serialize_partitioning(
366357
)
367358
}
368359

369-
/// Thin shim over `TryFrom<&PartitionedFile>`, which owns the wire logic.
370-
impl TryFromProto<&PartitionedFile> for protobuf::PartitionedFile {
371-
type Error = DataFusionError;
372-
373-
fn try_from_proto(pf: &PartitionedFile) -> Result<Self> {
374-
pf.try_into()
375-
}
376-
}
377-
378-
/// Thin shim over `TryFrom<&FileRange>`, which owns the wire logic.
379-
impl TryFromProto<&FileRange> for protobuf::FileRange {
380-
type Error = DataFusionError;
381-
382-
fn try_from_proto(value: &FileRange) -> Result<Self> {
383-
value.try_into()
384-
}
385-
}
386-
387-
/// Thin shim over `TryFrom<&PartitionedFile>`, which owns the wire logic.
388-
///
389-
/// The slice form cannot be a `TryFrom` impl: the orphan rule only accepts a
390-
/// type this crate owns, and `&[PartitionedFile]` is not one (`&FileGroup` is,
391-
/// hence the impl next to the type). Callers inside DataFusion go through
392-
/// `FileGroup`; this stays for downstream users of the published signature.
393-
impl TryFromProto<&[PartitionedFile]> for protobuf::FileGroup {
394-
type Error = DataFusionError;
395-
396-
fn try_from_proto(gr: &[PartitionedFile]) -> Result<Self, Self::Error> {
397-
Ok(protobuf::FileGroup {
398-
files: gr
399-
.iter()
400-
.map(TryInto::try_into)
401-
.collect::<Result<Vec<_>>>()?,
402-
})
403-
}
404-
}
405-
406360
pub fn serialize_file_scan_config(
407361
conf: &FileScanConfig,
408362
codec: &dyn PhysicalExtensionCodec,
@@ -445,36 +399,3 @@ pub fn serialize_record_batches(batches: &[RecordBatch]) -> Result<Vec<u8>> {
445399
writer.finish()?;
446400
Ok(buf)
447401
}
448-
449-
impl TryFromProto<&JsonSink> for protobuf::JsonSink {
450-
type Error = DataFusionError;
451-
452-
fn try_from_proto(value: &JsonSink) -> Result<Self, Self::Error> {
453-
Self::try_from(value)
454-
}
455-
}
456-
457-
impl TryFromProto<&CsvSink> for protobuf::CsvSink {
458-
type Error = DataFusionError;
459-
460-
fn try_from_proto(value: &CsvSink) -> Result<Self, Self::Error> {
461-
Self::try_from(value)
462-
}
463-
}
464-
465-
#[cfg(feature = "parquet")]
466-
impl TryFromProto<&ParquetSink> for protobuf::ParquetSink {
467-
type Error = DataFusionError;
468-
469-
fn try_from_proto(value: &ParquetSink) -> Result<Self, Self::Error> {
470-
Self::try_from(value)
471-
}
472-
}
473-
474-
impl TryFromProto<&FileSinkConfig> for protobuf::FileSinkConfig {
475-
type Error = DataFusionError;
476-
477-
fn try_from_proto(conf: &FileSinkConfig) -> Result<Self, Self::Error> {
478-
conf.try_into()
479-
}
480-
}

0 commit comments

Comments
 (0)