Skip to content

Commit 2f98a97

Browse files
adriangbclaude
andcommitted
refactor(proto): drop TryFromProto shims for datasource and sink types
The real `TryFrom` impls for `PartitionedFile`, `FileRange`, `FileGroup` (apache#24006) and for `JsonSink` / `CsvSink` / `ParquetSink` / `FileSinkConfig` (apache#23781) now live next to the types, so the `TryFromProto` copies in `datafusion-proto` were pure delegation. `TryFrom<&[PartitionedFile]> for protobuf::FileGroup` goes away here and comes back in the `datafusion-proto-models` commit, which is the one crate that can express it. The `PartitionedFile` tests move to `datafusion-datasource` alongside the logic they cover; the two that duplicated existing coverage there are dropped. Part of apache#24019. Co-Authored-By: Claude Opus 5 <[email protected]>
1 parent fc846dd commit 2f98a97

5 files changed

Lines changed: 54 additions & 250 deletions

File tree

‎Cargo.lock‎

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/datasource/src/proto.rs‎

Lines changed: 50 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,9 @@
1818
//! Protobuf conversions for the file-scan leaf types owned by this crate:
1919
//! [`FileRange`], [`PartitionedFile`] and [`FileGroup`].
2020
//!
21-
//! These are the single copy of that wire logic. `datafusion-proto`'s
22-
//! `TryFromProto` implementations for the same types are thin shims that
23-
//! delegate here, so the format cannot drift between the central serializer and
24-
//! the per-source `try_to_proto` hooks.
21+
//! These are the single copy of that wire logic, used both by the central
22+
//! serializer in `datafusion-proto` and by the per-source `try_to_proto` hooks,
23+
//! so the format cannot drift between them.
2524
//!
2625
//! None of these conversions need a codec or an encode/decode context: every
2726
//! field is plain data or goes through `datafusion-proto-common`. That is why
@@ -204,6 +203,53 @@ mod tests {
204203
Ok(())
205204
}
206205

206+
#[test]
207+
fn partitioned_file_path_roundtrip_percent_encoded() -> Result<()> {
208+
// The wire format carries the *encoded* path, so a location that already
209+
// contains percent escapes must survive without a second round of
210+
// encoding or decoding.
211+
let path_str = "foo/foo%2Fbar/baz%252Fqux";
212+
let pf = PartitionedFile::new_from_meta(ObjectMeta {
213+
location: Path::parse(path_str)?,
214+
last_modified: Utc.timestamp_nanos(1_000),
215+
size: 42,
216+
e_tag: None,
217+
version: None,
218+
});
219+
220+
let encoded = protobuf::PartitionedFile::try_from(&pf)?;
221+
assert_eq!(encoded.path, path_str);
222+
223+
let decoded = PartitionedFile::try_from(&encoded)?;
224+
assert_eq!(decoded.object_meta.location.as_ref(), path_str);
225+
assert_eq!(decoded.object_meta.location, pf.object_meta.location);
226+
Ok(())
227+
}
228+
229+
#[test]
230+
fn partitioned_file_arrow_schema_roundtrip_preserves_metadata() -> Result<()> {
231+
use std::collections::HashMap;
232+
233+
let arrow_schema = Arc::new(Schema::new_with_metadata(
234+
vec![
235+
Field::new("id", DataType::Int64, false),
236+
Field::new("value", DataType::Utf8, true).with_metadata(HashMap::from([
237+
("field_meta".to_string(), "field_value".to_string()),
238+
])),
239+
],
240+
HashMap::from([("schema_meta".to_string(), "schema_value".to_string())]),
241+
));
242+
let pf = PartitionedFile::new("foo/bar.parquet", 10)
243+
.with_arrow_schema(Arc::clone(&arrow_schema));
244+
245+
let encoded = protobuf::PartitionedFile::try_from(&pf)?;
246+
assert!(encoded.arrow_schema.is_some());
247+
248+
let decoded = PartitionedFile::try_from(&encoded)?;
249+
assert_eq!(decoded.arrow_schema.as_deref(), Some(arrow_schema.as_ref()));
250+
Ok(())
251+
}
252+
207253
#[test]
208254
fn partitioned_file_from_proto_rejects_invalid_path() {
209255
let proto = protobuf::PartitionedFile {

‎datafusion/proto/Cargo.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ avro = ["datafusion-datasource-avro"]
5656

5757
[dependencies]
5858
arrow = { workspace = true }
59-
chrono = { workspace = true }
6059
datafusion-catalog = { workspace = true }
6160
datafusion-catalog-listing = { workspace = true }
6261
datafusion-common = { workspace = true }

‎datafusion/proto/src/physical_plan/from_proto.rs‎

Lines changed: 2 additions & 163 deletions
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,10 @@ use arrow::array::RecordBatch;
2323
use arrow::compute::SortOptions;
2424
use arrow::datatypes::{Field, Schema};
2525
use arrow::ipc::reader::StreamReader;
26-
use datafusion_common::{DataFusionError, Result, internal_datafusion_err, not_impl_err};
26+
use datafusion_common::{Result, internal_datafusion_err, not_impl_err};
27+
use datafusion_datasource::TableSchema;
2728
use datafusion_datasource::file::FileSource;
28-
use datafusion_datasource::file_groups::FileGroup;
2929
use datafusion_datasource::file_scan_config::FileScanConfig;
30-
use datafusion_datasource::file_sink_config::FileSinkConfig;
31-
use datafusion_datasource::{FileRange, PartitionedFile, TableSchema};
32-
use datafusion_datasource_csv::file_format::CsvSink;
33-
use datafusion_datasource_json::file_format::JsonSink;
34-
#[cfg(feature = "parquet")]
35-
use datafusion_datasource_parquet::file_format::ParquetSink;
3630
use datafusion_execution::{FunctionRegistry, TaskContext};
3731
use datafusion_expr::WindowFunctionDefinition;
3832
use datafusion_physical_expr::expressions::{LambdaExpr, LambdaVariable};
@@ -467,66 +461,6 @@ pub fn parse_record_batches(buf: &[u8]) -> Result<Vec<RecordBatch>> {
467461
Ok(batches)
468462
}
469463

470-
/// Thin shim over `TryFrom<&protobuf::PartitionedFile>`, which owns the wire logic.
471-
impl TryFromProto<&protobuf::PartitionedFile> for PartitionedFile {
472-
type Error = DataFusionError;
473-
474-
fn try_from_proto(val: &protobuf::PartitionedFile) -> Result<Self, Self::Error> {
475-
PartitionedFile::try_from(val)
476-
}
477-
}
478-
479-
/// Thin shim over `TryFrom<&protobuf::FileRange>`, which owns the wire logic.
480-
impl TryFromProto<&protobuf::FileRange> for FileRange {
481-
type Error = DataFusionError;
482-
483-
fn try_from_proto(value: &protobuf::FileRange) -> Result<Self, Self::Error> {
484-
FileRange::try_from(value)
485-
}
486-
}
487-
488-
/// Thin shim over `TryFrom<&protobuf::FileGroup>`, which owns the wire logic.
489-
impl TryFromProto<&protobuf::FileGroup> for FileGroup {
490-
type Error = DataFusionError;
491-
492-
fn try_from_proto(val: &protobuf::FileGroup) -> Result<Self, Self::Error> {
493-
FileGroup::try_from(val)
494-
}
495-
}
496-
497-
impl TryFromProto<&protobuf::JsonSink> for JsonSink {
498-
type Error = DataFusionError;
499-
500-
fn try_from_proto(value: &protobuf::JsonSink) -> Result<Self, Self::Error> {
501-
Self::try_from(value)
502-
}
503-
}
504-
505-
#[cfg(feature = "parquet")]
506-
impl TryFromProto<&protobuf::ParquetSink> for ParquetSink {
507-
type Error = DataFusionError;
508-
509-
fn try_from_proto(value: &protobuf::ParquetSink) -> Result<Self, Self::Error> {
510-
Self::try_from(value)
511-
}
512-
}
513-
514-
impl TryFromProto<&protobuf::CsvSink> for CsvSink {
515-
type Error = DataFusionError;
516-
517-
fn try_from_proto(value: &protobuf::CsvSink) -> Result<Self, Self::Error> {
518-
Self::try_from(value)
519-
}
520-
}
521-
522-
impl TryFromProto<&protobuf::FileSinkConfig> for FileSinkConfig {
523-
type Error = DataFusionError;
524-
525-
fn try_from_proto(conf: &protobuf::FileSinkConfig) -> Result<Self, Self::Error> {
526-
conf.try_into()
527-
}
528-
}
529-
530464
/// Concrete [`PhysicalExprDecode`] driver that backs
531465
/// [`PhysicalExprDecodeCtx`] inside `parse_physical_expr_with_converter`.
532466
///
@@ -553,98 +487,3 @@ impl datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprD
553487
.proto_to_physical_expr(node, schema, self.ctx)
554488
}
555489
}
556-
557-
#[cfg(test)]
558-
mod tests {
559-
use super::*;
560-
use arrow::datatypes::{DataType, Field, Schema};
561-
use chrono::{TimeZone, Utc};
562-
use datafusion_common::ScalarValue;
563-
use object_store::ObjectMeta;
564-
use object_store::path::Path;
565-
566-
#[test]
567-
fn partitioned_file_path_roundtrip_percent_encoded() {
568-
let path_str = "foo/foo%2Fbar/baz%252Fqux";
569-
let pf = PartitionedFile::new_from_meta(ObjectMeta {
570-
location: Path::parse(path_str).unwrap(),
571-
last_modified: Utc.timestamp_nanos(1_000),
572-
size: 42,
573-
e_tag: None,
574-
version: None,
575-
});
576-
577-
let proto = protobuf::PartitionedFile::try_from_proto(&pf).unwrap();
578-
assert_eq!(proto.path, path_str);
579-
580-
let pf2 = PartitionedFile::try_from_proto(&proto).unwrap();
581-
assert_eq!(pf2.object_meta.location.as_ref(), path_str);
582-
assert_eq!(pf2.object_meta.location, pf.object_meta.location);
583-
assert_eq!(pf2.object_meta.size, pf.object_meta.size);
584-
assert_eq!(pf2.object_meta.last_modified, pf.object_meta.last_modified);
585-
}
586-
587-
#[test]
588-
fn partitioned_file_arrow_schema_roundtrip() {
589-
use std::collections::HashMap;
590-
591-
let arrow_schema = Arc::new(Schema::new_with_metadata(
592-
vec![
593-
Field::new("id", DataType::Int64, false),
594-
Field::new("value", DataType::Utf8, true).with_metadata(HashMap::from([
595-
("field_meta".to_string(), "field_value".to_string()),
596-
])),
597-
],
598-
HashMap::from([("schema_meta".to_string(), "schema_value".to_string())]),
599-
));
600-
let pf = PartitionedFile::new("foo/bar.parquet", 10)
601-
.with_arrow_schema(Arc::clone(&arrow_schema));
602-
603-
let proto = protobuf::PartitionedFile::try_from_proto(&pf).unwrap();
604-
assert!(proto.arrow_schema.is_some());
605-
606-
let decoded = PartitionedFile::try_from_proto(&proto).unwrap();
607-
assert_eq!(
608-
decoded.arrow_schema.as_ref().map(|s| s.as_ref()),
609-
Some(arrow_schema.as_ref())
610-
);
611-
}
612-
613-
#[test]
614-
fn partitioned_file_statistics_roundtrip_with_partition_values() {
615-
use datafusion_common::Statistics;
616-
let file_schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);
617-
let pf = PartitionedFile::new("foo/bar.parquet", 1234)
618-
.with_partition_values(vec![ScalarValue::from("2024-01-01")])
619-
.with_statistics(Arc::new(Statistics::new_unknown(&file_schema)));
620-
621-
// `statistics` covers the full table schema: file columns followed by one
622-
// entry per partition column.
623-
let expected_len = file_schema.fields().len() + pf.partition_values.len();
624-
assert_eq!(
625-
pf.statistics.as_ref().unwrap().column_statistics.len(),
626-
expected_len
627-
);
628-
629-
let proto = protobuf::PartitionedFile::try_from_proto(&pf).unwrap();
630-
let decoded = PartitionedFile::try_from_proto(&proto).unwrap();
631-
632-
assert_eq!(decoded.statistics, pf.statistics);
633-
}
634-
635-
#[test]
636-
fn partitioned_file_from_proto_invalid_path() {
637-
let proto = protobuf::PartitionedFile {
638-
arrow_schema: None,
639-
path: "foo//bar".to_string(),
640-
size: 1,
641-
last_modified_ns: 0,
642-
partition_values: vec![],
643-
range: None,
644-
statistics: None,
645-
};
646-
647-
let err = PartitionedFile::try_from_proto(&proto).unwrap_err();
648-
assert!(err.to_string().contains("Invalid object_store path"));
649-
}
650-
}

‎datafusion/proto/src/physical_plan/to_proto.rs‎

Lines changed: 2 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,8 @@ use std::sync::Arc;
2020
use arrow::array::RecordBatch;
2121
use arrow::datatypes::Schema;
2222
use arrow::ipc::writer::StreamWriter;
23-
use datafusion_common::{
24-
DataFusionError, Result, internal_datafusion_err, internal_err, not_impl_err,
25-
};
23+
use datafusion_common::{Result, internal_datafusion_err, internal_err, not_impl_err};
2624
use datafusion_datasource::file_scan_config::FileScanConfig;
27-
use datafusion_datasource::file_sink_config::FileSinkConfig;
28-
use datafusion_datasource::{FileRange, PartitionedFile};
29-
use datafusion_datasource_csv::file_format::CsvSink;
30-
use datafusion_datasource_json::file_format::JsonSink;
31-
#[cfg(feature = "parquet")]
32-
use datafusion_datasource_parquet::file_format::ParquetSink;
3325
use datafusion_expr::WindowFrame;
3426
use datafusion_physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};
3527
use datafusion_physical_expr::{HigherOrderFunctionExpr, ScalarFunctionExpr};
@@ -43,7 +35,6 @@ use super::{
4335
ConverterPlanEncoder, DefaultPhysicalProtoConverter, PhysicalExtensionCodec,
4436
PhysicalProtoConverterExtension, encode_human_display_alias,
4537
};
46-
use crate::convert::TryFromProto;
4738
use crate::protobuf::{
4839
self, PhysicalSortExprNode, physical_aggregate_expr_node, physical_window_expr_node,
4940
};
@@ -172,7 +163,7 @@ pub fn serialize_physical_window_expr(
172163
codec,
173164
proto_converter,
174165
)?;
175-
let window_frame = protobuf::WindowFrame::try_from_proto(window_frame.as_ref())
166+
let window_frame = protobuf::WindowFrame::try_from(window_frame.as_ref())
176167
.map_err(|e| internal_datafusion_err!("{e}"))?;
177168

178169
Ok(protobuf::PhysicalWindowExprNode {
@@ -365,43 +356,6 @@ pub fn serialize_partitioning(
365356
)
366357
}
367358

368-
/// Thin shim over `TryFrom<&PartitionedFile>`, which owns the wire logic.
369-
impl TryFromProto<&PartitionedFile> for protobuf::PartitionedFile {
370-
type Error = DataFusionError;
371-
372-
fn try_from_proto(pf: &PartitionedFile) -> Result<Self> {
373-
pf.try_into()
374-
}
375-
}
376-
377-
/// Thin shim over `TryFrom<&FileRange>`, which owns the wire logic.
378-
impl TryFromProto<&FileRange> for protobuf::FileRange {
379-
type Error = DataFusionError;
380-
381-
fn try_from_proto(value: &FileRange) -> Result<Self> {
382-
value.try_into()
383-
}
384-
}
385-
386-
/// Thin shim over `TryFrom<&PartitionedFile>`, which owns the wire logic.
387-
///
388-
/// The slice form cannot be a `TryFrom` impl: the orphan rule only accepts a
389-
/// type this crate owns, and `&[PartitionedFile]` is not one (`&FileGroup` is,
390-
/// hence the impl next to the type). Callers inside DataFusion go through
391-
/// `FileGroup`; this stays for downstream users of the published signature.
392-
impl TryFromProto<&[PartitionedFile]> for protobuf::FileGroup {
393-
type Error = DataFusionError;
394-
395-
fn try_from_proto(gr: &[PartitionedFile]) -> Result<Self, Self::Error> {
396-
Ok(protobuf::FileGroup {
397-
files: gr
398-
.iter()
399-
.map(TryInto::try_into)
400-
.collect::<Result<Vec<_>>>()?,
401-
})
402-
}
403-
}
404-
405359
pub fn serialize_file_scan_config(
406360
conf: &FileScanConfig,
407361
codec: &dyn PhysicalExtensionCodec,
@@ -440,36 +394,3 @@ pub fn serialize_record_batches(batches: &[RecordBatch]) -> Result<Vec<u8>> {
440394
writer.finish()?;
441395
Ok(buf)
442396
}
443-
444-
impl TryFromProto<&JsonSink> for protobuf::JsonSink {
445-
type Error = DataFusionError;
446-
447-
fn try_from_proto(value: &JsonSink) -> Result<Self, Self::Error> {
448-
Self::try_from(value)
449-
}
450-
}
451-
452-
impl TryFromProto<&CsvSink> for protobuf::CsvSink {
453-
type Error = DataFusionError;
454-
455-
fn try_from_proto(value: &CsvSink) -> Result<Self, Self::Error> {
456-
Self::try_from(value)
457-
}
458-
}
459-
460-
#[cfg(feature = "parquet")]
461-
impl TryFromProto<&ParquetSink> for protobuf::ParquetSink {
462-
type Error = DataFusionError;
463-
464-
fn try_from_proto(value: &ParquetSink) -> Result<Self, Self::Error> {
465-
Self::try_from(value)
466-
}
467-
}
468-
469-
impl TryFromProto<&FileSinkConfig> for protobuf::FileSinkConfig {
470-
type Error = DataFusionError;
471-
472-
fn try_from_proto(conf: &FileSinkConfig) -> Result<Self, Self::Error> {
473-
conf.try_into()
474-
}
475-
}

0 commit comments

Comments
 (0)