Skip to content

Commit a684b04

Browse files
adriangbclaude
andcommitted
refactor(proto): drop TryFromProto shims for datasource and sink types
The real `TryFrom` impls for `PartitionedFile`, `FileRange`, `FileGroup` (apache#24006) and for `JsonSink` / `CsvSink` / `ParquetSink` / `FileSinkConfig` (apache#23781) now live next to the types, so the `TryFromProto` copies in `datafusion-proto` were pure delegation. `TryFrom<&[PartitionedFile]> for protobuf::FileGroup` has no `TryFrom` equivalent — a slice is not a local type in any crate that could host the impl — so it becomes `datafusion_datasource::proto::partitioned_files_to_proto`, re-exported from `datafusion_proto::physical_plan::to_proto`. The `PartitionedFile` tests move to `datafusion-datasource` alongside the logic they cover; the two that duplicated existing coverage there are dropped. Part of apache#24019. Co-Authored-By: Claude Opus 5 <[email protected]>
1 parent fc846dd commit a684b04

6 files changed

Lines changed: 75 additions & 248 deletions

File tree

‎Cargo.lock‎

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/datasource/src/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ pub mod projection;
4444
/// Protobuf conversions for [`FileRange`], [`PartitionedFile`] and
4545
/// [`FileGroup`](crate::file_groups::FileGroup), gated on the `proto` feature.
4646
#[cfg(feature = "proto")]
47-
mod proto;
47+
pub mod proto;
4848
pub mod schema_adapter;
4949
pub mod sink;
5050
pub mod source;

‎datafusion/datasource/src/proto.rs‎

Lines changed: 67 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,9 @@
1818
//! Protobuf conversions for the file-scan leaf types owned by this crate:
1919
//! [`FileRange`], [`PartitionedFile`] and [`FileGroup`].
2020
//!
21-
//! These are the single copy of that wire logic. `datafusion-proto`'s
22-
//! `TryFromProto` implementations for the same types are thin shims that
23-
//! delegate here, so the format cannot drift between the central serializer and
24-
//! the per-source `try_to_proto` hooks.
21+
//! These are the single copy of that wire logic, used both by the central
22+
//! serializer in `datafusion-proto` and by the per-source `try_to_proto` hooks,
23+
//! so the format cannot drift between them.
2524
//!
2625
//! None of these conversions need a codec or an encode/decode context: every
2726
//! field is plain data or goes through `datafusion-proto-common`. That is why
@@ -160,6 +159,23 @@ impl TryFrom<&protobuf::FileGroup> for FileGroup {
160159
}
161160
}
162161

162+
/// Encode a slice of [`PartitionedFile`]s as a [`protobuf::FileGroup`].
163+
///
164+
/// This is the one conversion in this module that cannot be a [`TryFrom`] impl:
165+
/// the orphan rule needs one side to be a type some crate owns, and
166+
/// `&[PartitionedFile]` is not one — `&FileGroup` is, hence the impl above.
167+
/// Prefer that impl when you already hold a [`FileGroup`].
168+
pub fn partitioned_files_to_proto(
169+
files: &[PartitionedFile],
170+
) -> Result<protobuf::FileGroup> {
171+
Ok(protobuf::FileGroup {
172+
files: files
173+
.iter()
174+
.map(TryInto::try_into)
175+
.collect::<Result<Vec<_>>>()?,
176+
})
177+
}
178+
163179
#[cfg(test)]
164180
mod tests {
165181
use arrow::datatypes::{DataType, Field, Schema};
@@ -204,6 +220,53 @@ mod tests {
204220
Ok(())
205221
}
206222

223+
#[test]
224+
fn partitioned_file_path_roundtrip_percent_encoded() -> Result<()> {
225+
// The wire format carries the *encoded* path, so a location that already
226+
// contains percent escapes must survive without a second round of
227+
// encoding or decoding.
228+
let path_str = "foo/foo%2Fbar/baz%252Fqux";
229+
let pf = PartitionedFile::new_from_meta(ObjectMeta {
230+
location: Path::parse(path_str)?,
231+
last_modified: Utc.timestamp_nanos(1_000),
232+
size: 42,
233+
e_tag: None,
234+
version: None,
235+
});
236+
237+
let encoded = protobuf::PartitionedFile::try_from(&pf)?;
238+
assert_eq!(encoded.path, path_str);
239+
240+
let decoded = PartitionedFile::try_from(&encoded)?;
241+
assert_eq!(decoded.object_meta.location.as_ref(), path_str);
242+
assert_eq!(decoded.object_meta.location, pf.object_meta.location);
243+
Ok(())
244+
}
245+
246+
#[test]
247+
fn partitioned_file_arrow_schema_roundtrip_preserves_metadata() -> Result<()> {
248+
use std::collections::HashMap;
249+
250+
let arrow_schema = Arc::new(Schema::new_with_metadata(
251+
vec![
252+
Field::new("id", DataType::Int64, false),
253+
Field::new("value", DataType::Utf8, true).with_metadata(HashMap::from([
254+
("field_meta".to_string(), "field_value".to_string()),
255+
])),
256+
],
257+
HashMap::from([("schema_meta".to_string(), "schema_value".to_string())]),
258+
));
259+
let pf = PartitionedFile::new("foo/bar.parquet", 10)
260+
.with_arrow_schema(Arc::clone(&arrow_schema));
261+
262+
let encoded = protobuf::PartitionedFile::try_from(&pf)?;
263+
assert!(encoded.arrow_schema.is_some());
264+
265+
let decoded = PartitionedFile::try_from(&encoded)?;
266+
assert_eq!(decoded.arrow_schema.as_deref(), Some(arrow_schema.as_ref()));
267+
Ok(())
268+
}
269+
207270
#[test]
208271
fn partitioned_file_from_proto_rejects_invalid_path() {
209272
let proto = protobuf::PartitionedFile {

‎datafusion/proto/Cargo.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ avro = ["datafusion-datasource-avro"]
5656

5757
[dependencies]
5858
arrow = { workspace = true }
59-
chrono = { workspace = true }
6059
datafusion-catalog = { workspace = true }
6160
datafusion-catalog-listing = { workspace = true }
6261
datafusion-common = { workspace = true }

‎datafusion/proto/src/physical_plan/from_proto.rs‎

Lines changed: 2 additions & 163 deletions
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,10 @@ use arrow::array::RecordBatch;
2323
use arrow::compute::SortOptions;
2424
use arrow::datatypes::{Field, Schema};
2525
use arrow::ipc::reader::StreamReader;
26-
use datafusion_common::{DataFusionError, Result, internal_datafusion_err, not_impl_err};
26+
use datafusion_common::{Result, internal_datafusion_err, not_impl_err};
27+
use datafusion_datasource::TableSchema;
2728
use datafusion_datasource::file::FileSource;
28-
use datafusion_datasource::file_groups::FileGroup;
2929
use datafusion_datasource::file_scan_config::FileScanConfig;
30-
use datafusion_datasource::file_sink_config::FileSinkConfig;
31-
use datafusion_datasource::{FileRange, PartitionedFile, TableSchema};
32-
use datafusion_datasource_csv::file_format::CsvSink;
33-
use datafusion_datasource_json::file_format::JsonSink;
34-
#[cfg(feature = "parquet")]
35-
use datafusion_datasource_parquet::file_format::ParquetSink;
3630
use datafusion_execution::{FunctionRegistry, TaskContext};
3731
use datafusion_expr::WindowFunctionDefinition;
3832
use datafusion_physical_expr::expressions::{LambdaExpr, LambdaVariable};
@@ -467,66 +461,6 @@ pub fn parse_record_batches(buf: &[u8]) -> Result<Vec<RecordBatch>> {
467461
Ok(batches)
468462
}
469463

470-
/// Thin shim over `TryFrom<&protobuf::PartitionedFile>`, which owns the wire logic.
471-
impl TryFromProto<&protobuf::PartitionedFile> for PartitionedFile {
472-
type Error = DataFusionError;
473-
474-
fn try_from_proto(val: &protobuf::PartitionedFile) -> Result<Self, Self::Error> {
475-
PartitionedFile::try_from(val)
476-
}
477-
}
478-
479-
/// Thin shim over `TryFrom<&protobuf::FileRange>`, which owns the wire logic.
480-
impl TryFromProto<&protobuf::FileRange> for FileRange {
481-
type Error = DataFusionError;
482-
483-
fn try_from_proto(value: &protobuf::FileRange) -> Result<Self, Self::Error> {
484-
FileRange::try_from(value)
485-
}
486-
}
487-
488-
/// Thin shim over `TryFrom<&protobuf::FileGroup>`, which owns the wire logic.
489-
impl TryFromProto<&protobuf::FileGroup> for FileGroup {
490-
type Error = DataFusionError;
491-
492-
fn try_from_proto(val: &protobuf::FileGroup) -> Result<Self, Self::Error> {
493-
FileGroup::try_from(val)
494-
}
495-
}
496-
497-
impl TryFromProto<&protobuf::JsonSink> for JsonSink {
498-
type Error = DataFusionError;
499-
500-
fn try_from_proto(value: &protobuf::JsonSink) -> Result<Self, Self::Error> {
501-
Self::try_from(value)
502-
}
503-
}
504-
505-
#[cfg(feature = "parquet")]
506-
impl TryFromProto<&protobuf::ParquetSink> for ParquetSink {
507-
type Error = DataFusionError;
508-
509-
fn try_from_proto(value: &protobuf::ParquetSink) -> Result<Self, Self::Error> {
510-
Self::try_from(value)
511-
}
512-
}
513-
514-
impl TryFromProto<&protobuf::CsvSink> for CsvSink {
515-
type Error = DataFusionError;
516-
517-
fn try_from_proto(value: &protobuf::CsvSink) -> Result<Self, Self::Error> {
518-
Self::try_from(value)
519-
}
520-
}
521-
522-
impl TryFromProto<&protobuf::FileSinkConfig> for FileSinkConfig {
523-
type Error = DataFusionError;
524-
525-
fn try_from_proto(conf: &protobuf::FileSinkConfig) -> Result<Self, Self::Error> {
526-
conf.try_into()
527-
}
528-
}
529-
530464
/// Concrete [`PhysicalExprDecode`] driver that backs
531465
/// [`PhysicalExprDecodeCtx`] inside `parse_physical_expr_with_converter`.
532466
///
@@ -553,98 +487,3 @@ impl datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprD
553487
.proto_to_physical_expr(node, schema, self.ctx)
554488
}
555489
}
556-
557-
#[cfg(test)]
558-
mod tests {
559-
use super::*;
560-
use arrow::datatypes::{DataType, Field, Schema};
561-
use chrono::{TimeZone, Utc};
562-
use datafusion_common::ScalarValue;
563-
use object_store::ObjectMeta;
564-
use object_store::path::Path;
565-
566-
#[test]
567-
fn partitioned_file_path_roundtrip_percent_encoded() {
568-
let path_str = "foo/foo%2Fbar/baz%252Fqux";
569-
let pf = PartitionedFile::new_from_meta(ObjectMeta {
570-
location: Path::parse(path_str).unwrap(),
571-
last_modified: Utc.timestamp_nanos(1_000),
572-
size: 42,
573-
e_tag: None,
574-
version: None,
575-
});
576-
577-
let proto = protobuf::PartitionedFile::try_from_proto(&pf).unwrap();
578-
assert_eq!(proto.path, path_str);
579-
580-
let pf2 = PartitionedFile::try_from_proto(&proto).unwrap();
581-
assert_eq!(pf2.object_meta.location.as_ref(), path_str);
582-
assert_eq!(pf2.object_meta.location, pf.object_meta.location);
583-
assert_eq!(pf2.object_meta.size, pf.object_meta.size);
584-
assert_eq!(pf2.object_meta.last_modified, pf.object_meta.last_modified);
585-
}
586-
587-
#[test]
588-
fn partitioned_file_arrow_schema_roundtrip() {
589-
use std::collections::HashMap;
590-
591-
let arrow_schema = Arc::new(Schema::new_with_metadata(
592-
vec![
593-
Field::new("id", DataType::Int64, false),
594-
Field::new("value", DataType::Utf8, true).with_metadata(HashMap::from([
595-
("field_meta".to_string(), "field_value".to_string()),
596-
])),
597-
],
598-
HashMap::from([("schema_meta".to_string(), "schema_value".to_string())]),
599-
));
600-
let pf = PartitionedFile::new("foo/bar.parquet", 10)
601-
.with_arrow_schema(Arc::clone(&arrow_schema));
602-
603-
let proto = protobuf::PartitionedFile::try_from_proto(&pf).unwrap();
604-
assert!(proto.arrow_schema.is_some());
605-
606-
let decoded = PartitionedFile::try_from_proto(&proto).unwrap();
607-
assert_eq!(
608-
decoded.arrow_schema.as_ref().map(|s| s.as_ref()),
609-
Some(arrow_schema.as_ref())
610-
);
611-
}
612-
613-
#[test]
614-
fn partitioned_file_statistics_roundtrip_with_partition_values() {
615-
use datafusion_common::Statistics;
616-
let file_schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);
617-
let pf = PartitionedFile::new("foo/bar.parquet", 1234)
618-
.with_partition_values(vec![ScalarValue::from("2024-01-01")])
619-
.with_statistics(Arc::new(Statistics::new_unknown(&file_schema)));
620-
621-
// `statistics` covers the full table schema: file columns followed by one
622-
// entry per partition column.
623-
let expected_len = file_schema.fields().len() + pf.partition_values.len();
624-
assert_eq!(
625-
pf.statistics.as_ref().unwrap().column_statistics.len(),
626-
expected_len
627-
);
628-
629-
let proto = protobuf::PartitionedFile::try_from_proto(&pf).unwrap();
630-
let decoded = PartitionedFile::try_from_proto(&proto).unwrap();
631-
632-
assert_eq!(decoded.statistics, pf.statistics);
633-
}
634-
635-
#[test]
636-
fn partitioned_file_from_proto_invalid_path() {
637-
let proto = protobuf::PartitionedFile {
638-
arrow_schema: None,
639-
path: "foo//bar".to_string(),
640-
size: 1,
641-
last_modified_ns: 0,
642-
partition_values: vec![],
643-
range: None,
644-
statistics: None,
645-
};
646-
647-
let err = PartitionedFile::try_from_proto(&proto).unwrap_err();
648-
assert!(err.to_string().contains("Invalid object_store path"));
649-
}
650-
}

0 commit comments

Comments
 (0)