Skip to content

Commit 5e0b2d0

Browse files
authored
fix(datafusion-proto): support serializing/deserilizing ArrowFormat tables (#16875)
Fixes #16874
1 parent 2ff02b2 commit 5e0b2d0

11 files changed

Lines changed: 127 additions & 7 deletions

File tree

‎datafusion/proto-common/proto/datafusion_common.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ message NdJsonFormat {
5555
JsonOptions options = 1;
5656
}
5757

58+
message ArrowFormat {}
59+
5860

5961
message PrimaryKeyConstraint{
6062
repeated uint64 indices = 1;

‎datafusion/proto-common/src/generated/pbjson.rs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,74 @@
1+
impl serde::Serialize for ArrowFormat {
2+
#[allow(deprecated)]
3+
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
4+
where
5+
S: serde::Serializer,
6+
{
7+
use serde::ser::SerializeStruct;
8+
let len = 0;
9+
let struct_ser = serializer.serialize_struct("datafusion_common.ArrowFormat", len)?;
10+
struct_ser.end()
11+
}
12+
}
13+
impl<'de> serde::Deserialize<'de> for ArrowFormat {
14+
#[allow(deprecated)]
15+
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
16+
where
17+
D: serde::Deserializer<'de>,
18+
{
19+
const FIELDS: &[&str] = &[
20+
];
21+
22+
#[allow(clippy::enum_variant_names)]
23+
enum GeneratedField {
24+
}
25+
impl<'de> serde::Deserialize<'de> for GeneratedField {
26+
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
27+
where
28+
D: serde::Deserializer<'de>,
29+
{
30+
struct GeneratedVisitor;
31+
32+
impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {
33+
type Value = GeneratedField;
34+
35+
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
36+
write!(formatter, "expected one of: {:?}", &FIELDS)
37+
}
38+
39+
#[allow(unused_variables)]
40+
fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>
41+
where
42+
E: serde::de::Error,
43+
{
44+
Err(serde::de::Error::unknown_field(value, FIELDS))
45+
}
46+
}
47+
deserializer.deserialize_identifier(GeneratedVisitor)
48+
}
49+
}
50+
struct GeneratedVisitor;
51+
impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {
52+
type Value = ArrowFormat;
53+
54+
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
55+
formatter.write_str("struct datafusion_common.ArrowFormat")
56+
}
57+
58+
fn visit_map<V>(self, mut map_: V) -> std::result::Result<ArrowFormat, V::Error>
59+
where
60+
V: serde::de::MapAccess<'de>,
61+
{
62+
while map_.next_key::<GeneratedField>()?.is_some() {
63+
let _ = map_.next_value::<serde::de::IgnoredAny>()?;
64+
}
65+
Ok(ArrowFormat {
66+
})
67+
}
68+
}
69+
deserializer.deserialize_struct("datafusion_common.ArrowFormat", FIELDS, GeneratedVisitor)
70+
}
71+
}
172
impl serde::Serialize for ArrowOptions {
273
#[allow(deprecated)]
374
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>

‎datafusion/proto-common/src/generated/prost.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ pub struct NdJsonFormat {
4545
#[prost(message, optional, tag = "1")]
4646
pub options: ::core::option::Option<JsonOptions>,
4747
}
48+
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
49+
pub struct ArrowFormat {}
4850
#[derive(Clone, PartialEq, ::prost::Message)]
4951
pub struct PrimaryKeyConstraint {
5052
#[prost(uint64, repeated, tag = "1")]

‎datafusion/proto/proto/datafusion.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ message ListingTableScanNode {
9898
datafusion_common.ParquetFormat parquet = 11;
9999
datafusion_common.AvroFormat avro = 12;
100100
datafusion_common.NdJsonFormat json = 15;
101+
datafusion_common.ArrowFormat arrow = 16;
101102
}
102103
repeated SortExprNodeCollection file_sort_order = 13;
103104
}

‎datafusion/proto/src/generated/datafusion_proto_common.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ pub struct NdJsonFormat {
4545
#[prost(message, optional, tag = "1")]
4646
pub options: ::core::option::Option<JsonOptions>,
4747
}
48+
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
49+
pub struct ArrowFormat {}
4850
#[derive(Clone, PartialEq, ::prost::Message)]
4951
pub struct PrimaryKeyConstraint {
5052
#[prost(uint64, repeated, tag = "1")]

‎datafusion/proto/src/generated/pbjson.rs‎

Lines changed: 13 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/proto/src/generated/prost.rs‎

Lines changed: 6 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/proto/src/lib.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,8 +130,9 @@ pub mod protobuf {
130130
pub use crate::generated::datafusion::*;
131131
pub use datafusion_proto_common::common::proto_error;
132132
pub use datafusion_proto_common::protobuf_common::{
133-
ArrowOptions, ArrowType, AvroFormat, AvroOptions, CsvFormat, DfSchema,
134-
EmptyMessage, Field, JoinSide, NdJsonFormat, ParquetFormat, ScalarValue, Schema,
133+
ArrowFormat, ArrowOptions, ArrowType, AvroFormat, AvroOptions, CsvFormat,
134+
DfSchema, EmptyMessage, Field, JoinSide, NdJsonFormat, ParquetFormat,
135+
ScalarValue, Schema,
135136
};
136137
pub use datafusion_proto_common::{FromProtoError, ToProtoError};
137138
}

‎datafusion/proto/src/logical_plan/mod.rs‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ use crate::{
3535
use crate::protobuf::{proto_error, ToProtoError};
3636
use arrow::datatypes::{DataType, Schema, SchemaBuilder, SchemaRef};
3737
use datafusion::datasource::cte_worktable::CteWorkTable;
38+
use datafusion::datasource::file_format::arrow::ArrowFormat;
3839
#[cfg(feature = "avro")]
3940
use datafusion::datasource::file_format::avro::AvroFormat;
4041
#[cfg(feature = "parquet")]
@@ -439,13 +440,16 @@ impl AsLogicalPlan for LogicalPlanNode {
439440
}
440441
#[cfg_attr(not(feature = "avro"), allow(unused_variables))]
441442
FileFormatType::Avro(..) => {
442-
#[cfg(feature = "avro")]
443+
#[cfg(feature = "avro")]
443444
{
444445
Arc::new(AvroFormat)
445446
}
446447
#[cfg(not(feature = "avro"))]
447448
panic!("Unable to process avro file since `avro` feature is not enabled");
448449
}
450+
FileFormatType::Arrow(..) => {
451+
Arc::new(ArrowFormat)
452+
}
449453
};
450454

451455
let table_paths = &scan
@@ -1057,13 +1061,18 @@ impl AsLogicalPlan for LogicalPlanNode {
10571061
Some(FileFormatType::Avro(protobuf::AvroFormat {}))
10581062
}
10591063

1064+
if any.is::<ArrowFormat>() {
1065+
maybe_some_type =
1066+
Some(FileFormatType::Arrow(protobuf::ArrowFormat {}))
1067+
}
1068+
10601069
if let Some(file_format_type) = maybe_some_type {
10611070
file_format_type
10621071
} else {
10631072
return Err(proto_error(format!(
1064-
"Error converting file format, {:?} is invalid as a datafusion format.",
1065-
listing_table.options().format
1066-
)));
1073+
"Error deserializing unknown file format: {:?}",
1074+
listing_table.options().format
1075+
)));
10671076
}
10681077
};
10691078

‎datafusion/proto/tests/cases/roundtrip_logical_plan.rs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ use datafusion::datasource::file_format::json::{JsonFormat, JsonFormatFactory};
2828
use datafusion::datasource::listing::{
2929
ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,
3030
};
31+
use datafusion::execution::options::ArrowReadOptions;
3132
use datafusion::optimizer::eliminate_nested_union::EliminateNestedUnion;
3233
use datafusion::optimizer::Optimizer;
3334
use datafusion_common::parsers::CompressionTypeVariant;
@@ -2656,3 +2657,16 @@ async fn roundtrip_custom_listing_tables_schema() -> Result<()> {
26562657
assert_eq!(plan, new_plan);
26572658
Ok(())
26582659
}
2660+
2661+
#[tokio::test]
2662+
async fn roundtrip_arrow_scan() -> Result<()> {
2663+
let ctx = SessionContext::new();
2664+
let plan = ctx
2665+
.read_arrow("tests/testdata/test.arrow", ArrowReadOptions::default())
2666+
.await?
2667+
.into_optimized_plan()?;
2668+
let bytes = logical_plan_to_bytes(&plan)?;
2669+
let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
2670+
assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));
2671+
Ok(())
2672+
}

0 commit comments

Comments
 (0)