Skip to content

Commit f956fd1

Browse files
authored
fix(proto): preserve Arrow IPC stream format (apache#24224)
## Which issue does this PR close? - Closes apache#24196. ## Rationale for this change `ArrowSource` distinguishes between Arrow IPC file and stream formats, but this format was not serialized. As a result, a stream scan round-tripped through protobuf as a file scan, selecting the wrong opener and potentially allowing unsupported range-based repartitioning. ## What changes are included in this PR? - Add an `ArrowIpcFormat` discriminator to `ArrowScanExecNode`. - Serialize and restore the `ArrowSource` IPC format. - Decode payloads without the new field as file format, preserving the previous behavior. - Regenerate the prost and pbjson models. - Add regression coverage for file, stream, and older payloads without the format field. ## Are these changes tested? Yes: - `cargo test -p datafusion-proto --test proto_integration roundtrip_arrow` - `cargo test -p datafusion-proto --test proto_integration arrow_scan_without_format_field_decodes_as_file_format` - `cargo fmt --all` - Targeted all-feature clippy for `datafusion-datasource-arrow` and `datafusion-proto` ## Are there any user-facing changes? Arrow IPC stream scans now preserve their format across protobuf round trips. The binary protobuf change is additive. Older payloads continue to decode as file format. Stream preservation requires both producer and consumer to include this change because older versions do not carry or read the discriminator. The generated Rust `ArrowScanExecNode` struct gains a `format` field.
1 parent b9399dc commit f956fd1

5 files changed

Lines changed: 256 additions & 9 deletions

File tree

‎datafusion/datasource-arrow/src/source.rs‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -403,10 +403,8 @@ impl FileSource for ArrowSource {
403403
datafusion_physical_plan::apply_expression_roots(self.projection.source.iter(), f)
404404
}
405405

406-
/// Emit an `ArrowScan` node wrapping the shared base config.
407-
///
408-
/// Decoding defaults to the IPC file format because protobuf does not
409-
/// distinguish it from the IPC stream format.
406+
/// Emit an `ArrowScan` node wrapping the shared base config and recording
407+
/// which Arrow IPC format (file or stream) this source reads.
410408
#[cfg(feature = "proto")]
411409
fn try_to_proto(
412410
&self,
@@ -416,10 +414,16 @@ impl FileSource for ArrowSource {
416414
use datafusion_proto_models::protobuf;
417415
use protobuf::physical_plan_node::PhysicalPlanType;
418416

417+
let format = match self.format {
418+
ArrowFormat::File => protobuf::ArrowIpcFormat::File,
419+
ArrowFormat::Stream => protobuf::ArrowIpcFormat::Stream,
420+
};
421+
419422
Ok(Some(protobuf::PhysicalPlanNode {
420423
physical_plan_type: Some(PhysicalPlanType::ArrowScan(
421424
protobuf::ArrowScanExecNode {
422425
base_conf: Some(base.try_to_proto(ctx)?),
426+
format: format as i32,
423427
},
424428
)),
425429
}))
@@ -430,8 +434,8 @@ impl FileSource for ArrowSource {
430434
impl ArrowSource {
431435
/// Reconstructs a `DataSourceExec` from a protobuf `ArrowScan`.
432436
///
433-
/// Defaults to the IPC file format because protobuf does not distinguish it
434-
/// from the IPC stream format.
437+
/// Payloads encoded before the `format` field existed leave it unset;
438+
/// those decode as the IPC file format, matching the historical behavior.
435439
pub fn try_from_proto(
436440
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
437441
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
@@ -455,9 +459,21 @@ impl ArrowSource {
455459
)
456460
})?;
457461

462+
let format = protobuf::ArrowIpcFormat::try_from(scan.format).map_err(|_| {
463+
datafusion_common::internal_datafusion_err!(
464+
"Unknown ArrowIpcFormat: {}",
465+
scan.format
466+
)
467+
})?;
468+
458469
let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;
459-
let source = Arc::new(ArrowSource::new_file_source(table_schema));
460-
let scan_conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?;
470+
let source = match format {
471+
protobuf::ArrowIpcFormat::Stream => {
472+
ArrowSource::new_stream_file_source(table_schema)
473+
}
474+
protobuf::ArrowIpcFormat::File => ArrowSource::new_file_source(table_schema),
475+
};
476+
let scan_conf = FileScanConfig::try_from_proto(base_conf, ctx, Arc::new(source))?;
461477
Ok(DataSourceExec::from_data_source(scan_conf))
462478
}
463479
}

‎datafusion/proto-models/proto/datafusion.proto‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1296,8 +1296,18 @@ message AvroScanExecNode {
12961296
FileScanExecConf base_conf = 1;
12971297
}
12981298

1299+
// Identifies which Arrow IPC format an ArrowScanExecNode reads.
1300+
enum ArrowIpcFormat {
1301+
// Arrow IPC file format (with footer, supports range-based parallel reading).
1302+
// This is the default for payloads encoded before the format field existed.
1303+
ARROW_IPC_FORMAT_FILE = 0;
1304+
// Arrow IPC stream format (without footer, sequential reading only)
1305+
ARROW_IPC_FORMAT_STREAM = 1;
1306+
}
1307+
12991308
message ArrowScanExecNode {
13001309
FileScanExecConf base_conf = 1;
1310+
ArrowIpcFormat format = 2;
13011311
}
13021312

13031313
message MemoryScanExecNode {

‎datafusion/proto-models/src/generated/pbjson.rs‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1445,6 +1445,77 @@ impl<'de> serde::Deserialize<'de> for AnalyzedLogicalPlanType {
14451445
deserializer.deserialize_struct("datafusion.AnalyzedLogicalPlanType", FIELDS, GeneratedVisitor)
14461446
}
14471447
}
1448+
impl serde::Serialize for ArrowIpcFormat {
1449+
#[allow(deprecated)]
1450+
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1451+
where
1452+
S: serde::Serializer,
1453+
{
1454+
let variant = match self {
1455+
Self::File => "ARROW_IPC_FORMAT_FILE",
1456+
Self::Stream => "ARROW_IPC_FORMAT_STREAM",
1457+
};
1458+
serializer.serialize_str(variant)
1459+
}
1460+
}
1461+
impl<'de> serde::Deserialize<'de> for ArrowIpcFormat {
1462+
#[allow(deprecated)]
1463+
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
1464+
where
1465+
D: serde::Deserializer<'de>,
1466+
{
1467+
const FIELDS: &[&str] = &[
1468+
"ARROW_IPC_FORMAT_FILE",
1469+
"ARROW_IPC_FORMAT_STREAM",
1470+
];
1471+
1472+
struct GeneratedVisitor;
1473+
1474+
impl serde::de::Visitor<'_> for GeneratedVisitor {
1475+
type Value = ArrowIpcFormat;
1476+
1477+
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1478+
write!(formatter, "expected one of: {:?}", &FIELDS)
1479+
}
1480+
1481+
fn visit_i64<E>(self, v: i64) -> std::result::Result<Self::Value, E>
1482+
where
1483+
E: serde::de::Error,
1484+
{
1485+
i32::try_from(v)
1486+
.ok()
1487+
.and_then(|x| x.try_into().ok())
1488+
.ok_or_else(|| {
1489+
serde::de::Error::invalid_value(serde::de::Unexpected::Signed(v), &self)
1490+
})
1491+
}
1492+
1493+
fn visit_u64<E>(self, v: u64) -> std::result::Result<Self::Value, E>
1494+
where
1495+
E: serde::de::Error,
1496+
{
1497+
i32::try_from(v)
1498+
.ok()
1499+
.and_then(|x| x.try_into().ok())
1500+
.ok_or_else(|| {
1501+
serde::de::Error::invalid_value(serde::de::Unexpected::Unsigned(v), &self)
1502+
})
1503+
}
1504+
1505+
fn visit_str<E>(self, value: &str) -> std::result::Result<Self::Value, E>
1506+
where
1507+
E: serde::de::Error,
1508+
{
1509+
match value {
1510+
"ARROW_IPC_FORMAT_FILE" => Ok(ArrowIpcFormat::File),
1511+
"ARROW_IPC_FORMAT_STREAM" => Ok(ArrowIpcFormat::Stream),
1512+
_ => Err(serde::de::Error::unknown_variant(value, FIELDS)),
1513+
}
1514+
}
1515+
}
1516+
deserializer.deserialize_any(GeneratedVisitor)
1517+
}
1518+
}
14481519
impl serde::Serialize for ArrowScanExecNode {
14491520
#[allow(deprecated)]
14501521
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
@@ -1456,10 +1527,18 @@ impl serde::Serialize for ArrowScanExecNode {
14561527
if self.base_conf.is_some() {
14571528
len += 1;
14581529
}
1530+
if self.format != 0 {
1531+
len += 1;
1532+
}
14591533
let mut struct_ser = serializer.serialize_struct("datafusion.ArrowScanExecNode", len)?;
14601534
if let Some(v) = self.base_conf.as_ref() {
14611535
struct_ser.serialize_field("baseConf", v)?;
14621536
}
1537+
if self.format != 0 {
1538+
let v = ArrowIpcFormat::try_from(self.format)
1539+
.map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.format)))?;
1540+
struct_ser.serialize_field("format", &v)?;
1541+
}
14631542
struct_ser.end()
14641543
}
14651544
}
@@ -1472,11 +1551,13 @@ impl<'de> serde::Deserialize<'de> for ArrowScanExecNode {
14721551
const FIELDS: &[&str] = &[
14731552
"base_conf",
14741553
"baseConf",
1554+
"format",
14751555
];
14761556

14771557
#[allow(clippy::enum_variant_names)]
14781558
enum GeneratedField {
14791559
BaseConf,
1560+
Format,
14801561
}
14811562
impl<'de> serde::Deserialize<'de> for GeneratedField {
14821563
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -1499,6 +1580,7 @@ impl<'de> serde::Deserialize<'de> for ArrowScanExecNode {
14991580
{
15001581
match value {
15011582
"baseConf" | "base_conf" => Ok(GeneratedField::BaseConf),
1583+
"format" => Ok(GeneratedField::Format),
15021584
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
15031585
}
15041586
}
@@ -1519,6 +1601,7 @@ impl<'de> serde::Deserialize<'de> for ArrowScanExecNode {
15191601
V: serde::de::MapAccess<'de>,
15201602
{
15211603
let mut base_conf__ = None;
1604+
let mut format__ = None;
15221605
while let Some(k) = map_.next_key()? {
15231606
match k {
15241607
GeneratedField::BaseConf => {
@@ -1527,10 +1610,17 @@ impl<'de> serde::Deserialize<'de> for ArrowScanExecNode {
15271610
}
15281611
base_conf__ = map_.next_value()?;
15291612
}
1613+
GeneratedField::Format => {
1614+
if format__.is_some() {
1615+
return Err(serde::de::Error::duplicate_field("format"));
1616+
}
1617+
format__ = Some(map_.next_value::<ArrowIpcFormat>()? as i32);
1618+
}
15301619
}
15311620
}
15321621
Ok(ArrowScanExecNode {
15331622
base_conf: base_conf__,
1623+
format: format__.unwrap_or_default(),
15341624
})
15351625
}
15361626
}

‎datafusion/proto-models/src/generated/prost.rs‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1996,6 +1996,8 @@ pub struct AvroScanExecNode {
19961996
pub struct ArrowScanExecNode {
19971997
#[prost(message, optional, tag = "1")]
19981998
pub base_conf: ::core::option::Option<FileScanExecConf>,
1999+
#[prost(enumeration = "ArrowIpcFormat", tag = "2")]
2000+
pub format: i32,
19992001
}
20002002
#[derive(Clone, PartialEq, ::prost::Message)]
20012003
pub struct MemoryScanExecNode {
@@ -2794,6 +2796,36 @@ impl InsertOp {
27942796
}
27952797
}
27962798
}
2799+
/// Identifies which Arrow IPC format an ArrowScanExecNode reads.
2800+
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
2801+
#[repr(i32)]
2802+
pub enum ArrowIpcFormat {
2803+
/// Arrow IPC file format (with footer, supports range-based parallel reading).
2804+
/// This is the default for payloads encoded before the format field existed.
2805+
File = 0,
2806+
/// Arrow IPC stream format (without footer, sequential reading only)
2807+
Stream = 1,
2808+
}
2809+
impl ArrowIpcFormat {
2810+
/// String value of the enum field names used in the ProtoBuf definition.
2811+
///
2812+
/// The values are not transformed in any way and thus are considered stable
2813+
/// (if the ProtoBuf definition does not change) and safe for programmatic use.
2814+
pub fn as_str_name(&self) -> &'static str {
2815+
match self {
2816+
Self::File => "ARROW_IPC_FORMAT_FILE",
2817+
Self::Stream => "ARROW_IPC_FORMAT_STREAM",
2818+
}
2819+
}
2820+
/// Creates an enum from field names used in the ProtoBuf definition.
2821+
pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
2822+
match value {
2823+
"ARROW_IPC_FORMAT_FILE" => Some(Self::File),
2824+
"ARROW_IPC_FORMAT_STREAM" => Some(Self::Stream),
2825+
_ => None,
2826+
}
2827+
}
2828+
}
27972829
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
27982830
#[repr(i32)]
27992831
pub enum PartitionMode {

0 commit comments

Comments
 (0)