Skip to content

Commit 7b1e55d

Browse files
authored
fix: preserve dotted column relation qualifiers (#25334)
## Which issue does this PR close? * Closes #24776. ## Rationale for this change `ColumnRelation` currently round-trips `TableReference` qualifiers through a single dotted string. When a catalog, schema, or table identifier itself contains `.`, the qualifier boundaries are lost during deserialization, causing the reconstructed `TableReference` to differ from the original. For example, a schema named `my.schema` can be incorrectly interpreted as separate qualifier components after a protobuf round-trip. ## What changes are included in this PR? * Add a structured `parts` field to `ColumnRelation`. * Serialize `TableReference` components using `TableReference::to_vec()`. * Prefer the structured `parts` representation when deserializing. * Fall back to the existing `relation` string for legacy protobuf messages. * Add round-trip tests for dotted bare, partial, and full table references. * Add a test covering decoding of legacy relation-only messages. * Regenerate the protobuf bindings. ## What is the testing strategy for this PR? Added regression tests covering the `Column -> protobuf -> TableReference` round-trip for: * Bare table references containing `.` * Partial references with dotted schema identifiers * Full references with dotted schema identifiers * Legacy `relation`-only protobuf messages Also ran: `cargo test -p datafusion-proto-common column_relation -- --nocapture` `cargo test -p datafusion-proto-common` All tests pass. ## Are there any user-facing changes? No.
1 parent e8535d3 commit 7b1e55d

6 files changed

Lines changed: 148 additions & 1 deletion

File tree

‎datafusion/proto-common/proto/datafusion_common.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ package datafusion_common;
2222

2323
message ColumnRelation {
2424
string relation = 1;
25+
repeated string parts = 2;
2526
}
2627

2728
message Column {

‎datafusion/proto-common/src/from_proto/mod.rs‎

Lines changed: 124 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,14 @@ where
147147

148148
impl From<protobuf::ColumnRelation> for TableReference {
149149
fn from(rel: protobuf::ColumnRelation) -> Self {
150-
Self::parse_str_normalized(rel.relation.as_str(), true)
150+
match rel.parts.as_slice() {
151+
[table] => Self::bare(table.as_str()),
152+
[schema, table] => Self::partial(schema.as_str(), table.as_str()),
153+
[catalog, schema, table] => {
154+
Self::full(catalog.as_str(), schema.as_str(), table.as_str())
155+
}
156+
_ => Self::parse_str_normalized(rel.relation.as_str(), true),
157+
}
151158
}
152159
}
153160

@@ -1440,6 +1447,7 @@ where
14401447

14411448
#[cfg(test)]
14421449
mod tests {
1450+
use datafusion_common::TableReference;
14431451
use datafusion_common::config::{
14441452
MaxRowGroupBytes, ParquetCdcOptions, ParquetOptions, TableParquetOptions,
14451453
};
@@ -1461,6 +1469,121 @@ mod tests {
14611469
);
14621470
}
14631471

1472+
#[test]
1473+
fn column_relation_round_trip_preserves_dotted_bare_table() {
1474+
let column = datafusion_common::Column::new(
1475+
Some(TableReference::bare("has.dot")),
1476+
"column",
1477+
);
1478+
1479+
let proto: crate::protobuf_common::Column = (&column).into();
1480+
let relation = proto.relation.expect("relation should be present");
1481+
1482+
assert_eq!(relation.relation, "has.dot");
1483+
assert_eq!(relation.parts, vec!["has.dot".to_string()]);
1484+
1485+
let recovered = TableReference::from(relation);
1486+
assert_eq!(recovered, TableReference::bare("has.dot"));
1487+
}
1488+
1489+
#[test]
1490+
fn column_relation_round_trip_preserves_dotted_partial_reference() {
1491+
let column = datafusion_common::Column::new(
1492+
Some(TableReference::partial("my.schema", "table")),
1493+
"column",
1494+
);
1495+
1496+
let proto: crate::protobuf_common::Column = (&column).into();
1497+
let relation = proto.relation.expect("relation should be present");
1498+
1499+
assert_eq!(relation.relation, "my.schema.table");
1500+
assert_eq!(
1501+
relation.parts,
1502+
vec!["my.schema".to_string(), "table".to_string()]
1503+
);
1504+
1505+
let recovered = TableReference::from(relation);
1506+
assert_eq!(recovered, TableReference::partial("my.schema", "table"));
1507+
}
1508+
1509+
#[test]
1510+
fn column_relation_round_trip_preserves_dotted_full_reference() {
1511+
let column = datafusion_common::Column::new(
1512+
Some(TableReference::full("catalog", "my.schema", "table")),
1513+
"column",
1514+
);
1515+
1516+
let proto: crate::protobuf_common::Column = (&column).into();
1517+
let relation = proto.relation.expect("relation should be present");
1518+
1519+
assert_eq!(relation.relation, "catalog.my.schema.table");
1520+
assert_eq!(
1521+
relation.parts,
1522+
vec![
1523+
"catalog".to_string(),
1524+
"my.schema".to_string(),
1525+
"table".to_string()
1526+
]
1527+
);
1528+
1529+
let recovered = TableReference::from(relation);
1530+
assert_eq!(
1531+
recovered,
1532+
TableReference::full("catalog", "my.schema", "table")
1533+
);
1534+
}
1535+
1536+
#[test]
1537+
fn df_schema_round_trip_preserves_dotted_qualifiers() {
1538+
use std::sync::Arc;
1539+
1540+
use datafusion_common::DFSchema;
1541+
use datafusion_common::arrow::datatypes::{DataType, Field};
1542+
1543+
let schema = DFSchema::new_with_metadata(
1544+
vec![
1545+
(
1546+
Some(TableReference::partial("my.schema", "table")),
1547+
Arc::new(Field::new("partial_col", DataType::Utf8, true)),
1548+
),
1549+
(
1550+
Some(TableReference::full("catalog", "my.schema", "table")),
1551+
Arc::new(Field::new("full_col", DataType::Utf8, true)),
1552+
),
1553+
],
1554+
std::collections::HashMap::<String, String>::new(),
1555+
)
1556+
.unwrap();
1557+
1558+
let proto: crate::protobuf_common::DfSchema = (&schema).try_into().unwrap();
1559+
let recovered = DFSchema::try_from(&proto).unwrap();
1560+
1561+
let qualifiers = recovered
1562+
.iter()
1563+
.map(|(qualifier, _)| qualifier.cloned())
1564+
.collect::<Vec<_>>();
1565+
1566+
assert_eq!(
1567+
qualifiers,
1568+
vec![
1569+
Some(TableReference::partial("my.schema", "table")),
1570+
Some(TableReference::full("catalog", "my.schema", "table")),
1571+
]
1572+
);
1573+
}
1574+
1575+
#[test]
1576+
fn column_relation_decodes_legacy_relation() {
1577+
let proto = crate::protobuf_common::ColumnRelation {
1578+
relation: "schema.table".to_string(),
1579+
parts: vec![],
1580+
};
1581+
1582+
let recovered = TableReference::from(proto);
1583+
1584+
assert_eq!(recovered, TableReference::partial("schema", "table"));
1585+
}
1586+
14641587
#[test]
14651588
fn table_parquet_options_defaults_missing_global() {
14661589
let recovered = TableParquetOptions::try_from(

‎datafusion/proto-common/src/generated/pbjson.rs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1030,10 +1030,16 @@ impl serde::Serialize for ColumnRelation {
10301030
if !self.relation.is_empty() {
10311031
len += 1;
10321032
}
1033+
if !self.parts.is_empty() {
1034+
len += 1;
1035+
}
10331036
let mut struct_ser = serializer.serialize_struct("datafusion_common.ColumnRelation", len)?;
10341037
if !self.relation.is_empty() {
10351038
struct_ser.serialize_field("relation", &self.relation)?;
10361039
}
1040+
if !self.parts.is_empty() {
1041+
struct_ser.serialize_field("parts", &self.parts)?;
1042+
}
10371043
struct_ser.end()
10381044
}
10391045
}
@@ -1045,11 +1051,13 @@ impl<'de> serde::Deserialize<'de> for ColumnRelation {
10451051
{
10461052
const FIELDS: &[&str] = &[
10471053
"relation",
1054+
"parts",
10481055
];
10491056

10501057
#[allow(clippy::enum_variant_names)]
10511058
enum GeneratedField {
10521059
Relation,
1060+
Parts,
10531061
}
10541062
impl<'de> serde::Deserialize<'de> for GeneratedField {
10551063
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -1072,6 +1080,7 @@ impl<'de> serde::Deserialize<'de> for ColumnRelation {
10721080
{
10731081
match value {
10741082
"relation" => Ok(GeneratedField::Relation),
1083+
"parts" => Ok(GeneratedField::Parts),
10751084
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
10761085
}
10771086
}
@@ -1092,6 +1101,7 @@ impl<'de> serde::Deserialize<'de> for ColumnRelation {
10921101
V: serde::de::MapAccess<'de>,
10931102
{
10941103
let mut relation__ = None;
1104+
let mut parts__ = None;
10951105
while let Some(k) = map_.next_key()? {
10961106
match k {
10971107
GeneratedField::Relation => {
@@ -1100,10 +1110,17 @@ impl<'de> serde::Deserialize<'de> for ColumnRelation {
11001110
}
11011111
relation__ = Some(map_.next_value()?);
11021112
}
1113+
GeneratedField::Parts => {
1114+
if parts__.is_some() {
1115+
return Err(serde::de::Error::duplicate_field("parts"));
1116+
}
1117+
parts__ = Some(map_.next_value()?);
1118+
}
11031119
}
11041120
}
11051121
Ok(ColumnRelation {
11061122
relation: relation__.unwrap_or_default(),
1123+
parts: parts__.unwrap_or_default(),
11071124
})
11081125
}
11091126
}

‎datafusion/proto-common/src/generated/prost.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
pub struct ColumnRelation {
44
#[prost(string, tag = "1")]
55
pub relation: ::prost::alloc::string::String,
6+
#[prost(string, repeated, tag = "2")]
7+
pub parts: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
68
}
79
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
810
pub struct Column {

‎datafusion/proto-common/src/to_proto/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,7 @@ impl From<Column> for protobuf::Column {
248248
Self {
249249
relation: c.relation.map(|relation| protobuf::ColumnRelation {
250250
relation: relation.to_string(),
251+
parts: relation.to_vec(),
251252
}),
252253
name: c.name,
253254
}
@@ -293,6 +294,7 @@ impl TryFrom<&DFSchema> for protobuf::DfSchema {
293294
field: Some(field.as_ref().try_into()?),
294295
qualifier: qualifier.map(|r| protobuf::ColumnRelation {
295296
relation: r.to_string(),
297+
parts: r.to_vec(),
296298
}),
297299
})
298300
})

‎datafusion/proto-models/src/generated/datafusion_proto_common.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
pub struct ColumnRelation {
44
#[prost(string, tag = "1")]
55
pub relation: ::prost::alloc::string::String,
6+
#[prost(string, repeated, tag = "2")]
7+
pub parts: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
68
}
79
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
810
pub struct Column {

0 commit comments

Comments
 (0)