Skip to content

Commit 37afb61

Browse files
cetra3claude
andcommitted
fix: column ids, stage 7: fix decorrelation and remaining id bugs
Bind pulled up references by id (apache#26234, apache#26033), build join schemas with ids (apache#14147), fix fallback and join key id allocation, read by id where DataFrame methods rebuild columns, keep same-named UNNEST outputs apart, and keep rejecting ambiguous joins. Adds regression tests for apache#13476, apache#26620 and apache#21066. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent e37e55b commit 37afb61

22 files changed

Lines changed: 723 additions & 78 deletions

File tree

‎datafusion-examples/examples/query_planning/parse_sql_expr.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,9 @@ async fn simple_dataframe_parse_sql_expr_demo() -> Result<()> {
7676

7777
let parsed_expr = df.parse_sql_expr(sql)?;
7878

79-
assert_eq!(parsed_expr, expr);
79+
// the parsed columns are bound to the columns of `df`: they have column
80+
// ids, which `==` compares, and `col` gives none
81+
assert_eq!(parsed_expr.without_column_ids(), expr);
8082

8183
Ok(())
8284
}

‎datafusion-examples/examples/query_planning/plan_walk.rs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,9 @@ pub fn plan_walk() -> Result<()> {
154154
#[derive(Debug, Default)]
155155
struct PlanSummary {
156156
tables: BTreeSet<String>,
157+
/// The predicates without their column ids: the plan binds the columns
158+
/// they read (`==` compares the ids), and this example compares them with
159+
/// predicates written by name
157160
filters: Vec<Expr>,
158161
}
159162

@@ -163,10 +166,14 @@ fn summarize(plan: &LogicalPlan) -> Result<PlanSummary> {
163166
match node {
164167
LogicalPlan::TableScan(scan) => {
165168
summary.tables.insert(scan.table_name.to_string());
166-
summary.filters.extend(scan.filters.iter().cloned());
169+
summary
170+
.filters
171+
.extend(scan.filters.iter().cloned().map(Expr::without_column_ids));
167172
}
168173
LogicalPlan::Filter(filter) => {
169-
summary.filters.push(filter.predicate.clone());
174+
summary
175+
.filters
176+
.push(filter.predicate.clone().without_column_ids());
170177
}
171178
_ => {}
172179
}

‎datafusion/common/src/column.rs‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -441,19 +441,20 @@ impl Column {
441441
}
442442

443443
/// Returns this qualified column with the id of the field it resolves to
444-
/// in `schemas` (searched in order), if it has no id yet. Returned as is
445-
/// if it has an id, or if no field has its qualifier and name (the lookup
446-
/// then uses the qualifier and name). An error if the first schema that
447-
/// has fields with its qualifier and name has several, with different
448-
/// ids: the reference is ambiguous.
444+
/// in `schemas` (searched level by level), if it has no id yet. Returned
445+
/// as is if it has an id, or if no field has its qualifier and name (the
446+
/// lookup then uses the qualifier and name). An error if the first level
447+
/// that has fields with its qualifier and name has several, with
448+
/// different ids, in one schema or across its schemas (the two sides of
449+
/// a join): the reference is ambiguous.
449450
fn bind_qualified(self, schemas: &[&[&DFSchema]]) -> Result<Self> {
450451
if self.id.is_some() {
451452
return Ok(self);
452453
}
453-
for schema in schemas.iter().flat_map(|level| level.iter()) {
454-
let mut ids = schema
454+
for level in schemas {
455+
let mut ids = level
455456
.iter()
456-
.zip(schema.field_ids())
457+
.flat_map(|schema| schema.iter().zip(schema.field_ids()))
457458
.filter(|((qualifier, field), _)| {
458459
field.name() == &self.name
459460
&& qualifier.is_some_and(|q| {

‎datafusion/core/src/dataframe/mod.rs‎

Lines changed: 29 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -398,7 +398,9 @@ impl DataFrame {
398398
}
399399

400400
/// Creates logical expression from a SQL query text.
401-
/// The expression is created and processed against the current schema.
401+
/// The expression is created and processed against the current schema:
402+
/// its column references are bound to the columns of this `DataFrame`
403+
/// (they have column ids, see [`Column`]).
402404
///
403405
/// # Example: Parsing SQL queries
404406
/// ```
@@ -417,7 +419,9 @@ impl DataFrame {
417419
/// .read_csv("tests/data/example.csv", CsvReadOptions::new())
418420
/// .await?;
419421
/// let expr = df.parse_sql_expr(sql)?;
420-
/// assert_eq!(expected, expr);
422+
/// // the columns of `expr` are bound to the columns of `df`: they have
423+
/// // column ids, which `==` compares, and `col` gives none
424+
/// assert_eq!(expected, expr.without_column_ids());
421425
/// # Ok(())
422426
/// # }
423427
/// ```
@@ -688,9 +692,15 @@ impl DataFrame {
688692
.fields()
689693
.into_iter()
690694
.enumerate()
691-
.map(|(idx, _)| self.plan.schema().qualified_field(idx))
692-
.filter(|(qualifier, f)| !fields_to_drop.contains(&(*qualifier, f)))
693-
.map(|(qualifier, field)| Expr::Column(Column::from((qualifier, field))))
695+
.map(|(idx, _)| {
696+
let (qualifier, field) = self.plan.schema().qualified_field(idx);
697+
(qualifier, field, self.plan.schema().field_id(idx))
698+
})
699+
.filter(|(qualifier, f, _)| !fields_to_drop.contains(&(*qualifier, f)))
700+
// the kept columns are read by id: their names may repeat
701+
.map(|(qualifier, field, id)| {
702+
Expr::Column(Column::from((qualifier, field)).with_id(id))
703+
})
694704
.collect();
695705
self.select(expr)
696706
}
@@ -2521,10 +2531,12 @@ impl DataFrame {
25212531
let new_column = expr.alias(name);
25222532
let mut col_exists = false;
25232533

2534+
// the other columns are read by id: their names may repeat
25242535
let mut fields: Vec<(Expr, bool)> = plan
25252536
.schema()
25262537
.iter()
2527-
.filter_map(|(qualifier, field)| {
2538+
.zip(plan.schema().field_ids())
2539+
.filter_map(|((qualifier, field), id)| {
25282540
// Skip new fields introduced by window_plan
25292541
if !original_names.contains(field.name()) {
25302542
return None;
@@ -2534,7 +2546,7 @@ impl DataFrame {
25342546
col_exists = true;
25352547
Some((new_column.clone(), true))
25362548
} else {
2537-
let e = col(Column::from((qualifier, field)));
2549+
let e = col(Column::from((qualifier, field)).with_id(*id));
25382550
Some((e, self.projection_requires_validation))
25392551
}
25402552
})
@@ -2605,19 +2617,18 @@ impl DataFrame {
26052617
}
26062618
Err(err) => return Err(err),
26072619
};
2620+
// the columns are read by id: their names may repeat
26082621
let projection = self
26092622
.plan
26102623
.schema()
26112624
.iter()
2612-
.map(|(qualifier, field)| {
2625+
.zip(self.plan.schema().field_ids())
2626+
.map(|((qualifier, field), id)| {
2627+
let column = col(Column::from((qualifier, field)).with_id(*id));
26132628
if qualifier.eq(&qualifier_rename) && field == field_rename {
2614-
(
2615-
col(Column::from((qualifier, field)))
2616-
.alias_qualified(qualifier.cloned(), new_name),
2617-
false,
2618-
)
2629+
(column.alias_qualified(qualifier.cloned(), new_name), false)
26192630
} else {
2620-
(col(Column::from((qualifier, field))), false)
2631+
(column, false)
26212632
}
26222633
})
26232634
.collect::<Vec<_>>();
@@ -2847,8 +2858,10 @@ impl DataFrame {
28472858
.logical_plan()
28482859
.schema()
28492860
.iter()
2850-
.map(|(qualifier, field)| {
2851-
let column = Expr::Column(Column::from((qualifier, field)));
2861+
.zip(self.logical_plan().schema().field_ids())
2862+
.map(|((qualifier, field), id)| {
2863+
// read by id: the names of the columns may repeat
2864+
let column = Expr::Column(Column::from((qualifier, field)).with_id(*id));
28522865
if cols.contains(field) && applies(field) {
28532866
// Try to cast fill value to column type. If the cast fails, fallback to the original column.
28542867
match value.clone().cast_to(field.data_type()) {

‎datafusion/core/src/physical_planner.rs‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1489,12 +1489,11 @@ impl DefaultPhysicalPlanner {
14891489
keys.iter().cloned().unzip();
14901490

14911491
// the projections define columns: their ids must not
1492-
// collide with the ids of either side
1493-
let generator = Arc::new(ColumnIdGenerator::new_after(
1494-
original_left
1495-
.max_column_id()
1496-
.max(original_right.max_column_id()),
1497-
));
1492+
// collide with the ids of either side, nor with the ids
1493+
// the join itself defines (the mark column of a mark
1494+
// join), which the projection above the join keeps
1495+
let generator =
1496+
Arc::new(ColumnIdGenerator::new_after(node.max_column_id()));
14981497
let (left, left_col_keys, left_projected) =
14991498
wrap_projection_for_join_if_necessary(
15001499
&left_keys,

‎datafusion/core/tests/dataframe/mod.rs‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1643,6 +1643,153 @@ async fn join_asof() -> Result<()> {
16431643
Ok(())
16441644
}
16451645

1646+
/// Regression test for <https://github.com/apache/datafusion/issues/14147>:
1647+
/// joining two unnamed DataFrames (both qualified `?table?`) whose columns
1648+
/// have the same names
1649+
#[tokio::test]
1650+
async fn join_unnamed_same_column_names() -> Result<()> {
1651+
let ctx = SessionContext::new();
1652+
let left = ctx.read_batch(record_batch!(
1653+
("id1", Int32, [1, 2, 3]),
1654+
("val1", Utf8, ["a", "b", "c"])
1655+
)?)?;
1656+
let right = ctx.read_batch(record_batch!(
1657+
("id1", Int32, [2, 3, 4]),
1658+
("val1", Utf8, ["b", "c", "d"])
1659+
)?)?;
1660+
1661+
// a self join: the builder renumbers the right side
1662+
let self_joined =
1663+
left.clone()
1664+
.join(left.clone(), JoinType::Inner, &["id1"], &["id1"], None)?;
1665+
assert_eq!(self_joined.count().await?, 3);
1666+
1667+
let joined =
1668+
left.clone()
1669+
.join(right.clone(), JoinType::Inner, &["id1"], &["id1"], None)?;
1670+
let results = joined.clone().collect().await?;
1671+
assert_snapshot!(
1672+
batches_to_sort_string(&results),
1673+
@r"
1674+
+-----+------+-----+------+
1675+
| id1 | val1 | id1 | val1 |
1676+
+-----+------+-----+------+
1677+
| 2 | b | 2 | b |
1678+
| 3 | c | 3 | c |
1679+
+-----+------+-----+------+
1680+
"
1681+
);
1682+
1683+
// `id1` and `?table?.id1` name a column of each side: ambiguous, as in
1684+
// Postgres
1685+
let err = joined.clone().select(vec![col("id1")]).unwrap_err();
1686+
assert_contains!(
1687+
err.to_string(),
1688+
"Ambiguous reference to unqualified field id1"
1689+
);
1690+
let qualified = datafusion_common::Column::new(Some("?table?"), "id1");
1691+
let err = joined
1692+
.clone()
1693+
.select(vec![Expr::Column(qualified.clone())])
1694+
.unwrap_err();
1695+
assert_contains!(err.to_string(), "would be ambiguous");
1696+
// in a join condition, the qualified name matches a column of each side
1697+
let err = left
1698+
.join_on(right, JoinType::Inner, [Expr::Column(qualified).eq(lit(2))])
1699+
.unwrap_err();
1700+
assert_contains!(err.to_string(), "would be ambiguous");
1701+
1702+
// the columns of the schema read each side by position
1703+
let columns = joined.schema().columns();
1704+
let results = joined
1705+
.select(vec![
1706+
Expr::Column(columns[1].clone()),
1707+
Expr::Column(columns[3].clone()),
1708+
])?
1709+
.collect()
1710+
.await?;
1711+
assert_snapshot!(
1712+
batches_to_sort_string(&results),
1713+
@r"
1714+
+------+------+
1715+
| val1 | val1 |
1716+
+------+------+
1717+
| b | b |
1718+
| c | c |
1719+
+------+------+
1720+
"
1721+
);
1722+
Ok(())
1723+
}
1724+
1725+
/// The DataFrame methods that rebuild every column of a DataFrame keep
1726+
/// working when columns have the same qualifier and name (the join of
1727+
/// <https://github.com/apache/datafusion/issues/14147>)
1728+
#[tokio::test]
1729+
async fn dataframe_methods_with_repeated_column_names() -> Result<()> {
1730+
let ctx = SessionContext::new();
1731+
let left = ctx.read_batch(record_batch!(
1732+
("id1", Int32, [1, 2, 3]),
1733+
("val1", Utf8, [Some("a"), None, Some("c")])
1734+
)?)?;
1735+
let right = ctx.read_batch(record_batch!(
1736+
("id1", Int32, [2, 3, 4]),
1737+
("val1", Utf8, [Some("b"), None, Some("d")])
1738+
)?)?;
1739+
let joined = left.join(right, JoinType::Inner, &["id1"], &["id1"], None)?;
1740+
1741+
let df = joined
1742+
.clone()
1743+
.with_column("one", lit(1))?
1744+
.with_column_renamed("one", "uno")?
1745+
.fill_null(&ScalarValue::from("-"), &[])?;
1746+
assert_snapshot!(
1747+
batches_to_sort_string(&df.collect().await?),
1748+
@r"
1749+
+-----+------+-----+------+-----+
1750+
| id1 | val1 | id1 | val1 | uno |
1751+
+-----+------+-----+------+-----+
1752+
| 2 | - | 2 | b | 1 |
1753+
| 3 | c | 3 | - | 1 |
1754+
+-----+------+-----+------+-----+
1755+
"
1756+
);
1757+
1758+
let df = joined
1759+
.clone()
1760+
.with_column("l", make_array(vec![lit(1), lit(2)]))?
1761+
.unnest_columns(&["l"])?
1762+
.drop_columns(&["val1"])?;
1763+
assert_snapshot!(
1764+
batches_to_sort_string(&df.collect().await?),
1765+
@r"
1766+
+-----+-----+---+
1767+
| id1 | id1 | l |
1768+
+-----+-----+---+
1769+
| 2 | 2 | 1 |
1770+
| 2 | 2 | 2 |
1771+
| 3 | 3 | 1 |
1772+
| 3 | 3 | 2 |
1773+
+-----+-----+---+
1774+
"
1775+
);
1776+
1777+
// an unqualified name drops the column of each side
1778+
let df = joined.drop_columns(&["val1"])?;
1779+
assert_snapshot!(
1780+
batches_to_sort_string(&df.collect().await?),
1781+
@r"
1782+
+-----+-----+
1783+
| id1 | id1 |
1784+
+-----+-----+
1785+
| 2 | 2 |
1786+
| 3 | 3 |
1787+
+-----+-----+
1788+
"
1789+
);
1790+
Ok(())
1791+
}
1792+
16461793
#[tokio::test]
16471794
async fn join_coercion_unnamed() -> Result<()> {
16481795
let ctx = SessionContext::new();

0 commit comments

Comments
 (0)