Skip to content

Commit cd82723

Browse files
adriangbclaude
andcommitted
fix: prevent duplicate alias collision with user-provided __datafusion_extracted names (apache#20430)
When a user query contains an explicit alias using the reserved `__datafusion_extracted` prefix, the optimizer's AliasGenerator could generate the same alias name, causing a "Schema contains duplicate unqualified field name" error. Fix by scanning each plan node's expressions for pre-existing `__datafusion_extracted_N` aliases during the TopDown traversal in ExtractLeafExpressions, advancing the generator counter past them before any extraction occurs. Closes apache#20430 Co-Authored-By: Claude Opus 4.6 <[email protected]>
1 parent c3f0807 commit cd82723

3 files changed

Lines changed: 95 additions & 0 deletions

File tree

‎datafusion/common/src/alias.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,16 @@ impl AliasGenerator {
3737
Self::default()
3838
}
3939

40+
/// Advance the counter to at least `min_id`, ensuring future aliases
41+
/// won't collide with already-existing ones.
42+
///
43+
/// For example, if the query already contains an alias `alias_42`, then calling
44+
/// `update_min_id(42)` will ensure that future aliases generated by this
45+
/// [`AliasGenerator`] will start from `alias_43`.
46+
pub fn update_min_id(&self, min_id: usize) {
47+
self.next_id.fetch_max(min_id + 1, Ordering::Relaxed);
48+
}
49+
4050
/// Return a unique alias with the provided prefix
4151
pub fn next(&self, prefix: &str) -> String {
4252
let id = self.next_id.fetch_add(1, Ordering::Relaxed);

‎datafusion/optimizer/src/extract_leaf_expressions.rs‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,10 +127,41 @@ impl OptimizerRule for ExtractLeafExpressions {
127127
return Ok(Transformed::no(plan));
128128
}
129129
let alias_generator = config.alias_generator();
130+
131+
// Advance the alias generator past any user-provided __datafusion_extracted_N
132+
// aliases to prevent collisions when generating new extraction aliases.
133+
advance_generator_past_existing(&plan, alias_generator);
134+
130135
extract_from_plan(plan, alias_generator)
131136
}
132137
}
133138

139+
/// Scans the current plan node's expressions for pre-existing
140+
/// `__datafusion_extracted_N` aliases and advances the generator
141+
/// counter past them to avoid collisions with user-provided aliases.
142+
fn advance_generator_past_existing(
143+
plan: &LogicalPlan,
144+
alias_generator: &AliasGenerator,
145+
) {
146+
for expr in plan.expressions() {
147+
expr.apply(|e| {
148+
if let Expr::Alias(alias) = e {
149+
if let Some(id_str) = alias
150+
.name
151+
.strip_prefix(EXTRACTED_EXPR_PREFIX)
152+
.and_then(|s| s.strip_prefix('_'))
153+
{
154+
if let Ok(id) = id_str.parse::<usize>() {
155+
alias_generator.update_min_id(id);
156+
}
157+
}
158+
}
159+
Ok(TreeNodeRecursion::Continue)
160+
})
161+
.ok();
162+
}
163+
}
164+
134165
/// Extracts `MoveTowardsLeafNodes` sub-expressions from a plan node.
135166
///
136167
/// Works for any number of inputs (0, 1, 2, …N). For multi-input nodes
@@ -3019,4 +3050,45 @@ mod tests {
30193050

30203051
Ok(())
30213052
}
3053+
3054+
/// Regression test: user-provided `__datafusion_extracted_2` alias in a
3055+
/// Projection must not collide with optimizer-generated aliases.
3056+
/// See <https://github.com/apache/datafusion/issues/20430>
3057+
#[test]
3058+
fn test_user_provided_extracted_alias_no_collision() -> Result<()> {
3059+
let table_scan = test_table_scan_with_struct()?;
3060+
3061+
// Projection with a user-provided __datafusion_extracted_2 alias
3062+
let proj = LogicalPlanBuilder::from(table_scan)
3063+
.project(vec![
3064+
leaf_udf(col("user"), "status")
3065+
.alias(format!("{EXTRACTED_EXPR_PREFIX}_2")),
3066+
col("id"),
3067+
])?
3068+
.build()?;
3069+
3070+
// Filter below will trigger extraction; the generator must skip past _2
3071+
let plan = LogicalPlanBuilder::from(proj)
3072+
.filter(
3073+
leaf_udf(col("id"), "check").eq(lit(1)),
3074+
)?
3075+
.build()?;
3076+
3077+
let ctx = OptimizerContext::new().with_max_passes(1);
3078+
let optimizer = Optimizer::with_rules(vec![
3079+
Arc::new(OptimizeProjections::new()),
3080+
Arc::new(ExtractLeafExpressions::new()),
3081+
]);
3082+
// Should not error with "duplicate unqualified field name"
3083+
let result = optimizer.optimize(plan, &ctx, |_, _| {})?;
3084+
3085+
// Verify the generated alias is _3 or higher (not _2)
3086+
let plan_str = format!("{result}");
3087+
assert!(
3088+
!plan_str.contains("__datafusion_extracted_2, __datafusion_extracted_2"),
3089+
"Found duplicate alias in plan:\n{plan_str}"
3090+
);
3091+
3092+
Ok(())
3093+
}
30223094
}

‎datafusion/sqllogictest/test_files/projection_pushdown.slt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1949,3 +1949,16 @@ ORDER BY simple_struct.id;
19491949
3 3
19501950
4 4
19511951
5 5
1952+
1953+
# =========================================================================
1954+
# Regression: user-provided __datafusion_extracted aliases must not
1955+
# collide with optimizer-generated ones
1956+
# (https://github.com/apache/datafusion/issues/20430)
1957+
# =========================================================================
1958+
1959+
query I
1960+
SELECT s['value'] AS __datafusion_extracted_2
1961+
FROM simple_struct
1962+
WHERE COALESCE(s['value'], s['value']) = 100;
1963+
----
1964+
100

0 commit comments

Comments
 (0)