Skip to content

Commit e0677c4

Browse files
cetra3claude
andcommitted
fix: column ids, stage 9: outer references two levels up, no allowlist
Targeted id-based fixes for #15558 (substitute through an equality of the middle subquery) and #21727 (flatten EXISTS or IN into EXISTS or IN), to rules slated for replacement by #5492. Shapes these do not cover now error with a stated reason. Remove the verifier's allowlist. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 4b46e30 commit e0677c4

10 files changed

Lines changed: 1318 additions & 121 deletions

File tree

‎datafusion/optimizer/src/column_id_check.rs‎

Lines changed: 14 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,7 @@
2626
//! [`Stale`](ColumnIdIssueKind::Stale) references than its
2727
//! input (which is the output of the planner, or a plan optimized before
2828
//! for a prepared statement);
29-
//! - after an optimizer rule, if it introduced issues of a kind it may not
30-
//! introduce: a rule in [`UNMIGRATED_RULES`] may introduce issues of the
31-
//! kind listed with it (a `Missing` or `Stale` reference), any other
32-
//! rule nothing.
29+
//! - after an optimizer rule, if it introduced any issue.
3330
//!
3431
//! A plan without any id (built without a generator) is not checked: all
3532
//! its references are `Missing`, as before ids. Neither is the output of
@@ -50,33 +47,6 @@ use datafusion_common::{Result, internal_err};
5047
use datafusion_expr::{ColumnIdIssueKind, LogicalPlan};
5148
use log::{Level, debug, log_enabled};
5249

53-
/// The rules that may still introduce column id issues, each with the one
54-
/// kind of issue it may introduce. Every other rule may introduce none;
55-
/// remove a rule once it keeps its ids, CI then fails if it regresses.
56-
///
57-
/// The rules listed are migrated: the issues they may introduce come from
58-
/// plans that are wrong without ids (on `main`), which the decorrelation
59-
/// rules build and the unnesting rule of #5492 replaces. See the
60-
/// [module documentation](self).
61-
///
62-
/// The analyzer is checked as a whole, after all its rules, and may not
63-
/// introduce any issue.
64-
pub const UNMIGRATED_RULES: &[(&str, ColumnIdIssueKind)] = &[
65-
// decorrelating a subquery that reads a column two levels up (an
66-
// `EXISTS` inside an `EXISTS` or a scalar subquery) leaves the
67-
// reference as a plain column in a join filter where it is not in scope,
68-
// as on `main` (merge_into.slt, explain-only tests in subquery.slt; the
69-
// SELECT form fails on `main` with "No field named t.id"): the
70-
// reference has the right id, which is stale there
71-
("decorrelate_predicate_subquery", ColumnIdIssueKind::Stale),
72-
// a subquery nested in a correlated scalar subquery that reads a column
73-
// two levels up (an `EXISTS` inside a scalar subquery) ends up reading
74-
// it as a plain column in a join filter where it is not in scope, as on
75-
// `main` (explain-only tests in subquery.slt): the reference has the
76-
// right id, which is stale there
77-
("scalar_subquery_to_join", ColumnIdIssueKind::Stale),
78-
];
79-
8050
/// The log target of the reports
8151
const LOG_TARGET: &str = "datafusion::column_id";
8252

@@ -109,11 +79,6 @@ static VERIFY: LazyLock<bool> = LazyLock::new(|| {
10979
/// The number of issues of each kind, indexed like [`ColumnIdIssueKind::ALL`]
11080
type Counts = [usize; ColumnIdIssueKind::ALL.len()];
11181

112-
/// Whether `rule` may introduce issues of `kind` (see [`UNMIGRATED_RULES`])
113-
fn may_introduce(rule: &str, kind: ColumnIdIssueKind) -> bool {
114-
UNMIGRATED_RULES.contains(&(rule, kind))
115-
}
116-
11782
/// The kinds of issues that a plan may not have after the analyzer at all
11883
const AFTER_ANALYZER_FORBIDDEN: [ColumnIdIssueKind; 2] = [
11984
ColumnIdIssueKind::Disagree,
@@ -186,7 +151,7 @@ impl ColumnIdCheck {
186151

187152
/// Records the issues that `rule` introduced, `plan` being its output,
188153
/// and returns an error if verification is on and `rule` introduced
189-
/// issues of a kind it may not introduce (see [`UNMIGRATED_RULES`])
154+
/// issues
190155
pub(crate) fn after_rule(&mut self, rule: &str, plan: &LogicalPlan) -> Result<()> {
191156
let before = match &self.before {
192157
Some(before) => *before,
@@ -200,10 +165,7 @@ impl ColumnIdCheck {
200165
}
201166
let failed: Vec<_> = ColumnIdIssueKind::ALL
202167
.into_iter()
203-
.filter(|kind| {
204-
after[*kind as usize] > before[*kind as usize]
205-
&& !may_introduce(rule, *kind)
206-
})
168+
.filter(|kind| after[*kind as usize] > before[*kind as usize])
207169
.collect();
208170
fail_if_any(
209171
&format!(
@@ -400,22 +362,22 @@ mod tests {
400362
}
401363

402364
#[test]
403-
fn listed_rules_may_lose_ids() -> Result<()> {
365+
fn no_rule_may_lose_ids() -> Result<()> {
404366
let plan = scan(true)?;
367+
// a stale reference (`t.a#9` resolves by name to a field without an
368+
// id), also from a decorrelation rule
405369
let mut check = ColumnIdCheck::new(&plan);
406-
// a listed rule adds a stale reference (`t.a#9` resolves by name to
407-
// a field without an id), the kind listed with it
408-
check.after_rule("scalar_subquery_to_join", &filter(scan(false)?, Some(9))?)?;
409-
// but not a missing one
410370
let err = check
411-
.after_rule(
412-
"scalar_subquery_to_join",
413-
&filter(filter(scan(false)?, Some(9))?, None)?,
414-
)
371+
.after_rule("scalar_subquery_to_join", &filter(scan(false)?, Some(9))?)
415372
.unwrap_err();
416-
assert!(err.to_string().contains("kinds missing"), "{err}");
373+
assert!(
374+
err.to_string().contains(
375+
"optimizer rule 'scalar_subquery_to_join' introduced column id issues of kinds stale"
376+
),
377+
"{err}"
378+
);
417379

418-
// another rule may not
380+
// a missing one
419381
let mut check = ColumnIdCheck::new(&plan);
420382
let err = check
421383
.after_rule("my_rule", &filter(scan(true)?, None)?)
@@ -488,19 +450,4 @@ mod tests {
488450
let mut check = ColumnIdCheck::new(&plan);
489451
check.after_rule("my_rule", &filter(scan(false)?, None)?)
490452
}
491-
492-
#[test]
493-
fn unmigrated_rules_exist() {
494-
let optimizer = crate::Optimizer::new();
495-
let analyzer = crate::Analyzer::new();
496-
let names: Vec<&str> = optimizer
497-
.rules
498-
.iter()
499-
.map(|rule| rule.name())
500-
.chain(analyzer.rules.iter().map(|rule| rule.name()))
501-
.collect();
502-
for (rule, _) in UNMIGRATED_RULES {
503-
assert!(names.contains(rule), "{rule} is not a rule");
504-
}
505-
}
506453
}

0 commit comments

Comments
 (0)