-
-
Save neilconway/f956e8ecec36a2979fdb799036dd1ea4 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| diff --git a/datafusion/sql/src/unparser/ast.rs b/datafusion/sql/src/unparser/ast.rs | |
| index 7f4ad9a05..bc1f654e7 100644 | |
| --- a/datafusion/sql/src/unparser/ast.rs | |
| +++ b/datafusion/sql/src/unparser/ast.rs | |
| @@ -34,8 +34,6 @@ pub struct QueryBuilder { | |
| fetch: Option<ast::Fetch>, | |
| locks: Vec<ast::LockClause>, | |
| for_clause: Option<ast::ForClause>, | |
| - // If true, we need to unparse LogicalPlan::Union as a SQL `UNION` rather than a `UNION ALL`. | |
| - distinct_union: bool, | |
| } | |
| impl QueryBuilder { | |
| @@ -78,13 +76,6 @@ impl QueryBuilder { | |
| self.for_clause = value; | |
| self | |
| } | |
| - pub fn distinct_union(&mut self) -> &mut Self { | |
| - self.distinct_union = true; | |
| - self | |
| - } | |
| - pub fn is_distinct_union(&self) -> bool { | |
| - self.distinct_union | |
| - } | |
| /// Whether this query carries clauses that are scoped to a single query | |
| /// rather than to a set operation. When such a query is used as an operand | |
| /// of a set operation (e.g. a branch of a `UNION`), it must be wrapped in | |
| @@ -140,7 +131,6 @@ impl QueryBuilder { | |
| fetch: Default::default(), | |
| locks: Default::default(), | |
| for_clause: Default::default(), | |
| - distinct_union: false, | |
| } | |
| } | |
| } | |
| diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs | |
| index 108b38b64..738a18c47 100644 | |
| --- a/datafusion/sql/src/unparser/plan.rs | |
| +++ b/datafusion/sql/src/unparser/plan.rs | |
| @@ -51,7 +51,7 @@ use datafusion_expr::expr::{OUTER_REFERENCE_COLUMN_PREFIX, UNNEST_COLUMN_PREFIX} | |
| use datafusion_expr::{ | |
| Aggregate, BinaryExpr, Distinct, Expr, FetchType, JoinConstraint, JoinType, | |
| LogicalPlan, LogicalPlanBuilder, Operator, Projection, SkipType, Sort, SortExpr, | |
| - TableScan, Unnest, UserDefinedLogicalNode, Window, expr::Alias, | |
| + TableScan, Union, Unnest, UserDefinedLogicalNode, Window, expr::Alias, | |
| }; | |
| use sqlparser::ast::{self, Ident, OrderByKind, SetExpr, TableAliasColumnDef}; | |
| use std::{sync::Arc, vec}; | |
| @@ -719,6 +719,86 @@ impl Unparser<'_> { | |
| } | |
| } | |
| + /// Unparses `union` as a chain of SQL set operations, one `UNION` per | |
| + /// input, and installs the result as the body of `query`. `distinct` | |
| + /// selects the set quantifier: a bare `UNION` (duplicates removed) when | |
| + /// true, `UNION ALL` when false. The planner encodes a SQL `UNION` as | |
| + /// `Distinct(Union)` and `UNION ALL` as a bare `Union`, so the `Distinct` | |
| + /// and `Union` arms of [`Self::select_to_sql_recursively`] each call this | |
| + /// with the quantifier implied by the node they matched. | |
| + fn union_to_sql( | |
| + &self, | |
| + union: &Union, | |
| + distinct: bool, | |
| + query: &mut Option<QueryBuilder>, | |
| + ) -> Result<()> { | |
| + // Each branch is unparsed in its own query context so that clauses | |
| + // scoped to the branch (`ORDER BY`, `LIMIT`, `OFFSET`, ...) stay with | |
| + // the branch instead of accumulating on the enclosing statement. | |
| + // Branches that are themselves set operations, or that carry such | |
| + // clauses, are parenthesized: sqlparser renders nested | |
| + // `SetExpr::SetOperation`s without parentheses, so e.g. | |
| + // `a UNION ALL (b UNION c LIMIT 1)` would otherwise collapse into | |
| + // `a UNION b UNION c LIMIT 1`, changing both the set quantifier and | |
| + // the scope of the LIMIT. | |
| + let input_exprs: Vec<SetExpr> = union | |
| + .inputs | |
| + .iter() | |
| + .map(|input| { | |
| + let mut branch_query = Some(QueryBuilder::default()); | |
| + let body = self.select_to_sql_expr(input, &mut branch_query)?; | |
| + | |
| + // Inline a branch only when it is a plain SELECT with no | |
| + // query-scoped clauses; otherwise wrap it in a | |
| + // parenthesized subquery so its set quantifier and | |
| + // clauses stay bound to the branch. | |
| + match branch_query { | |
| + Some(mut branch_query) | |
| + if !matches!(body, SetExpr::Select(_)) | |
| + || branch_query.has_operand_scoped_clauses() => | |
| + { | |
| + let query = branch_query.body(Box::new(body)).build()?; | |
| + Ok(SetExpr::Query(Box::new(query))) | |
| + } | |
| + _ => Ok(body), | |
| + } | |
| + }) | |
| + .collect::<Result<Vec<_>>>()?; | |
| + | |
| + assert_or_internal_err!( | |
| + input_exprs.len() >= 2, | |
| + "UNION operator requires at least 2 inputs" | |
| + ); | |
| + | |
| + let set_quantifier = if distinct { | |
| + // `SetQuantifier::None` renders as a bare `UNION`, which removes | |
| + // duplicates; `All` renders `UNION ALL`. | |
| + ast::SetQuantifier::None | |
| + } else { | |
| + ast::SetQuantifier::All | |
| + }; | |
| + | |
| + // Build the union expression tree bottom-up by reversing the order | |
| + // note that we are also swapping left and right inputs because of the rev | |
| + let union_expr = input_exprs | |
| + .into_iter() | |
| + .rev() | |
| + .reduce(|a, b| SetExpr::SetOperation { | |
| + op: ast::SetOperator::Union, | |
| + set_quantifier, | |
| + left: Box::new(b), | |
| + right: Box::new(a), | |
| + }) | |
| + .unwrap(); | |
| + | |
| + let Some(query) = query.as_mut() else { | |
| + return internal_err!("UNION operator only valid in a statement context"); | |
| + }; | |
| + query.body(Box::new(union_expr)); | |
| + | |
| + Ok(()) | |
| + } | |
| + | |
| #[cfg_attr(feature = "recursive_protection", recursive::recursive)] | |
| fn select_to_sql_recursively( | |
| &self, | |
| @@ -1251,19 +1331,13 @@ impl Unparser<'_> { | |
| ); | |
| } | |
| - // If this distinct is the parent of a Union and we're in a query context, | |
| - // then we need to unparse as a `UNION` rather than a `UNION ALL`. | |
| + // The planner encodes a SQL `UNION` (as opposed to `UNION ALL`) | |
| + // as `Distinct(Union)`, so unparse that shape as a distinct set | |
| + // operation rather than as `SELECT DISTINCT` over a `UNION ALL`. | |
| if let Distinct::All(input) = distinct | |
| - && matches!(input.as_ref(), LogicalPlan::Union(_)) | |
| - && let Some(query_mut) = query.as_mut() | |
| + && let LogicalPlan::Union(union) = input.as_ref() | |
| { | |
| - query_mut.distinct_union(); | |
| - return self.select_to_sql_recursively( | |
| - input.as_ref(), | |
| - query, | |
| - select, | |
| - relation, | |
| - ); | |
| + return self.union_to_sql(union, true, query); | |
| } | |
| let (select_distinct, input) = match distinct { | |
| @@ -1591,80 +1665,7 @@ impl Unparser<'_> { | |
| ); | |
| } | |
| - // Each UNION branch is unparsed in its own isolated query | |
| - // context. A branch is inlined only when it is a plain SELECT | |
| - // with no query-scoped clauses; otherwise it is wrapped in a | |
| - // parenthesized subquery. | |
| - // | |
| - // Sharing this statement's `QueryBuilder` across branches is | |
| - // unsound: its single `distinct_union` flag leaks a nested | |
| - // distinct `UNION` up to the enclosing `UNION ALL`, and a | |
| - // branch's own `ORDER BY`/`LIMIT`/`OFFSET` would bind to the | |
| - // whole set operation. Combined with sqlparser rendering nested | |
| - // `SetExpr::SetOperation`s without parentheses, this silently | |
| - // rewrites e.g. `a UNION ALL (b UNION c LIMIT 1)` into | |
| - // `a UNION b UNION c LIMIT 1`. Isolating each branch and | |
| - // parenthesizing non-trivial ones preserves precedence, the set | |
| - // quantifier, and operand-scoped clauses. | |
| - let input_exprs: Vec<SetExpr> = union | |
| - .inputs | |
| - .iter() | |
| - .map(|input| { | |
| - let mut branch_query = Some(QueryBuilder::default()); | |
| - let body = self.select_to_sql_expr(input, &mut branch_query)?; | |
| - | |
| - // Inline a branch only when it is a plain SELECT with no | |
| - // query-scoped clauses; otherwise wrap it in a | |
| - // parenthesized subquery so its set quantifier and | |
| - // clauses stay bound to the branch. | |
| - match branch_query { | |
| - Some(mut branch_query) | |
| - if !matches!(body, SetExpr::Select(_)) | |
| - || branch_query.has_operand_scoped_clauses() => | |
| - { | |
| - let query = branch_query.body(Box::new(body)).build()?; | |
| - Ok(SetExpr::Query(Box::new(query))) | |
| - } | |
| - _ => Ok(body), | |
| - } | |
| - }) | |
| - .collect::<Result<Vec<_>>>()?; | |
| - | |
| - assert_or_internal_err!( | |
| - input_exprs.len() >= 2, | |
| - "UNION operator requires at least 2 inputs" | |
| - ); | |
| - | |
| - let set_quantifier = | |
| - if query.as_ref().is_some_and(|q| q.is_distinct_union()) { | |
| - // Setting the SetQuantifier to None will unparse as a `UNION` | |
| - // rather than a `UNION ALL`. | |
| - ast::SetQuantifier::None | |
| - } else { | |
| - ast::SetQuantifier::All | |
| - }; | |
| - | |
| - // Build the union expression tree bottom-up by reversing the order | |
| - // note that we are also swapping left and right inputs because of the rev | |
| - let union_expr = input_exprs | |
| - .into_iter() | |
| - .rev() | |
| - .reduce(|a, b| SetExpr::SetOperation { | |
| - op: ast::SetOperator::Union, | |
| - set_quantifier, | |
| - left: Box::new(b), | |
| - right: Box::new(a), | |
| - }) | |
| - .unwrap(); | |
| - | |
| - let Some(query) = query.as_mut() else { | |
| - return internal_err!( | |
| - "UNION ALL operator only valid in a statement context" | |
| - ); | |
| - }; | |
| - query.body(Box::new(union_expr)); | |
| - | |
| - Ok(()) | |
| + self.union_to_sql(union, false, query) | |
| } | |
| LogicalPlan::Window(window) => { | |
| // Window nodes are usually handled simultaneously with Projection | |
| diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md | |
| index ef2785edb..eba489c4b 100644 | |
| --- a/docs/source/library-user-guide/upgrading/55.0.0.md | |
| +++ b/docs/source/library-user-guide/upgrading/55.0.0.md | |
| @@ -1408,3 +1408,13 @@ let table_opts = TableParquetOptions::try_from(&proto_table_opts)?; | |
| ``` | |
| See [issue #24019](https://github.com/apache/datafusion/issues/24019) for details. | |
| + | |
| +### `QueryBuilder::distinct_union` and `QueryBuilder::is_distinct_union` removed | |
| + | |
| +`datafusion_sql::unparser::ast::QueryBuilder` no longer carries a | |
| +`distinct_union` flag. The unparser now chooses between `UNION` and `UNION ALL` | |
| +from the shape of the logical plan (`Distinct(Union)` vs. `Union`) at the point | |
| +where it renders the set operation, rather than by setting a flag on the | |
| +`QueryBuilder` that a later step reads back. Only the built-in unparser ever | |
| +read the flag, so there is nothing to migrate: `plan_to_sql` produces the same | |
| +SQL as before. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment