Skip to content

Instantly share code, notes, and snippets.

@neilconway
Created August 20, 2026 13:57
Show Gist options
  • Select an option

  • Save neilconway/f956e8ecec36a2979fdb799036dd1ea4 to your computer and use it in GitHub Desktop.

Select an option

Save neilconway/f956e8ecec36a2979fdb799036dd1ea4 to your computer and use it in GitHub Desktop.
diff --git a/datafusion/sql/src/unparser/ast.rs b/datafusion/sql/src/unparser/ast.rs
index 7f4ad9a05..bc1f654e7 100644
--- a/datafusion/sql/src/unparser/ast.rs
+++ b/datafusion/sql/src/unparser/ast.rs
@@ -34,8 +34,6 @@ pub struct QueryBuilder {
fetch: Option<ast::Fetch>,
locks: Vec<ast::LockClause>,
for_clause: Option<ast::ForClause>,
- // If true, we need to unparse LogicalPlan::Union as a SQL `UNION` rather than a `UNION ALL`.
- distinct_union: bool,
}
impl QueryBuilder {
@@ -78,13 +76,6 @@ impl QueryBuilder {
self.for_clause = value;
self
}
- pub fn distinct_union(&mut self) -> &mut Self {
- self.distinct_union = true;
- self
- }
- pub fn is_distinct_union(&self) -> bool {
- self.distinct_union
- }
/// Whether this query carries clauses that are scoped to a single query
/// rather than to a set operation. When such a query is used as an operand
/// of a set operation (e.g. a branch of a `UNION`), it must be wrapped in
@@ -140,7 +131,6 @@ impl QueryBuilder {
fetch: Default::default(),
locks: Default::default(),
for_clause: Default::default(),
- distinct_union: false,
}
}
}
diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs
index 108b38b64..738a18c47 100644
--- a/datafusion/sql/src/unparser/plan.rs
+++ b/datafusion/sql/src/unparser/plan.rs
@@ -51,7 +51,7 @@ use datafusion_expr::expr::{OUTER_REFERENCE_COLUMN_PREFIX, UNNEST_COLUMN_PREFIX}
use datafusion_expr::{
Aggregate, BinaryExpr, Distinct, Expr, FetchType, JoinConstraint, JoinType,
LogicalPlan, LogicalPlanBuilder, Operator, Projection, SkipType, Sort, SortExpr,
- TableScan, Unnest, UserDefinedLogicalNode, Window, expr::Alias,
+ TableScan, Union, Unnest, UserDefinedLogicalNode, Window, expr::Alias,
};
use sqlparser::ast::{self, Ident, OrderByKind, SetExpr, TableAliasColumnDef};
use std::{sync::Arc, vec};
@@ -719,6 +719,86 @@ impl Unparser<'_> {
}
}
+ /// Unparses `union` as a chain of SQL set operations, one `UNION` per
+ /// input, and installs the result as the body of `query`. `distinct`
+ /// selects the set quantifier: a bare `UNION` (duplicates removed) when
+ /// true, `UNION ALL` when false. The planner encodes a SQL `UNION` as
+ /// `Distinct(Union)` and `UNION ALL` as a bare `Union`, so the `Distinct`
+ /// and `Union` arms of [`Self::select_to_sql_recursively`] each call this
+ /// with the quantifier implied by the node they matched.
+ fn union_to_sql(
+ &self,
+ union: &Union,
+ distinct: bool,
+ query: &mut Option<QueryBuilder>,
+ ) -> Result<()> {
+ // Each branch is unparsed in its own query context so that clauses
+ // scoped to the branch (`ORDER BY`, `LIMIT`, `OFFSET`, ...) stay with
+ // the branch instead of accumulating on the enclosing statement.
+ // Branches that are themselves set operations, or that carry such
+ // clauses, are parenthesized: sqlparser renders nested
+ // `SetExpr::SetOperation`s without parentheses, so e.g.
+ // `a UNION ALL (b UNION c LIMIT 1)` would otherwise collapse into
+ // `a UNION b UNION c LIMIT 1`, changing both the set quantifier and
+ // the scope of the LIMIT.
+ let input_exprs: Vec<SetExpr> = union
+ .inputs
+ .iter()
+ .map(|input| {
+ let mut branch_query = Some(QueryBuilder::default());
+ let body = self.select_to_sql_expr(input, &mut branch_query)?;
+
+ // Inline a branch only when it is a plain SELECT with no
+ // query-scoped clauses; otherwise wrap it in a
+ // parenthesized subquery so its set quantifier and
+ // clauses stay bound to the branch.
+ match branch_query {
+ Some(mut branch_query)
+ if !matches!(body, SetExpr::Select(_))
+ || branch_query.has_operand_scoped_clauses() =>
+ {
+ let query = branch_query.body(Box::new(body)).build()?;
+ Ok(SetExpr::Query(Box::new(query)))
+ }
+ _ => Ok(body),
+ }
+ })
+ .collect::<Result<Vec<_>>>()?;
+
+ assert_or_internal_err!(
+ input_exprs.len() >= 2,
+ "UNION operator requires at least 2 inputs"
+ );
+
+ let set_quantifier = if distinct {
+ // `SetQuantifier::None` renders as a bare `UNION`, which removes
+ // duplicates; `All` renders `UNION ALL`.
+ ast::SetQuantifier::None
+ } else {
+ ast::SetQuantifier::All
+ };
+
+ // Build the union expression tree bottom-up by reversing the order
+ // note that we are also swapping left and right inputs because of the rev
+ let union_expr = input_exprs
+ .into_iter()
+ .rev()
+ .reduce(|a, b| SetExpr::SetOperation {
+ op: ast::SetOperator::Union,
+ set_quantifier,
+ left: Box::new(b),
+ right: Box::new(a),
+ })
+ .unwrap();
+
+ let Some(query) = query.as_mut() else {
+ return internal_err!("UNION operator only valid in a statement context");
+ };
+ query.body(Box::new(union_expr));
+
+ Ok(())
+ }
+
#[cfg_attr(feature = "recursive_protection", recursive::recursive)]
fn select_to_sql_recursively(
&self,
@@ -1251,19 +1331,13 @@ impl Unparser<'_> {
);
}
- // If this distinct is the parent of a Union and we're in a query context,
- // then we need to unparse as a `UNION` rather than a `UNION ALL`.
+ // The planner encodes a SQL `UNION` (as opposed to `UNION ALL`)
+ // as `Distinct(Union)`, so unparse that shape as a distinct set
+ // operation rather than as `SELECT DISTINCT` over a `UNION ALL`.
if let Distinct::All(input) = distinct
- && matches!(input.as_ref(), LogicalPlan::Union(_))
- && let Some(query_mut) = query.as_mut()
+ && let LogicalPlan::Union(union) = input.as_ref()
{
- query_mut.distinct_union();
- return self.select_to_sql_recursively(
- input.as_ref(),
- query,
- select,
- relation,
- );
+ return self.union_to_sql(union, true, query);
}
let (select_distinct, input) = match distinct {
@@ -1591,80 +1665,7 @@ impl Unparser<'_> {
);
}
- // Each UNION branch is unparsed in its own isolated query
- // context. A branch is inlined only when it is a plain SELECT
- // with no query-scoped clauses; otherwise it is wrapped in a
- // parenthesized subquery.
- //
- // Sharing this statement's `QueryBuilder` across branches is
- // unsound: its single `distinct_union` flag leaks a nested
- // distinct `UNION` up to the enclosing `UNION ALL`, and a
- // branch's own `ORDER BY`/`LIMIT`/`OFFSET` would bind to the
- // whole set operation. Combined with sqlparser rendering nested
- // `SetExpr::SetOperation`s without parentheses, this silently
- // rewrites e.g. `a UNION ALL (b UNION c LIMIT 1)` into
- // `a UNION b UNION c LIMIT 1`. Isolating each branch and
- // parenthesizing non-trivial ones preserves precedence, the set
- // quantifier, and operand-scoped clauses.
- let input_exprs: Vec<SetExpr> = union
- .inputs
- .iter()
- .map(|input| {
- let mut branch_query = Some(QueryBuilder::default());
- let body = self.select_to_sql_expr(input, &mut branch_query)?;
-
- // Inline a branch only when it is a plain SELECT with no
- // query-scoped clauses; otherwise wrap it in a
- // parenthesized subquery so its set quantifier and
- // clauses stay bound to the branch.
- match branch_query {
- Some(mut branch_query)
- if !matches!(body, SetExpr::Select(_))
- || branch_query.has_operand_scoped_clauses() =>
- {
- let query = branch_query.body(Box::new(body)).build()?;
- Ok(SetExpr::Query(Box::new(query)))
- }
- _ => Ok(body),
- }
- })
- .collect::<Result<Vec<_>>>()?;
-
- assert_or_internal_err!(
- input_exprs.len() >= 2,
- "UNION operator requires at least 2 inputs"
- );
-
- let set_quantifier =
- if query.as_ref().is_some_and(|q| q.is_distinct_union()) {
- // Setting the SetQuantifier to None will unparse as a `UNION`
- // rather than a `UNION ALL`.
- ast::SetQuantifier::None
- } else {
- ast::SetQuantifier::All
- };
-
- // Build the union expression tree bottom-up by reversing the order
- // note that we are also swapping left and right inputs because of the rev
- let union_expr = input_exprs
- .into_iter()
- .rev()
- .reduce(|a, b| SetExpr::SetOperation {
- op: ast::SetOperator::Union,
- set_quantifier,
- left: Box::new(b),
- right: Box::new(a),
- })
- .unwrap();
-
- let Some(query) = query.as_mut() else {
- return internal_err!(
- "UNION ALL operator only valid in a statement context"
- );
- };
- query.body(Box::new(union_expr));
-
- Ok(())
+ self.union_to_sql(union, false, query)
}
LogicalPlan::Window(window) => {
// Window nodes are usually handled simultaneously with Projection
diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md
index ef2785edb..eba489c4b 100644
--- a/docs/source/library-user-guide/upgrading/55.0.0.md
+++ b/docs/source/library-user-guide/upgrading/55.0.0.md
@@ -1408,3 +1408,13 @@ let table_opts = TableParquetOptions::try_from(&proto_table_opts)?;
```
See [issue #24019](https://github.com/apache/datafusion/issues/24019) for details.
+
+### `QueryBuilder::distinct_union` and `QueryBuilder::is_distinct_union` removed
+
+`datafusion_sql::unparser::ast::QueryBuilder` no longer carries a
+`distinct_union` flag. The unparser now chooses between `UNION` and `UNION ALL`
+from the shape of the logical plan (`Distinct(Union)` vs. `Union`) at the point
+where it renders the set operation, rather than by setting a flag on the
+`QueryBuilder` that a later step reads back. Only the built-in unparser ever
+read the flag, so there is nothing to migrate: `plan_to_sql` produces the same
+SQL as before.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment