Skip to content
Prev Previous commit
Next Next commit
more tests including type mismatch, operators and NULL
Signed-off-by: Ruihang Xia <[email protected]>
  • Loading branch information
waynexia committed Dec 25, 2025
commit 07e23bd0303dac19e7b26f6138993183511041de
95 changes: 93 additions & 2 deletions datafusion/core/tests/set_comparison.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@

use std::sync::Arc;

use arrow::array::Int32Array;
use arrow::array::{Int32Array, StringArray};
use arrow::datatypes::{DataType, Field, Schema};
use arrow::record_batch::RecordBatch;
use datafusion::prelude::SessionContext;
use datafusion_common::{Result, assert_batches_eq};
use datafusion_common::{Result, assert_batches_eq, assert_contains};

fn build_table(values: &[i32]) -> Result<RecordBatch> {
let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int32, true)]));
Expand Down Expand Up @@ -82,3 +82,94 @@ async fn set_comparison_all_empty() -> Result<()> {
);
Ok(())
}

#[tokio::test]
async fn set_comparison_type_mismatch() -> Result<()> {
let ctx = SessionContext::new();

ctx.register_batch("t", build_table(&[1])?)?;
ctx.register_batch("strings", {
let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, true)]));
let array = Arc::new(StringArray::from(vec![Some("a"), Some("b")]))
as Arc<dyn arrow::array::Array>;
RecordBatch::try_new(schema, vec![array])?
})?;

let df = ctx
.sql("select v from t where v > any(select s from strings)")
.await?;
let err = df.collect().await.unwrap_err();
assert_contains!(
err.to_string(),
"expr type Int32 can't cast to Utf8 in SetComparison"
);
Ok(())
}

#[tokio::test]
async fn set_comparison_multiple_operators() -> Result<()> {
let ctx = SessionContext::new();

ctx.register_batch("t", build_table(&[1, 2, 3, 4])?)?;
ctx.register_batch("s", build_table(&[2, 3])?)?;

let df = ctx
.sql("select v from t where v = any(select v from s) order by v")
.await?;
let results = df.collect().await?;
assert_batches_eq!(
&["+---+", "| v |", "+---+", "| 2 |", "| 3 |", "+---+",],
&results
);

let df = ctx
.sql("select v from t where v != all(select v from s) order by v")
.await?;
let results = df.collect().await?;
assert_batches_eq!(
&["+---+", "| v |", "+---+", "| 1 |", "| 4 |", "+---+",],
&results
);

let df = ctx
.sql("select v from t where v >= all(select v from s) order by v")
.await?;
let results = df.collect().await?;
assert_batches_eq!(
&["+---+", "| v |", "+---+", "| 3 |", "| 4 |", "+---+",],
&results
);

let df = ctx
.sql("select v from t where v <= any(select v from s) order by v")
.await?;
let results = df.collect().await?;
assert_batches_eq!(
&[
"+---+", "| v |", "+---+", "| 1 |", "| 2 |", "| 3 |", "+---+",
],
&results
);
Ok(())
}

#[tokio::test]
async fn set_comparison_null_semantics_all() -> Result<()> {
let ctx = SessionContext::new();

ctx.register_batch("t", build_table(&[5])?)?;
ctx.register_batch("s", {
let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int32, true)]));
let array = Arc::new(Int32Array::from(vec![Some(1), None]))
as Arc<dyn arrow::array::Array>;
RecordBatch::try_new(schema, vec![array])?
})?;

let df = ctx
.sql("select v from t where v != all(select v from s)")
.await?;
let results = df.collect().await?;
let row_count: usize = results.iter().map(|batch| batch.num_rows()).sum();
assert_eq!(0, row_count);
Ok(())
}
9 changes: 9 additions & 0 deletions datafusion/optimizer/src/analyzer/type_coercion.rs
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,15 @@ impl TreeNodeRewriter for TypeCoercionRewriter<'_> {
.data;
let expr_type = expr.get_type(self.schema)?;
let subquery_type = new_plan.schema().field(0).data_type();
if (expr_type.is_numeric()
&& is_utf8_or_utf8view_or_large_utf8(subquery_type))
|| (subquery_type.is_numeric()
&& is_utf8_or_utf8view_or_large_utf8(&expr_type))
{
return plan_err!(
"expr type {expr_type} can't cast to {subquery_type} in SetComparison"
);
}
let common_type = comparison_coercion(&expr_type, subquery_type).ok_or(
plan_datafusion_err!(
"expr type {expr_type} can't cast to {subquery_type} in SetComparison"
Expand Down
22 changes: 11 additions & 11 deletions datafusion/sql/src/unparser/expr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,27 +25,27 @@ use sqlparser::ast::{
use std::sync::Arc;
use std::vec;

use super::dialect::IntervalStyle;
use super::Unparser;
use super::dialect::IntervalStyle;
use arrow::array::{
ArrayRef, Date32Array, Date64Array, PrimitiveArray,
types::{
ArrowTemporalType, Time32MillisecondType, Time32SecondType,
Time64MicrosecondType, Time64NanosecondType, TimestampMicrosecondType,
TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType,
},
ArrayRef, Date32Array, Date64Array, PrimitiveArray,
};
use arrow::datatypes::{
DataType, Decimal128Type, Decimal256Type, Decimal32Type, Decimal64Type, DecimalType,
DataType, Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, DecimalType,
};
use arrow::util::display::array_value_to_string;
use datafusion_common::{
assert_eq_or_internal_err, assert_or_internal_err, internal_datafusion_err,
internal_err, not_impl_err, plan_err, Column, Result, ScalarValue,
Column, Result, ScalarValue, assert_eq_or_internal_err, assert_or_internal_err,
internal_datafusion_err, internal_err, not_impl_err, plan_err,
};
use datafusion_expr::{
expr::{Alias, Exists, InList, ScalarFunction, SetQuantifier, Sort, WindowFunction},
Between, BinaryExpr, Case, Cast, Expr, GroupingSet, Like, Operator, TryCast,
expr::{Alias, Exists, InList, ScalarFunction, SetQuantifier, Sort, WindowFunction},
};
use sqlparser::ast::helpers::attached_token::AttachedToken;
use sqlparser::tokenizer::Span;
Expand Down Expand Up @@ -1831,12 +1831,12 @@ mod tests {
use datafusion_common::{Spans, TableReference};
use datafusion_expr::expr::WildcardOptions;
use datafusion_expr::{
case, cast, col, cube, exists, grouping_set, interval_datetime_lit,
interval_year_month_lit, lit, not, not_exists, out_ref_col, placeholder, rollup,
table_scan, try_cast, when, ColumnarValue, ScalarFunctionArgs, ScalarUDF,
ScalarUDFImpl, Signature, Volatility, WindowFrame, WindowFunctionDefinition,
ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,
Volatility, WindowFrame, WindowFunctionDefinition, case, cast, col, cube, exists,
grouping_set, interval_datetime_lit, interval_year_month_lit, lit, not,
not_exists, out_ref_col, placeholder, rollup, table_scan, try_cast, when,
};
use datafusion_expr::{interval_month_day_nano_lit, ExprFunctionExt};
use datafusion_expr::{ExprFunctionExt, interval_month_day_nano_lit};
use datafusion_functions::datetime::from_unixtime::FromUnixtimeFunc;
use datafusion_functions::expr_fn::{get_field, named_struct};
use datafusion_functions_aggregate::count::count_udaf;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@
// under the License.

use crate::logical_plan::consumer::SubstraitConsumer;
use datafusion::common::{substrait_datafusion_err, substrait_err, DFSchema, Spans};
use datafusion::common::{DFSchema, Spans, substrait_datafusion_err, substrait_err};
use datafusion::logical_expr::expr::{Exists, InSubquery, SetComparison, SetQuantifier};
use datafusion::logical_expr::{Expr, Operator, Subquery};
use std::sync::Arc;
use substrait::proto::expression as substrait_expression;
use substrait::proto::expression::subquery::SubqueryType;
use substrait::proto::expression::subquery::set_comparison::{ComparisonOp, ReductionOp};
use substrait::proto::expression::subquery::set_predicate::PredicateOp;
use substrait::proto::expression::subquery::SubqueryType;

pub async fn from_subquery(
consumer: &impl SubstraitConsumer,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,16 @@ use crate::logical_plan::producer::{
from_union, from_values, from_window, from_window_function, to_substrait_rel,
to_substrait_rex,
};
use datafusion::common::{substrait_err, Column, DFSchemaRef, ScalarValue};
use datafusion::execution::registry::SerializerRegistry;
use datafusion::common::{Column, DFSchemaRef, ScalarValue, substrait_err};
use datafusion::execution::SessionState;
use datafusion::execution::registry::SerializerRegistry;
use datafusion::logical_expr::expr::{
Alias, InList, InSubquery, SetComparison, WindowFunction,
};
use datafusion::logical_expr::{
expr, Aggregate, Between, BinaryExpr, Case, Cast, Distinct, EmptyRelation, Expr,
Extension, Filter, Join, Like, Limit, LogicalPlan, Projection, Repartition, Sort,
SubqueryAlias, TableScan, TryCast, Union, Values, Window,
Aggregate, Between, BinaryExpr, Case, Cast, Distinct, EmptyRelation, Expr, Extension,
Filter, Join, Like, Limit, LogicalPlan, Projection, Repartition, Sort, SubqueryAlias,
TableScan, TryCast, Union, Values, Window, expr,
};
use pbjson_types::Any as ProtoAny;
use substrait::proto::aggregate_rel::Measure;
Expand Down
4 changes: 2 additions & 2 deletions datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ use std::mem::size_of_val;

use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema, TimeUnit};
use datafusion::common::tree_node::Transformed;
use datafusion::common::{not_impl_err, plan_err, DFSchema, DFSchemaRef, Spans};
use datafusion::common::{DFSchema, DFSchemaRef, Spans, not_impl_err, plan_err};
use datafusion::error::Result;
use datafusion::execution::registry::SerializerRegistry;
use datafusion::execution::runtime_env::RuntimeEnv;
Expand All @@ -46,7 +46,7 @@ use std::hash::Hash;
use std::sync::Arc;
use substrait::proto::extensions::simple_extension_declaration::MappingType;
use substrait::proto::rel::RelType;
use substrait::proto::{plan_rel, Plan, Rel};
use substrait::proto::{Plan, Rel, plan_rel};

#[derive(Debug)]
struct MockSerializerRegistry;
Expand Down
Loading