Skip to content

Commit 070951d

Browse files
authored
feat: convert OneRowRelation to Arrow by default (apache#6666)
* feat: convert OneRowRelation to Arrow by default Turn on spark.comet.convert.oneRowRelation.enabled by default. The row a query without a FROM clause reads has no columns, so converting it costs almost nothing, and it lets the operators above it run in Comet. Where the parent cannot run in Comet, EliminateRedundantTransitions drops the conversion again, so no extra transition is added. On Spark 4.2 the TPC-DS q77a golden now records the remaining blocker, the struct-typed scalar subquery result the projection above each OneRowRelation reads (apache#5834), instead of the unsupported leaf. * fix: handle expressions exposed by converting OneRowRelation by default Folded array literals are UnsafeArrayData and have no backing array, so serialize them through toArray. might_contain accepts an untyped NULL for either argument. parse_url raises Spark's INVALID_URL error under ANSI instead of a generic native error. An Expand with no output columns falls back to Spark. * fix: decline codegen dispatch for expressions that contain a subquery * test: adapt Spark explain tests to the Comet plan for OneRowRelation queries
1 parent 8d8d1a2 commit 070951d

26 files changed

Lines changed: 300 additions & 43 deletions

File tree

‎dev/diffs/3.4.3.diff‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -593,7 +593,7 @@ index f33432ddb6f..b375e285dde 100644
593593
}
594594
assert(scanOption.isDefined)
595595
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
596-
index a6b295578d6..1167bbe6554 100644
596+
index a6b295578d6..1fa898bf840 100644
597597
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
598598
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
599599
@@ -260,7 +260,8 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
@@ -606,6 +606,15 @@ index a6b295578d6..1167bbe6554 100644
606606
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
607607
withTempView("df") {
608608
val df1 = spark.range(1, 100)
609+
@@ -331,7 +332,7 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
610+
611+
val sqlText = "EXPLAIN FORMATTED SELECT (SELECT min(id) FROM df) as v"
612+
val expected_pattern =
613+
- "Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#x"
614+
+ "Subquery:1 Hosting operator id = \\d+ Hosting Expression = Subquery subquery#x"
615+
616+
withNormalizedExplain(sqlText) { normalizedOutput =>
617+
assert(expected_pattern.r.findAllMatchIn(normalizedOutput).length == 1)
609618
@@ -463,7 +464,8 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
610619
}
611620
}

‎dev/diffs/3.5.9.diff‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -574,7 +574,7 @@ index f33432ddb6f..b375e285dde 100644
574574
}
575575
assert(scanOption.isDefined)
576576
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
577-
index 739557bef30..718fbc5b786 100644
577+
index 739557bef30..51682036f38 100644
578578
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
579579
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
580580
@@ -265,7 +265,8 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
@@ -587,6 +587,15 @@ index 739557bef30..718fbc5b786 100644
587587
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
588588
withTempView("df") {
589589
val df1 = spark.range(1, 100)
590+
@@ -336,7 +337,7 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
591+
592+
val sqlText = "EXPLAIN FORMATTED SELECT (SELECT min(id) FROM df) as v"
593+
val expected_pattern =
594+
- "Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#x"
595+
+ "Subquery:1 Hosting operator id = \\d+ Hosting Expression = Subquery subquery#x"
596+
597+
withNormalizedExplain(sqlText) { normalizedOutput =>
598+
assert(expected_pattern.r.findAllMatchIn(normalizedOutput).length == 1)
590599
@@ -468,7 +469,8 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
591600
}
592601
}

‎dev/diffs/4.0.4.diff‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -711,7 +711,7 @@ index 2c24cc7d570..12d897866da 100644
711711
}
712712
assert(scanOption.isDefined)
713713
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
714-
index ef9c301a66a..49fad8cbbf0 100644
714+
index ef9c301a66a..d97e9f76dc3 100644
715715
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
716716
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
717717
@@ -268,7 +268,8 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
@@ -724,6 +724,15 @@ index ef9c301a66a..49fad8cbbf0 100644
724724
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
725725
withTempView("df") {
726726
val df1 = spark.range(1, 100)
727+
@@ -339,7 +340,7 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
728+
729+
val sqlText = "EXPLAIN FORMATTED SELECT (SELECT min(id) FROM df) as v"
730+
val expected_pattern =
731+
- "Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#x"
732+
+ "Subquery:1 Hosting operator id = \\d+ Hosting Expression = Subquery subquery#x"
733+
734+
withNormalizedExplain(sqlText) { normalizedOutput =>
735+
assert(expected_pattern.r.findAllMatchIn(normalizedOutput).length == 1)
727736
@@ -471,7 +472,8 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
728737
}
729738
}
@@ -1192,7 +1201,7 @@ index ad424b3a7cc..4ece0117a34 100644
11921201
protected val baseResourcePath = {
11931202
// use the same way as `SQLQueryTestSuite` to get the resource path
11941203
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
1195-
index f294ff81021..02d72be8d29 100644
1204+
index f294ff81021..62ca9a9bcd5 100644
11961205
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
11971206
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
11981207
@@ -1524,7 +1524,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
@@ -1222,6 +1231,16 @@ index f294ff81021..02d72be8d29 100644
12221231

12231232
verifyCallCount(
12241233
df.selectExpr("testUdf(a + 1) + testUdf(1 + a)", "testUdf(a + 1)"), Row(4, 2), 1)
1234+
@@ -4744,7 +4752,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
1235+
)
1236+
}
1237+
1238+
- test("SPARK-47939: Explain should work with parameterized queries") {
1239+
+ test("SPARK-47939: Explain should work with parameterized queries",
1240+
+ IgnoreComet("Comet explain output is different")) {
1241+
def checkQueryPlan(df: DataFrame, plan: String): Unit = assert(
1242+
df.collect()
1243+
.map(_.getString(0))
12251244
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
12261245
index 575a4ae69d1..129d9f27232 100644
12271246
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala

‎dev/diffs/4.1.3.diff‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -813,7 +813,7 @@ index e1a2fd33c7c..632f4b695df 100644
813813
}
814814
assert(scanOption.isDefined)
815815
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
816-
index 4c62c47971a..ecc7ed21546 100644
816+
index 4c62c47971a..fe788debf70 100644
817817
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
818818
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
819819
@@ -268,7 +268,8 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
@@ -826,6 +826,15 @@ index 4c62c47971a..ecc7ed21546 100644
826826
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
827827
withTempView("df") {
828828
val df1 = spark.range(1, 100)
829+
@@ -339,7 +340,7 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
830+
831+
val sqlText = "EXPLAIN FORMATTED SELECT (SELECT min(id) FROM df) as v"
832+
val expected_pattern =
833+
- "Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#x"
834+
+ "Subquery:1 Hosting operator id = \\d+ Hosting Expression = Subquery subquery#x"
835+
836+
withNormalizedExplain(sqlText) { normalizedOutput =>
837+
assert(expected_pattern.r.findAllMatchIn(normalizedOutput).length == 1)
829838
@@ -471,7 +472,8 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
830839
}
831840
}
@@ -1313,7 +1322,7 @@ index cb9d0909554..084d6515e8b 100644
13131322
}
13141323

13151324
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
1316-
index 74cdee49e55..f7452c9abb7 100644
1325+
index 74cdee49e55..1a7bc4567b4 100644
13171326
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
13181327
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
13191328
@@ -1521,7 +1521,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
@@ -1343,6 +1352,16 @@ index 74cdee49e55..f7452c9abb7 100644
13431352

13441353
verifyCallCount(
13451354
df.selectExpr("testUdf(a + 1) + testUdf(1 + a)", "testUdf(a + 1)"), Row(4, 2), 1)
1355+
@@ -4741,7 +4749,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
1356+
)
1357+
}
1358+
1359+
- test("SPARK-47939: Explain should work with parameterized queries") {
1360+
+ test("SPARK-47939: Explain should work with parameterized queries",
1361+
+ IgnoreComet("Comet explain output is different")) {
1362+
def checkQueryPlan(df: DataFrame, plan: String): Unit = assert(
1363+
df.collect()
1364+
.map(_.getString(0))
13461365
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
13471366
index 23f0144dcec..40d536bb23a 100644
13481367
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala

‎dev/diffs/4.2.0.diff‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -842,7 +842,7 @@ index c68f64d52f2..32489f2cb15 100644
842842
}
843843
assert(scanOption.isDefined)
844844
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
845-
index 618f0ab675e..be186f99b66 100644
845+
index 618f0ab675e..efcd4853251 100644
846846
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
847847
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
848848
@@ -268,7 +268,8 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
@@ -855,6 +855,15 @@ index 618f0ab675e..be186f99b66 100644
855855
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
856856
withTempView("df") {
857857
val df1 = spark.range(1, 100)
858+
@@ -339,7 +340,7 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
859+
860+
val sqlText = "EXPLAIN FORMATTED SELECT (SELECT min(id) FROM df) as v"
861+
val expected_pattern =
862+
- "Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#x"
863+
+ "Subquery:1 Hosting operator id = \\d+ Hosting Expression = Subquery subquery#x"
864+
865+
withNormalizedExplain(sqlText) { normalizedOutput =>
866+
assert(expected_pattern.r.findAllMatchIn(normalizedOutput).length == 1)
858867
@@ -471,7 +472,8 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
859868
}
860869
}
@@ -1373,7 +1382,7 @@ index 291aa7cab72..7783c37683e 100644
13731382
super.test(testName, testTags: _*) {
13741383
withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
13751384
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
1376-
index da6f6aca2ad..c02b7c99490 100644
1385+
index da6f6aca2ad..12f48c4bb5d 100644
13771386
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
13781387
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
13791388
@@ -1529,7 +1529,8 @@ class SQLQuerySuite extends SharedSparkSession with AdaptiveSparkPlanHelper
@@ -1403,6 +1412,16 @@ index da6f6aca2ad..c02b7c99490 100644
14031412

14041413
verifyCallCount(
14051414
df.selectExpr("testUdf(a + 1) + testUdf(1 + a)", "testUdf(a + 1)"), Row(4, 2), 1)
1415+
@@ -4755,7 +4763,8 @@ class SQLQuerySuite extends SharedSparkSession with AdaptiveSparkPlanHelper
1416+
)
1417+
}
1418+
1419+
- test("SPARK-47939: Explain should work with parameterized queries") {
1420+
+ test("SPARK-47939: Explain should work with parameterized queries",
1421+
+ IgnoreComet("Comet explain output is different")) {
1422+
def checkQueryPlan(df: DataFrame, plan: String): Unit = assert(
1423+
df.collect()
1424+
.map(_.getString(0))
14061425
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
14071426
index 395cb67f441..33ac6ed19af 100644
14081427
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala

‎docs/source/user-guide/latest/compatibility/spark-versions.md‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -89,11 +89,13 @@ Spark 4.2.0 is supported with Java 17 and Scala 2.13.
8989
- **`FILTER` on window aggregates**: Spark 4.2 accepts a `FILTER (WHERE ...)` clause on an
9090
aggregate function used as a window function. Comet does not support the clause there, so the
9191
window operator falls back to Spark.
92-
- **`OneRowRelation` in `UNION` branches**
93-
([#4949](https://github.com/apache/datafusion-comet/issues/4949)): Spark 4.2 plans a
94-
`OneRowRelation` into the `UNION` branches of some queries, for example TPC-DS q77a. Comet does
95-
not convert `OneRowRelation` by default, so the union and the aggregates above it fall back to
96-
Spark.
92+
- **Merged subqueries in `UNION` branches**
93+
([#4949](https://github.com/apache/datafusion-comet/issues/4949)): Spark 4.2 turns a single-row
94+
aggregate in some queries, for example TPC-DS q77a, into a scalar subquery that returns a struct,
95+
and reads its fields in a projection over a `OneRowRelation`. Comet does not support a
96+
struct-typed scalar subquery result yet
97+
([#5834](https://github.com/apache/datafusion-comet/issues/5834)), so that projection, and the
98+
union and the aggregates above it, fall back to Spark.
9799
- **ANSI arithmetic overflow**
98100
([#4967](https://github.com/apache/datafusion-comet/issues/4967)): Spark 4.2 changed some
99101
ANSI-mode arithmetic overflow behavior, and Comet does not match these changes yet. See

‎docs/source/user-guide/latest/datasources.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,8 @@ converted into Arrow format, allowing the Comet pipeline to take over after that
5252
### Other Spark inputs
5353

5454
Comet can also convert the output of these Spark inputs to Arrow format, so that the operators
55-
above them can run in Comet. Each conversion is off by default.
55+
above them can run in Comet. Only `spark.comet.convert.oneRowRelation.enabled` is on by default,
56+
because the row it converts has no columns.
5657

5758
- `spark.comet.convert.range.enabled`: `spark.range` and SQL `range()`, for ranges that Comet does
5859
not generate natively with `spark.comet.exec.range.enabled`.

‎docs/source/user-guide/latest/migration-guide.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,9 @@ set, `spark.comet.sparkToColumnar.enabled=true` still converts the first four, w
7979
by default, and a list that names any of the five still converts it, as before. In both cases the
8080
driver logs a warning that names the config to use instead. Before Spark 4.1, Spark plans a query
8181
without a `FROM` clause as an `RDDScan`, so on those versions `RDDScan` in the list also converted
82-
it. The list remains the way to convert other leaf operators, such as the scan of a Data Source V2
83-
connector.
82+
it. `spark.comet.convert.oneRowRelation.enabled` is on by default, so such a query is now converted
83+
without either setting, and logs no warning. The list remains the way to convert other leaf
84+
operators, such as the scan of a Data Source V2 connector.
8485

8586
## Upgrading to Comet 1.1.0
8687

‎native/common/src/error.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,9 @@ pub enum SparkError {
215215
group_index: i32,
216216
},
217217

218+
#[error("[INVALID_URL] The url is invalid: {url}. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")]
219+
InvalidUrl { url: String },
220+
218221
#[error("[DATATYPE_CANNOT_ORDER] Cannot order by type: {data_type}.")]
219222
DatatypeCannotOrder { data_type: String },
220223

@@ -379,6 +382,7 @@ impl SparkError {
379382
SparkError::UnexpectedPositiveValue { .. } => "UnexpectedPositiveValue",
380383
SparkError::UnexpectedNegativeValue { .. } => "UnexpectedNegativeValue",
381384
SparkError::InvalidRegexGroupIndex { .. } => "InvalidRegexGroupIndex",
385+
SparkError::InvalidUrl { .. } => "InvalidUrl",
382386
SparkError::DatatypeCannotOrder { .. } => "DatatypeCannotOrder",
383387
SparkError::ScalarSubqueryTooManyRows => "ScalarSubqueryTooManyRows",
384388
SparkError::MergeCardinalityViolation => "MergeCardinalityViolation",
@@ -613,6 +617,11 @@ impl SparkError {
613617
"groupIndex": group_index,
614618
})
615619
}
620+
SparkError::InvalidUrl { url } => {
621+
serde_json::json!({
622+
"url": url,
623+
})
624+
}
616625
SparkError::DatatypeCannotOrder { data_type } => {
617626
serde_json::json!({
618627
"dataType": data_type,
@@ -742,6 +751,7 @@ impl SparkError {
742751

743752
// IllegalArgumentException
744753
SparkError::DatatypeCannotOrder { .. }
754+
| SparkError::InvalidUrl { .. }
745755
| SparkError::InvalidUtf8String { .. }
746756
| SparkError::IllegalDayOfWeek { .. }
747757
| SparkError::SequenceIllegalBoundaries { .. } => {
@@ -853,6 +863,9 @@ impl SparkError {
853863
// Regex errors
854864
SparkError::InvalidRegexGroupIndex { .. } => Some("INVALID_PARAMETER_VALUE"),
855865

866+
// URL errors
867+
SparkError::InvalidUrl { .. } => Some("INVALID_URL"),
868+
856869
// Unsupported operation errors
857870
SparkError::DatatypeCannotOrder { .. } => Some("DATATYPE_CANNOT_ORDER"),
858871

‎native/spark-expr/src/bloom_filter/bloom_filter_might_contain.rs‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ fn evaluate_bloom_filter(
5757
ColumnarValue::Scalar(ScalarValue::Binary(v)) => {
5858
Ok(v.map(|v| SparkBloomFilter::from(v.as_slice())))
5959
}
60+
// An untyped NULL literal, as in `might_contain(null, 1L)`
61+
ColumnarValue::Scalar(ScalarValue::Null) => Ok(None),
6062
_ => internal_err!("Bloom filter expression should be evaluated as a scalar binary value"),
6163
}
6264
}
@@ -90,6 +92,10 @@ fn execute_might_contain(
9092
.and_then(|filter| optional_value.map(|value| filter.might_contain_long(value)));
9193
Ok(ColumnarValue::Scalar(ScalarValue::Boolean(result)))
9294
}
95+
// An untyped NULL literal, as in `might_contain(bf, null)`
96+
ColumnarValue::Scalar(ScalarValue::Null) => {
97+
Ok(ColumnarValue::Scalar(ScalarValue::Boolean(None)))
98+
}
9399
ColumnarValue::Array(values_array) => {
94100
let values = values_array
95101
.as_any()
@@ -109,14 +115,15 @@ fn execute_might_contain(
109115
})
110116
.unwrap_or_else(|| Ok(ColumnarValue::Scalar(ScalarValue::Boolean(None))))
111117
}
112-
_ => internal_err!("Expected Int64Array or Int64 Scalar as arguments"),
118+
other => internal_err!("Expected Int64Array or Int64 Scalar as arguments, got {other:?}"),
113119
}
114120
}
115121

116122
#[cfg(test)]
117123
mod tests {
118124
use super::*;
119125
use arrow::array::BooleanArray;
126+
use datafusion::physical_expr::expressions::Literal;
120127

121128
fn assert_result_eq<T: Into<Option<bool>>>(result: ColumnarValue, expected: Vec<T>) {
122129
let array = result.to_array(1).unwrap();
@@ -171,6 +178,24 @@ mod tests {
171178
assert_all_null(result);
172179
}
173180

181+
#[test]
182+
fn test_execute_untyped_null_value() {
183+
let mut filter = SparkBloomFilter::from((4, 64));
184+
filter.put_long(123);
185+
186+
let args = [ColumnarValue::Scalar(ScalarValue::Null)];
187+
188+
let result = execute_might_contain(&Some(filter), &args).unwrap();
189+
assert_all_null(result);
190+
}
191+
192+
#[test]
193+
fn test_untyped_null_bloom_filter() {
194+
let bloom_filter_expr: Arc<dyn PhysicalExpr> = Arc::new(Literal::new(ScalarValue::Null));
195+
let expr = BloomFilterMightContain::try_new(bloom_filter_expr).unwrap();
196+
assert!(expr.bloom_filter.is_none());
197+
}
198+
174199
#[test]
175200
fn test_execute_array() {
176201
let mut filter = SparkBloomFilter::from((4, 64));

0 commit comments

Comments
 (0)