Skip to content

Commit ae5cbe3

Browse files
committed
test: cover booleans in native explode output past the first batch
The native explode slices each output batch out of the exploded child instead of gathering it, so an exploded boolean, or a boolean field of an exploded struct, leaves native at a non-zero bit offset. Arrow Java ignores that offset on import, so on 1.1.0-rc1 these booleans came back wrong from the second output batch on. apache#6339 already fixed this on main by zeroing offsets at every level before export, but none of its tests go through explode. Add tests to CometGenerateExecSuite for explode, explode_outer, posexplode and posexplode_outer of structs with boolean fields, for one row longer than the batch size, and for a native named_struct and a boolean Scala UDF over an exploded boolean. All of them fail when native zeroes only top-level offsets and leaves the UDF bridge inputs alone, as rc1 does. Closes apache#6464.
1 parent b38a60b commit ae5cbe3

1 file changed

Lines changed: 63 additions & 0 deletions

File tree

‎spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -659,4 +659,67 @@ class CometGenerateExecSuite extends CometTestBase {
659659
}
660660
}
661661

662+
// The native explode slices each output batch out of the exploded child instead of gathering
663+
// it, so an exploded boolean, or a boolean field of an exploded struct, leaves native at a
664+
// non-zero bit offset. Arrow Java ignores that offset on import, so native has to zero it at
665+
// every level before export, including in a struct built over the booleans and in the input to
666+
// a Scala UDF, or they come back wrong after the first output batch.
667+
// https://github.com/apache/datafusion-comet/issues/6464
668+
private def withBooleanArrays(numRows: Int, arrayLength: Int)(f: => Unit): Unit = {
669+
withTempPath { dir =>
670+
// One file, so a single input batch explodes into several output batches.
671+
withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
672+
spark
673+
.range(0, numRows, 1, 1)
674+
.selectExpr(
675+
"id",
676+
s"transform(sequence(1, $arrayLength), i -> named_struct(" +
677+
"'b', hash(id, i) % 2 = 0, " +
678+
"'bn', IF(hash(id, i, 7) % 5 = 0, NULL, hash(id, i, 3) % 2 = 0), " +
679+
"'n', id * 1000 + i)) AS structs",
680+
// No NULLs, so the null check Spark wraps around a primitive UDF argument selects
681+
// every row, and the UDF reads the exploded batch rather than a filtered copy.
682+
s"transform(sequence(1, $arrayLength), i -> hash(id, i, 11) % 2 = 0) AS bools")
683+
.write
684+
.parquet(dir.getCanonicalPath)
685+
}
686+
withParquetTable(dir.getCanonicalPath, "t")(f)
687+
}
688+
}
689+
690+
for (generator <- Seq("explode", "explode_outer", "posexplode", "posexplode_outer")) {
691+
test(s"$generator of structs keeps boolean fields past the first output batch") {
692+
// With the default batch size, 13 elements a row give output batches of 8190 rows, so the
693+
// later batches start both on and off a byte boundary.
694+
withBooleanArrays(numRows = 3000, arrayLength = 13) {
695+
checkSparkAnswerAndOperator(sql(s"SELECT id, $generator(structs) FROM t"))
696+
}
697+
}
698+
}
699+
700+
test("explode of structs keeps boolean fields when one row exceeds the batch size") {
701+
// A row longer than the batch size is unnested in one build, which is then sliced into
702+
// batch-size pieces on the way out, here at elements 100 and 200.
703+
withSQLConf(CometConf.COMET_BATCH_SIZE.key -> "100") {
704+
withBooleanArrays(numRows = 1, arrayLength = 250) {
705+
checkSparkAnswerAndOperator(sql("SELECT id, explode(structs) FROM t"))
706+
}
707+
}
708+
}
709+
710+
test("named_struct over an exploded boolean keeps its values") {
711+
withBooleanArrays(numRows = 3000, arrayLength = 13) {
712+
checkSparkAnswerAndOperator(
713+
sql("SELECT id, named_struct('v', v) FROM (SELECT id, explode(bools) AS v FROM t)"))
714+
}
715+
}
716+
717+
test("boolean ScalaUDF over an exploded boolean keeps its values") {
718+
spark.udf.register("flip", (x: Boolean) => !x)
719+
withBooleanArrays(numRows = 3000, arrayLength = 13) {
720+
checkSparkAnswerAndOperator(
721+
sql("SELECT id, flip(v) FROM (SELECT id, explode(bools) AS v FROM t)"))
722+
}
723+
}
724+
662725
}

0 commit comments

Comments
 (0)