@@ -23,9 +23,9 @@ import org.apache.spark.sql.{CometTestBase, DataFrame, Row}
2323import org .apache .spark .sql .comet .CometSparkToColumnarExec
2424import org .apache .spark .sql .comet .execution .shuffle .{CometColumnarShuffle , CometNativeShuffle , CometShuffleExchangeExec }
2525import org .apache .spark .sql .execution .{ColumnarToRowExec , ColumnarToRowTransition , SparkPlan }
26- import org .apache .spark .sql .functions .{avg , col , count , length , max , min , size , sum }
26+ import org .apache .spark .sql .functions .{array , avg , col , count , length , max , min , size , sum }
2727import org .apache .spark .sql .internal .SQLConf
28- import org .apache .spark .sql .types .{ArrayType , DataTypes , DecimalType , IntegerType , LongType , StructType }
28+ import org .apache .spark .sql .types .{ArrayType , BinaryType , DataTypes , DecimalType , IntegerType , LongType , StringType , StructType }
2929
3030import org .apache .comet .{CometConf , ExtendedExplainInfo }
3131
@@ -145,17 +145,17 @@ class CometShuffleInputConversionSuite extends CometTestBase {
145145 val ds = spark.sql(" SELECT _1 AS a, _2 AS b FROM tbl" ).as[ShuffleInputRec ]
146146 // The shuffle reads the typed operation's SerializeFromObject.
147147 val (_, repartitioned) = checkSparkAnswer(
148- ds.map(r => ShuffleInputRec (r.a + 1 , r.b))
149- .repartition(7 , col(" b " ))
150- .groupBy(" b " )
151- .agg(sum( " a " )))
148+ ds.map(r => ShuffleInputRec (r.a % 13 , r.b))
149+ .repartition(7 , col(" a " ))
150+ .groupBy(" a " )
151+ .agg(count( " b " )))
152152 assert(convertedShuffles(repartitioned).length == 1 , s " AQE $aqe: \n $repartitioned" )
153153 assertRowOperatorsReadThroughTransitions(repartitioned)
154154 // The shuffle reads a Spark partial aggregate over the typed operation. The final
155155 // aggregate stays on Spark too, so the shuffle would go back to Spark's own.
156156 withSQLConf(CometConf .COMET_SHUFFLE_REVERT_REDUNDANT_COLUMNAR_ENABLED .key -> " false" ) {
157157 val (_, aggregated) =
158- checkSparkAnswer(ds.map(r => ShuffleInputRec (r.a * 2 , r.b)).groupBy(" b " ).count())
158+ checkSparkAnswer(ds.map(r => ShuffleInputRec (r.a % 7 , r.b)).groupBy(" a " ).count())
159159 assert(convertedShuffles(aggregated).length == 1 , s " AQE $aqe: \n $aggregated" )
160160 assertRowOperatorsReadThroughTransitions(aggregated)
161161 }
@@ -215,6 +215,31 @@ class CometShuffleInputConversionSuite extends CometTestBase {
215215 }
216216 }
217217
218+ convertTest(" a shuffle that hashes a string stays on the JVM columnar shuffle" ) {
219+ // Spark's partitioner hashes a string's bytes as they are, but native shuffle hashes them
220+ // after the import into native has replaced invalid UTF-8. The right input, with its
221+ // array<int> column, stays on the JVM columnar shuffle, so the left one has to as well, or
222+ // matching keys with invalid UTF-8 land in different partitions. Both shuffles write the rows
223+ // with invalid UTF-8 replaced, so the four keys stay apart only because they hash to four
224+ // different partitions.
225+ withSQLConf(
226+ SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
227+ SQLConf .AUTO_BROADCASTJOIN_THRESHOLD .key -> " -1" ,
228+ SQLConf .SHUFFLE_PARTITIONS .key -> " 10" ) {
229+ val schema = new StructType ().add(" b" , BinaryType ).add(" v" , LongType )
230+ val data = (0 until 40 ).map(i => Row (Array ((0x80 + i % 4 ).toByte), i.toLong))
231+ val keys = spark.createDataFrame(spark.sparkContext.parallelize(data, 4 ), schema)
232+ val left = keys.select(col(" b" ).cast(StringType ).as(" lk" ), col(" v" ))
233+ val right = keys.select(
234+ col(" b" ).cast(StringType ).as(" rk" ),
235+ array(col(" v" ).cast(IntegerType )).as(" xs" ))
236+ val df = left.join(right, col(" lk" ) === col(" rk" ))
237+ val (_, plan) = checkSparkAnswer(df)
238+ assert(conversions(plan).isEmpty, plan)
239+ checkCometExchange(df, 2 , native = false )
240+ }
241+ }
242+
218243 convertTest(" columns the conversion does not support keep the JVM columnar shuffle" ) {
219244 val df = arraysDf().repartition(5 , col(" k" )).groupBy(" k" ).agg(sum(size(col(" xs" ))))
220245 val (_, plan) = checkSparkAnswer(df)
0 commit comments