Skip to content

Commit d6182f3

Browse files
committed
feat: support Spark unbase64 expression (apache#423)
Add CometUnBase64 handler that maps Spark's unbase64(input) to DataFusion's built-in decode(input, 'base64') function. The failOnError variant (triggered by to_binary(x, 'base64')) is not supported by DataFusion's decode and falls back to Spark. Closes apache#423
1 parent 45b670a commit d6182f3

4 files changed

Lines changed: 50 additions & 2 deletions

File tree

‎docs/spark_expressions_support.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -460,7 +460,7 @@
460460
- [ ] try_to_binary
461461
- [ ] try_to_number
462462
- [x] ucase
463-
- [ ] unbase64
463+
- [x] unbase64
464464
- [x] upper
465465

466466
### struct_funcs

‎spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,7 @@ object QueryPlanSerde extends Logging with CometExprShim {
178178
classOf[Left] -> CometLeft,
179179
classOf[Right] -> CometRight,
180180
classOf[Substring] -> CometSubstring,
181+
classOf[UnBase64] -> CometUnBase64,
181182
classOf[Upper] -> CometUpper)
182183

183184
private val bitwiseExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map(

‎spark/src/main/scala/org/apache/comet/serde/strings.scala‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ package org.apache.comet.serde
2121

2222
import java.util.Locale
2323

24-
import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Concat, ConcatWs, Expression, If, InitCap, IsNull, Left, Length, Like, Literal, Lower, RegExpReplace, Right, RLike, StringLPad, StringRepeat, StringRPad, StringSplit, Substring, Upper}
24+
import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Concat, ConcatWs, Expression, If, InitCap, IsNull, Left, Length, Like, Literal, Lower, RegExpReplace, Right, RLike, StringLPad, StringRepeat, StringRPad, StringSplit, Substring, UnBase64, Upper}
2525
import org.apache.spark.sql.types.{BinaryType, DataTypes, LongType, StringType}
2626
import org.apache.spark.unsafe.types.UTF8String
2727

@@ -31,6 +31,28 @@ import org.apache.comet.expressions.{CometCast, CometEvalMode, RegExp}
3131
import org.apache.comet.serde.ExprOuterClass.Expr
3232
import org.apache.comet.serde.QueryPlanSerde.{createBinaryExpr, exprToProtoInternal, optExprWithInfo, scalarFunctionExprToProto, scalarFunctionExprToProtoWithReturnType}
3333

34+
object CometUnBase64 extends CometExpressionSerde[UnBase64] {
35+
36+
override def getSupportLevel(expr: UnBase64): SupportLevel = {
37+
if (expr.failOnError) {
38+
Unsupported(Some("unbase64 with failOnError is not supported"))
39+
} else {
40+
Compatible()
41+
}
42+
}
43+
44+
override def convert(
45+
expr: UnBase64,
46+
inputs: Seq[Attribute],
47+
binding: Boolean): Option[ExprOuterClass.Expr] = {
48+
val inputExpr = exprToProtoInternal(expr.child, inputs, binding)
49+
val encodingExpr = exprToProtoInternal(Literal("base64"), inputs, binding)
50+
val optExpr = scalarFunctionExprToProto("decode", inputExpr, encodingExpr)
51+
optExprWithInfo(optExpr, expr, expr.child)
52+
}
53+
}
54+
55+
3456
object CometStringRepeat extends CometExpressionSerde[StringRepeat] {
3557

3658
override def convert(

‎spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,31 @@ class CometStringExpressionSuite extends CometTestBase {
148148
}
149149
}
150150

151+
test("unbase64") {
152+
val data = Seq(
153+
"SGVsbG8=", // base64("Hello")
154+
"U3BhcmsgU1FM", // base64("Spark SQL")
155+
"", // empty
156+
null).map(Tuple1(_))
157+
withParquetTable(data, "tbl") {
158+
// unbase64 decoding from column
159+
checkSparkAnswerAndOperator("SELECT unbase64(_1) FROM tbl")
160+
// unbase64 with inline literal
161+
checkSparkAnswerAndOperator("SELECT unbase64('U3BhcmsgU1FM') FROM tbl")
162+
// null handling
163+
checkSparkAnswerAndOperator("SELECT unbase64(NULL) FROM tbl")
164+
}
165+
}
166+
167+
test("to_binary with base64 falls back (failOnError)") {
168+
val data = Seq("SGVsbG8=", "U3BhcmsgU1FM").map(Tuple1(_))
169+
withParquetTable(data, "tbl") {
170+
checkSparkAnswerAndFallbackReason(
171+
"SELECT to_binary(_1, 'base64') FROM tbl",
172+
"unbase64 with failOnError is not supported")
173+
}
174+
}
175+
151176
test("split string basic") {
152177
withSQLConf("spark.comet.expression.StringSplit.allowIncompatible" -> "true") {
153178
withParquetTable((0 until 5).map(i => (s"value$i,test$i", i)), "tbl") {

0 commit comments

Comments
 (0)