Skip to content

Commit 85bd183

Browse files
committed
fix: repair CometIcebergSystemFunctionSuite registration and bind BucketInt
The ApplyFunctionExpression coverage added in this branch registered its handler-map test twice. ScalaTest rejects a duplicate test name while constructing the suite, before any test body or the icebergAvailable assumption runs, so every scan job aborted the whole suite and none of it validated the change. The wrapper test also looked up a no-arg BucketInt constructor. Iceberg's per-type implementations take the bound input type, and no runtime these profiles build against (1.5.2, 1.8.1, 1.10.0, 1.11.0) declares a no-arg constructor, so that lookup would have failed once registration was repaired. Also strengthen the wrapper test: assert the serialized call rather than only the support level, since ApplyFunctionExpression carries its arguments and result type on children/dataType rather than the arguments/dataType the StaticInvoke path reads, and add an unlisted V2 catalog function to pin that a class with no handler stays on Spark instead of matching by argument shape.
1 parent 9805783 commit 85bd183

1 file changed

Lines changed: 38 additions & 20 deletions

File tree

‎spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala‎

Lines changed: 38 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import org.scalatest.Tag
3131

3232
import org.apache.spark.SparkConf
3333
import org.apache.spark.sql.{CometTestBase, DataFrame, Row}
34+
import org.apache.spark.sql.catalyst.InternalRow
3435
import org.apache.spark.sql.catalyst.expressions.{ApplyFunctionExpression, AttributeReference, Expression, Literal}
3536
import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke
3637
import org.apache.spark.sql.comet.{CometIcebergWriteExec, CometSortExec}
@@ -344,37 +345,40 @@ class CometIcebergSystemFunctionSuite
344345
val value = AttributeReference("v", IntegerType)()
345346
val bucketIntCls =
346347
Class.forName("org.apache.iceberg.spark.functions.BucketFunction$BucketInt")
347-
val bucketInt =
348-
bucketIntCls.getDeclaredConstructor().newInstance().asInstanceOf[ScalarFunction[_]]
348+
// Iceberg's per-type implementations take the bound input type; there is no no-arg
349+
// constructor on any of the runtimes these profiles build against.
350+
val bucketInt = bucketIntCls
351+
.getDeclaredConstructor(classOf[DataType])
352+
.newInstance(IntegerType)
353+
.asInstanceOf[ScalarFunction[_]]
349354
val expr = ApplyFunctionExpression(bucketInt, Seq(Literal(4), value))
350355
assert(CometApplyFunctionExpression.getSupportLevel(expr) == Compatible())
351356

357+
// Support level alone would still leave the arguments and result type unread: the wrapper
358+
// carries them on `children` / `dataType` rather than the `arguments` / `dataType` the
359+
// StaticInvoke path uses, so assert the serialized call to cover that extraction.
360+
val proto = CometApplyFunctionExpression.convert(expr, Seq(value), binding = true)
361+
assert(
362+
proto.exists(_.getScalarFunc.getFunc == "iceberg_bucket"),
363+
s"expected a native iceberg_bucket call, got $proto")
364+
352365
// The same argument-shape checks that gate the StaticInvoke path have to gate this one, or a
353366
// zero-bucket call would land natively and diverge from Iceberg's own ArithmeticException.
354367
val zeroBuckets = ApplyFunctionExpression(bucketInt, Seq(Literal(0), value))
355368
assert(CometApplyFunctionExpression.getSupportLevel(zeroBuckets).isInstanceOf[Unsupported])
356-
}
357369

358-
test("Iceberg handler map is keyed by Iceberg implementation class names") {
359-
// Guards against a rename of one of Iceberg's per-type implementation classes silently
360-
// dropping the native path: if any of these classes is on the classpath, its name has to be
361-
// in the handler map.
362-
val expected = Seq(
363-
"org.apache.iceberg.spark.functions.BucketFunction$BucketInt" -> CometIcebergBucket,
364-
"org.apache.iceberg.spark.functions.BucketFunction$BucketLong" -> CometIcebergBucket,
365-
"org.apache.iceberg.spark.functions.TruncateFunction$TruncateInt" -> CometIcebergTruncate,
366-
"org.apache.iceberg.spark.functions.TruncateFunction$TruncateString" -> CometIcebergTruncate)
367-
expected.foreach { case (className, expectedHandler) =>
368-
assert(
369-
CometIcebergSystemFunctions.handlers.get(className).contains(expectedHandler),
370-
s"missing handler for $className")
371-
}
370+
// A V2 catalog function Comet has no handler for has to stay on Spark rather than reaching
371+
// the Iceberg handlers by argument shape alone: `ApplyFunctionExpression` is the generic DSv2
372+
// lowering, so most expressions arriving here belong to some other catalog entirely.
373+
val unlisted = ApplyFunctionExpression(new UnlistedScalarFunction, Seq(Literal(4), value))
374+
assert(CometApplyFunctionExpression.getSupportLevel(unlisted).isInstanceOf[Unsupported])
375+
assert(CometApplyFunctionExpression.convert(unlisted, Seq(value), binding = true).isEmpty)
372376
}
373377

374378
test("Iceberg handler map is keyed by Iceberg implementation class names") {
375-
// Guards against a rename of one of Iceberg's per-type implementation classes silently
376-
// dropping the native path: if any of these classes is on the classpath, its name has to be
377-
// in the handler map.
379+
// The map is keyed by class name because Iceberg is not on Comet's compile classpath, so a
380+
// rename on Iceberg's side would silently drop the native path rather than fail to compile.
381+
// These names are the ones both lowerings carry as their identity.
378382
val expected = Seq(
379383
"org.apache.iceberg.spark.functions.BucketFunction$BucketInt" -> CometIcebergBucket,
380384
"org.apache.iceberg.spark.functions.BucketFunction$BucketLong" -> CometIcebergBucket,
@@ -531,3 +535,17 @@ class CometIcebergSystemFunctionSuite
531535
sourceSchema)
532536
}
533537
}
538+
539+
/**
540+
* A DataSourceV2 catalog scalar function with no static `invoke` magic method, so Spark's
541+
* `V2ExpressionUtils.resolveScalarFunction` would lower a call to it as
542+
* `ApplyFunctionExpression`. Stands in for the third-party catalogs that share that lowering with
543+
* Iceberg but have no native handler in Comet.
544+
*/
545+
private class UnlistedScalarFunction extends ScalarFunction[Integer] {
546+
override def inputTypes(): Array[DataType] = Array(IntegerType, IntegerType)
547+
override def resultType(): DataType = IntegerType
548+
override def name(): String = "unlisted"
549+
override def canonicalName(): String = "test.unlisted"
550+
override def produceResult(input: InternalRow): Integer = input.getInt(0)
551+
}

0 commit comments

Comments
 (0)