@@ -75,6 +75,9 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
7575 val AND = " And"
7676 val OR = " Or"
7777 val NOT = " Not"
78+ // The only UnboundTerm shape this serde converts. The others, UnboundTransform and (on
79+ // Iceberg versions that have it) UnboundExtract, are declined; see icebergExprToProto.
80+ val NAMED_REFERENCE = " NamedReference"
7881 }
7982 }
8083
@@ -643,31 +646,48 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
643646 }
644647
645648 /**
646- * Converts an Iceberg residual Expression into an IcebergPredicate for native row-group
647- * pruning.
649+ * Converts an Iceberg residual Expression into an IcebergPredicate for the native scan.
648650 *
649651 * Residuals come from Iceberg's ResidualEvaluator (partial evaluation of the scan filter
650- * against each file's partition data). This is only a pruning hint: the CometFilter above the
651- * scan enforces correctness, so any node or literal we cannot represent yields None (no
652- * pushdown). Predicates over `pageIndexUnsupportedColumns` also yield None (iceberg-rust cannot
653- * use those columns in the page index). Uses reflection because Iceberg's expression classes
652+ * against each file's partition data). iceberg-rust prunes row groups and pages with the
653+ * predicate and also filters rows by it, so what this returns may be weaker than the residual
654+ * but never stronger. Iceberg keeps every predicate that can survive into a residual in the
655+ * scan's postScanFilters, so the filter above the scan re-applies whatever is not pushed, but
656+ * it cannot restore rows the scan dropped. Uses reflection because Iceberg's expression classes
654657 * are not on Spark's classpath at planning time; residuals are unbound predicates carrying a
655658 * NamedReference (column name) and a literal.
659+ *
660+ * Not pushing a residual therefore takes one of two forms, kept deliberately distinct:
661+ *
662+ * - `None` is a decision. Every point that returns it has established that this serde does
663+ * not model the node, term shape, operation, column or literal in front of it, so the
664+ * predicate is left to the post-scan filter.
665+ * - An exception means reflection over Iceberg's expression API did something unexpected, so
666+ * no such decision was reached. It propagates instead of folding into `None`, which would
667+ * make the two indistinguishable; see [[serializeResidual ]].
656668 */
657669 def icebergExprToProto (
658670 icebergExpr : Any ,
659671 output : Seq [Attribute ],
660672 pageIndexUnsupportedColumns : Set [String ]): Option [OperatorOuterClass .IcebergPredicate ] = {
661- try {
662- val exprClass = icebergExpr.getClass
663- val attributeMap = output.map(attr => attr.name -> attr).toMap
664-
665- if (exprClass.getName.endsWith(Constants .ExpressionTypes .UNBOUND_PREDICATE )) {
666- val operation = IcebergReflection .getMethod(exprClass, " op" ).invoke(icebergExpr).toString
667- val term = IcebergReflection .getMethod(exprClass, " term" ).invoke(icebergExpr)
668- val ref = IcebergReflection .getMethod(term.getClass, " ref" ).invoke(term)
673+ val exprClass = icebergExpr.getClass
674+ val attributeMap = output.map(attr => attr.name -> attr).toMap
675+
676+ if (exprClass.getName.endsWith(Constants .ExpressionTypes .UNBOUND_PREDICATE )) {
677+ val operation = IcebergReflection .getMethod(exprClass, " op" ).invoke(icebergExpr).toString
678+ val term = IcebergReflection .getMethod(exprClass, " term" ).invoke(icebergExpr)
679+
680+ // Only a bare column reference converts. UnboundTransform (and UnboundExtract, on Iceberg
681+ // versions that have it) answers ref() with its *source* column, so reading the name off
682+ // it would push `bucket(4, id) = 2` as `id = 2` and drop every matching row whose id is
683+ // not 2. CometScanRule declines a non-identity transform at planning time, but only when
684+ // the residual is a bare predicate rather than one under AND/OR/NOT, so the term shape is
685+ // checked here too rather than assumed.
686+ if (! term.getClass.getName.endsWith(Constants .ExpressionTypes .NAMED_REFERENCE )) {
687+ None
688+ } else {
669689 val columnName =
670- IcebergReflection .getMethod(ref .getClass, " name" ).invoke(ref ).asInstanceOf [String ]
690+ IcebergReflection .getMethod(term .getClass, " name" ).invoke(term ).asInstanceOf [String ]
671691
672692 // Iceberg names a nested reference by its dotted path ("struct.field"), which never matches
673693 // a top-level scan output attribute, so a residual on a nested field drops here. That miss
@@ -694,54 +714,68 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
694714 }
695715 }
696716 }
697- } else if (exprClass.getName.endsWith(Constants .ExpressionTypes .AND )) {
698- val left = icebergExprToProto(
699- IcebergReflection .getMethod(exprClass, " left" ).invoke(icebergExpr),
700- output,
701- pageIndexUnsupportedColumns)
702- val right = icebergExprToProto(
703- IcebergReflection .getMethod(exprClass, " right" ).invoke(icebergExpr),
704- output,
705- pageIndexUnsupportedColumns)
706- (left, right) match {
707- // Push the residual only if it converts whole. Dropping a conjunct is safe in positive
708- // position but strengthens the predicate under a NOT (De Morgan), which would wrongly
709- // prune, and tracking polarity across arbitrary nesting is error prone. So an
710- // unconvertible conjunct elides the whole residual; the post-scan CometFilter is exact.
711- case (Some (l), Some (r)) => Some (logicalPredicate(isAnd = true , l, r))
712- case _ => None
713- }
714- } else if (exprClass.getName.endsWith(Constants .ExpressionTypes .OR )) {
715- val left = icebergExprToProto(
716- IcebergReflection .getMethod(exprClass, " left" ).invoke(icebergExpr),
717- output,
718- pageIndexUnsupportedColumns)
719- val right = icebergExprToProto(
720- IcebergReflection .getMethod(exprClass, " right" ).invoke(icebergExpr),
721- output,
722- pageIndexUnsupportedColumns)
723- // Dropping a disjunct would strengthen the predicate and wrongly prune, so require both.
724- (left, right) match {
725- case (Some (l), Some (r)) => Some (logicalPredicate(isAnd = false , l, r))
726- case _ => None
727- }
728- } else if (exprClass.getName.endsWith(Constants .ExpressionTypes .NOT )) {
729- val child = IcebergReflection .getMethod(exprClass, " child" ).invoke(icebergExpr)
730- icebergExprToProto(child, output, pageIndexUnsupportedColumns).map(notPredicate)
731- } else {
732- None
733717 }
718+ } else if (exprClass.getName.endsWith(Constants .ExpressionTypes .AND )) {
719+ val left = icebergExprToProto(
720+ IcebergReflection .getMethod(exprClass, " left" ).invoke(icebergExpr),
721+ output,
722+ pageIndexUnsupportedColumns)
723+ val right = icebergExprToProto(
724+ IcebergReflection .getMethod(exprClass, " right" ).invoke(icebergExpr),
725+ output,
726+ pageIndexUnsupportedColumns)
727+ (left, right) match {
728+ // Push the residual only if it converts whole. Dropping a conjunct is safe in positive
729+ // position but strengthens the predicate under a NOT (De Morgan), which would wrongly
730+ // prune, and tracking polarity across arbitrary nesting is error prone. So an
731+ // unconvertible conjunct elides the whole residual; the post-scan CometFilter is exact.
732+ case (Some (l), Some (r)) => Some (logicalPredicate(isAnd = true , l, r))
733+ case _ => None
734+ }
735+ } else if (exprClass.getName.endsWith(Constants .ExpressionTypes .OR )) {
736+ val left = icebergExprToProto(
737+ IcebergReflection .getMethod(exprClass, " left" ).invoke(icebergExpr),
738+ output,
739+ pageIndexUnsupportedColumns)
740+ val right = icebergExprToProto(
741+ IcebergReflection .getMethod(exprClass, " right" ).invoke(icebergExpr),
742+ output,
743+ pageIndexUnsupportedColumns)
744+ // Dropping a disjunct would strengthen the predicate and wrongly prune, so require both.
745+ (left, right) match {
746+ case (Some (l), Some (r)) => Some (logicalPredicate(isAnd = false , l, r))
747+ case _ => None
748+ }
749+ } else if (exprClass.getName.endsWith(Constants .ExpressionTypes .NOT )) {
750+ val child = IcebergReflection .getMethod(exprClass, " child" ).invoke(icebergExpr)
751+ icebergExprToProto(child, output, pageIndexUnsupportedColumns).map(notPredicate)
752+ } else {
753+ // Anything else, such as Expressions.alwaysTrue or alwaysFalse, or a node type a future
754+ // Iceberg introduces, carries no pushdown this serde can express.
755+ None
756+ }
757+ }
758+
759+ /**
760+ * Converts a FileScanTask's residual with [[icebergExprToProto ]], failing the query if that
761+ * throws. The converter returns None only for a residual it deliberately leaves to the
762+ * post-scan filter, so a failure means this serde has misread Iceberg's expression API. Serde
763+ * runs after CometScanRule committed the scan to native execution, so there is no fallback to
764+ * take.
765+ */
766+ private [operator] def serializeResidual (
767+ residual : Any ,
768+ output : Seq [Attribute ],
769+ pageIndexUnsupportedColumns : Set [String ]): Option [OperatorOuterClass .IcebergPredicate ] =
770+ try {
771+ icebergExprToProto(residual, output, pageIndexUnsupportedColumns)
734772 } catch {
735- // Reflection over Iceberg's expression classes can fail on an unexpected shape (e.g. an
736- // Iceberg version change). A residual is only a pruning hint, so skip pushdown rather than
737- // fail the scan, but log it: a persistent warning here signals a real API drift to fix.
738773 case e : Exception =>
739- logWarning(
740- " Skipping Iceberg residual pushdown; could not convert expression: " +
741- s " ${e.getMessage} " )
742- None
774+ val msg = " Iceberg reflection failure: Failed to convert residual expression " +
775+ s " ' $ residual' from FileScanTask: ${e.getMessage} "
776+ logError(msg )
777+ throw new RuntimeException (msg, e)
743778 }
744- }
745779
746780 private def unaryPredicate (
747781 column : String ,
@@ -1193,18 +1227,10 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
11931227 }
11941228
11951229 val residualExprOpt =
1196- try {
1197- icebergExprToProto(
1198- residualMethod.invoke(task),
1199- output,
1200- pageIndexUnsupportedColumns)
1201- } catch {
1202- case e : Exception =>
1203- logWarning(
1204- " Failed to extract residual expression from FileScanTask: " +
1205- s " ${e.getMessage}" )
1206- None
1207- }
1230+ serializeResidual(
1231+ residualMethod.invoke(task),
1232+ output,
1233+ pageIndexUnsupportedColumns)
12081234
12091235 residualExprOpt.foreach { residual =>
12101236 val residualIdx = residualToPoolIndex.getOrElseUpdate(
0 commit comments