Skip to content

Commit 37509cb

Browse files
committed
test: assert native scan for list/map complex-null predicates
Convert five list/map fallback assertions in CometIcebergNativeSuite to checkIcebergNativeScan now that the scan rule keeps complex-null checks native for list/map columns: array/map IS NULL, array-element filter, whole-array equality, and map-key access. iceberg-rust rejects these predicates, so the residual is skipped and the post-scan Comet filter enforces them while the scan stays native.
1 parent 68f2f47 commit 37509cb

1 file changed

Lines changed: 22 additions & 16 deletions

File tree

‎spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala‎

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2245,7 +2245,7 @@ class CometIcebergNativeSuite
22452245

22462246
checkIcebergNativeScanFallback(
22472247
"SELECT * FROM test_cat.db.struct_filter_test WHERE address IS NULL ORDER BY id",
2248-
"iceberg-rust does not support IS NULL on complex type columns")
2248+
"iceberg-rust does not support IS NULL on struct type columns")
22492249

22502250
spark.sql("DROP TABLE test_cat.db.struct_filter_test")
22512251
}
@@ -2355,9 +2355,9 @@ class CometIcebergNativeSuite
23552355
(3, 'Charlie', NULL)
23562356
""")
23572357

2358-
checkIcebergNativeScanFallback(
2359-
"SELECT * FROM test_cat.db.array_filter_test WHERE values IS NULL ORDER BY id",
2360-
"iceberg-rust does not support IS NULL on complex type columns")
2358+
// iceberg-rust evaluates IS NULL on list columns natively, so the scan stays native
2359+
checkIcebergNativeScan(
2360+
"SELECT * FROM test_cat.db.array_filter_test WHERE values IS NULL ORDER BY id")
23612361

23622362
spark.sql("DROP TABLE test_cat.db.array_filter_test")
23632363
}
@@ -2392,9 +2392,11 @@ class CometIcebergNativeSuite
23922392
(3, 'Charlie', array(1, 7, 8))
23932393
""")
23942394

2395-
checkIcebergNativeScanFallback(
2396-
"SELECT * FROM test_cat.db.array_element_filter_test WHERE array_contains(values, 1) ORDER BY id",
2397-
"Iceberg Java only pushes down NOT NULL, which iceberg-rust rejects")
2395+
// The element predicate is not pushed to iceberg-rust (Iceberg Java only pushes NOT
2396+
// NULL, which iceberg-rust rejects); the residual is skipped and the post-scan Comet
2397+
// filter enforces it while the scan stays native
2398+
checkIcebergNativeScan(
2399+
"SELECT * FROM test_cat.db.array_element_filter_test WHERE array_contains(values, 1) ORDER BY id")
23982400

23992401
spark.sql("DROP TABLE test_cat.db.array_element_filter_test")
24002402
}
@@ -2429,9 +2431,11 @@ class CometIcebergNativeSuite
24292431
(3, 'Charlie', array(1, 2, 3))
24302432
""")
24312433

2432-
checkIcebergNativeScanFallback(
2433-
"SELECT * FROM test_cat.db.array_value_filter_test WHERE values = array(1, 2, 3) ORDER BY id",
2434-
"Iceberg Java only pushes down NOT NULL, which iceberg-rust rejects")
2434+
// The whole-array equality is not pushed to iceberg-rust (Iceberg Java only pushes NOT
2435+
// NULL, which iceberg-rust rejects); the residual is skipped and the post-scan Comet
2436+
// filter enforces it while the scan stays native
2437+
checkIcebergNativeScan(
2438+
"SELECT * FROM test_cat.db.array_value_filter_test WHERE values = array(1, 2, 3) ORDER BY id")
24352439

24362440
spark.sql("DROP TABLE test_cat.db.array_value_filter_test")
24372441
}
@@ -2466,9 +2470,9 @@ class CometIcebergNativeSuite
24662470
(3, 'Charlie', NULL)
24672471
""")
24682472

2469-
checkIcebergNativeScanFallback(
2470-
"SELECT * FROM test_cat.db.map_filter_test WHERE properties IS NULL ORDER BY id",
2471-
"iceberg-rust does not support IS NULL on complex type columns")
2473+
// iceberg-rust evaluates IS NULL on map columns natively, so the scan stays native
2474+
checkIcebergNativeScan(
2475+
"SELECT * FROM test_cat.db.map_filter_test WHERE properties IS NULL ORDER BY id")
24722476

24732477
spark.sql("DROP TABLE test_cat.db.map_filter_test")
24742478
}
@@ -2503,9 +2507,11 @@ class CometIcebergNativeSuite
25032507
(3, 'Charlie', map('age', 30, 'score', 80))
25042508
""")
25052509

2506-
checkIcebergNativeScanFallback(
2507-
"SELECT * FROM test_cat.db.map_key_filter_test WHERE properties['age'] = 30 ORDER BY id",
2508-
"Iceberg Java only pushes down NOT NULL, which iceberg-rust rejects")
2510+
// The map-key predicate is not pushed to iceberg-rust (Iceberg Java only pushes NOT
2511+
// NULL, which iceberg-rust rejects); the residual is skipped and the post-scan Comet
2512+
// filter enforces it while the scan stays native
2513+
checkIcebergNativeScan(
2514+
"SELECT * FROM test_cat.db.map_key_filter_test WHERE properties['age'] = 30 ORDER BY id")
25092515

25102516
spark.sql("DROP TABLE test_cat.db.map_key_filter_test")
25112517
}

0 commit comments

Comments
 (0)