Skip to content

Commit 42a72d9

Browse files
authored
test(spark): characterize SparkPredicateToVortexExpression conversion (#9452)
## Rationale for this change `isPushable` documents an invariant: when it returns true, `convert` must succeed. Otherwise `VortexScanBuilder.pushPredicates` lets Spark drop a predicate that `VortexPartitionReader` silently skips, and the scan returns rows the query excluded. Only `isPushable` was tested; `convert` had no coverage. ## What changes are included in this PR? Test only. Asserts both sides of the invariant over every accepted shape: the comparison operators with the column on either side, column-to-column, null checks, `IN`, the string matches with `%`, `_` and `\` in the needle, bare boolean columns, the constants, nested `AND`/`OR`/`NOT`, and all 13 literal types. Rejections too: wrong arity, unknown predicate name, empty named reference, and a decimal that does not fit its scale. Verified non-vacuous: five mutations — dropping `BOOLEAN_EXPRESSION` from `convert`, the null decimal literal, the commuted comparison, `columnOf`'s zero-part guard, and `default -> false` — each fail a targeted test. 27 pass on `:vortex-spark_2.13:test` and on `:vortex-spark_2.12:test`; the full suites and `spotlessCheck` are clean apart from the S3Mock test, which needs Docker. ## What APIs are changed? Are there any user-facing changes? None. ## AI assistance Prepared with agentic AI assistance. I read the translator end to end and confirmed the mutations above fail. Signed-off-by: jackylee <[email protected]>
1 parent 6482227 commit 42a72d9

1 file changed

Lines changed: 263 additions & 3 deletions

File tree

‎java/vortex-spark/src/test/java/dev/vortex/spark/read/SparkPredicateToVortexExpressionTest.java‎

Lines changed: 263 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,36 @@
66
import static org.junit.jupiter.api.Assertions.assertFalse;
77
import static org.junit.jupiter.api.Assertions.assertTrue;
88

9+
import dev.vortex.jni.NativeLoader;
10+
import java.math.BigDecimal;
11+
import java.util.List;
912
import java.util.Map;
1013
import org.apache.spark.sql.connector.expressions.Expression;
1114
import org.apache.spark.sql.connector.expressions.LiteralValue;
1215
import org.apache.spark.sql.connector.expressions.NamedReference;
16+
import org.apache.spark.sql.connector.expressions.filter.AlwaysFalse;
17+
import org.apache.spark.sql.connector.expressions.filter.AlwaysTrue;
18+
import org.apache.spark.sql.connector.expressions.filter.And;
19+
import org.apache.spark.sql.connector.expressions.filter.Not;
20+
import org.apache.spark.sql.connector.expressions.filter.Or;
1321
import org.apache.spark.sql.connector.expressions.filter.Predicate;
1422
import org.apache.spark.sql.types.DataType;
1523
import org.apache.spark.sql.types.DataTypes;
24+
import org.apache.spark.sql.types.Decimal;
1625
import org.apache.spark.sql.types.StructType;
26+
import org.junit.jupiter.api.BeforeAll;
1727
import org.junit.jupiter.api.DisplayName;
1828
import org.junit.jupiter.api.Test;
1929

20-
/** Unit tests for {@link SparkPredicateToVortexExpression#isPushable(Predicate, Map)}. */
30+
/**
31+
* Unit tests for {@link SparkPredicateToVortexExpression#isPushable(Predicate, Map)} and
32+
* {@link SparkPredicateToVortexExpression#convert(Predicate)}.
33+
*
34+
* <p>{@code isPushable} decides which predicates {@code VortexScanBuilder.pushPredicates} lets Spark drop, while
35+
* {@code convert} builds the filter {@code VortexPartitionReader} actually pushes down. A predicate that passes the
36+
* first but fails the second is silently skipped by the reader, so the scan returns rows the query excluded. The tests
37+
* below pin that {@code isPushable} implies {@code convert().isPresent()} across every accepted shape.
38+
*/
2139
final class SparkPredicateToVortexExpressionTest {
2240

2341
private static final StructType ADDRESS = DataTypes.createStructType(new org.apache.spark.sql.types.StructField[] {
@@ -30,8 +48,48 @@ final class SparkPredicateToVortexExpressionTest {
3048
DataTypes.createStructField("address", ADDRESS, true)
3149
});
3250

33-
private static final Map<String, DataType> SCHEMA =
34-
Map.of("id", DataTypes.IntegerType, "name", DataTypes.StringType, "profile", PROFILE);
51+
private static final DataType DECIMAL = DataTypes.createDecimalType(10, 2);
52+
53+
/** One data column per literal type {@code convertLiteral} maps, so every literal meets a same-typed column. */
54+
private static final Map<String, DataType> SCHEMA = Map.ofEntries(
55+
Map.entry("id", DataTypes.IntegerType),
56+
Map.entry("name", DataTypes.StringType),
57+
Map.entry("active", DataTypes.BooleanType),
58+
Map.entry("tiny", DataTypes.ByteType),
59+
Map.entry("small", DataTypes.ShortType),
60+
Map.entry("big", DataTypes.LongType),
61+
Map.entry("ratio", DataTypes.FloatType),
62+
Map.entry("weight", DataTypes.DoubleType),
63+
Map.entry("payload", DataTypes.BinaryType),
64+
Map.entry("birthday", DataTypes.DateType),
65+
Map.entry("createdAt", DataTypes.TimestampType),
66+
Map.entry("createdLocal", DataTypes.TimestampNTZType),
67+
Map.entry("amount", DECIMAL),
68+
Map.entry("profile", PROFILE));
69+
70+
private static final List<String> COMPARISON_OPERATORS = List.of("=", "<>", "!=", ">", ">=", "<", "<=");
71+
72+
/** The columns whose type {@code isPushableLiteral} accepts a {@code null} value for. */
73+
private static final List<String> NULLABLE_LITERAL_COLUMNS = List.of(
74+
"active",
75+
"tiny",
76+
"small",
77+
"id",
78+
"big",
79+
"ratio",
80+
"weight",
81+
"name",
82+
"payload",
83+
"birthday",
84+
"createdAt",
85+
"createdLocal",
86+
"amount");
87+
88+
@BeforeAll
89+
static void loadNativeLibrary() {
90+
// `convert` allocates native expressions; `isPushable` does not.
91+
NativeLoader.loadJni();
92+
}
3593

3694
@Test
3795
@DisplayName("Top-level column reference is pushable when present in the schema")
@@ -83,6 +141,208 @@ void emptyReferenceIsNotPushable() {
83141
assertFalse(SparkPredicateToVortexExpression.isPushable(equality, SCHEMA));
84142
}
85143

144+
@Test
145+
@DisplayName("Every accepted comparison operator converts, with the column on either side")
146+
void everyComparisonOperatorConverts() {
147+
for (String op : COMPARISON_OPERATORS) {
148+
assertPushableAndConvertible(predicate(op, ref("id"), literal(42)), op + " with column on the left");
149+
// Spark's V2 builder sometimes commutes; `convertComparison` swaps the operator back.
150+
assertPushableAndConvertible(predicate(op, literal(42), ref("id")), op + " with column on the right");
151+
}
152+
}
153+
154+
@Test
155+
@DisplayName("Comparison between two columns converts")
156+
void columnToColumnComparisonConverts() {
157+
assertPushableAndConvertible(equality(ref("id"), ref("profile", "address", "zip")));
158+
}
159+
160+
@Test
161+
@DisplayName("Comparison between two literals is rejected by both sides")
162+
void literalToLiteralComparisonIsRejected() {
163+
assertNotPushableAndNotConvertible(equality(literal(1), literal(2)));
164+
}
165+
166+
@Test
167+
@DisplayName("Comparison with the wrong number of children is rejected by both sides")
168+
void comparisonWithWrongArityIsRejected() {
169+
assertNotPushableAndNotConvertible(predicate("=", ref("id")));
170+
assertNotPushableAndNotConvertible(predicate("=", ref("id"), literal(1), literal(2)));
171+
}
172+
173+
@Test
174+
@DisplayName("Nested column comparison converts")
175+
void nestedColumnComparisonConverts() {
176+
assertPushableAndConvertible(equality(ref("profile", "email"), literal("[email protected]")));
177+
}
178+
179+
@Test
180+
@DisplayName("IS_NULL and IS_NOT_NULL convert for top-level and nested columns")
181+
void nullChecksConvert() {
182+
assertPushableAndConvertible(predicate("IS_NULL", ref("name")));
183+
assertPushableAndConvertible(predicate("IS_NOT_NULL", ref("name")));
184+
assertPushableAndConvertible(predicate("IS_NULL", ref("profile", "address", "city")));
185+
}
186+
187+
@Test
188+
@DisplayName("IN converts for a single literal and for many literals")
189+
void inConverts() {
190+
assertPushableAndConvertible(predicate("IN", ref("id"), literal(1)));
191+
assertPushableAndConvertible(predicate("IN", ref("id"), literal(1), literal(2), literal(3)));
192+
}
193+
194+
@Test
195+
@DisplayName("IN with no literals is rejected by both sides")
196+
void inWithoutLiteralsIsRejected() {
197+
assertNotPushableAndNotConvertible(predicate("IN", ref("id")));
198+
}
199+
200+
@Test
201+
@DisplayName("String matching predicates convert, including LIKE meta-characters in the needle")
202+
void stringMatchesConvert() {
203+
for (String name : List.of("STARTS_WITH", "ENDS_WITH", "CONTAINS")) {
204+
assertPushableAndConvertible(predicate(name, ref("name"), literal("ali")), name);
205+
// `buildLikePattern` escapes `%`, `_` and `\` so the match stays an exact substring.
206+
assertPushableAndConvertible(predicate(name, ref("name"), literal("100%_a\\b")), name + " with escapes");
207+
}
208+
}
209+
210+
@Test
211+
@DisplayName("String matching against a non-string literal is rejected by both sides")
212+
void stringMatchAgainstNonStringLiteralIsRejected() {
213+
assertNotPushableAndNotConvertible(predicate("STARTS_WITH", ref("name"), literal(1)));
214+
}
215+
216+
@Test
217+
@DisplayName("BOOLEAN_EXPRESSION over a column reference converts")
218+
void bareBooleanColumnConverts() {
219+
assertPushableAndConvertible(predicate("BOOLEAN_EXPRESSION", ref("active")));
220+
}
221+
222+
@Test
223+
@DisplayName("An unrecognised predicate name is rejected by both sides")
224+
void unknownPredicateNameIsRejected() {
225+
assertNotPushableAndNotConvertible(predicate("BLOOM_FILTER", ref("id"), literal(1)));
226+
}
227+
228+
@Test
229+
@DisplayName("AlwaysTrue and AlwaysFalse convert to boolean literals")
230+
void constantPredicatesConvert() {
231+
assertPushableAndConvertible(new AlwaysTrue());
232+
assertPushableAndConvertible(new AlwaysFalse());
233+
}
234+
235+
@Test
236+
@DisplayName("AND, OR and NOT convert when every leaf converts")
237+
void compoundPredicatesConvert() {
238+
Predicate left = equality(ref("id"), literal(1));
239+
Predicate right = predicate("IS_NOT_NULL", ref("name"));
240+
assertPushableAndConvertible(new And(left, right));
241+
assertPushableAndConvertible(new Or(left, right));
242+
assertPushableAndConvertible(new Not(left));
243+
assertPushableAndConvertible(new Not(new And(left, new Or(right, new AlwaysFalse()))));
244+
}
245+
246+
@Test
247+
@DisplayName("A compound predicate with one unconvertible leaf is rejected by both sides")
248+
void compoundPredicateWithBadLeafIsRejected() {
249+
Predicate good = equality(ref("id"), literal(1));
250+
Predicate bad = predicate("BLOOM_FILTER", ref("id"), literal(1));
251+
assertNotPushableAndNotConvertible(new And(good, bad));
252+
assertNotPushableAndNotConvertible(new Or(bad, good));
253+
assertNotPushableAndNotConvertible(new Not(bad));
254+
}
255+
256+
@Test
257+
@DisplayName("Every literal type that accepts a null value converts")
258+
void nullLiteralsConvert() {
259+
for (String column : NULLABLE_LITERAL_COLUMNS) {
260+
assertPushableAndConvertible(
261+
equality(ref(column), new LiteralValue<>(null, SCHEMA.get(column))), "null literal for " + column);
262+
}
263+
}
264+
265+
@Test
266+
@DisplayName("Every non-null literal type the translator maps converts")
267+
void nonNullLiteralsConvert() {
268+
assertPushableAndConvertible(equality(ref("active"), new LiteralValue<>(true, DataTypes.BooleanType)));
269+
assertPushableAndConvertible(equality(ref("tiny"), new LiteralValue<>((byte) 1, DataTypes.ByteType)));
270+
assertPushableAndConvertible(equality(ref("small"), new LiteralValue<>((short) 1, DataTypes.ShortType)));
271+
assertPushableAndConvertible(equality(ref("id"), literal(42)));
272+
assertPushableAndConvertible(equality(ref("big"), new LiteralValue<>(1L, DataTypes.LongType)));
273+
assertPushableAndConvertible(equality(ref("ratio"), new LiteralValue<>(1.5f, DataTypes.FloatType)));
274+
assertPushableAndConvertible(equality(ref("weight"), new LiteralValue<>(1.5d, DataTypes.DoubleType)));
275+
assertPushableAndConvertible(equality(ref("name"), literal("alice")));
276+
assertPushableAndConvertible(
277+
equality(ref("payload"), new LiteralValue<>(new byte[] {1, 2, 3}, DataTypes.BinaryType)));
278+
// Spark encodes DateType as an epoch-day int and both timestamp types as epoch micros.
279+
assertPushableAndConvertible(equality(ref("birthday"), new LiteralValue<>(19_000, DataTypes.DateType)));
280+
assertPushableAndConvertible(
281+
equality(ref("createdAt"), new LiteralValue<>(1_700_000_000L, DataTypes.TimestampType)));
282+
assertPushableAndConvertible(
283+
equality(ref("createdLocal"), new LiteralValue<>(1_700_000_000L, DataTypes.TimestampNTZType)));
284+
assertPushableAndConvertible(equality(ref("amount"), decimalLiteral("12.34")));
285+
}
286+
287+
@Test
288+
@DisplayName("A literal type with no Vortex representation is rejected by both sides")
289+
void unrepresentableLiteralIsRejected() {
290+
assertNotPushableAndNotConvertible(equality(ref("id"), new LiteralValue<>(null, DataTypes.NullType)));
291+
}
292+
293+
@Test
294+
@DisplayName("A decimal literal that does not fit the declared scale is rejected by both sides")
295+
void decimalThatDoesNotFitTheScaleIsRejected() {
296+
// `unscaledValueOf` calls `setScale(2)` without a rounding mode, so 12.345 throws and
297+
// `isPushableLiteral` falls through to `literalOf`, which is empty.
298+
assertNotPushableAndNotConvertible(equality(ref("amount"), decimalLiteral("12.345")));
299+
}
300+
301+
@Test
302+
@DisplayName("An empty named reference is rejected by convert as well as by isPushable")
303+
void emptyReferenceIsAlsoNotConvertible() {
304+
// `isFieldRefExpr` on the convert path only checks `instanceof NamedReference`, so the
305+
// zero-part guard lives in `columnOf`. Without it a pushable-looking predicate would reach
306+
// the reader and be silently dropped.
307+
assertNotPushableAndNotConvertible(equality(ref(), literal(1)));
308+
assertNotPushableAndNotConvertible(predicate("IS_NULL", ref()));
309+
assertNotPushableAndNotConvertible(predicate("IN", ref(), literal(1)));
310+
assertNotPushableAndNotConvertible(predicate("STARTS_WITH", ref(), literal("a")));
311+
assertNotPushableAndNotConvertible(predicate("BOOLEAN_EXPRESSION", ref()));
312+
}
313+
314+
/**
315+
* Asserts the invariant documented on {@link SparkPredicateToVortexExpression#isPushable(Predicate, Map)}: a
316+
* predicate Spark is allowed to drop must produce a Vortex expression.
317+
*/
318+
private static void assertPushableAndConvertible(Predicate predicate) {
319+
assertPushableAndConvertible(predicate, predicate.name());
320+
}
321+
322+
private static void assertPushableAndConvertible(Predicate predicate, String what) {
323+
assertTrue(SparkPredicateToVortexExpression.isPushable(predicate, SCHEMA), () -> "not pushable: " + what);
324+
assertTrue(
325+
SparkPredicateToVortexExpression.convert(predicate).isPresent(),
326+
() -> "pushable but not convertible: " + what);
327+
}
328+
329+
private static void assertNotPushableAndNotConvertible(Predicate predicate) {
330+
assertNotPushableAndNotConvertible(predicate, predicate.name());
331+
}
332+
333+
private static void assertNotPushableAndNotConvertible(Predicate predicate, String what) {
334+
assertFalse(SparkPredicateToVortexExpression.isPushable(predicate, SCHEMA), () -> "pushable: " + what);
335+
assertFalse(SparkPredicateToVortexExpression.convert(predicate).isPresent(), () -> "convertible: " + what);
336+
}
337+
338+
private static Predicate predicate(String name, Expression... children) {
339+
return new Predicate(name, children);
340+
}
341+
342+
private static LiteralValue<Object> decimalLiteral(String value) {
343+
return new LiteralValue<>(Decimal.apply(new BigDecimal(value)), DECIMAL);
344+
}
345+
86346
private static Predicate equality(Expression left, Expression right) {
87347
return new Predicate("=", new Expression[] {left, right});
88348
}

0 commit comments

Comments
 (0)