66import static org .junit .jupiter .api .Assertions .assertFalse ;
77import static org .junit .jupiter .api .Assertions .assertTrue ;
88
9+ import dev .vortex .jni .NativeLoader ;
10+ import java .math .BigDecimal ;
11+ import java .util .List ;
912import java .util .Map ;
1013import org .apache .spark .sql .connector .expressions .Expression ;
1114import org .apache .spark .sql .connector .expressions .LiteralValue ;
1215import org .apache .spark .sql .connector .expressions .NamedReference ;
16+ import org .apache .spark .sql .connector .expressions .filter .AlwaysFalse ;
17+ import org .apache .spark .sql .connector .expressions .filter .AlwaysTrue ;
18+ import org .apache .spark .sql .connector .expressions .filter .And ;
19+ import org .apache .spark .sql .connector .expressions .filter .Not ;
20+ import org .apache .spark .sql .connector .expressions .filter .Or ;
1321import org .apache .spark .sql .connector .expressions .filter .Predicate ;
1422import org .apache .spark .sql .types .DataType ;
1523import org .apache .spark .sql .types .DataTypes ;
24+ import org .apache .spark .sql .types .Decimal ;
1625import org .apache .spark .sql .types .StructType ;
26+ import org .junit .jupiter .api .BeforeAll ;
1727import org .junit .jupiter .api .DisplayName ;
1828import org .junit .jupiter .api .Test ;
1929
20- /** Unit tests for {@link SparkPredicateToVortexExpression#isPushable(Predicate, Map)}. */
30+ /**
31+ * Unit tests for {@link SparkPredicateToVortexExpression#isPushable(Predicate, Map)} and
32+ * {@link SparkPredicateToVortexExpression#convert(Predicate)}.
33+ *
34+ * <p>{@code isPushable} decides which predicates {@code VortexScanBuilder.pushPredicates} lets Spark drop, while
35+ * {@code convert} builds the filter {@code VortexPartitionReader} actually pushes down. A predicate that passes the
36+ * first but fails the second is silently skipped by the reader, so the scan returns rows the query excluded. The tests
37+ * below pin that {@code isPushable} implies {@code convert().isPresent()} across every accepted shape.
38+ */
2139final class SparkPredicateToVortexExpressionTest {
2240
2341 private static final StructType ADDRESS = DataTypes .createStructType (new org .apache .spark .sql .types .StructField [] {
@@ -30,8 +48,48 @@ final class SparkPredicateToVortexExpressionTest {
3048 DataTypes .createStructField ("address" , ADDRESS , true )
3149 });
3250
33- private static final Map <String , DataType > SCHEMA =
34- Map .of ("id" , DataTypes .IntegerType , "name" , DataTypes .StringType , "profile" , PROFILE );
51+ private static final DataType DECIMAL = DataTypes .createDecimalType (10 , 2 );
52+
53+ /** One data column per literal type {@code convertLiteral} maps, so every literal meets a same-typed column. */
54+ private static final Map <String , DataType > SCHEMA = Map .ofEntries (
55+ Map .entry ("id" , DataTypes .IntegerType ),
56+ Map .entry ("name" , DataTypes .StringType ),
57+ Map .entry ("active" , DataTypes .BooleanType ),
58+ Map .entry ("tiny" , DataTypes .ByteType ),
59+ Map .entry ("small" , DataTypes .ShortType ),
60+ Map .entry ("big" , DataTypes .LongType ),
61+ Map .entry ("ratio" , DataTypes .FloatType ),
62+ Map .entry ("weight" , DataTypes .DoubleType ),
63+ Map .entry ("payload" , DataTypes .BinaryType ),
64+ Map .entry ("birthday" , DataTypes .DateType ),
65+ Map .entry ("createdAt" , DataTypes .TimestampType ),
66+ Map .entry ("createdLocal" , DataTypes .TimestampNTZType ),
67+ Map .entry ("amount" , DECIMAL ),
68+ Map .entry ("profile" , PROFILE ));
69+
70+ private static final List <String > COMPARISON_OPERATORS = List .of ("=" , "<>" , "!=" , ">" , ">=" , "<" , "<=" );
71+
72+ /** The columns whose type {@code isPushableLiteral} accepts a {@code null} value for. */
73+ private static final List <String > NULLABLE_LITERAL_COLUMNS = List .of (
74+ "active" ,
75+ "tiny" ,
76+ "small" ,
77+ "id" ,
78+ "big" ,
79+ "ratio" ,
80+ "weight" ,
81+ "name" ,
82+ "payload" ,
83+ "birthday" ,
84+ "createdAt" ,
85+ "createdLocal" ,
86+ "amount" );
87+
88+ @ BeforeAll
89+ static void loadNativeLibrary () {
90+ // `convert` allocates native expressions; `isPushable` does not.
91+ NativeLoader .loadJni ();
92+ }
3593
3694 @ Test
3795 @ DisplayName ("Top-level column reference is pushable when present in the schema" )
@@ -83,6 +141,208 @@ void emptyReferenceIsNotPushable() {
83141 assertFalse (SparkPredicateToVortexExpression .isPushable (equality , SCHEMA ));
84142 }
85143
144+ @ Test
145+ @ DisplayName ("Every accepted comparison operator converts, with the column on either side" )
146+ void everyComparisonOperatorConverts () {
147+ for (String op : COMPARISON_OPERATORS ) {
148+ assertPushableAndConvertible (predicate (op , ref ("id" ), literal (42 )), op + " with column on the left" );
149+ // Spark's V2 builder sometimes commutes; `convertComparison` swaps the operator back.
150+ assertPushableAndConvertible (predicate (op , literal (42 ), ref ("id" )), op + " with column on the right" );
151+ }
152+ }
153+
154+ @ Test
155+ @ DisplayName ("Comparison between two columns converts" )
156+ void columnToColumnComparisonConverts () {
157+ assertPushableAndConvertible (equality (ref ("id" ), ref ("profile" , "address" , "zip" )));
158+ }
159+
160+ @ Test
161+ @ DisplayName ("Comparison between two literals is rejected by both sides" )
162+ void literalToLiteralComparisonIsRejected () {
163+ assertNotPushableAndNotConvertible (equality (literal (1 ), literal (2 )));
164+ }
165+
166+ @ Test
167+ @ DisplayName ("Comparison with the wrong number of children is rejected by both sides" )
168+ void comparisonWithWrongArityIsRejected () {
169+ assertNotPushableAndNotConvertible (predicate ("=" , ref ("id" )));
170+ assertNotPushableAndNotConvertible (predicate ("=" , ref ("id" ), literal (1 ), literal (2 )));
171+ }
172+
173+ @ Test
174+ @ DisplayName ("Nested column comparison converts" )
175+ void nestedColumnComparisonConverts () {
176+ assertPushableAndConvertible (
equality (
ref (
"profile" ,
"email" ),
literal (
"[email protected] " )));
177+ }
178+
179+ @ Test
180+ @ DisplayName ("IS_NULL and IS_NOT_NULL convert for top-level and nested columns" )
181+ void nullChecksConvert () {
182+ assertPushableAndConvertible (predicate ("IS_NULL" , ref ("name" )));
183+ assertPushableAndConvertible (predicate ("IS_NOT_NULL" , ref ("name" )));
184+ assertPushableAndConvertible (predicate ("IS_NULL" , ref ("profile" , "address" , "city" )));
185+ }
186+
187+ @ Test
188+ @ DisplayName ("IN converts for a single literal and for many literals" )
189+ void inConverts () {
190+ assertPushableAndConvertible (predicate ("IN" , ref ("id" ), literal (1 )));
191+ assertPushableAndConvertible (predicate ("IN" , ref ("id" ), literal (1 ), literal (2 ), literal (3 )));
192+ }
193+
194+ @ Test
195+ @ DisplayName ("IN with no literals is rejected by both sides" )
196+ void inWithoutLiteralsIsRejected () {
197+ assertNotPushableAndNotConvertible (predicate ("IN" , ref ("id" )));
198+ }
199+
200+ @ Test
201+ @ DisplayName ("String matching predicates convert, including LIKE meta-characters in the needle" )
202+ void stringMatchesConvert () {
203+ for (String name : List .of ("STARTS_WITH" , "ENDS_WITH" , "CONTAINS" )) {
204+ assertPushableAndConvertible (predicate (name , ref ("name" ), literal ("ali" )), name );
205+ // `buildLikePattern` escapes `%`, `_` and `\` so the match stays an exact substring.
206+ assertPushableAndConvertible (predicate (name , ref ("name" ), literal ("100%_a\\ b" )), name + " with escapes" );
207+ }
208+ }
209+
210+ @ Test
211+ @ DisplayName ("String matching against a non-string literal is rejected by both sides" )
212+ void stringMatchAgainstNonStringLiteralIsRejected () {
213+ assertNotPushableAndNotConvertible (predicate ("STARTS_WITH" , ref ("name" ), literal (1 )));
214+ }
215+
216+ @ Test
217+ @ DisplayName ("BOOLEAN_EXPRESSION over a column reference converts" )
218+ void bareBooleanColumnConverts () {
219+ assertPushableAndConvertible (predicate ("BOOLEAN_EXPRESSION" , ref ("active" )));
220+ }
221+
222+ @ Test
223+ @ DisplayName ("An unrecognised predicate name is rejected by both sides" )
224+ void unknownPredicateNameIsRejected () {
225+ assertNotPushableAndNotConvertible (predicate ("BLOOM_FILTER" , ref ("id" ), literal (1 )));
226+ }
227+
228+ @ Test
229+ @ DisplayName ("AlwaysTrue and AlwaysFalse convert to boolean literals" )
230+ void constantPredicatesConvert () {
231+ assertPushableAndConvertible (new AlwaysTrue ());
232+ assertPushableAndConvertible (new AlwaysFalse ());
233+ }
234+
235+ @ Test
236+ @ DisplayName ("AND, OR and NOT convert when every leaf converts" )
237+ void compoundPredicatesConvert () {
238+ Predicate left = equality (ref ("id" ), literal (1 ));
239+ Predicate right = predicate ("IS_NOT_NULL" , ref ("name" ));
240+ assertPushableAndConvertible (new And (left , right ));
241+ assertPushableAndConvertible (new Or (left , right ));
242+ assertPushableAndConvertible (new Not (left ));
243+ assertPushableAndConvertible (new Not (new And (left , new Or (right , new AlwaysFalse ()))));
244+ }
245+
246+ @ Test
247+ @ DisplayName ("A compound predicate with one unconvertible leaf is rejected by both sides" )
248+ void compoundPredicateWithBadLeafIsRejected () {
249+ Predicate good = equality (ref ("id" ), literal (1 ));
250+ Predicate bad = predicate ("BLOOM_FILTER" , ref ("id" ), literal (1 ));
251+ assertNotPushableAndNotConvertible (new And (good , bad ));
252+ assertNotPushableAndNotConvertible (new Or (bad , good ));
253+ assertNotPushableAndNotConvertible (new Not (bad ));
254+ }
255+
256+ @ Test
257+ @ DisplayName ("Every literal type that accepts a null value converts" )
258+ void nullLiteralsConvert () {
259+ for (String column : NULLABLE_LITERAL_COLUMNS ) {
260+ assertPushableAndConvertible (
261+ equality (ref (column ), new LiteralValue <>(null , SCHEMA .get (column ))), "null literal for " + column );
262+ }
263+ }
264+
265+ @ Test
266+ @ DisplayName ("Every non-null literal type the translator maps converts" )
267+ void nonNullLiteralsConvert () {
268+ assertPushableAndConvertible (equality (ref ("active" ), new LiteralValue <>(true , DataTypes .BooleanType )));
269+ assertPushableAndConvertible (equality (ref ("tiny" ), new LiteralValue <>((byte ) 1 , DataTypes .ByteType )));
270+ assertPushableAndConvertible (equality (ref ("small" ), new LiteralValue <>((short ) 1 , DataTypes .ShortType )));
271+ assertPushableAndConvertible (equality (ref ("id" ), literal (42 )));
272+ assertPushableAndConvertible (equality (ref ("big" ), new LiteralValue <>(1L , DataTypes .LongType )));
273+ assertPushableAndConvertible (equality (ref ("ratio" ), new LiteralValue <>(1.5f , DataTypes .FloatType )));
274+ assertPushableAndConvertible (equality (ref ("weight" ), new LiteralValue <>(1.5d , DataTypes .DoubleType )));
275+ assertPushableAndConvertible (equality (ref ("name" ), literal ("alice" )));
276+ assertPushableAndConvertible (
277+ equality (ref ("payload" ), new LiteralValue <>(new byte [] {1 , 2 , 3 }, DataTypes .BinaryType )));
278+ // Spark encodes DateType as an epoch-day int and both timestamp types as epoch micros.
279+ assertPushableAndConvertible (equality (ref ("birthday" ), new LiteralValue <>(19_000 , DataTypes .DateType )));
280+ assertPushableAndConvertible (
281+ equality (ref ("createdAt" ), new LiteralValue <>(1_700_000_000L , DataTypes .TimestampType )));
282+ assertPushableAndConvertible (
283+ equality (ref ("createdLocal" ), new LiteralValue <>(1_700_000_000L , DataTypes .TimestampNTZType )));
284+ assertPushableAndConvertible (equality (ref ("amount" ), decimalLiteral ("12.34" )));
285+ }
286+
287+ @ Test
288+ @ DisplayName ("A literal type with no Vortex representation is rejected by both sides" )
289+ void unrepresentableLiteralIsRejected () {
290+ assertNotPushableAndNotConvertible (equality (ref ("id" ), new LiteralValue <>(null , DataTypes .NullType )));
291+ }
292+
293+ @ Test
294+ @ DisplayName ("A decimal literal that does not fit the declared scale is rejected by both sides" )
295+ void decimalThatDoesNotFitTheScaleIsRejected () {
296+ // `unscaledValueOf` calls `setScale(2)` without a rounding mode, so 12.345 throws and
297+ // `isPushableLiteral` falls through to `literalOf`, which is empty.
298+ assertNotPushableAndNotConvertible (equality (ref ("amount" ), decimalLiteral ("12.345" )));
299+ }
300+
301+ @ Test
302+ @ DisplayName ("An empty named reference is rejected by convert as well as by isPushable" )
303+ void emptyReferenceIsAlsoNotConvertible () {
304+ // `isFieldRefExpr` on the convert path only checks `instanceof NamedReference`, so the
305+ // zero-part guard lives in `columnOf`. Without it a pushable-looking predicate would reach
306+ // the reader and be silently dropped.
307+ assertNotPushableAndNotConvertible (equality (ref (), literal (1 )));
308+ assertNotPushableAndNotConvertible (predicate ("IS_NULL" , ref ()));
309+ assertNotPushableAndNotConvertible (predicate ("IN" , ref (), literal (1 )));
310+ assertNotPushableAndNotConvertible (predicate ("STARTS_WITH" , ref (), literal ("a" )));
311+ assertNotPushableAndNotConvertible (predicate ("BOOLEAN_EXPRESSION" , ref ()));
312+ }
313+
314+ /**
315+ * Asserts the invariant documented on {@link SparkPredicateToVortexExpression#isPushable(Predicate, Map)}: a
316+ * predicate Spark is allowed to drop must produce a Vortex expression.
317+ */
318+ private static void assertPushableAndConvertible (Predicate predicate ) {
319+ assertPushableAndConvertible (predicate , predicate .name ());
320+ }
321+
322+ private static void assertPushableAndConvertible (Predicate predicate , String what ) {
323+ assertTrue (SparkPredicateToVortexExpression .isPushable (predicate , SCHEMA ), () -> "not pushable: " + what );
324+ assertTrue (
325+ SparkPredicateToVortexExpression .convert (predicate ).isPresent (),
326+ () -> "pushable but not convertible: " + what );
327+ }
328+
329+ private static void assertNotPushableAndNotConvertible (Predicate predicate ) {
330+ assertNotPushableAndNotConvertible (predicate , predicate .name ());
331+ }
332+
333+ private static void assertNotPushableAndNotConvertible (Predicate predicate , String what ) {
334+ assertFalse (SparkPredicateToVortexExpression .isPushable (predicate , SCHEMA ), () -> "pushable: " + what );
335+ assertFalse (SparkPredicateToVortexExpression .convert (predicate ).isPresent (), () -> "convertible: " + what );
336+ }
337+
338+ private static Predicate predicate (String name , Expression ... children ) {
339+ return new Predicate (name , children );
340+ }
341+
342+ private static LiteralValue <Object > decimalLiteral (String value ) {
343+ return new LiteralValue <>(Decimal .apply (new BigDecimal (value )), DECIMAL );
344+ }
345+
86346 private static Predicate equality (Expression left , Expression right ) {
87347 return new Predicate ("=" , new Expression [] {left , right });
88348 }
0 commit comments