Skip to content

Commit 0cbb528

Browse files
jliworkjliwork
committed
address comments to improve tests
1 parent fc34568 commit 0cbb528

File tree

2 files changed

+10
-24
lines changed

2 files changed

+10
-24
lines changed

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ class DataSourceStrategySuite extends PlanTest with SharedSQLContext {
150150

151151
test("translate complex expression") {
152152
val attrInt = 'cint.int
153-
153+
// Functions such as 'Abs' are not supported
154154
assertResult(None) {
155155
DataSourceStrategy.translateFilter(
156156
expressions.LessThanOrEqual(
@@ -176,6 +176,7 @@ class DataSourceStrategySuite extends PlanTest with SharedSQLContext {
176176
))
177177
}
178178
// SPARK-22548 Incorrect nested AND expression pushed down to JDBC data source
179+
// Functions such as 'Abs' are not supported
179180
assertResult(None) {
180181
DataSourceStrategy.translateFilter(expressions.Or(
181182
expressions.And(
@@ -189,6 +190,7 @@ class DataSourceStrategySuite extends PlanTest with SharedSQLContext {
189190
)
190191
))
191192
}
193+
// Functions such as 'Abs' are not supported
192194
assertResult(None) {
193195
DataSourceStrategy.translateFilter(
194196
expressions.Not(expressions.And(
@@ -223,6 +225,7 @@ class DataSourceStrategySuite extends PlanTest with SharedSQLContext {
223225
)
224226
))
225227
}
228+
// Functions such as 'Abs' are not supported
226229
assertResult(None) {
227230
DataSourceStrategy.translateFilter(expressions.Or(
228231
expressions.Or(
@@ -255,6 +258,7 @@ class DataSourceStrategySuite extends PlanTest with SharedSQLContext {
255258
)
256259
))
257260
}
261+
// Functions such as 'Abs' are not supported
258262
assertResult(None) {
259263
DataSourceStrategy.translateFilter(expressions.And(
260264
expressions.And(
@@ -286,6 +290,7 @@ class DataSourceStrategySuite extends PlanTest with SharedSQLContext {
286290
)
287291
))
288292
}
293+
// Functions such as 'Abs' are not supported
289294
assertResult(None) {
290295
DataSourceStrategy.translateFilter(expressions.And(
291296
expressions.Or(

‎sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala‎

Lines changed: 4 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -296,33 +296,14 @@ class JDBCSuite extends SparkFunSuite
296296
// The older versions of spark have this kind of bugs in parquet data source.
297297
val df1 = sql("SELECT * FROM foobar WHERE NOT (THEID != 2 AND NAME != 'mary')")
298298
val df2 = sql("SELECT * FROM foobar WHERE NOT (THEID != 2) OR NOT (NAME != 'mary')")
299-
val df3 = sql("SELECT * FROM foobar WHERE (THEID > 0 AND NAME = 'mary') OR (NAME = 'fred')")
300-
val df4 = sql("SELECT * FROM foobar " +
301-
"WHERE (THEID > 0 AND TRIM(NAME) = 'mary') OR (NAME = 'fred')")
302-
val df5 = sql("SELECT * FROM foobar " +
303-
"WHERE THEID > 0 AND TRIM(NAME) = 'mary' AND LENGTH(NAME) > 3")
304-
val df6 = sql("SELECT * FROM foobar " +
305-
"WHERE THEID < 0 OR NAME = 'mary' OR NAME = 'fred'")
306-
val df7 = sql("SELECT * FROM foobar " +
307-
"WHERE THEID < 0 OR TRIM(NAME) = 'mary' OR NAME = 'fred'")
308-
val df8 = sql("SELECT * FROM foobar " +
309-
"WHERE NOT((THEID < 0 OR NAME != 'mary') AND (THEID != 1 OR NAME != 'fred'))")
310-
val df9 = sql("SELECT * FROM foobar " +
311-
"WHERE NOT((THEID < 0 OR NAME != 'mary') AND (THEID != 1 OR TRIM(NAME) != 'fred'))")
312-
val df10 = sql("SELECT * FROM foobar " +
313-
"WHERE (NOT(THEID < 0 OR TRIM(NAME) != 'mary')) OR (THEID = 1 AND NAME = 'fred')")
314299

315300
assert(df1.collect.toSet === Set(Row("mary", 2)))
316301
assert(df2.collect.toSet === Set(Row("mary", 2)))
317-
assert(df3.collect.toSet === Set(Row("fred", 1), Row("mary", 2)))
318-
assert(df4.collect.toSet === Set(Row("fred", 1), Row("mary", 2)))
319-
assert(df5.collect.toSet === Set(Row("mary", 2)))
320-
assert(df6.collect.toSet === Set(Row("fred", 1), Row("mary", 2)))
321-
assert(df7.collect.toSet === Set(Row("fred", 1), Row("mary", 2)))
322-
assert(df8.collect.toSet === Set(Row("fred", 1), Row("mary", 2)))
323-
assert(df9.collect.toSet === Set(Row("fred", 1), Row("mary", 2)))
324-
assert(df10.collect.toSet === Set(Row("fred", 1), Row("mary", 2)))
325302

303+
// SPARK-22548: Incorrect nested AND expression pushed down to JDBC data source
304+
val df3 = sql("SELECT * FROM foobar " +
305+
"WHERE (THEID > 0 AND TRIM(NAME) = 'mary') OR (NAME = 'fred')")
306+
assert(df3.collect.toSet === Set(Row("fred", 1), Row("mary", 2)))
326307

327308
def checkNotPushdown(df: DataFrame): DataFrame = {
328309
val parentPlan = df.queryExecution.executedPlan

0 commit comments

Comments
 (0)