Skip to content

Commit a159357

Browse files
committed
roll back diff changes
1 parent c12096e commit a159357

4 files changed

Lines changed: 7 additions & 198 deletions

File tree

‎dev/diffs/3.4.3.diff‎

Lines changed: 2 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -918,7 +918,7 @@ index b5b34922694..a72403780c4 100644
918918
protected val baseResourcePath = {
919919
// use the same way as `SQLQueryTestSuite` to get the resource path
920920
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
921-
index 525d97e4998..e205689a6a9 100644
921+
index 525d97e4998..f600e162da3 100644
922922
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
923923
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
924924
@@ -1508,7 +1508,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
@@ -931,25 +931,7 @@ index 525d97e4998..e205689a6a9 100644
931931
AccumulatorSuite.verifyPeakExecutionMemorySet(sparkContext, "external sort") {
932932
sql("SELECT * FROM testData2 ORDER BY a ASC, b ASC").collect()
933933
}
934-
@@ -1960,8 +1961,16 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
935-
countAcc.add(1)
936-
x
937-
})
938-
+ // Comet's `CometProject` and `CometHashAggregate` do not implement Spark's cross-sibling
939-
+ // subexpression elimination over `ScalaUDF`, so each reference invokes the UDF body
940-
+ // separately. The other call sites in this test pass against Comet because the source
941-
+ // (`testData2`, a `LocalRelation`) is not Comet-scannable and the project runs on Spark's
942-
+ // path; the `agg` case routes through `CometHashAggregate` once an Exchange enters the
943-
+ // plan. TODO(comet#XXXX): add cross-sibling CSE to both `CometProject` and the aggregate
944-
+ // operator's input projection.
945-
verifyCallCount(
946-
- df.agg(sum(testUdf($"b") + testUdf($"b") + testUdf($"b"))), Row(3.0), 1)
947-
+ df.agg(sum(testUdf($"b") + testUdf($"b") + testUdf($"b"))), Row(3.0),
948-
+ if (isCometEnabled) 3 else 1)
949-
950-
verifyCallCount(
951-
df.selectExpr("testUdf(a + 1) + testUdf(1 + a)", "testUdf(a + 1)"), Row(4, 2), 1)
952-
@@ -3730,7 +3739,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
934+
@@ -3730,7 +3731,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
953935
}
954936
}
955937

@@ -959,36 +941,6 @@ index 525d97e4998..e205689a6a9 100644
959941
val sc = spark.sparkContext
960942
val hiveVersion = "2.3.9"
961943
// transitive=false, only download specified jar
962-
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
963-
index 2dabcf01be7..9bc0be5d9aa 100644
964-
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
965-
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
966-
@@ -491,8 +491,23 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession with SQLHelper
967-
s"Schema did not match for query #$i\n${expected.sql}: $output") {
968-
output.schema
969-
}
970-
- assertResult(expected.output, s"Result did not match" +
971-
- s" for query #$i\n${expected.sql}") { output.output }
972-
+ // Comet may surface errors as `CometNativeException` instead of the matching Spark
973-
+ // exception class when DataFusion's parquet row filter wraps the typed error via
974-
+ // `format!("{e:?}")`, dropping the JNI bridge's ability to downcast. Same category,
975-
+ // different surface. Collapse both sides to a placeholder when this happens so the
976-
+ // literal compare passes. TODO(comet#XXXX): remove once DataFusion preserves the typed
977-
+ // error end to end.
978-
+ val (expectedOut, actualOut) = if (isCometEnabled &&
979-
+ expected.output.startsWith("org.apache.spark.SparkArithmeticException") &&
980-
+ expected.output.contains("\"DIVIDE_BY_ZERO\"") &&
981-
+ output.output.startsWith("org.apache.comet.CometNativeException") &&
982-
+ output.output.contains("DivideByZero")) {
983-
+ ("[DIVIDE_BY_ZERO]", "[DIVIDE_BY_ZERO]")
984-
+ } else {
985-
+ (expected.output, output.output)
986-
+ }
987-
+ assertResult(expectedOut, s"Result did not match" +
988-
+ s" for query #$i\n${expected.sql}") { actualOut }
989-
}
990-
}
991-
}
992944
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
993945
index 48ad10992c5..51d1ee65422 100644
994946
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala

‎dev/diffs/3.5.8.diff‎

Lines changed: 2 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -937,7 +937,7 @@ index c26757c9cff..d55775f09d7 100644
937937
protected val baseResourcePath = {
938938
// use the same way as `SQLQueryTestSuite` to get the resource path
939939
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
940-
index 3cf2bfd17ab..8a166271e65 100644
940+
index 3cf2bfd17ab..a3effb1eeb8 100644
941941
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
942942
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
943943
@@ -1521,7 +1521,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
@@ -950,25 +950,7 @@ index 3cf2bfd17ab..8a166271e65 100644
950950
AccumulatorSuite.verifyPeakExecutionMemorySet(sparkContext, "external sort") {
951951
sql("SELECT * FROM testData2 ORDER BY a ASC, b ASC").collect()
952952
}
953-
@@ -1979,8 +1980,16 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
954-
countAcc.add(1)
955-
x
956-
})
957-
+ // Comet's `CometProject` and `CometHashAggregate` do not implement Spark's cross-sibling
958-
+ // subexpression elimination over `ScalaUDF`, so each reference invokes the UDF body
959-
+ // separately. The other call sites in this test pass against Comet because the source
960-
+ // (`testData2`, a `LocalRelation`) is not Comet-scannable and the project runs on Spark's
961-
+ // path; the `agg` case routes through `CometHashAggregate` once an Exchange enters the
962-
+ // plan. TODO(comet#XXXX): add cross-sibling CSE to both `CometProject` and the aggregate
963-
+ // operator's input projection.
964-
verifyCallCount(
965-
- df.agg(sum(testUdf($"b") + testUdf($"b") + testUdf($"b"))), Row(3.0), 1)
966-
+ df.agg(sum(testUdf($"b") + testUdf($"b") + testUdf($"b"))), Row(3.0),
967-
+ if (isCometEnabled) 3 else 1)
968-
969-
verifyCallCount(
970-
df.selectExpr("testUdf(a + 1) + testUdf(1 + a)", "testUdf(a + 1)"), Row(4, 2), 1)
971-
@@ -3750,7 +3759,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
953+
@@ -3750,7 +3751,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
972954
}
973955
}
974956

@@ -978,37 +960,6 @@ index 3cf2bfd17ab..8a166271e65 100644
978960
val sc = spark.sparkContext
979961
val hiveVersion = "2.3.9"
980962
// transitive=false, only download specified jar
981-
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
982-
index 71af1fd69c3..da40c939b78 100644
983-
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
984-
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
985-
@@ -872,9 +872,24 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession with SQLHelper
986-
s"Schema did not match for query #$i\n${expected.sql}: $output") {
987-
output.schema
988-
}
989-
- assertResult(expected.output, s"Result did not match" +
990-
+ // Comet may surface errors as `CometNativeException` instead of the matching Spark
991-
+ // exception class when DataFusion's parquet row filter wraps the typed error via
992-
+ // `format!("{e:?}")`, dropping the JNI bridge's ability to downcast. Same category,
993-
+ // different surface. Collapse both sides to a placeholder when this happens so the
994-
+ // literal compare passes. TODO(comet#XXXX): remove once DataFusion preserves the typed
995-
+ // error end to end.
996-
+ val (expectedOut, actualOut) = if (isCometEnabled &&
997-
+ expected.output.startsWith("org.apache.spark.SparkArithmeticException") &&
998-
+ expected.output.contains("\"DIVIDE_BY_ZERO\"") &&
999-
+ output.output.startsWith("org.apache.comet.CometNativeException") &&
1000-
+ output.output.contains("DivideByZero")) {
1001-
+ ("[DIVIDE_BY_ZERO]", "[DIVIDE_BY_ZERO]")
1002-
+ } else {
1003-
+ (expected.output, output.output)
1004-
+ }
1005-
+ assertResult(expectedOut, s"Result did not match" +
1006-
s" for query #$i\n${expected.sql}") {
1007-
- output.output
1008-
+ actualOut
1009-
}
1010-
}
1011-
}
1012963
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
1013964
index 8b4ac474f87..3f79f20822f 100644
1014965
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala

‎dev/diffs/4.0.2.diff‎

Lines changed: 1 addition & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1072,7 +1072,7 @@ index ad424b3a7cc..4ece0117a34 100644
10721072
protected val baseResourcePath = {
10731073
// use the same way as `SQLQueryTestSuite` to get the resource path
10741074
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
1075-
index f294ff81021..37793afed44 100644
1075+
index f294ff81021..7775027bcee 100644
10761076
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
10771077
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
10781078
@@ -1524,7 +1524,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
@@ -1085,55 +1085,6 @@ index f294ff81021..37793afed44 100644
10851085
AccumulatorSuite.verifyPeakExecutionMemorySet(sparkContext, "external sort") {
10861086
sql("SELECT * FROM testData2 ORDER BY a ASC, b ASC").collect()
10871087
}
1088-
@@ -1985,8 +1986,16 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
1089-
countAcc.add(1)
1090-
x
1091-
})
1092-
+ // Comet's `CometProject` and `CometHashAggregate` do not implement Spark's cross-sibling
1093-
+ // subexpression elimination over `ScalaUDF`, so each reference invokes the UDF body
1094-
+ // separately. The other call sites in this test pass against Comet because the source
1095-
+ // (`testData2`, a `LocalRelation`) is not Comet-scannable and the project runs on Spark's
1096-
+ // path; the `agg` case routes through `CometHashAggregate` once an Exchange enters the
1097-
+ // plan. TODO(comet#XXXX): add cross-sibling CSE to both `CometProject` and the aggregate
1098-
+ // operator's input projection.
1099-
verifyCallCount(
1100-
- df.agg(sum(testUdf($"b") + testUdf($"b") + testUdf($"b"))), Row(3.0), 1)
1101-
+ df.agg(sum(testUdf($"b") + testUdf($"b") + testUdf($"b"))), Row(3.0),
1102-
+ if (isCometEnabled) 3 else 1)
1103-
1104-
verifyCallCount(
1105-
df.selectExpr("testUdf(a + 1) + testUdf(1 + a)", "testUdf(a + 1)"), Row(4, 2), 1)
1106-
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
1107-
index 575a4ae69d1..37f975c0e21 100644
1108-
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
1109-
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
1110-
@@ -679,9 +679,24 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession with SQLHelper
1111-
s"Schema did not match for query #$i\n${expected.sql}: $output") {
1112-
output.schema
1113-
}
1114-
- assertResult(expected.output, s"Result did not match" +
1115-
+ // Comet may surface errors as `CometNativeException` instead of the matching Spark
1116-
+ // exception class when DataFusion's parquet row filter wraps the typed error via
1117-
+ // `format!("{e:?}")`, dropping the JNI bridge's ability to downcast. Same category,
1118-
+ // different surface. Collapse both sides to a placeholder when this happens so the
1119-
+ // literal compare passes. TODO(comet#XXXX): remove once DataFusion preserves the typed
1120-
+ // error end to end.
1121-
+ val (expectedOut, actualOut) = if (isCometEnabled &&
1122-
+ expected.output.startsWith("org.apache.spark.SparkArithmeticException") &&
1123-
+ expected.output.contains("\"DIVIDE_BY_ZERO\"") &&
1124-
+ output.output.startsWith("org.apache.comet.CometNativeException") &&
1125-
+ output.output.contains("DivideByZero")) {
1126-
+ ("[DIVIDE_BY_ZERO]", "[DIVIDE_BY_ZERO]")
1127-
+ } else {
1128-
+ (expected.output, output.output)
1129-
+ }
1130-
+ assertResult(expectedOut, s"Result did not match" +
1131-
s" for query #$i\n${expected.sql}") {
1132-
- output.output
1133-
+ actualOut
1134-
}
1135-
}
1136-
}
11371088
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
11381089
index c1c041509c3..7d463e4b85e 100644
11391090
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala

‎dev/diffs/4.1.1.diff‎

Lines changed: 2 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1143,7 +1143,7 @@ index e4b5e10f7c3..c6efde09c8a 100644
11431143
protected val baseResourcePath = {
11441144
// use the same way as `SQLQueryTestSuite` to get the resource path
11451145
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
1146-
index 74cdee49e55..0b2607579bc 100644
1146+
index 74cdee49e55..3decf393ed0 100644
11471147
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
11481148
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
11491149
@@ -1521,7 +1521,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
@@ -1156,26 +1156,8 @@ index 74cdee49e55..0b2607579bc 100644
11561156
AccumulatorSuite.verifyPeakExecutionMemorySet(sparkContext, "external sort") {
11571157
sql("SELECT * FROM testData2 ORDER BY a ASC, b ASC").collect()
11581158
}
1159-
@@ -1982,8 +1983,16 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
1160-
countAcc.add(1)
1161-
x
1162-
})
1163-
+ // Comet's `CometProject` and `CometHashAggregate` do not implement Spark's cross-sibling
1164-
+ // subexpression elimination over `ScalaUDF`, so each reference invokes the UDF body
1165-
+ // separately. The other call sites in this test pass against Comet because the source
1166-
+ // (`testData2`, a `LocalRelation`) is not Comet-scannable and the project runs on Spark's
1167-
+ // path; the `agg` case routes through `CometHashAggregate` once an Exchange enters the
1168-
+ // plan. TODO(comet#XXXX): add cross-sibling CSE to both `CometProject` and the aggregate
1169-
+ // operator's input projection.
1170-
verifyCallCount(
1171-
- df.agg(sum(testUdf($"b") + testUdf($"b") + testUdf($"b"))), Row(3.0), 1)
1172-
+ df.agg(sum(testUdf($"b") + testUdf($"b") + testUdf($"b"))), Row(3.0),
1173-
+ if (isCometEnabled) 3 else 1)
1174-
1175-
verifyCallCount(
1176-
df.selectExpr("testUdf(a + 1) + testUdf(1 + a)", "testUdf(a + 1)"), Row(4, 2), 1)
11771159
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
1178-
index 23f0144dcec..4672b1b6513 100644
1160+
index 23f0144dcec..df845f7295a 100644
11791161
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
11801162
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
11811163
@@ -166,7 +166,16 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession with SQLHelper
@@ -1196,33 +1178,6 @@ index 23f0144dcec..4672b1b6513 100644
11961178
) ++ otherIgnoreList
11971179
/** List of test cases that require TPCDS table schemas to be loaded. */
11981180
private def requireTPCDSCases: Seq[String] = Seq("pipe-operators.sql")
1199-
@@ -682,9 +691,24 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession with SQLHelper
1200-
s"Schema did not match for query #$i\n${expected.sql}: $output") {
1201-
output.schema
1202-
}
1203-
- assertResult(expected.output, s"Result did not match" +
1204-
+ // Comet may surface errors as `CometNativeException` instead of the matching Spark
1205-
+ // exception class when DataFusion's parquet row filter wraps the typed error via
1206-
+ // `format!("{e:?}")`, dropping the JNI bridge's ability to downcast. Same category,
1207-
+ // different surface. Collapse both sides to a placeholder when this happens so the
1208-
+ // literal compare passes. TODO(comet#XXXX): remove once DataFusion preserves the typed
1209-
+ // error end to end.
1210-
+ val (expectedOut, actualOut) = if (isCometEnabled &&
1211-
+ expected.output.startsWith("org.apache.spark.SparkArithmeticException") &&
1212-
+ expected.output.contains("\"DIVIDE_BY_ZERO\"") &&
1213-
+ output.output.startsWith("org.apache.comet.CometNativeException") &&
1214-
+ output.output.contains("DivideByZero")) {
1215-
+ ("[DIVIDE_BY_ZERO]", "[DIVIDE_BY_ZERO]")
1216-
+ } else {
1217-
+ (expected.output, output.output)
1218-
+ }
1219-
+ assertResult(expectedOut, s"Result did not match" +
1220-
s" for query #$i\n${expected.sql}") {
1221-
- output.output
1222-
+ actualOut
1223-
}
1224-
}
1225-
}
12261181
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
12271182
index 66826a9ca76..ab4265a5fb9 100644
12281183
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala

0 commit comments

Comments
 (0)