Skip to content

Commit 36ab57c

Browse files
authored
fix: inject Comet session extension rules only once per session (#6230)
* fix: inject Comet session extension rules only once per session A session can receive CometSparkSessionExtensions more than once, for example through both spark.sql.extensions and SparkSession.Builder.withExtensions. Each application injected another copy of every Comet rule, so the rules ran twice per plan. Skip the injection when the extension has already been applied to the same SparkSessionExtensions instance. * ci: register CometSessionExtensionsSuite in the PR build workflows
1 parent 68c4e9b commit 36ab57c

4 files changed

Lines changed: 75 additions & 0 deletions

File tree

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -539,6 +539,7 @@ jobs:
539539
org.apache.comet.QueryContextInternerSuite
540540
org.apache.comet.CometSetOpWithGroupBySuite
541541
org.apache.comet.CometSparkSessionExtensionsSuite
542+
org.apache.spark.sql.comet.CometSessionExtensionsSuite
542543
org.apache.spark.CometPluginsSuite
543544
org.apache.spark.CometRuntimeShutdownSuite
544545
org.apache.spark.CometTaskMemoryManagerSuite

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ jobs:
187187
org.apache.comet.QueryContextInternerSuite
188188
org.apache.comet.CometSetOpWithGroupBySuite
189189
org.apache.comet.CometSparkSessionExtensionsSuite
190+
org.apache.spark.sql.comet.CometSessionExtensionsSuite
190191
org.apache.spark.CometPluginsSuite
191192
org.apache.spark.CometRuntimeShutdownSuite
192193
org.apache.spark.CometTaskMemoryManagerSuite

‎spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,13 @@ class CometSparkSessionExtensions
9090
with Logging
9191
with ShimCometSparkSessionExtensions {
9292
override def apply(extensions: SparkSessionExtensions): Unit = {
93+
// A session can be handed this extension more than once, for example through both
94+
// spark.sql.extensions and SparkSession.Builder.withExtensions. Injecting twice would run
95+
// every Comet rule twice per plan.
96+
if (!CometSparkSessionExtensions.markConfigured(extensions)) {
97+
logDebug("Comet extension already applied to these session extensions; skipping")
98+
return
99+
}
93100
extensions.injectColumnar { session => CometColumnar(session) }
94101
// Pre-3.5 only: tag AQE DPP regions so the conversion rules below leave them Spark-native.
95102
// Registered before CometRule so tags are in place when conversion runs.
@@ -117,6 +124,15 @@ object CometSparkSessionExtensions extends Logging {
117124
lazy val isBigEndian: Boolean = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)
118125
private val SHUFFLE_MANAGER_KEY = "spark.shuffle.manager"
119126

127+
/** Session extensions Comet has already been injected into. Weak so sessions can be GC'd. */
128+
private val configuredExtensions =
129+
java.util.Collections.synchronizedMap(
130+
new java.util.WeakHashMap[SparkSessionExtensions, java.lang.Boolean]())
131+
132+
/** Records that Comet is being injected into `extensions`; false if it already was. */
133+
private def markConfigured(extensions: SparkSessionExtensions): Boolean =
134+
configuredExtensions.put(extensions, java.lang.Boolean.TRUE) == null
135+
120136
/**
121137
* Checks whether Comet extension should be loaded for Spark.
122138
*/
Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.spark.sql.comet
21+
22+
import org.apache.spark.sql.{CometTestBase, SparkSessionExtensions}
23+
24+
import org.apache.comet.CometSparkSessionExtensions
25+
import org.apache.comet.rules.CometRule
26+
27+
class CometSessionExtensionsSuite extends CometTestBase {
28+
29+
private def countCometRules(extensions: SparkSessionExtensions): (Int, Int) = {
30+
val columnar = extensions
31+
.buildColumnarRules(spark)
32+
.count(_.isInstanceOf[CometSparkSessionExtensions#CometColumnar])
33+
val queryStagePrep = extensions
34+
.buildQueryStagePrepRules(spark)
35+
.count(_.isInstanceOf[CometRule])
36+
(columnar, queryStagePrep)
37+
}
38+
39+
test("Comet rules are injected once when the extension is applied twice") {
40+
val extensions = new SparkSessionExtensions
41+
// Separate instances, as when the extension is registered both through
42+
// spark.sql.extensions and SparkSession.Builder.withExtensions.
43+
new CometSparkSessionExtensions().apply(extensions)
44+
new CometSparkSessionExtensions().apply(extensions)
45+
assert(countCometRules(extensions) == ((1, 1)))
46+
}
47+
48+
test("each session extensions instance gets its own Comet rules") {
49+
val first = new SparkSessionExtensions
50+
val second = new SparkSessionExtensions
51+
val comet = new CometSparkSessionExtensions()
52+
comet.apply(first)
53+
comet.apply(second)
54+
assert(countCometRules(first) == ((1, 1)))
55+
assert(countCometRules(second) == ((1, 1)))
56+
}
57+
}

0 commit comments

Comments
 (0)