Skip to content

Commit aaf57c8

Browse files
authored
feat: override executor overhead memory only when comet unified memory manager is disabled (apache#1379)
1 parent 71c95fc commit aaf57c8

4 files changed

Lines changed: 41 additions & 11 deletions

File tree

‎spark/src/main/scala/org/apache/comet/CometExecIterator.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,8 @@ class CometExecIterator(
7575
metricsUpdateInterval = COMET_METRICS_UPDATE_INTERVAL.get(),
7676
new CometTaskMemoryManager(id),
7777
batchSize = COMET_BATCH_SIZE.get(),
78-
use_unified_memory_manager = conf.getBoolean("spark.memory.offHeap.enabled", false),
78+
use_unified_memory_manager =
79+
CometSparkSessionExtensions.cometUnifiedMemoryManagerEnabled(conf),
7980
memory_pool_type = COMET_EXEC_MEMORY_POOL_TYPE.get(),
8081
memory_limit = CometSparkSessionExtensions.getCometMemoryOverhead(conf),
8182
memory_limit_per_task = getMemoryLimitPerTask(conf),

‎spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1363,9 +1363,14 @@ object CometSparkSessionExtensions extends Logging {
13631363

13641364
/** Calculates required memory overhead in MB per executor process for Comet. */
13651365
def getCometMemoryOverheadInMiB(sparkConf: SparkConf): Long = {
1366-
// `spark.executor.memory` default value is 1g
1367-
val executorMemoryMiB = ConfigHelpers
1368-
.byteFromString(sparkConf.get("spark.executor.memory", "1024MB"), ByteUnit.MiB)
1366+
val baseMemoryMiB = if (cometUnifiedMemoryManagerEnabled(sparkConf)) {
1367+
ConfigHelpers
1368+
.byteFromString(sparkConf.get("spark.memory.offHeap.size"), ByteUnit.MiB)
1369+
} else {
1370+
// `spark.executor.memory` default value is 1g
1371+
ConfigHelpers
1372+
.byteFromString(sparkConf.get("spark.executor.memory", "1024MB"), ByteUnit.MiB)
1373+
}
13691374

13701375
val minimum = ConfigHelpers
13711376
.byteFromString(
@@ -1381,7 +1386,7 @@ object CometSparkSessionExtensions extends Logging {
13811386
.getOption(COMET_MEMORY_OVERHEAD.key)
13821387
.map(ConfigHelpers.byteFromString(_, ByteUnit.MiB))
13831388

1384-
overHeadMemFromConf.getOrElse(math.max((overheadFactor * executorMemoryMiB).toLong, minimum))
1389+
overHeadMemFromConf.getOrElse(math.max((overheadFactor * baseMemoryMiB).toLong, minimum))
13851390
}
13861391

13871392
/** Calculates required memory overhead in bytes per executor process for Comet. */
@@ -1390,7 +1395,7 @@ object CometSparkSessionExtensions extends Logging {
13901395
}
13911396

13921397
/** Calculates required shuffle memory size in bytes per executor process for Comet. */
1393-
def getCometShuffleMemorySize(sparkConf: SparkConf, conf: SQLConf): Long = {
1398+
def getCometShuffleMemorySize(sparkConf: SparkConf, conf: SQLConf = SQLConf.get): Long = {
13941399
val cometMemoryOverhead = getCometMemoryOverheadInMiB(sparkConf)
13951400

13961401
val overheadFactor = COMET_COLUMNAR_SHUFFLE_MEMORY_FACTOR.get(conf)
@@ -1408,6 +1413,16 @@ object CometSparkSessionExtensions extends Logging {
14081413
}
14091414
}
14101415

1416+
def cometUnifiedMemoryManagerEnabled(sparkConf: SparkConf): Boolean = {
1417+
sparkConf.getBoolean("spark.memory.offHeap.enabled", false)
1418+
}
1419+
1420+
def cometShuffleUnifiedMemoryManagerInTestEnabled(sparkConf: SparkConf): Boolean = {
1421+
sparkConf.getBoolean(
1422+
CometConf.COMET_COLUMNAR_SHUFFLE_UNIFIED_MEMORY_ALLOCATOR_IN_TEST.key,
1423+
CometConf.COMET_COLUMNAR_SHUFFLE_UNIFIED_MEMORY_ALLOCATOR_IN_TEST.defaultValue.get)
1424+
}
1425+
14111426
/**
14121427
* Attaches explain information to a TreeNode, rolling up the corresponding information tags
14131428
* from any child nodes. For now, we are using this to attach the reasons why certain Spark

‎spark/src/main/scala/org/apache/spark/Plugins.scala‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,13 @@ class CometDriverPlugin extends DriverPlugin with Logging with ShimCometDriverPl
6262
Math.max((executorMemory * memoryOverheadFactor).toLong, memoryOverheadMinMib)
6363
}
6464

65-
val cometMemOverhead = CometSparkSessionExtensions.getCometMemoryOverheadInMiB(sc.getConf)
65+
val cometMemOverhead =
66+
if (!CometSparkSessionExtensions.cometUnifiedMemoryManagerEnabled(sc.getConf)) {
67+
CometSparkSessionExtensions.getCometMemoryOverheadInMiB(sc.getConf)
68+
} else {
69+
// comet shuffle unified memory manager is disabled, so we need to add overhead memory
70+
CometSparkSessionExtensions.getCometShuffleMemorySize(sc.getConf)
71+
}
6672
sc.conf.set(EXECUTOR_MEMORY_OVERHEAD.key, s"${execMemOverhead + cometMemOverhead}M")
6773
val newExecMemOverhead = sc.getConf.getSizeAsMb(EXECUTOR_MEMORY_OVERHEAD.key)
6874

@@ -90,13 +96,19 @@ class CometDriverPlugin extends DriverPlugin with Logging with ShimCometDriverPl
9096

9197
/**
9298
* Whether we should override Spark memory configuration for Comet. This only returns true when
93-
* Comet native execution is enabled and/or Comet shuffle is enabled
99+
* Comet native execution is enabled and/or Comet shuffle is enabled and Comet doesn't use
100+
* unified memory manager.
94101
*/
95102
private def shouldOverrideMemoryConf(conf: SparkConf): Boolean = {
96103
conf.getBoolean(CometConf.COMET_ENABLED.key, true) && (
97-
conf.getBoolean(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, false) ||
98-
conf.getBoolean(CometConf.COMET_EXEC_ENABLED.key, false)
99-
)
104+
conf.getBoolean(
105+
CometConf.COMET_EXEC_SHUFFLE_ENABLED.key,
106+
CometConf.COMET_EXEC_SHUFFLE_ENABLED.defaultValue.get) ||
107+
conf.getBoolean(
108+
CometConf.COMET_EXEC_ENABLED.key,
109+
CometConf.COMET_EXEC_ENABLED.defaultValue.get)
110+
) && (!CometSparkSessionExtensions.cometUnifiedMemoryManagerEnabled(conf) ||
111+
!CometSparkSessionExtensions.cometShuffleUnifiedMemoryManagerInTestEnabled(conf))
100112
}
101113
}
102114

‎spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,8 @@ class CometPluginsNonOverrideSuite extends CometTestBase {
126126
conf.set("spark.executor.memoryOverheadFactor", "0.5")
127127
conf.set("spark.plugins", "org.apache.spark.CometPlugin")
128128
conf.set("spark.comet.enabled", "true")
129+
conf.set("spark.comet.exec.shuffle.enabled", "false")
130+
conf.set("spark.comet.exec.enabled", "false")
129131
conf
130132
}
131133

0 commit comments

Comments
 (0)