@@ -1363,9 +1363,14 @@ object CometSparkSessionExtensions extends Logging {
13631363
13641364 /** Calculates required memory overhead in MB per executor process for Comet. */
13651365 def getCometMemoryOverheadInMiB (sparkConf : SparkConf ): Long = {
1366- // `spark.executor.memory` default value is 1g
1367- val executorMemoryMiB = ConfigHelpers
1368- .byteFromString(sparkConf.get(" spark.executor.memory" , " 1024MB" ), ByteUnit .MiB )
1366+ val baseMemoryMiB = if (cometUnifiedMemoryManagerEnabled(sparkConf)) {
1367+ ConfigHelpers
1368+ .byteFromString(sparkConf.get(" spark.memory.offHeap.size" ), ByteUnit .MiB )
1369+ } else {
1370+ // `spark.executor.memory` default value is 1g
1371+ ConfigHelpers
1372+ .byteFromString(sparkConf.get(" spark.executor.memory" , " 1024MB" ), ByteUnit .MiB )
1373+ }
13691374
13701375 val minimum = ConfigHelpers
13711376 .byteFromString(
@@ -1381,7 +1386,7 @@ object CometSparkSessionExtensions extends Logging {
13811386 .getOption(COMET_MEMORY_OVERHEAD .key)
13821387 .map(ConfigHelpers .byteFromString(_, ByteUnit .MiB ))
13831388
1384- overHeadMemFromConf.getOrElse(math.max((overheadFactor * executorMemoryMiB ).toLong, minimum))
1389+ overHeadMemFromConf.getOrElse(math.max((overheadFactor * baseMemoryMiB ).toLong, minimum))
13851390 }
13861391
13871392 /** Calculates required memory overhead in bytes per executor process for Comet. */
@@ -1390,7 +1395,7 @@ object CometSparkSessionExtensions extends Logging {
13901395 }
13911396
13921397 /** Calculates required shuffle memory size in bytes per executor process for Comet. */
1393- def getCometShuffleMemorySize (sparkConf : SparkConf , conf : SQLConf ): Long = {
1398+ def getCometShuffleMemorySize (sparkConf : SparkConf , conf : SQLConf = SQLConf .get ): Long = {
13941399 val cometMemoryOverhead = getCometMemoryOverheadInMiB(sparkConf)
13951400
13961401 val overheadFactor = COMET_COLUMNAR_SHUFFLE_MEMORY_FACTOR .get(conf)
@@ -1408,6 +1413,16 @@ object CometSparkSessionExtensions extends Logging {
14081413 }
14091414 }
14101415
1416+ def cometUnifiedMemoryManagerEnabled (sparkConf : SparkConf ): Boolean = {
1417+ sparkConf.getBoolean(" spark.memory.offHeap.enabled" , false )
1418+ }
1419+
1420+ def cometShuffleUnifiedMemoryManagerInTestEnabled (sparkConf : SparkConf ): Boolean = {
1421+ sparkConf.getBoolean(
1422+ CometConf .COMET_COLUMNAR_SHUFFLE_UNIFIED_MEMORY_ALLOCATOR_IN_TEST .key,
1423+ CometConf .COMET_COLUMNAR_SHUFFLE_UNIFIED_MEMORY_ALLOCATOR_IN_TEST .defaultValue.get)
1424+ }
1425+
14111426 /**
14121427 * Attaches explain information to a TreeNode, rolling up the corresponding information tags
14131428 * from any child nodes. For now, we are using this to attach the reasons why certain Spark
0 commit comments