Use BUFFER_WRITE_CHUNK_SIZE

caneGuy · caneGuy · commit 9d3004d5fc91 · 2017-08-24T22:46:31.000+08:00
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -321,6 +321,13 @@ package object config {
       .intConf
       .createWithDefault(3)
 
+  private[spark] val BUFFER_WRITE_CHUNK_SIZE =
+    ConfigBuilder("spark.buffer.write.chunkSize")
+      .internal()
+      .doc("The block size limit when use ChunkedByteBuffer to writeFully bytes.")
+      .bytesConf(ByteUnit.BYTE)
+      .createWithDefault(64 * 1024 * 1024)
+
   private[spark] val REDUCER_MAX_REQ_SIZE_SHUFFLE_TO_MEM =
     ConfigBuilder("spark.reducer.maxReqSizeShuffleToMem")
       .internal()
@@ -336,11 +343,4 @@ package object config {
         "spark.")
       .booleanConf
       .createWithDefault(false)
-
-  private[spark] val STORAGE_NIO_BUFFER_LIMIT =
-    ConfigBuilder("spark.storage.nioBufferLimit")
-      .internal()
-      .doc("The block size limit when use ChunkedByteBuffer to writeFully bytes.")
-      .bytesConf(ByteUnit.BYTE)
-      .createWithDefault(64 * 1024 * 1024)
 }
diff --git a/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala b/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala
@@ -42,7 +42,7 @@ private[spark] class ChunkedByteBuffer(var chunks: Array[ByteBuffer]) {
   require(chunks.forall(_.position() == 0), "chunks' positions must be 0")
 
   // Chunk size in bytes
-  private val NIO_BUFFER_LIMIT = SparkEnv.get.conf.get(config.STORAGE_NIO_BUFFER_LIMIT)
+  private val NIO_BUFFER_LIMIT = SparkEnv.get.conf.get(config.BUFFER_WRITE_CHUNK_SIZE)
 
   private[this] var disposed: Boolean = false
 
