support setting maxCapacity to something different than capacity in kryo Output

koertkuipers · koertkuipers · commit 07324450f3ae · 2014-05-11T16:40:35.000-04:00
diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -39,11 +39,12 @@ class KryoSerializer(conf: SparkConf)
   with Logging
   with Serializable {
 
-  private val bufferSize = conf.getInt("spark.kryoserializer.buffer.mb", 2) * 1024 * 1024
+  private val bufferSizeMb = conf.getInt("spark.kryoserializer.buffer.mb", 2)
+  private val maxBufferSizeMb = conf.getInt("spark.kryoserializer.buffer.max.mb", bufferSizeMb)
   private val referenceTracking = conf.getBoolean("spark.kryo.referenceTracking", true)
   private val registrator = conf.getOption("spark.kryo.registrator")
 
-  def newKryoOutput() = new KryoOutput(bufferSize)
+  def newKryoOutput() = new KryoOutput(bufferSizeMb * 1024 * 1024, maxBufferSizeMb * 1024 * 1024)
 
   def newKryo(): Kryo = {
     val instantiator = new EmptyScalaKryoInstantiator
