Adding some comments.

kellrott · kellrott · commit 44ec35a3733a · 2013-11-16T22:35:51.000-08:00
diff --git a/core/src/main/scala/org/apache/spark/serializer/JavaSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/JavaSerializer.scala
@@ -25,7 +25,10 @@ import org.apache.spark.util.ByteBufferInputStream
 private[spark] class JavaSerializationStream(out: OutputStream) extends SerializationStream {
   val objOut = new ObjectOutputStream(out)
   var counter = 0;
-  //Calling reset to avoid memory leak: http://stackoverflow.com/questions/1281549/memory-leak-traps-in-the-java-standard-api
+  /* Calling reset to avoid memory leak: http://stackoverflow.com/questions/1281549/memory-leak-traps-in-the-java-standard-api
+   * But only call it every 1000th time to avoid bloated serialization streams (when
+   * the stream 'resets' object class descriptions have to be re-written)  
+   */
   def writeObject[T](t: T): SerializationStream = {
     objOut.writeObject(t);
     if (counter >= 1000) {
diff --git a/core/src/test/scala/org/apache/spark/storage/LargeIteratorSuite.scala b/core/src/test/scala/org/apache/spark/storage/LargeIteratorSuite.scala
@@ -35,7 +35,12 @@ object Expander {
 }
 
 class LargeIteratorSuite extends FunSuite with LocalSparkContext {
-
+  /* Tests the ability of Spark to deal with user provided iterators that
+   * generate more data then available memory. In any memory based persistance
+   * Spark will unroll the iterator into an ArrayBuffer for caching, however in
+   * the case that the use defines DISK_ONLY persistance, the iterator will be 
+   * fed directly to the serializer and written to disk.
+   */
   val clusterUrl = "local-cluster[1,1,512]"
   test("Flatmap iterator") {
     sc = new SparkContext(clusterUrl, "mem_test");
