Changes according to the feedback

jerryshao · jerryshao · commit c068f434db53 · 2017-06-01T10:05:27.000+08:00
Change-Id: I6e7870698108a52d577a59478a0f88bc645d1133
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
@@ -149,16 +149,15 @@ class SparkHadoopUtil extends Logging {
     val f = () => FileSystem.getAllStatistics.asScala.map(_.getThreadStatistics.getBytesRead).sum
     val baseline = (Thread.currentThread().getId, f())
 
+    /**
+     * This function may be called in both spawned child threads and parent task thread (in
+     * PythonRDD), and Hadoop FileSystem uses thread local variables to track the statistics.
+     * So we need a map to track the bytes read from the child threads and parent thread,
+     * summing them together to get the bytes read of this task.
+     */
     new Function0[Long] {
       private val bytesReadMap = new mutable.HashMap[Long, Long]()
 
-      /**
-       * Returns a function that can be called to calculate Hadoop FileSystem bytes read.
-       * This function may be called in both spawned child threads and parent task thread (in
-       * PythonRDD), and Hadoop FileSystem uses thread local variables to track the statistics.
-       * So we need a map to track the bytes read from the child threads and parent thread,
-       * summing them together to get the bytes read of this task.
-       */
       override def apply(): Long = {
         bytesReadMap.synchronized {
           bytesReadMap.put(Thread.currentThread().getId, f())
diff --git a/core/src/test/scala/org/apache/spark/metrics/InputOutputMetricsSuite.scala b/core/src/test/scala/org/apache/spark/metrics/InputOutputMetricsSuite.scala
@@ -331,7 +331,6 @@ class InputOutputMetricsSuite extends SparkFunSuite with SharedSparkContext
         buf.iterator
       }.count()
     }
-    assert(bytesRead != 0)
     assert(bytesRead >= tmpFile.length())
   }
 
@@ -347,7 +346,6 @@ class InputOutputMetricsSuite extends SparkFunSuite with SharedSparkContext
         buf.iterator
       }.count()
     }
-    assert(bytesRead != 0)
     assert(bytesRead >= tmpFile.length())
   }
 }

Original file line number	Diff line number	Diff line change
`@@ -331,7 +331,6 @@ class InputOutputMetricsSuite extends SparkFunSuite with SharedSparkContext`
`331`	`331`	`buf.iterator`
`332`	`332`	`}.count()`
`333`	`333`	`}`
`334`		`- assert(bytesRead != 0)`
`335`	`334`	`assert(bytesRead >= tmpFile.length())`
`336`	`335`	`}`
`337`	`336`
`@@ -347,7 +346,6 @@ class InputOutputMetricsSuite extends SparkFunSuite with SharedSparkContext`
`347`	`346`	`buf.iterator`
`348`	`347`	`}.count()`
`349`	`348`	`}`
`350`		`- assert(bytesRead != 0)`
`351`	`349`	`assert(bytesRead >= tmpFile.length())`
`352`	`350`	`}`
`353`	`351`	`}`