[SPARK-20591][WEB UI] Change succeeded task num of All jobs page to the same as the job detail page(by excluding speculated tasks).

fjh100456 · fjh100456 · commit d80f14013d9a · 2017-05-13T15:15:44.000+08:00
## What changes were proposed in this pull request?

Count the completed tasks num by taskId to excluding speculated tasks.
Add a "completedIndices"(OpenHashSet) to record the the completed tasks by id, and show the completed tasks num by "job.completedIndices.size" instead of "job.numCompletedTasks".

## How was this patch tested?

manual tests
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
@@ -629,7 +629,8 @@ private[ui] class JobPagedTable(
         {if (job.numSkippedStages > 0) s"(${job.numSkippedStages} skipped)"}
       </td>
       <td class="progress-cell">
-        {UIUtils.makeProgressBar(started = job.numActiveTasks, completed = job.numCompletedTasks,
+        {UIUtils.makeProgressBar(started = job.numActiveTasks,
+        completed = job.completedStageIndices.size,
         failed = job.numFailedTasks, skipped = job.numSkippedTasks,
         reasonToNumKilled = job.reasonToNumKilled, total = job.numTasks - job.numSkippedTasks)}
       </td>
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
@@ -423,6 +423,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
         jobData.numActiveTasks -= 1
         taskEnd.reason match {
           case Success =>
+            jobData.completedIndices.add(info.index)
             jobData.numCompletedTasks += 1
           case kill: TaskKilled =>
             jobData.reasonToNumKilled = jobData.reasonToNumKilled.updated(
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
@@ -299,7 +299,7 @@ private[ui] class StagePagedTable(
         <td>{data.formattedDuration}</td>
         <td class="progress-cell">
           {UIUtils.makeProgressBar(started = stageData.numActiveTasks,
-          completed = stageData.numCompleteTasks, failed = stageData.numFailedTasks,
+          completed = stageData.completedIndices.size, failed = stageData.numFailedTasks,
           skipped = 0, reasonToNumKilled = stageData.reasonToNumKilled, total = info.numTasks)}
         </td>
         <td>{data.inputReadWithUnit}</td>
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
@@ -62,6 +62,7 @@ private[spark] object UIData {
     var numTasks: Int = 0,
     var numActiveTasks: Int = 0,
     var numCompletedTasks: Int = 0,
+    var completedIndices: OpenHashSet[Int] = new OpenHashSet[Int](),
     var numSkippedTasks: Int = 0,
     var numFailedTasks: Int = 0,
     var reasonToNumKilled: Map[String, Int] = Map.empty,
