Adding support for downloading logs directly from Spark UI (apache#23)

hkothari · Curtis Howard · commit 0ddb04a599e6 · 2018-02-15T15:04:13.000-05:00
(cherry picked from commit 38fa491) (cherry picked from commit 5ff2e10)
diff --git a/cook/src/main/scala/org/apache/spark/scheduler/cluster/cook/CoarseCookSchedulerBackend.scala b/cook/src/main/scala/org/apache/spark/scheduler/cluster/cook/CoarseCookSchedulerBackend.scala
@@ -306,13 +306,20 @@ class CoarseCookSchedulerBackend(
       s"[ -z $$KEEP_SPARK_LOCAL_TARS ] || rm -f $$(basename $uri)"
     }
 
+    val logUrlCommand = Seq(
+      "export BASE_SPARK_LOG_URL=\"http://`hostname`:5051/files/download?path=$MESOS_SANDBOX\"",
+      "export SPARK_LOG_URL_STDERR=\"$BASE_SPARK_LOG_URL/stderr\"",
+      "export SPARK_LOG_URL_STDOUT=\"$BASE_SPARK_LOG_URL/stdout\""
+    )
+
     val commandSeq =
       debugCommand ++
         envSettingCommand ++
         uriFetchCommand ++
         shippedTarballsCommand ++
         remoteConfFetchCommand ++
         keystorePullCommand.map(Seq(_)).getOrElse(Seq[String]()) ++
+        logUrlCommand ++
         Seq("set", commandString) ++
         cleanup
 
