Adding support for downloading logs directly from Spark UI (apache#23)

(cherry picked from commit 38fa491) (cherry picked from commit 5ff2e10) (cherry picked from commit 0ddb04a) (cherry picked from commit c265023) (cherry picked from commit db6dacb)
curtishoward · Oct 8, 2018 · 8b96932 · 8b96932
1 parent 744ce57
commit 8b96932
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/cook/src/main/scala/org/apache/spark/scheduler/cluster/cook/CoarseCookSchedulerBackend.scala b/cook/src/main/scala/org/apache/spark/scheduler/cluster/cook/CoarseCookSchedulerBackend.scala
@@ -306,13 +306,20 @@ class CoarseCookSchedulerBackend(
       s"[ -z $$KEEP_SPARK_LOCAL_TARS ] || rm -f $$(basename $uri)"
     }
 
+    val logUrlCommand = Seq(
+      "export BASE_SPARK_LOG_URL=\"http://`hostname`:5051/files/download?path=$MESOS_SANDBOX\"",
+      "export SPARK_LOG_URL_STDERR=\"$BASE_SPARK_LOG_URL/stderr\"",
+      "export SPARK_LOG_URL_STDOUT=\"$BASE_SPARK_LOG_URL/stdout\""
+    )
+
     val commandSeq =
       debugCommand ++
         envSettingCommand ++
         uriFetchCommand ++
         shippedTarballsCommand ++
         remoteConfFetchCommand ++
         keystorePullCommand.map(Seq(_)).getOrElse(Seq[String]()) ++
+        logUrlCommand ++
         Seq("set", commandString) ++
         cleanup