tests for taskSummary

apache · Apr 1, 2015 · 63eb4a6 · 63eb4a6
1 parent ad27de8
commit 63eb4a6
Show file tree

Hide file tree

Showing 7 changed files with 12,144 additions and 0 deletions.
diff --git a/core/src/test/resources/HistoryServerExpectations/applications/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/json_expectation
@@ -1,4 +1,11 @@
 [ {
+  "id" : "local-1427397477963",
+  "name" : "Spark shell",
+  "startTime" : "2015-03-26T19:17:57.184GMT",
+  "endTime" : "2015-03-26T19:20:02.949GMT",
+  "sparkUser" : "irashid",
+  "completed" : true
+}, {
   "id" : "local-1426533911241",
   "name" : "Spark shell",
   "startTime" : "2015-03-16T19:25:10.242GMT",

diff --git a/...verExpectations/applications/local-1427397477963/stages/20/0/taskSummary/json_expectation b/...verExpectations/applications/local-1427397477963/stages/20/0/taskSummary/json_expectation
@@ -0,0 +1,15 @@
+{
+  "quantiles" : [ 0.05, 0.25, 0.5, 0.75, 0.95 ],
+  "executorDeserializeTime" : [ 0.0, 0.0, 0.0, 1.0, 3.0 ],
+  "executorRunTime" : [ 6.0, 6.0, 7.0, 10.0, 48.0 ],
+  "resultSize" : [ 930.0, 930.0, 930.0, 930.0, 930.0 ],
+  "jvmGcTime" : [ 0.0, 0.0, 0.0, 0.0, 7.0 ],
+  "resultSerializationTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+  "memoryBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+  "diskBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+  "shuffleWriteMetrics" : {
+    "writeBytes" : [ 9598.0, 9599.0, 9600.0, 9601.0, 9603.0 ],
+    "writeRecords" : [ 100.0, 100.0, 100.0, 100.0, 100.0 ],
+    "writeTime" : [ 278805.0, 308809.0, 327953.0, 367487.0, 944783.0 ]
+  }
+}
diff --git a/...ions/local-1427397477963/stages/20/0/taskSummary?quantiles=0.01,0.5,0.99/json_expectation b/...ions/local-1427397477963/stages/20/0/taskSummary?quantiles=0.01,0.5,0.99/json_expectation
@@ -0,0 +1,15 @@
+{
+  "quantiles" : [ 0.01, 0.5, 0.99 ],
+  "executorDeserializeTime" : [ 0.0, 0.0, 25.0 ],
+  "executorRunTime" : [ 5.0, 7.0, 123.0 ],
+  "resultSize" : [ 930.0, 930.0, 930.0 ],
+  "jvmGcTime" : [ 0.0, 0.0, 14.0 ],
+  "resultSerializationTime" : [ 0.0, 0.0, 1.0 ],
+  "memoryBytesSpilled" : [ 0.0, 0.0, 0.0 ],
+  "diskBytesSpilled" : [ 0.0, 0.0, 0.0 ],
+  "shuffleWriteMetrics" : {
+    "writeBytes" : [ 9597.0, 9600.0, 9614.0 ],
+    "writeRecords" : [ 100.0, 100.0, 100.0 ],
+    "writeTime" : [ 260991.0, 327953.0, 8980526.0 ]
+  }
+}
diff --git a/...test/resources/HistoryServerExpectations/applications?minDate=2015-02-10/json_expectation b/...test/resources/HistoryServerExpectations/applications?minDate=2015-02-10/json_expectation
@@ -1,4 +1,11 @@
 [ {
+  "id" : "local-1427397477963",
+  "name" : "Spark shell",
+  "startTime" : "2015-03-26T19:17:57.184GMT",
+  "endTime" : "2015-03-26T19:20:02.949GMT",
+  "sparkUser" : "irashid",
+  "completed" : true
+}, {
   "id" : "local-1426533911241",
   "name" : "Spark shell",
   "startTime" : "2015-03-16T19:25:10.242GMT",

diff --git a/...c/test/resources/HistoryServerExpectations/applications?status=completed/json_expectation b/...c/test/resources/HistoryServerExpectations/applications?status=completed/json_expectation
@@ -1,4 +1,11 @@
 [ {
+  "id" : "local-1427397477963",
+  "name" : "Spark shell",
+  "startTime" : "2015-03-26T19:17:57.184GMT",
+  "endTime" : "2015-03-26T19:20:02.949GMT",
+  "sparkUser" : "irashid",
+  "completed" : true
+}, {
   "id" : "local-1426533911241",
   "name" : "Spark shell",
   "startTime" : "2015-03-16T19:25:10.242GMT",

diff --git a/core/src/test/resources/spark-events/local-1427397477963 b/core/src/test/resources/spark-events/local-1427397477963
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -80,6 +80,9 @@ class HistoryServerSuite extends FunSuite with BeforeAndAfter with Matchers with
     "failed stage list json" -> "applications/local-1422981780767/stages?status=failed",
     "one stage json" -> "applications/local-1422981780767/stages/1",
     "one stage attempt json" -> "applications/local-1422981780767/stages/1/0",
+    "stage task summary" -> "applications/local-1427397477963/stages/20/0/taskSummary",
+    "stage task summary w/ custom quantiles" ->
+      "applications/local-1427397477963/stages/20/0/taskSummary?quantiles=0.01,0.5,0.99",
     "stage list with accumulable json" -> "applications/local-1426533911241/stages",
     "stage with accumulable json" -> "applications/local-1426533911241/stages/0/0",
     "rdd list storage json" -> "applications/local-1422981780767/storage/rdd",
@@ -150,6 +153,13 @@ class HistoryServerSuite extends FunSuite with BeforeAndAfter with Matchers with
     badStageId2._1 should be (HttpServletResponse.SC_NOT_FOUND)
     // will take some mucking w/ jersey to get a better error msg in this case
 
+
+    val badQuantiles = getContentAndCode(
+      "applications/local-1427397477963/stages/20/0/taskSummary?quantiles=foo,0.1")
+    badQuantiles._1 should be (HttpServletResponse.SC_BAD_REQUEST)
+    badQuantiles._3 should be (Some("Bad value for parameter \"quantiles\".  Expected a double, " +
+      "got \"foo\""))
+
     getContentAndCode("foobar")._1 should be (HttpServletResponse.SC_NOT_FOUND)
   }