lazily convert ApplicationInfo to avoid memory overhead

jeanlyn · Apr 27, 2015 · 2e19be2 · 2e19be2
1 parent ba3d9d2
commit 2e19be2
Show file tree

Hide file tree

Showing 5 changed files with 9 additions and 9 deletions.
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -162,8 +162,8 @@ class HistoryServer(
     provider.getListing()
   }
 
-  def getApplicationInfoList: Seq[ApplicationInfo] = {
-    getApplicationList().map { ApplicationsListResource.appHistoryInfoToPublicAppInfo }.toSeq
+  def getApplicationInfoList: Iterator[ApplicationInfo] = {
+    getApplicationList().iterator.map(ApplicationsListResource.appHistoryInfoToPublicAppInfo)
   }
 
   /**

diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
@@ -66,12 +66,12 @@ class MasterWebUI(val master: Master, requestedPort: Int)
     ui.getHandlers.foreach(detachHandler)
   }
 
-  def getApplicationInfoList: Seq[ApplicationInfo] = {
+  def getApplicationInfoList: Iterator[ApplicationInfo] = {
     val state = masterPage.getMasterState
     val activeApps = state.activeApps.sortBy(_.startTime).reverse
     val completedApps = state.completedApps.sortBy(_.endTime).reverse
-    activeApps.map { ApplicationsListResource.convertApplicationInfo(_, false) } ++
-      completedApps.map { ApplicationsListResource.convertApplicationInfo(_, true) }
+    activeApps.iterator.map { ApplicationsListResource.convertApplicationInfo(_, false) } ++
+      completedApps.iterator.map { ApplicationsListResource.convertApplicationInfo(_, true) }
   }
 
   def getSparkUI(appId: String): Option[SparkUI] = {

diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
@@ -31,7 +31,7 @@ private[v1] class ApplicationListResource(uiRoot: UIRoot) {
     @QueryParam("status") status: JList[ApplicationStatus],
     @DefaultValue("2010-01-01") @QueryParam("minDate") minDate: SimpleDateParam,
     @DefaultValue("3000-01-01") @QueryParam("maxDate") maxDate: SimpleDateParam
-  ): Seq[ApplicationInfo] = {
+  ): Iterator[ApplicationInfo] = {
     val allApps = uiRoot.getApplicationInfoList
     val adjStatus = {
       if (status.isEmpty) {

diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/JsonRootResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/JsonRootResource.scala
@@ -103,7 +103,7 @@ private[spark] object JsonRootResource {
  */
 private[spark] trait UIRoot {
   def getSparkUI(appKey: String): Option[SparkUI]
-  def getApplicationInfoList: Seq[ApplicationInfo]
+  def getApplicationInfoList: Iterator[ApplicationInfo]
 
   /**
    * Get the spark UI with the given appID, and apply a function

diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -92,8 +92,8 @@ private[spark] class SparkUI private (
     if (appId == appName) Some(this) else None
   }
 
-  def getApplicationInfoList: Seq[ApplicationInfo] = {
-    Seq(new ApplicationInfo(
+  def getApplicationInfoList: Iterator[ApplicationInfo] = {
+    Iterator(new ApplicationInfo(
       id = appName,
       name = appName,
       startTime = new Date(startTime),