Skip to content

Commit

Permalink
style
Browse files Browse the repository at this point in the history
  • Loading branch information
squito committed Mar 10, 2015
1 parent 017c755 commit 56edce0
Show file tree
Hide file tree
Showing 12 changed files with 37 additions and 24 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ private[spark] class HistoryPage(parent: HistoryServer) extends WebUIPage("") {
</h4> ++
appTable
} else {
<h4>No {if (requestedIncomplete) "running" else "completed"} applications found!</h4> ++
<h4>No {if (requestedIncomplete) "running" else "completed"}
applications found!</h4> ++
<p>Did you specify the correct logging directory?
Please verify your setting of <span style="font-style:italic">
spark.history.fs.logDirectory</span> and whether you have the permissions to
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ class HistoryServer(

private val appLoader = new CacheLoader[String, SparkUI] {
override def load(key: String): SparkUI = {
val ui = provider.getAppUI(key).getOrElse(throw new NoSuchElementException(s"no app with key $key"))
val ui = provider.getAppUI(key).getOrElse(
throw new NoSuchElementException(s"no app with key $key"))
attachSparkUI(ui)
ui
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ import org.apache.spark.util.AkkaUtils
*/
private[spark]
class MasterWebUI(val master: Master, requestedPort: Int)
extends WebUI(master.securityMgr, requestedPort, master.conf, name = "MasterUI") with Logging with UIRoot {
extends WebUI(master.securityMgr, requestedPort, master.conf, name = "MasterUI") with Logging
with UIRoot {

val masterActorRef = master.self
val timeout = AkkaUtils.askTimeout(master.conf)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,12 @@ class AllRDDResource(uiRoot: UIRoot) {
@PathParam("appId") appId: String
): Seq[RDDStorageInfo] = {
uiRoot.withSparkUI(appId) { ui =>
//should all access on storageListener also be synchronized?
// should all access on storageListener also be synchronized?
val storageStatusList = ui.storageListener.storageStatusList
val rddInfos = ui.storageListener.rddInfoList
rddInfos.map{rddInfo =>
AllRDDResource.getRDDStorageInfo(rddInfo.id, rddInfo, storageStatusList, includeDetails = false)
AllRDDResource.getRDDStorageInfo(rddInfo.id, rddInfo, storageStatusList,
includeDetails = false)
}

}
Expand All @@ -44,7 +45,10 @@ class AllRDDResource(uiRoot: UIRoot) {

object AllRDDResource {

def getRDDStorageInfo(rddId: Int, listener: StorageListener, includeDetails: Boolean): Option[RDDStorageInfo] = {
def getRDDStorageInfo(
rddId: Int,
listener: StorageListener,
includeDetails: Boolean): Option[RDDStorageInfo] = {
val storageStatusList = listener.storageStatusList
listener.rddInfoList.find(_.id == rddId).map{rddInfo =>
getRDDStorageInfo(rddId, rddInfo, storageStatusList, includeDetails)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ class ApplicationListResource(uiRoot: UIRoot) {
allApps.filter{app =>
val statusOk = (app.completed && includeCompleted) ||
(!app.completed && includeRunning)
val dateOk = app.startTime.getTime >= minDate.timestamp && app.startTime.getTime <= maxDate.timestamp
val dateOk = app.startTime.getTime >= minDate.timestamp &&
app.startTime.getTime <= maxDate.timestamp
statusOk && dateOk
}
}
Expand All @@ -64,7 +65,9 @@ object ApplicationsListResource {
)
}

def convertApplicationInfo(internal: InternalApplicationInfo, completed: Boolean): ApplicationInfo = {
def convertApplicationInfo(
internal: InternalApplicationInfo,
completed: Boolean): ApplicationInfo = {
ApplicationInfo(
id = internal.id,
name = internal.desc.name,
Expand All @@ -75,4 +78,4 @@ object ApplicationsListResource {
)
}

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,4 @@ class ExecutorListResource(uiRoot: UIRoot) {
}
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -126,4 +126,4 @@ class NotFoundException(msg: String) extends WebApplicationException(
.status(Response.Status.NOT_FOUND)
.entity(msg)
.build()
)
)
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,12 @@ class OneStageResource(uiRoot: UIRoot) {
case Some((status,stageInfo)) =>
val stageUiData = listener.synchronized{
listener.stageIdToData.get((stageInfo.stageId, stageInfo.attemptId)).
getOrElse{ throw new SparkException("failed to get full stage data for stage: " + stageInfo.stageId +
":" + stageInfo.attemptId)
getOrElse{ throw new SparkException("failed to get full stage data for stage: " +
stageInfo.stageId + ":" + stageInfo.attemptId)
}
}
AllStagesResource.stageUiToStageData(status, stageInfo, stageUiData, includeDetails = true)
AllStagesResource.stageUiToStageData(status, stageInfo, stageUiData,
includeDetails = true)
case None =>
throw new NotFoundException("unknown stage: " + stageId)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
*/
package org.apache.spark.status.api.v1

//Q: should Tachyon size go in here as well? currently the UI only shows it on the overall storage page ... does
// anybody pay attention to it?
// Q: should Tachyon size go in here as well? currently the UI only shows it on the overall storage
// page ... does anybody pay attention to it?
case class RDDStorageInfo(
id: Int,
name: String,
Expand All @@ -43,4 +43,4 @@ case class RDDPartitionInfo(
memoryUsed: Long,
diskUsed: Long,
executors: Seq[String]
)
)
Original file line number Diff line number Diff line change
Expand Up @@ -53,4 +53,4 @@ case class ShuffleWriteMetrics(
bytesWritten: Long,
writeTime: Long,
recordsWritten: Long
)
)
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ private[ui] class ExecutorsPage(
val maxMem = storageStatusList.map(_.maxMem).sum
val memUsed = storageStatusList.map(_.memUsed).sum
val diskUsed = storageStatusList.map(_.diskUsed).sum
val execInfo = for (statusId <- 0 until storageStatusList.size) yield ExecutorsPage.getExecInfo(listener, statusId)
val execInfo = for (statusId <- 0 until storageStatusList.size) yield
ExecutorsPage.getExecInfo(listener, statusId)
val execInfoSorted = execInfo.sortBy(_.id)
val logsExist = execInfo.filter(_.executorLogs.nonEmpty).nonEmpty

Expand Down
11 changes: 6 additions & 5 deletions core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,15 @@ private[ui] class RDDPage(parent: StorageTab) extends WebUIPage("rdd") {
val parameterId = request.getParameter("id")
require(parameterId != null && parameterId.nonEmpty, "Missing id parameter")
val rddId = parameterId.toInt
val rddStorageInfo = AllRDDResource.getRDDStorageInfo(rddId, listener, includeDetails = true).getOrElse {
// Rather than crashing, render an "RDD Not Found" page
return UIUtils.headerSparkPage("RDD Not Found", Seq[Node](), parent)
val rddStorageInfo = AllRDDResource.getRDDStorageInfo(rddId, listener,
includeDetails = true).getOrElse {
// Rather than crashing, render an "RDD Not Found" page
return UIUtils.headerSparkPage("RDD Not Found", Seq[Node](), parent)
}

// Worker table
val workerTable = UIUtils.listingTable(workerHeader, workerRow, rddStorageInfo.dataDistribution.get,
id = Some("rdd-storage-by-worker-table"))
val workerTable = UIUtils.listingTable(workerHeader, workerRow,
rddStorageInfo.dataDistribution.get, id = Some("rdd-storage-by-worker-table"))

// Block table
val blockTable = UIUtils.listingTable(blockHeader, blockRow, rddStorageInfo.partitions.get,
Expand Down

0 comments on commit 56edce0

Please sign in to comment.