diff --git a/app/main.go b/app/main.go index 159d0b7d6..bab7deea2 100644 --- a/app/main.go +++ b/app/main.go @@ -30,6 +30,7 @@ func init() { registerRPC("/api/get-status", commands.GetStatus) registerRPC("/api/refresh-github-commits", commands.RefreshGithubCommits) registerRPC("/api/refresh-travis-status", commands.RefreshTravisStatus) + registerRPC("/api/refresh-chromebot-status", commands.RefreshChromebotStatus) registerRPC("/api/reserve-task", commands.ReserveTask) registerRPC("/api/update-task-status", commands.UpdateTaskStatus) diff --git a/commands/refresh_chromebot_status.go b/commands/refresh_chromebot_status.go new file mode 100644 index 000000000..427d0e678 --- /dev/null +++ b/commands/refresh_chromebot_status.go @@ -0,0 +1,203 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package commands + +import ( + "cocoon/db" + "encoding/json" + "fmt" + "io/ioutil" + + "google.golang.org/appengine/urlfetch" +) + +// RefreshChromebotStatusResult contains chromebot status results. +type RefreshChromebotStatusResult struct { + Results []*ChromebotResult +} + +// ChromebotResult describes a chromebot build result. +type ChromebotResult struct { + Commit string + State db.TaskStatus +} + +// RefreshChromebotStatus pulls down the latest chromebot builds and updates the +// corresponding task statuses. +func RefreshChromebotStatus(cocoon *db.Cocoon, inputJSON []byte) (interface{}, error) { + linuxResults, err := refreshChromebot(cocoon, "linux_bot", "Linux") + + if err != nil { + return nil, err + } + + macResults, err := refreshChromebot(cocoon, "mac_bot", "Mac") + + if err != nil { + return nil, err + } + + var allResults []*ChromebotResult + allResults = append(allResults, linuxResults...) + allResults = append(allResults, macResults...) + return RefreshChromebotStatusResult{allResults}, nil +} + +func refreshChromebot(cocoon *db.Cocoon, taskName string, builderName string) ([]*ChromebotResult, error) { + tasks, err := cocoon.QueryPendingTasks(taskName) + + if err != nil { + return nil, err + } + + if len(tasks) == 0 { + // Short-circuit. Don't bother fetching Chromebot data if there are no tasks to + // to update. + return make([]*ChromebotResult, 0), nil + } + + buildStatuses, err := fetchChromebotBuildStatuses(cocoon, builderName) + + if err != nil { + return nil, err + } + + for _, fullTask := range tasks { + for _, status := range buildStatuses { + if status.Commit == fullTask.ChecklistEntity.Checklist.Commit.Sha { + task := fullTask.TaskEntity.Task + task.Status = status.State + cocoon.PutTask(fullTask.TaskEntity.Key, task) + } + } + } + + return buildStatuses, nil +} + +func fetchChromebotBuildStatuses(cocoon *db.Cocoon, builderName string) ([]*ChromebotResult, error) { + builderURL := fmt.Sprintf("https://build.chromium.org/p/client.flutter/json/builders/%v", builderName) + + jsonResponse, err := fetchJSON(cocoon, builderURL) + + if err != nil { + return nil, err + } + + buildIds := jsonResponse.(map[string]interface{})["cachedBuilds"].([]interface{}) + + if len(buildIds) > 10 { + // Build IDs are sorted in ascending order, to get the latest 10 builds we + // grab the tail. + buildIds = buildIds[len(buildIds)-10:] + } + + var results []*ChromebotResult + for i := len(buildIds) - 1; i >= 0; i-- { + buildID := buildIds[i] + buildJSON, err := fetchJSON(cocoon, fmt.Sprintf("%v/builds/%v", builderURL, buildID)) + + if err != nil { + return nil, err + } + + results = append(results, &ChromebotResult{ + Commit: getBuildProperty(buildJSON.(map[string]interface{}), "git_revision"), + State: getStatus(buildJSON.(map[string]interface{})), + }) + } + + return results, nil +} + +func fetchJSON(cocoon *db.Cocoon, url string) (interface{}, error) { + httpClient := urlfetch.Client(cocoon.Ctx) + response, err := httpClient.Get(url) + if err != nil { + return nil, err + } + + defer response.Body.Close() + + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, err + } + + var js interface{} + if json.Unmarshal(body, &js) != nil { + return nil, err + } + + return js, nil +} + +// Properties are encoded as: +// +// { +// "properties": [ +// [ +// "name1", +// value1, +// ... things we don't care about ... +// ], +// [ +// "name2", +// value2, +// ... things we don't care about ... +// ] +// ] +// } +func getBuildProperty(buildJSON map[string]interface{}, propertyName string) string { + properties := buildJSON["properties"].([]interface{}) + for _, property := range properties { + if property.([]interface{})[0] == propertyName { + return property.([]interface{})[1].(string) + } + } + return "" +} + +// Parses out whether the build was successful. +// +// Successes are encoded like this: +// +// "text": [ +// "build", +// "successful" +// ] +// +// Exceptions are encoded like this: +// +// "text": [ +// "exception", +// "steps", +// "exception", +// "flutter build apk material_gallery" +// ] +// +// Errors are encoded like this: +// +// "text": [ +// "failed", +// "steps", +// "failed", +// "flutter build ios simulator stocks" +// ] +// +// In-progress builds are encoded like this: +// +// "text": [] +// +func getStatus(buildJSON map[string]interface{}) db.TaskStatus { + text := buildJSON["text"].([]interface{}) + if text == nil || len(text) < 2 { + return db.TaskInProgress + } else if text[1].(string) == "successful" { + return db.TaskSucceeded + } else { + return db.TaskFailed + } +} diff --git a/commands/refresh_travis_status.go b/commands/refresh_travis_status.go index 3bcdf92df..b40676d1c 100644 --- a/commands/refresh_travis_status.go +++ b/commands/refresh_travis_status.go @@ -39,22 +39,8 @@ func RefreshTravisStatus(cocoon *db.Cocoon, inputJSON []byte) (interface{}, erro return RefreshTravisStatusResult{}, nil } - // Maps from checklist key to checklist - checklists := make(map[string]*db.ChecklistEntity) - for _, taskEntity := range travisTasks { - key := taskEntity.Task.ChecklistKey - keyString := taskEntity.Task.ChecklistKey.Encode() - if checklists[keyString] == nil { - checklists[keyString], err = cocoon.GetChecklist(key) - if err != nil { - return nil, err - } - } - } - // Fetch data from Travis httpClient := urlfetch.Client(cocoon.Ctx) - travisResp, err := httpClient.Get("https://api.travis-ci.org/repos/flutter/flutter/builds") if err != nil { return nil, err @@ -73,9 +59,9 @@ func RefreshTravisStatus(cocoon *db.Cocoon, inputJSON []byte) (interface{}, erro return nil, err } - for _, taskEntity := range travisTasks { - task := taskEntity.Task - checklistEntity := checklists[task.ChecklistKey.Encode()] + for _, fullTask := range travisTasks { + task := fullTask.TaskEntity.Task + checklistEntity := fullTask.ChecklistEntity for _, travisResult := range travisResults { if travisResult.Commit == checklistEntity.Checklist.Commit.Sha { if travisResult.State == "finished" { @@ -85,7 +71,7 @@ func RefreshTravisStatus(cocoon *db.Cocoon, inputJSON []byte) (interface{}, erro } else { task.Status = db.TaskFailed } - cocoon.PutTask(taskEntity.Key, task) + cocoon.PutTask(fullTask.TaskEntity.Key, task) } } } diff --git a/db/db.go b/db/db.go index f04d54a2c..206ae2574 100644 --- a/db/db.go +++ b/db/db.go @@ -170,22 +170,30 @@ func (c *Cocoon) QueryTasks(checklistKey *datastore.Key) ([]*TaskEntity, error) return c.runTaskQuery(query) } +// FullTask contains information about a Task as well as surrounding metadata. +// It is generally more expensive to query this data than to query just the task +// records. +type FullTask struct { + TaskEntity *TaskEntity + ChecklistEntity *ChecklistEntity +} + // QueryPendingTasks lists the latest tasks with the given name that are not yet // in a final status. // // See also IsFinal. -func (c *Cocoon) QueryPendingTasks(name string) ([]*TaskEntity, error) { +func (c *Cocoon) QueryPendingTasks(taskName string) ([]*FullTask, error) { checklists, err := c.QueryLatestChecklists() if err != nil { return nil, err } - tasks := make([]*TaskEntity, 0, 20) + tasks := make([]*FullTask, 0, 20) for i := len(checklists) - 1; i >= 0; i-- { query := datastore.NewQuery("Task"). Ancestor(checklists[i].Key). - Filter("Name =", name). + Filter("Name =", taskName). Order("-CreateTimestamp"). Limit(20) candidates, err := c.runTaskQuery(query) @@ -196,7 +204,10 @@ func (c *Cocoon) QueryPendingTasks(name string) ([]*TaskEntity, error) { for _, candidate := range candidates { if !candidate.Task.Status.IsFinal() { - tasks = append(tasks, candidate) + tasks = append(tasks, &FullTask{ + TaskEntity: candidate, + ChecklistEntity: checklists[i], + }) } } }