diff --git a/BUILD.bazel b/BUILD.bazel index 14befef5d8..475d1647e1 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -31,6 +31,7 @@ pkg_tar( "//go/examples/pingpong/pp_integration:pp_integration", "//go/tools/scion-custpk-load:scion-custpk-load", "//go/sciond:sciond", + "//go/tools/buildkite_log_downloader:buildkite_log_downloader", "//go/tools/scion-pki:scion-pki", "//go/tools/scmp:scmp", "//go/integration/scmp_error_pyintegration:scmp_error_pyintegration", diff --git a/WORKSPACE b/WORKSPACE index 4f20073c35..bf4cfa10a4 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -462,3 +462,15 @@ go_repository( commit = "34c6fa2dc70986bccbbffcc6130f6920a924b075", importpath = "github.com/stretchr/testify", ) + +go_repository( + name = "com_github_buildkite_go_buildkite", + commit = "568b6651b687ccf6893ada08086ce58b072538b6", + importpath = "github.com/buildkite/go-buildkite", +) + +go_repository( + name = "com_github_google_go_querystring", + commit = "c8c88dbee036db4e4808d1f2ec8c2e15e11c3f80", + importpath = "github.com/google/go-querystring", +) diff --git a/go/tools/buildkite_log_downloader/BUILD.bazel b/go/tools/buildkite_log_downloader/BUILD.bazel new file mode 100644 index 0000000000..343d79ceaa --- /dev/null +++ b/go/tools/buildkite_log_downloader/BUILD.bazel @@ -0,0 +1,18 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "github.com/scionproto/scion/go/tools/buildkite_log_downloader", + visibility = ["//visibility:private"], + deps = [ + "//go/lib/common:go_default_library", + "@com_github_buildkite_go_buildkite//buildkite:go_default_library", + ], +) + +go_binary( + name = "buildkite_log_downloader", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/go/tools/buildkite_log_downloader/main.go b/go/tools/buildkite_log_downloader/main.go new file mode 100644 index 0000000000..02e5dfccc9 --- /dev/null +++ b/go/tools/buildkite_log_downloader/main.go @@ -0,0 +1,156 @@ +// Copyright 2019 Anapaya Systems +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A tool to download all logs from a specific build. + +package main + +import ( + "flag" + "fmt" + "os" + "strconv" + "strings" + + "github.com/buildkite/go-buildkite/buildkite" + + "github.com/scionproto/scion/go/lib/common" +) + +var ( + tokenFlag = flag.String("token", "", "The buildkite API token") + destDir = flag.String("dst", "", + "The destination dir for the downloaded logs, must be writable (default: current dir)") + orgFlag = flag.String("org", "scionproto", + "The organization slug on buildkite (default: scionproto)") + pipelineFlag = flag.String("pipeline", "scionproto", + "The name of the pipeline (default: scionproto)") + buildFlag = flag.Int("build", 0, "The build number") +) + +func main() { + os.Exit(realMain()) +} + +// buildDesc describes the relevant info to get build information from the buildkite API. +type buildDesc struct { + org string + pipeline string + id string +} + +func realMain() int { + flag.Parse() + if err := createDstDir(); err != nil { + fmt.Fprintf(os.Stderr, "Failed to create dest dir: %s\n", err) + } + bd, err := verifyFlags() + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + return 1 + } + config, err := buildkite.NewTokenConfig(*tokenFlag, false) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to init buildkite cfg: %s\n", err) + return 1 + } + client := buildkite.NewClient(config.Client()) + if err := downloadBuildArtifacts(client, bd); err != nil { + fmt.Fprintf(os.Stderr, "Failed to download artifacts %s\n", err) + return 1 + } + return 0 +} + +func createDstDir() error { + return os.MkdirAll(*destDir, os.ModePerm) +} + +func verifyFlags() (buildDesc, error) { + var problems []string + if *tokenFlag == "" { + problems = append(problems, "API-Token not provided") + } + if *buildFlag == 0 { + problems = append(problems, "Build number not provided") + } + if len(problems) > 0 { + return buildDesc{}, common.NewBasicError("Not all required flags provided", nil, + "problems", strings.Join(problems, "\n")) + } + return buildDesc{ + org: *orgFlag, + pipeline: *pipelineFlag, + id: strconv.Itoa(*buildFlag), + }, nil +} + +func downloadBuildArtifacts(c *buildkite.Client, bd buildDesc) error { + b, _, err := c.Builds.Get(bd.org, bd.pipeline, bd.id) + if err != nil { + return err + } + artifacts, _, err := c.Artifacts.ListByBuild(bd.org, bd.pipeline, bd.id, + &buildkite.ArtifactListOptions{ + ListOptions: buildkite.ListOptions{ + PerPage: 100, + }, + }) + if err != nil { + return err + } + for _, artifact := range artifacts { + if artifact.DownloadURL == nil { + fmt.Fprintf(os.Stderr, "Artifact %s has no download URL, ignored\n", *artifact.ID) + continue + } + if artifact.Filename == nil || !strings.HasPrefix(*artifact.Filename, "buildkite") { + fmt.Fprintf(os.Stderr, "Ignore artifiact %s because of filename %s\n", + *artifact.ID, *artifact.Filename) + continue + } + job := jobForArtifact(b, &artifact) + if job == nil { + fmt.Fprintf(os.Stderr, "No job found for artifact: %s\n", *artifact.ID) + continue + } + if err := downloadJobArtifacts(c, job, &artifact); err != nil { + fmt.Fprintf(os.Stderr, "Failed to download artifacts for job %s: %s\n", *job.Name, err) + } + } + return nil +} + +func jobForArtifact(b *buildkite.Build, a *buildkite.Artifact) *buildkite.Job { + if a.JobID == nil { + return nil + } + for _, job := range b.Jobs { + if job.ID != nil && *job.ID == *a.JobID { + return job + } + } + return nil +} + +func downloadJobArtifacts(c *buildkite.Client, j *buildkite.Job, a *buildkite.Artifact) error { + mangledJobName := strings.Replace(*j.Name, " ", "_", -1) + f, err := os.Create(fmt.Sprintf("%s/%s_%s_%s", *destDir, mangledJobName, *j.ID, *a.Filename)) + if err != nil { + return err + } + defer f.Close() + _, err = c.Artifacts.DownloadArtifactByURL(*a.DownloadURL, f) + return err +} diff --git a/nogo.json b/nogo.json index f6cb931bfc..bbb5fa8712 100644 --- a/nogo.json +++ b/nogo.json @@ -42,6 +42,11 @@ "/com_github_lucas_clemente_quic_go/": "" } }, + "structtag": { + "exclude_files": { + "com_github_buildkite_go_buildkite": "" + } + }, "gocall": { "exclude_files": { "(vendor|external)/*": "Only applies to internal files",