buildkite: Add tool to download logs

BUILD.bazel

@@ -31,6 +31,7 @@ pkg_tar(
         "//go/examples/pingpong/pp_integration:pp_integration",
         "//go/tools/scion-custpk-load:scion-custpk-load",
         "//go/sciond:sciond",
+        "//go/tools/buildkite_log_downloader:buildkite_log_downloader",
         "//go/tools/scion-pki:scion-pki",
         "//go/tools/scmp:scmp",
         "//go/integration/scmp_error_pyintegration:scmp_error_pyintegration",

WORKSPACE

@@ -462,3 +462,15 @@ go_repository(
     commit = "34c6fa2dc70986bccbbffcc6130f6920a924b075",
     importpath = "github.com/stretchr/testify",
 )
+
+go_repository(
+    name = "com_github_buildkite_go_buildkite",
+    commit = "568b6651b687ccf6893ada08086ce58b072538b6",
+    importpath = "github.com/buildkite/go-buildkite",
+)
+
+go_repository(
+    name = "com_github_google_go_querystring",
+    commit = "c8c88dbee036db4e4808d1f2ec8c2e15e11c3f80",
+    importpath = "github.com/google/go-querystring",
+)

go/tools/buildkite_log_downloader/BUILD.bazel

@@ -0,0 +1,18 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "github.com/scionproto/scion/go/tools/buildkite_log_downloader",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//go/lib/common:go_default_library",
+        "@com_github_buildkite_go_buildkite//buildkite:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "buildkite_log_downloader",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)

go/tools/buildkite_log_downloader/main.go

@@ -0,0 +1,156 @@
+// Copyright 2019 Anapaya Systems
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// A tool to download all logs from a specific build.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/buildkite/go-buildkite/buildkite"
+
+	"github.com/scionproto/scion/go/lib/common"
+)
+
+var (
+	tokenFlag = flag.String("token", "", "The buildkite API token")
+	destDir   = flag.String("dst", "",
+		"The destination dir for the downloaded logs, must be writable (default: current dir)")
+	orgFlag = flag.String("org", "scionproto",
+		"The organization slug on buildkite (default: scionproto)")
+	pipelineFlag = flag.String("pipeline", "scionproto",
+		"The name of the pipeline (default: scionproto)")
+	buildFlag = flag.Int("build", 0, "The build number")
+)
+
+func main() {
+	os.Exit(realMain())
+}
+
+// buildDesc describes the relevant info to get build information from the buildkite API.
+type buildDesc struct {
+	org      string
+	pipeline string
+	id       string
+}
+
+func realMain() int {
+	flag.Parse()
+	if err := createDstDir(); err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to create dest dir: %s\n", err)
+	}
+	bd, err := verifyFlags()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		return 1
+	}
+	config, err := buildkite.NewTokenConfig(*tokenFlag, false)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to init buildkite cfg: %s\n", err)
+		return 1
+	}
+	client := buildkite.NewClient(config.Client())
+	if err := downloadBuildArtifacts(client, bd); err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to download artifacts %s\n", err)
+		return 1
+	}
+	return 0
+}
+
+func createDstDir() error {
+	return os.MkdirAll(*destDir, os.ModePerm)
+}
+
+func verifyFlags() (buildDesc, error) {
+	var problems []string
+	if *tokenFlag == "" {
+		problems = append(problems, "API-Token not provided")
+	}
+	if *buildFlag == 0 {
+		problems = append(problems, "Build number not provided")
+	}
+	if len(problems) > 0 {
+		return buildDesc{}, common.NewBasicError("Not all required flags provided", nil,
+			"problems", strings.Join(problems, "\n"))
+	}
+	return buildDesc{
+		org:      *orgFlag,
+		pipeline: *pipelineFlag,
+		id:       strconv.Itoa(*buildFlag),
+	}, nil
+}
+
+func downloadBuildArtifacts(c *buildkite.Client, bd buildDesc) error {
+	b, _, err := c.Builds.Get(bd.org, bd.pipeline, bd.id)
+	if err != nil {
+		return err
+	}
+	artifacts, _, err := c.Artifacts.ListByBuild(bd.org, bd.pipeline, bd.id,
+		&buildkite.ArtifactListOptions{
+			ListOptions: buildkite.ListOptions{
+				PerPage: 100,
+			},
+		})
+	if err != nil {
+		return err
+	}
+	for _, artifact := range artifacts {
+		if artifact.DownloadURL == nil {
+			fmt.Fprintf(os.Stderr, "Artifact %s has no download URL, ignored\n", *artifact.ID)
+			continue
+		}
+		if artifact.Filename == nil || !strings.HasPrefix(*artifact.Filename, "buildkite") {
+			fmt.Fprintf(os.Stderr, "Ignore artifiact %s because of filename %s\n",
+				*artifact.ID, *artifact.Filename)
+			continue
+		}
+		job := jobForArtifact(b, &artifact)
+		if job == nil {
+			fmt.Fprintf(os.Stderr, "No job found for artifact: %s\n", *artifact.ID)
+			continue
+		}
+		if err := downloadJobArtifacts(c, job, &artifact); err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to download artifacts for job %s: %s\n", *job.Name, err)
+		}
+	}
+	return nil
+}
+
+func jobForArtifact(b *buildkite.Build, a *buildkite.Artifact) *buildkite.Job {
+	if a.JobID == nil {
+		return nil
+	}
+	for _, job := range b.Jobs {
+		if job.ID != nil && *job.ID == *a.JobID {
+			return job
+		}
+	}
+	return nil
+}
+
+func downloadJobArtifacts(c *buildkite.Client, j *buildkite.Job, a *buildkite.Artifact) error {
+	mangledJobName := strings.Replace(*j.Name, " ", "_", -1)
+	f, err := os.Create(fmt.Sprintf("%s/%s_%s_%s", *destDir, mangledJobName, *j.ID, *a.Filename))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	_, err = c.Artifacts.DownloadArtifactByURL(*a.DownloadURL, f)
+	return err
+}

nogo.json

@@ -42,6 +42,11 @@
       "/com_github_lucas_clemente_quic_go/": ""
     }
   },
+  "structtag": {
+    "exclude_files": {
+      "com_github_buildkite_go_buildkite": ""
+    }
+  },
   "gocall": {
     "exclude_files": {
       "(vendor|external)/*": "Only applies to internal files",