Update pipelines for new Docker VM config

joeleba · Jun 17, 2019 · f49f8f8 · f49f8f8
1 parent 0d0d95e
commit f49f8f8
Show file tree

Hide file tree

Showing 7 changed files with 146 additions and 76 deletions.
diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py
@@ -1543,7 +1543,6 @@ def create_docker_step(label, image, commands=None, additional_env_vars=None):
         "plugins": {
             "docker#v3.2.0": {
                 "always-pull": True,
-                "debug": True,
                 "environment": env,
                 "image": image,
                 "network": "host",

diff --git a/pipelines/bazel-release.yml b/pipelines/bazel-release.yml
@@ -4,45 +4,57 @@ steps:
     agents:
       - "queue=default"
     plugins:
-      philwo/docker:
+      docker#v3.2.0:
         always-pull: true
-        debug: true
-        environment: [BUILDKITE_ARTIFACT_UPLOAD_DESTINATION, BUILDKITE_GS_ACL]
+        environment:
+          - ANDROID_HOME
+          - ANDROID_NDK_HOME
+          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
         image: gcr.io/bazel-public/ubuntu1804:java11
         network: host
         privileged: true
         propagate-environment: true
+        propagate-uid-gid: true
         shell: ["/bin/bash", "-e", "-c"]
         volumes:
-          - ".:/workdir"
-          - "/var/lib/buildkite-agent/builds:/var/lib/buildkite-agent/builds"
-          - "/var/lib/bazelbuild:/var/lib/bazelbuild:ro"
-        workdir: "/workdir"
+          - "/etc/group:/etc/group:ro"
+          - "/etc/passwd:/etc/passwd:ro"
+          - "/opt:/opt:ro"
+          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
+          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
+          - "/var/run/docker.sock:/var/run/docker.sock"
     command: |
       git fetch origin master
       git fetch --force origin refs/notes/*:refs/notes/*
       release_name=\$(source scripts/release/common.sh; get_full_release_name)
       echo "release_name = \"\$release_name\""
       buildkite-agent meta-data set "release_name" "\$release_name"
+
   - wait
+
   - label: "Ubuntu 14.04"
     agents:
       - "queue=default"
     plugins:
-      philwo/docker:
+      docker#v3.2.0:
         always-pull: true
-        debug: true
-        environment: [BUILDKITE_ARTIFACT_UPLOAD_DESTINATION, BUILDKITE_GS_ACL]
-        image: gcr.io/bazel-public/ubuntu1404:java8
+        environment:
+          - ANDROID_HOME
+          - ANDROID_NDK_HOME
+          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
+        image: gcr.io/bazel-public/ubuntu1804:java11
         network: host
         privileged: true
         propagate-environment: true
+        propagate-uid-gid: true
         shell: ["/bin/bash", "-e", "-c"]
         volumes:
-          - ".:/workdir"
-          - "/var/lib/buildkite-agent/builds:/var/lib/buildkite-agent/builds"
-          - "/var/lib/bazelbuild:/var/lib/bazelbuild:ro"
-        workdir: "/workdir"
+          - "/etc/group:/etc/group:ro"
+          - "/etc/passwd:/etc/passwd:ro"
+          - "/opt:/opt:ro"
+          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
+          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
+          - "/var/run/docker.sock:/var/run/docker.sock"
     command: |
       git fetch origin master
       git fetch --force origin refs/notes/*:refs/notes/*
@@ -66,6 +78,7 @@ steps:
 
       cd artifacts
       buildkite-agent artifact upload "*"
+
   - label: macOS
     agents:
       - queue=macos
@@ -88,6 +101,7 @@ steps:
 
       cd artifacts
       buildkite-agent artifact upload "*"
+
   - label: Windows
     agents:
       - queue=windows
@@ -113,25 +127,32 @@ steps:
 
       cd artifacts
       buildkite-agent artifact upload "*"
+
   - wait
+
   - label: "Test on Ubuntu 14.04"
     agents:
       - "queue=default"
     plugins:
-      philwo/docker:
+      docker#v3.2.0:
         always-pull: true
-        debug: true
-        environment: [BUILDKITE_ARTIFACT_UPLOAD_DESTINATION, BUILDKITE_GS_ACL]
-        image: gcr.io/bazel-public/ubuntu1404:java8
+        environment:
+          - ANDROID_HOME
+          - ANDROID_NDK_HOME
+          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
+        image: gcr.io/bazel-public/ubuntu1804:java11
         network: host
         privileged: true
         propagate-environment: true
+        propagate-uid-gid: true
         shell: ["/bin/bash", "-e", "-c"]
         volumes:
-          - ".:/workdir"
-          - "/var/lib/buildkite-agent/builds:/var/lib/buildkite-agent/builds"
-          - "/var/lib/bazelbuild:/var/lib/bazelbuild:ro"
-        workdir: "/workdir"
+          - "/etc/group:/etc/group:ro"
+          - "/etc/passwd:/etc/passwd:ro"
+          - "/opt:/opt:ro"
+          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
+          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
+          - "/var/run/docker.sock:/var/run/docker.sock"
     command: |
       git fetch origin master
       git fetch --force origin refs/notes/*:refs/notes/*
@@ -144,6 +165,7 @@ steps:
       chmod +x "bazel-\${release_name}-linux-x86_64"
 
       "./bazel-\${release_name}-linux-x86_64" info
+
   - label: "Test on macOS"
     agents:
       - queue=macos
@@ -175,26 +197,34 @@ steps:
       buildkite-agent artifact download bazel-%RELEASE_NAME%-windows-x86_64.exe .
 
       bazel-%RELEASE_NAME%-windows-x86_64.exe info
+
   - wait
+
   - block: ":rocket: Deploy release artifacts"
+
   - label: "Deploy release artifacts"
     agents:
       - "queue=default"
     plugins:
-      philwo/docker:
+      docker#v3.2.0:
         always-pull: true
-        debug: true
-        environment: [BUILDKITE_ARTIFACT_UPLOAD_DESTINATION, BUILDKITE_GS_ACL]
+        environment:
+          - ANDROID_HOME
+          - ANDROID_NDK_HOME
+          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
         image: gcr.io/bazel-public/ubuntu1804:java11
         network: host
         privileged: true
         propagate-environment: true
+        propagate-uid-gid: true
         shell: ["/bin/bash", "-e", "-c"]
         volumes:
-          - ".:/workdir"
-          - "/var/lib/buildkite-agent/builds:/var/lib/buildkite-agent/builds"
-          - "/var/lib/bazelbuild:/var/lib/bazelbuild:ro"
-        workdir: "/workdir"
+          - "/etc/group:/etc/group:ro"
+          - "/etc/passwd:/etc/passwd:ro"
+          - "/opt:/opt:ro"
+          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
+          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
+          - "/var/run/docker.sock:/var/run/docker.sock"
     command: |
       echo "+++ Fetching Git notes"
       git fetch origin master
@@ -214,26 +244,34 @@ steps:
       echo "+++ Deploying release"
       source scripts/ci/build.sh
       deploy_release "\${ARTIFACTS}"
+
   - wait
+
   - block: ":e-mail: Generate announcement mail text"
+
   - label: "Generate announcement mail text"
     agents:
       - "queue=default"
     plugins:
-      philwo/docker:
+      docker#v3.2.0:
         always-pull: true
-        debug: true
-        environment: [BUILDKITE_ARTIFACT_UPLOAD_DESTINATION, BUILDKITE_GS_ACL]
+        environment:
+          - ANDROID_HOME
+          - ANDROID_NDK_HOME
+          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
         image: gcr.io/bazel-public/ubuntu1804:java11
         network: host
         privileged: true
         propagate-environment: true
+        propagate-uid-gid: true
         shell: ["/bin/bash", "-e", "-c"]
         volumes:
-          - ".:/workdir"
-          - "/var/lib/buildkite-agent/builds:/var/lib/buildkite-agent/builds"
-          - "/var/lib/bazelbuild:/var/lib/bazelbuild:ro"
-        workdir: "/workdir"
+          - "/etc/group:/etc/group:ro"
+          - "/etc/passwd:/etc/passwd:ro"
+          - "/opt:/opt:ro"
+          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
+          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
+          - "/var/run/docker.sock:/var/run/docker.sock"
     command: |
       git fetch origin master
       git fetch --force origin refs/notes/*:refs/notes/*

diff --git a/pipelines/bazelisk-binaries.yml b/pipelines/bazelisk-binaries.yml
@@ -6,19 +6,25 @@ steps:
     agents:
       - "queue=default"
     plugins:
-      philwo/docker:
+      docker#v3.2.0:
         always-pull: true
-        debug: true
-        environment: [BUILDKITE_ARTIFACT_UPLOAD_DESTINATION, BUILDKITE_GS_ACL]
-        image: gcr.io/bazel-public/ubuntu1404:java8
+        environment:
+          - ANDROID_HOME
+          - ANDROID_NDK_HOME
+          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
+        image: gcr.io/bazel-public/ubuntu1804:java11
         network: host
         privileged: true
         propagate-environment: true
+        propagate-uid-gid: true
+        shell: ["/bin/bash", "-e", "-c"]
         volumes:
-          - ".:/workdir"
-          - "/var/lib/buildkite-agent/builds:/var/lib/buildkite-agent/builds"
-          - "/var/lib/bazelbuild:/var/lib/bazelbuild:ro"
-        workdir: "/workdir"
+          - "/etc/group:/etc/group:ro"
+          - "/etc/passwd:/etc/passwd:ro"
+          - "/opt:/opt:ro"
+          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
+          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
+          - "/var/run/docker.sock:/var/run/docker.sock"
 
   - command: "powershell -Command \"bazel build //:bazelisk ; if (\\$LastExitCode -ne 0) { exit 1 } ; buildkite-agent artifact upload ./bazel-bin/windows_amd64_stripped/bazelisk.exe ; if (\\$LastExitCode -ne 0) { exit 1 } ;\""
     label: ":windows:"

diff --git a/pipelines/build-embedded-minimized-jdk.yml b/pipelines/build-embedded-minimized-jdk.yml
@@ -9,19 +9,25 @@ steps:
     agents:
       - "queue=default"
     plugins:
-      philwo/docker:
+      docker#v3.2.0:
         always-pull: true
-        debug: true
-        environment: [BUILDKITE_ARTIFACT_UPLOAD_DESTINATION, BUILDKITE_GS_ACL]
+        environment:
+          - ANDROID_HOME
+          - ANDROID_NDK_HOME
+          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
         image: gcr.io/bazel-public/ubuntu1804:java11
         network: host
         privileged: true
         propagate-environment: true
+        propagate-uid-gid: true
+        shell: ["/bin/bash", "-e", "-c"]
         volumes:
-          - ".:/workdir"
-          - "/var/lib/buildkite-agent/builds:/var/lib/buildkite-agent/builds"
-          - "/var/lib/bazelbuild:/var/lib/bazelbuild:ro"
-        workdir: "/workdir"
+          - "/etc/group:/etc/group:ro"
+          - "/etc/passwd:/etc/passwd:ro"
+          - "/opt:/opt:ro"
+          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
+          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
+          - "/var/run/docker.sock:/var/run/docker.sock"
 
   - command: "powershell -Command \"bazel --nomaster_bazelrc --bazelrc=NUL version ; if (\\$LastExitCode -ne 0) { exit 1 } ; bazel build //src:embedded_jdk_minimal //src:embedded_jdk_allmodules ; if (\\$LastExitCode -ne 0) { exit 1 } ; buildkite-agent artifact upload ./bazel-genfiles/src/minimal_jdk.zip ; if (\\$LastExitCode -ne 0) { exit 1 } ; buildkite-agent artifact upload ./bazel-genfiles/src/allmodules_jdk.zip ; if (\\$LastExitCode -ne 0) { exit 1 } ; \\$revision=(git rev-parse HEAD) ; if (\\$LastExitCode -ne 0) { exit 1 } ; \\$epoch=(date +%%s) ; if (\\$LastExitCode -ne 0) { exit 1 } ; gsutil cp ./bazel-genfiles/src/minimal_jdk.zip gs://bazel-mirror/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-win_x64-minimal-\\${revision}-\\${epoch}.zip ; if (\\$LastExitCode -ne 0) { exit 1 } ; gsutil cp ./bazel-genfiles/src/allmodules_jdk.zip gs://bazel-mirror/openjdk/azul-zulu11.29.3-ca-jdk11.0.2/zulu11.29.3-ca-jdk11.0.2-win_x64-allmodules-\\${revision}-\\${epoch}.zip ; if (\\$LastExitCode -ne 0) { exit 1 } ;\""
     label: ":windows:"

diff --git a/pipelines/buildtools-binaries.yml b/pipelines/buildtools-binaries.yml
@@ -7,10 +7,12 @@ steps:
     label: ":darwin:"
     agents:
       - "queue=macos"
+
   - command: "powershell -Command \"bazel build -c opt --workspace_status_command='status.py' unused_deps buildifier buildozer ; if (\\$LastExitCode -ne 0) { exit 1 } ; buildkite-agent artifact upload ./bazel-bin/buildozer/windows_amd64_stripped/buildozer.exe ; if (\\$LastExitCode -ne 0) { exit 1 } ; buildkite-agent artifact upload ./bazel-bin/buildifier/windows_amd64_stripped/buildifier.exe ; if (\\$LastExitCode -ne 0) { exit 1 } ; buildkite-agent artifact upload ./bazel-bin/unused_deps/windows_amd64_stripped/unused_deps.exe ; if (\\$LastExitCode -ne 0) { exit 1 } ;\""
     label: ":windows:"
     agents:
       - "queue=windows"
+
   - command: |-
       bazel build -c opt --workspace_status_command="$$PWD/status.py" unused_deps buildifier buildozer
       buildkite-agent artifact upload "./bazel-bin/buildozer/linux_amd64_stripped/buildozer"
@@ -20,16 +22,22 @@ steps:
     agents:
       - "queue=default"
     plugins:
-      philwo/docker:
+      docker#v3.2.0:
         always-pull: true
-        debug: true
-        environment: [BUILDKITE_ARTIFACT_UPLOAD_DESTINATION, BUILDKITE_GS_ACL]
-        image: gcr.io/bazel-public/ubuntu1404:java8
+        environment:
+          - ANDROID_HOME
+          - ANDROID_NDK_HOME
+          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
+        image: gcr.io/bazel-public/ubuntu1804:java11
         network: host
         privileged: true
         propagate-environment: true
+        propagate-uid-gid: true
+        shell: ["/bin/bash", "-e", "-c"]
         volumes:
-          - ".:/workdir"
-          - "/var/lib/buildkite-agent/builds:/var/lib/buildkite-agent/builds"
-          - "/var/lib/bazelbuild:/var/lib/bazelbuild:ro"
-        workdir: "/workdir"
+          - "/etc/group:/etc/group:ro"
+          - "/etc/passwd:/etc/passwd:ro"
+          - "/opt:/opt:ro"
+          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
+          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
+          - "/var/run/docker.sock:/var/run/docker.sock"
diff --git a/pipelines/java_tools-binaries.yml b/pipelines/java_tools-binaries.yml
@@ -6,19 +6,26 @@ steps:
     agents:
       - "queue=default"
     plugins:
-      philwo/docker:
+      docker#v3.2.0:
         always-pull: true
-        debug: true
-        environment: [BUILDKITE_ARTIFACT_UPLOAD_DESTINATION, BUILDKITE_GS_ACL]
-        image: gcr.io/bazel-public/ubuntu1404:java8
+        environment:
+          - ANDROID_HOME
+          - ANDROID_NDK_HOME
+          - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
+        image: gcr.io/bazel-public/ubuntu1804:java11
         network: host
         privileged: true
         propagate-environment: true
+        propagate-uid-gid: true
+        shell: ["/bin/bash", "-e", "-c"]
         volumes:
-          - ".:/workdir"
-          - "/var/lib/buildkite-agent/builds:/var/lib/buildkite-agent/builds"
-          - "/var/lib/bazelbuild:/var/lib/bazelbuild:ro"
-        workdir: "/workdir"
+          - "/etc/group:/etc/group:ro"
+          - "/etc/passwd:/etc/passwd:ro"
+          - "/opt:/opt:ro"
+          - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
+          - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
+          - "/var/run/docker.sock:/var/run/docker.sock"
+
   - command: "powershell -Command \"bazel --nomaster_bazelrc --bazelrc=NUL version ; if (\\$LastExitCode -ne 0) { exit 1 } ; sh ./src/upload_all_java_tools.sh ; if (\\$LastExitCode -ne 0) { exit 1 } ;\""
     label: ":windows:"
     agents: