Enable x86_64-linux-gnu packager

.buildkite/hooks/post-checkout

@@ -4,10 +4,12 @@ ls -A1 | xargs rm -rf
 echo 'Clone testing branch of julia, main branch only:'
 git clone -b master https://github.com/JuliaCI/julia-buildkite-testing ./
 echo
+git log -1
+echo
 
-echo 'Set buildkite metadata'
 # Force external-buildkite plugin to use the current branch's commit as the version to download for this and all child jobs
 if ! buildkite-agent meta-data exists BUILDKITE_PLUGIN_EXTERNAL_BUILDKITE_VERSION; then
+    echo "Override external-buildkite metadata to VERSION '${BUILDKITE_COMMIT}' and REPO_URL 'https://github.com/JuliaCI/julia-buildkite'"
     buildkite-agent meta-data set BUILDKITE_PLUGIN_EXTERNAL_BUILDKITE_VERSION "${BUILDKITE_COMMIT:?}"
     buildkite-agent meta-data set BUILDKITE_PLUGIN_EXTERNAL_BUILDKITE_REPO_URL "https://github.com/JuliaCI/julia-buildkite"
 fi

Makefile

@@ -0,0 +1,5 @@
+sign_treehashes:
+	cd .buildkite/cryptic_repo_root && sign_treehashes --repo-root=$$(pwd) --verbose
+
+verify_treehashes:
+	cd .buildkite/cryptic_repo_root && verify_treehashes --repo-root=$$(pwd) --verbose

pipelines/main/0_webui.yml

@@ -2,26 +2,29 @@
 # It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect.
 # We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps.
 steps:
-  - label: ":unlock: Unlock secrets, launch pipelines"
-    agents:
-      queue: "julia"
-      sandbox.jl: "true"
-    plugins:
-      - JuliaCI/external-buildkite#v1:
-          version: ".buildkite-external-version"
-          repo_url: "https://github.com/JuliaCI/julia-buildkite"
-      - staticfloat/cryptic#v2:
-          # Our list of pipelines that should be launched (but don't require a signature)
-          # These pipelines can be modified by any contributor and CI will still run.
-          # Build secrets will not be available in these pipelines (or their children)
-          # but some of our signed pipelines can wait upon the completion of these unsigned
-          # pipelines.
-          unsigned_pipelines:
-            - .buildkite/pipelines/main/launch_unsigned_builders.yml
+  - group: ":buildkite: Infrastructure"
+    steps:
+      - label: ":unlock: Unlock secrets, launch pipelines"
+        agents:
+          queue: "julia"
+          sandbox_capable: "true"
+        plugins:
+          - JuliaCI/external-buildkite#v1:
+              version: ".buildkite-external-version"
+              repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - staticfloat/cryptic#v2:
+              # Our list of pipelines that should be launched (but don't require a signature)
+              # These pipelines can be modified by any contributor and CI will still run.
+              # Build secrets will not be available in these pipelines (or their children)
+              # but some of our signed pipelines can wait upon the completion of these unsigned
+              # pipelines.
+              unsigned_pipelines:
+                - .buildkite/pipelines/main/launch_unsigned_builders.yml
 
-          # Our signed pipelines must have a `signature` or `signature_file` parameter that
-          # verifies the treehash of the pipeline itself and the inputs listed in `inputs`
-          signed_pipelines:
-            - pipeline: .buildkite/pipelines/main/misc/signed_pipeline_test.yml
-              signature_file: .buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature
-    command: "true"
+              # Our signed pipelines must have a `signature` or `signature_file` parameter that
+              # verifies the treehash of the pipeline itself and the inputs listed in `inputs`
+              signed_pipelines:
+                - pipeline: .buildkite/pipelines/main/launch_signed_jobs.yml
+                  signature_file: .buildkite/pipelines/main/launch_signed_jobs.yml.signature
+                  allow_hash_override: true
+        command: "true"

pipelines/main/launch_signed_jobs.yml

@@ -0,0 +1,24 @@
+# This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3
+steps:
+  - label: ":buildkite: :unlock: Launch signed pipelines"
+    plugins:
+      - JuliaCI/external-buildkite#v1:
+          version: ".buildkite-external-version"
+          repo_url: "https://github.com/JuliaCI/julia-buildkite"
+    commands: |
+      # Explicitly pass along the cryptic token to child pipelines
+      export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET
+
+      # Launch `upload_linux` jobs to store tarballs into S3 once tests are done
+      bash .buildkite/utilities/arches_pipeline_upload.sh \
+          .buildkite/pipelines/main/platforms/build_linux.arches \
+          .buildkite/pipelines/main/platforms/upload_linux.yml
+
+      # Don't share this with buildkite's env display
+      unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET
+    agents:
+      queue: "julia"
+      cryptic_capable: "true"
+    env:
+      # Receive cryptic token from parent job
+      BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?}

pipelines/main/launch_signed_jobs.yml.signature

@@ -0,0 +1 @@
+Salted__��E����'��s�nm�~H�S�u�u��)���ԥ��A�ˋ ���D����l���<渳�(aݣ���;��鼏��}m� 5-�

pipelines/main/launch_unsigned_builders.yml

@@ -13,25 +13,40 @@
 # something about the privileged steps.
 
 steps:
-  - label: ":buildkite: Launch unsigned jobs"
-    plugins:
-      - JuliaCI/external-buildkite#v1:
-          version: ".buildkite-external-version"
-          repo_url: "https://github.com/JuliaCI/julia-buildkite"
-    commands: |
-      # Launch the miscellaneous jobs in alphabetical order.
-      #buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
-      #buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
-      #buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
-      #buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml
+  - group: ":buildkite: Infrastructure"
+    steps:
+      - label: ":buildkite: Launch misc. jobs"
+        plugins:
+          - JuliaCI/external-buildkite#v1:
+              version: ".buildkite-external-version"
+              repo_url: "https://github.com/JuliaCI/julia-buildkite"
+        commands: |
+          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml
+        agents:
+          queue: "julia"
 
-      # Launch all of the platform jobs.
-      #bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml
-      #bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/tester_linux.arches .buildkite/pipelines/main/platforms/tester_linux.yml
+  - group: ":linux: Linux"
+    steps:
+      - label: ":buildkite: Launch jobs"
+        plugins:
+          - JuliaCI/external-buildkite#v1:
+              version: ".buildkite-external-version"
+              repo_url: "https://github.com/JuliaCI/julia-buildkite"
+        commands: |
+          # Launch the miscellaneous jobs in alphabetical order.
+          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
+          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
+          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
+          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml
 
-      # Launch the `whitespace` job last. Uploading it last actually causes it to start
-      # first. We want this job to start first because we want it to finish as quickly
-      # as possible.
-      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml
-    agents:
-      queue: julia
+          # Launch Linux packaging jobs
+          bash .buildkite/utilities/arches_pipeline_upload.sh \
+              .buildkite/pipelines/main/platforms/build_linux.arches \
+              .buildkite/pipelines/main/platforms/build_linux.yml
+
+          # Launch Linux testing jobs
+          bash .buildkite/utilities/arches_pipeline_upload.sh \
+              .buildkite/pipelines/main/platforms/test_linux.arches \
+              .buildkite/pipelines/main/platforms/test_linux.yml
+        agents:
+          queue: "julia"

pipelines/main/misc/doctest.yml

@@ -1,34 +1,38 @@
-agents:
-  queue: "julia"
-  # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
-  sandbox.jl: "true"
-  os: "linux"
 steps:
-  - label: "doctest"
-    key: doctest
-    plugins:
-      - JuliaCI/julia#v1:
-          # Drop default "registries" directory, so it is not persisted from execution to execution
-          persist_depot_dirs: packages,artifacts,compiled
-          version: '1.6'
-      - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
-          rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
-          uid: 1000
-          gid: 1000
-          workspaces:
-            # Include `/cache/repos` so that our `git` version introspection works.
-            - "/cache/repos:/cache/repos"
-    commands: |
-      echo "--- Build Julia from source"
-      make --output-sync -j 6
+  - group: ":linux: Linux"
+    steps:
+      - label: "doctest"
+        key: doctest
+        plugins:
+          - JuliaCI/external-buildkite#v1:
+              version: ".buildkite-external-version"
+              repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - JuliaCI/julia#v1:
+              # Drop default "registries" directory, so it is not persisted from execution to execution
+              persist_depot_dirs: packages,artifacts,compiled
+              version: '1.6'
+          - staticfloat/sandbox#v1:
+              rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
+              rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
+              uid: 1000
+              gid: 1000
+              workspaces:
+                # Include `/cache/repos` so that our `git` version introspection works.
+                - "/cache/repos:/cache/repos"
+        commands: |
+          echo "--- Build Julia from source"
+          make --output-sync -j 6
 
-      echo "--- Print Julia version info"
-      ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
+          echo "--- Print Julia version info"
+          ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
 
-      echo "--- Build Julia docs"
-      make docs
+          echo "--- Build Julia docs"
+          make docs
 
-      echo "--- Run Julia doctests"
-      JULIA_NUM_THREADS=1 make -C doc doctest=true
-    timeout_in_minutes: 45
+          echo "--- Run Julia doctests"
+          JULIA_NUM_THREADS=1 make -C doc doctest=true
+        timeout_in_minutes: 45
+        agents:
+          queue: "julia"
+          sandbox_capable: "true"
+          os: "linux"

pipelines/main/misc/embedding.yml

@@ -1,31 +1,35 @@
-agents:
-  queue: "julia"
-  # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
-  sandbox.jl: "true"
-  os: "linux"
 steps:
-  - label: "embedding"
-    key: "embedding"
-    plugins:
-      - JuliaCI/julia#v1:
-          # Drop default "registries" directory, so it is not persisted from execution to execution
-          persist_depot_dirs: packages,artifacts,compiled
-          version: '1.6'
-      - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
-          rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
-          uid: 1000
-          gid: 1000
-          workspaces:
-            # Include `/cache/repos` so that our `git` version introspection works.
-            - "/cache/repos:/cache/repos"
-    commands: |
-      prefix="/tmp/prefix"
-      echo "+++ Build julia, deploy to $${prefix:?}"
-      make --output-sync -j$${JULIA_CPU_THREADS:?} JULIA_PRECOMPILE=0 prefix=$${prefix:?} install
+  - group: ":linux: Linux"
+    steps:
+      - label: "embedding"
+        key: "embedding"
+        plugins:
+          - JuliaCI/external-buildkite#v1:
+              version: ".buildkite-external-version"
+              repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - JuliaCI/julia#v1:
+              # Drop default "registries" directory, so it is not persisted from execution to execution
+              persist_depot_dirs: packages,artifacts,compiled
+              version: '1.6'
+          - staticfloat/sandbox#v1:
+              rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
+              rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
+              uid: 1000
+              gid: 1000
+              workspaces:
+                # Include `/cache/repos` so that our `git` version introspection works.
+                - "/cache/repos:/cache/repos"
+        commands: |
+          prefix="/tmp/prefix"
+          echo "+++ Build julia, deploy to $${prefix:?}"
+          make --output-sync -j$${JULIA_CPU_THREADS:?} JULIA_PRECOMPILE=0 prefix=$${prefix:?} install
 
-      embedding_output="/tmp/embedding-test"
-      echo "+++ Run embedding tests, deploy to $${embedding_output:?}"
-      mkdir -p "$${embedding_output:?}"
-      make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/embedding JULIA="$${prefix:?}/bin/julia" BIN="$${embedding_output:?}"
-    timeout_in_minutes: 60
+          embedding_output="/tmp/embedding-test"
+          echo "+++ Run embedding tests, deploy to $${embedding_output:?}"
+          mkdir -p "$${embedding_output:?}"
+          make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/embedding JULIA="$${prefix:?}/bin/julia" BIN="$${embedding_output:?}"
+        timeout_in_minutes: 60
+        agents:
+          queue: "julia"
+          sandbox_capable: "true"
+          os: "linux"