From 57c60a4ce796e3f22fa0041d673886d9725dd21c Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Mon, 28 Feb 2022 16:21:21 -0800 Subject: [PATCH 1/2] Enable `x86_64-linux-gnu` packager with new rootfs Use new rootfs image with custom-built glibc and gcc version. --- .buildkite/hooks/post-checkout | 4 +- Makefile | 5 + pipelines/main/0_webui.yml | 47 +++---- pipelines/main/launch_signed_jobs.yml | 27 ++++ .../main/launch_signed_jobs.yml.signature | Bin 0 -> 96 bytes pipelines/main/launch_unsigned_builders.yml | 55 +++++--- pipelines/main/misc/doctest.yml | 64 +++++----- pipelines/main/misc/embedding.yml | 62 ++++----- pipelines/main/misc/llvmpasses.yml | 109 ++++++++-------- pipelines/main/misc/sanitizers.yml | 101 ++++++++------- pipelines/main/misc/signed_pipeline_test.yml | 21 --- .../misc/signed_pipeline_test.yml.signature | Bin 96 -> 0 bytes pipelines/main/misc/whitespace.yml | 48 +++---- pipelines/main/platforms/build_linux.arches | 17 +++ pipelines/main/platforms/build_linux.yml | 57 +++++++++ pipelines/main/platforms/package_linux.arches | 7 - pipelines/main/platforms/package_linux.yml | 54 -------- pipelines/main/platforms/test_linux.arches | 20 +++ pipelines/main/platforms/test_linux.yml | 84 ++++++++++++ pipelines/main/platforms/tester_linux.arches | 25 ---- pipelines/main/platforms/tester_linux.yml | 120 ------------------ pipelines/main/platforms/upload_linux.yml | 55 ++++++++ pipelines/scheduled/0_webui.yml | 6 +- .../scheduled/coverage/coverage_linux64.yml | 11 +- .../coverage/coverage_linux64.yml.signature | Bin 96 -> 96 bytes .../scheduled/coverage/upload_coverage.jl | 13 +- pipelines/scheduled/launch_unsigned_jobs.yml | 4 +- .../no_bb/no_bb_package_linux.arches | 4 +- .../scheduled/no_bb/no_bb_tester_linux.arches | 12 +- utilities/arches_env.sh | 80 ++++++++++++ utilities/arches_pipeline_upload.sh | 28 ++++ utilities/calc_version_envs.sh | 83 ++++++++++++ utilities/platforms/platforms.sh | 76 ----------- 33 files changed, 741 insertions(+), 558 deletions(-) create mode 100644 Makefile create mode 100644 pipelines/main/launch_signed_jobs.yml create mode 100644 pipelines/main/launch_signed_jobs.yml.signature delete mode 100644 pipelines/main/misc/signed_pipeline_test.yml delete mode 100644 pipelines/main/misc/signed_pipeline_test.yml.signature create mode 100644 pipelines/main/platforms/build_linux.arches create mode 100644 pipelines/main/platforms/build_linux.yml delete mode 100644 pipelines/main/platforms/package_linux.arches delete mode 100644 pipelines/main/platforms/package_linux.yml create mode 100644 pipelines/main/platforms/test_linux.arches create mode 100644 pipelines/main/platforms/test_linux.yml delete mode 100644 pipelines/main/platforms/tester_linux.arches delete mode 100644 pipelines/main/platforms/tester_linux.yml create mode 100644 pipelines/main/platforms/upload_linux.yml create mode 100755 utilities/arches_env.sh create mode 100644 utilities/arches_pipeline_upload.sh create mode 100755 utilities/calc_version_envs.sh delete mode 100755 utilities/platforms/platforms.sh diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index eff8acb4..b77793c1 100755 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -4,10 +4,12 @@ ls -A1 | xargs rm -rf echo 'Clone testing branch of julia, main branch only:' git clone -b master https://github.com/JuliaCI/julia-buildkite-testing ./ echo +git log -1 +echo -echo 'Set buildkite metadata' # Force external-buildkite plugin to use the current branch's commit as the version to download for this and all child jobs if ! buildkite-agent meta-data exists BUILDKITE_PLUGIN_EXTERNAL_BUILDKITE_VERSION; then + echo "Override external-buildkite metadata to VERSION '${BUILDKITE_COMMIT}' and REPO_URL 'https://github.com/JuliaCI/julia-buildkite'" buildkite-agent meta-data set BUILDKITE_PLUGIN_EXTERNAL_BUILDKITE_VERSION "${BUILDKITE_COMMIT:?}" buildkite-agent meta-data set BUILDKITE_PLUGIN_EXTERNAL_BUILDKITE_REPO_URL "https://github.com/JuliaCI/julia-buildkite" fi diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..6c6049d3 --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +sign_treehashes: + cd .buildkite/cryptic_repo_root && sign_treehashes --repo-root=$$(pwd) --verbose + +verify_treehashes: + cd .buildkite/cryptic_repo_root && verify_treehashes --repo-root=$$(pwd) --verbose diff --git a/pipelines/main/0_webui.yml b/pipelines/main/0_webui.yml index 15bfb24b..d476adc1 100644 --- a/pipelines/main/0_webui.yml +++ b/pipelines/main/0_webui.yml @@ -2,26 +2,29 @@ # It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. # We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. steps: - - label: ":unlock: Unlock secrets, launch pipelines" - agents: - queue: "julia" - sandbox.jl: "true" - plugins: - - JuliaCI/external-buildkite#v1: - version: ".buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - staticfloat/cryptic#v2: - # Our list of pipelines that should be launched (but don't require a signature) - # These pipelines can be modified by any contributor and CI will still run. - # Build secrets will not be available in these pipelines (or their children) - # but some of our signed pipelines can wait upon the completion of these unsigned - # pipelines. - unsigned_pipelines: - - .buildkite/pipelines/main/launch_unsigned_builders.yml + - group: ":buildkite: Infrastructure" + steps: + - label: ":unlock: Unlock secrets, launch pipelines" + agents: + queue: "julia" + sandbox_capable: "true" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - staticfloat/cryptic#v2: + # Our list of pipelines that should be launched (but don't require a signature) + # These pipelines can be modified by any contributor and CI will still run. + # Build secrets will not be available in these pipelines (or their children) + # but some of our signed pipelines can wait upon the completion of these unsigned + # pipelines. + unsigned_pipelines: + - .buildkite/pipelines/main/launch_unsigned_builders.yml - # Our signed pipelines must have a `signature` or `signature_file` parameter that - # verifies the treehash of the pipeline itself and the inputs listed in `inputs` - signed_pipelines: - - pipeline: .buildkite/pipelines/main/misc/signed_pipeline_test.yml - signature_file: .buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature - command: "true" + # Our signed pipelines must have a `signature` or `signature_file` parameter that + # verifies the treehash of the pipeline itself and the inputs listed in `inputs` + signed_pipelines: + - pipeline: .buildkite/pipelines/main/launch_signed_jobs.yml + signature_file: .buildkite/pipelines/main/launch_signed_jobs.yml.signature + allow_hash_override: true + command: "true" diff --git a/pipelines/main/launch_signed_jobs.yml b/pipelines/main/launch_signed_jobs.yml new file mode 100644 index 00000000..7e519b92 --- /dev/null +++ b/pipelines/main/launch_signed_jobs.yml @@ -0,0 +1,27 @@ +# This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3 +steps: + - group: ":buildkite: Infrastructure" + depends_on: + steps: + - label: ":buildkite: :unlock: Launch signed pipelines" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # Explicitly pass along the cryptic token to child pipelines + export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET + + # Launch `upload_linux` jobs to store tarballs into S3 once tests are done + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_linux.arches \ + .buildkite/pipelines/main/platforms/upload_linux.yml + + # Don't share this with buildkite's env display + unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET + agents: + queue: "julia" + cryptic_capable: "true" + env: + # Receive cryptic token from parent job + BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/pipelines/main/launch_signed_jobs.yml.signature b/pipelines/main/launch_signed_jobs.yml.signature new file mode 100644 index 0000000000000000000000000000000000000000..1f331d7539adc76f1d8887a9acc111f750b8fae4 GIT binary patch literal 96 zcmV-m0H6O;VQh3|WM5xUtdFJuxr0S=XH0o%FkFV1rW4vSvR(^cAb1w*@k?hGOYa>y zc9bT%DSyY1i2SbswOo2C|Jl~Y6f&;5j(ky?s%HB8F^Bu@!~CnMk$VF(1q`(=U6E5g Ci7mJQ literal 0 HcmV?d00001 diff --git a/pipelines/main/launch_unsigned_builders.yml b/pipelines/main/launch_unsigned_builders.yml index 28e0a9cf..02bab7c4 100644 --- a/pipelines/main/launch_unsigned_builders.yml +++ b/pipelines/main/launch_unsigned_builders.yml @@ -13,25 +13,40 @@ # something about the privileged steps. steps: - - label: ":buildkite: Launch unsigned jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: ".buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - commands: | - # Launch the miscellaneous jobs in alphabetical order. - #buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml - #buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml - #buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml - #buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml + - group: ":buildkite: Infrastructure" + steps: + - label: ":buildkite: Launch misc. jobs" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml + agents: + queue: "julia" - # Launch all of the platform jobs. - #bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml - #bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/tester_linux.arches .buildkite/pipelines/main/platforms/tester_linux.yml + - group: ":linux: Linux" + steps: + - label: ":buildkite: Launch jobs" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # Launch the miscellaneous jobs in alphabetical order. + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml - # Launch the `whitespace` job last. Uploading it last actually causes it to start - # first. We want this job to start first because we want it to finish as quickly - # as possible. - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml - agents: - queue: julia + # Launch Linux packaging jobs + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_linux.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml + + # Launch Linux testing jobs + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_linux.arches \ + .buildkite/pipelines/main/platforms/test_linux.yml + agents: + queue: "julia" diff --git a/pipelines/main/misc/doctest.yml b/pipelines/main/misc/doctest.yml index b83139dd..eaea8d2e 100644 --- a/pipelines/main/misc/doctest.yml +++ b/pipelines/main/misc/doctest.yml @@ -1,34 +1,38 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" steps: - - label: "doctest" - key: doctest - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - commands: | - echo "--- Build Julia from source" - make --output-sync -j 6 + - group: ":linux: Linux" + steps: + - label: "doctest" + key: doctest + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz + rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + commands: | + echo "--- Build Julia from source" + make --output-sync -j 6 - echo "--- Print Julia version info" - ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' + echo "--- Print Julia version info" + ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - echo "--- Build Julia docs" - make docs + echo "--- Build Julia docs" + make docs - echo "--- Run Julia doctests" - JULIA_NUM_THREADS=1 make -C doc doctest=true - timeout_in_minutes: 45 + echo "--- Run Julia doctests" + JULIA_NUM_THREADS=1 make -C doc doctest=true + timeout_in_minutes: 45 + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/main/misc/embedding.yml b/pipelines/main/misc/embedding.yml index bdd2a0a6..3394231d 100644 --- a/pipelines/main/misc/embedding.yml +++ b/pipelines/main/misc/embedding.yml @@ -1,31 +1,35 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" steps: - - label: "embedding" - key: "embedding" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - commands: | - prefix="/tmp/prefix" - echo "+++ Build julia, deploy to $${prefix:?}" - make --output-sync -j$${JULIA_CPU_THREADS:?} JULIA_PRECOMPILE=0 prefix=$${prefix:?} install + - group: ":linux: Linux" + steps: + - label: "embedding" + key: "embedding" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz + rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + commands: | + prefix="/tmp/prefix" + echo "+++ Build julia, deploy to $${prefix:?}" + make --output-sync -j$${JULIA_CPU_THREADS:?} JULIA_PRECOMPILE=0 prefix=$${prefix:?} install - embedding_output="/tmp/embedding-test" - echo "+++ Run embedding tests, deploy to $${embedding_output:?}" - mkdir -p "$${embedding_output:?}" - make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/embedding JULIA="$${prefix:?}/bin/julia" BIN="$${embedding_output:?}" - timeout_in_minutes: 60 + embedding_output="/tmp/embedding-test" + echo "+++ Run embedding tests, deploy to $${embedding_output:?}" + mkdir -p "$${embedding_output:?}" + make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/embedding JULIA="$${prefix:?}/bin/julia" BIN="$${embedding_output:?}" + timeout_in_minutes: 60 + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/main/misc/llvmpasses.yml b/pipelines/main/misc/llvmpasses.yml index 97ea2e09..c5b9932f 100644 --- a/pipelines/main/misc/llvmpasses.yml +++ b/pipelines/main/misc/llvmpasses.yml @@ -1,51 +1,60 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" steps: - - label: "analyzegc" - key: "analyzegc" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1.2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz - rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd" - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - commands: | - echo "--- Install in-tree LLVM dependencies" - make --output-sync -j$${JULIA_CPU_THREADS:?} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind - echo "+++ run clangsa/analyzegc" - make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/clangsa - make --output-sync -j$${JULIA_CPU_THREADS:?} -C src analyzegc - timeout_in_minutes: 60 - - label: "llvmpasses" - key: "llvmpasses" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1.2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" - uid: 1000 - gid: 1000 - workspaces: - - "/cache/repos:/cache/repos" - commands: | - echo "--- make release" - # Enable Julia assertions: FORCE_ASSERTIONS=1 - # Enable LLVM assertions: LLVM_ASSERTIONS=1 - export MAKE_ASSERT_FLAGS="FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1" - make --output-sync -j$${JULIA_CPU_THREADS:?} release JULIA_PRECOMPILE=0 $${MAKE_ASSERT_FLAGS:?} - echo "--- make src/install-analysis-deps" - make --output-sync -j$${JULIA_CPU_THREADS:?} -C src install-analysis-deps $${MAKE_ASSERT_FLAGS:?} - echo "+++ make test/llvmpasses" - make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/llvmpasses $${MAKE_ASSERT_FLAGS:?} - timeout_in_minutes: 60 + - group: ":linux: Linux" + steps: + - label: "analyzegc" + key: "analyzegc" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1.2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz + rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd" + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + commands: | + echo "--- Install in-tree LLVM dependencies" + make --output-sync -j$${JULIA_CPU_THREADS:?} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind + echo "+++ run clangsa/analyzegc" + make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/clangsa + make --output-sync -j$${JULIA_CPU_THREADS:?} -C src analyzegc + timeout_in_minutes: 60 + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" + + - label: "llvmpasses" + key: "llvmpasses" + plugins: + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1.2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz + rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" + uid: 1000 + gid: 1000 + workspaces: + - "/cache/repos:/cache/repos" + commands: | + echo "--- make release" + # Enable Julia assertions: FORCE_ASSERTIONS=1 + # Enable LLVM assertions: LLVM_ASSERTIONS=1 + export MAKE_ASSERT_FLAGS="FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1" + make --output-sync -j$${JULIA_CPU_THREADS:?} release JULIA_PRECOMPILE=0 $${MAKE_ASSERT_FLAGS:?} + echo "--- make src/install-analysis-deps" + make --output-sync -j$${JULIA_CPU_THREADS:?} -C src install-analysis-deps $${MAKE_ASSERT_FLAGS:?} + echo "+++ make test/llvmpasses" + make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/llvmpasses $${MAKE_ASSERT_FLAGS:?} + timeout_in_minutes: 60 + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/main/misc/sanitizers.yml b/pipelines/main/misc/sanitizers.yml index a0c40dda..42feb06b 100644 --- a/pipelines/main/misc/sanitizers.yml +++ b/pipelines/main/misc/sanitizers.yml @@ -1,47 +1,56 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" steps: - - label: "asan" - key: "asan" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz - rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd" - uid: 1000 - gid: 1000 - workspaces: - - "/cache/repos:/cache/repos" - timeout_in_minutes: 120 - if: | # We only run the `asan` job on Julia 1.8 and later. - (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7") - soft_fail: true # TODO: delete this line (and thus disallow failures) once JuliaLang/julia#42540 is fixed - commands: | - echo "--- Build julia-debug with ASAN" - contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} debug - - label: "tsan" - key: "tsan" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz - rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd" - uid: 1000 - gid: 1000 - workspaces: - - "/cache/repos:/cache/repos" - timeout_in_minutes: 120 - if: | # We only run the `tsan` job on Julia 1.8 and later. - (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7") - commands: | - echo "--- Build julia-debug runtime with TSAN" - contrib/tsan/build.sh ./tmp/test-tsan -j$${JULIA_CPU_THREADS:?} julia-src-debug + - group: ":linux: Linux" + steps: + - label: "asan" + key: "asan" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz + rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd" + uid: 1000 + gid: 1000 + workspaces: + - "/cache/repos:/cache/repos" + timeout_in_minutes: 120 + if: | # We only run the `asan` job on Julia 1.8 and later. + (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7") + soft_fail: true # TODO: delete this line (and thus disallow failures) once JuliaLang/julia#42540 is fixed + commands: | + echo "--- Build julia-debug with ASAN" + contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} debug + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" + + - label: "tsan" + key: "tsan" + plugins: + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz + rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd" + uid: 1000 + gid: 1000 + workspaces: + - "/cache/repos:/cache/repos" + timeout_in_minutes: 120 + if: | # We only run the `tsan` job on Julia 1.8 and later. + (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7") + commands: | + echo "--- Build julia-debug runtime with TSAN" + contrib/tsan/build.sh ./tmp/test-tsan -j$${JULIA_CPU_THREADS:?} julia-src-debug + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/main/misc/signed_pipeline_test.yml b/pipelines/main/misc/signed_pipeline_test.yml deleted file mode 100644 index a4e6b897..00000000 --- a/pipelines/main/misc/signed_pipeline_test.yml +++ /dev/null @@ -1,21 +0,0 @@ -agents: - queue: "julia" - os: "linux" - -## pipeline that showcases decryption of environment variable -steps: - - label: ":lock: :rocket: Signed pipeline test" - # We must accept the signed job id secret in order to propagate secrets - env: - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - depends_on: - plugins: - - JuliaCI/external-buildkite#v1: - version: ".buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - staticfloat/cryptic#v2: - variables: - - SECRET_KEY="U2FsdGVkX18tb7st0SuQAvh4Yv4xENxOAu8q9XkmOeDVKBNY4FngEwK3xmiKUqaS" - commands: | - echo "SECRET_KEY: $${SECRET_KEY}" - diff --git a/pipelines/main/misc/signed_pipeline_test.yml.signature b/pipelines/main/misc/signed_pipeline_test.yml.signature deleted file mode 100644 index 66b35421388ab66134756fcfe5e9d65fa4c7c164..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 96 zcmV-m0H6O;VQh3|WM5x!yVI=b&+sTQ@=UrAv#-7lF;-!01oY<1kypIEPvaSc{v~$V zo6KEE(ObOn3{&j$F^N!?r6%PhI(UV#9sqb#+)9E;0U?rCN9|A$Z+^^nBnDl;yT^w= C3@;G? diff --git a/pipelines/main/misc/whitespace.yml b/pipelines/main/misc/whitespace.yml index 4d6f896a..5b37b8ec 100644 --- a/pipelines/main/misc/whitespace.yml +++ b/pipelines/main/misc/whitespace.yml @@ -1,24 +1,26 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" steps: - - label: "whitespace" - key: "whitespace" - plugins: - - JuliaCI/external-buildkite#v1: - version: ".buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" - workspaces: - - "/cache/repos:/cache/repos" - timeout_in_minutes: 10 - commands: | - make --output-sync -j$${JULIA_CPU_THREADS:?} check-whitespace + - group: ":buildkite: Infrastructure" + steps: + - label: ":buildkite: Whitespace check" + key: "whitespace" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz + rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" + workspaces: + - "/cache/repos:/cache/repos" + timeout_in_minutes: 10 + commands: | + make --output-sync -j$${JULIA_CPU_THREADS:?} check-whitespace + agents: + queue: "julia" + # Only run on `sandbox.jl`-capable machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/main/platforms/build_linux.arches b/pipelines/main/platforms/build_linux.arches new file mode 100644 index 00000000..851bd17f --- /dev/null +++ b/pipelines/main/platforms/build_linux.arches @@ -0,0 +1,17 @@ +# OS TRIPLET ALLOW_FAIL ARCH_ROOTFS MAKE_FLAGS TIMEOUT ROOTFS_TAG ROOTFS_HASH +# linux i686-linux-gnu . i686 . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 +linux x86_64-linux-gnu . x86_64 . . v5.1 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0 +# linux aarch64-linux-gnu . aarch64 . . ---- ---------------------------------------- +# linux armv7l-linux-gnueabihf . armv7l . . ---- ---------------------------------------- +# linux powerpc64le-linux-gnu . powerpc64le . . ---- ---------------------------------------- +# musl x86_64-linux-musl . x86_64 . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 + + +# These special lines allow us to embed default values for the columns above. +# Any column without a default mapping here will simply substitute a `.` to the empty string + +# Of course we do not allow jobs to fail typically +#default ALLOW_FAIL false + +# Most jobs should finish within 1.5 hours, barring exceptionally slow hardware +#default TIMEOUT 90 diff --git a/pipelines/main/platforms/build_linux.yml b/pipelines/main/platforms/build_linux.yml new file mode 100644 index 00000000..29f5027a --- /dev/null +++ b/pipelines/main/platforms/build_linux.yml @@ -0,0 +1,57 @@ +steps: + - group: ":linux: Linux" + steps: + - label: ":linux: build ${TRIPLET?}" + key: "build_${TRIPLET?}" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${OS?}.${ARCH_ROOTFS?}.tar.gz + rootfs_treehash: "${ROOTFS_HASH?}" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + timeout_in_minutes: ${TIMEOUT?} + commands: | + # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc... + TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh + + echo "--- Build Julia" + make --output-sync -j 8 ${MAKE_FLAGS?} + + echo "--- Check that the working directory is clean" + if [ -z "$(git status --short)" ]; then + echo "INFO: The working directory is clean." + else + echo "ERROR: The working directory is dirty." + echo "Output of git status:" + git status + exit 1 + fi + + echo "--- Print Julia version info" + ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' + + echo "--- Create build artifacts" + make --output-sync -j 8 binary-dist ${MAKE_FLAGS?} + + # Rename the build artifact in case we want to name it differently, as is the case on `musl`. + if [[ "$${JULIA_BINARYDIST_FILENAME}" != "$${UPLOAD_FILENAME}" ]]; then + mv $${JULIA_BINARYDIST_FILENAME} $${UPLOAD_FILENAME} + fi + + echo "--- Upload build artifacts" + buildkite-agent artifact upload $${UPLOAD_FILENAME} + agents: + queue: "julia" + # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/main/platforms/package_linux.arches b/pipelines/main/platforms/package_linux.arches deleted file mode 100644 index dec82f53..00000000 --- a/pipelines/main/platforms/package_linux.arches +++ /dev/null @@ -1,7 +0,0 @@ -# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -linux 32 . . 32 i686 . . . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 -linux 64 . . 64 x86_64 . . . . . . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -# linux aarch64 . . aarch64 aarch64 . . . . . . . .... ........................................ -# linux armv7l . . armv7l armv7l . . . . . . . .... ........................................ -# linux ppc64le . . ppc64le powerpc64le . . . . . . . .... ........................................ -musl 64 . . 64 x86_64 . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 diff --git a/pipelines/main/platforms/package_linux.yml b/pipelines/main/platforms/package_linux.yml deleted file mode 100644 index ce778c39..00000000 --- a/pipelines/main/platforms/package_linux.yml +++ /dev/null @@ -1,54 +0,0 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" -steps: - - label: "package_${PLATFORM?}${LABEL?}" - key: package_${PLATFORM?}${LABEL?} - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz - rootfs_treehash: "${ROOTFS_HASH?}" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - timeout_in_minutes: ${TIMEOUT_BK?} - commands: | - echo "--- Print the full and short commit hashes" - SHORT_COMMIT_LENGTH=10 - SHORT_COMMIT=`echo $${BUILDKITE_COMMIT:?} | cut -c1-$${SHORT_COMMIT_LENGTH:?}` - ARTIFACT_FILE_EXTENSION="tar.gz" - ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-${PLATFORM?}${ARCH?}.$${ARTIFACT_FILE_EXTENSION:?}" - JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME ${MAKE_FLAGS?} | cut -c27- | tr -s ' '` - JULIA_BINARYDIST="$${JULIA_BINARYDIST_FILENAME:?}.$${ARTIFACT_FILE_EXTENSION:?}" - - echo "The full commit is: $${BUILDKITE_COMMIT:?}" - echo "The short commit is: $${SHORT_COMMIT:?}" - echo "The artifact filename will be: $${ARTIFACT_FILENAME:?}" - - echo "--- Build Julia from source" - rm -rf $${ARTIFACT_FILENAME:?} - make --output-sync -j 8 ${MAKE_FLAGS?} - - echo "--- Check that the working directory is clean" - if [ -z "$(git status --short)" ]; then echo "INFO: The working directory is clean."; else echo "ERROR: The working directory is dirty."; echo "Output of git status:"; git status; exit 1; fi - - echo "--- Print Julia version info" - ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - - echo "--- Create build artifacts" - make --output-sync -j 8 binary-dist ${MAKE_FLAGS?} - ls -l $${JULIA_BINARYDIST:?} - if [[ "$${JULIA_BINARYDIST:?}" != "$${ARTIFACT_FILENAME:?}" ]]; then - mv $${JULIA_BINARYDIST:?} $${ARTIFACT_FILENAME:?} - fi - ls -l $${ARTIFACT_FILENAME:?} - echo "--- Upload build artifacts" - buildkite-agent artifact upload $${ARTIFACT_FILENAME:?} diff --git a/pipelines/main/platforms/test_linux.arches b/pipelines/main/platforms/test_linux.arches new file mode 100644 index 00000000..6975716e --- /dev/null +++ b/pipelines/main/platforms/test_linux.arches @@ -0,0 +1,20 @@ +# OS TRIPLET ALLOW_FAIL ARCH_ROOTFS TIMEOUT USE_RR ROOTFS_TAG ROOTFS_HASH +# linux 686-linux-gnu . i686 . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 +linux x86_64-linux-gnu . x86_64 . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a +linux x86_64-linux-gnu . x86_64 120 rr v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a + +# linux aarch64-linux-gnu true aarch64 . . ---- ---------------------------------------- +# linux armv7l-linux-gnueabihf true armv7l . . ---- ---------------------------------------- +# linux powerpc64le-linux-gnu true powerpc64le . . ---- ---------------------------------------- + +# musl x86_64-linux-musl true x86_64 . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 + + +# These special lines allow us to embed default values for the columns above. +# Any column without a default mapping here will simply substitute a `.` to the empty string + +# Of course we do not allow jobs to fail typically +#default ALLOW_FAIL false + +# Most jobs should finish within 1.5 hours, barring exceptionally slow hardware +#default TIMEOUT 90 diff --git a/pipelines/main/platforms/test_linux.yml b/pipelines/main/platforms/test_linux.yml new file mode 100644 index 00000000..46837a35 --- /dev/null +++ b/pipelines/main/platforms/test_linux.yml @@ -0,0 +1,84 @@ +steps: + - group: ":linux: Linux" + steps: + - label: ":linux: :test_tube: test ${TRIPLET?}${USE_RR-}" + key: "test_${TRIPLET?}${USE_RR-}" + depends_on: + - "build_${TRIPLET?}" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${OS?}.${ARCH_ROOTFS?}.tar.gz + # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${OS?}.${ARCH_ROOTFS?}.tar.gz + rootfs_treehash: "${ROOTFS_HASH?}" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + env: + JULIA_SHELL: "/bin/bash" + timeout_in_minutes: ${TIMEOUT?} + soft_fail: ${ALLOW_FAIL?} + commands: | + # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc... + TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh + + echo "--- Download build artifacts" + buildkite-agent artifact download "$${UPLOAD_FILENAME}" . + + echo "--- Extract build artifacts" + tar xzf "$${UPLOAD_FILENAME}" "$${JULIA_INSTALL_DIR}/" + + echo "--- Print Julia version info" + $${JULIA_BINARY} -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' + echo "JULIA_CPU_THREADS is: $${JULIA_CPU_THREADS}" + $${JULIA_BINARY} -e '@info "" Sys.CPU_THREADS' + + echo "--- Set some environment variables" + export OPENBLAS_NUM_THREADS=8 + unset JULIA_DEPOT_PATH + unset JULIA_PKG_SERVER + + # Make sure that temp files and temp directories are created in a location that is + # backed by real storage. + export TMPDIR="$(pwd)/tmp" + mkdir -p $${TMPDIR} + + # By default, run all tests. + export TESTS="all LibGit2/online --ci" + + # If we're running inside of `rr`, limit the number of threads + if [[ "${USE_RR?}" == "rr" ]]; then + export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY}" + export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])" + export JULIA_NUM_THREADS=1 + + # Give `rr` 60 minutes to run the tests + export JULIA_TEST_RR_TIMEOUT="60" + else + export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY}" + export NCORES_FOR_TESTS="Sys.CPU_THREADS" + export JULIA_NUM_THREADS=16 + fi + + echo "--- Print the list of test sets, and other useful environment variables" + echo "JULIA_CMD_FOR_TESTS is: $${JULIA_CMD_FOR_TESTS:?}" + echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set + echo "NCORES_FOR_TESTS is: $${NCORES_FOR_TESTS:?}" + echo "OPENBLAS_NUM_THREADS is: $${OPENBLAS_NUM_THREADS:?}" + echo "TESTS is: $${TESTS:?}" + echo "USE_RR is: ${USE_RR?}" + + echo "--- Run the Julia test suite" + $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})" + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/main/platforms/tester_linux.arches b/pipelines/main/platforms/tester_linux.arches deleted file mode 100644 index d1304563..00000000 --- a/pipelines/main/platforms/tester_linux.arches +++ /dev/null @@ -1,25 +0,0 @@ -# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -linux 32_g1 g1 . 32 i686 . . . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 -linux 32_g2 g2 . 32 i686 . . . 3 . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 - -linux 64_g1_mt g1 . 64 x86_64 . . . . . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g2_mt g2 . 64 x86_64 . . . 3 . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -linux 64_g1_st g1 . 64 x86_64 . . . . . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g2_st g2 . 64 x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -linux 64_g1_rrst g1 . 64 x86_64 . 300 240 . yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g2_rrst g2 . 64 x86_64 . 180 120 3 yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g3_st g3 . 64 x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -# linux aarch64_g1 g1 true aarch64 aarch64 . . . . . . . ---- ---------------------------------------- -# linux aarch64_g2 g2 true aarch64 aarch64 . . . . . . . ---- ---------------------------------------- - -# linux armv7l_g1 g1 true armv7l armv7l . . . . . . . ---- ---------------------------------------- -# linux armv7l_g2 g2 true armv7l armv7l . . . . . . . ---- ---------------------------------------- - -# linux ppc64le_g1 g1 true ppc64le powerpc64le . . . . . . . ---- ---------------------------------------- -# linux ppc64le_g2 g2 true ppc64le powerpc64le . . . . . . . ---- ---------------------------------------- - -musl 64_g1 g1 true 64 x86_64 . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 -musl 64_g2 g2 true 64 x86_64 . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 diff --git a/pipelines/main/platforms/tester_linux.yml b/pipelines/main/platforms/tester_linux.yml deleted file mode 100644 index 18b019e6..00000000 --- a/pipelines/main/platforms/tester_linux.yml +++ /dev/null @@ -1,120 +0,0 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" -steps: - - label: "tester_${PLATFORM?}${LABEL?}" - key: tester_${PLATFORM?}${LABEL?} - depends_on: package_${PLATFORM?}${ARCH?} - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz - # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz - rootfs_treehash: "${ROOTFS_HASH?}" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - env: - JULIA_SHELL: "/bin/bash" - timeout_in_minutes: ${TIMEOUT_BK?} - retry: - automatic: - - exit_status: "*" - limit: ${RETRIES?} - soft_fail: ${ALLOW_FAIL?} - commands: | - echo "--- Print the full and short commit hashes" - SHORT_COMMIT_LENGTH=10 - SHORT_COMMIT=`echo $${BUILDKITE_COMMIT:?} | cut -c1-$${SHORT_COMMIT_LENGTH:?}` - JULIA_DIR="julia-$${SHORT_COMMIT:?}" - JULIA_BINARY="$${JULIA_DIR:?}/bin/julia" - ARTIFACT_FILE_EXTENSION="tar.gz" - ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-${PLATFORM?}${ARCH?}.$${ARTIFACT_FILE_EXTENSION:?}" - echo "The full commit is: $${BUILDKITE_COMMIT:?}" - echo "The short commit is: $${SHORT_COMMIT:?}" - echo "The artifact filename will be: $${ARTIFACT_FILENAME:?}" - echo "The Julia directory name will be: $${JULIA_DIR:?}" - echo "The Julia binary will be: $${JULIA_BINARY:?}" - - echo "--- Download build artifacts" - rm -rf $${ARTIFACT_FILENAME:?} - buildkite-agent artifact download $${ARTIFACT_FILENAME:?} . - - echo "--- Extract build artifacts" - rm -rf $${JULIA_DIR:?}/ - tar xzf $${ARTIFACT_FILENAME:?} $${JULIA_DIR:?}/ - - echo "--- Print Julia version info" - $${JULIA_BINARY:?} -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - echo "JULIA_CPU_THREADS is: $${JULIA_CPU_THREADS:?}" - $${JULIA_BINARY:?} -e '@info "" Sys.CPU_THREADS' - - echo "--- Set some environment variables" - export OPENBLAS_NUM_THREADS=8 - unset JULIA_DEPOT_PATH - unset JULIA_PKG_SERVER - - # Make sure that temp files and temp directories are created in a location that is - # backed by real storage. - export TMPDIR="$(pwd)/tmp" - mkdir -p $${TMPDIR:?} - - export NETWORK_RELATED_TESTS="Artifacts Downloads download LazyArtifacts LibGit2/online Pkg" - - if [[ "${GROUP?}" == "all" ]]; then - export TESTS="all LibGit2/online --force-net" - elif [[ "${GROUP?}" == "all_except_pkg" ]]; then - export TESTS="all LibGit2/online --force-net --skip Pkg" - elif [[ "${GROUP?}" == "g1" ]]; then - # Group 1: ALL tests EXCEPT the network-related tests. - export TESTS="all --force-net --skip $${NETWORK_RELATED_TESTS:?}" - elif [[ "${GROUP?}" == "g2" ]]; then - # Group 2: ONLY the network-related tests. - # In Group 2, we use whatever the default setting is with regards to the Pkg server. - export TESTS="$${NETWORK_RELATED_TESTS:?} --force-net" - elif [[ "${GROUP?}" == "g3" ]]; then - # Group 3: only Pkg. - # In Group 3, we explicitly opt-out of the Pkg server. - # The purpose of group 3 is to test the non-Pkg-server codepaths of Pkg. - export TESTS="Pkg --force-net" - export JULIA_PKG_SERVER="" - else - echo "Invalid value for GROUP: ${GROUP?}" - exit 1 - fi - - export JULIA_TEST_RR_TIMEOUT="${TIMEOUT_RR?}" - - if [[ "${IS_RR?}" == "yes" ]]; then - export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}" - export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])" - else - export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?}" - export NCORES_FOR_TESTS="Sys.CPU_THREADS" - fi - - if [[ "${IS_ST?}" == "yes" ]]; then - export JULIA_NUM_THREADS=1 - fi - - if [[ "${IS_MT?}" == "yes" ]]; then - export JULIA_NUM_THREADS=16 - fi - - echo "--- Print the test group, list of test sets, and other useful environment variables" - echo "JULIA_CMD_FOR_TESTS is: $${JULIA_CMD_FOR_TESTS:?}" - echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set - echo "NCORES_FOR_TESTS is: $${NCORES_FOR_TESTS:?}" - echo "OPENBLAS_NUM_THREADS is: $${OPENBLAS_NUM_THREADS:?}" - echo "GROUP is: ${GROUP?}" - echo "TESTS is: $${TESTS:?}" - - echo "--- Run the Julia test suite" - $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})" diff --git a/pipelines/main/platforms/upload_linux.yml b/pipelines/main/platforms/upload_linux.yml new file mode 100644 index 00000000..baa18d71 --- /dev/null +++ b/pipelines/main/platforms/upload_linux.yml @@ -0,0 +1,55 @@ +steps: + - label: ":linux: upload ${TRIPLET?}" + key: "upload_${TRIPLET?}" + depends_on: + # Wait for the builder to finish + - "build_${TRIPLET?}" + # Wait for the tester to finish + - "test_${TRIPLET?}" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.1/aws_uploader.x86_64.tar.gz + rootfs_treehash: "f865a3a68e33de10b3b4c9c5d0c9e5f2ff5a6965" + uid: 1000 + gid: 1000 + - staticfloat/cryptic#v2: + variables: + - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" + - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" + timeout_in_minutes: ${TIMEOUT?} + commands: | + # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc... + TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh + + echo "--- Download $${UPLOAD_FILENAME} to ." + buildkite-agent artifact download "$${UPLOAD_FILENAME}" . + + # We first upload the canonical fully-specified upload target, which is the first one: + echo "--- Upload to S3" + aws s3 cp --acl public-read "$${UPLOAD_FILENAME}" "s3://$${UPLOAD_TARGETS[0]}" + + echo "--- Copy to secondary upload targets" + for SECONDARY_TARGET in $${UPLOAD_TARGETS[@]:1}; do + aws s3 cp --acl public-read "s3://$${UPLOAD_TARGETS[0]}" "s3://$${SECONDARY_TARGET}" + done + + # Report to the user some URLs that they can use to download this from + echo "+++ Uploaded to targets" + for UPLOAD_TARGET in $${UPLOAD_TARGETS[@]}; do + echo " -> s3://$${UPLOAD_TARGET}" + done + agents: + queue: "julia" + # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox_capable: "true" + os: "linux" + env: + # Receive cryptic token from parent job + BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/pipelines/scheduled/0_webui.yml b/pipelines/scheduled/0_webui.yml index 78031b49..c90cf937 100644 --- a/pipelines/scheduled/0_webui.yml +++ b/pipelines/scheduled/0_webui.yml @@ -1,9 +1,6 @@ # This file represents what is put into the webUI. # It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. # We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. -agents: - queue: "julia" - sandbox.jl: "true" steps: - label: ":unlock: Unlock secrets, launch pipelines" plugins: @@ -25,3 +22,6 @@ steps: - .buildkite/pipelines/scheduled/coverage/coverage_linux64.yml - .buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl - .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + agents: + queue: "julia" + sandbox_capable: "true" diff --git a/pipelines/scheduled/coverage/coverage_linux64.yml b/pipelines/scheduled/coverage/coverage_linux64.yml index 1ff88577..4d011cec 100644 --- a/pipelines/scheduled/coverage/coverage_linux64.yml +++ b/pipelines/scheduled/coverage/coverage_linux64.yml @@ -1,14 +1,9 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" steps: - label: ":unlock: :coverage: Run coverage test" # We must accept the signed job id secret in order to propagate secrets env: BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - depends_on: + depends_on: ~ plugins: - staticfloat/cryptic: variables: @@ -42,3 +37,7 @@ steps: echo "--- Process and upload coverage information" ./julia .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/scheduled/coverage/coverage_linux64.yml.signature b/pipelines/scheduled/coverage/coverage_linux64.yml.signature index b28f81ebfe9f5b6fad981794f2779cb1b3314679..84d07008f20a6f3b3996500d4c5c63c7543866a7 100644 GIT binary patch literal 96 zcmV-m0H6O;VQh3|WM5wpkTm`8lJ!l@g)!N?fr)T&8QUiXBu5G`j;dzV8~i&2 + exit 1 + fi + done +} + + +# Determine variable names from the header of the .arches file: +readarray -d ' ' -s 1 -t var_names < <(head -1 "${ARCHES_FILE}" | tr -s ' ') + +# Determine any embedded defaults +declare -A defaults_map +readarray -t default_mapping_lines < <(grep "^#default" "${ARCHES_FILE}" | tr -s ' ') +for idx in "${!default_mapping_lines[@]}"; do + key="$( cut -d' ' -f2 <<<"${default_mapping_lines[${idx}]}" | xargs)" + value="$(cut -d' ' -f3 <<<"${default_mapping_lines[${idx}]}" | xargs)" + enforce_sanitized "${key}" "${value}" + defaults_map["${key}"]="${value}" +done + +cat "${ARCHES_FILE}" | while read line; do + # Remove whitespace from the beginning and end of each line + line="$(tr -s ' ' <<<"${line}")" + + # Skip any line that begins with the `#` character + if [[ $line == \#* ]]; then + continue + fi + + # Skip any empty line + if [[ $line == "" ]]; then + continue + fi + + # Skip any line that contains suspicious characters, to prevent shell escaping bugs + enforce_sanitized "${line}" + + # Convert line to array + readarray -d ' ' -t line_array <<<"${line}" + + # Panic if we don't have the same number of items as our column names: + if [[ "${#line_array[@]}" != "${#var_names[@]}" ]]; then + echo "ERROR: The following line does not contain ${#var_names[@]} columns as we would expect from the header of ${ARCHES_FILE}" >&2 + echo "${line}" + exit 1 + fi + + # Loop over columns, bind values to their column name + for idx in "${!var_names[@]}"; do + # Get the name and value + name="$(xargs <<<"${var_names[${idx}]}")" + value="$(xargs <<<"${line_array[${idx}]}")" + + # Apply default values to our special `.` token + if [[ "${value}" == "." ]]; then + value="${defaults_map[${name}]:-}" + fi + + echo -n "${name}=\"${value}\" " + done + echo +done diff --git a/utilities/arches_pipeline_upload.sh b/utilities/arches_pipeline_upload.sh new file mode 100644 index 00000000..5ac88acd --- /dev/null +++ b/utilities/arches_pipeline_upload.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -eou pipefail +shopt -s nullglob + +# This script reads in an `.arches` file, processes the columns and default value mappings +# within it, and outputs an environment block (e.g. a line of the form "X=a B= C=123") for +# each architecture defined within, to be used by other tools such as the brother script +# `arches_pipeline_upload.sh`, which uses those environment mappings to template pipeline +# YAML files that are being uploaded by `buildkite-agent pipeline upload`. + +ARCHES_FILE="${1:-}" +if [[ ! -f "${ARCHES_FILE}" ]] ; then + echo "Arches file does not exist: '${ARCHES_FILE}'" + exit 1 +fi + +YAML_FILE="${2:-}" +if [[ ! -f "${YAML_FILE}" ]] ; then + echo "YAML file does not exist: '${YAML_FILE}'" + exit 1 +fi + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +"${BASH}" "${SCRIPT_DIR}/arches_env.sh" "${ARCHES_FILE}" | while read env_map; do + # Export the environment mappings, then launch the yaml file + eval "export ${env_map}" + buildkite-agent pipeline upload "${YAML_FILE}" +done diff --git a/utilities/calc_version_envs.sh b/utilities/calc_version_envs.sh new file mode 100755 index 00000000..60c54b37 --- /dev/null +++ b/utilities/calc_version_envs.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Extract information from our triplet +export ARCH="$(cut -d- -f1 <<<"${TRIPLET}")" +case "${TRIPLET}" in + *-apple-*) + OS="macos" + ;; + *-freebsd*) + OS="freebsd" + ;; + *-mingw*) + OS="windows" + ;; + *-gnu*) + OS="linux" + ;; + *-musl*) + OS="musl" + ;; + *) + echo "Unknown triplet OS '${TRIPLET}'" >&2 + exit 1 + ;; +esac +export OS + +# Extract git information +SHORT_COMMIT_LENGTH=10 +LONG_COMMIT="$(git rev-parse HEAD)" +SHORT_COMMIT="$(echo ${LONG_COMMIT} | cut -c1-${SHORT_COMMIT_LENGTH})" + +# Extract information about the current julia version number +JULIA_VERSION="$(cat VERSION)" +MAJMIN="${JULIA_VERSION:0:3}" + +# If we're on a tag, then our "tar version" will be the julia version. +# Otherwise, it's the short commit. +if git describe --tags --exact-match >/dev/null 2>/dev/null; then + TAR_VERSION="${JULIA_VERSION}" +else + TAR_VERSION="${SHORT_COMMIT}" +fi + +# Build the filename that we'll upload as, and get the filename that will be built +# These are not the same in situations such as `musl`, where the build system doesn't +# differentiate but we need to give it a different name. +JULIA_BINARYDIST_FILENAME="$(make print-JULIA_BINARYDIST_FILENAME | cut -c27- | tr -s ' ').tar.gz" + +JULIA_INSTALL_DIR="julia-${TAR_VERSION}" +JULIA_BINARY="${JULIA_INSTALL_DIR}/bin/julia" + +# We generally upload to multiple upload targets +UPLOAD_TARGETS=( + # First, we have the canonical fully-specified upload target + "julialangnightlies/bin/${OS?}/${ARCH?}/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}.tar.gz" + + # Next, we have the "latest" upload target + "julialangnightlies/bin/${OS?}/${ARCH?}/${MAJMIN?}/julia-latest-${OS?}-${ARCH?}.tar.gz" +) +UPLOAD_FILENAME="julia-${TAR_VERSION?}-${OS?}-${ARCH?}.tar.gz" + +# Finally, for compatibility, we keep on uploading x86_64 and i686 targets to folders called `x64` +# and `x86`, respectively, although I would very much like to stop doing that. +if [[ "${ARCH}" == "x86_64" ]]; then + UPLOAD_TARGETS+=( "julialangnightlies/bin/${OS?}/x64/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}.tar.gz" ) + UPLOAD_TARGETS+=( "julialangnightlies/bin/${OS?}/x64/${MAJMIN?}/julia-latest-${OS?}-${ARCH?}.tar.gz" ) +elif [[ "${ARCH}" == "i686" ]]; then + UPLOAD_TARGETS+=( "julialangnightlies/bin/${OS?}/x86/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}.tar.gz" ) + UPLOAD_TARGETS+=( "julialangnightlies/bin/${OS?}/x86/${MAJMIN?}/julia-latest-${OS?}-${ARCH?}.tar.gz" ) +fi + +echo "--- Print the full and short commit hashes" +echo "The full commit is: ${LONG_COMMIT}" +echo "The short commit is: ${SHORT_COMMIT}" +echo "Julia will be installed to: ${JULIA_BINARY}" +echo "Detected Julia version: ${MAJMIN} (${JULIA_VERSION})" +echo "Detected build platform: ${TRIPLET} (${ARCH}, ${OS})" +echo "Julia will be uploaded to: s3://${UPLOAD_TARGETS[0]}" +echo "With additional upload targets:" +for UPLOAD_TARGET in ${UPLOAD_TARGETS[@]:1}; do + echo " -> s3://${UPLOAD_TARGET}" +done diff --git a/utilities/platforms/platforms.sh b/utilities/platforms/platforms.sh deleted file mode 100755 index 9a47c18e..00000000 --- a/utilities/platforms/platforms.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -ARCHES="$1" -YAML="$2" - -if [[ ! -f "${ARCHES:?}" ]] ; then - echo "Arches file does not exist: ${ARCHES:?}" - exit 1 -fi - -if [[ ! -f "${YAML:?}" ]] ; then - echo "YAML file does not exist: ${YAML:?}" - exit 1 -fi - -cat "${ARCHES:?}" | tr -s ' ' | while read _line; do - # Remove whitespace from the beginning and end of each line - line=`echo $_line | tr -s ' '` - - # Skip any line that begins with the `#` character - if [[ $line == \#* ]]; then - continue - fi - - # Skip any empty line - if [[ $line == "" ]]; then - continue - fi - - export PLATFORM=`echo $line | cut -d ' ' -f 1 | tr -s ' '` - export LABEL=`echo $line | cut -d ' ' -f 2 | tr -s ' '` - export GROUP=`echo $line | cut -d ' ' -f 3 | tr -s ' '` - - export ALLOW_FAIL=`echo $line | cut -d ' ' -f 4 | tr -s ' '` - export ARCH=`echo $line | cut -d ' ' -f 5 | tr -s ' '` - export ARCH_ROOTFS=`echo $line | cut -d ' ' -f 6 | tr -s ' '` - - export MAKE_FLAGS=`echo $line | cut -d ' ' -f 7 | tr -s ' '` - export TIMEOUT_BK=`echo $line | cut -d ' ' -f 8 | tr -s ' '` - export TIMEOUT_RR=`echo $line | cut -d ' ' -f 9 | tr -s ' '` - export RETRIES=`echo $line | cut -d ' ' -f 10 | tr -s ' '` - export IS_RR=`echo $line | cut -d ' ' -f 11 | tr -s ' '` - export IS_ST=`echo $line | cut -d ' ' -f 12 | tr -s ' '` - export IS_MT=`echo $line | cut -d ' ' -f 13 | tr -s ' '` - export ROOTFS_TAG=`echo $line | cut -d ' ' -f 14 | tr -s ' '` - export ROOTFS_HASH=`echo $line | cut -d ' ' -f 15 | tr -s ' '` - - if [[ "${IS_ST:?}" == "yes" ]]; then - if [[ "${IS_MT:?}" == "yes" ]]; then - echo "You cannot set both IS_ST and IS_MT to yes" - exit 1 - fi - fi - - if [[ "${ALLOW_FAIL:?}" == "." ]]; then - export ALLOW_FAIL="false" - fi - - if [[ "${MAKE_FLAGS:?}" == "." ]]; then - export MAKE_FLAGS="" - fi - - if [[ "${TIMEOUT_BK:?}" == "." ]]; then - export TIMEOUT_BK="90" # minutes - fi - - if [[ "${TIMEOUT_RR:?}" == "." ]]; then - export TIMEOUT_RR="60" # minutes - fi - - if [[ "${RETRIES:?}" == "." ]]; then - export RETRIES="0" - fi - - buildkite-agent pipeline upload "${YAML:?}" -done From cdabb29a8fee1b181dda5dd18c4b3098dc787d20 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Wed, 9 Mar 2022 00:16:03 -0800 Subject: [PATCH 2/2] Remove this from the grouping for now due to cryptic requirements --- pipelines/main/launch_signed_jobs.yml | 43 ++++++++---------- .../main/launch_signed_jobs.yml.signature | Bin 96 -> 96 bytes .../coverage/coverage_linux64.yml.signature | 3 +- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/pipelines/main/launch_signed_jobs.yml b/pipelines/main/launch_signed_jobs.yml index 7e519b92..f8e1e776 100644 --- a/pipelines/main/launch_signed_jobs.yml +++ b/pipelines/main/launch_signed_jobs.yml @@ -1,27 +1,24 @@ # This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3 steps: - - group: ":buildkite: Infrastructure" - depends_on: - steps: - - label: ":buildkite: :unlock: Launch signed pipelines" - plugins: - - JuliaCI/external-buildkite#v1: - version: ".buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - commands: | - # Explicitly pass along the cryptic token to child pipelines - export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET + - label: ":buildkite: :unlock: Launch signed pipelines" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # Explicitly pass along the cryptic token to child pipelines + export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - # Launch `upload_linux` jobs to store tarballs into S3 once tests are done - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_linux.arches \ - .buildkite/pipelines/main/platforms/upload_linux.yml + # Launch `upload_linux` jobs to store tarballs into S3 once tests are done + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_linux.arches \ + .buildkite/pipelines/main/platforms/upload_linux.yml - # Don't share this with buildkite's env display - unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - agents: - queue: "julia" - cryptic_capable: "true" - env: - # Receive cryptic token from parent job - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} + # Don't share this with buildkite's env display + unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET + agents: + queue: "julia" + cryptic_capable: "true" + env: + # Receive cryptic token from parent job + BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/pipelines/main/launch_signed_jobs.yml.signature b/pipelines/main/launch_signed_jobs.yml.signature index 1f331d7539adc76f1d8887a9acc111f750b8fae4..80020daf38ea442f21403b9891d7a6b866104cc0 100644 GIT binary patch literal 96 zcmV-m0H6O;VQh3|WM5x}{6&|F+Vm#^=mXnxoNjHden_t$Q{QzIgLTmADfi%r)TIQl z2>C(%%ZnhQ_NYXL_xt%2Y>}y zc9bT%DSyY1i2SbswOo2C|Jl~Y6f&;5j(ky?s%HB8F^Bu@!~CnMk$VF(1q`(=U6E5g Ci7mJQ diff --git a/pipelines/scheduled/coverage/coverage_linux64.yml.signature b/pipelines/scheduled/coverage/coverage_linux64.yml.signature index 84d07008..ab24be2e 100644 --- a/pipelines/scheduled/coverage/coverage_linux64.yml.signature +++ b/pipelines/scheduled/coverage/coverage_linux64.yml.signature @@ -1,2 +1 @@ -Salted__�4���M̅1ٻ��pq�'$G -0��f��=�1�Ǜ�Ϩ|z��{Y��!�2g�3Ɔ3�\��$�4��k�� Ȅh���z�Sɚ/z \ No newline at end of file +Salted__��˕,���#�W�3�8O %��U�F]�_Ch1'�bB�MD#��%7=)Bw+�x�.b�mdud�"�g]d8{i^��=d� g��7 \ No newline at end of file