diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index eff8acb4..b77793c1 100755 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -4,10 +4,12 @@ ls -A1 | xargs rm -rf echo 'Clone testing branch of julia, main branch only:' git clone -b master https://github.com/JuliaCI/julia-buildkite-testing ./ echo +git log -1 +echo -echo 'Set buildkite metadata' # Force external-buildkite plugin to use the current branch's commit as the version to download for this and all child jobs if ! buildkite-agent meta-data exists BUILDKITE_PLUGIN_EXTERNAL_BUILDKITE_VERSION; then + echo "Override external-buildkite metadata to VERSION '${BUILDKITE_COMMIT}' and REPO_URL 'https://github.com/JuliaCI/julia-buildkite'" buildkite-agent meta-data set BUILDKITE_PLUGIN_EXTERNAL_BUILDKITE_VERSION "${BUILDKITE_COMMIT:?}" buildkite-agent meta-data set BUILDKITE_PLUGIN_EXTERNAL_BUILDKITE_REPO_URL "https://github.com/JuliaCI/julia-buildkite" fi diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..6c6049d3 --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +sign_treehashes: + cd .buildkite/cryptic_repo_root && sign_treehashes --repo-root=$$(pwd) --verbose + +verify_treehashes: + cd .buildkite/cryptic_repo_root && verify_treehashes --repo-root=$$(pwd) --verbose diff --git a/pipelines/main/0_webui.yml b/pipelines/main/0_webui.yml index 15bfb24b..d476adc1 100644 --- a/pipelines/main/0_webui.yml +++ b/pipelines/main/0_webui.yml @@ -2,26 +2,29 @@ # It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. # We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. steps: - - label: ":unlock: Unlock secrets, launch pipelines" - agents: - queue: "julia" - sandbox.jl: "true" - plugins: - - JuliaCI/external-buildkite#v1: - version: ".buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - staticfloat/cryptic#v2: - # Our list of pipelines that should be launched (but don't require a signature) - # These pipelines can be modified by any contributor and CI will still run. - # Build secrets will not be available in these pipelines (or their children) - # but some of our signed pipelines can wait upon the completion of these unsigned - # pipelines. - unsigned_pipelines: - - .buildkite/pipelines/main/launch_unsigned_builders.yml + - group: ":buildkite: Infrastructure" + steps: + - label: ":unlock: Unlock secrets, launch pipelines" + agents: + queue: "julia" + sandbox_capable: "true" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - staticfloat/cryptic#v2: + # Our list of pipelines that should be launched (but don't require a signature) + # These pipelines can be modified by any contributor and CI will still run. + # Build secrets will not be available in these pipelines (or their children) + # but some of our signed pipelines can wait upon the completion of these unsigned + # pipelines. + unsigned_pipelines: + - .buildkite/pipelines/main/launch_unsigned_builders.yml - # Our signed pipelines must have a `signature` or `signature_file` parameter that - # verifies the treehash of the pipeline itself and the inputs listed in `inputs` - signed_pipelines: - - pipeline: .buildkite/pipelines/main/misc/signed_pipeline_test.yml - signature_file: .buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature - command: "true" + # Our signed pipelines must have a `signature` or `signature_file` parameter that + # verifies the treehash of the pipeline itself and the inputs listed in `inputs` + signed_pipelines: + - pipeline: .buildkite/pipelines/main/launch_signed_jobs.yml + signature_file: .buildkite/pipelines/main/launch_signed_jobs.yml.signature + allow_hash_override: true + command: "true" diff --git a/pipelines/main/launch_signed_jobs.yml b/pipelines/main/launch_signed_jobs.yml new file mode 100644 index 00000000..7e519b92 --- /dev/null +++ b/pipelines/main/launch_signed_jobs.yml @@ -0,0 +1,27 @@ +# This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3 +steps: + - group: ":buildkite: Infrastructure" + depends_on: + steps: + - label: ":buildkite: :unlock: Launch signed pipelines" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # Explicitly pass along the cryptic token to child pipelines + export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET + + # Launch `upload_linux` jobs to store tarballs into S3 once tests are done + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_linux.arches \ + .buildkite/pipelines/main/platforms/upload_linux.yml + + # Don't share this with buildkite's env display + unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET + agents: + queue: "julia" + cryptic_capable: "true" + env: + # Receive cryptic token from parent job + BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/pipelines/main/launch_signed_jobs.yml.signature b/pipelines/main/launch_signed_jobs.yml.signature new file mode 100644 index 00000000..1f331d75 Binary files /dev/null and b/pipelines/main/launch_signed_jobs.yml.signature differ diff --git a/pipelines/main/launch_unsigned_builders.yml b/pipelines/main/launch_unsigned_builders.yml index 28e0a9cf..02bab7c4 100644 --- a/pipelines/main/launch_unsigned_builders.yml +++ b/pipelines/main/launch_unsigned_builders.yml @@ -13,25 +13,40 @@ # something about the privileged steps. steps: - - label: ":buildkite: Launch unsigned jobs" - plugins: - - JuliaCI/external-buildkite#v1: - version: ".buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - commands: | - # Launch the miscellaneous jobs in alphabetical order. - #buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml - #buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml - #buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml - #buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml + - group: ":buildkite: Infrastructure" + steps: + - label: ":buildkite: Launch misc. jobs" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml + agents: + queue: "julia" - # Launch all of the platform jobs. - #bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml - #bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/tester_linux.arches .buildkite/pipelines/main/platforms/tester_linux.yml + - group: ":linux: Linux" + steps: + - label: ":buildkite: Launch jobs" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: | + # Launch the miscellaneous jobs in alphabetical order. + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml - # Launch the `whitespace` job last. Uploading it last actually causes it to start - # first. We want this job to start first because we want it to finish as quickly - # as possible. - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml - agents: - queue: julia + # Launch Linux packaging jobs + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_linux.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml + + # Launch Linux testing jobs + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_linux.arches \ + .buildkite/pipelines/main/platforms/test_linux.yml + agents: + queue: "julia" diff --git a/pipelines/main/misc/doctest.yml b/pipelines/main/misc/doctest.yml index b83139dd..eaea8d2e 100644 --- a/pipelines/main/misc/doctest.yml +++ b/pipelines/main/misc/doctest.yml @@ -1,34 +1,38 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" steps: - - label: "doctest" - key: doctest - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - commands: | - echo "--- Build Julia from source" - make --output-sync -j 6 + - group: ":linux: Linux" + steps: + - label: "doctest" + key: doctest + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz + rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + commands: | + echo "--- Build Julia from source" + make --output-sync -j 6 - echo "--- Print Julia version info" - ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' + echo "--- Print Julia version info" + ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - echo "--- Build Julia docs" - make docs + echo "--- Build Julia docs" + make docs - echo "--- Run Julia doctests" - JULIA_NUM_THREADS=1 make -C doc doctest=true - timeout_in_minutes: 45 + echo "--- Run Julia doctests" + JULIA_NUM_THREADS=1 make -C doc doctest=true + timeout_in_minutes: 45 + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/main/misc/embedding.yml b/pipelines/main/misc/embedding.yml index bdd2a0a6..3394231d 100644 --- a/pipelines/main/misc/embedding.yml +++ b/pipelines/main/misc/embedding.yml @@ -1,31 +1,35 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" steps: - - label: "embedding" - key: "embedding" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - commands: | - prefix="/tmp/prefix" - echo "+++ Build julia, deploy to $${prefix:?}" - make --output-sync -j$${JULIA_CPU_THREADS:?} JULIA_PRECOMPILE=0 prefix=$${prefix:?} install + - group: ":linux: Linux" + steps: + - label: "embedding" + key: "embedding" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz + rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + commands: | + prefix="/tmp/prefix" + echo "+++ Build julia, deploy to $${prefix:?}" + make --output-sync -j$${JULIA_CPU_THREADS:?} JULIA_PRECOMPILE=0 prefix=$${prefix:?} install - embedding_output="/tmp/embedding-test" - echo "+++ Run embedding tests, deploy to $${embedding_output:?}" - mkdir -p "$${embedding_output:?}" - make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/embedding JULIA="$${prefix:?}/bin/julia" BIN="$${embedding_output:?}" - timeout_in_minutes: 60 + embedding_output="/tmp/embedding-test" + echo "+++ Run embedding tests, deploy to $${embedding_output:?}" + mkdir -p "$${embedding_output:?}" + make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/embedding JULIA="$${prefix:?}/bin/julia" BIN="$${embedding_output:?}" + timeout_in_minutes: 60 + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/main/misc/llvmpasses.yml b/pipelines/main/misc/llvmpasses.yml index 97ea2e09..c5b9932f 100644 --- a/pipelines/main/misc/llvmpasses.yml +++ b/pipelines/main/misc/llvmpasses.yml @@ -1,51 +1,60 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" steps: - - label: "analyzegc" - key: "analyzegc" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1.2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz - rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd" - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - commands: | - echo "--- Install in-tree LLVM dependencies" - make --output-sync -j$${JULIA_CPU_THREADS:?} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind - echo "+++ run clangsa/analyzegc" - make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/clangsa - make --output-sync -j$${JULIA_CPU_THREADS:?} -C src analyzegc - timeout_in_minutes: 60 - - label: "llvmpasses" - key: "llvmpasses" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1.2: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" - uid: 1000 - gid: 1000 - workspaces: - - "/cache/repos:/cache/repos" - commands: | - echo "--- make release" - # Enable Julia assertions: FORCE_ASSERTIONS=1 - # Enable LLVM assertions: LLVM_ASSERTIONS=1 - export MAKE_ASSERT_FLAGS="FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1" - make --output-sync -j$${JULIA_CPU_THREADS:?} release JULIA_PRECOMPILE=0 $${MAKE_ASSERT_FLAGS:?} - echo "--- make src/install-analysis-deps" - make --output-sync -j$${JULIA_CPU_THREADS:?} -C src install-analysis-deps $${MAKE_ASSERT_FLAGS:?} - echo "+++ make test/llvmpasses" - make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/llvmpasses $${MAKE_ASSERT_FLAGS:?} - timeout_in_minutes: 60 + - group: ":linux: Linux" + steps: + - label: "analyzegc" + key: "analyzegc" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1.2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz + rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd" + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + commands: | + echo "--- Install in-tree LLVM dependencies" + make --output-sync -j$${JULIA_CPU_THREADS:?} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind + echo "+++ run clangsa/analyzegc" + make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/clangsa + make --output-sync -j$${JULIA_CPU_THREADS:?} -C src analyzegc + timeout_in_minutes: 60 + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" + + - label: "llvmpasses" + key: "llvmpasses" + plugins: + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1.2: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz + rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" + uid: 1000 + gid: 1000 + workspaces: + - "/cache/repos:/cache/repos" + commands: | + echo "--- make release" + # Enable Julia assertions: FORCE_ASSERTIONS=1 + # Enable LLVM assertions: LLVM_ASSERTIONS=1 + export MAKE_ASSERT_FLAGS="FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1" + make --output-sync -j$${JULIA_CPU_THREADS:?} release JULIA_PRECOMPILE=0 $${MAKE_ASSERT_FLAGS:?} + echo "--- make src/install-analysis-deps" + make --output-sync -j$${JULIA_CPU_THREADS:?} -C src install-analysis-deps $${MAKE_ASSERT_FLAGS:?} + echo "+++ make test/llvmpasses" + make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/llvmpasses $${MAKE_ASSERT_FLAGS:?} + timeout_in_minutes: 60 + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/main/misc/sanitizers.yml b/pipelines/main/misc/sanitizers.yml index a0c40dda..42feb06b 100644 --- a/pipelines/main/misc/sanitizers.yml +++ b/pipelines/main/misc/sanitizers.yml @@ -1,47 +1,56 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" steps: - - label: "asan" - key: "asan" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz - rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd" - uid: 1000 - gid: 1000 - workspaces: - - "/cache/repos:/cache/repos" - timeout_in_minutes: 120 - if: | # We only run the `asan` job on Julia 1.8 and later. - (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7") - soft_fail: true # TODO: delete this line (and thus disallow failures) once JuliaLang/julia#42540 is fixed - commands: | - echo "--- Build julia-debug with ASAN" - contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} debug - - label: "tsan" - key: "tsan" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz - rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd" - uid: 1000 - gid: 1000 - workspaces: - - "/cache/repos:/cache/repos" - timeout_in_minutes: 120 - if: | # We only run the `tsan` job on Julia 1.8 and later. - (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7") - commands: | - echo "--- Build julia-debug runtime with TSAN" - contrib/tsan/build.sh ./tmp/test-tsan -j$${JULIA_CPU_THREADS:?} julia-src-debug + - group: ":linux: Linux" + steps: + - label: "asan" + key: "asan" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz + rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd" + uid: 1000 + gid: 1000 + workspaces: + - "/cache/repos:/cache/repos" + timeout_in_minutes: 120 + if: | # We only run the `asan` job on Julia 1.8 and later. + (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7") + soft_fail: true # TODO: delete this line (and thus disallow failures) once JuliaLang/julia#42540 is fixed + commands: | + echo "--- Build julia-debug with ASAN" + contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} debug + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" + + - label: "tsan" + key: "tsan" + plugins: + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz + rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd" + uid: 1000 + gid: 1000 + workspaces: + - "/cache/repos:/cache/repos" + timeout_in_minutes: 120 + if: | # We only run the `tsan` job on Julia 1.8 and later. + (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7") + commands: | + echo "--- Build julia-debug runtime with TSAN" + contrib/tsan/build.sh ./tmp/test-tsan -j$${JULIA_CPU_THREADS:?} julia-src-debug + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/main/misc/signed_pipeline_test.yml b/pipelines/main/misc/signed_pipeline_test.yml deleted file mode 100644 index a4e6b897..00000000 --- a/pipelines/main/misc/signed_pipeline_test.yml +++ /dev/null @@ -1,21 +0,0 @@ -agents: - queue: "julia" - os: "linux" - -## pipeline that showcases decryption of environment variable -steps: - - label: ":lock: :rocket: Signed pipeline test" - # We must accept the signed job id secret in order to propagate secrets - env: - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - depends_on: - plugins: - - JuliaCI/external-buildkite#v1: - version: ".buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - staticfloat/cryptic#v2: - variables: - - SECRET_KEY="U2FsdGVkX18tb7st0SuQAvh4Yv4xENxOAu8q9XkmOeDVKBNY4FngEwK3xmiKUqaS" - commands: | - echo "SECRET_KEY: $${SECRET_KEY}" - diff --git a/pipelines/main/misc/signed_pipeline_test.yml.signature b/pipelines/main/misc/signed_pipeline_test.yml.signature deleted file mode 100644 index 66b35421..00000000 Binary files a/pipelines/main/misc/signed_pipeline_test.yml.signature and /dev/null differ diff --git a/pipelines/main/misc/whitespace.yml b/pipelines/main/misc/whitespace.yml index 4d6f896a..5b37b8ec 100644 --- a/pipelines/main/misc/whitespace.yml +++ b/pipelines/main/misc/whitespace.yml @@ -1,24 +1,26 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" steps: - - label: "whitespace" - key: "whitespace" - plugins: - - JuliaCI/external-buildkite#v1: - version: ".buildkite-external-version" - repo_url: "https://github.com/JuliaCI/julia-buildkite" - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" - workspaces: - - "/cache/repos:/cache/repos" - timeout_in_minutes: 10 - commands: | - make --output-sync -j$${JULIA_CPU_THREADS:?} check-whitespace + - group: ":buildkite: Infrastructure" + steps: + - label: ":buildkite: Whitespace check" + key: "whitespace" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz + rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" + workspaces: + - "/cache/repos:/cache/repos" + timeout_in_minutes: 10 + commands: | + make --output-sync -j$${JULIA_CPU_THREADS:?} check-whitespace + agents: + queue: "julia" + # Only run on `sandbox.jl`-capable machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/main/platforms/build_linux.arches b/pipelines/main/platforms/build_linux.arches new file mode 100644 index 00000000..851bd17f --- /dev/null +++ b/pipelines/main/platforms/build_linux.arches @@ -0,0 +1,17 @@ +# OS TRIPLET ALLOW_FAIL ARCH_ROOTFS MAKE_FLAGS TIMEOUT ROOTFS_TAG ROOTFS_HASH +# linux i686-linux-gnu . i686 . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 +linux x86_64-linux-gnu . x86_64 . . v5.1 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0 +# linux aarch64-linux-gnu . aarch64 . . ---- ---------------------------------------- +# linux armv7l-linux-gnueabihf . armv7l . . ---- ---------------------------------------- +# linux powerpc64le-linux-gnu . powerpc64le . . ---- ---------------------------------------- +# musl x86_64-linux-musl . x86_64 . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 + + +# These special lines allow us to embed default values for the columns above. +# Any column without a default mapping here will simply substitute a `.` to the empty string + +# Of course we do not allow jobs to fail typically +#default ALLOW_FAIL false + +# Most jobs should finish within 1.5 hours, barring exceptionally slow hardware +#default TIMEOUT 90 diff --git a/pipelines/main/platforms/build_linux.yml b/pipelines/main/platforms/build_linux.yml new file mode 100644 index 00000000..29f5027a --- /dev/null +++ b/pipelines/main/platforms/build_linux.yml @@ -0,0 +1,57 @@ +steps: + - group: ":linux: Linux" + steps: + - label: ":linux: build ${TRIPLET?}" + key: "build_${TRIPLET?}" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${OS?}.${ARCH_ROOTFS?}.tar.gz + rootfs_treehash: "${ROOTFS_HASH?}" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + timeout_in_minutes: ${TIMEOUT?} + commands: | + # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc... + TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh + + echo "--- Build Julia" + make --output-sync -j 8 ${MAKE_FLAGS?} + + echo "--- Check that the working directory is clean" + if [ -z "$(git status --short)" ]; then + echo "INFO: The working directory is clean." + else + echo "ERROR: The working directory is dirty." + echo "Output of git status:" + git status + exit 1 + fi + + echo "--- Print Julia version info" + ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' + + echo "--- Create build artifacts" + make --output-sync -j 8 binary-dist ${MAKE_FLAGS?} + + # Rename the build artifact in case we want to name it differently, as is the case on `musl`. + if [[ "$${JULIA_BINARYDIST_FILENAME}" != "$${UPLOAD_FILENAME}" ]]; then + mv $${JULIA_BINARYDIST_FILENAME} $${UPLOAD_FILENAME} + fi + + echo "--- Upload build artifacts" + buildkite-agent artifact upload $${UPLOAD_FILENAME} + agents: + queue: "julia" + # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/main/platforms/package_linux.arches b/pipelines/main/platforms/package_linux.arches deleted file mode 100644 index dec82f53..00000000 --- a/pipelines/main/platforms/package_linux.arches +++ /dev/null @@ -1,7 +0,0 @@ -# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -linux 32 . . 32 i686 . . . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 -linux 64 . . 64 x86_64 . . . . . . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -# linux aarch64 . . aarch64 aarch64 . . . . . . . .... ........................................ -# linux armv7l . . armv7l armv7l . . . . . . . .... ........................................ -# linux ppc64le . . ppc64le powerpc64le . . . . . . . .... ........................................ -musl 64 . . 64 x86_64 . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 diff --git a/pipelines/main/platforms/package_linux.yml b/pipelines/main/platforms/package_linux.yml deleted file mode 100644 index ce778c39..00000000 --- a/pipelines/main/platforms/package_linux.yml +++ /dev/null @@ -1,54 +0,0 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" -steps: - - label: "package_${PLATFORM?}${LABEL?}" - key: package_${PLATFORM?}${LABEL?} - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz - rootfs_treehash: "${ROOTFS_HASH?}" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - timeout_in_minutes: ${TIMEOUT_BK?} - commands: | - echo "--- Print the full and short commit hashes" - SHORT_COMMIT_LENGTH=10 - SHORT_COMMIT=`echo $${BUILDKITE_COMMIT:?} | cut -c1-$${SHORT_COMMIT_LENGTH:?}` - ARTIFACT_FILE_EXTENSION="tar.gz" - ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-${PLATFORM?}${ARCH?}.$${ARTIFACT_FILE_EXTENSION:?}" - JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME ${MAKE_FLAGS?} | cut -c27- | tr -s ' '` - JULIA_BINARYDIST="$${JULIA_BINARYDIST_FILENAME:?}.$${ARTIFACT_FILE_EXTENSION:?}" - - echo "The full commit is: $${BUILDKITE_COMMIT:?}" - echo "The short commit is: $${SHORT_COMMIT:?}" - echo "The artifact filename will be: $${ARTIFACT_FILENAME:?}" - - echo "--- Build Julia from source" - rm -rf $${ARTIFACT_FILENAME:?} - make --output-sync -j 8 ${MAKE_FLAGS?} - - echo "--- Check that the working directory is clean" - if [ -z "$(git status --short)" ]; then echo "INFO: The working directory is clean."; else echo "ERROR: The working directory is dirty."; echo "Output of git status:"; git status; exit 1; fi - - echo "--- Print Julia version info" - ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - - echo "--- Create build artifacts" - make --output-sync -j 8 binary-dist ${MAKE_FLAGS?} - ls -l $${JULIA_BINARYDIST:?} - if [[ "$${JULIA_BINARYDIST:?}" != "$${ARTIFACT_FILENAME:?}" ]]; then - mv $${JULIA_BINARYDIST:?} $${ARTIFACT_FILENAME:?} - fi - ls -l $${ARTIFACT_FILENAME:?} - echo "--- Upload build artifacts" - buildkite-agent artifact upload $${ARTIFACT_FILENAME:?} diff --git a/pipelines/main/platforms/test_linux.arches b/pipelines/main/platforms/test_linux.arches new file mode 100644 index 00000000..6975716e --- /dev/null +++ b/pipelines/main/platforms/test_linux.arches @@ -0,0 +1,20 @@ +# OS TRIPLET ALLOW_FAIL ARCH_ROOTFS TIMEOUT USE_RR ROOTFS_TAG ROOTFS_HASH +# linux 686-linux-gnu . i686 . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 +linux x86_64-linux-gnu . x86_64 . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a +linux x86_64-linux-gnu . x86_64 120 rr v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a + +# linux aarch64-linux-gnu true aarch64 . . ---- ---------------------------------------- +# linux armv7l-linux-gnueabihf true armv7l . . ---- ---------------------------------------- +# linux powerpc64le-linux-gnu true powerpc64le . . ---- ---------------------------------------- + +# musl x86_64-linux-musl true x86_64 . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 + + +# These special lines allow us to embed default values for the columns above. +# Any column without a default mapping here will simply substitute a `.` to the empty string + +# Of course we do not allow jobs to fail typically +#default ALLOW_FAIL false + +# Most jobs should finish within 1.5 hours, barring exceptionally slow hardware +#default TIMEOUT 90 diff --git a/pipelines/main/platforms/test_linux.yml b/pipelines/main/platforms/test_linux.yml new file mode 100644 index 00000000..46837a35 --- /dev/null +++ b/pipelines/main/platforms/test_linux.yml @@ -0,0 +1,84 @@ +steps: + - group: ":linux: Linux" + steps: + - label: ":linux: :test_tube: test ${TRIPLET?}${USE_RR-}" + key: "test_${TRIPLET?}${USE_RR-}" + depends_on: + - "build_${TRIPLET?}" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${OS?}.${ARCH_ROOTFS?}.tar.gz + # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${OS?}.${ARCH_ROOTFS?}.tar.gz + rootfs_treehash: "${ROOTFS_HASH?}" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + env: + JULIA_SHELL: "/bin/bash" + timeout_in_minutes: ${TIMEOUT?} + soft_fail: ${ALLOW_FAIL?} + commands: | + # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc... + TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh + + echo "--- Download build artifacts" + buildkite-agent artifact download "$${UPLOAD_FILENAME}" . + + echo "--- Extract build artifacts" + tar xzf "$${UPLOAD_FILENAME}" "$${JULIA_INSTALL_DIR}/" + + echo "--- Print Julia version info" + $${JULIA_BINARY} -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' + echo "JULIA_CPU_THREADS is: $${JULIA_CPU_THREADS}" + $${JULIA_BINARY} -e '@info "" Sys.CPU_THREADS' + + echo "--- Set some environment variables" + export OPENBLAS_NUM_THREADS=8 + unset JULIA_DEPOT_PATH + unset JULIA_PKG_SERVER + + # Make sure that temp files and temp directories are created in a location that is + # backed by real storage. + export TMPDIR="$(pwd)/tmp" + mkdir -p $${TMPDIR} + + # By default, run all tests. + export TESTS="all LibGit2/online --ci" + + # If we're running inside of `rr`, limit the number of threads + if [[ "${USE_RR?}" == "rr" ]]; then + export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY}" + export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])" + export JULIA_NUM_THREADS=1 + + # Give `rr` 60 minutes to run the tests + export JULIA_TEST_RR_TIMEOUT="60" + else + export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY}" + export NCORES_FOR_TESTS="Sys.CPU_THREADS" + export JULIA_NUM_THREADS=16 + fi + + echo "--- Print the list of test sets, and other useful environment variables" + echo "JULIA_CMD_FOR_TESTS is: $${JULIA_CMD_FOR_TESTS:?}" + echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set + echo "NCORES_FOR_TESTS is: $${NCORES_FOR_TESTS:?}" + echo "OPENBLAS_NUM_THREADS is: $${OPENBLAS_NUM_THREADS:?}" + echo "TESTS is: $${TESTS:?}" + echo "USE_RR is: ${USE_RR?}" + + echo "--- Run the Julia test suite" + $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})" + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/main/platforms/tester_linux.arches b/pipelines/main/platforms/tester_linux.arches deleted file mode 100644 index d1304563..00000000 --- a/pipelines/main/platforms/tester_linux.arches +++ /dev/null @@ -1,25 +0,0 @@ -# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -linux 32_g1 g1 . 32 i686 . . . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 -linux 32_g2 g2 . 32 i686 . . . 3 . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 - -linux 64_g1_mt g1 . 64 x86_64 . . . . . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g2_mt g2 . 64 x86_64 . . . 3 . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -linux 64_g1_st g1 . 64 x86_64 . . . . . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g2_st g2 . 64 x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -linux 64_g1_rrst g1 . 64 x86_64 . 300 240 . yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g2_rrst g2 . 64 x86_64 . 180 120 3 yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g3_st g3 . 64 x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -# linux aarch64_g1 g1 true aarch64 aarch64 . . . . . . . ---- ---------------------------------------- -# linux aarch64_g2 g2 true aarch64 aarch64 . . . . . . . ---- ---------------------------------------- - -# linux armv7l_g1 g1 true armv7l armv7l . . . . . . . ---- ---------------------------------------- -# linux armv7l_g2 g2 true armv7l armv7l . . . . . . . ---- ---------------------------------------- - -# linux ppc64le_g1 g1 true ppc64le powerpc64le . . . . . . . ---- ---------------------------------------- -# linux ppc64le_g2 g2 true ppc64le powerpc64le . . . . . . . ---- ---------------------------------------- - -musl 64_g1 g1 true 64 x86_64 . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 -musl 64_g2 g2 true 64 x86_64 . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 diff --git a/pipelines/main/platforms/tester_linux.yml b/pipelines/main/platforms/tester_linux.yml deleted file mode 100644 index 18b019e6..00000000 --- a/pipelines/main/platforms/tester_linux.yml +++ /dev/null @@ -1,120 +0,0 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" -steps: - - label: "tester_${PLATFORM?}${LABEL?}" - key: tester_${PLATFORM?}${LABEL?} - depends_on: package_${PLATFORM?}${ARCH?} - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz - # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz - rootfs_treehash: "${ROOTFS_HASH?}" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - env: - JULIA_SHELL: "/bin/bash" - timeout_in_minutes: ${TIMEOUT_BK?} - retry: - automatic: - - exit_status: "*" - limit: ${RETRIES?} - soft_fail: ${ALLOW_FAIL?} - commands: | - echo "--- Print the full and short commit hashes" - SHORT_COMMIT_LENGTH=10 - SHORT_COMMIT=`echo $${BUILDKITE_COMMIT:?} | cut -c1-$${SHORT_COMMIT_LENGTH:?}` - JULIA_DIR="julia-$${SHORT_COMMIT:?}" - JULIA_BINARY="$${JULIA_DIR:?}/bin/julia" - ARTIFACT_FILE_EXTENSION="tar.gz" - ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-${PLATFORM?}${ARCH?}.$${ARTIFACT_FILE_EXTENSION:?}" - echo "The full commit is: $${BUILDKITE_COMMIT:?}" - echo "The short commit is: $${SHORT_COMMIT:?}" - echo "The artifact filename will be: $${ARTIFACT_FILENAME:?}" - echo "The Julia directory name will be: $${JULIA_DIR:?}" - echo "The Julia binary will be: $${JULIA_BINARY:?}" - - echo "--- Download build artifacts" - rm -rf $${ARTIFACT_FILENAME:?} - buildkite-agent artifact download $${ARTIFACT_FILENAME:?} . - - echo "--- Extract build artifacts" - rm -rf $${JULIA_DIR:?}/ - tar xzf $${ARTIFACT_FILENAME:?} $${JULIA_DIR:?}/ - - echo "--- Print Julia version info" - $${JULIA_BINARY:?} -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - echo "JULIA_CPU_THREADS is: $${JULIA_CPU_THREADS:?}" - $${JULIA_BINARY:?} -e '@info "" Sys.CPU_THREADS' - - echo "--- Set some environment variables" - export OPENBLAS_NUM_THREADS=8 - unset JULIA_DEPOT_PATH - unset JULIA_PKG_SERVER - - # Make sure that temp files and temp directories are created in a location that is - # backed by real storage. - export TMPDIR="$(pwd)/tmp" - mkdir -p $${TMPDIR:?} - - export NETWORK_RELATED_TESTS="Artifacts Downloads download LazyArtifacts LibGit2/online Pkg" - - if [[ "${GROUP?}" == "all" ]]; then - export TESTS="all LibGit2/online --force-net" - elif [[ "${GROUP?}" == "all_except_pkg" ]]; then - export TESTS="all LibGit2/online --force-net --skip Pkg" - elif [[ "${GROUP?}" == "g1" ]]; then - # Group 1: ALL tests EXCEPT the network-related tests. - export TESTS="all --force-net --skip $${NETWORK_RELATED_TESTS:?}" - elif [[ "${GROUP?}" == "g2" ]]; then - # Group 2: ONLY the network-related tests. - # In Group 2, we use whatever the default setting is with regards to the Pkg server. - export TESTS="$${NETWORK_RELATED_TESTS:?} --force-net" - elif [[ "${GROUP?}" == "g3" ]]; then - # Group 3: only Pkg. - # In Group 3, we explicitly opt-out of the Pkg server. - # The purpose of group 3 is to test the non-Pkg-server codepaths of Pkg. - export TESTS="Pkg --force-net" - export JULIA_PKG_SERVER="" - else - echo "Invalid value for GROUP: ${GROUP?}" - exit 1 - fi - - export JULIA_TEST_RR_TIMEOUT="${TIMEOUT_RR?}" - - if [[ "${IS_RR?}" == "yes" ]]; then - export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}" - export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])" - else - export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?}" - export NCORES_FOR_TESTS="Sys.CPU_THREADS" - fi - - if [[ "${IS_ST?}" == "yes" ]]; then - export JULIA_NUM_THREADS=1 - fi - - if [[ "${IS_MT?}" == "yes" ]]; then - export JULIA_NUM_THREADS=16 - fi - - echo "--- Print the test group, list of test sets, and other useful environment variables" - echo "JULIA_CMD_FOR_TESTS is: $${JULIA_CMD_FOR_TESTS:?}" - echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set - echo "NCORES_FOR_TESTS is: $${NCORES_FOR_TESTS:?}" - echo "OPENBLAS_NUM_THREADS is: $${OPENBLAS_NUM_THREADS:?}" - echo "GROUP is: ${GROUP?}" - echo "TESTS is: $${TESTS:?}" - - echo "--- Run the Julia test suite" - $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})" diff --git a/pipelines/main/platforms/upload_linux.yml b/pipelines/main/platforms/upload_linux.yml new file mode 100644 index 00000000..baa18d71 --- /dev/null +++ b/pipelines/main/platforms/upload_linux.yml @@ -0,0 +1,55 @@ +steps: + - label: ":linux: upload ${TRIPLET?}" + key: "upload_${TRIPLET?}" + depends_on: + # Wait for the builder to finish + - "build_${TRIPLET?}" + # Wait for the tester to finish + - "test_${TRIPLET?}" + plugins: + - JuliaCI/external-buildkite#v1: + version: ".buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.1/aws_uploader.x86_64.tar.gz + rootfs_treehash: "f865a3a68e33de10b3b4c9c5d0c9e5f2ff5a6965" + uid: 1000 + gid: 1000 + - staticfloat/cryptic#v2: + variables: + - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" + - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" + timeout_in_minutes: ${TIMEOUT?} + commands: | + # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc... + TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh + + echo "--- Download $${UPLOAD_FILENAME} to ." + buildkite-agent artifact download "$${UPLOAD_FILENAME}" . + + # We first upload the canonical fully-specified upload target, which is the first one: + echo "--- Upload to S3" + aws s3 cp --acl public-read "$${UPLOAD_FILENAME}" "s3://$${UPLOAD_TARGETS[0]}" + + echo "--- Copy to secondary upload targets" + for SECONDARY_TARGET in $${UPLOAD_TARGETS[@]:1}; do + aws s3 cp --acl public-read "s3://$${UPLOAD_TARGETS[0]}" "s3://$${SECONDARY_TARGET}" + done + + # Report to the user some URLs that they can use to download this from + echo "+++ Uploaded to targets" + for UPLOAD_TARGET in $${UPLOAD_TARGETS[@]}; do + echo " -> s3://$${UPLOAD_TARGET}" + done + agents: + queue: "julia" + # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox_capable: "true" + os: "linux" + env: + # Receive cryptic token from parent job + BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/pipelines/scheduled/0_webui.yml b/pipelines/scheduled/0_webui.yml index 78031b49..c90cf937 100644 --- a/pipelines/scheduled/0_webui.yml +++ b/pipelines/scheduled/0_webui.yml @@ -1,9 +1,6 @@ # This file represents what is put into the webUI. # It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. # We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. -agents: - queue: "julia" - sandbox.jl: "true" steps: - label: ":unlock: Unlock secrets, launch pipelines" plugins: @@ -25,3 +22,6 @@ steps: - .buildkite/pipelines/scheduled/coverage/coverage_linux64.yml - .buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl - .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + agents: + queue: "julia" + sandbox_capable: "true" diff --git a/pipelines/scheduled/coverage/coverage_linux64.yml b/pipelines/scheduled/coverage/coverage_linux64.yml index 1ff88577..4d011cec 100644 --- a/pipelines/scheduled/coverage/coverage_linux64.yml +++ b/pipelines/scheduled/coverage/coverage_linux64.yml @@ -1,14 +1,9 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" steps: - label: ":unlock: :coverage: Run coverage test" # We must accept the signed job id secret in order to propagate secrets env: BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - depends_on: + depends_on: ~ plugins: - staticfloat/cryptic: variables: @@ -42,3 +37,7 @@ steps: echo "--- Process and upload coverage information" ./julia .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + agents: + queue: "julia" + sandbox_capable: "true" + os: "linux" diff --git a/pipelines/scheduled/coverage/coverage_linux64.yml.signature b/pipelines/scheduled/coverage/coverage_linux64.yml.signature index b28f81eb..84d07008 100644 Binary files a/pipelines/scheduled/coverage/coverage_linux64.yml.signature and b/pipelines/scheduled/coverage/coverage_linux64.yml.signature differ diff --git a/pipelines/scheduled/coverage/upload_coverage.jl b/pipelines/scheduled/coverage/upload_coverage.jl index d995e97f..7253ba6d 100644 --- a/pipelines/scheduled/coverage/upload_coverage.jl +++ b/pipelines/scheduled/coverage/upload_coverage.jl @@ -112,18 +112,9 @@ end function buildkite_branch_and_commit() branch = buildkite_env("BUILDKITE_BRANCH") - commit = buildkite_env("BUILDKITE_COMMIT") - head_rev_parse = String(strip(read(`git rev-parse HEAD`, String))) - if strip(commit) == "HEAD" - commit = head_rev_parse - end - if commit !== head_rev_parse - msg = "mismatch" - @error msg commit head_rev_parse - throw(ErrorException(msg)) - end + commit = String(strip(read(`git rev-parse HEAD`, String))) if !occursin(r"^[a-f0-9]{40}$", commit) - msg = "BUILDKITE_COMMIT does not look like a long commit SHA" + msg = "'$(commit)' does not look like a long commit SHA" @error msg commit throw(ErrorException(msg)) end diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_unsigned_jobs.yml index 300c8d84..2ebf1bbd 100644 --- a/pipelines/scheduled/launch_unsigned_jobs.yml +++ b/pipelines/scheduled/launch_unsigned_jobs.yml @@ -2,7 +2,7 @@ steps: - label: ":buildkite: Launch unsigned jobs" commands: | # Launch all of the `USE_BINARYBUILDER=0` jobs. - bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml - bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches .buildkite/pipelines/main/platforms/tester_linux.yml + bash .buildkite/utilities/arches_pipeline_upload.sh .buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml + bash .buildkite/utilities/arches_pipeline_upload.sh .buildkite/pipelines/scheduled/no_bb/no_bb_test_linux.arches .buildkite/pipelines/main/platforms/test_linux.yml agents: queue: julia diff --git a/pipelines/scheduled/no_bb/no_bb_package_linux.arches b/pipelines/scheduled/no_bb/no_bb_package_linux.arches index dff2aab4..734ce3a3 100644 --- a/pipelines/scheduled/no_bb/no_bb_package_linux.arches +++ b/pipelines/scheduled/no_bb/no_bb_package_linux.arches @@ -1,2 +1,2 @@ -# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -linux 64src . . 64src x86_64 USE_BINARYBUILDER=0 180 . . . . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a +# OS ALLOW_FAIL TAR_ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT ROOTFS_TAG ROOTFS_HASH +linux . 64src x86_64 USE_BINARYBUILDER=0 180 v4.8 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0 diff --git a/pipelines/scheduled/no_bb/no_bb_tester_linux.arches b/pipelines/scheduled/no_bb/no_bb_tester_linux.arches index 0b1fbdf6..dfd808f8 100644 --- a/pipelines/scheduled/no_bb/no_bb_tester_linux.arches +++ b/pipelines/scheduled/no_bb/no_bb_tester_linux.arches @@ -1,10 +1,2 @@ -# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -linux 64src_g1_mt g1 . 64src x86_64 . . . . . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64src_g2_mt g2 . 64src x86_64 . . . 3 . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -linux 64src_g1_st g1 . 64src x86_64 . . . . . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64src_g2_st g2 . 64src x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -linux 64src_g1_rrst g1 . 64src x86_64 . 300 240 . yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64src_g2_rrst g2 . 64src x86_64 . 180 120 3 yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64src_g3_st g3 . 64src x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a +# OS LABEL ALLOW_FAIL TAR_ARCH ARCH_ROOTFS TIMEOUT_BK TIMEOUT_RR IS_RR ROOTFS_TAG ROOTFS_HASH +linux 64src . 64src x86_64 . . . v4.8 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0 diff --git a/utilities/arches_env.sh b/utilities/arches_env.sh new file mode 100755 index 00000000..bbdd4ca2 --- /dev/null +++ b/utilities/arches_env.sh @@ -0,0 +1,80 @@ +#!/bin/bash +set -eou pipefail + +# This script reads in an `.arches` file, processes the columns and default value mappings +# within it, and outputs an environment block (e.g. a line of the form "X=a B= C=123") for +# each architecture defined within, to be used by other tools such as the brother script +# `arches_pipeline_upload.sh`, which uses those environment mappings to template pipeline +# YAML files that are being uploaded by `buildkite-agent pipeline upload`. + +ARCHES_FILE="${1:-}" +if [[ ! -f "${ARCHES_FILE}" ]] ; then + echo "Arches file does not exist: '${ARCHES_FILE}'" + exit 1 +fi + +enforce_sanitized() { + for value in "$@"; do + if ! [[ "${value}" =~ ^[a-zA-Z0-9_\.[:space:]-]*$ ]]; then + echo "Arches file '${ARCHES_FILE}' contains value '${value}' with non-alphanumeric characters; refusing to parse!" >&2 + exit 1 + fi + done +} + + +# Determine variable names from the header of the .arches file: +readarray -d ' ' -s 1 -t var_names < <(head -1 "${ARCHES_FILE}" | tr -s ' ') + +# Determine any embedded defaults +declare -A defaults_map +readarray -t default_mapping_lines < <(grep "^#default" "${ARCHES_FILE}" | tr -s ' ') +for idx in "${!default_mapping_lines[@]}"; do + key="$( cut -d' ' -f2 <<<"${default_mapping_lines[${idx}]}" | xargs)" + value="$(cut -d' ' -f3 <<<"${default_mapping_lines[${idx}]}" | xargs)" + enforce_sanitized "${key}" "${value}" + defaults_map["${key}"]="${value}" +done + +cat "${ARCHES_FILE}" | while read line; do + # Remove whitespace from the beginning and end of each line + line="$(tr -s ' ' <<<"${line}")" + + # Skip any line that begins with the `#` character + if [[ $line == \#* ]]; then + continue + fi + + # Skip any empty line + if [[ $line == "" ]]; then + continue + fi + + # Skip any line that contains suspicious characters, to prevent shell escaping bugs + enforce_sanitized "${line}" + + # Convert line to array + readarray -d ' ' -t line_array <<<"${line}" + + # Panic if we don't have the same number of items as our column names: + if [[ "${#line_array[@]}" != "${#var_names[@]}" ]]; then + echo "ERROR: The following line does not contain ${#var_names[@]} columns as we would expect from the header of ${ARCHES_FILE}" >&2 + echo "${line}" + exit 1 + fi + + # Loop over columns, bind values to their column name + for idx in "${!var_names[@]}"; do + # Get the name and value + name="$(xargs <<<"${var_names[${idx}]}")" + value="$(xargs <<<"${line_array[${idx}]}")" + + # Apply default values to our special `.` token + if [[ "${value}" == "." ]]; then + value="${defaults_map[${name}]:-}" + fi + + echo -n "${name}=\"${value}\" " + done + echo +done diff --git a/utilities/arches_pipeline_upload.sh b/utilities/arches_pipeline_upload.sh new file mode 100644 index 00000000..5ac88acd --- /dev/null +++ b/utilities/arches_pipeline_upload.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -eou pipefail +shopt -s nullglob + +# This script reads in an `.arches` file, processes the columns and default value mappings +# within it, and outputs an environment block (e.g. a line of the form "X=a B= C=123") for +# each architecture defined within, to be used by other tools such as the brother script +# `arches_pipeline_upload.sh`, which uses those environment mappings to template pipeline +# YAML files that are being uploaded by `buildkite-agent pipeline upload`. + +ARCHES_FILE="${1:-}" +if [[ ! -f "${ARCHES_FILE}" ]] ; then + echo "Arches file does not exist: '${ARCHES_FILE}'" + exit 1 +fi + +YAML_FILE="${2:-}" +if [[ ! -f "${YAML_FILE}" ]] ; then + echo "YAML file does not exist: '${YAML_FILE}'" + exit 1 +fi + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +"${BASH}" "${SCRIPT_DIR}/arches_env.sh" "${ARCHES_FILE}" | while read env_map; do + # Export the environment mappings, then launch the yaml file + eval "export ${env_map}" + buildkite-agent pipeline upload "${YAML_FILE}" +done diff --git a/utilities/calc_version_envs.sh b/utilities/calc_version_envs.sh new file mode 100755 index 00000000..60c54b37 --- /dev/null +++ b/utilities/calc_version_envs.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Extract information from our triplet +export ARCH="$(cut -d- -f1 <<<"${TRIPLET}")" +case "${TRIPLET}" in + *-apple-*) + OS="macos" + ;; + *-freebsd*) + OS="freebsd" + ;; + *-mingw*) + OS="windows" + ;; + *-gnu*) + OS="linux" + ;; + *-musl*) + OS="musl" + ;; + *) + echo "Unknown triplet OS '${TRIPLET}'" >&2 + exit 1 + ;; +esac +export OS + +# Extract git information +SHORT_COMMIT_LENGTH=10 +LONG_COMMIT="$(git rev-parse HEAD)" +SHORT_COMMIT="$(echo ${LONG_COMMIT} | cut -c1-${SHORT_COMMIT_LENGTH})" + +# Extract information about the current julia version number +JULIA_VERSION="$(cat VERSION)" +MAJMIN="${JULIA_VERSION:0:3}" + +# If we're on a tag, then our "tar version" will be the julia version. +# Otherwise, it's the short commit. +if git describe --tags --exact-match >/dev/null 2>/dev/null; then + TAR_VERSION="${JULIA_VERSION}" +else + TAR_VERSION="${SHORT_COMMIT}" +fi + +# Build the filename that we'll upload as, and get the filename that will be built +# These are not the same in situations such as `musl`, where the build system doesn't +# differentiate but we need to give it a different name. +JULIA_BINARYDIST_FILENAME="$(make print-JULIA_BINARYDIST_FILENAME | cut -c27- | tr -s ' ').tar.gz" + +JULIA_INSTALL_DIR="julia-${TAR_VERSION}" +JULIA_BINARY="${JULIA_INSTALL_DIR}/bin/julia" + +# We generally upload to multiple upload targets +UPLOAD_TARGETS=( + # First, we have the canonical fully-specified upload target + "julialangnightlies/bin/${OS?}/${ARCH?}/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}.tar.gz" + + # Next, we have the "latest" upload target + "julialangnightlies/bin/${OS?}/${ARCH?}/${MAJMIN?}/julia-latest-${OS?}-${ARCH?}.tar.gz" +) +UPLOAD_FILENAME="julia-${TAR_VERSION?}-${OS?}-${ARCH?}.tar.gz" + +# Finally, for compatibility, we keep on uploading x86_64 and i686 targets to folders called `x64` +# and `x86`, respectively, although I would very much like to stop doing that. +if [[ "${ARCH}" == "x86_64" ]]; then + UPLOAD_TARGETS+=( "julialangnightlies/bin/${OS?}/x64/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}.tar.gz" ) + UPLOAD_TARGETS+=( "julialangnightlies/bin/${OS?}/x64/${MAJMIN?}/julia-latest-${OS?}-${ARCH?}.tar.gz" ) +elif [[ "${ARCH}" == "i686" ]]; then + UPLOAD_TARGETS+=( "julialangnightlies/bin/${OS?}/x86/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}.tar.gz" ) + UPLOAD_TARGETS+=( "julialangnightlies/bin/${OS?}/x86/${MAJMIN?}/julia-latest-${OS?}-${ARCH?}.tar.gz" ) +fi + +echo "--- Print the full and short commit hashes" +echo "The full commit is: ${LONG_COMMIT}" +echo "The short commit is: ${SHORT_COMMIT}" +echo "Julia will be installed to: ${JULIA_BINARY}" +echo "Detected Julia version: ${MAJMIN} (${JULIA_VERSION})" +echo "Detected build platform: ${TRIPLET} (${ARCH}, ${OS})" +echo "Julia will be uploaded to: s3://${UPLOAD_TARGETS[0]}" +echo "With additional upload targets:" +for UPLOAD_TARGET in ${UPLOAD_TARGETS[@]:1}; do + echo " -> s3://${UPLOAD_TARGET}" +done diff --git a/utilities/platforms/platforms.sh b/utilities/platforms/platforms.sh deleted file mode 100755 index 9a47c18e..00000000 --- a/utilities/platforms/platforms.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -ARCHES="$1" -YAML="$2" - -if [[ ! -f "${ARCHES:?}" ]] ; then - echo "Arches file does not exist: ${ARCHES:?}" - exit 1 -fi - -if [[ ! -f "${YAML:?}" ]] ; then - echo "YAML file does not exist: ${YAML:?}" - exit 1 -fi - -cat "${ARCHES:?}" | tr -s ' ' | while read _line; do - # Remove whitespace from the beginning and end of each line - line=`echo $_line | tr -s ' '` - - # Skip any line that begins with the `#` character - if [[ $line == \#* ]]; then - continue - fi - - # Skip any empty line - if [[ $line == "" ]]; then - continue - fi - - export PLATFORM=`echo $line | cut -d ' ' -f 1 | tr -s ' '` - export LABEL=`echo $line | cut -d ' ' -f 2 | tr -s ' '` - export GROUP=`echo $line | cut -d ' ' -f 3 | tr -s ' '` - - export ALLOW_FAIL=`echo $line | cut -d ' ' -f 4 | tr -s ' '` - export ARCH=`echo $line | cut -d ' ' -f 5 | tr -s ' '` - export ARCH_ROOTFS=`echo $line | cut -d ' ' -f 6 | tr -s ' '` - - export MAKE_FLAGS=`echo $line | cut -d ' ' -f 7 | tr -s ' '` - export TIMEOUT_BK=`echo $line | cut -d ' ' -f 8 | tr -s ' '` - export TIMEOUT_RR=`echo $line | cut -d ' ' -f 9 | tr -s ' '` - export RETRIES=`echo $line | cut -d ' ' -f 10 | tr -s ' '` - export IS_RR=`echo $line | cut -d ' ' -f 11 | tr -s ' '` - export IS_ST=`echo $line | cut -d ' ' -f 12 | tr -s ' '` - export IS_MT=`echo $line | cut -d ' ' -f 13 | tr -s ' '` - export ROOTFS_TAG=`echo $line | cut -d ' ' -f 14 | tr -s ' '` - export ROOTFS_HASH=`echo $line | cut -d ' ' -f 15 | tr -s ' '` - - if [[ "${IS_ST:?}" == "yes" ]]; then - if [[ "${IS_MT:?}" == "yes" ]]; then - echo "You cannot set both IS_ST and IS_MT to yes" - exit 1 - fi - fi - - if [[ "${ALLOW_FAIL:?}" == "." ]]; then - export ALLOW_FAIL="false" - fi - - if [[ "${MAKE_FLAGS:?}" == "." ]]; then - export MAKE_FLAGS="" - fi - - if [[ "${TIMEOUT_BK:?}" == "." ]]; then - export TIMEOUT_BK="90" # minutes - fi - - if [[ "${TIMEOUT_RR:?}" == "." ]]; then - export TIMEOUT_RR="60" # minutes - fi - - if [[ "${RETRIES:?}" == "." ]]; then - export RETRIES="0" - fi - - buildkite-agent pipeline upload "${YAML:?}" -done