From 5e7f8336c21dbbf6536dae6ebe1fca42464ea2eb Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 2 Oct 2022 16:30:24 -0400 Subject: [PATCH] [`release-1.6` branch] CI (Buildkite): remove the `.buildkite` folder (#44380) --- .buildkite-external-version | 1 + .buildkite/README.md | 7 - .buildkite/cryptic_repo_keys/.gitignore | 5 - .buildkite/cryptic_repo_keys/README.md | 6 - .../cryptic_repo_keys/repo_key.2297e5e7 | Bin 256 -> 0 bytes .buildkite/pipelines/experimental/0_webui.yml | 22 -- .buildkite/pipelines/experimental/README.md | 7 - .../experimental/launch_unsigned_builders.yml | 6 - .buildkite/pipelines/main/0_webui.yml | 23 -- .buildkite/pipelines/main/README.md | 15 -- .../main/launch_unsigned_builders.yml | 33 --- .buildkite/pipelines/main/misc/doctest.yml | 35 --- .buildkite/pipelines/main/misc/embedding.yml | 32 --- .buildkite/pipelines/main/misc/llvmpasses.yml | 49 ---- .buildkite/pipelines/main/misc/sanitizers.yml | 48 ---- .../main/misc/signed_pipeline_test.yml | 18 -- .../misc/signed_pipeline_test.yml.signature | 1 - .buildkite/pipelines/main/misc/whitespace.yml | 25 -- .../main/platforms/package_linux.arches | 7 - .../main/platforms/package_linux.yml | 55 ----- .../main/platforms/tester_linux.arches | 25 -- .../pipelines/main/platforms/tester_linux.yml | 121 ---------- .buildkite/pipelines/scheduled/0_webui.yml | 27 --- .buildkite/pipelines/scheduled/README.md | 5 - .../scheduled/coverage/coverage_linux64.yml | 44 ---- .../coverage/coverage_linux64.yml.signature | 1 - .../scheduled/coverage/run_tests_parallel.jl | 29 --- .../scheduled/coverage/upload_coverage.jl | 228 ------------------ .../scheduled/launch_unsigned_jobs.yml | 8 - .../no_bb/no_bb_package_linux.arches | 2 - .../scheduled/no_bb/no_bb_tester_linux.arches | 10 - .buildkite/secrets/.gitignore | 11 - .buildkite/secrets/ssh_docs_deploy.encrypted | Bin 1856 -> 0 bytes .buildkite/secrets/ssh_docs_deploy.pub | 1 - .buildkite/utilities/platforms/platforms.sh | 76 ------ .buildkite/utilities/rr/rr_capture.jl | 200 --------------- 36 files changed, 1 insertion(+), 1182 deletions(-) create mode 100644 .buildkite-external-version delete mode 100644 .buildkite/README.md delete mode 100644 .buildkite/cryptic_repo_keys/.gitignore delete mode 100644 .buildkite/cryptic_repo_keys/README.md delete mode 100644 .buildkite/cryptic_repo_keys/repo_key.2297e5e7 delete mode 100644 .buildkite/pipelines/experimental/0_webui.yml delete mode 100644 .buildkite/pipelines/experimental/README.md delete mode 100644 .buildkite/pipelines/experimental/launch_unsigned_builders.yml delete mode 100644 .buildkite/pipelines/main/0_webui.yml delete mode 100644 .buildkite/pipelines/main/README.md delete mode 100644 .buildkite/pipelines/main/launch_unsigned_builders.yml delete mode 100644 .buildkite/pipelines/main/misc/doctest.yml delete mode 100644 .buildkite/pipelines/main/misc/embedding.yml delete mode 100644 .buildkite/pipelines/main/misc/llvmpasses.yml delete mode 100644 .buildkite/pipelines/main/misc/sanitizers.yml delete mode 100644 .buildkite/pipelines/main/misc/signed_pipeline_test.yml delete mode 100644 .buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature delete mode 100644 .buildkite/pipelines/main/misc/whitespace.yml delete mode 100644 .buildkite/pipelines/main/platforms/package_linux.arches delete mode 100644 .buildkite/pipelines/main/platforms/package_linux.yml delete mode 100644 .buildkite/pipelines/main/platforms/tester_linux.arches delete mode 100644 .buildkite/pipelines/main/platforms/tester_linux.yml delete mode 100644 .buildkite/pipelines/scheduled/0_webui.yml delete mode 100644 .buildkite/pipelines/scheduled/README.md delete mode 100644 .buildkite/pipelines/scheduled/coverage/coverage_linux64.yml delete mode 100644 .buildkite/pipelines/scheduled/coverage/coverage_linux64.yml.signature delete mode 100644 .buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl delete mode 100644 .buildkite/pipelines/scheduled/coverage/upload_coverage.jl delete mode 100644 .buildkite/pipelines/scheduled/launch_unsigned_jobs.yml delete mode 100644 .buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches delete mode 100644 .buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches delete mode 100644 .buildkite/secrets/.gitignore delete mode 100644 .buildkite/secrets/ssh_docs_deploy.encrypted delete mode 100644 .buildkite/secrets/ssh_docs_deploy.pub delete mode 100755 .buildkite/utilities/platforms/platforms.sh delete mode 100644 .buildkite/utilities/rr/rr_capture.jl diff --git a/.buildkite-external-version b/.buildkite-external-version new file mode 100644 index 00000000000000..ba2906d0666cf7 --- /dev/null +++ b/.buildkite-external-version @@ -0,0 +1 @@ +main diff --git a/.buildkite/README.md b/.buildkite/README.md deleted file mode 100644 index b3f74f2b231378..00000000000000 --- a/.buildkite/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Buildkite - -This directory contains the Buildkite configuration files for Base Julia CI. - -The rootfs image definitions are located in the [rootfs-images](https://github.com/JuliaCI/rootfs-images) repository. - -The documentation for the Base Julia CI setup is located in the [base-buildkite-docs](https://github.com/JuliaCI/base-buildkite-docs) repository. diff --git a/.buildkite/cryptic_repo_keys/.gitignore b/.buildkite/cryptic_repo_keys/.gitignore deleted file mode 100644 index 8d18931dbcf7c5..00000000000000 --- a/.buildkite/cryptic_repo_keys/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Ignore the unencrypted repo_key -repo_key - -# Ignore any agent keys (public or private) we have stored -agent_key* diff --git a/.buildkite/cryptic_repo_keys/README.md b/.buildkite/cryptic_repo_keys/README.md deleted file mode 100644 index 93ed17ce4757bb..00000000000000 --- a/.buildkite/cryptic_repo_keys/README.md +++ /dev/null @@ -1,6 +0,0 @@ -## Cryptic repository keys - -This folder contains RSA-encrypted symmetric AES keys. -These are used by buildkite agents to decrypt the secrets embedded within this repository. -Each buildkite agent contains an RSA secret key that is used to unlock the symmetric AES key that was used to encrypt the secrets within this repository. -For more information, see the [`cryptic` buildkite plugin repository](https://github.com/staticfloat/cryptic-buildkite-plugin). diff --git a/.buildkite/cryptic_repo_keys/repo_key.2297e5e7 b/.buildkite/cryptic_repo_keys/repo_key.2297e5e7 deleted file mode 100644 index 2ab9198b4ce2d7c7a9327935e18aaef5943629d2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 256 zcmV+b0ssE4{g>pIb#Eq&=vC$Qr^8S_O6!cOYE`=}q%}#an@eaJd$RWGY{B`_JI5h4 zkH`Ok?yS%>4AD6gqiiL&E|`(uBGp1*D8z1}bTOSJGyXKmUCrMUN%Q;gJ{M_gVt6DF z#aRD2;iY!|^9jgEcBXhT=jxw2yplIkGkb~?zh)kpuL|wT!R7{E{^jQhsqTt2Woi5X z6&zlfSSPoyKVU`TIJAL8I6(F*9}>v<(M$c2U_*AHb;CSzxbLIaZK0;0gYIZ?hzwB# zq^tZ&SA{ud8h!8dRd!Rzp7pWdE3F(RM#CkMHZoXOfk*SkHmzrDCiOUb@l`xQjdi^f GB)!2VDuP}B diff --git a/.buildkite/pipelines/experimental/0_webui.yml b/.buildkite/pipelines/experimental/0_webui.yml deleted file mode 100644 index eee9743d39f6aa..00000000000000 --- a/.buildkite/pipelines/experimental/0_webui.yml +++ /dev/null @@ -1,22 +0,0 @@ -# This file represents what is put into the webUI. -# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. -# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. -agents: - queue: "julia" - sandbox.jl: "true" -steps: - - label: ":unlock: Unlock secrets, launch pipelines" - plugins: - - staticfloat/cryptic: - # Our list of pipelines that should be launched (but don't require a signature) - # These pipelines can be modified by any contributor and CI will still run. - # Build secrets will not be available in these pipelines (or their children) - # but some of our signed pipelines can wait upon the completion of these unsigned - # pipelines. - unsigned_pipelines: - - .buildkite/pipelines/experimental/launch_unsigned_builders.yml - # Our signed pipelines must have a `signature` or `signature_file` parameter that - # verifies the treehash of the pipeline itself and the inputs listed in `inputs` - # signed_pipelines: - # - pipeline: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml - # signature_file: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml.signature diff --git a/.buildkite/pipelines/experimental/README.md b/.buildkite/pipelines/experimental/README.md deleted file mode 100644 index f92aac7a1af02f..00000000000000 --- a/.buildkite/pipelines/experimental/README.md +++ /dev/null @@ -1,7 +0,0 @@ -## Experimental pipeline (`master` branch only) - -This is the [`julia-master->experimental`](https://buildkite.com/julialang/julia-master-experimental) pipeline. - -We use this pipeline for builders that are not yet stable enough to go into the main pipeline. - -These builders are triggered by GitHub webhook events, such as pushes and pull requests. diff --git a/.buildkite/pipelines/experimental/launch_unsigned_builders.yml b/.buildkite/pipelines/experimental/launch_unsigned_builders.yml deleted file mode 100644 index 04d82a6e39a5e8..00000000000000 --- a/.buildkite/pipelines/experimental/launch_unsigned_builders.yml +++ /dev/null @@ -1,6 +0,0 @@ -steps: - - label: ":buildkite: Launch unsigned pipelines" - commands: | - true - agents: - queue: julia diff --git a/.buildkite/pipelines/main/0_webui.yml b/.buildkite/pipelines/main/0_webui.yml deleted file mode 100644 index af68158f9a51f0..00000000000000 --- a/.buildkite/pipelines/main/0_webui.yml +++ /dev/null @@ -1,23 +0,0 @@ -# This file represents what is put into the webUI. -# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. -# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. -agents: - queue: "julia" - sandbox.jl: "true" -steps: - - label: ":unlock: Unlock secrets, launch pipelines" - plugins: - - staticfloat/cryptic: - # Our list of pipelines that should be launched (but don't require a signature) - # These pipelines can be modified by any contributor and CI will still run. - # Build secrets will not be available in these pipelines (or their children) - # but some of our signed pipelines can wait upon the completion of these unsigned - # pipelines. - unsigned_pipelines: - - .buildkite/pipelines/main/launch_unsigned_builders.yml - - # Our signed pipelines must have a `signature` or `signature_file` parameter that - # verifies the treehash of the pipeline itself and the inputs listed in `inputs` - signed_pipelines: - - pipeline: .buildkite/pipelines/main/misc/signed_pipeline_test.yml - signature_file: .buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature diff --git a/.buildkite/pipelines/main/README.md b/.buildkite/pipelines/main/README.md deleted file mode 100644 index 6b9d67bd7cc3ad..00000000000000 --- a/.buildkite/pipelines/main/README.md +++ /dev/null @@ -1,15 +0,0 @@ -## Main pipeline - -This is the main pipeline. It contains most of the builders. These builders are triggered by GitHub webhook events, such as pushes and pull requests. - -We have a different main pipeline for each permanent branch. - -For example: - -| Permanent Branch | Pipeline | -| ---------------- | -------------------------------------------------------------------------------- | -| `master` | [`julia-master`](https://buildkite.com/julialang/julia-master) | -| `release-1.6` | [`julia-release-1.6`](https://buildkite.com/julialang/julia-release-1-dot-6) | -| `release-1.7` | [`julia-release-1.7`](https://buildkite.com/julialang/julia-release-1-dot-7) | - -(This is not a complete list.) diff --git a/.buildkite/pipelines/main/launch_unsigned_builders.yml b/.buildkite/pipelines/main/launch_unsigned_builders.yml deleted file mode 100644 index 2b6794ed13bd14..00000000000000 --- a/.buildkite/pipelines/main/launch_unsigned_builders.yml +++ /dev/null @@ -1,33 +0,0 @@ -# This file launches all the build jobs that _don't_ require secrets access. -# These jobs can pass their output off to jobs that do require secrets access, -# but those privileged steps require signing before they can be run. -# -# Yes, this is creating another layer of indirection; the flow now looks like: -# -# [webui] -> launch_unsigned_builders.yml -> misc/whitespace.yml -# -# when we could theoretically just have the `webui` launch `misc/whitespace.yml`, -# however this raises the bar for contributors to add new (unsigned) steps to -# our CI configuration, so I'd rather live with an extra layer of indirection -# and only need to touch the webui configuration when we need to alter -# something about the privileged steps. - -steps: - - label: ":buildkite: Launch unsigned jobs" - commands: | - # Launch the miscellaneous jobs in alphabetical order. - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml - - # Launch all of the platform jobs. - bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml - bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/tester_linux.arches .buildkite/pipelines/main/platforms/tester_linux.yml - - # Launch the `whitespace` job last. Uploading it last actually causes it to start - # first. We want this job to start first because we want it to finish as quickly - # as possible. - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml - agents: - queue: julia diff --git a/.buildkite/pipelines/main/misc/doctest.yml b/.buildkite/pipelines/main/misc/doctest.yml deleted file mode 100644 index 33e42debb478d6..00000000000000 --- a/.buildkite/pipelines/main/misc/doctest.yml +++ /dev/null @@ -1,35 +0,0 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" - arch: "x86_64" -steps: - - label: "doctest" - key: doctest - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - commands: | - echo "--- Build Julia from source" - make --output-sync -j 6 - - echo "--- Print Julia version info" - ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - - echo "--- Build Julia docs" - make docs - - echo "--- Run Julia doctests" - JULIA_NUM_THREADS=1 make -C doc doctest=true - timeout_in_minutes: 45 diff --git a/.buildkite/pipelines/main/misc/embedding.yml b/.buildkite/pipelines/main/misc/embedding.yml deleted file mode 100644 index ba273e239d69f9..00000000000000 --- a/.buildkite/pipelines/main/misc/embedding.yml +++ /dev/null @@ -1,32 +0,0 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" - arch: "x86_64" -steps: - - label: "embedding" - key: "embedding" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - commands: | - prefix="/tmp/prefix" - echo "+++ Build julia, deploy to $${prefix:?}" - make --output-sync -j$${JULIA_CPU_THREADS:?} JULIA_PRECOMPILE=0 prefix=$${prefix:?} install - - embedding_output="/tmp/embedding-test" - echo "+++ Run embedding tests, deploy to $${embedding_output:?}" - mkdir -p "$${embedding_output:?}" - make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/embedding JULIA="$${prefix:?}/bin/julia" BIN="$${embedding_output:?}" - timeout_in_minutes: 60 diff --git a/.buildkite/pipelines/main/misc/llvmpasses.yml b/.buildkite/pipelines/main/misc/llvmpasses.yml deleted file mode 100644 index 25bcad1247db9e..00000000000000 --- a/.buildkite/pipelines/main/misc/llvmpasses.yml +++ /dev/null @@ -1,49 +0,0 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" - arch: "x86_64" -steps: - - label: "analyzegc" - key: "analyzegc" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz - rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd" - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - commands: | - echo "--- Install in-tree LLVM dependencies" - make --output-sync -j$${JULIA_CPU_THREADS:?} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind - echo "+++ run clangsa/analyzegc" - make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/clangsa - make --output-sync -j$${JULIA_CPU_THREADS:?} -C src analyzegc - timeout_in_minutes: 60 - - label: "llvmpasses" - key: "llvmpasses" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" - uid: 1000 - gid: 1000 - workspaces: - - "/cache/repos:/cache/repos" - commands: | - echo "--- make release" - make --output-sync -j$${JULIA_CPU_THREADS:?} release JULIA_PRECOMPILE=0 - echo "--- make src/install-analysis-deps" - make --output-sync -j$${JULIA_CPU_THREADS:?} -C src install-analysis-deps - echo "+++ make test/llvmpasses" - make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/llvmpasses - timeout_in_minutes: 60 diff --git a/.buildkite/pipelines/main/misc/sanitizers.yml b/.buildkite/pipelines/main/misc/sanitizers.yml deleted file mode 100644 index d406bcf403d370..00000000000000 --- a/.buildkite/pipelines/main/misc/sanitizers.yml +++ /dev/null @@ -1,48 +0,0 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" - arch: "x86_64" -steps: - - label: "asan" - key: "asan" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz - rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd" - uid: 1000 - gid: 1000 - workspaces: - - "/cache/repos:/cache/repos" - timeout_in_minutes: 120 - if: | # We only run the `asan` job on Julia 1.8 and later. - (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7") - soft_fail: true # TODO: delete this line (and thus disallow failures) once JuliaLang/julia#42540 is fixed - commands: | - echo "--- Build julia-debug with ASAN" - contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} debug - - label: "tsan" - key: "tsan" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz - rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd" - uid: 1000 - gid: 1000 - workspaces: - - "/cache/repos:/cache/repos" - timeout_in_minutes: 120 - if: | # We only run the `tsan` job on Julia 1.8 and later. - (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7") - commands: | - echo "--- Build julia-debug runtime with TSAN" - contrib/tsan/build.sh ./tmp/test-tsan -j$${JULIA_CPU_THREADS:?} julia-src-debug diff --git a/.buildkite/pipelines/main/misc/signed_pipeline_test.yml b/.buildkite/pipelines/main/misc/signed_pipeline_test.yml deleted file mode 100644 index 1d59253d43bcee..00000000000000 --- a/.buildkite/pipelines/main/misc/signed_pipeline_test.yml +++ /dev/null @@ -1,18 +0,0 @@ -agents: - queue: "julia" - os: "linux" - -## pipeline that showcases decryption of environment variable -steps: - - label: ":lock: :rocket: Signed pipeline test" - # We must accept the signed job id secret in order to propagate secrets - env: - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - depends_on: - plugins: - - staticfloat/cryptic#v1: - variables: - - SECRET_KEY="U2FsdGVkX18tb7st0SuQAvh4Yv4xENxOAu8q9XkmOeDVKBNY4FngEwK3xmiKUqaS" - commands: | - echo "SECRET_KEY: $${SECRET_KEY}" - diff --git a/.buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature b/.buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature deleted file mode 100644 index b0844748c486f3..00000000000000 --- a/.buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature +++ /dev/null @@ -1 +0,0 @@ -Salted__���J0�Q?���rۀ�g�~�d��ۛŧ�ө��o���Ujʀ���p�)�$�U$� ���y@gZM}{�m��,۠�K��e�r� \ No newline at end of file diff --git a/.buildkite/pipelines/main/misc/whitespace.yml b/.buildkite/pipelines/main/misc/whitespace.yml deleted file mode 100644 index 076e71b6fd5148..00000000000000 --- a/.buildkite/pipelines/main/misc/whitespace.yml +++ /dev/null @@ -1,25 +0,0 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" - arch: "x86_64" -steps: - - label: "whitespace" - key: "whitespace" - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" - workspaces: - - "/cache/repos:/cache/repos" - timeout_in_minutes: 10 - notify: - - github_commit_status: - context: "whitespace" - commands: | - make --output-sync -j$${JULIA_CPU_THREADS:?} check-whitespace diff --git a/.buildkite/pipelines/main/platforms/package_linux.arches b/.buildkite/pipelines/main/platforms/package_linux.arches deleted file mode 100644 index dec82f530a8324..00000000000000 --- a/.buildkite/pipelines/main/platforms/package_linux.arches +++ /dev/null @@ -1,7 +0,0 @@ -# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -linux 32 . . 32 i686 . . . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 -linux 64 . . 64 x86_64 . . . . . . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -# linux aarch64 . . aarch64 aarch64 . . . . . . . .... ........................................ -# linux armv7l . . armv7l armv7l . . . . . . . .... ........................................ -# linux ppc64le . . ppc64le powerpc64le . . . . . . . .... ........................................ -musl 64 . . 64 x86_64 . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 diff --git a/.buildkite/pipelines/main/platforms/package_linux.yml b/.buildkite/pipelines/main/platforms/package_linux.yml deleted file mode 100644 index 0e49cab2d6fc9e..00000000000000 --- a/.buildkite/pipelines/main/platforms/package_linux.yml +++ /dev/null @@ -1,55 +0,0 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" - arch: "x86_64" -steps: - - label: "package_${PLATFORM?}${LABEL?}" - key: package_${PLATFORM?}${LABEL?} - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz - rootfs_treehash: "${ROOTFS_HASH?}" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - timeout_in_minutes: ${TIMEOUT_BK?} - commands: | - echo "--- Print the full and short commit hashes" - SHORT_COMMIT_LENGTH=10 - SHORT_COMMIT=`echo $${BUILDKITE_COMMIT:?} | cut -c1-$${SHORT_COMMIT_LENGTH:?}` - ARTIFACT_FILE_EXTENSION="tar.gz" - ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-${PLATFORM?}${ARCH?}.$${ARTIFACT_FILE_EXTENSION:?}" - JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME ${MAKE_FLAGS?} | cut -c27- | tr -s ' '` - JULIA_BINARYDIST="$${JULIA_BINARYDIST_FILENAME:?}.$${ARTIFACT_FILE_EXTENSION:?}" - - echo "The full commit is: $${BUILDKITE_COMMIT:?}" - echo "The short commit is: $${SHORT_COMMIT:?}" - echo "The artifact filename will be: $${ARTIFACT_FILENAME:?}" - - echo "--- Build Julia from source" - rm -rf $${ARTIFACT_FILENAME:?} - make --output-sync -j 8 ${MAKE_FLAGS?} - - echo "--- Check that the working directory is clean" - if [ -z "$(git status --short)" ]; then echo "INFO: The working directory is clean."; else echo "ERROR: The working directory is dirty."; echo "Output of git status:"; git status; exit 1; fi - - echo "--- Print Julia version info" - ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - - echo "--- Create build artifacts" - make --output-sync -j 8 binary-dist ${MAKE_FLAGS?} - ls -l $${JULIA_BINARYDIST:?} - if [[ "$${JULIA_BINARYDIST:?}" != "$${ARTIFACT_FILENAME:?}" ]]; then - mv $${JULIA_BINARYDIST:?} $${ARTIFACT_FILENAME:?} - fi - ls -l $${ARTIFACT_FILENAME:?} - echo "--- Upload build artifacts" - buildkite-agent artifact upload $${ARTIFACT_FILENAME:?} diff --git a/.buildkite/pipelines/main/platforms/tester_linux.arches b/.buildkite/pipelines/main/platforms/tester_linux.arches deleted file mode 100644 index d1304563cb815e..00000000000000 --- a/.buildkite/pipelines/main/platforms/tester_linux.arches +++ /dev/null @@ -1,25 +0,0 @@ -# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -linux 32_g1 g1 . 32 i686 . . . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 -linux 32_g2 g2 . 32 i686 . . . 3 . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 - -linux 64_g1_mt g1 . 64 x86_64 . . . . . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g2_mt g2 . 64 x86_64 . . . 3 . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -linux 64_g1_st g1 . 64 x86_64 . . . . . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g2_st g2 . 64 x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -linux 64_g1_rrst g1 . 64 x86_64 . 300 240 . yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g2_rrst g2 . 64 x86_64 . 180 120 3 yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g3_st g3 . 64 x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -# linux aarch64_g1 g1 true aarch64 aarch64 . . . . . . . ---- ---------------------------------------- -# linux aarch64_g2 g2 true aarch64 aarch64 . . . . . . . ---- ---------------------------------------- - -# linux armv7l_g1 g1 true armv7l armv7l . . . . . . . ---- ---------------------------------------- -# linux armv7l_g2 g2 true armv7l armv7l . . . . . . . ---- ---------------------------------------- - -# linux ppc64le_g1 g1 true ppc64le powerpc64le . . . . . . . ---- ---------------------------------------- -# linux ppc64le_g2 g2 true ppc64le powerpc64le . . . . . . . ---- ---------------------------------------- - -musl 64_g1 g1 true 64 x86_64 . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 -musl 64_g2 g2 true 64 x86_64 . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 diff --git a/.buildkite/pipelines/main/platforms/tester_linux.yml b/.buildkite/pipelines/main/platforms/tester_linux.yml deleted file mode 100644 index 85bc64f51903e6..00000000000000 --- a/.buildkite/pipelines/main/platforms/tester_linux.yml +++ /dev/null @@ -1,121 +0,0 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" - arch: "x86_64" -steps: - - label: "tester_${PLATFORM?}${LABEL?}" - key: tester_${PLATFORM?}${LABEL?} - depends_on: package_${PLATFORM?}${ARCH?} - plugins: - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz - # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz - rootfs_treehash: "${ROOTFS_HASH?}" - uid: 1000 - gid: 1000 - workspaces: - # Include `/cache/repos` so that our `git` version introspection works. - - "/cache/repos:/cache/repos" - env: - JULIA_SHELL: "/bin/bash" - timeout_in_minutes: ${TIMEOUT_BK?} - retry: - automatic: - - exit_status: "*" - limit: ${RETRIES?} - soft_fail: ${ALLOW_FAIL?} - commands: | - echo "--- Print the full and short commit hashes" - SHORT_COMMIT_LENGTH=10 - SHORT_COMMIT=`echo $${BUILDKITE_COMMIT:?} | cut -c1-$${SHORT_COMMIT_LENGTH:?}` - JULIA_DIR="julia-$${SHORT_COMMIT:?}" - JULIA_BINARY="$${JULIA_DIR:?}/bin/julia" - ARTIFACT_FILE_EXTENSION="tar.gz" - ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-${PLATFORM?}${ARCH?}.$${ARTIFACT_FILE_EXTENSION:?}" - echo "The full commit is: $${BUILDKITE_COMMIT:?}" - echo "The short commit is: $${SHORT_COMMIT:?}" - echo "The artifact filename will be: $${ARTIFACT_FILENAME:?}" - echo "The Julia directory name will be: $${JULIA_DIR:?}" - echo "The Julia binary will be: $${JULIA_BINARY:?}" - - echo "--- Download build artifacts" - rm -rf $${ARTIFACT_FILENAME:?} - buildkite-agent artifact download $${ARTIFACT_FILENAME:?} . - - echo "--- Extract build artifacts" - rm -rf $${JULIA_DIR:?}/ - tar xzf $${ARTIFACT_FILENAME:?} $${JULIA_DIR:?}/ - - echo "--- Print Julia version info" - $${JULIA_BINARY:?} -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - echo "JULIA_CPU_THREADS is: $${JULIA_CPU_THREADS:?}" - $${JULIA_BINARY:?} -e '@info "" Sys.CPU_THREADS' - - echo "--- Set some environment variables" - export OPENBLAS_NUM_THREADS=8 - unset JULIA_DEPOT_PATH - unset JULIA_PKG_SERVER - - # Make sure that temp files and temp directories are created in a location that is - # backed by real storage. - export TMPDIR="$(pwd)/tmp" - mkdir -p $${TMPDIR:?} - - export NETWORK_RELATED_TESTS="Artifacts Downloads download LazyArtifacts LibGit2/online Pkg" - - if [[ "${GROUP?}" == "all" ]]; then - export TESTS="all LibGit2/online --force-net" - elif [[ "${GROUP?}" == "all_except_pkg" ]]; then - export TESTS="all LibGit2/online --force-net --skip Pkg" - elif [[ "${GROUP?}" == "g1" ]]; then - # Group 1: ALL tests EXCEPT the network-related tests. - export TESTS="all --force-net --skip $${NETWORK_RELATED_TESTS:?}" - elif [[ "${GROUP?}" == "g2" ]]; then - # Group 2: ONLY the network-related tests. - # In Group 2, we use whatever the default setting is with regards to the Pkg server. - export TESTS="$${NETWORK_RELATED_TESTS:?} --force-net" - elif [[ "${GROUP?}" == "g3" ]]; then - # Group 3: only Pkg. - # In Group 3, we explicitly opt-out of the Pkg server. - # The purpose of group 3 is to test the non-Pkg-server codepaths of Pkg. - export TESTS="Pkg --force-net" - export JULIA_PKG_SERVER="" - else - echo "Invalid value for GROUP: ${GROUP?}" - exit 1 - fi - - export JULIA_TEST_RR_TIMEOUT="${TIMEOUT_RR?}" - - if [[ "${IS_RR?}" == "yes" ]]; then - export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}" - export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])" - else - export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?}" - export NCORES_FOR_TESTS="Sys.CPU_THREADS" - fi - - if [[ "${IS_ST?}" == "yes" ]]; then - export JULIA_NUM_THREADS=1 - fi - - if [[ "${IS_MT?}" == "yes" ]]; then - export JULIA_NUM_THREADS=16 - fi - - echo "--- Print the test group, list of test sets, and other useful environment variables" - echo "JULIA_CMD_FOR_TESTS is: $${JULIA_CMD_FOR_TESTS:?}" - echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set - echo "NCORES_FOR_TESTS is: $${NCORES_FOR_TESTS:?}" - echo "OPENBLAS_NUM_THREADS is: $${OPENBLAS_NUM_THREADS:?}" - echo "GROUP is: ${GROUP?}" - echo "TESTS is: $${TESTS:?}" - - echo "--- Run the Julia test suite" - $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})" diff --git a/.buildkite/pipelines/scheduled/0_webui.yml b/.buildkite/pipelines/scheduled/0_webui.yml deleted file mode 100644 index 78031b49d9f17d..00000000000000 --- a/.buildkite/pipelines/scheduled/0_webui.yml +++ /dev/null @@ -1,27 +0,0 @@ -# This file represents what is put into the webUI. -# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. -# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. -agents: - queue: "julia" - sandbox.jl: "true" -steps: - - label: ":unlock: Unlock secrets, launch pipelines" - plugins: - - staticfloat/cryptic: - # Our list of pipelines that should be launched (but don't require a signature) - # These pipelines can be modified by any contributor and CI will still run. - # Build secrets will not be available in these pipelines (or their children) - # but some of our signed pipelines can wait upon the completion of these unsigned - # pipelines. - unsigned_pipelines: - - .buildkite/pipelines/scheduled/launch_unsigned_jobs.yml - - # Our signed pipelines must have a `signature` or `signature_file` parameter that - # verifies the treehash of the pipeline itself and the inputs listed in `inputs` - signed_pipelines: - - pipeline: .buildkite/pipelines/scheduled/coverage/coverage_linux64.yml - signature_file: .buildkite/pipelines/scheduled/coverage/coverage_linux64.yml.signature - inputs: - - .buildkite/pipelines/scheduled/coverage/coverage_linux64.yml - - .buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl - - .buildkite/pipelines/scheduled/coverage/upload_coverage.jl diff --git a/.buildkite/pipelines/scheduled/README.md b/.buildkite/pipelines/scheduled/README.md deleted file mode 100644 index ca071dceb2a444..00000000000000 --- a/.buildkite/pipelines/scheduled/README.md +++ /dev/null @@ -1,5 +0,0 @@ -## Scheduled pipeline (`master` branch only) - -This is the [`julia-master->scheduled`](https://buildkite.com/julialang/julia-master-scheduled) pipeline. - -We use this pipeline for scheduled builds. The builders in this pipeline run on a schedule once per day. They are not triggered by GitHub webhooks. diff --git a/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml b/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml deleted file mode 100644 index 1ff88577e2be4e..00000000000000 --- a/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml +++ /dev/null @@ -1,44 +0,0 @@ -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" -steps: - - label: ":unlock: :coverage: Run coverage test" - # We must accept the signed job id secret in order to propagate secrets - env: - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} - depends_on: - plugins: - - staticfloat/cryptic: - variables: - - CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA==" - - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw==" - - JuliaCI/julia#v1: - # Drop default "registries" directory, so it is not persisted from execution to execution - persist_depot_dirs: packages,artifacts,compiled - version: '1.6' - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz - rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a" - uid: 1000 - gid: 1000 - timeout_in_minutes: 360 # 360 minutes = 6 hours - commands: | - echo "--- Build Julia from source" - make --output-sync -j 6 - - echo "--- Print Julia version info" - ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - ./julia -e '@info "" Sys.CPU_THREADS' - # this is necessary to make sure that the LibGit2 tests passes - git config --global init.defaultBranch master - - echo "--- Run Julia tests in parallel with code coverage enabled" - export JULIA_NUM_THREADS=1 - export JULIA_WORKER_TIMEOUT=1200 # 1200 seconds = 20 minutes - ./julia -e 'import Distributed; @info "" Distributed.worker_timeout()' - ./julia .buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl - - echo "--- Process and upload coverage information" - ./julia .buildkite/pipelines/scheduled/coverage/upload_coverage.jl diff --git a/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml.signature b/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml.signature deleted file mode 100644 index 4ecec8e8bb72cc..00000000000000 --- a/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml.signature +++ /dev/null @@ -1 +0,0 @@ -Salted__�I�y�֌��>y�Nc kB�v�n�+�Hvrލ��Ƽ�r����/�uY�����u����I�iiE��(��v�L��!��?��v� \ No newline at end of file diff --git a/.buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl b/.buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl deleted file mode 100644 index b6eed225f652d4..00000000000000 --- a/.buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl +++ /dev/null @@ -1,29 +0,0 @@ -# Important note: even if one or more tests fail, we will still exit with status code 0. -# -# The reason for this is that we always want to upload code coverage, even if some of the -# tests fail. Therefore, even if the `coverage_linux64` builder passes, you should not -# assume that all of the tests passed. If you want to know if all of the tests are passing, -# please look at the status of the `tester_*` builders (e.g. `tester_linux64`). - -const ncores = Sys.CPU_THREADS -@info "" Sys.CPU_THREADS -@info "" ncores - -script_native_yes = """ - Base.runtests(["cmdlineargs"]; ncores = $(ncores)) -""" -script_native_no = """ - Base.runtests(["all", "--skip", "cmdlineargs"]; ncores = $(ncores)) -""" - -base_cmd = `$(Base.julia_cmd()) --code-coverage=all` -cmd_native_yes = `$(base_cmd) --sysimage-native-code=yes -e $(script_native_yes)` -cmd_native_no = `$(base_cmd) --sysimage-native-code=no -e $(script_native_no)` - -@info "Running command" cmd_native_yes -p1 = run(pipeline(cmd_native_yes; stdin, stdout, stderr); wait = false) -wait(p1) - -@info "Running command" cmd_native_no -p2 = run(pipeline(cmd_native_no; stdin, stdout, stderr); wait = false) -wait(p2) diff --git a/.buildkite/pipelines/scheduled/coverage/upload_coverage.jl b/.buildkite/pipelines/scheduled/coverage/upload_coverage.jl deleted file mode 100644 index d995e97fc17fb9..00000000000000 --- a/.buildkite/pipelines/scheduled/coverage/upload_coverage.jl +++ /dev/null @@ -1,228 +0,0 @@ -empty!(Base.DEPOT_PATH) -push!(Base.DEPOT_PATH, mktempdir(; cleanup = true)) - -import Pkg -import Logging -import TOML - -Pkg.add(; name = "Coverage", uuid = "a2441757-f6aa-5fb2-8edb-039e3f45d037", version = "1") -Pkg.precompile() - -import Coverage - -function process_folders() - # `Coverage.process_folder` will have a LOT of `@info` statements that will make the log - # way too long. So before we run `Coverage.process_folder`, we disable logging for `@info` - # statements. After we run `Coverage.process_folder`, we re-enable logging for `@info` - # statements. - Logging.disable_logging(Logging.Info) - fcs_base = Coverage.process_folder("base"); - fcs_stdlib = Coverage.process_folder("stdlib"); - Logging.disable_logging(Logging.Debug) - - fcs = Coverage.merge_coverage_counts( - fcs_base, - fcs_stdlib, - ); - - return fcs -end - -function get_external_stdlib_names(stdlib_dir::AbstractString) - filename_list = filter(x -> isfile(joinpath(stdlib_dir, x)), readdir(stdlib_dir)) - # find all of the files like `Pkg.version`, `Statistics.version`, etc. - regex_matches_or_nothing = match.(Ref(r"^([\w].*?)\.version$"), filename_list) - regex_matches = filter(x -> x !== nothing, regex_matches_or_nothing) - # get the names of the external stdlibs, like `Pkg`, `Statistics`, etc. - external_stdlib_names = only.(regex_matches) - unique!(external_stdlib_names) - sort!(external_stdlib_names) - @info "# Begin list of external stdlibs" - for (i, x) in enumerate(external_stdlib_names) - @info "$(i). $(x)" - end - @info "# End list of external stdlibs" - return external_stdlib_names -end - -function get_external_stdlib_prefixes(stdlib_dir::AbstractString) - external_stdlib_names = get_external_stdlib_names(stdlib_dir) - prefixes_1 = joinpath.(Ref(stdlib_dir), external_stdlib_names, Ref("")) - prefixes_2 = joinpath.(Ref(stdlib_dir), string.(external_stdlib_names, Ref("-"))) - prefixes = vcat(prefixes_1, prefixes_2) - unique!(prefixes) - sort!(prefixes) - # example of what `prefixes` might look like: - # 4-element Vector{String}: - # "stdlib/Pkg-" - # "stdlib/Pkg/" - # "stdlib/Statistics-" - # "stdlib/Statistics/" - return prefixes -end - -function print_coverage_summary(fc::Coverage.FileCoverage) - cov_lines, tot_lines = Coverage.get_summary(fc) - if cov_lines == tot_lines == 0 - cov_pct = 0 - else - cov_pct = floor(Int, cov_lines/tot_lines * 100) - end - pad_1 = 71 - pad_2 = 15 - pad_3 = 15 - col_1 = rpad(fc.filename, pad_1) - col_2 = rpad(string(cov_pct, " %"), pad_2) - col_3 = string( - rpad(string(cov_lines), pad_3), - string(tot_lines), - ) - @info "$(col_1) $(col_2) $(col_3)" - return nothing -end - -function print_coverage_summary( - fcs::Vector{Coverage.FileCoverage}, description::AbstractString, - ) - cov_lines, tot_lines = Coverage.get_summary(fcs) - if cov_lines == tot_lines == 0 - cov_pct = 0 - else - cov_pct = floor(Int, cov_lines/tot_lines * 100) - end - @info "$(description): $(cov_pct)% ($(cov_lines)/$(tot_lines))" - return (; cov_pct) -end - -function buildkite_env(name::String) - value = String(strip(ENV[name])) - if isempty(value) - throw(ErrorException("environment variable $(name) is empty")) - end - return value -end - -function buildkite_env(name_1::String, name_2::String, default::String) - value_1 = String(strip(ENV[name_1])) - value_2 = String(strip(ENV[name_2])) - !isempty(value_1) && return value_1 - !isempty(value_2) && return value_2 - return default -end - -function buildkite_branch_and_commit() - branch = buildkite_env("BUILDKITE_BRANCH") - commit = buildkite_env("BUILDKITE_COMMIT") - head_rev_parse = String(strip(read(`git rev-parse HEAD`, String))) - if strip(commit) == "HEAD" - commit = head_rev_parse - end - if commit !== head_rev_parse - msg = "mismatch" - @error msg commit head_rev_parse - throw(ErrorException(msg)) - end - if !occursin(r"^[a-f0-9]{40}$", commit) - msg = "BUILDKITE_COMMIT does not look like a long commit SHA" - @error msg commit - throw(ErrorException(msg)) - end - return (; branch, commit) -end - -function codecov_buildkite_add_local_to_kwargs() - branch, commit = buildkite_branch_and_commit() - kwargs = Coverage.Codecov.set_defaults( - Dict(); - branch, - commit, - ) - return kwargs -end - -function coveralls_buildkite_query_git_info() - branch, commit = buildkite_branch_and_commit() - remote_name = "origin" - remote = buildkite_env("BUILDKITE_REPO") - message = buildkite_env("BUILDKITE_MESSAGE") - author_name = buildkite_env( - "BUILDKITE_BUILD_AUTHOR", - "BUILDKITE_BUILD_CREATOR", - "", - ) - author_email = buildkite_env( - "BUILDKITE_BUILD_AUTHOR_EMAIL", - "BUILDKITE_BUILD_CREATOR_EMAIL", - "", - ) - remotes = [ - Dict( - "name" => remote_name, - "url" => remote, - ) - ] - head = Dict( - "id" => commit, - "author_name" => author_name, - "author_email" => author_email, - "committer_name" => author_name, - "committer_email" => author_email, - "message" => message, - ) - git_info = Dict( - "branch" => branch, - "remotes" => remotes, - "head" => head, - ) - return git_info -end - -const fcs = process_folders() - -# Only include source code files. Exclude test files, benchmarking files, etc. -filter!(fcs) do fc - occursin(r"^base\/", fc.filename) || occursin("/src/", fc.filename) -end; - -# Exclude all external stdlibs (stdlibs that live in external repos). -const external_stdlib_prefixes = get_external_stdlib_prefixes("stdlib") -filter!(fcs) do fc - all(x -> !startswith(fc.filename, x), external_stdlib_prefixes) -end; - -# Exclude all stdlib JLLs (stdlibs of the form `stdlib/*_jll/`). -filter!(fcs) do fc - !occursin(r"^stdlib\/[A-Za-z0-9]*?_jll\/", fc.filename) -end; - -sort!(fcs; by = fc -> fc.filename); - -print_coverage_summary.(fcs); -const total_cov_pct = print_coverage_summary(fcs, "Total").cov_pct - -let - git_info = coveralls_buildkite_query_git_info() - @info "" git_info - @info "" git_info["branch"] - @info "" git_info["head"] - - # In order to upload to Coveralls, you need to have the `COVERALLS_TOKEN` environment variable defined. - Coverage.Coveralls.submit_local(fcs, git_info) -end - -let - kwargs = codecov_buildkite_add_local_to_kwargs() - @info "" kwargs - - # In order to upload to Codecov, you need to have the `CODECOV_TOKEN` environment variable defined. - Coverage.Codecov.submit_generic(fcs, kwargs) -end - -if total_cov_pct < 50 - msg = string( - "The total coverage is less than 50%. This should never happen, ", - "so it means that something has probably gone wrong with the code coverage job.", - ) - @error msg total_cov_pct - throw(ErrorException(msg)) -end diff --git a/.buildkite/pipelines/scheduled/launch_unsigned_jobs.yml b/.buildkite/pipelines/scheduled/launch_unsigned_jobs.yml deleted file mode 100644 index 300c8d8466aea1..00000000000000 --- a/.buildkite/pipelines/scheduled/launch_unsigned_jobs.yml +++ /dev/null @@ -1,8 +0,0 @@ -steps: - - label: ":buildkite: Launch unsigned jobs" - commands: | - # Launch all of the `USE_BINARYBUILDER=0` jobs. - bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml - bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches .buildkite/pipelines/main/platforms/tester_linux.yml - agents: - queue: julia diff --git a/.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches b/.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches deleted file mode 100644 index dff2aab4591e2f..00000000000000 --- a/.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches +++ /dev/null @@ -1,2 +0,0 @@ -# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -linux 64src . . 64src x86_64 USE_BINARYBUILDER=0 180 . . . . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a diff --git a/.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches b/.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches deleted file mode 100644 index 0b1fbdf63b7967..00000000000000 --- a/.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches +++ /dev/null @@ -1,10 +0,0 @@ -# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -linux 64src_g1_mt g1 . 64src x86_64 . . . . . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64src_g2_mt g2 . 64src x86_64 . . . 3 . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -linux 64src_g1_st g1 . 64src x86_64 . . . . . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64src_g2_st g2 . 64src x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -linux 64src_g1_rrst g1 . 64src x86_64 . 300 240 . yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64src_g2_rrst g2 . 64src x86_64 . 180 120 3 yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64src_g3_st g3 . 64src x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a diff --git a/.buildkite/secrets/.gitignore b/.buildkite/secrets/.gitignore deleted file mode 100644 index 2a84f48682a04f..00000000000000 --- a/.buildkite/secrets/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -# Ignore everything -* - -# Don't ignore this `.gitignore` file -!.gitignore - -# Don't ignore encrypted files -!*.encrypted - -# Don't ignore public keys, that's fine to include -!*.pub diff --git a/.buildkite/secrets/ssh_docs_deploy.encrypted b/.buildkite/secrets/ssh_docs_deploy.encrypted deleted file mode 100644 index 8b7e2ffe279402c74a57005d327220a4806500d0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1856 zcmV-G2fz4JVQh3|WM5x6_BQu$(kQ%->4|0fa6`t(&xPEIE=okh>cTg)MnE6hWl4%W z%<3W-{MyzND_s>N=jAkwv=;0N%hTRP{Cvd@J_VNs`*gk&F>exm{qkM5tsA}DTlf{R zE3wfuA1RqY3nBxTkXkcD(Homn*7scj70QL=N#-$yyBFcv>M0p_5i7lvUA=JQhG3Z+2M#8S>T*R*KB;@y_|LL#USeKun zkfFh$9|tFr_kr7O0Ql?;<^&yF6V>Ef_^4H5{F9V5G3qwi<-e|_aeeqH6Owo8;Rrxo zER^|Tk@}rbeW>kdJH7Id@zC2jnZu)5rJL0_p-$QD*LUAw?J(|;mrWh`aEL}&i${cZ zY)9c(4&4g(YI$*P$Uy@y4*%%H4eDuo_PIMyknL`x$yy}5&8_!DWuxhtrLErV$9Yk!bromBYjTI}xQ6_n+A$HgyXBLV)@ybZI~VFDh6B2@^*>Lj*b-pbw{G=x0sV~412 zD^63?S_Jp7Yw_f9vE6|`Rzeh$kOKcHQ#g#$nc5-bokDPt0>54uD*tsi`jrC?C6Hnp zyta39hntcy$WBN?9Lzx69D+`$2(Ofh)~8PNRvU@V*T`0o;K)roVPfVBwCkalL{maZ5qqQLtJG; zPN^s$@56cDc1#rP?@^P~J=N7j6JB`kMqKKE%1zt~u}KiyTt{NsNUL5c$j^c_AHyOl ziX5j1M3+!k&H%g}NZ9n}!wQ^c!`@wtk70?|6IWO^3+RzA&t;XE@Qgeo*f|$?HNc#v zroCb}MgdxKJ1&nVgVk(ADAtVfMA*sdWr5li%*zneS9atN$?KHX6#h92?9s;f57^ya zwO}Q*b*P=ao`JFwLr9tH*%M&;EB`lZ)XvM)%znRP_lHz~^Y^V4r$)=d@Q8q5N;|riRzw4GteZ)2amD zu<@cbLZp~&>IO(=XVO6c62;Kj*;x)plTO2Q!68R*?$j|bnm_Eoh%i^v_6mn*HF7a1 zZv_!pari;`$#s97Z*XA>vg~`325A;w0W0Vw{1h%&{fKK1oM`Eeg^I~}WtpZ^ch{yd z%lS?c8IqGcJCDb!LI2wP9R_NG@}l;~Tj&xjC}98gTC|vs2iXz^DkT-j6{m~aFz`xf zH_+Cd`skfz8WSz>I<$zoRitWpm@@!zcJ$K3rn~Y0ebJLgz)Un_68_D1Lk7K)o=w}v zm`MzC4hhfT2d8758RTL=q+sKAxexZ1n7v-c4%Bw=c zvfVjEpgF=yzWJ;HC;fqYJg&3car(huXIu zTy-=%EaXCo$MpXLIYe&JpuG9O*g=d9-maj{#}IEyzeRcUguycE(_agoB<=P>B46VH za)c3lWQrd2sWAp6c;OeEjOnGInOh?7zR#b7V%4x0$sB#%9_K{@9Y6rJ+^iAg+yv4` z7~bPZGm>{d1NhK}DMKyfLwaxAH z;(X}Vd;1P}+WzJ5u1k`KR6bB$I$}XI5B6xx%}Z|)Z4mFEyU3=${b) z-YJO2(&)IoTz6MidbNwn3Mv#8(69wQmsj6Y1I3Svw5o+r(*XNBO_<%$mmh?vjKI1ze;=FMc9)PSS&U0!++8Co}z4{($e zWfbj0!zl{njIk$UDNDGhSLFu*Fg|WQBX7T2=Dm3rb)_pHap strip |> lowercase - result = parse(Bool, value)::Bool - return result -end - -const is_buildkite = get_bool_from_env("BUILDKITE", false) -const always_save_rr_trace = get_bool_from_env("JULIA_ALWAYS_SAVE_RR_TRACE", false) - -function get_from_env(name::AbstractString) - if is_buildkite - value = ENV[name] - else - value = get(ENV, name, "") - end - result = convert(String, strip(value))::String - return result -end - -function my_exit(process::Base.Process) - wait(process) - - @info( - "", - process.exitcode, - process.termsignal, - ) - - # Pass the exit code back up - if process.termsignal != 0 - ccall(:raise, Cvoid, (Cint,), process.termsignal) - - # If for some reason the signal did not cause an exit, we'll exit manually. - # We need to make sure that we exit with a non-zero exit code. - if process.exitcode != 0 - exit(process.exitcode) - else - exit(1) - end - end - exit(process.exitcode) -end - -if Base.VERSION < v"1.6" - throw(ErrorException("The `$(basename(@__FILE__))` script requires Julia 1.6 or greater")) -end - -if length(ARGS) < 1 - throw(ErrorException("Usage: julia $(basename(@__FILE__)) [command...]")) -end - -@info "We will run the command under rr" - -const build_number = get_from_env("BUILDKITE_BUILD_NUMBER") -const job_name = get_from_env("BUILDKITE_STEP_KEY") -const commit_full = get_from_env("BUILDKITE_COMMIT") -const commit_short = first(commit_full, 10) -const JULIA_TEST_RR_TIMEOUT = get(ENV, "JULIA_TEST_RR_TIMEOUT", "120") -const timeout_minutes = parse(Int, JULIA_TEST_RR_TIMEOUT) -const JULIA_TEST_NUM_CORES = get(ENV, "JULIA_TEST_NUM_CORES", "8") -const julia_test_num_cores_int = parse(Int, JULIA_TEST_NUM_CORES) -const num_cores = min( - 8, - Sys.CPU_THREADS, - julia_test_num_cores_int + 1, -) - -ENV["JULIA_RRCAPTURE_NUM_CORES"] = "$(num_cores)" - -@info( - "", - build_number, - job_name, - commit_full, - commit_short, - timeout_minutes, - num_cores, -) - -const dumps_dir = joinpath(pwd(), "dumps") -const temp_parent_dir = joinpath(pwd(), "temp_for_rr") - -mkpath(dumps_dir) -mkpath(temp_parent_dir) - -proc = nothing - -mktempdir(temp_parent_dir) do dir - Pkg.activate(dir) - Pkg.add("rr_jll") - Pkg.add("Zstd_jll") - - rr_jll = Base.require(Base.PkgId(Base.UUID((0xe86bdf43_55f7_5ea2_9fd0_e7daa2c0f2b4)), "rr_jll")) - zstd_jll = Base.require(Base.PkgId(Base.UUID((0x3161d3a3_bdf6_5164_811a_617609db77b4)), "Zstd_jll")) - rr(func) = Base.invokelatest(rr_jll.rr, func; adjust_LIBPATH=false) - rr() do rr_path - capture_script_path = joinpath(dir, "capture_output.sh") - loader = Sys.WORD_SIZE == 64 ? "/lib64/ld-linux-x86-64.so.2" : "/lib/ld-linux.so.2" - open(capture_script_path, "w") do io - write(io, """ - #!/bin/bash - - $(rr_path) record --nested=detach "\$@" > >(tee -a $(dir)/stdout.log) 2> >(tee -a $(dir)/stderr.log >&2) - """) - end - chmod(capture_script_path, 0o755) - - new_env = copy(ENV) - new_env["_RR_TRACE_DIR"] = joinpath(dir, "rr_traces") - new_env["RR_LOG"]="all:debug" - new_env["RR_LOG_BUFFER"]="100000" - new_env["JULIA_RR"] = capture_script_path - t_start = time() - global proc = run(setenv(`$(rr_path) record --num-cores=$(num_cores) $ARGS`, new_env), (stdin, stdout, stderr); wait=false) - - # Start asynchronous timer that will kill `rr` - @async begin - sleep(timeout_minutes * 60) - - # If we've exceeded the timeout and `rr` is still running, kill it. - if isopen(proc) - println(stderr, "\n\nProcess timed out (with a timeout of $(timeout_minutes) minutes). Signalling `rr` for force-cleanup!") - kill(proc, Base.SIGTERM) - - # Give `rr` a chance to cleanup and upload. - # Note: this time period includes the time to upload the `rr` trace files - # as Buildkite artifacts, so make sure it is long enough to allow the - # uploads to finish. - cleanup_minutes = 30 - sleep(cleanup_minutes * 60) - - if isopen(proc) - println(stderr, "\n\n`rr` failed to cleanup and upload within $(cleanup_minutes) minutes, killing and exiting immediately!") - kill(proc, Base.SIGKILL) - exit(1) - end - end - end - - # Wait for `rr` to finish, either through naturally finishing its run, or `SIGTERM`. - wait(proc) - process_failed = !success(proc) - - if process_failed || always_save_rr_trace || is_buildkite - println(stderr, "`rr` returned $(proc.exitcode), packing and uploading traces...") - - if !isdir(joinpath(dir, "rr_traces")) - println(stderr, "No `rr_traces` directory! Did `rr` itself fail?") - exit(1) - end - - # Clean up non-traces - rm(joinpath(dir, "rr_traces", "latest-trace")) - rm(joinpath(dir, "rr_traces", "cpu_lock")) - - # Create a directory for the pack files to go - pack_dir = joinpath(dir, "pack") - mkdir(pack_dir) - - # Pack all traces - trace_dirs = [joinpath(dir, "rr_traces", f) for f in readdir(joinpath(dir, "rr_traces"))] - filter!(isdir, trace_dirs) - run(ignorestatus(`$(rr_path) pack --pack-dir=$pack_dir $(trace_dirs)`)) - - # Tar it up - mkpath(dumps_dir) - date_str = Dates.format(Dates.now(), Dates.dateformat"yyyy_mm_dd_HH_MM_SS") - dst_file_name = string( - "rr", - "--build_$(build_number)", - "--$(job_name)", - "--commit_$(commit_short)", - "--$(date_str)", - ".tar.zst", - ) - dst_full_path = joinpath(dumps_dir, dst_file_name) - zstd_jll.zstdmt() do zstdp - tarproc = open(`$(zstdp) -o $(dst_full_path)`, "w") - Tar.create(dir, tarproc) - close(tarproc.in) - end - - @info "The `rr` trace file has been saved to: $(dst_full_path)" - if is_buildkite - @info "Since this is a Buildkite run, we will upload the `rr` trace file." - cd(dumps_dir) do - run(`buildkite-agent artifact upload $(dst_file_name)`) - end - end - end - - end -end - -@info "Finished running the command under rr" -my_exit(proc)