diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index 8f987172..786f8c78 100755 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -26,7 +26,7 @@ rm -rf ..?* .[!.]* * # git clone, then force checkout the given gitref UPSTREAM_GITREF="$(buildkite-agent meta-data get --default "origin/${UPSTREAM_BRANCH}" BUILDKITE_JULIA_VERSION)" -git clone --reference "${UPSTREAM_CACHE}" "${UPSTREAM_URL}" "./" +git clone --dissociate --reference "${UPSTREAM_CACHE}" "${UPSTREAM_URL}" "./" git reset --hard "${UPSTREAM_GITREF}" echo git --no-pager log -1 diff --git a/Makefile b/Makefile index 6c6049d3..6b3cfe78 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,6 @@ +decrypt: + cd .buildkite/cryptic_repo_root && decrypt --repo-root=$$(pwd) --verbose + sign_treehashes: cd .buildkite/cryptic_repo_root && sign_treehashes --repo-root=$$(pwd) --verbose diff --git a/pipelines/main/launch_signed_jobs.yml.signature b/pipelines/main/launch_signed_jobs.yml.signature index f9e31eca..22749346 100644 Binary files a/pipelines/main/launch_signed_jobs.yml.signature and b/pipelines/main/launch_signed_jobs.yml.signature differ diff --git a/pipelines/main/platforms/build_linux.arches b/pipelines/main/platforms/build_linux.arches index eb81652c..b54b252b 100644 --- a/pipelines/main/platforms/build_linux.arches +++ b/pipelines/main/platforms/build_linux.arches @@ -1,11 +1,11 @@ -# OS TRIPLET ALLOW_FAIL ARCH_ROOTFS MAKE_FLAGS TIMEOUT ROOTFS_TAG ROOTFS_HASH -# linux i686-linux-gnu . i686 . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 -linux x86_64-linux-gnu . x86_64 CFLAGS=-Werror,CXXFLAGS=-Werror . v5.1 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0 -linux x86_64-linux-gnuassert . x86_64 FORCE_ASSERTIONS=1,LLVM_ASSERTIONS=1,CFLAGS=-Werror,CXXFLAGS=-Werror . v5.1 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0 -# linux aarch64-linux-gnu . aarch64 . . ---- ---------------------------------------- -# linux armv7l-linux-gnueabihf . armv7l . . ---- ---------------------------------------- -# linux powerpc64le-linux-gnu . powerpc64le . . ---- ---------------------------------------- -musl x86_64-linux-musl . x86_64 . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 +# OS TRIPLET ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT ROOTFS_TAG ROOTFS_HASH +# linux i686-linux-gnu . x86_64 i686 . . ---- ---------------------------------------- +linux x86_64-linux-gnu . x86_64 x86_64 CFLAGS=-Werror,CXXFLAGS=-Werror . v5.8 582b5d44bbc24d2a58ab609c5a520b3fd102e504 +linux x86_64-linux-gnuassert . x86_64 x86_64 FORCE_ASSERTIONS=1,LLVM_ASSERTIONS=1,CFLAGS=-Werror,CXXFLAGS=-Werror . v5.8 582b5d44bbc24d2a58ab609c5a520b3fd102e504 +# linux aarch64-linux-gnu . aarch64 aarch64 . . ---- ---------------------------------------- +# linux armv7l-linux-gnueabihf . armv7l armv7l . . ---- ---------------------------------------- +# linux powerpc64le-linux-gnu . powerpc64le powerpc64le . . ---- ---------------------------------------- +musl x86_64-linux-musl . x86_64 x86_64 . . v5.8 dc23dbeb02f9b85c8c8c7991ee8cfcae8f4c809b # These special lines allow us to embed default values for the columns above. # Any column without a default mapping here will simply substitute a `.` to the empty string diff --git a/pipelines/main/platforms/build_linux.yml b/pipelines/main/platforms/build_linux.yml index 26136663..e9c706b2 100644 --- a/pipelines/main/platforms/build_linux.yml +++ b/pipelines/main/platforms/build_linux.yml @@ -20,53 +20,14 @@ steps: # Include `/cache/repos` so that our `git` version introspection works. - "/cache/repos:/cache/repos" timeout_in_minutes: ${TIMEOUT?} - commands: | - # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc... - TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh - - # export JULIA_CPU_TARGET, which is picked up by the build system - ARCH="$${ARCH??}" source .buildkite/utilities/export_julia_cpu_target.sh - - echo "--- Create the Make.user file" - echo "${MAKE_FLAGS?}" | tr "," "\n" > Make.user - if [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-master" ]] || [[ "$${BUILDKITE_PIPELINE_SLUG:?}" =~ ^julia-release- ]]; then - banner="Official https://julialang.org/ release" - echo "TAGGED_RELEASE_BANNER=\"$${banner:?}\"" >> Make.user - elif [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-buildkite" ]]; then - banner="Not an official binary; built as part of the CI infrastructure testing" - echo "TAGGED_RELEASE_BANNER=\"$${banner:?}\"" >> Make.user - fi - echo "VERBOSE=1" >> Make.user - cat Make.user - - echo "--- Build Julia" - make --output-sync -j 8 - - echo "--- Check that the working directory is clean" - if [ -z "$(git status --short)" ]; then - echo "INFO: The working directory is clean." - else - echo "ERROR: The working directory is dirty." - echo "Output of git status:" - git status - exit 1 - fi - - echo "--- Print Julia version info" - ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - - echo "--- Create build artifacts" - make --output-sync -j 8 binary-dist - - # Rename the build artifact in case we want to name it differently, as is the case on `musl`. - if [[ "$${JULIA_BINARYDIST_FILENAME}" != "$${UPLOAD_FILENAME}" ]]; then - mv $${JULIA_BINARYDIST_FILENAME} $${UPLOAD_FILENAME} - fi - - echo "--- Upload build artifacts" - buildkite-agent artifact upload $${UPLOAD_FILENAME} + commands: "bash .buildkite/utilities/build_julia.sh" agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing sandbox_capable: "true" os: "linux" + arch: "${ARCH?}" + env: + # Receive some environment mappings from our templating engine + TRIPLET: "${TRIPLET?}" + MAKE_FLAGS: "${MAKE_FLAGS?}" diff --git a/pipelines/main/platforms/build_macos.yml b/pipelines/main/platforms/build_macos.yml index 92b9fb08..38813a3b 100644 --- a/pipelines/main/platforms/build_macos.yml +++ b/pipelines/main/platforms/build_macos.yml @@ -5,57 +5,16 @@ steps: key: "build_${TRIPLET?}" plugins: - JuliaCI/external-buildkite#v1: - version: ".buildkite-external-version" + version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" timeout_in_minutes: ${TIMEOUT?} - commands: | - # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc... - TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh - - # export JULIA_CPU_TARGET, which is picked up by the build system - ARCH="$${ARCH??}" source .buildkite/utilities/export_julia_cpu_target.sh - - echo "--- Create the Make.user file" - echo "${MAKE_FLAGS?}" | tr "," "\n" > Make.user - if [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-master" ]] || [[ "$${BUILDKITE_PIPELINE_SLUG:?}" =~ ^julia-release- ]]; then - banner="Official https://julialang.org/ release" - echo "TAGGED_RELEASE_BANNER=\"$${banner:?}\"" >> Make.user - elif [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-buildkite" ]]; then - banner="Not an official binary; built as part of the CI infrastructure testing" - echo "TAGGED_RELEASE_BANNER=\"$${banner:?}\"" >> Make.user - fi - echo "VERBOSE=1" >> Make.user - cat Make.user - - echo "--- Build Julia" - make -j 8 ${MAKE_FLAGS?} - - echo "--- Check that the working directory is clean" - if [ -z "$(git status --short)" ]; then - echo "INFO: The working directory is clean." - else - echo "ERROR: The working directory is dirty." - echo "Output of git status:" - git status - exit 1 - fi - echo "--- Print Julia version info" - ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - - echo "--- Create build artifacts" - make -j 8 binary-dist ${MAKE_FLAGS?} - - # Rename the build artifact in case we want to name it differently, as is the case on `musl`. - if [[ "$${JULIA_BINARYDIST_FILENAME}" != "$${UPLOAD_FILENAME}" ]]; then - mv $${JULIA_BINARYDIST_FILENAME} $${UPLOAD_FILENAME} - fi - - echo "--- Upload build artifacts" - buildkite-agent artifact upload $${UPLOAD_FILENAME} + commands: "bash .buildkite/utilities/build_julia.sh" agents: queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - # sandbox_capable: "true" os: "macos" - arch: "${ARCH}" + arch: "${ARCH?}" + env: + # Receive some environment mappings from our templating engine + TRIPLET: "${TRIPLET?}" + MAKE_FLAGS: "${MAKE_FLAGS?}" diff --git a/pipelines/main/platforms/test_linux.arches b/pipelines/main/platforms/test_linux.arches index fdc31905..b33ee9cd 100644 --- a/pipelines/main/platforms/test_linux.arches +++ b/pipelines/main/platforms/test_linux.arches @@ -1,12 +1,12 @@ -# OS TRIPLET ALLOW_FAIL ARCH_ROOTFS TIMEOUT USE_RR ROOTFS_TAG ROOTFS_HASH -# linux 686-linux-gnu . i686 . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 -linux x86_64-linux-gnu . x86_64 . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux x86_64-linux-gnuassert . x86_64 360 rr v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -# linux aarch64-linux-gnu true aarch64 . . ---- ---------------------------------------- -# linux armv7l-linux-gnueabihf true armv7l . . ---- ---------------------------------------- -# linux powerpc64le-linux-gnu true powerpc64le . . ---- ---------------------------------------- +# OS TRIPLET ALLOW_FAIL ARCH ARCH_ROOTFS TIMEOUT USE_RR ROOTFS_TAG ROOTFS_HASH +# linux i686-linux-gnu . x86_64 i686 . . ---- ---------------------------------------- +linux x86_64-linux-gnu . x86_64 x86_64 . . v5.9 2212d69a3bc8a516ae8731ac20771d90d62206bc +linux x86_64-linux-gnuassert . x86_64 x86_64 360 rr v5.9 2212d69a3bc8a516ae8731ac20771d90d62206bc +# linux aarch64-linux-gnu true aarch64 aarch64 . . ---- ---------------------------------------- +# linux armv7l-linux-gnueabihf true armv7l armv7l . . ---- ---------------------------------------- +# linux powerpc64le-linux-gnu true powerpc64le powerpc64le . . ---- ---------------------------------------- -musl x86_64-linux-musl true x86_64 . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 +musl x86_64-linux-musl true x86_64 x86_64 . . v5.9 417adcf6facf0933d496fa45a831a10632a7313f # These special lines allow us to embed default values for the columns above. # Any column without a default mapping here will simply substitute a `.` to the empty string diff --git a/pipelines/main/platforms/test_linux.yml b/pipelines/main/platforms/test_linux.yml index 6a55d8e8..c023362c 100644 --- a/pipelines/main/platforms/test_linux.yml +++ b/pipelines/main/platforms/test_linux.yml @@ -14,69 +14,23 @@ steps: persist_depot_dirs: packages,artifacts,compiled version: '1.6' - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${OS?}.${ARCH_ROOTFS?}.tar.gz - # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${OS?}.${ARCH_ROOTFS?}.tar.gz + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester_${OS?}.${ARCH_ROOTFS?}.tar.gz rootfs_treehash: "${ROOTFS_HASH?}" uid: 1000 gid: 1000 workspaces: # Include `/cache/repos` so that our `git` version introspection works. - "/cache/repos:/cache/repos" - env: - JULIA_SHELL: "/bin/bash" timeout_in_minutes: ${TIMEOUT?} soft_fail: ${ALLOW_FAIL?} - commands: | - # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc... - TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh - - echo "--- Download build artifacts" - buildkite-agent artifact download "$${UPLOAD_FILENAME}" . - - echo "--- Extract build artifacts" - tar xzf "$${UPLOAD_FILENAME}" "$${JULIA_INSTALL_DIR}/" - - echo "--- Print Julia version info" - $${JULIA_BINARY} -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - echo "JULIA_CPU_THREADS is: $${JULIA_CPU_THREADS}" - $${JULIA_BINARY} -e '@info "" Sys.CPU_THREADS' - - echo "--- Set some environment variables" - export OPENBLAS_NUM_THREADS=8 - unset JULIA_DEPOT_PATH - unset JULIA_PKG_SERVER - - # Make sure that temp files and temp directories are created in a location that is - # backed by real storage. - export TMPDIR="$(pwd)/tmp" - mkdir -p $${TMPDIR} - - # If we're running inside of `rr`, limit the number of threads - if [[ "${USE_RR?}" == "rr" ]]; then - export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY}" - export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])" - export JULIA_NUM_THREADS=1 - - # Don't run network tests on `rr`, as it causes the trace size to explode. - export TESTS="all --ci --skip Artifacts Downloads download LazyArtifacts LibGit2/online Pkg" - else - export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY}" - export NCORES_FOR_TESTS="Sys.CPU_THREADS" - export JULIA_NUM_THREADS=16 - export TESTS="all LibGit2/online --ci" - fi - - echo "--- Print the list of test sets, and other useful environment variables" - echo "JULIA_CMD_FOR_TESTS is: $${JULIA_CMD_FOR_TESTS:?}" - echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set - echo "NCORES_FOR_TESTS is: $${NCORES_FOR_TESTS:?}" - echo "OPENBLAS_NUM_THREADS is: $${OPENBLAS_NUM_THREADS:?}" - echo "TESTS is: $${TESTS:?}" - echo "USE_RR is: ${USE_RR?}" - - echo "--- Run the Julia test suite" - $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})" + commands: "bash .buildkite/utilities/test_julia.sh" agents: queue: "julia" sandbox_capable: "true" os: "linux" + arch: "${ARCH?}" + env: + JULIA_SHELL: "/bin/bash" + TRIPLET: "${TRIPLET?}" + USE_RR: "${USE_RR?}" + \ No newline at end of file diff --git a/pipelines/main/platforms/test_macos.yml b/pipelines/main/platforms/test_macos.yml index ae82b9ca..dd91cf81 100644 --- a/pipelines/main/platforms/test_macos.yml +++ b/pipelines/main/platforms/test_macos.yml @@ -7,52 +7,15 @@ steps: - "build_${TRIPLET?}" plugins: - JuliaCI/external-buildkite#v1: - version: ".buildkite-external-version" + version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" - env: - JULIA_SHELL: "/bin/bash" timeout_in_minutes: ${TIMEOUT?} soft_fail: ${ALLOW_FAIL?} - commands: | - # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc... - TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh - - echo "--- Download build artifacts" - buildkite-agent artifact download "$${UPLOAD_FILENAME}" . - - echo "--- Extract build artifacts" - tar xzf "$${UPLOAD_FILENAME}" "$${JULIA_INSTALL_DIR}/" - - echo "--- Ad-hoc sign for testing" - contrib/codesign.sh "-" "$${JULIA_INSTALL_DIR}" - - echo "--- Print Julia version info" - $${JULIA_BINARY} -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - echo "JULIA_CPU_THREADS is: $${JULIA_CPU_THREADS}" - $${JULIA_BINARY} -e '@info "" Sys.CPU_THREADS' - - echo "--- Set some environment variables" - export OPENBLAS_NUM_THREADS=4 - unset JULIA_DEPOT_PATH - unset JULIA_PKG_SERVER - - # By default, run all tests. - export TESTS="all LibGit2/online --ci" - - export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY}" - export NCORES_FOR_TESTS="Sys.CPU_THREADS" - - - echo "--- Print the list of test sets, and other useful environment variables" - echo "JULIA_CMD_FOR_TESTS is: $${JULIA_CMD_FOR_TESTS:?}" - echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set - echo "NCORES_FOR_TESTS is: $${NCORES_FOR_TESTS:?}" - echo "OPENBLAS_NUM_THREADS is: $${OPENBLAS_NUM_THREADS:?}" - echo "TESTS is: $${TESTS:?}" - - echo "--- Run the Julia test suite" - $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})" + commands: "bash .buildkite/utilities/test_julia.sh" agents: queue: "julia" os: "macos" arch: "${ARCH}" + env: + JULIA_SHELL: "/bin/bash" + TRIPLET: "${TRIPLET?}" diff --git a/pipelines/main/platforms/upload_linux.yml b/pipelines/main/platforms/upload_linux.yml index e1fff458..4add5730 100644 --- a/pipelines/main/platforms/upload_linux.yml +++ b/pipelines/main/platforms/upload_linux.yml @@ -15,7 +15,7 @@ steps: # Prevent multiple pipelines from uploading to S3 simultaneously # It is okay for two different triplets to upload simultaneously concurrency: 1 - concurrency_group: 'upload/julialangnightlies/upload_linux/${TRIPLET?}' + concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' plugins: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" @@ -25,8 +25,8 @@ steps: persist_depot_dirs: packages,artifacts,compiled version: '1' - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.4/aws_uploader.x86_64.tar.gz - rootfs_treehash: "a3a152e5cdb00ef12abfc22351449ae62f7085ae" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.7/aws_uploader.x86_64.tar.gz + rootfs_treehash: "0d5ad94ba902eea4d9a7d220b49c144626a102ea" uid: 1000 gid: 1000 - staticfloat/cryptic#v2: @@ -36,34 +36,7 @@ steps: files: - .buildkite/secrets/tarball_signing.gpg timeout_in_minutes: ${TIMEOUT?} - commands: | - # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc... - TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh - - echo "--- Download $${UPLOAD_FILENAME} to ." - buildkite-agent artifact download "$${UPLOAD_FILENAME}" . - - echo "--- GPG-sign the tarball" - .buildkite/utilities/sign_tarball.sh .buildkite/secrets/tarball_signing.gpg "$${UPLOAD_FILENAME}" - - # We first upload the canonical fully-specified upload target, which is the first one: - echo "--- Upload tarballs and signatures to S3" - aws s3 cp --acl public-read "$${UPLOAD_FILENAME}" "s3://$${UPLOAD_TARGETS[0]}" - aws s3 cp --acl public-read "$${UPLOAD_FILENAME}.asc" "s3://$${UPLOAD_TARGETS[0]}.asc" - - echo "--- Copy to secondary upload targets" - # We'll do these in parallel, then wait on the background jobs - for SECONDARY_TARGET in $${UPLOAD_TARGETS[@]:1}; do - aws s3 cp --acl public-read "s3://$${UPLOAD_TARGETS[0]}" "s3://$${SECONDARY_TARGET}" & - aws s3 cp --acl public-read "s3://$${UPLOAD_TARGETS[0]}.asc" "s3://$${SECONDARY_TARGET}.asc" & - done - wait - - # Report to the user some URLs that they can use to download this from - echo "+++ Uploaded to targets" - for UPLOAD_TARGET in $${UPLOAD_TARGETS[@]}; do - echo " -> s3://$${UPLOAD_TARGET}" - done + commands: "bash .buildkite/utilities/upload_julia.sh" agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -72,3 +45,4 @@ steps: env: # Receive cryptic token from parent job BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} + TRIPLET: "${TRIPLET?}" diff --git a/pipelines/main/platforms/upload_macos.yml b/pipelines/main/platforms/upload_macos.yml index 2aa7aadd..3bcb6cd5 100644 --- a/pipelines/main/platforms/upload_macos.yml +++ b/pipelines/main/platforms/upload_macos.yml @@ -15,7 +15,7 @@ steps: # Prevent multiple pipelines from uploading to S3 simultaneously # It is okay for two different triplets to upload simultaneously concurrency: 1 - concurrency_group: 'upload/julialangnightlies/upload_macos/${TRIPLET?}' + concurrency_group: 'upload/julialangnightlies/${TRIPLET?}' plugins: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" @@ -27,103 +27,8 @@ steps: files: - .buildkite/secrets/tarball_signing.gpg - .buildkite/secrets/macos_codesigning.keychain - - .buildkite/secrets/keychain_password.txt timeout_in_minutes: ${TIMEOUT?} - commands: | - # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc... - TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh - - echo "--- Variables for the dmg" - MACOS_CODESIGN_IDENTITY="2053E9292809B66582CA9F042B470C0929340362" - APP_NAME="Julia-$${MAJMIN}.app" - DMG_NAME="$$(basename $$JULIA_BINARYDIST_FILENAME .tar.gz).dmg" - APP_PATH="dmg/$${APP_NAME}" - MAC_PATH="contrib/mac/app" - KEYCHAIN_PATH="$$(pwd)/.buildkite/secrets/macos_codesigning.keychain" - APP_ID="org.julialang.launcherapp" - APP_COPYRIGHT="$$(date '+%Y') The Julia Project" - JULIA_VERSION_OPT_COMMIT="$${JULIA_VERSION}-$${SHORT_COMMIT}" - VOL_NAME="Julia-$${JULIA_VERSION_OPT_COMMIT}" - KEYCHAIN_PASSWORD="$$(cat .buildkite/secrets/keychain_password.txt)" - - echo "--- Download $${UPLOAD_FILENAME} to ." - buildkite-agent artifact download "$${UPLOAD_FILENAME}" . - - #Get the current keychains and put into a list - echo "--- Add keychain to list and unlock it" - security -v list-keychains -s -d user "$${KEYCHAIN_PATH}" - security unlock-keychain -p "$${KEYCHAIN_PASSWORD}" "$${KEYCHAIN_PATH}" - - security find-identity "$${MACOS_CODESIGN_IDENTITY}" "$${KEYCHAIN_PATH}" - security -v list-keychains -d user - - echo "--- Prepare .app file" - mkdir -p dmg - osacompile -o "$${APP_PATH}" "$${MAC_PATH}/startup.applescript" - rm "$${APP_PATH}/Contents/Resources/applet.icns" - cp "$${MAC_PATH}/julia.icns" "$${APP_PATH}/Contents/Resources/" - plutil -replace CFBundleDevelopmentRegion -string "en" "$${APP_PATH}/Contents/Info.plist" - plutil -insert CFBundleDisplayName -string "Julia" "$${APP_PATH}/Contents/Info.plist" - plutil -replace CFBundleIconFile -string "julia.icns" "$${APP_PATH}/Contents/Info.plist" - plutil -insert CFBundleIdentifier -string "$${APP_ID}" "$${APP_PATH}/Contents/Info.plist" - plutil -replace CFBundleName -string "Julia" "$${APP_PATH}/Contents/Info.plist" - plutil -insert CFBundleShortVersionString -string "$${MAJMINPAT}" "$${APP_PATH}/Contents/Info.plist" - plutil -insert CFBundleVersion -string "$${JULIA_VERSION_OPT_COMMIT}" "$${APP_PATH}/Contents/Info.plist" - plutil -insert NSHumanReadableCopyright -string "$${APP_COPYRIGHT}" "$${APP_PATH}/Contents/Info.plist" - mkdir -p "$${APP_PATH}/Contents/Resources/julia" - - echo "--- Extract build artifacts" - tar zxf "$${UPLOAD_FILENAME}" -C "$${APP_PATH}/Contents/Resources/julia" --strip-components 1 - - find $${APP_PATH}/Contents/Resources/julia -type f -exec chmod -w {} \; - echo "--- Codesign the files" - cp -f "$${MAC_PATH}/julia.icns" dmg/.VolumeIcon.icns - ln -fs /Applications dmg/Applications - .buildkite/utilities/macos/codesign.sh --keychain "$${KEYCHAIN_PATH}" --identity "$${MACOS_CODESIGN_IDENTITY}" "$${APP_PATH}/Contents/Resources/julia" - codesign -s $$MACOS_CODESIGN_IDENTITY "$${APP_PATH}/Contents/MacOS/applet" --keychain "$${KEYCHAIN_PATH}" --force - echo "--- create dmg" - hdiutil create $$DMG_NAME -size 1t -fs HFS+ -ov -volname "$${VOL_NAME}" -imagekey zlib-level=9 -srcfolder dmg - codesign -s $$MACOS_CODESIGN_IDENTITY --timestamp -i org.julialang.launcherapp $$DMG_NAME --keychain "$${KEYCHAIN_PATH}" --force - - # remake tar file after signing - mkdir "$${JULIA_INSTALL_DIR}" - cp -R "$${APP_PATH}/Contents/Resources/julia/" "$${JULIA_INSTALL_DIR}" - rm -rf "$${UPLOAD_FILENAME}" - tar zcvf "$${UPLOAD_FILENAME}" "$${JULIA_INSTALL_DIR}" - - echo "--- GPG-sign the tarball" - .buildkite/utilities/sign_tarball.sh .buildkite/secrets/tarball_signing.gpg "$${UPLOAD_FILENAME}" - - echo "--- Upload signed packages to buildkite" - buildkite-agent artifact upload "$${DMG_NAME}" - buildkite-agent artifact upload "$${UPLOAD_FILENAME}" - buildkite-agent artifact upload "$${UPLOAD_FILENAME}.asc" - - # We first upload the canonical fully-specified upload target, which is the first one: - echo "--- Upload tarballs, dmgs and signatures to S3" - DMG_TARGET="$$(dirname $$UPLOAD_TARGETS[0])/$$DMG_NAME" - aws s3 cp --acl public-read "$${UPLOAD_FILENAME}" "s3://$${UPLOAD_TARGETS[0]}" - aws s3 cp --acl public-read "$${UPLOAD_FILENAME}.asc" "s3://$${UPLOAD_TARGETS[0]}.asc" - aws s3 cp --acl public-read "$${DMG_NAME}" "s3://$${DMG_TARGET}" - - echo "--- Copy to secondary upload targets" - # We'll do these in parallel, then wait on the background jobs - for SECONDARY_TARGET in $${UPLOAD_TARGETS[@]:1}; do - SECONDARY_DMG_TARGET=""$$(dirname $$SECONDARY_TARGET)/$$DMG_NAME"" - aws s3 cp --acl public-read "s3://$${DMG_TARGET}" "s3://$${SECONDARY_DMG_TARGET}" & - aws s3 cp --acl public-read "s3://$${UPLOAD_TARGETS[0]}" "s3://$${SECONDARY_TARGET}" & - aws s3 cp --acl public-read "s3://$${UPLOAD_TARGETS[0]}.asc" "s3://$${SECONDARY_TARGET}.asc" & - done - wait - - # Report to the user some URLs that they can use to download this from - echo "+++ Uploaded to targets" - for UPLOAD_TARGET in $${UPLOAD_TARGETS[@]}; do - echo " -> s3://$${UPLOAD_TARGET}" - echo " -> s3://$$(dirname $$UPLOAD_TARGET)/$$DMG_NAME" - done - - security -v delete-keychain "$$KEYCHAIN_PATH" + commands: "bash .buildkite/utilities/upload_julia.sh" agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -132,3 +37,4 @@ steps: env: # Receive cryptic token from parent job BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} + TRIPLET: "${TRIPLET?}" diff --git a/pipelines/scheduled/coverage/coverage_linux64.yml.signature b/pipelines/scheduled/coverage/coverage_linux64.yml.signature index 9afd823c..f46fcc94 100644 Binary files a/pipelines/scheduled/coverage/coverage_linux64.yml.signature and b/pipelines/scheduled/coverage/coverage_linux64.yml.signature differ diff --git a/secrets/keychain_password.txt.encrypted b/secrets/keychain_password.txt.encrypted deleted file mode 100644 index 5d55017b..00000000 --- a/secrets/keychain_password.txt.encrypted +++ /dev/null @@ -1 +0,0 @@ -Salted__��TT���c�C����X����K��9Z�$�T� �.��t|| \ No newline at end of file diff --git a/utilities/calc_version_envs.sh b/utilities/build_envs.sh similarity index 54% rename from utilities/calc_version_envs.sh rename to utilities/build_envs.sh index 4013c83d..46c43796 100755 --- a/utilities/calc_version_envs.sh +++ b/utilities/build_envs.sh @@ -28,15 +28,76 @@ case "${TRIPLET}" in esac export OS +# Determine JULIA_CPU_TARGETS for different architectures +JUlIA_CPU_TARGETS=() +case "${ARCH?}" in + x86_64) + JULIA_CPU_TARGETS+=( + # Absolute base x86_64 feature set + "generic" + # Add sandybridge level (without xsaveopt) and that clones all functions + "sandybridge,-xsaveopt,clone_all" + # Add haswell level (without rdrnd) that is a diff of the sandybridge level + "haswell,-rdrnd,base(1)" + ) + ;; + i686) + JULIA_CPU_TARGETS+=( + # We require SSE2, etc.. so `pentium4` is our base i686 feature set + "pentium4" + # Add sandybridge level similar to x86_64 above + "sandybridge,-xsaveopt,clone_all" + ) + ;; + armv7l) + JULIA_CPU_TARGETS+=( + # Absolute base armv7-a feature set + "armv7-a" + # Add NEON level on top of that + "armv7-a,neon" + # Add NEON with VFP4 on top of that + "armv7-a,neon,vfp4" + ) + ;; + aarch64) + JULIA_CPU_TARGETS+=( + # Absolute base aarch64 feature set + "generic" + # Cortex A57, Example: NVIDIA Jetson TX1, Jetson Nano + "cortex-a57" + # Cavium ThunderX2T99, a common server architecture + "thunderx2t99" + # NVidia Carmel, e.g. Jetson AGX Xavier + "carmel" + ) + ;; + powerpc64le) + JULIA_CPU_TARGETS+=( + # Absolute base POWER-8 feature set + "pwr8" + ) + ;; + *) + echo "Unknown target processor architecture '${ARCH}'" >&2 + exit 1 + ;; +esac + +# Join and output +JULIA_CPU_TARGET="$(printf ";%s" "${JULIA_CPU_TARGETS[@]}")" +export JULIA_CPU_TARGET="${JULIA_CPU_TARGET:1}" + + + # Extract git information SHORT_COMMIT_LENGTH=10 -LONG_COMMIT="$(git rev-parse HEAD)" -SHORT_COMMIT="$(echo ${LONG_COMMIT} | cut -c1-${SHORT_COMMIT_LENGTH})" +export LONG_COMMIT="$(git rev-parse HEAD)" +export SHORT_COMMIT="$(echo ${LONG_COMMIT} | cut -c1-${SHORT_COMMIT_LENGTH})" # Extract information about the current julia version number -JULIA_VERSION="$(cat VERSION)" -MAJMIN="${JULIA_VERSION:0:3}" -MAJMINPAT="${JULIA_VERSION:0:5}" +export JULIA_VERSION="$(cat VERSION)" +export MAJMIN="$(cut -d. -f1-2 <<<"${JULIA_VERSION}")" +export MAJMINPAT="$(cut -d- -f1 <<<"${JULIA_VERSION}")" # If we're on a tag, then our "tar version" will be the julia version. # Otherwise, it's the short commit. if git describe --tags --exact-match >/dev/null 2>/dev/null; then @@ -44,13 +105,16 @@ if git describe --tags --exact-match >/dev/null 2>/dev/null; then else TAR_VERSION="${SHORT_COMMIT}" fi +export TAR_VERSION + + # Build the filename that we'll upload as, and get the filename that will be built # These are not the same in situations such as `musl`, where the build system doesn't # differentiate but we need to give it a different name. -JULIA_BINARYDIST_FILENAME="$(make print-JULIA_BINARYDIST_FILENAME | cut -c27- | tr -s ' ').tar.gz" +export JULIA_BINARYDIST_FILENAME="$(make print-JULIA_BINARYDIST_FILENAME | cut -c27- | tr -s ' ')" -JULIA_INSTALL_DIR="julia-${TAR_VERSION}" +export JULIA_INSTALL_DIR="julia-${TAR_VERSION}" JULIA_BINARY="${JULIA_INSTALL_DIR}/bin/julia" # By default, we upload to `julialangnightlies/bin`, but we allow this to be overridden @@ -60,46 +124,48 @@ S3_BUCKET_PREFIX="${S3_BUCKET_PREFIX:-bin}" # We generally upload to multiple upload targets UPLOAD_TARGETS=( # First, we have the canonical fully-specified upload target - "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}.tar.gz" + "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}" # Next, we have the "majmin/latest" upload target - "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/${MAJMIN?}/julia-latest-${OS?}-${ARCH?}.tar.gz" + "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/${MAJMIN?}/julia-latest-${OS?}-${ARCH?}" ) # The absolute latest upload target is only for if we're on the `master` branch if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/julia-latest-${OS?}-${ARCH?}.tar.gz" ) + UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/julia-latest-${OS?}-${ARCH?}" ) fi -UPLOAD_FILENAME="julia-${TAR_VERSION?}-${OS?}-${ARCH?}.tar.gz" + # Finally, for compatibility, we keep on uploading x86_64 and i686 targets to folders called `x64` # and `x86`, and ending in `-linux64` and `-linux32`, although I would very much like to stop doing that. if [[ "${ARCH}" == "x86_64" ]]; then - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}64.tar.gz" ) - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/${MAJMIN?}/julia-latest-${OS?}64.tar.gz" ) + UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}64" ) + UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/${MAJMIN?}/julia-latest-${OS?}64" ) # Only upload to absolute latest if we're on `master` if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/julia-latest-${OS?}64.tar.gz" ) + UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/julia-latest-${OS?}64" ) fi elif [[ "${ARCH}" == "i686" ]]; then - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}32.tar.gz" ) - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/${MAJMIN?}/julia-latest-${OS?}32.tar.gz" ) + UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}32" ) + UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/${MAJMIN?}/julia-latest-${OS?}32" ) # Only upload to absolute latest if we're on `master` if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then - UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/julia-latest-${OS?}32.tar.gz" ) + UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/julia-latest-${OS?}32" ) fi fi +export UPLOAD_FILENAME="julia-${TAR_VERSION?}-${OS?}-${ARCH?}" + echo "--- Print the full and short commit hashes" echo "The full commit is: ${LONG_COMMIT}" echo "The short commit is: ${SHORT_COMMIT}" echo "Julia will be installed to: ${JULIA_BINARY}" echo "Detected Julia version: ${MAJMIN} (${JULIA_VERSION})" echo "Detected build platform: ${TRIPLET} (${ARCH}, ${OS})" -echo "Julia will be uploaded to: s3://${UPLOAD_TARGETS[0]}" +echo "Julia will be uploaded to: s3://${UPLOAD_TARGETS[0]}.tar.gz" echo "With additional upload targets:" for UPLOAD_TARGET in ${UPLOAD_TARGETS[@]:1}; do - echo " -> s3://${UPLOAD_TARGET}" + echo " -> s3://${UPLOAD_TARGET}.tar.gz" done diff --git a/utilities/build_julia.sh b/utilities/build_julia.sh new file mode 100644 index 00000000..a8d3264e --- /dev/null +++ b/utilities/build_julia.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# This script performs the basic steps needed to build Julia from source +# It requires the following environment variables to be defined: +# - TRIPLET +# - MAKE_FLAGS +set -euo pipefail + +# First, get things like `SHORT_COMMIT`, `JULIA_CPU_TARGET`, `UPLOAD_TARGETS`, etc... +source .buildkite/utilities/build_envs.sh + +echo "--- Collect make options" +# These are the flags we'll provide to `make` +MFLAGS=() + +# If we have the option, let's use `--output-sync` +if make --help | grep output-sync >/dev/null 2>/dev/null; then + MFLAGS+=( "--output-sync" ) +fi + +# Always use this much parallelism +MFLAGS+=( "-j${JULIA_CPU_THREADS}") + +# Add a few default flags to our make flags: +MFLAGS+=( "VERBOSE=1" ) +MFLAGS+=( "TAGGED_RELEASE_BANNER=Official https://julialang.org/ release" ) +MFLAGS+=( "JULIA_CPU_TARGET=${JULIA_CPU_TARGET}" ) + +# Finish off with any extra make flags from the `.arches` file +MFLAGS+=( $(tr "," " " <<<"${MAKE_FLAGS}") ) + +echo "Make Options:" +for FLAG in "${MFLAGS[@]}"; do + echo " -> ${FLAG}" +done + +echo "--- Build Julia" +make "${MFLAGS[@]}" + +echo "--- Check that the working directory is clean" +if [ -n "$(git status --short)" ]; then + echo "ERROR: The working directory is dirty." >&2 + echo "Output of git status:" >&2 + git status + exit 1 +fi + +echo "--- Print Julia version info" +./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' + + +echo "--- Create build artifacts" +make "${MFLAGS[@]}" binary-dist + + +# Rename the build artifact in case we want to name it differently, as is the case on `musl`. +if [[ "${JULIA_BINARYDIST_FILENAME}.tar.gz" != "${UPLOAD_FILENAME}.tar.gz" ]]; then + mv "${JULIA_BINARYDIST_FILENAME}.tar.gz" "${UPLOAD_FILENAME}.tar.gz" +fi + + +echo "--- Upload build artifacts to buildkite" +buildkite-agent artifact upload "${UPLOAD_FILENAME}.tar.gz" diff --git a/utilities/export_julia_cpu_target.sh b/utilities/export_julia_cpu_target.sh deleted file mode 100755 index 53f9c1c0..00000000 --- a/utilities/export_julia_cpu_target.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Determine JULIA_CPU_TARGETS for different architectures -JUlIA_CPU_TARGETS=() -case "${ARCH?}" in - x86_64) - JULIA_CPU_TARGETS+=( - # Absolute base x86_64 feature set - "generic" - # Add sandybridge level (without xsaveopt) and that clones all functions - "sandybridge,-xsaveopt,clone_all" - # Add haswell level (without rdrnd) that is a diff of the sandybridge level - "haswell,-rdrnd,base(1)" - ) - ;; - i686) - JULIA_CPU_TARGETS+=( - # We require SSE2, etc.. so `pentium4` is our base i686 feature set - "pentium4" - # Add sandybridge level similar to x86_64 above - "sandybridge,-xsaveopt,clone_all" - ) - ;; - armv7l) - JULIA_CPU_TARGETS+=( - # Absolute base armv7-a feature set - "armv7-a" - # Add NEON level on top of that - "armv7-a,neon" - # Add NEON with VFP4 on top of that - "armv7-a,neon,vfp4" - ) - ;; - aarch64) - JULIA_CPU_TARGETS+=( - # Absolute base aarch64 feature set - "generic" - # Cortex A57, Example: NVIDIA Jetson TX1, Jetson Nano - "cortex-a57" - # Cavium ThunderX2T99, a common server architecture - "thunderx2t99" - # NVidia Carmel, e.g. Jetson AGX Xavier - "carmel" - ) - ;; - powerpc64le) - JULIA_CPU_TARGETS+=( - # Absolute base POWER-8 feature set - "pwr8" - ) - ;; - *) - echo "Unknown target processor architecture '${ARCH}'" >&2 - exit 1 - ;; -esac - -# Join and output -JULIA_CPU_TARGET="$(printf ";%s" "${JULIA_CPU_TARGETS[@]}")" -export JULIA_CPU_TARGET="${JULIA_CPU_TARGET:1}" diff --git a/utilities/macos/build_dmg.sh b/utilities/macos/build_dmg.sh new file mode 100755 index 00000000..725cb0d1 --- /dev/null +++ b/utilities/macos/build_dmg.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -euo pipefail + +DMG_PATH="dmg" +mkdir -p "${DMG_PATH}" +APP_PATH="${DMG_PATH}/Julia-${MAJMIN?}.app" +DMG_NAME="${UPLOAD_FILENAME?}.dmg" + +# Start by compiling an applescript into a `.app`, which creates the skeleton, which we will fill out +osacompile -o "${APP_PATH}" "contrib/mac/app/startup.applescript" + +# Use `plutil` to fill out the `Info.plist` appropriately +plutil -replace CFBundleDevelopmentRegion -string "en" "${APP_PATH}/Contents/Info.plist" +plutil -insert CFBundleDisplayName -string "Julia" "${APP_PATH}/Contents/Info.plist" +plutil -replace CFBundleIconFile -string "julia.icns" "${APP_PATH}/Contents/Info.plist" +plutil -insert CFBundleIdentifier -string "org.julialang.launcherapp" "${APP_PATH}/Contents/Info.plist" +plutil -replace CFBundleName -string "Julia" "${APP_PATH}/Contents/Info.plist" +plutil -insert CFBundleShortVersionString -string "${MAJMINPAT?}" "${APP_PATH}/Contents/Info.plist" +plutil -insert CFBundleVersion -string "${JULIA_VERSION?}-${SHORT_COMMIT?}" "${APP_PATH}/Contents/Info.plist" +plutil -insert NSHumanReadableCopyright -string "$(date '+%Y') The Julia Project" "${APP_PATH}/Contents/Info.plist" + +# Add icon file for the application and the .dmg +cp "contrib/mac/app/julia.icns" "${APP_PATH}/Contents/Resources/" +cp "contrib/mac/app/julia.icns" "${DMG_PATH}/.VolumeIcon.icns" + +# Add link to `/Applications` +ln -s /Applications "${DMG_PATH}/Applications" + +# Copy our signed tarball into the `.dmg` +cp -aR "${JULIA_INSTALL_DIR?}" "${APP_PATH}/Contents/Resources/julia" + +# Sign the `.app` launcher +.buildkite/utilities/macos/codesign.sh \ + --keychain "${KEYCHAIN_PATH}" \ + --identity "${MACOS_CODESIGN_IDENTITY}" \ + "${APP_PATH}/Contents/MacOS/applet" + +# Create `.dmg`. We create it with 1TB size, but since that is +# a maximum, it has no effect on download or unpack size. +hdiutil create \ + "${DMG_NAME}" \ + -size 1t \ + -fs HFS+ \ + -volname "Julia-${TAR_VERSION?}" \ + -imagekey zlib-level=9 \ + -srcfolder "${DMG_PATH}" + +# Sign the `.dmg` itself +.buildkite/utilities/macos/codesign.sh \ + --keychain "${KEYCHAIN_PATH}" \ + --identity "${MACOS_CODESIGN_IDENTITY}" \ + "${DMG_NAME}" + +rm -rf "${DMG_PATH}" diff --git a/utilities/test_julia.sh b/utilities/test_julia.sh new file mode 100644 index 00000000..159eda1d --- /dev/null +++ b/utilities/test_julia.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# This script performs the basic steps needed to test Julia previously +# built and uploaded as a `.tar.gz`. +# It requires the following environment variables to be defined: +# - TRIPLET +# - USE_RR +set -euo pipefail + +# First, get things like `SHORT_COMMIT`, `JULIA_CPU_TARGET`, `UPLOAD_TARGETS`, etc... +source .buildkite/utilities/build_envs.sh + +echo "--- Download build artifacts" +buildkite-agent artifact download "${UPLOAD_FILENAME}.tar.gz" . + +echo "--- Extract build artifacts" +tar xzf "${UPLOAD_FILENAME}.tar.gz" "${JULIA_INSTALL_DIR}/" + +# If we're on macOS, we need to re-sign the downloaded tarball so it will +# execute on this machine +if [[ "${OS}" == "macos" ]]; then + .buildkite/utilities/macos/codesign.sh "${JULIA_INSTALL_DIR}" +fi + + +echo "--- Print Julia version info" +${JULIA_BINARY} -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' + + +echo "--- Set some environment variables" +# Prevent OpenBLAS from spinning up a large number of threads on our big machines +export OPENBLAS_NUM_THREADS="${JULIA_CPU_THREADS}" +unset JULIA_DEPOT_PATH +unset JULIA_PKG_SERVER + +# Make sure that temp files and temp directories are created in a location that is +# backed by real storage, and not by a tmpfs, as some tests don't like that on Linux +if [[ "${OS}" == "linux" ]]; then + export TMPDIR="$(pwd)/tmp" + mkdir -p ${TMPDIR} +fi + +# If we're running inside of `rr`, limit the number of threads +if [[ "${USE_RR-}" == "rr" ]]; then + export JULIA_CMD_FOR_TESTS="${JULIA_BINARY} .buildkite/utilities/rr/rr_capture.jl ${JULIA_BINARY}" + export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])" + export JULIA_NUM_THREADS=1 + + # Don't run network tests on `rr`, as it causes the trace size to explode. + export TESTS="all --ci --skip Artifacts Downloads download LazyArtifacts LibGit2/online Pkg" +else + export JULIA_CMD_FOR_TESTS="${JULIA_BINARY}" + export NCORES_FOR_TESTS="${JULIA_CPU_THREADS}" + export JULIA_NUM_THREADS="${JULIA_CPU_THREADS}" + + # By default, run all tests + export TESTS="all LibGit2/online --ci" +fi + +echo "--- Print the list of test sets, and other useful environment variables" +echo "JULIA_CMD_FOR_TESTS is: ${JULIA_CMD_FOR_TESTS:?}" +echo "JULIA_NUM_THREADS is: ${JULIA_NUM_THREADS:?}" +echo "NCORES_FOR_TESTS is: ${NCORES_FOR_TESTS:?}" +echo "OPENBLAS_NUM_THREADS is: ${OPENBLAS_NUM_THREADS:?}" +echo "TESTS is: ${TESTS:?}" +echo "USE_RR is: ${USE_RR-}" + +echo "--- Run the Julia test suite" +${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"${TESTS:?}\"; ncores = ${NCORES_FOR_TESTS:?})" diff --git a/utilities/upload_julia.sh b/utilities/upload_julia.sh new file mode 100644 index 00000000..0e272293 --- /dev/null +++ b/utilities/upload_julia.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# This script performs the basic steps needed to sign and upload a +# Julia previously built and uploaded as a `.tar.gz`. +# It requires the following environment variables to be defined: +# - TRIPLET +set -euo pipefail + +# First, get things like `SHORT_COMMIT`, `JULIA_CPU_TARGET`, `UPLOAD_TARGETS`, etc... +source .buildkite/utilities/build_envs.sh + +echo "--- Download ${UPLOAD_FILENAME}.tar.gz to ." +buildkite-agent artifact download "${UPLOAD_FILENAME}.tar.gz" . + +# These are the extensions that we will always upload +UPLOAD_EXTENSIONS=("tar.gz" "tar.gz.asc") + +# If we're on macOS, we need to re-sign the tarball +if [[ "${OS}" == "macos" ]]; then + echo "--- [mac] Unlock keychain" + + # This _must_ be an absolute path + KEYCHAIN_PATH="$(pwd)/.buildkite/secrets/macos_codesigning.keychain" + MACOS_CODESIGN_IDENTITY="2053E9292809B66582CA9F042B470C0929340362" + + # Add the keychain to the list of keychains to search, then unlock it + security -v list-keychains -s -d user "${KEYCHAIN_PATH}" + security unlock-keychain -p "keychainpassword" "${KEYCHAIN_PATH}" + security find-identity -p codesigning "${KEYCHAIN_PATH}" + + echo "--- [mac] Codesign tarball contents" + mkdir -p "${JULIA_INSTALL_DIR}" + tar zxf "${UPLOAD_FILENAME}.tar.gz" -C "${JULIA_INSTALL_DIR}" --strip-components 1 + .buildkite/utilities/macos/codesign.sh \ + --keychain "${KEYCHAIN_PATH}" \ + --identity "${MACOS_CODESIGN_IDENTITY}" \ + "${JULIA_INSTALL_DIR}" + + # Immediately re-compress that tarball for upload + echo "--- [mac] Re-compress codesigned tarball" + rm -f "${UPLOAD_FILENAME}.tar.gz" + tar zcf "${UPLOAD_FILENAME}.tar.gz" "${JULIA_INSTALL_DIR}" + + # Make a `.dmg` out of those files + echo "--- [mac] Build .dmg" + KEYCHAIN_PATH="${KEYCHAIN_PATH}" MACOS_CODESIGN_IDENTITY="${MACOS_CODESIGN_IDENTITY}" \ + .buildkite/utilities/macos/build_dmg.sh + + # Add the `.dmg` to our upload targets + UPLOAD_EXTENSIONS+=( "dmg" ) +fi + +echo "--- GPG-sign the tarball" +.buildkite/utilities/sign_tarball.sh .buildkite/secrets/tarball_signing.gpg "${UPLOAD_FILENAME}.tar.gz" + +# First, upload our signed products to buildkite, for easy downloading +echo "--- Upload signed products to buildkite" +for EXT in "${UPLOAD_EXTENSIONS[@]}"; do + buildkite-agent artifact upload "${UPLOAD_FILENAME}.${EXT}" & +done +wait + +# Next, upload primary files to S3 +echo "--- Upload primary products to S3" +for EXT in "${UPLOAD_EXTENSIONS[@]}"; do + aws s3 cp --acl public-read "${UPLOAD_FILENAME}.${EXT}" "s3://${UPLOAD_TARGETS[0]}.${EXT}" & +done +wait + +echo "--- Copy to secondary upload targets" +# We'll do these in parallel, then wait on the background jobs +for SECONDARY_TARGET in ${UPLOAD_TARGETS[@]:1}; do + for EXT in "${UPLOAD_EXTENSIONS[@]}"; do + aws s3 cp --acl public-read "s3://${UPLOAD_TARGETS[0]}.${EXT}" "s3://${SECONDARY_TARGET}.${EXT}" & + done +done +wait + +# Report to the user some URLs that they can use to download this from +echo "+++ Uploaded to targets" +for UPLOAD_TARGET in ${UPLOAD_TARGETS[@]}; do + for EXT in "${UPLOAD_EXTENSIONS[@]}"; do + echo " -> s3://${UPLOAD_TARGET}.${EXT}" + done +done