Skip to content

Commit

Permalink
More reorganization and simplification
Browse files Browse the repository at this point in the history
  • Loading branch information
staticfloat committed Mar 8, 2022
1 parent 9e510eb commit 85ae354
Show file tree
Hide file tree
Showing 11 changed files with 131 additions and 43 deletions.
1 change: 1 addition & 0 deletions pipelines/main/0_webui.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,5 @@ steps:
signed_pipelines:
- pipeline: .buildkite/pipelines/main/launch_signed_jobs.yml
signature_file: .buildkite/pipelines/main/launch_signed_jobs.yml.signature
allow_hash_override: true
command: "true"
3 changes: 0 additions & 3 deletions pipelines/main/launch_signed_jobs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,6 @@ steps:
# Explicitly pass along the cryptic token to child pipelines
export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET
# Determine the julia version
export JULIA_VERSION=$$(cat VERSION)
# Launch `upload_linux` jobs to store tarballs into S3 once tests are done
bash .buildkite/utilities/arches_pipeline_upload.sh \
.buildkite/pipelines/main/platforms/build_linux.arches \
Expand Down
3 changes: 2 additions & 1 deletion pipelines/main/launch_signed_jobs.yml.signature
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
Salted__ �#z:G�Lj�]�"��u4�`�,�-Ӱ�@�b_7�m�{�;���$����@�� ��VJ��<����ګs�+6ū������| ���
Salted__�3�Lx�
K��xڳ5g�x�ڨ !RN�,V5��N��?|�4����@���+7��7���F"� �_3l#2�S��J9�:ӣNi�
Expand Down
14 changes: 7 additions & 7 deletions pipelines/main/platforms/build_linux.arches
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
# OS TRIPLET ALLOW_FAIL TAR_ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT ROOTFS_TAG ROOTFS_HASH
# linux i686-linux-gnu . 32 i686 . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
linux x86_64-linux-gnu . 64 x86_64 . . v5.1 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0
# linux aarch64-linux-gnu . aarch64 aarch64 . . ---- ----------------------------------------
# linux armv7l-linux-gnueabihf . armv7l armv7l . . ---- ----------------------------------------
# linux powerpc64le-linux-gnu . ppc64le powerpc64le . . ---- ----------------------------------------
# musl x86_64-linux-musl . 64 x86_64 . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90
# TRIPLET ALLOW_FAIL ARCH_ROOTFS MAKE_FLAGS TIMEOUT ROOTFS_TAG ROOTFS_HASH
# i686-linux-gnu . i686 . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
x86_64-linux-gnu . x86_64 . . v5.1 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0
# aarch64-linux-gnu . aarch64 . . ---- ----------------------------------------
# armv7l-linux-gnueabihf . armv7l . . ---- ----------------------------------------
# powerpc64le-linux-gnu . powerpc64le . . ---- ----------------------------------------
# x86_64-linux-musl . x86_64 . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90


# These special lines allow us to embed default values for the columns above.
Expand Down
11 changes: 5 additions & 6 deletions pipelines/main/platforms/build_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,9 @@ steps:
timeout_in_minutes: ${TIMEOUT?}
commands: |
# First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
TAR_ARCH=${TAR_ARCH?} OS=${OS?} MAKE_FLAGS=${MAKE_FLAGS?} source .buildkite/utilities/calc_version_envs.sh
TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh
echo "--- Build Julia from source"
rm -rf $${ARTIFACT_FILENAME}
echo "--- Build Julia"
make --output-sync -j 8 ${MAKE_FLAGS?}
echo "--- Check that the working directory is clean"
Expand All @@ -45,12 +44,12 @@ steps:
make --output-sync -j 8 binary-dist ${MAKE_FLAGS?}
# Rename the build artifact in case we want to name it differently, as is the case on `musl`.
if [[ "$${JULIA_BINARYDIST}" != "$${ARTIFACT_FILENAME}" ]]; then
mv $${JULIA_BINARYDIST} $${ARTIFACT_FILENAME}
if [[ "$${JULIA_BINARYDIST_FILENAME}" != "$${UPLOAD_FILENAME}" ]]; then
mv $${JULIA_BINARYDIST_FILENAME} $${UPLOAD_FILENAME}
fi
echo "--- Upload build artifacts"
buildkite-agent artifact upload $${ARTIFACT_FILENAME}
buildkite-agent artifact upload $${UPLOAD_FILENAME}
agents:
queue: "julia"
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
Expand Down
16 changes: 8 additions & 8 deletions pipelines/main/platforms/test_linux.arches
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
# OS TRIPLET ALLOW_FAIL TAR_ARCH ARCH_ROOTFS TIMEOUT USE_RR ROOTFS_TAG ROOTFS_HASH
# linux 686-linux-gnu . 32 i686 . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
linux x86_64-linux-gnu . 64 x86_64 . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
linux x86_64-linux-gnu . 64 x86_64 120 rr v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
# TRIPLET ALLOW_FAIL ARCH_ROOTFS TIMEOUT USE_RR ROOTFS_TAG ROOTFS_HASH
# 686-linux-gnu . i686 . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
x86_64-linux-gnu . x86_64 . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
x86_64-linux-gnu . x86_64 120 rr v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a

# linux aarch64-linux-gnu true aarch64 aarch64 . . ---- ----------------------------------------
# linux armv7l-linux-gnueabihf true armv7l armv7l . . ---- ----------------------------------------
# linux powerpc64le-linux-gnu true ppc64le powerpc64le . . ---- ----------------------------------------
# aarch64-linux-gnu true aarch64 . . ---- ----------------------------------------
# armv7l-linux-gnueabihf true armv7l . . ---- ----------------------------------------
# powerpc64le-linux-gnu true powerpc64le . . ---- ----------------------------------------

# musl x86_64-linux-musl true 64 x86_64 . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90
# x86_64-linux-musl true x86_64 . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90


# These special lines allow us to embed default values for the columns above.
Expand Down
6 changes: 3 additions & 3 deletions pipelines/main/platforms/test_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,13 @@ steps:
soft_fail: ${ALLOW_FAIL?}
commands: |
# First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
TAR_ARCH=${TAR_ARCH?} OS=${OS?} MAKE_FLAGS= source .buildkite/utilities/calc_version_envs.sh
TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh
echo "--- Download build artifacts"
buildkite-agent artifact download $${ARTIFACT_FILENAME} .
buildkite-agent artifact download "$${UPLOAD_FILENAME}" .
echo "--- Extract build artifacts"
tar xzf $${ARTIFACT_FILENAME} $${JULIA_INSTALL_DIR}/
tar xzf "$${UPLOAD_FILENAME}" "$${JULIA_INSTALL_DIR}/"
echo "--- Print Julia version info"
$${JULIA_BINARY} -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
Expand Down
23 changes: 16 additions & 7 deletions pipelines/main/platforms/upload_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,25 @@ steps:
timeout_in_minutes: ${TIMEOUT?}
commands: |
# First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
TAR_ARCH=${TAR_ARCH?} OS=${OS?} MAKE_FLAGS=${MAKE_FLAGS?} source .buildkite/utilities/calc_version_envs.sh
TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh
echo "--- Download $${ARTIFACT_FILENAME} to ."
buildkite-agent artifact download $${ARTIFACT_FILENAME} .
echo "--- Download $${UPLOAD_FILENAME} to ."
buildkite-agent artifact download "$${UPLOAD_FILENAME}" .
# We first upload the canonical fully-specified upload target, which is the first one:
echo "--- Upload to S3"
MAJMIN="$${JULIA_VERSION:0:3}"
UPLOAD_TARGET="s3://julialang2/bin/${OS?}_buildkite/${TAR_ARCH?}/$${MAJMIN}/$${ARTIFACT_FILENAME}"
aws s3 cp $${ARTIFACT_FILENAME} $${UPLOAD_TARGET}
echo "Uploaded to: $${UPLOAD_TARGET}"
aws s3 cp "$${UPLOAD_FILENAME}" "s3://$${UPLOAD_TARGETS[0]}"
echo "--- Copy to secondary upload targets"
for SECONDARY_TARGET in $${UPLOAD_TARGETS[@]:1}; do
aws s3 cp "s3://$${UPLOAD_TARGETS[0]}" "s3://$${SECONDARY_TARGET}"
done
# Report to the user some URLs that they can use to download this from
echo "Uploaded to targets:"
for UPLOAD_TARGET in $${UPLOAD_TARGETS[@]}; do
echo " -> s3://$${UPLOAD_TARGET}"
done
agents:
queue: "julia"
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
Expand Down
Binary file modified pipelines/scheduled/coverage/coverage_linux64.yml.signature
Binary file not shown.
13 changes: 13 additions & 0 deletions utilities/arches_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,15 @@ if [[ ! -f "${ARCHES_FILE}" ]] ; then
exit 1
fi

enforce_sanitized() {
for value in "$@"; do
if ! [[ "${value}" =~ ^[a-zA-Z0-9_[:space:]-]*$ ]]; then
echo "Arches file '${ARCHES_FILE}' contains value '${value}' with non-alphanumeric characters; refusing to parse!" >&2
exit 1
fi
done
}


# Determine variable names from the header of the .arches file:
readarray -d ' ' -s 1 -t var_names < <(head -1 "${ARCHES_FILE}" | tr -s ' ')
Expand All @@ -23,6 +32,7 @@ readarray -t default_mapping_lines < <(grep "^#default" "${ARCHES_FILE}" | tr -s
for idx in "${!default_mapping_lines[@]}"; do
key="$( cut -d' ' -f2 <<<"${default_mapping_lines[${idx}]}" | xargs)"
value="$(cut -d' ' -f3 <<<"${default_mapping_lines[${idx}]}" | xargs)"
enforce_sanitized "${key}" "${value}"
defaults_map["${key}"]="${value}"
done

Expand All @@ -40,6 +50,9 @@ cat "${ARCHES_FILE}" | while read line; do
continue
fi

# Skip any line that contains suspicious characters, to prevent shell escaping bugs
enforce_sanitized "${line}"

# Convert line to array
readarray -d ' ' -t line_array <<<"${line}"

Expand Down
84 changes: 76 additions & 8 deletions utilities/calc_version_envs.sh
Original file line number Diff line number Diff line change
@@ -1,15 +1,83 @@
#!/bin/bash

echo "--- Print the full and short commit hashes"
# Extract information from our triplet
export ARCH="$(cut -d- -f1 <<<"${TRIPLET}")"
case "${TRIPLET}" in
*-apple-*)
OS="macos"
;;
*-freebsd*)
OS="freebsd"
;;
*-mingw*)
OS="windows"
;;
*-gnu*)
OS="linux"
;;
*-musl*)
OS="musl"
;;
*)
echo "Unknown triplet OS '${TRIPLET}'" >&2
exit 1
;;
esac
export OS

# Extract git information
SHORT_COMMIT_LENGTH=10
LONG_COMMIT="$(git rev-parse HEAD)"
SHORT_COMMIT="$(echo ${LONG_COMMIT} | cut -c1-${SHORT_COMMIT_LENGTH})"
ARTIFACT_FILENAME="julia-${SHORT_COMMIT}-${OS?}${TAR_ARCH?}.tar.gz"
JULIA_BINARYDIST="$(make print-JULIA_BINARYDIST_FILENAME ${MAKE_FLAGS?} | cut -c27- | tr -s ' ').tar.gz"
JULIA_INSTALL_DIR="julia-${SHORT_COMMIT}"

# Extract information about the current julia version number
JULIA_VERSION="$(cat VERSION)"
MAJMIN="${JULIA_VERSION:0:3}"

# If we're on a tag, then our "tar version" will be the julia version.
# Otherwise, it's the short commit.
if git describe --tags --exact-match >/dev/null 2>/dev/null; then
TAR_VERSION="${JULIA_VERSION}"
else
TAR_VERSION="${SHORT_COMMIT}"
fi

# Build the filename that we'll upload as, and get the filename that will be built
# These are not the same in situations such as `musl`, where the build system doesn't
# differentiate but we need to give it a different name.
JULIA_BINARYDIST_FILENAME="$(make print-JULIA_BINARYDIST_FILENAME | cut -c27- | tr -s ' ').tar.gz"

JULIA_INSTALL_DIR="julia-${TAR_VERSION}"
JULIA_BINARY="${JULIA_INSTALL_DIR}/bin/julia"

echo "The full commit is: ${LONG_COMMIT}"
echo "The short commit is: ${SHORT_COMMIT}"
echo "The artifact filename will be: ${ARTIFACT_FILENAME}"
echo "Julia will be installed to: ${JULIA_BINARY}"
# We generally upload to multiple upload targets
UPLOAD_TARGETS=(
# First, we have the canonical fully-specified upload target
"julialangnightlies/bin/${OS?}/${ARCH?}/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}.tar.gz"

# Next, we have the "latest" upload target
"julialangnightlies/bin/${OS?}/${ARCH?}/${MAJMIN?}/julia-latest-${OS?}-${ARCH?}.tar.gz"
)
UPLOAD_FILENAME="julia-${TAR_VERSION?}-${OS?}-${ARCH?}.tar.gz"

# Finally, for compatibility, we keep on uploading x86_64 and i686 targets to folders called `x64`
# and `x86`, respectively, although I would very much like to stop doing that.
if [[ "${ARCH}" == "x86_64" ]]; then
UPLOAD_TARGETS+=( "julialangnightlies/bin/${OS?}/x64/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}.tar.gz" )
UPLOAD_TARGETS+=( "julialangnightlies/bin/${OS?}/x64/${MAJMIN?}/julia-latest-${OS?}-${ARCH?}.tar.gz" )
elif [[ "${ARCH}" == "i686" ]]; then
UPLOAD_TARGETS+=( "julialangnightlies/bin/${OS?}/x86/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}.tar.gz" )
UPLOAD_TARGETS+=( "julialangnightlies/bin/${OS?}/x86/${MAJMIN?}/julia-latest-${OS?}-${ARCH?}.tar.gz" )
fi

echo "--- Print the full and short commit hashes"
echo "The full commit is: ${LONG_COMMIT}"
echo "The short commit is: ${SHORT_COMMIT}"
echo "Julia will be installed to: ${JULIA_BINARY}"
echo "Detected Julia version: ${MAJMIN} (${JULIA_VERSION})"
echo "Detected build platform: ${TRIPLET} (${ARCH}, ${OS})"
echo "Julia will be uploaded to: s3://${UPLOAD_TARGETS[0]}"
echo "With additional upload targets:"
for UPLOAD_TARGET in ${UPLOAD_TARGETS[@]:1}; do
echo " -> s3://${UPLOAD_TARGET}"
done

0 comments on commit 85ae354

Please sign in to comment.