-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add macos packaging, testing and uploading
- Loading branch information
1 parent
e20da7a
commit 43f62de
Showing
15 changed files
with
466 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| # OS TRIPLET ALLOW_FAIL ARCH MAKE_FLAGS TIMEOUT | ||
| macos x86_64-apple-darwin . x86_64 . . | ||
| macos aarch64-apple-darwin . aarch64 . . | ||
|
|
||
|
|
||
| # These special lines allow us to embed default values for the columns above. | ||
| # Any column without a default mapping here will simply substitute a `.` to the empty string | ||
|
|
||
| # Of course we do not allow jobs to fail typically | ||
| #default ALLOW_FAIL false | ||
|
|
||
| # Most jobs should finish within 1.5 hours, barring exceptionally slow hardware | ||
| #default TIMEOUT 90 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| steps: | ||
| - group: ":macos: Macos" | ||
| steps: | ||
| - label: ":macos: build ${TRIPLET?}" | ||
| key: "build_${TRIPLET?}" | ||
| plugins: | ||
| - JuliaCI/external-buildkite#v1: | ||
| version: ".buildkite-external-version" | ||
| repo_url: "https://github.com/JuliaCI/julia-buildkite" | ||
| timeout_in_minutes: ${TIMEOUT?} | ||
| commands: | | ||
| # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc... | ||
| TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh | ||
| # export JULIA_CPU_TARGET, which is picked up by the build system | ||
| ARCH="$${ARCH??}" source .buildkite/utilities/export_julia_cpu_target.sh | ||
| echo "--- Create the Make.user file" | ||
| echo "${MAKE_FLAGS?}" | tr "," "\n" > Make.user | ||
| if [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-master" ]] || [[ "$${BUILDKITE_PIPELINE_SLUG:?}" =~ ^julia-release- ]]; then | ||
| banner="Official https://julialang.org/ release" | ||
| echo "TAGGED_RELEASE_BANNER=\"$${banner:?}\"" >> Make.user | ||
| elif [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-buildkite" ]]; then | ||
| banner="Not an official binary; built as part of the CI infrastructure testing" | ||
| echo "TAGGED_RELEASE_BANNER=\"$${banner:?}\"" >> Make.user | ||
| fi | ||
| echo "VERBOSE=1" >> Make.user | ||
| cat Make.user | ||
| echo "--- Build Julia" | ||
| make -j 8 ${MAKE_FLAGS?} | ||
| echo "--- Check that the working directory is clean" | ||
| if [ -z "$(git status --short)" ]; then | ||
| echo "INFO: The working directory is clean." | ||
| else | ||
| echo "ERROR: The working directory is dirty." | ||
| echo "Output of git status:" | ||
| git status | ||
| exit 1 | ||
| fi | ||
| echo "--- Print Julia version info" | ||
| ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' | ||
| echo "--- Create build artifacts" | ||
| make -j 8 binary-dist ${MAKE_FLAGS?} | ||
| # Rename the build artifact in case we want to name it differently, as is the case on `musl`. | ||
| if [[ "$${JULIA_BINARYDIST_FILENAME}" != "$${UPLOAD_FILENAME}" ]]; then | ||
| mv $${JULIA_BINARYDIST_FILENAME} $${UPLOAD_FILENAME} | ||
| fi | ||
| echo "--- Upload build artifacts" | ||
| buildkite-agent artifact upload $${UPLOAD_FILENAME} | ||
| agents: | ||
| queue: "julia" | ||
| # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing | ||
| # sandbox_capable: "true" | ||
| os: "macos" | ||
| arch: "${ARCH}" | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| # OS TRIPLET ALLOW_FAIL ARCH MAKE_FLAGS TIMEOUT | ||
| macos x86_64-apple-darwin . x86_64 . . | ||
| macos aarch64-apple-darwin true aarch64 . . | ||
|
|
||
|
|
||
| # These special lines allow us to embed default values for the columns above. | ||
| # Any column without a default mapping here will simply substitute a `.` to the empty string | ||
|
|
||
| # Of course we do not allow jobs to fail typically | ||
| #default ALLOW_FAIL false | ||
|
|
||
| # Most jobs should finish within 1.5 hours, barring exceptionally slow hardware | ||
| #default TIMEOUT 90 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| steps: | ||
| - group: ":macos: Macos" | ||
| steps: | ||
| - label: ":macos: :test_tube: test ${TRIPLET?}" | ||
| key: "test_${TRIPLET?}" | ||
| depends_on: | ||
| - "build_${TRIPLET?}" | ||
| plugins: | ||
| - JuliaCI/external-buildkite#v1: | ||
| version: ".buildkite-external-version" | ||
| repo_url: "https://github.com/JuliaCI/julia-buildkite" | ||
| env: | ||
| JULIA_SHELL: "/bin/bash" | ||
| timeout_in_minutes: ${TIMEOUT?} | ||
| soft_fail: ${ALLOW_FAIL?} | ||
| commands: | | ||
| # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc... | ||
| TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh | ||
| echo "--- Download build artifacts" | ||
| buildkite-agent artifact download "$${UPLOAD_FILENAME}" . | ||
| echo "--- Extract build artifacts" | ||
| tar xzf "$${UPLOAD_FILENAME}" "$${JULIA_INSTALL_DIR}/" | ||
| echo "--- Ad-hoc sign for testing" | ||
| contrib/codesign.sh "-" "$${JULIA_INSTALL_DIR}" | ||
| echo "--- Print Julia version info" | ||
| $${JULIA_BINARY} -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' | ||
| echo "JULIA_CPU_THREADS is: $${JULIA_CPU_THREADS}" | ||
| $${JULIA_BINARY} -e '@info "" Sys.CPU_THREADS' | ||
| echo "--- Set some environment variables" | ||
| export OPENBLAS_NUM_THREADS=4 | ||
| unset JULIA_DEPOT_PATH | ||
| unset JULIA_PKG_SERVER | ||
| # By default, run all tests. | ||
| export TESTS="all LibGit2/online --ci" | ||
| export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY}" | ||
| export NCORES_FOR_TESTS="Sys.CPU_THREADS" | ||
| echo "--- Print the list of test sets, and other useful environment variables" | ||
| echo "JULIA_CMD_FOR_TESTS is: $${JULIA_CMD_FOR_TESTS:?}" | ||
| echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set | ||
| echo "NCORES_FOR_TESTS is: $${NCORES_FOR_TESTS:?}" | ||
| echo "OPENBLAS_NUM_THREADS is: $${OPENBLAS_NUM_THREADS:?}" | ||
| echo "TESTS is: $${TESTS:?}" | ||
| echo "--- Run the Julia test suite" | ||
| $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})" | ||
| agents: | ||
| queue: "julia" | ||
| os: "macos" | ||
| arch: "${ARCH}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| # OS TRIPLET ALLOW_FAIL ARCH MAKE_FLAGS TIMEOUT | ||
| macos x86_64-apple-darwin . x86_64 . . | ||
| macos aarch64-apple-darwin . aarch64 . . | ||
|
|
||
|
|
||
| # These special lines allow us to embed default values for the columns above. | ||
| # Any column without a default mapping here will simply substitute a `.` to the empty string | ||
|
|
||
| # Of course we do not allow jobs to fail typically | ||
| #default ALLOW_FAIL false | ||
|
|
||
| # Most jobs should finish within 1.5 hours, barring exceptionally slow hardware | ||
| #default TIMEOUT 90 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,134 @@ | ||
| steps: | ||
| - label: ":macos: upload ${TRIPLET?}" | ||
| key: "upload_${TRIPLET?}" | ||
| # We only upload to S3 if one of the following criteria are true: | ||
| # 1. The branch is `master`. | ||
| # 2. The branch is `release-*`. | ||
| # 3. The build is a tag build AND the tag is `v*`. | ||
| # 4. The pipeline is `julia-buildkite`. | ||
| if: (build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/) || (pipeline.slug == "julia-buildkite") | ||
| depends_on: | ||
| # Wait for the builder to finish | ||
| - "build_${TRIPLET?}" | ||
| # Wait for the tester to finish | ||
| - "test_${TRIPLET?}" | ||
| # Prevent multiple pipelines from uploading to S3 simultaneously | ||
| # It is okay for two different triplets to upload simultaneously | ||
| concurrency: 1 | ||
| concurrency_group: 'upload/julialangnightlies/upload_macos/${TRIPLET?}' | ||
| plugins: | ||
| - JuliaCI/external-buildkite#v1: | ||
| version: "./.buildkite-external-version" | ||
| repo_url: "https://github.com/JuliaCI/julia-buildkite" | ||
| - staticfloat/cryptic#v2: | ||
| variables: | ||
| - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1" | ||
| - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg==" | ||
| files: | ||
| - .buildkite/secrets/tarball_signing.gpg | ||
| - .buildkite/secrets/macos_codesigning.keychain | ||
| - .buildkite/secrets/keychain_password.txt | ||
| timeout_in_minutes: ${TIMEOUT?} | ||
| commands: | | ||
| # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc... | ||
| TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh | ||
| echo "--- Variables for the dmg" | ||
| MACOS_CODESIGN_IDENTITY="2053E9292809B66582CA9F042B470C0929340362" | ||
| APP_NAME="Julia-$${MAJMIN}.app" | ||
| DMG_NAME="$$(basename $$JULIA_BINARYDIST_FILENAME .tar.gz).dmg" | ||
| APP_PATH="dmg/$${APP_NAME}" | ||
| MAC_PATH="contrib/mac/app" | ||
| KEYCHAIN_PATH="$$(pwd)/.buildkite/secrets/macos_codesigning.keychain" | ||
| APP_ID="org.julialang.launcherapp" | ||
| APP_COPYRIGHT="$$(date '+%Y') The Julia Project" | ||
| JULIA_VERSION_OPT_COMMIT="$${JULIA_VERSION}-$${SHORT_COMMIT}" | ||
| VOL_NAME="Julia-$${JULIA_VERSION_OPT_COMMIT}" | ||
| KEYCHAIN_PASSWORD="$$(cat .buildkite/secrets/keychain_password.txt)" | ||
| echo "--- Download $${UPLOAD_FILENAME} to ." | ||
| buildkite-agent artifact download "$${UPLOAD_FILENAME}" . | ||
| #Get the current keychains and put into a list | ||
| echo "--- Add keychain to list and unlock it" | ||
| security -v list-keychains -s -d user "$${KEYCHAIN_PATH}" | ||
| security unlock-keychain -p "$${KEYCHAIN_PASSWORD}" "$${KEYCHAIN_PATH}" | ||
| security find-identity "$${MACOS_CODESIGN_IDENTITY}" "$${KEYCHAIN_PATH}" | ||
| security -v list-keychains -d user | ||
| echo "--- Prepare .app file" | ||
| mkdir -p dmg | ||
| osacompile -o "$${APP_PATH}" "$${MAC_PATH}/startup.applescript" | ||
| rm "$${APP_PATH}/Contents/Resources/applet.icns" | ||
| cp "$${MAC_PATH}/julia.icns" "$${APP_PATH}/Contents/Resources/" | ||
| plutil -replace CFBundleDevelopmentRegion -string "en" "$${APP_PATH}/Contents/Info.plist" | ||
| plutil -insert CFBundleDisplayName -string "Julia" "$${APP_PATH}/Contents/Info.plist" | ||
| plutil -replace CFBundleIconFile -string "julia.icns" "$${APP_PATH}/Contents/Info.plist" | ||
| plutil -insert CFBundleIdentifier -string "$${APP_ID}" "$${APP_PATH}/Contents/Info.plist" | ||
| plutil -replace CFBundleName -string "Julia" "$${APP_PATH}/Contents/Info.plist" | ||
| plutil -insert CFBundleShortVersionString -string "$${MAJMINPAT}" "$${APP_PATH}/Contents/Info.plist" | ||
| plutil -insert CFBundleVersion -string "$${JULIA_VERSION_OPT_COMMIT}" "$${APP_PATH}/Contents/Info.plist" | ||
| plutil -insert NSHumanReadableCopyright -string "$${APP_COPYRIGHT}" "$${APP_PATH}/Contents/Info.plist" | ||
| mkdir -p "$${APP_PATH}/Contents/Resources/julia" | ||
| echo "--- Extract build artifacts" | ||
| tar zxf "$${UPLOAD_FILENAME}" -C "$${APP_PATH}/Contents/Resources/julia" --strip-components 1 | ||
| find $${APP_PATH}/Contents/Resources/julia -type f -exec chmod -w {} \; | ||
| echo "--- Codesign the files" | ||
| cp -f "$${MAC_PATH}/julia.icns" dmg/.VolumeIcon.icns | ||
| ln -fs /Applications dmg/Applications | ||
| .buildkite/utilities/macos/codesign.sh --keychain "$${KEYCHAIN_PATH}" --identity "$${MACOS_CODESIGN_IDENTITY}" "$${APP_PATH}/Contents/Resources/julia" | ||
| codesign -s $$MACOS_CODESIGN_IDENTITY "$${APP_PATH}/Contents/MacOS/applet" --keychain "$${KEYCHAIN_PATH}" --force | ||
| echo "--- create dmg" | ||
| hdiutil create $$DMG_NAME -size 1t -fs HFS+ -ov -volname "$${VOL_NAME}" -imagekey zlib-level=9 -srcfolder dmg | ||
| codesign -s $$MACOS_CODESIGN_IDENTITY --timestamp -i org.julialang.launcherapp $$DMG_NAME --keychain "$${KEYCHAIN_PATH}" --force | ||
| # remake tar file after signing | ||
| mkdir "$${JULIA_INSTALL_DIR}" | ||
| cp -R "$${APP_PATH}/Contents/Resources/julia/" "$${JULIA_INSTALL_DIR}" | ||
| rm -rf "$${UPLOAD_FILENAME}" | ||
| tar zcvf "$${UPLOAD_FILENAME}" "$${JULIA_INSTALL_DIR}" | ||
| echo "--- GPG-sign the tarball" | ||
| .buildkite/utilities/sign_tarball.sh .buildkite/secrets/tarball_signing.gpg "$${UPLOAD_FILENAME}" | ||
| echo "--- Upload signed packages to buildkite" | ||
| buildkite-agent artifact upload "$${DMG_NAME}" | ||
| buildkite-agent artifact upload "$${UPLOAD_FILENAME}" | ||
| buildkite-agent artifact upload "$${UPLOAD_FILENAME}.asc" | ||
| # We first upload the canonical fully-specified upload target, which is the first one: | ||
| echo "--- Upload tarballs, dmgs and signatures to S3" | ||
| DMG_TARGET="$$(dirname $$UPLOAD_TARGETS[0])/$$DMG_NAME" | ||
| aws s3 cp --acl public-read "$${UPLOAD_FILENAME}" "s3://$${UPLOAD_TARGETS[0]}" | ||
| aws s3 cp --acl public-read "$${UPLOAD_FILENAME}.asc" "s3://$${UPLOAD_TARGETS[0]}.asc" | ||
| aws s3 cp --acl public-read "$${DMG_NAME}" "s3://$${DMG_TARGET}" | ||
| echo "--- Copy to secondary upload targets" | ||
| # We'll do these in parallel, then wait on the background jobs | ||
| for SECONDARY_TARGET in $${UPLOAD_TARGETS[@]:1}; do | ||
| SECONDARY_DMG_TARGET=""$$(dirname $$SECONDARY_TARGET)/$$DMG_NAME"" | ||
| aws s3 cp --acl public-read "s3://$${DMG_TARGET}" "s3://$${SECONDARY_DMG_TARGET}" & | ||
| aws s3 cp --acl public-read "s3://$${UPLOAD_TARGETS[0]}" "s3://$${SECONDARY_TARGET}" & | ||
| aws s3 cp --acl public-read "s3://$${UPLOAD_TARGETS[0]}.asc" "s3://$${SECONDARY_TARGET}.asc" & | ||
| done | ||
| wait | ||
| # Report to the user some URLs that they can use to download this from | ||
| echo "+++ Uploaded to targets" | ||
| for UPLOAD_TARGET in $${UPLOAD_TARGETS[@]}; do | ||
| echo " -> s3://$${UPLOAD_TARGET}" | ||
| echo " -> s3://$$(dirname $$UPLOAD_TARGET)/$$DMG_NAME" | ||
| done | ||
| security -v delete-keychain "$$KEYCHAIN_PATH" | ||
| agents: | ||
| queue: "julia" | ||
| # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing | ||
| os: "macos" | ||
| arch: "${ARCH}" | ||
| env: | ||
| # Receive cryptic token from parent job | ||
| BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} |
Binary file modified
BIN
+0 Bytes
(100%)
pipelines/scheduled/coverage/coverage_linux64.yml.signature
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| Salted__��TT���c�C����X����K��9Z�$�T� �.��t|| |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
| <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> | ||
| <plist version="1.0"> | ||
| <dict> | ||
| <key>com.apple.security.automation.apple-events</key> | ||
| <true/> | ||
| <key>com.apple.security.get-task-allow</key> | ||
| <true/> | ||
| <key>com.apple.security.cs.allow-dyld-environment-variables</key> | ||
| <true/> | ||
| <key>com.apple.security.cs.allow-jit</key> | ||
| <true/> | ||
| <key>com.apple.security.cs.allow-unsigned-executable-memory</key> | ||
| <true/> | ||
| <key>com.apple.security.cs.debugger</key> | ||
| <true/> | ||
| <key>com.apple.security.cs.disable-library-validation</key> | ||
| <true/> | ||
| <key>com.apple.security.device.audio-input</key> | ||
| <true/> | ||
| <key>com.apple.security.device.camera</key> | ||
| <true/> | ||
| </dict> | ||
| </plist> |
Oops, something went wrong.