Rename to build

JuliaCI · Mar 4, 2022 · 351975b · 351975b
1 parent b694dfa
commit 351975b
Show file tree

Hide file tree

Showing 9 changed files with 93 additions and 81 deletions.
diff --git a/pipelines/main/launch_signed_jobs.yml b/pipelines/main/launch_signed_jobs.yml
@@ -10,8 +10,11 @@ steps:
       # Explicitly pass along the cryptic token
       export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET
 
+      # Determine the julia version
+      export JULIA_VERSION=$$(cat VERSION)
+
       # Launch `upload_linux` jobs to store tarballs into S3 once tests are done
-      bash .buildkite/utilities/arches_pipeline_upload.sh .buildkite/pipelines/main/platforms/package_linux.arches .buildkite/pipelines/main/platforms/upload_linux.yml
+      bash .buildkite/utilities/arches_pipeline_upload.sh .buildkite/pipelines/main/platforms/build_linux.arches .buildkite/pipelines/main/platforms/upload_linux.yml
     agents:
       queue: julia
     env:

diff --git a/pipelines/main/launch_signed_jobs.yml.signature b/pipelines/main/launch_signed_jobs.yml.signature
@@ -1,2 +1 @@
-Salted__}�2|�����o<V\������,������p��o�F7v^s��G��.H�`iŘG���?ҭP�XR"���M�h�
-���ZMa��Ha�{�P
+Salted__�QRԿ��D���/�	[#�aI,HfmI$�/��J3���Rb�o�����i_���lBP/�Q�\~�{�&�7��0'($L��4/�4

diff --git a/pipelines/main/launch_unsigned_builders.yml b/pipelines/main/launch_unsigned_builders.yml
@@ -27,8 +27,8 @@ steps:
 
       # Launch all of the packaging jobs
       bash .buildkite/utilities/arches_pipeline_upload.sh \
-           .buildkite/pipelines/main/platforms/package_linux.arches \
-           .buildkite/pipelines/main/platforms/package_linux.yml
+           .buildkite/pipelines/main/platforms/build_linux.arches \
+           .buildkite/pipelines/main/platforms/build_linux.yml
 
       # Launch all of the testing jobs
       bash .buildkite/utilities/arches_pipeline_upload.sh \

diff --git a/...lines/main/platforms/package_linux.arches → pipelines/main/platforms/build_linux.arches b/...lines/main/platforms/package_linux.arches → pipelines/main/platforms/build_linux.arches
diff --git a/pipelines/main/platforms/package_linux.yml → pipelines/main/platforms/build_linux.yml b/pipelines/main/platforms/package_linux.yml → pipelines/main/platforms/build_linux.yml
@@ -4,11 +4,11 @@ agents:
   sandbox.jl: "true"
   os: "linux"
 steps:
-  - group: ":package: Packaging"
-    key: "package"
+  - group: ":package: Build"
+    key: "build"
     steps:
       - label: ":linux: package ${TRIPLET?}"
-        key: "package_${TRIPLET?}"
+        key: "build_${TRIPLET?}"
         plugins:
           - JuliaCI/external-buildkite#v1:
               version: ".buildkite-external-version"

diff --git a/pipelines/main/platforms/test_linux.arches b/pipelines/main/platforms/test_linux.arches
@@ -1,7 +1,7 @@
 # OS       TRIPLET                 ALLOW_FAIL    TAR_ARCH    ARCH_ROOTFS    TIMEOUT    USE_RR   ROOTFS_TAG    ROOTFS_HASH
 # linux    686-linux-gnu           .             32          i686           .          .        v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
 linux      x86_64-linux-gnu        .             64          x86_64         .          .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
-linux      x86_64-linux-gnu        .             64          x86_64         .          rr       v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+linux      x86_64-linux-gnu        .             64          x86_64         120        rr       v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
 
 # linux    aarch64-linux-gnu       true          aarch64     aarch64        .          .        ----          ----------------------------------------
 # linux    armv7l-linux-gnueabihf  true          armv7l      armv7l         .          .        ----          ----------------------------------------

diff --git a/pipelines/main/platforms/test_linux.yml b/pipelines/main/platforms/test_linux.yml
@@ -4,79 +4,83 @@ agents:
   sandbox.jl: "true"
   os: "linux"
 steps:
-  - label: ":linux: test ${TRIPLET?}"
-    key: "test_${TRIPLET?}${USE_RR-}"
-    depends_on:
-      - "package_${TRIPLET?}"
-    plugins:
-      - JuliaCI/external-buildkite#v1:
-          version: ".buildkite-external-version"
-          repo_url: "https://github.com/JuliaCI/julia-buildkite"
-      - JuliaCI/julia#v1:
-          # Drop default "registries" directory, so it is not persisted from execution to execution
-          persist_depot_dirs: packages,artifacts,compiled
-          version: '1.6'
-      - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${OS?}.${ARCH_ROOTFS?}.tar.gz
-          # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${OS?}.${ARCH_ROOTFS?}.tar.gz
-          rootfs_treehash: "${ROOTFS_HASH?}"
-          uid: 1000
-          gid: 1000
-          workspaces:
-            # Include `/cache/repos` so that our `git` version introspection works.
-            - "/cache/repos:/cache/repos"
-    env:
-      JULIA_SHELL: "/bin/bash"
-    timeout_in_minutes: ${TIMEOUT?}
-    soft_fail: ${ALLOW_FAIL?}
-    commands: |
-      # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
-      TAR_ARCH=${TAR_ARCH?} OS=${OS?} MAKE_FLAGS= source .buildkite/utilities/calc_version_envs.sh
+  - group: ":test_tube: Test"
+    key: "test"
+    steps:
+      - label: ":linux: test ${TRIPLET?}${USE_RR-}"
+        key: "test_${TRIPLET?}${USE_RR-}"
+        depends_on:
+          - "build_${TRIPLET?}"
+        plugins:
+          - JuliaCI/external-buildkite#v1:
+              version: ".buildkite-external-version"
+              repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - JuliaCI/julia#v1:
+              # Drop default "registries" directory, so it is not persisted from execution to execution
+              persist_depot_dirs: packages,artifacts,compiled
+              version: '1.6'
+          - staticfloat/sandbox#v1:
+              rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${OS?}.${ARCH_ROOTFS?}.tar.gz
+              # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${OS?}.${ARCH_ROOTFS?}.tar.gz
+              rootfs_treehash: "${ROOTFS_HASH?}"
+              uid: 1000
+              gid: 1000
+              workspaces:
+                # Include `/cache/repos` so that our `git` version introspection works.
+                - "/cache/repos:/cache/repos"
+        env:
+          JULIA_SHELL: "/bin/bash"
+        timeout_in_minutes: ${TIMEOUT?}
+        soft_fail: ${ALLOW_FAIL?}
+        commands: |
+          # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
+          TAR_ARCH=${TAR_ARCH?} OS=${OS?} MAKE_FLAGS= source .buildkite/utilities/calc_version_envs.sh
 
-      echo "--- Download build artifacts"
-      buildkite-agent artifact download $${ARTIFACT_FILENAME} .
+          echo "--- Download build artifacts"
+          buildkite-agent artifact download $${ARTIFACT_FILENAME} .
 
-      echo "--- Extract build artifacts"
-      tar xzf $${ARTIFACT_FILENAME} $${JULIA_INSTALL_DIR}/
+          echo "--- Extract build artifacts"
+          tar xzf $${ARTIFACT_FILENAME} $${JULIA_INSTALL_DIR}/
 
-      echo "--- Print Julia version info"
-      $${JULIA_BINARY} -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
-      echo "JULIA_CPU_THREADS is: $${JULIA_CPU_THREADS}"
-      $${JULIA_BINARY} -e '@info "" Sys.CPU_THREADS'
+          echo "--- Print Julia version info"
+          $${JULIA_BINARY} -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
+          echo "JULIA_CPU_THREADS is: $${JULIA_CPU_THREADS}"
+          $${JULIA_BINARY} -e '@info "" Sys.CPU_THREADS'
 
-      echo "--- Set some environment variables"
-      export OPENBLAS_NUM_THREADS=8
-      unset JULIA_DEPOT_PATH
-      unset JULIA_PKG_SERVER
+          echo "--- Set some environment variables"
+          export OPENBLAS_NUM_THREADS=8
+          unset JULIA_DEPOT_PATH
+          unset JULIA_PKG_SERVER
 
-      # Make sure that temp files and temp directories are created in a location that is
-      # backed by real storage.
-      export TMPDIR="$(pwd)/tmp"
-      mkdir -p $${TMPDIR}
+          # Make sure that temp files and temp directories are created in a location that is
+          # backed by real storage.
+          export TMPDIR="$(pwd)/tmp"
+          mkdir -p $${TMPDIR}
 
-      # By default, run all tests.
-      export TESTS="all LibGit2/online --ci"
+          # By default, run all tests.
+          export TESTS="all LibGit2/online --ci"
 
-      # If we're running inside of `rr`, limit the number of threads
-      if [[ "${USE_RR?}" == "rr" ]]; then
-        export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY}"
-        export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])"
-        export JULIA_NUM_THREADS=1
+          # If we're running inside of `rr`, limit the number of threads
+          if [[ "${USE_RR?}" == "rr" ]]; then
+            export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY}"
+            export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])"
+            export JULIA_NUM_THREADS=1
 
-        # Give `rr` 60 minutes to run the tests
-        export JULIA_TEST_RR_TIMEOUT="60"
-      else
-        export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY}"
-        export NCORES_FOR_TESTS="Sys.CPU_THREADS"
-        export JULIA_NUM_THREADS=16
-      fi
+            # Give `rr` 60 minutes to run the tests
+            export JULIA_TEST_RR_TIMEOUT="60"
+          else
+            export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY}"
+            export NCORES_FOR_TESTS="Sys.CPU_THREADS"
+            export JULIA_NUM_THREADS=16
+          fi
 
-      echo "--- Print the list of test sets, and other useful environment variables"
-      echo "JULIA_CMD_FOR_TESTS is:    $${JULIA_CMD_FOR_TESTS:?}"
-      echo "JULIA_NUM_THREADS is:      $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set
-      echo "NCORES_FOR_TESTS is:       $${NCORES_FOR_TESTS:?}"
-      echo "OPENBLAS_NUM_THREADS is:   $${OPENBLAS_NUM_THREADS:?}"
-      echo "TESTS is:                  $${TESTS:?}"
+          echo "--- Print the list of test sets, and other useful environment variables"
+          echo "JULIA_CMD_FOR_TESTS is:    $${JULIA_CMD_FOR_TESTS:?}"
+          echo "JULIA_NUM_THREADS is:      $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set
+          echo "NCORES_FOR_TESTS is:       $${NCORES_FOR_TESTS:?}"
+          echo "OPENBLAS_NUM_THREADS is:   $${OPENBLAS_NUM_THREADS:?}"
+          echo "TESTS is:                  $${TESTS:?}"
+          echo "USE_RR is:                 $${USE_RR:?}"
 
-      echo "--- Run the Julia test suite"
-      $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})"
+          echo "--- Run the Julia test suite"
+          $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})"
diff --git a/pipelines/main/platforms/upload_linux.yml b/pipelines/main/platforms/upload_linux.yml
@@ -7,13 +7,13 @@ steps:
   - label: ":linux: upload ${TRIPLET?}"
     key:   "upload_${TRIPLET?}"
     depends_on:
+      # Wait for the builder to finish
+      - "build_${TRIPLET?}"
       # Wait for the tester to finish
       - "test_${TRIPLET?}"
-
     plugins:
-      - JuliaCI/external-buildkite#v1:
-          version: ".buildkite-external-version"
-          repo_url: "https://github.com/JuliaCI/julia-buildkite"
+      # Avoid checking out Julia, as we don't actually need the code for this step
+      - thedyrt/skip-checkout#v0.1.1: ~
       - JuliaCI/julia#v1:
           # Drop default "registries" directory, so it is not persisted from execution to execution
           persist_depot_dirs: packages,artifacts,compiled
@@ -29,4 +29,10 @@ steps:
       TAR_ARCH=${TAR_ARCH?} OS=${OS?} MAKE_FLAGS=${MAKE_FLAGS?} source .buildkite/utilities/calc_version_envs.sh
 
       echo "--- Download $${ARTIFACT_FILENAME}"
-      buildkite-agent artifact upload $${ARTIFACT_FILENAME}
+      buildkite-agent artifact download $${ARTIFACT_FILENAME}
+
+      echo "--- Upload to S3"
+      MAJMIN="$${JULIA_VERSION:0:3}"
+      UPLOAD_TARGET="s3://julialang2/bin/${OS?}/${TAR_ARCH?}/$${MAJMIN}/$${ARTIFACT_FILENAME}"
+      aws s3 cp $${ARTIFACT_FILENAME} $${UPLOAD_TARGET}
+      echo "Uploaded to: $${UPLOAD_TARGET}"
diff --git a/pipelines/scheduled/coverage/coverage_linux64.yml.signature b/pipelines/scheduled/coverage/coverage_linux64.yml.signature
@@ -1 +1 @@
-Salted__��#��8�Ыڵk�m��&�o���6��p����I����#����{�*؞oZ0ě�w���ĵd�&�i��kå�1�W��_��<(
+Salted__� R�˺o���%�(�	���n8<��lro&g��_��/��Ny��5�$4��OD��.É>T������P}__���7C��� ��j�
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		Salted__��#��8�Ыڵk�m��&�o��6��p��I��#��{�*؞oZ0ě�w��ĵd�&�i��kå�1�W��_��<(
		Salted__� R�˺o��%�(� ��n8<��lro&g��_��/��Ny��5�$4��OD��.É>T��P}__��7C�� j�
Expand Down