Merge in cibuildwheels-imgs repository

.github/workflows/build-and-publish-images.yml

@@ -46,6 +46,7 @@ jobs:
       PYTHON_VER: ${{ matrix.PYTHON_VER }}
       IMAGE_REPO: ${{ matrix.IMAGE_REPO }}
       IMAGE_NAME: ${{ matrix.IMAGE_NAME }}
+      MANYLINUX_VER: ${{ matrix.MANYLINUX_VER }}
 
   build-multiarch-manifest:
     name: manifest (${{ matrix.CUDA_VER }}, ${{ matrix.PYTHON_VER }}, ${{ matrix.LINUX_VER }}, ${{ matrix.IMAGE_REPO }})

.github/workflows/build-image.yml

@@ -21,6 +21,9 @@ on:
       IMAGE_NAME:
         required: true
         type: string
+      MANYLINUX_VER:
+        required: false
+        type: string
 
 jobs:
   run:
@@ -49,15 +52,22 @@ jobs:
         with:
           driver: docker
           endpoint: builders
+      - name: Generate Build Args
+        id: generate-build-args
+        run: ci/compute-build-args.sh
+        env:
+          IMAGE_REPO: ${{ inputs.IMAGE_REPO }}
+          CUDA_VER: ${{ inputs.CUDA_VER }}
+          LINUX_VER: ${{ inputs.LINUX_VER }}
+          PYTHON_VER: ${{ inputs.PYTHON_VER }}
+          ARCH: ${{ matrix.ARCH }}
       - name: Build image
         uses: docker/build-push-action@v4
         with:
           context: context
-          file: Dockerfile
+          file: dockerfiles/${{ inputs.IMAGE_REPO }}.Dockerfile
           push: true
           pull: true
           build-args: |
-            CUDA_VER=${{ inputs.CUDA_VER }}
-            LINUX_VER=${{ inputs.LINUX_VER }}
-            PYTHON_VER=${{ inputs.PYTHON_VER }}
+            ${{ steps.generate-build-args.outputs.ARGS }}
           tags: ${{ inputs.IMAGE_NAME }}-${{ matrix.ARCH }}

ci/compute-build-args.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -euo pipefail
+
+ARGS=""
+case "${IMAGE_REPO}" in
+  ci)
+    ARGS="CUDA_VER=${CUDA_VER}
+LINUX_VER=${LINUX_VER}
+PYTHON_VER=${PYTHON_VER}"
+    ;;
+  *)
+    MANYLINUX_VER="manylinux_2_17"
+    if [[
+        "${LINUX_VER}" == "ubuntu18.04" ||
+        "${LINUX_VER}" == "ubuntu20.04"
+    ]]; then
+        MANYLINUX_VER="manylinux_2_31"
+    fi
+    ARGS="CUDA_VER=${CUDA_VER}
+LINUX_VER=${LINUX_VER}
+PYTHON_VER=${PYTHON_VER}
+CPU_ARCH=${ARCH}
+REAL_ARCH=$(arch)
+MANYLINUX_VER=${MANYLINUX_VER}"
+    ;;
+esac
+
+echo "ARGS<<EOF" >> "$GITHUB_OUTPUT"
+echo "$ARGS" >> "$GITHUB_OUTPUT"
+echo "EOF" >> "$GITHUB_OUTPUT"

ci/compute-latest-versions.sh

@@ -10,15 +10,17 @@ export LINUX_KEY="LINUX_VER"
 export CUDA_KEY="CUDA_VER"
 export PYTHON_KEY="PYTHON_VER"
 
+PREFIX="ci"
+
 # Get latest values
-LATEST_LINUX_VER=$(yq '.LATEST_VERSIONS.[strenv(LINUX_KEY)]' matrix.yaml)
-LATEST_CUDA_VER=$(yq '.LATEST_VERSIONS.[strenv(CUDA_KEY)]' matrix.yaml)
-LATEST_PYTHON_VER=$(yq '.LATEST_VERSIONS.[strenv(PYTHON_KEY)]' matrix.yaml)
+LATEST_LINUX_VER=$(yq ".${PREFIX}.LATEST_VERSIONS.[strenv(LINUX_KEY)]" matrix.yaml)
+LATEST_CUDA_VER=$(yq ".${PREFIX}.LATEST_VERSIONS.[strenv(CUDA_KEY)]" matrix.yaml)
+LATEST_PYTHON_VER=$(yq ".${PREFIX}.LATEST_VERSIONS.[strenv(PYTHON_KEY)]" matrix.yaml)
 
 # Get matrix array values
-LINUX_VERS=$(yq '.[strenv(LINUX_KEY)]' matrix.yaml)
-CUDA_VERS=$(yq '.[strenv(CUDA_KEY)]' matrix.yaml)
-PYTHON_VERS=$(yq '.[strenv(PYTHON_KEY)]' matrix.yaml)
+LINUX_VERS=$(yq ".${PREFIX}.[strenv(LINUX_KEY)]" matrix.yaml)
+CUDA_VERS=$(yq ".${PREFIX}.[strenv(CUDA_KEY)]" matrix.yaml)
+PYTHON_VERS=$(yq ".${PREFIX}.[strenv(PYTHON_KEY)]" matrix.yaml)
 
 # Ensure matrix array values contain latest values
 for KEY in "${LINUX_KEY}" "${CUDA_KEY}" "${PYTHON_KEY}"; do

ci/compute-matrix.jq

@@ -1,62 +1,89 @@
-def compute_arch($x):
-  ["amd64"] |
-  if
-    $x.CUDA_VER > "11.2.2" and
-    $x.LINUX_VER != "centos7"
-  then
-    . + ["arm64"]
-  else
-    .
-  end |
-  $x + {ARCHES: .};
-
-def compute_repo($x):
-  if
-    env.BUILD_TYPE == "pull-request"
-  then
-    "staging"
-  else
-    $x.IMAGE_REPO
-  end;
+include "ci/shared";
 
-def compute_tag_prefix($x):
-  if
-    env.BUILD_TYPE == "branch"
-  then
-    ""
-  else
-    $x.IMAGE_REPO + "-" + env.PR_NUM + "-"
-  end;
+# CI specific computations
+def compute_ci($x):
+  def compute_ci_arch($x):
+    ["amd64"] |
+    if
+      $x.CUDA_VER > "11.2.2" and
+      $x.LINUX_VER != "centos7"
+    then
+      . + ["arm64"]
+    else
+      .
+    end |
+    $x + {ARCHES: .};
 
-def compute_image_name($x):
-  compute_repo($x) as $repo |
-  compute_tag_prefix($x) as $tag_prefix |
-  "rapidsai/" + $repo + ":" + $tag_prefix + "cuda" + $x.CUDA_VER + "-" + $x.LINUX_VER + "-" + "py" + $x.PYTHON_VER |
-  $x + {IMAGE_NAME: .};
+  def compute_image_name($x):
+    compute_repo($x) as $repo |
+    compute_tag_prefix($x) as $tag_prefix |
+    "rapidsai/" + $repo + ":" + $tag_prefix + "cuda" + $x.CUDA_VER + "-" + $x.LINUX_VER + "-" + "py" + $x.PYTHON_VER |
+    $x + {IMAGE_NAME: .};
 
-# Checks the current entry to see if it matches the given exclude
-def matches($entry; $exclude):
-  all($exclude | to_entries | .[]; $entry[.key] == .value);
+  ($x.exclude // []) as $excludes |
+  $x | del(.exclude) |
+  keys_unsorted as $matrix_keys |
+  to_entries |
+  map(.value) |
+  [
+    combinations |
+    lists2dict($matrix_keys; .) |
+    filter_excludes(.; $excludes) |
+    compute_ci_arch(.) |
+    compute_image_name(.)
+  ] |
+  {include: .};
 
-# Checks the current entry to see if it matches any of the excludes.
-# If so, produce no output. Otherwise, output the entry.
-def filter_excludes($entry; $excludes):
-  select(any($excludes[]; matches($entry; .)) | not);
+# Wheels specific computations
+def compute_wheels($x):
+  def compute_wheels_arch($x):
+    ["amd64"] |
+    if
+      ["ubuntu18.04", "centos7"] | index($x.LINUX_VER) != null
+    then
+      .
+    else
+      . + ["arm64"]
+    end |
+    $x + {ARCHES: .};
 
-def lists2dict($keys; $values):
-  reduce range($keys | length) as $ind ({}; . + {($keys[$ind]): $values[$ind]});
+  def compute_manylinux_version($x):
+    if
+      ["ubuntu18.04", "ubuntu20.04"] | index($x.LINUX_VER) != null
+    then
+      "manylinux_2_31"
+    else
+      "manylinux_2_17"
+    end |
+    $x + {MANYLINUX_VER: .};
 
-def compute_matrix($input):
-  ($input.exclude // []) as $excludes |
-  $input | del(.exclude) |
+  def compute_image_name($x):
+    compute_repo($x) as $repo |
+    compute_tag_prefix($x) as $tag_prefix |
+    "rapidsai/" + $repo + ":" + $tag_prefix + "cuda" + $x.CUDA_VER + "-" + $x.LINUX_VER + "-" + "py" + $x.PYTHON_VER |
+    $x + {IMAGE_NAME: .};
+
+  ($x.exclude // []) as $excludes |
+  $x | del(.exclude) |
   keys_unsorted as $matrix_keys |
   to_entries |
   map(.value) |
   [
     combinations |
     lists2dict($matrix_keys; .) |
     filter_excludes(.; $excludes) |
-    compute_arch(.) |
+    compute_wheels_arch(.) |
+    compute_manylinux_version(.) |
     compute_image_name(.)
   ] |
   {include: .};
+
+# Main function to compute matrix
+def compute_matrix($type; $input):
+  if $type == "ci" then
+    compute_ci($input)
+  elif $type == "wheels" then
+    compute_wheels($input)
+  else
+    error("Unknown matrix type: " + $type)
+  end;

ci/compute-matrix.sh

@@ -13,4 +13,20 @@ case "${BUILD_TYPE}" in
     ;;
 esac
 
-yq -o json '. | del(.LATEST_VERSIONS)' matrix.yaml | jq -c 'include "ci/compute-matrix"; compute_matrix(.)'
+COMBINED_MATRIX_YAML=$(yq -o json 'matrix.yaml')
+
+# Get all top-level keys (e.g., "ci", "wheels") from matrix.yaml
+CONFIGURATIONS=$(echo "$COMBINED_MATRIX_YAML" | jq -r 'keys[]')
+
+COMBINED_COMPUTED='{"include": []}'
+
+# Loop through each configuration and compute matrix
+for CONFIG in $CONFIGURATIONS; do
+  CONFIG_MATRIX=$(echo "$COMBINED_MATRIX_YAML" | jq -c --arg config "$CONFIG" '.[$config] | del(.LATEST_VERSIONS)')
+  CONFIG_COMPUTED=$(echo "$CONFIG_MATRIX" | jq -c --arg type "$CONFIG" 'include "ci/compute-matrix"; compute_matrix($type; .)')
+  COMBINED_COMPUTED=$(echo "$COMBINED_COMPUTED" | jq -c \
+    --argjson config_computed "$CONFIG_COMPUTED" \
+    '{"include": (.include + $config_computed.include)}')
+done
+
+echo "$COMBINED_COMPUTED"

ci/create-multiarch-manifest.sh

@@ -1,6 +1,12 @@
 #!/bin/bash
 set -euo pipefail
 
+if [[ "${IMAGE_REPO}" != "ci" ]]; then
+  LATEST_CUDA_VER=$(yq '.wheels.CUDA_VER | sort | .[-1]' matrix.yaml)
+  LATEST_PYTHON_VER=$(yq -o json '.wheels.PYTHON_VER' matrix.yaml | jq -r 'max_by(split(".") | map(tonumber))')
+  LATEST_UBUNTU_VER=$(yq '.wheels.LINUX_VER | map(select(. == "*ubuntu*")) | sort | .[-1]' matrix.yaml)
+fi
+
 source_tags=()
 tag="${IMAGE_NAME}"
 for arch in $(echo "${ARCHES}" | jq .[] -r); do

ci/shared.jq

@@ -0,0 +1,30 @@
+def compute_repo($x):
+    if
+      env.BUILD_TYPE == "pull-request"
+    then
+      "staging"
+    else
+      $x.IMAGE_REPO
+    end;
+
+# Compute tag prefix
+def compute_tag_prefix($x):
+if
+    env.BUILD_TYPE == "branch"
+then
+    ""
+else
+    $x.IMAGE_REPO + "-" + env.PR_NUM + "-"
+end;
+
+# Checks the current entry to see if it matches the given exclude
+def matches($entry; $exclude):
+all($exclude | to_entries | .[]; $entry[.key] == .value);
+
+# Checks the current entry to see if it matches any of the excludes.
+def filter_excludes($entry; $excludes):
+select(any($excludes[]; matches($entry; .)) | not);
+
+# Convert lists to dictionary
+def lists2dict($keys; $values):
+reduce range($keys | length) as $ind ({}; . + {($keys[$ind]): $values[$ind]});