Build RAPIDS. (#210)

* Build RAPIDS. --------- Co-authored-by: Paul Taylor <[email protected]> Co-authored-by: ptaylor <[email protected]>
rapidsai · Feb 20, 2024 · 32f398e · 32f398e
1 parent df3edf3
commit 32f398e
Show file tree

Hide file tree

Showing 11 changed files with 135 additions and 36 deletions.
diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile
@@ -7,10 +7,23 @@ FROM ${BASE} as pip-base
 
 RUN apt update -y \
  && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \
+    # C++ build tools
+    doxygen \
+    graphviz \
+    # C++ test dependencies
+    libgmock-dev \
+    libgtest-dev \
     # needed by libcudf_kafka
     librdkafka-dev \
+    # cuML/cuGraph dependencies
+    libblas-dev \
+    liblapack-dev \
     # needed by libcuspatial
-    sqlite3 libsqlite3-dev libtiff-dev libcurl4-openssl-dev \
+    libgdal-dev \
+    sqlite3 \
+    libsqlite3-dev \
+    libtiff-dev \
+    libcurl4-openssl-dev \
  && rm -rf /tmp/* /var/tmp/* /var/cache/apt/* /var/lib/apt/lists/*;
 
 ENV DEFAULT_VIRTUAL_ENV=rapids

diff --git a/.github/actions/feature-matrix/action.sh b/.github/actions/feature-matrix/action.sh
@@ -25,7 +25,7 @@ files="${@}";
 features="";
 scenarios="";
 
-if [ "${full_matrix}" == "1" ] || echo "$files" | grep -q ".github/"; then
+if [ "${full_matrix}" == "1" ] || grep -q "\.github/" <<< "$files"; then
   features="$(find_features_with_tests 'test.sh' 'features/test')";
   scenarios="$(find_features_with_tests 'scenarios.json' 'features/test')";
 elif echo "$files" | grep -q "features/"; then

diff --git a/.github/actions/image-matrix/action.sh b/.github/actions/image-matrix/action.sh
@@ -31,18 +31,18 @@ features="$(echo                        \
 || echo ''                              \
 )";
 
-if `# Include all images if full_matrix is true`                         \
-   [ "${full_matrix}" == "1" ]                                           \
-   `# Include all images if matrix or workflows changed`                 \
-|| grep -q 'matrix.yml'                                   <<< "${files}" \
-|| grep -q '.github/workflows/test.yml'                   <<< "${files}" \
-|| grep -q '.github/actions/build-linux-image/action.yml' <<< "${files}" \
-|| grep -q '.github/actions/image-matrix/action.sh'       <<< "${files}" \
-|| grep -q '.github/actions/image-matrix/action.yml'      <<< "${files}" \
-|| grep -q '.github/workflows/build-and-test-image.yml'   <<< "${files}" \
-   `# Include all images if cmake, ninja, sccache, `                     \
-   `# gitlab-cli, or utils features changed`                             \
-|| grep -qE "(${common_features})"  <<< "${features}"                    \
+if `# Include all images if full_matrix is true`                           \
+   [ "${full_matrix}" == "1" ]                                             \
+   `# Include all images if matrix or workflows changed`                   \
+|| grep -q 'matrix\.yml'                                   <<< "${files}"  \
+|| grep -q '\.github/workflows/test\.yml'                   <<< "${files}" \
+|| grep -q '\.github/actions/build-linux-image/action\.yml' <<< "${files}" \
+|| grep -q '\.github/actions/image-matrix/action\.sh'       <<< "${files}" \
+|| grep -q '\.github/actions/image-matrix/action\.yml'      <<< "${files}" \
+|| grep -q '\.github/workflows/build-and-test-image\.yml'   <<< "${files}" \
+   `# Include all images if cmake, ninja, sccache, `                       \
+   `# gitlab-cli, or utils features changed`                               \
+|| grep -qE "(${common_features})"  <<< "${features}"                      \
 ; then
   features="$(                                                  \
       find features/src -mindepth 1 -maxdepth 1 -type d -print0 \

diff --git a/.github/actions/image-matrix/action.yml b/.github/actions/image-matrix/action.yml
@@ -38,10 +38,10 @@ runs:
           windows/**
           matrix.yml
           .github/**/*windows*
-          .github/workflows/test.yml
           .github/actions/build-linux-image/action.yml
           .github/actions/image-matrix/action.sh
           .github/actions/image-matrix/action.yml
+          .github/workflows/test.yml
           .github/workflows/build-and-test-image.yml
 
     - name: Report changes

diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml
@@ -0,0 +1,53 @@
+name: Build all RAPIDS repositories
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_call:
+
+jobs:
+  check-event:
+    name: Check GH Event
+    runs-on: ubuntu-latest
+    outputs:
+      ok: ${{ steps.check_gh_event.outputs.ok }}
+    steps:
+      - id: check_gh_event
+        name: Check GH Event
+        shell: bash
+        run: |
+          [[ '${{ github.event_name }}' == 'push' && '${{ github.repository }}' == 'rapidsai/devcontainers' ]] || \
+          [[ '${{ github.event_name }}' == 'schedule' && '${{ github.repository }}' == 'rapidsai/devcontainers' ]] || \
+          [[ '${{ github.event_name }}' == 'pull_request' && '${{ github.repository }}' != 'rapidsai/devcontainers' ]] \
+          && echo "ok=true"  | tee -a $GITHUB_OUTPUT \
+          || echo "ok=false" | tee -a $GITHUB_OUTPUT;
+
+  build-all-rapids-repos:
+    name: Build all RAPIDS repos
+    if: needs.check-event.outputs.ok == 'true'
+    needs: check-event
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location
+    with:
+      arch: '["amd64"]'
+      cuda: '["12.0", "12.2"]'
+      node_type: cpu32
+      extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY
+      extra-repo-deploy-key-2: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
+      build_command: |
+        sccache -z;
+        sccache --show-adv-stats;
+        clone-all --verbose --clone-upstream --no-update-env;
+        rapids-generate-scripts;
+        rapids-make-${PYTHON_PACKAGE_MANAGER}-env;
+        build-all \
+          --verbose \
+          -j$(nproc --ignore=1) \
+          -DBUILD_SHARED_LIBS=ON \
+          -DBUILD_TESTS=ON \
+          -DBUILD_BENCHMARKS=ON \
+          -DBUILD_ANN_BENCH=ON \
+          -DBUILD_PRIMS_BENCH=ON \
+          $([ "pip" = ${PYTHON_PACKAGE_MANAGER} ] && echo -DUSE_LIBARROW_FROM_PYARROW=ON || echo) \
+          ;
+        sccache --show-adv-stats;
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -145,3 +145,10 @@ jobs:
       features: "${{ toJSON(matrix.features) }}"
       container_env: "${{ toJSON(matrix.env) }}"
       repo: "${{ vars.DOCKERHUB_REPOSITORY || github.repository }}"
+
+  build-all-rapids-repos:
+    name: Build all RAPIDS repos
+    if: needs.check-event.outputs.ok == 'true'
+    needs: check-event
+    secrets: inherit
+    uses: ./.github/workflows/build-all-rapids-repos.yml
diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh
@@ -99,6 +99,6 @@ make_conda_env() {
 . /opt/conda/etc/profile.d/conda.sh;
 . /opt/conda/etc/profile.d/mamba.sh;
 
-(make_conda_env "${DEFAULT_CONDA_ENV:-rapids}" "$@");
+make_conda_env "${DEFAULT_CONDA_ENV:-rapids}" "$@";
 
 . /etc/profile.d/*-mambaforge.sh;
diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh
@@ -121,7 +121,7 @@ make_pip_env() {
     fi
 }
 
-(make_pip_env "${DEFAULT_VIRTUAL_ENV:-rapids}" "$@");
+make_pip_env "${DEFAULT_VIRTUAL_ENV:-rapids}" "$@";
 
 if test -f ~/.local/share/venvs/${DEFAULT_VIRTUAL_ENV:-rapids}/bin/activate; then
     . ~/.local/share/venvs/${DEFAULT_VIRTUAL_ENV:-rapids}/bin/activate;

diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json
@@ -1,7 +1,7 @@
 {
   "name": "devcontainer-utils",
   "id": "utils",
-  "version": "24.4.6",
+  "version": "24.4.7",
   "description": "A feature to install RAPIDS devcontainer utility scripts",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"

diff --git a/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh
@@ -59,8 +59,6 @@ clone_github_repo() {
 
     set -euo pipefail;
 
-    source devcontainer-utils-init-github-cli;
-
     local branch=;
     local no_fork=;
     local clone_upstream=;
@@ -92,7 +90,17 @@ clone_github_repo() {
     local fork=;
     local owner=;
 
-    if test -z "${clone_upstream:-}"; then
+    if test -z "${no_fork:-}" && \
+       test -z "${clone_upstream:-}" && \
+       devcontainer-utils-shell-is-interactive; then
+        # shellcheck disable=SC1091
+        . devcontainer-utils-init-github-cli;
+        user="${GITHUB_USER:-}";
+    fi
+
+    if test -n "${clone_upstream:-}"; then
+        fork="${upstream}";
+    else
         name="$(get_repo_name "${upstream}")";
         owner="$(get_repo_owner "${upstream}")";
         user="${GITHUB_USER:-"${owner}"}";
@@ -106,7 +114,7 @@ clone_github_repo() {
          devcontainer-utils-shell-is-interactive; then
         while true; do
             local CHOICE;
-            read -p "'${GITHUB_HOST:-github.com}/${user}/${name}.git' not found.
+            read -rp "'${GITHUB_HOST:-github.com}/${user}/${name}.git' not found.
     Fork '${upstream}' into '${user}/${name}' now (y/n)? " CHOICE <$(tty)
             case "${CHOICE:-}" in
                 [Nn]* ) origin="${upstream}"; break;;
@@ -118,12 +126,17 @@ clone_github_repo() {
         done
     fi
 
-    if [ "$(gh config get git_protocol)" = "ssh" ]; then
-        origin="$(get_repo_ssh_url "${origin}")";
-        upstream="$(get_repo_ssh_url "${upstream}")";
+    if gh auth status >/dev/null 2>&1; then
+        if [ "$(gh config get git_protocol)" = "ssh" ]; then
+            origin="$(get_repo_ssh_url "${origin}")";
+            upstream="$(get_repo_ssh_url "${upstream}")";
+        else
+            origin="$(get_repo_git_url "${origin}")";
+            upstream="$(get_repo_git_url "${upstream}")";
+        fi
     else
-        origin="$(get_repo_git_url "${origin}")";
-        upstream="$(get_repo_git_url "${upstream}")";
+        origin="https://${GITHUB_HOST:-github.com}/${origin}.git";
+        upstream="https://${GITHUB_HOST:-github.com}/${upstream}.git";
     fi
 
     devcontainer-utils-clone-git-repo \

diff --git a/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh
@@ -85,8 +85,6 @@ clone_gitlab_repo() {
 
     set -euo pipefail;
 
-    source devcontainer-utils-init-gitlab-cli;
-
     local branch=;
     local no_fork=;
     local clone_upstream=;
@@ -118,7 +116,17 @@ clone_gitlab_repo() {
     local user=;
     local owner=;
 
-    if test -z "${clone_upstream:-}"; then
+    if test -z "${no_fork:-}" && \
+       test -z "${clone_upstream:-}" && \
+       devcontainer-utils-shell-is-interactive; then
+        # shellcheck disable=SC1091
+        . devcontainer-utils-init-gitlab-cli;
+        user="${GITLAB_USER:-}";
+    fi
+
+    if test -n "${clone_upstream:-}"; then
+        fork="${upstream}";
+    else
         name="$(get_repo_name "${upstream}")";
         owner="$(get_repo_owner "${upstream}")";
         user="${GITLAB_USER:-"${owner}"}";
@@ -132,7 +140,7 @@ clone_gitlab_repo() {
          devcontainer-utils-shell-is-interactive; then
         while true; do
             local CHOICE;
-            read -p "'${GITLAB_HOST:-gitlab.com}/${user}/${name}.git' not found.
+            read -rp "'${GITLAB_HOST:-gitlab.com}/${user}/${name}.git' not found.
     Fork '${upstream}' into '${user}/${name}' now (y/n)? " CHOICE <$(tty)
             case "${CHOICE:-}" in
                 [Nn]* ) origin="${upstream}"; break;;
@@ -144,12 +152,17 @@ clone_gitlab_repo() {
         done
     fi
 
-    if [ "$(glab config get git_protocol)" = "ssh" ]; then
-        origin="$(get_repo_ssh_url "${origin}")";
-        upstream="$(get_repo_ssh_url "${upstream}")";
+    if ! glab auth status 2>&1 | grep -q "No token provided"; then
+        if [ "$(glab config get git_protocol)" = "ssh" ]; then
+            origin="$(get_repo_ssh_url "${origin}")";
+            upstream="$(get_repo_ssh_url "${upstream}")";
+        else
+            origin="$(get_repo_git_url "${origin}")";
+            upstream="$(get_repo_git_url "${upstream}")";
+        fi
     else
-        origin="$(get_repo_git_url "${origin}")";
-        upstream="$(get_repo_git_url "${upstream}")";
+        origin="https://${GITLAB_HOST:-gitlab.com}/${origin}.git";
+        upstream="https://${GITLAB_HOST:-gitlab.com}/${upstream}.git";
     fi
 
     devcontainer-utils-clone-git-repo \