From 32f398eaafee10d8b765d4f6ac0d0056499911fb Mon Sep 17 00:00:00 2001
From: Bradley Dice <bdice@bradleydice.com>
Date: Tue, 20 Feb 2024 15:16:29 -0700
Subject: [PATCH] Build RAPIDS. (#210)
* Build RAPIDS.
---------
Co-authored-by: Paul Taylor <178183+trxcllnt@users.noreply.github.com>
Co-authored-by: ptaylor <paul.e.taylor@me.com>
---
.devcontainer/rapids.Dockerfile | 15 +++++-
.github/actions/feature-matrix/action.sh | 2 +-
.github/actions/image-matrix/action.sh | 24 ++++-----
.github/actions/image-matrix/action.yml | 2 +-
.github/workflows/build-all-rapids-repos.yml | 53 +++++++++++++++++++
.github/workflows/test.yml | 7 +++
.../rapids-build-utils/bin/make-conda-env.sh | 2 +-
.../rapids-build-utils/bin/make-pip-env.sh | 2 +-
features/src/utils/devcontainer-feature.json | 2 +-
.../opt/devcontainer/bin/github/repo/clone.sh | 31 +++++++----
.../opt/devcontainer/bin/gitlab/repo/clone.sh | 31 +++++++----
11 files changed, 135 insertions(+), 36 deletions(-)
create mode 100644 .github/workflows/build-all-rapids-repos.yml
diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile
index 9bb8d78a..65025a67 100644
--- a/.devcontainer/rapids.Dockerfile
+++ b/.devcontainer/rapids.Dockerfile
@@ -7,10 +7,23 @@ FROM ${BASE} as pip-base
RUN apt update -y \
&& DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \
+ # C++ build tools
+ doxygen \
+ graphviz \
+ # C++ test dependencies
+ libgmock-dev \
+ libgtest-dev \
# needed by libcudf_kafka
librdkafka-dev \
+ # cuML/cuGraph dependencies
+ libblas-dev \
+ liblapack-dev \
# needed by libcuspatial
- sqlite3 libsqlite3-dev libtiff-dev libcurl4-openssl-dev \
+ libgdal-dev \
+ sqlite3 \
+ libsqlite3-dev \
+ libtiff-dev \
+ libcurl4-openssl-dev \
&& rm -rf /tmp/* /var/tmp/* /var/cache/apt/* /var/lib/apt/lists/*;
ENV DEFAULT_VIRTUAL_ENV=rapids
diff --git a/.github/actions/feature-matrix/action.sh b/.github/actions/feature-matrix/action.sh
index 2765d8cf..a819d49b 100755
--- a/.github/actions/feature-matrix/action.sh
+++ b/.github/actions/feature-matrix/action.sh
@@ -25,7 +25,7 @@ files="${@}";
features="";
scenarios="";
-if [ "${full_matrix}" == "1" ] || echo "$files" | grep -q ".github/"; then
+if [ "${full_matrix}" == "1" ] || grep -q "\.github/" <<< "$files"; then
features="$(find_features_with_tests 'test.sh' 'features/test')";
scenarios="$(find_features_with_tests 'scenarios.json' 'features/test')";
elif echo "$files" | grep -q "features/"; then
diff --git a/.github/actions/image-matrix/action.sh b/.github/actions/image-matrix/action.sh
index 73a7a805..2ab15f42 100755
--- a/.github/actions/image-matrix/action.sh
+++ b/.github/actions/image-matrix/action.sh
@@ -31,18 +31,18 @@ features="$(echo \
|| echo '' \
)";
-if `# Include all images if full_matrix is true` \
- [ "${full_matrix}" == "1" ] \
- `# Include all images if matrix or workflows changed` \
-|| grep -q 'matrix.yml' <<< "${files}" \
-|| grep -q '.github/workflows/test.yml' <<< "${files}" \
-|| grep -q '.github/actions/build-linux-image/action.yml' <<< "${files}" \
-|| grep -q '.github/actions/image-matrix/action.sh' <<< "${files}" \
-|| grep -q '.github/actions/image-matrix/action.yml' <<< "${files}" \
-|| grep -q '.github/workflows/build-and-test-image.yml' <<< "${files}" \
- `# Include all images if cmake, ninja, sccache, ` \
- `# gitlab-cli, or utils features changed` \
-|| grep -qE "(${common_features})" <<< "${features}" \
+if `# Include all images if full_matrix is true` \
+ [ "${full_matrix}" == "1" ] \
+ `# Include all images if matrix or workflows changed` \
+|| grep -q 'matrix\.yml' <<< "${files}" \
+|| grep -q '\.github/workflows/test\.yml' <<< "${files}" \
+|| grep -q '\.github/actions/build-linux-image/action\.yml' <<< "${files}" \
+|| grep -q '\.github/actions/image-matrix/action\.sh' <<< "${files}" \
+|| grep -q '\.github/actions/image-matrix/action\.yml' <<< "${files}" \
+|| grep -q '\.github/workflows/build-and-test-image\.yml' <<< "${files}" \
+ `# Include all images if cmake, ninja, sccache, ` \
+ `# gitlab-cli, or utils features changed` \
+|| grep -qE "(${common_features})" <<< "${features}" \
; then
features="$( \
find features/src -mindepth 1 -maxdepth 1 -type d -print0 \
diff --git a/.github/actions/image-matrix/action.yml b/.github/actions/image-matrix/action.yml
index 326691b9..256f37b1 100644
--- a/.github/actions/image-matrix/action.yml
+++ b/.github/actions/image-matrix/action.yml
@@ -38,10 +38,10 @@ runs:
windows/**
matrix.yml
.github/**/*windows*
- .github/workflows/test.yml
.github/actions/build-linux-image/action.yml
.github/actions/image-matrix/action.sh
.github/actions/image-matrix/action.yml
+ .github/workflows/test.yml
.github/workflows/build-and-test-image.yml
- name: Report changes
diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml
new file mode 100644
index 00000000..06cc28f3
--- /dev/null
+++ b/.github/workflows/build-all-rapids-repos.yml
@@ -0,0 +1,53 @@
+name: Build all RAPIDS repositories
+
+on:
+ schedule:
+ - cron: '0 0 * * *'
+ workflow_call:
+
+jobs:
+ check-event:
+ name: Check GH Event
+ runs-on: ubuntu-latest
+ outputs:
+ ok: ${{ steps.check_gh_event.outputs.ok }}
+ steps:
+ - id: check_gh_event
+ name: Check GH Event
+ shell: bash
+ run: |
+ [[ '${{ github.event_name }}' == 'push' && '${{ github.repository }}' == 'rapidsai/devcontainers' ]] || \
+ [[ '${{ github.event_name }}' == 'schedule' && '${{ github.repository }}' == 'rapidsai/devcontainers' ]] || \
+ [[ '${{ github.event_name }}' == 'pull_request' && '${{ github.repository }}' != 'rapidsai/devcontainers' ]] \
+ && echo "ok=true" | tee -a $GITHUB_OUTPUT \
+ || echo "ok=false" | tee -a $GITHUB_OUTPUT;
+
+ build-all-rapids-repos:
+ name: Build all RAPIDS repos
+ if: needs.check-event.outputs.ok == 'true'
+ needs: check-event
+ secrets: inherit
+ uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location
+ with:
+ arch: '["amd64"]'
+ cuda: '["12.0", "12.2"]'
+ node_type: cpu32
+ extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY
+ extra-repo-deploy-key-2: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
+ build_command: |
+ sccache -z;
+ sccache --show-adv-stats;
+ clone-all --verbose --clone-upstream --no-update-env;
+ rapids-generate-scripts;
+ rapids-make-${PYTHON_PACKAGE_MANAGER}-env;
+ build-all \
+ --verbose \
+ -j$(nproc --ignore=1) \
+ -DBUILD_SHARED_LIBS=ON \
+ -DBUILD_TESTS=ON \
+ -DBUILD_BENCHMARKS=ON \
+ -DBUILD_ANN_BENCH=ON \
+ -DBUILD_PRIMS_BENCH=ON \
+ $([ "pip" = ${PYTHON_PACKAGE_MANAGER} ] && echo -DUSE_LIBARROW_FROM_PYARROW=ON || echo) \
+ ;
+ sccache --show-adv-stats;
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2a6653a3..644b1685 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -145,3 +145,10 @@ jobs:
features: "${{ toJSON(matrix.features) }}"
container_env: "${{ toJSON(matrix.env) }}"
repo: "${{ vars.DOCKERHUB_REPOSITORY || github.repository }}"
+
+ build-all-rapids-repos:
+ name: Build all RAPIDS repos
+ if: needs.check-event.outputs.ok == 'true'
+ needs: check-event
+ secrets: inherit
+ uses: ./.github/workflows/build-all-rapids-repos.yml
diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh
index a9c75e00..c7118afa 100755
--- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh
+++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh
@@ -99,6 +99,6 @@ make_conda_env() {
. /opt/conda/etc/profile.d/conda.sh;
. /opt/conda/etc/profile.d/mamba.sh;
-(make_conda_env "${DEFAULT_CONDA_ENV:-rapids}" "$@");
+make_conda_env "${DEFAULT_CONDA_ENV:-rapids}" "$@";
. /etc/profile.d/*-mambaforge.sh;
diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh
index 964ff095..24612d54 100755
--- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh
+++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh
@@ -121,7 +121,7 @@ make_pip_env() {
fi
}
-(make_pip_env "${DEFAULT_VIRTUAL_ENV:-rapids}" "$@");
+make_pip_env "${DEFAULT_VIRTUAL_ENV:-rapids}" "$@";
if test -f ~/.local/share/venvs/${DEFAULT_VIRTUAL_ENV:-rapids}/bin/activate; then
. ~/.local/share/venvs/${DEFAULT_VIRTUAL_ENV:-rapids}/bin/activate;
diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json
index 914e3fd3..acc028a1 100644
--- a/features/src/utils/devcontainer-feature.json
+++ b/features/src/utils/devcontainer-feature.json
@@ -1,7 +1,7 @@
{
"name": "devcontainer-utils",
"id": "utils",
- "version": "24.4.6",
+ "version": "24.4.7",
"description": "A feature to install RAPIDS devcontainer utility scripts",
"containerEnv": {
"BASH_ENV": "/etc/bash.bash_env"
diff --git a/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh
index fc2f1e87..aff6b36e 100755
--- a/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh
+++ b/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh
@@ -59,8 +59,6 @@ clone_github_repo() {
set -euo pipefail;
- source devcontainer-utils-init-github-cli;
-
local branch=;
local no_fork=;
local clone_upstream=;
@@ -92,7 +90,17 @@ clone_github_repo() {
local fork=;
local owner=;
- if test -z "${clone_upstream:-}"; then
+ if test -z "${no_fork:-}" && \
+ test -z "${clone_upstream:-}" && \
+ devcontainer-utils-shell-is-interactive; then
+ # shellcheck disable=SC1091
+ . devcontainer-utils-init-github-cli;
+ user="${GITHUB_USER:-}";
+ fi
+
+ if test -n "${clone_upstream:-}"; then
+ fork="${upstream}";
+ else
name="$(get_repo_name "${upstream}")";
owner="$(get_repo_owner "${upstream}")";
user="${GITHUB_USER:-"${owner}"}";
@@ -106,7 +114,7 @@ clone_github_repo() {
devcontainer-utils-shell-is-interactive; then
while true; do
local CHOICE;
- read -p "'${GITHUB_HOST:-github.com}/${user}/${name}.git' not found.
+ read -rp "'${GITHUB_HOST:-github.com}/${user}/${name}.git' not found.
Fork '${upstream}' into '${user}/${name}' now (y/n)? " CHOICE <$(tty)
case "${CHOICE:-}" in
[Nn]* ) origin="${upstream}"; break;;
@@ -118,12 +126,17 @@ clone_github_repo() {
done
fi
- if [ "$(gh config get git_protocol)" = "ssh" ]; then
- origin="$(get_repo_ssh_url "${origin}")";
- upstream="$(get_repo_ssh_url "${upstream}")";
+ if gh auth status >/dev/null 2>&1; then
+ if [ "$(gh config get git_protocol)" = "ssh" ]; then
+ origin="$(get_repo_ssh_url "${origin}")";
+ upstream="$(get_repo_ssh_url "${upstream}")";
+ else
+ origin="$(get_repo_git_url "${origin}")";
+ upstream="$(get_repo_git_url "${upstream}")";
+ fi
else
- origin="$(get_repo_git_url "${origin}")";
- upstream="$(get_repo_git_url "${upstream}")";
+ origin="https://${GITHUB_HOST:-github.com}/${origin}.git";
+ upstream="https://${GITHUB_HOST:-github.com}/${upstream}.git";
fi
devcontainer-utils-clone-git-repo \
diff --git a/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh
index 55d1b161..82c51ac5 100755
--- a/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh
+++ b/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh
@@ -85,8 +85,6 @@ clone_gitlab_repo() {
set -euo pipefail;
- source devcontainer-utils-init-gitlab-cli;
-
local branch=;
local no_fork=;
local clone_upstream=;
@@ -118,7 +116,17 @@ clone_gitlab_repo() {
local user=;
local owner=;
- if test -z "${clone_upstream:-}"; then
+ if test -z "${no_fork:-}" && \
+ test -z "${clone_upstream:-}" && \
+ devcontainer-utils-shell-is-interactive; then
+ # shellcheck disable=SC1091
+ . devcontainer-utils-init-gitlab-cli;
+ user="${GITLAB_USER:-}";
+ fi
+
+ if test -n "${clone_upstream:-}"; then
+ fork="${upstream}";
+ else
name="$(get_repo_name "${upstream}")";
owner="$(get_repo_owner "${upstream}")";
user="${GITLAB_USER:-"${owner}"}";
@@ -132,7 +140,7 @@ clone_gitlab_repo() {
devcontainer-utils-shell-is-interactive; then
while true; do
local CHOICE;
- read -p "'${GITLAB_HOST:-gitlab.com}/${user}/${name}.git' not found.
+ read -rp "'${GITLAB_HOST:-gitlab.com}/${user}/${name}.git' not found.
Fork '${upstream}' into '${user}/${name}' now (y/n)? " CHOICE <$(tty)
case "${CHOICE:-}" in
[Nn]* ) origin="${upstream}"; break;;
@@ -144,12 +152,17 @@ clone_gitlab_repo() {
done
fi
- if [ "$(glab config get git_protocol)" = "ssh" ]; then
- origin="$(get_repo_ssh_url "${origin}")";
- upstream="$(get_repo_ssh_url "${upstream}")";
+ if ! glab auth status 2>&1 | grep -q "No token provided"; then
+ if [ "$(glab config get git_protocol)" = "ssh" ]; then
+ origin="$(get_repo_ssh_url "${origin}")";
+ upstream="$(get_repo_ssh_url "${upstream}")";
+ else
+ origin="$(get_repo_git_url "${origin}")";
+ upstream="$(get_repo_git_url "${upstream}")";
+ fi
else
- origin="$(get_repo_git_url "${origin}")";
- upstream="$(get_repo_git_url "${upstream}")";
+ origin="https://${GITLAB_HOST:-gitlab.com}/${origin}.git";
+ upstream="https://${GITLAB_HOST:-gitlab.com}/${upstream}.git";
fi
devcontainer-utils-clone-git-repo \