diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index 9bb8d78a..65025a67 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -7,10 +7,23 @@ FROM ${BASE} as pip-base RUN apt update -y \ && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \ + # C++ build tools + doxygen \ + graphviz \ + # C++ test dependencies + libgmock-dev \ + libgtest-dev \ # needed by libcudf_kafka librdkafka-dev \ + # cuML/cuGraph dependencies + libblas-dev \ + liblapack-dev \ # needed by libcuspatial - sqlite3 libsqlite3-dev libtiff-dev libcurl4-openssl-dev \ + libgdal-dev \ + sqlite3 \ + libsqlite3-dev \ + libtiff-dev \ + libcurl4-openssl-dev \ && rm -rf /tmp/* /var/tmp/* /var/cache/apt/* /var/lib/apt/lists/*; ENV DEFAULT_VIRTUAL_ENV=rapids diff --git a/.github/actions/feature-matrix/action.sh b/.github/actions/feature-matrix/action.sh index 2765d8cf..a819d49b 100755 --- a/.github/actions/feature-matrix/action.sh +++ b/.github/actions/feature-matrix/action.sh @@ -25,7 +25,7 @@ files="${@}"; features=""; scenarios=""; -if [ "${full_matrix}" == "1" ] || echo "$files" | grep -q ".github/"; then +if [ "${full_matrix}" == "1" ] || grep -q "\.github/" <<< "$files"; then features="$(find_features_with_tests 'test.sh' 'features/test')"; scenarios="$(find_features_with_tests 'scenarios.json' 'features/test')"; elif echo "$files" | grep -q "features/"; then diff --git a/.github/actions/image-matrix/action.sh b/.github/actions/image-matrix/action.sh index 73a7a805..2ab15f42 100755 --- a/.github/actions/image-matrix/action.sh +++ b/.github/actions/image-matrix/action.sh @@ -31,18 +31,18 @@ features="$(echo \ || echo '' \ )"; -if `# Include all images if full_matrix is true` \ - [ "${full_matrix}" == "1" ] \ - `# Include all images if matrix or workflows changed` \ -|| grep -q 'matrix.yml' <<< "${files}" \ -|| grep -q '.github/workflows/test.yml' <<< "${files}" \ -|| grep -q '.github/actions/build-linux-image/action.yml' <<< "${files}" \ -|| grep -q '.github/actions/image-matrix/action.sh' <<< "${files}" \ -|| grep -q '.github/actions/image-matrix/action.yml' <<< "${files}" \ -|| grep -q '.github/workflows/build-and-test-image.yml' <<< "${files}" \ - `# Include all images if cmake, ninja, sccache, ` \ - `# gitlab-cli, or utils features changed` \ -|| grep -qE "(${common_features})" <<< "${features}" \ +if `# Include all images if full_matrix is true` \ + [ "${full_matrix}" == "1" ] \ + `# Include all images if matrix or workflows changed` \ +|| grep -q 'matrix\.yml' <<< "${files}" \ +|| grep -q '\.github/workflows/test\.yml' <<< "${files}" \ +|| grep -q '\.github/actions/build-linux-image/action\.yml' <<< "${files}" \ +|| grep -q '\.github/actions/image-matrix/action\.sh' <<< "${files}" \ +|| grep -q '\.github/actions/image-matrix/action\.yml' <<< "${files}" \ +|| grep -q '\.github/workflows/build-and-test-image\.yml' <<< "${files}" \ + `# Include all images if cmake, ninja, sccache, ` \ + `# gitlab-cli, or utils features changed` \ +|| grep -qE "(${common_features})" <<< "${features}" \ ; then features="$( \ find features/src -mindepth 1 -maxdepth 1 -type d -print0 \ diff --git a/.github/actions/image-matrix/action.yml b/.github/actions/image-matrix/action.yml index 326691b9..256f37b1 100644 --- a/.github/actions/image-matrix/action.yml +++ b/.github/actions/image-matrix/action.yml @@ -38,10 +38,10 @@ runs: windows/** matrix.yml .github/**/*windows* - .github/workflows/test.yml .github/actions/build-linux-image/action.yml .github/actions/image-matrix/action.sh .github/actions/image-matrix/action.yml + .github/workflows/test.yml .github/workflows/build-and-test-image.yml - name: Report changes diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml new file mode 100644 index 00000000..06cc28f3 --- /dev/null +++ b/.github/workflows/build-all-rapids-repos.yml @@ -0,0 +1,53 @@ +name: Build all RAPIDS repositories + +on: + schedule: + - cron: '0 0 * * *' + workflow_call: + +jobs: + check-event: + name: Check GH Event + runs-on: ubuntu-latest + outputs: + ok: ${{ steps.check_gh_event.outputs.ok }} + steps: + - id: check_gh_event + name: Check GH Event + shell: bash + run: | + [[ '${{ github.event_name }}' == 'push' && '${{ github.repository }}' == 'rapidsai/devcontainers' ]] || \ + [[ '${{ github.event_name }}' == 'schedule' && '${{ github.repository }}' == 'rapidsai/devcontainers' ]] || \ + [[ '${{ github.event_name }}' == 'pull_request' && '${{ github.repository }}' != 'rapidsai/devcontainers' ]] \ + && echo "ok=true" | tee -a $GITHUB_OUTPUT \ + || echo "ok=false" | tee -a $GITHUB_OUTPUT; + + build-all-rapids-repos: + name: Build all RAPIDS repos + if: needs.check-event.outputs.ok == 'true' + needs: check-event + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location + with: + arch: '["amd64"]' + cuda: '["12.0", "12.2"]' + node_type: cpu32 + extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY + extra-repo-deploy-key-2: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + build_command: | + sccache -z; + sccache --show-adv-stats; + clone-all --verbose --clone-upstream --no-update-env; + rapids-generate-scripts; + rapids-make-${PYTHON_PACKAGE_MANAGER}-env; + build-all \ + --verbose \ + -j$(nproc --ignore=1) \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTS=ON \ + -DBUILD_BENCHMARKS=ON \ + -DBUILD_ANN_BENCH=ON \ + -DBUILD_PRIMS_BENCH=ON \ + $([ "pip" = ${PYTHON_PACKAGE_MANAGER} ] && echo -DUSE_LIBARROW_FROM_PYARROW=ON || echo) \ + ; + sccache --show-adv-stats; diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a6653a3..644b1685 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -145,3 +145,10 @@ jobs: features: "${{ toJSON(matrix.features) }}" container_env: "${{ toJSON(matrix.env) }}" repo: "${{ vars.DOCKERHUB_REPOSITORY || github.repository }}" + + build-all-rapids-repos: + name: Build all RAPIDS repos + if: needs.check-event.outputs.ok == 'true' + needs: check-event + secrets: inherit + uses: ./.github/workflows/build-all-rapids-repos.yml diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index a9c75e00..c7118afa 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -99,6 +99,6 @@ make_conda_env() { . /opt/conda/etc/profile.d/conda.sh; . /opt/conda/etc/profile.d/mamba.sh; -(make_conda_env "${DEFAULT_CONDA_ENV:-rapids}" "$@"); +make_conda_env "${DEFAULT_CONDA_ENV:-rapids}" "$@"; . /etc/profile.d/*-mambaforge.sh; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh index 964ff095..24612d54 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh @@ -121,7 +121,7 @@ make_pip_env() { fi } -(make_pip_env "${DEFAULT_VIRTUAL_ENV:-rapids}" "$@"); +make_pip_env "${DEFAULT_VIRTUAL_ENV:-rapids}" "$@"; if test -f ~/.local/share/venvs/${DEFAULT_VIRTUAL_ENV:-rapids}/bin/activate; then . ~/.local/share/venvs/${DEFAULT_VIRTUAL_ENV:-rapids}/bin/activate; diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json index 914e3fd3..acc028a1 100644 --- a/features/src/utils/devcontainer-feature.json +++ b/features/src/utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "devcontainer-utils", "id": "utils", - "version": "24.4.6", + "version": "24.4.7", "description": "A feature to install RAPIDS devcontainer utility scripts", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh index fc2f1e87..aff6b36e 100755 --- a/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh +++ b/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh @@ -59,8 +59,6 @@ clone_github_repo() { set -euo pipefail; - source devcontainer-utils-init-github-cli; - local branch=; local no_fork=; local clone_upstream=; @@ -92,7 +90,17 @@ clone_github_repo() { local fork=; local owner=; - if test -z "${clone_upstream:-}"; then + if test -z "${no_fork:-}" && \ + test -z "${clone_upstream:-}" && \ + devcontainer-utils-shell-is-interactive; then + # shellcheck disable=SC1091 + . devcontainer-utils-init-github-cli; + user="${GITHUB_USER:-}"; + fi + + if test -n "${clone_upstream:-}"; then + fork="${upstream}"; + else name="$(get_repo_name "${upstream}")"; owner="$(get_repo_owner "${upstream}")"; user="${GITHUB_USER:-"${owner}"}"; @@ -106,7 +114,7 @@ clone_github_repo() { devcontainer-utils-shell-is-interactive; then while true; do local CHOICE; - read -p "'${GITHUB_HOST:-github.com}/${user}/${name}.git' not found. + read -rp "'${GITHUB_HOST:-github.com}/${user}/${name}.git' not found. Fork '${upstream}' into '${user}/${name}' now (y/n)? " CHOICE <$(tty) case "${CHOICE:-}" in [Nn]* ) origin="${upstream}"; break;; @@ -118,12 +126,17 @@ clone_github_repo() { done fi - if [ "$(gh config get git_protocol)" = "ssh" ]; then - origin="$(get_repo_ssh_url "${origin}")"; - upstream="$(get_repo_ssh_url "${upstream}")"; + if gh auth status >/dev/null 2>&1; then + if [ "$(gh config get git_protocol)" = "ssh" ]; then + origin="$(get_repo_ssh_url "${origin}")"; + upstream="$(get_repo_ssh_url "${upstream}")"; + else + origin="$(get_repo_git_url "${origin}")"; + upstream="$(get_repo_git_url "${upstream}")"; + fi else - origin="$(get_repo_git_url "${origin}")"; - upstream="$(get_repo_git_url "${upstream}")"; + origin="https://${GITHUB_HOST:-github.com}/${origin}.git"; + upstream="https://${GITHUB_HOST:-github.com}/${upstream}.git"; fi devcontainer-utils-clone-git-repo \ diff --git a/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh index 55d1b161..82c51ac5 100755 --- a/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh +++ b/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh @@ -85,8 +85,6 @@ clone_gitlab_repo() { set -euo pipefail; - source devcontainer-utils-init-gitlab-cli; - local branch=; local no_fork=; local clone_upstream=; @@ -118,7 +116,17 @@ clone_gitlab_repo() { local user=; local owner=; - if test -z "${clone_upstream:-}"; then + if test -z "${no_fork:-}" && \ + test -z "${clone_upstream:-}" && \ + devcontainer-utils-shell-is-interactive; then + # shellcheck disable=SC1091 + . devcontainer-utils-init-gitlab-cli; + user="${GITLAB_USER:-}"; + fi + + if test -n "${clone_upstream:-}"; then + fork="${upstream}"; + else name="$(get_repo_name "${upstream}")"; owner="$(get_repo_owner "${upstream}")"; user="${GITLAB_USER:-"${owner}"}"; @@ -132,7 +140,7 @@ clone_gitlab_repo() { devcontainer-utils-shell-is-interactive; then while true; do local CHOICE; - read -p "'${GITLAB_HOST:-gitlab.com}/${user}/${name}.git' not found. + read -rp "'${GITLAB_HOST:-gitlab.com}/${user}/${name}.git' not found. Fork '${upstream}' into '${user}/${name}' now (y/n)? " CHOICE <$(tty) case "${CHOICE:-}" in [Nn]* ) origin="${upstream}"; break;; @@ -144,12 +152,17 @@ clone_gitlab_repo() { done fi - if [ "$(glab config get git_protocol)" = "ssh" ]; then - origin="$(get_repo_ssh_url "${origin}")"; - upstream="$(get_repo_ssh_url "${upstream}")"; + if ! glab auth status 2>&1 | grep -q "No token provided"; then + if [ "$(glab config get git_protocol)" = "ssh" ]; then + origin="$(get_repo_ssh_url "${origin}")"; + upstream="$(get_repo_ssh_url "${upstream}")"; + else + origin="$(get_repo_git_url "${origin}")"; + upstream="$(get_repo_git_url "${upstream}")"; + fi else - origin="$(get_repo_git_url "${origin}")"; - upstream="$(get_repo_git_url "${upstream}")"; + origin="https://${GITLAB_HOST:-gitlab.com}/${origin}.git"; + upstream="https://${GITLAB_HOST:-gitlab.com}/${upstream}.git"; fi devcontainer-utils-clone-git-repo \