-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Enable wheels CI scripts to run locally (#57)
Taking inspiration from #35 and #62 This PR adds / updates: - `rapids-configure-sccache` (new) : configures sccache for CI/local, with the intention being to remove this information from CI containers - `rapids-date-string` (new) : configures date string - `rapids-configure-conda-channels` (new) : modifies conda channels based on build type - `rapids-env-update` (modified) : calls the `rapids-configure-sccache` - `rapids-prompt-local-repo-config` (new) : consolidates prompts for users to configure repo information locally - `rapids-download-wheels/conda-from-s3` (updated) : uses `rapids-prompt-local-repo-config` - `rapids-upload-wheels-to-s3` (modifed) : guards to ensure local builds don't try to upload
- Loading branch information
Showing
8 changed files
with
123 additions
and
92 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| #!/bin/bash | ||
| # A utility script that configures conda channels | ||
|
|
||
| # Remove nightly channels if build is a release build | ||
| if rapids-is-release-build; then | ||
| conda config --system --remove channels rapidsai-nightly | ||
| conda config --system --remove channels dask/label/dev | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| #!/bin/bash | ||
| # A utility script that configures sccache environment variables | ||
|
|
||
| export CMAKE_CUDA_COMPILER_LAUNCHER=sccache | ||
| export CMAKE_CXX_COMPILER_LAUNCHER=sccache | ||
| export CMAKE_C_COMPILER_LAUNCHER=sccache | ||
| export SCCACHE_BUCKET=rapids-sccache-east | ||
| export SCCACHE_REGION=us-east-2 | ||
| export SCCACHE_IDLE_TIMEOUT=32768 | ||
| export SCCACHE_S3_USE_SSL=true | ||
| export SCCACHE_S3_NO_CREDENTIALS=false | ||
| if [ "${CI:-false}" = "false" ]; then | ||
| # Configure sccache for read-only mode since no credentials | ||
| # are available in local builds. | ||
| export SCCACHE_S3_NO_CREDENTIALS=true | ||
| export PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)} | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| #!/bin/bash | ||
| # A utility script produces a date string | ||
|
|
||
| RAPIDS_DATE_STRING="$(date +%y%m%d)" | ||
| if [ "${GITHUB_ACTIONS:-false}" = "true" ]; then | ||
| WORKFLOW_DATE=$(rapids-retry gh run view "${GITHUB_RUN_ID}" --json createdAt | jq -r '.createdAt') | ||
| RAPIDS_DATE_STRING=$(date -d "${WORKFLOW_DATE}" +%y%m%d) | ||
| fi | ||
| export RAPIDS_DATE_STRING |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| #!/bin/bash | ||
| # A utility script that prompts user to setup repository information in | ||
| # local environments | ||
|
|
||
| if [ "${CI:-false}" = "false" ]; then | ||
| rapids-echo-stderr "Local run detected." | ||
| rapids-echo-stderr "NVIDIA VPN connectivity is required to download workflow artifacts." | ||
|
|
||
| if [ -z "${RAPIDS_BUILD_TYPE:-}" ]; then | ||
| { | ||
| echo "" | ||
| read -r -p "Enter workflow type (one of: pull-request|branch|nightly): " RAPIDS_BUILD_TYPE | ||
| export RAPIDS_BUILD_TYPE | ||
| echo "" | ||
| echo "Suppress this prompt in the future by setting the 'RAPIDS_BUILD_TYPE' environment variable:" | ||
| echo "export RAPIDS_BUILD_TYPE=${RAPIDS_BUILD_TYPE}" | ||
| echo "" | ||
| } >&2 | ||
| fi | ||
|
|
||
| if [ -z "${RAPIDS_REPOSITORY:-}" ]; then | ||
| { | ||
| echo "" | ||
| read -r -p "Enter org/repository name (e.g. rapidsai/cudf): " RAPIDS_REPOSITORY | ||
| export RAPIDS_REPOSITORY | ||
| echo "" | ||
| echo "Suppress this prompt in the future by setting the 'RAPIDS_REPOSITORY' environment variable:" | ||
| echo "export RAPIDS_REPOSITORY=${RAPIDS_REPOSITORY}" | ||
| echo "" | ||
| } >&2 | ||
| fi | ||
|
|
||
| if [ -z "${RAPIDS_REF_NAME:-}" ]; then | ||
| { | ||
| echo "" | ||
| if [ "${RAPIDS_BUILD_TYPE}" = "pull-request" ]; then | ||
| read -r -p "Enter pull-request number (e.g. 1546): " PR_NUMBER | ||
| RAPIDS_REF_NAME=pull-request/${PR_NUMBER} | ||
| else | ||
| read -r -p "Enter branch name (e.g. branch-23.08): " RAPIDS_REF_NAME | ||
| fi | ||
| export RAPIDS_REF_NAME | ||
| echo "" | ||
| echo "Suppress this prompt in the future by setting the 'RAPIDS_REF_NAME' environment variable:" | ||
| echo "export RAPIDS_REF_NAME=${RAPIDS_REF_NAME}" | ||
| echo "" | ||
| } >&2 | ||
| fi | ||
|
|
||
| if [ -z "${RAPIDS_SHA:-}" ]; then | ||
| { | ||
| echo "" | ||
| if RAPIDS_SHA=$(set -e; git rev-parse HEAD 2> /dev/null); then | ||
| echo "Using HEAD commit for artifact commit hash. Overwrite this by setting the 'RAPIDS_SHA' environment variable:" | ||
| echo "export RAPIDS_SHA=${RAPIDS_SHA}" | ||
| else | ||
| echo "There was a problem acquiring the HEAD commit sha from the current directory." | ||
| echo "Suppress this prompt in the future by ensuring the current directory is the '${RAPIDS_REPOSITORY}' git repository." | ||
| read -r -p "Enter full commit sha (e.g. f15776cc449b0c8345f044f713c9c06eb13622f3): " RAPIDS_SHA | ||
| fi | ||
| echo "" | ||
| export RAPIDS_SHA | ||
| } >&2 | ||
| fi | ||
|
|
||
| if [ "${RAPIDS_BUILD_TYPE}" = "nightly" ] && [ -z "${RAPIDS_NIGHTLY_DATE:-}" ]; then | ||
| { | ||
| echo "" | ||
| read -r -p "Enter nightly date (e.g. 2023-06-20): " RAPIDS_NIGHTLY_DATE | ||
| export RAPIDS_NIGHTLY_DATE | ||
| echo "" | ||
| echo "Suppress this prompt in the future by setting the 'RAPIDS_NIGHTLY_DATE' environment variable:" | ||
| echo "export RAPIDS_NIGHTLY_DATE=${RAPIDS_NIGHTLY_DATE}" | ||
| echo "" | ||
| } >&2 | ||
| fi | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters