Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[REVIEW] Use custom conda build directory for ccache integration #3658

Merged
merged 10 commits into from
Mar 25, 2021
Merged

[REVIEW] Use custom conda build directory for ccache integration #3658

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7fadcad
FIX Fix EXITCODE override in test_notebooks script
dillon-cullinan Nov 30, 2020
4343f6f
DOC Changelog update
dillon-cullinan Nov 30, 2020
1720319
FIX Move bash trap to after the GTests so they fail immediately
dillon-cullinan Nov 30, 2020
5afb4f7
FIX Move codecov block to gpu build
dillon-cullinan Nov 30, 2020
bc5056e
ENH Add conda build dir
dillon-cullinan Mar 25, 2021
c3d386d
Merge branch-0.19
dillon-cullinan Mar 25, 2021
d842d09
ENH Change conda build dir for ccache integration
dillon-cullinan Mar 25, 2021
c32159f
FIX Fix artifact folder
dillon-cullinan Mar 25, 2021
bd29f51
Add copyright header
dillon-cullinan Mar 25, 2021
704210f
FIX Fix conda bld dir ref in python build
dillon-cullinan Mar 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions ci/cpu/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ export CUDA_REL=${CUDA_VERSION%.*}
export GPUCI_CONDA_RETRY_MAX=1
export GPUCI_CONDA_RETRY_SLEEP=30

# Use Ninja to build
# Use Ninja to build, setup Conda Build Dir
export CMAKE_GENERATOR="Ninja"
export CONDA_BLD_DIR="${WORKSPACE}/.conda-bld"

# Switch to project root; also root of repo checkout
cd $WORKSPACE
Expand Down Expand Up @@ -61,22 +62,24 @@ conda config --set ssl_verify False
if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
if [ "$BUILD_LIBCUML" == '1' -o "$BUILD_CUML" == '1' ]; then
gpuci_logger "Build conda pkg for libcuml"
gpuci_conda_retry build conda/recipes/libcuml
gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcuml
fi
else
if [ "$BUILD_LIBCUML" == '1' ]; then
gpuci_logger "PROJECT FLASH: Build conda pkg for libcuml"
gpuci_conda_retry build conda/recipes/libcuml --dirty --no-remove-work-dir
gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcuml --dirty --no-remove-work-dir
mkdir -p ${CONDA_BLD_DIR}/libcuml/work
cp -r ${CONDA_BLD_DIR}/work/* ${CONDA_BLD_DIR}/libcuml/work
fi
fi

if [ "$BUILD_CUML" == '1' ]; then
if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
gpuci_logger "Build conda pkg for cuml"
gpuci_conda_retry build conda/recipes/cuml --python=${PYTHON}
gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/cuml --python=${PYTHON}
else
gpuci_logger "PROJECT FLASH: Build conda pkg for cuml"
gpuci_conda_retry build -c ci/artifacts/cuml/cpu/conda-bld/ --dirty --no-remove-work-dir conda/recipes/cuml --python=${PYTHON}
gpuci_conda_retry build --croot ${CONDA_BLD_DIR} -c ci/artifacts/cuml/cpu/conda-bld/ --dirty --no-remove-work-dir conda/recipes/cuml --python=${PYTHON}
fi
fi

Expand Down
7 changes: 3 additions & 4 deletions ci/cpu/upload.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
#!/bin/bash
#
# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
# Copyright (c) 2018-2021, NVIDIA CORPORATION.

set -e

Expand Down Expand Up @@ -29,8 +28,8 @@ fi

gpuci_logger "Get conda file output locations"

export LIBCUML_FILE=`conda build conda/recipes/libcuml --output`
export CUML_FILE=`conda build conda/recipes/cuml --python=$PYTHON --output`
export LIBCUML_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcuml --output`
export CUML_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/cuml --python=$PYTHON --output`

################################################################################
# UPLOAD - Conda packages
Expand Down
6 changes: 3 additions & 3 deletions ci/gpu/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ function hasArg {
# Set path and build parallel level
export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
export CONDA_ARTIFACT_PATH=${WORKSPACE}/ci/artifacts/cuml/cpu/.conda-bld/

# Set home to the job's workspace
export HOME=$WORKSPACE
Expand Down Expand Up @@ -108,7 +109,6 @@ if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
################################################################################
# TEST - Run GoogleTest and py.tests for libcuml and cuML
################################################################################

if hasArg --skip-tests; then
gpuci_logger "Skipping Tests"
exit 0
Expand Down Expand Up @@ -186,11 +186,11 @@ else
patchelf --replace-needed `patchelf --print-needed ./test/ml | grep faiss` libfaiss.so ./test/ml
GTEST_OUTPUT="xml:${WORKSPACE}/test-results/libcuml_cpp/" ./test/ml

CONDA_FILE=`find $WORKSPACE/ci/artifacts/cuml/cpu/conda-bld/ -name "libcuml*.tar.bz2"`
CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "libcuml*.tar.bz2"`
CONDA_FILE=`basename "$CONDA_FILE" .tar.bz2` #get filename without extension
CONDA_FILE=${CONDA_FILE//-/=} #convert to conda install
gpuci_logger "Installing $CONDA_FILE"
conda install -c $WORKSPACE/ci/artifacts/cuml/cpu/conda-bld/ "$CONDA_FILE"
conda install -c ${CONDA_ARTIFACT_PATH} "$CONDA_FILE"

gpuci_logger "Building cuml"
"$WORKSPACE/build.sh" -v cuml --codecov
Expand Down