Merge pull request rapidsai#1100 from raydouglass/project-flash

[REVIEW] New build process script changes

CHANGELOG.md

@@ -17,6 +17,7 @@
 - PR #1047 Updated select tests to use new dataset list that includes asymmetric directed graph
 - PR #1090 Add experimental Leiden function
 - PR #1077 Updated/added copyright notices, added copyright CI check from cuml
+- PR #1100 Add support for new build process (Project Flash)
 
 ## Improvements
 - PR #898 Add Edge Betweenness Centrality, and endpoints to BC

build.sh

@@ -34,7 +34,7 @@ HELP="$0 [<target> ...] [<flag> ...]
 
  default action (no args) is to build and install 'libcugraph' then 'cugraph' targets
 "
-LIBCUGRAPH_BUILD_DIR=${REPODIR}/cpp/build
+LIBCUGRAPH_BUILD_DIR=${LIBCUGRAPH_BUILD_DIR:=${REPODIR}/cpp/build}
 CUGRAPH_BUILD_DIR=${REPODIR}/python/build
 BUILD_DIRS="${LIBCUGRAPH_BUILD_DIR} ${CUGRAPH_BUILD_DIR}"
 
@@ -116,7 +116,7 @@ if (( ${NUMARGS} == 0 )) || hasArg cugraph; then
 
     cd ${REPODIR}/python
     if [[ ${INSTALL_TARGET} != "" ]]; then
-	python setup.py build_ext --inplace
+	python setup.py build_ext --inplace --library-dir=${LIBCUGRAPH_BUILD_DIR}
 	python setup.py install
     else
 	python setup.py build_ext --inplace --library-dir=${LIBCUGRAPH_BUILD_DIR}

ci/cpu/cugraph/build_cugraph.sh

@@ -17,6 +17,9 @@ set -e
 if [ "$BUILD_CUGRAPH" == "1" ]; then
   echo "Building cugraph"
   CUDA_REL=${CUDA_VERSION%.*}
-
-  conda build conda/recipes/cugraph --python=$PYTHON
+  if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    conda build conda/recipes/cugraph --python=$PYTHON
+  else
+    conda build conda/recipes/cugraph -c ci/artifacts/cugraph/cpu/conda-bld/ --dirty --no-remove-work-dir --python=$PYTHON
+  fi
 fi

ci/cpu/cugraph/upload-anaconda.sh

@@ -14,7 +14,7 @@
 
 set -e
 
-if [ "$UPLOAD_CUGRAPH" == "1" ]; then
+if [[ "$BUILD_CUGRAPH" == "1" && "$UPLOAD_CUGRAPH" == "1" ]]; then
   export UPLOADFILE=`conda build conda/recipes/cugraph -c rapidsai -c nvidia -c numba -c conda-forge -c defaults --python=$PYTHON --output`
 
   SOURCE_BRANCH=master

ci/cpu/libcugraph/build_libcugraph.sh

@@ -17,6 +17,9 @@ set -e
 if [ "$BUILD_LIBCUGRAPH" == '1' ]; then
   echo "Building libcugraph"
   CUDA_REL=${CUDA_VERSION%.*}
-
-  conda build conda/recipes/libcugraph
+  if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    conda build conda/recipes/libcugraph
+  else
+    conda build --dirty --no-remove-work-dir conda/recipes/libcugraph
+  fi
 fi

ci/cpu/libcugraph/upload-anaconda.sh

@@ -14,7 +14,7 @@
 
 set -e
 
-if [ "$UPLOAD_LIBCUGRAPH" == "1" ]; then
+if [[ "$BUILD_LIBCUGRAPH" == "1" && "$UPLOAD_LIBCUGRAPH" == "1" ]]; then
   CUDA_REL=${CUDA_VERSION%.*}
 
   export UPLOADFILE=`conda build conda/recipes/libcugraph --output`

ci/cpu/prebuild.sh

@@ -12,8 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export BUILD_CUGRAPH=1
-export BUILD_LIBCUGRAPH=1
+if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    #If project flash is not activate, always build both
+    export BUILD_CUGRAPH=1
+    export BUILD_LIBCUGRAPH=1
+fi
 
 if [[ "$CUDA" == "10.1" ]]; then
     export UPLOAD_CUGRAPH=1

ci/gpu/build.sh

@@ -89,8 +89,10 @@ conda list
 # BUILD - Build libcugraph and cuGraph from source
 ################################################################################
 
-logger "Build libcugraph..."
-$WORKSPACE/build.sh clean libcugraph cugraph
+if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+  logger "Build libcugraph..."
+  $WORKSPACE/build.sh clean libcugraph cugraph
+fi
 
 ################################################################################
 # TEST - Run GoogleTest and py.tests for libcugraph and cuGraph

ci/test.sh

@@ -57,7 +57,12 @@ else
     fi
 fi
 
-cd ${CUGRAPH_ROOT}/cpp/build
+if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    cd ${CUGRAPH_ROOT}/cpp/build
+else
+    export LD_LIBRARY_PATH="$WORKSPACE/ci/artifacts/cugraph/cpu/conda_work/cpp/build:$LD_LIBRARY_PATH"
+    cd $WORKSPACE/ci/artifacts/cugraph/cpu/conda_work/cpp/build
+fi
 
 for gt in gtests/*; do
     test_name=$(basename $gt)
@@ -66,6 +71,14 @@ for gt in gtests/*; do
     ERRORCODE=$((ERRORCODE | $?))
 done
 
+if [[ "$PROJECT_FLASH" == "1" ]]; then
+    echo "Installing libcugraph..."
+    conda install -c $WORKSPACE/ci/artifacts/cugraph/cpu/conda-bld/ libcugraph
+    export LIBCUGRAPH_BUILD_DIR="$WORKSPACE/ci/artifacts/cugraph/cpu/conda_work/cpp/build"
+    echo "Build cugraph..."
+    $WORKSPACE/build.sh cugraph
+fi
+
 echo "Python pytest for cuGraph..."
 cd ${CUGRAPH_ROOT}/python
 pytest --cache-clear --junitxml=${CUGRAPH_ROOT}/junit-cugraph.xml -v --cov-config=.coveragerc --cov=cugraph --cov-report=xml:${WORKSPACE}/python/cugraph/cugraph-coverage.xml --cov-report term --ignore=cugraph/raft