From 55d73a38bb399f789ff57b3d0eacdd8fbb360652 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Fri, 24 May 2019 11:53:05 -0700 Subject: [PATCH] Initial version of build.sh for cugraph, plus updates to corresponding scripts and docs. --- CONTRIBUTING.md | 75 ++++++++++++------- build.sh | 118 ++++++++++++++++++++++++++++++ ci/docs/build.sh | 18 +---- ci/gpu/build.sh | 18 +---- conda/recipes/cugraph/build.sh | 4 +- conda/recipes/libcugraph/build.sh | 12 +-- 6 files changed, 175 insertions(+), 70 deletions(-) create mode 100755 build.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c861396ec80..d184347caaf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,11 +51,11 @@ implementation of the issue, ask them in the issue instead of the PR. ## Setting Up Your Build Environment -### Build from Source +### Build from Source The following instructions are for developers and contributors to cuGraph OSS development. These instructions are tested on Linux Ubuntu 16.04 & 18.04. Use these instructions to build cuGraph from source and contribute to its development. Other operating systems may be compatible, but are not currently tested. -The cuGraph package include both a C/C++ CUDA portion and a python portion. Both libraries need to be installed in order for cuGraph to operate correctly. +The cuGraph package include both a C/C++ CUDA portion and a python portion. Both libraries need to be installed in order for cuGraph to operate correctly. The following instructions are tested on Linux systems. @@ -97,7 +97,7 @@ To install cuGraph from source, ensure the dependencies are met and follow the s 1) Clone the repository and submodules ```bash - # Set the localtion to cuGraph in an environment variable CUGRAPH_HOME + # Set the localtion to cuGraph in an environment variable CUGRAPH_HOME export CUGRAPH_HOME=$(pwd)/cugraph # Download the cuGraph repo @@ -108,7 +108,7 @@ To install cuGraph from source, ensure the dependencies are met and follow the s git submodule update --init --recursive ``` -2) Create the conda development environment +2) Create the conda development environment ```bash # create the conda environment (assuming in base `cugraph` directory) @@ -119,13 +119,13 @@ conda env create --name cugraph_dev --file conda/environments/cugraph_dev.yml conda env create --name cugraph_dev --file conda/environments/cugraph_dev_cuda10.yml # activate the environment -conda activate cugraph_dev +conda activate cugraph_dev # to deactivate an environment conda deactivate ``` - - The environment can be updated as development includes/changes the dependencies. To do so, run: + - The environment can be updated as development includes/changes the dependencies. To do so, run: ```bash @@ -135,7 +135,7 @@ conda env update --name cugraph_dev --file conda/environments/cugraph_dev.yml # for CUDA 10 conda env update --name cugraph_dev --file conda/environments/cugraph_dev_cuda10.yml -conda activate cugraph_dev +conda activate cugraph_dev ``` 3) Build and install `libcugraph`. CMake depends on the `nvcc` executable being on your path or defined in `$CUDACXX`. @@ -143,21 +143,27 @@ conda activate cugraph_dev This project uses cmake for building the C/C++ library. CMake will also automatically build and install nvGraph library (`$CUGRAPH_HOME/cpp/nvgraph`) which may take a few minutes. To configure cmake, run: ```bash - # Set the localtion to cuGraph in an environment variable CUGRAPH_HOME + # Set the localtion to cuGraph in an environment variable CUGRAPH_HOME export CUGRAPH_HOME=$(pwd)/cugraph - + cd $CUGRAPH_HOME - cd cpp # enter cpp directory - mkdir build # create build directory - cd build # enter the build directory - cmake .. -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX + cd cpp # enter cpp directory + mkdir build # create build directory + cd build # enter the build directory + cmake .. -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX # now build the code - make -j # "-j" starts multiple threads - make install # install the libraries + make -j # "-j" starts multiple threads + make install # install the libraries ``` +The default installation locations are `$CMAKE_INSTALL_PREFIX/lib` and `$CMAKE_INSTALL_PREFIX/include/cugraph` respectively. -The default installation locations are `$CMAKE_INSTALL_PREFIX/lib` and `$CMAKE_INSTALL_PREFIX/include/cugraph` respectively. +As a convenience, a `build.sh` script is provided in `$CUGRAPH_HOME`. To execute the same build commands above, run the script as shown below. Note that the libraries will be installed to the location set in `$INSTALL_PREFIX` if set (i.e. `export INSTALL_PREFIX=/install/path`), otherwise to `$CONDA_PREFIX`. +```bash +$ cd $CUGRAPH_HOME +$ ./build.sh libcugraph # build the cuGraph libraries and install them to + # $INSTALL_PREFIX if set, otherwise $CONDA_PREFIX +``` #### Building and installing the Python package @@ -169,20 +175,37 @@ cd python python setup.py install # install cugraph python bindings ``` +Like the `libcugraph` build step above, `build.sh` can also be used to build the `cugraph` python package, as shown below: +```bash +$ cd $CUGRAPH_HOME +$ ./build.sh cugraph # build the cuGraph python bindings and install them + # to $INSTALL_PREFIX if set, otherwise $CONDA_PREFIX +``` + +Note: other `build.sh` options include: +```bash +$ cd $CUGRAPH_HOME +$ ./build.sh clean # remove any prior build artifacts and configuration (start over) +$ ./build.sh libcugraph -v # compile and install libcugraph with verbose output +$ ./build.sh libcugraph -g # compile and install libcugraph for debug +$ PARALLEL_LEVEL=4 ./build.sh libcugraph # compile and install libcugraph limiting parallel build jobs to 4 (make -j4) +$ ./build.sh libcugraph -n # compile libcugraph but do not install +``` + #### Run tests 6. Run either the C++ or the Python tests with datasets - - **Python tests with datasets** + - **Python tests with datasets** ```bash cd $CUGRAPH_HOME cd python - pytest + pytest ``` - - **C++ stand alone tests** + - **C++ stand alone tests** - From the build directory : + From the build directory : ```bash # Run the cugraph tests @@ -191,21 +214,21 @@ python setup.py install # install cugraph python bindings gtests/GDFGRAPH_TEST # this is an executable file ``` - **C++ tests with larger datasets** - + If you already have the datasets: - + ```bash export RAPIDS_DATASET_ROOT_DIR= ``` If you do not have the datasets: - + ```bash cd $CUGRAPH_HOME/datasets source get_test_data.sh #This takes about 10 minutes and download 1GB data (>5 GB uncompressed) ``` - + Run the C++ tests on large input: - + ```bash cd $CUGRAPH_HOME/cpp/build #test one particular analytics (eg. pagerank) @@ -258,7 +281,7 @@ Python API documentation can be generated from [docs](docs) directory. ## C++ ABI issues -cuGraph builds with C++14 features. By default, we build cuGraph with the latest ABI (the ABI changed with C++11). The version of cuDF pointed to in +cuGraph builds with C++14 features. By default, we build cuGraph with the latest ABI (the ABI changed with C++11). The version of cuDF pointed to in the conda installation above is build with the new ABI. If you see link errors indicating trouble finding functions that use C++ strings when trying to build cuGraph you may have an ABI incompatibility. diff --git a/build.sh b/build.sh new file mode 100755 index 00000000000..743b9c533bf --- /dev/null +++ b/build.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +# Copyright (c) 2019, NVIDIA CORPORATION. + +# cugraph build script + +# This script is used to build the component(s) in this repo from +# source, and can be called with various options to customize the +# build as needed (see the help output for details) + +# Abort script on first error +set -e + +NUMARGS=$# +ARGS=$* + +# NOTE: ensure all dir changes are relative to the location of this +# script, and that this script resides in the repo dir! +REPODIR=$(cd $(dirname $0); pwd) + +VALIDARGS="clean libcugraph cugraph -v -g -n -h --help" +HELP="$0 [ ...] [ ...] + where is: + clean - remove all existing build artifacts and configuration (start over) + libcugraph - build the cugraph C++ code only + cugraph - build the cugraph Python package + and is: + -v - verbose build mode + -g - build for debug + -n - no install step + -h - print this text + + default action (no args) is to build and install 'libcugraph' then 'cugraph' targets +" +LIBCUGRAPH_BUILD_DIR=${REPODIR}/cpp/build +CUGRAPH_BUILD_DIR=${REPODIR}/python/build +BUILD_DIRS="${LIBCUGRAPH_BUILD_DIR} ${CUGRAPH_BUILD_DIR}" + +# Set defaults for vars modified by flags to this script +VERBOSE="" +BUILD_TYPE=Release +INSTALL_TARGET=install + +# Set defaults for vars that may not have been defined externally +# FIXME: if INSTALL_PREFIX is not set, check PREFIX, then check +# CONDA_PREFIX, but there is no fallback from there! +INSTALL_PREFIX=${INSTALL_PREFIX:=${PREFIX:=${CONDA_PREFIX}}} +PARALLEL_LEVEL=${PARALLEL_LEVEL:=""} +BUILD_ABI=${BUILD_ABI:=ON} + +function hasArg { + (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") +} + +if hasArg -h || hasArg --help; then + echo "${HELP}" + exit 0 +fi + +# Check for valid usage +if (( ${NUMARGS} != 0 )); then + for a in ${ARGS}; do + if ! (echo " ${VALIDARGS} " | grep -q " ${a} "); then + echo "Invalid option: ${a}" + exit 1 + fi + done +fi + +# Process flags +if hasArg -v; then + VERBOSE=1 +fi +if hasArg -g; then + BUILD_TYPE=Debug +fi +if hasArg -n; then + INSTALL_TARGET="" +fi + +# If clean given, run it prior to any other steps +if hasArg clean; then + # If the dirs to clean are mounted dirs in a container, the + # contents should be removed but the mounted dirs will remain. + # The find removes all contents but leaves the dirs, the rmdir + # attempts to remove the dirs but can fail safely. + for bd in ${BUILD_DIRS}; do + if [ -d ${bd} ]; then + find ${bd} -mindepth 1 -delete + rmdir ${bd} || true + fi + done +fi + +################################################################################ +# Configure, build, and install libcugraph +if (( ${NUMARGS} == 0 )) || hasArg libcugraph; then + + mkdir -p ${LIBCUGRAPH_BUILD_DIR} + cd ${LIBCUGRAPH_BUILD_DIR} + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ + -DCMAKE_CXX11_ABI=${BUILD_ABI} \ + -DNVG_PLUGIN=True \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} .. + make -j${PARALLEL_LEVEL} VERBOSE=${VERBOSE} ${INSTALL_TARGET} +fi + +# Build and install the cugraph Python package +if (( ${NUMARGS} == 0 )) || hasArg cugraph; then + + cd ${REPODIR}/python + if [[ ${INSTALL_TARGET} != "" ]]; then + python setup.py build_ext --inplace + python setup.py install --single-version-externally-managed --record=record.txt + else + python setup.py build_ext --inplace --library-dir=${LIBCUGRAPH_BUILD_DIR} + fi +fi diff --git a/ci/docs/build.sh b/ci/docs/build.sh index 7c72fcff0bc..31a41712d89 100644 --- a/ci/docs/build.sh +++ b/ci/docs/build.sh @@ -49,23 +49,7 @@ conda list ################################################################################ logger "Build libcugraph..." -mkdir -p $WORKSPACE/cpp/build -cd $WORKSPACE/cpp/build -logger "Run cmake libcugraph..." -cmake -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DCMAKE_CXX11_ABI=ON .. - -logger "Clean up make..." -make clean - -logger "Make libcugraph..." -make -j${PARALLEL_LEVEL} - -logger "Install libcugraph..." -make -j${PARALLEL_LEVEL} install - -logger "Build cuGraph..." -cd $WORKSPACE/python -python setup.py install +$WORKSPACE/build.sh clean libcugraph cugraph ################################################################################ # BUILD - Build docs diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 57c6ce38c57..ea1929c3ad5 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -46,23 +46,7 @@ conda list ################################################################################ logger "Build libcugraph..." -mkdir -p $WORKSPACE/cpp/build -cd $WORKSPACE/cpp/build -logger "Run cmake libcugraph..." -cmake -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DCMAKE_CXX11_ABI=ON .. - -logger "Clean up make..." -make clean - -logger "Make libcugraph..." -make -j${PARALLEL_LEVEL} - -logger "Install libcugraph..." -make -j${PARALLEL_LEVEL} install - -logger "Build cuGraph..." -cd $WORKSPACE/python -python setup.py install +$WORKSPACE/build.sh clean libcugraph cugraph ################################################################################ # TEST - Run GoogleTest and py.tests for libcugraph and cuGraph diff --git a/conda/recipes/cugraph/build.sh b/conda/recipes/cugraph/build.sh index 7f4a7693240..de2f668b862 100644 --- a/conda/recipes/cugraph/build.sh +++ b/conda/recipes/cugraph/build.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash -cd python -python setup.py install \ No newline at end of file +# This assumes the script is executed from the root of the repo directory +./build.sh cugraph diff --git a/conda/recipes/libcugraph/build.sh b/conda/recipes/libcugraph/build.sh index f3f14a9634a..ae73202ca35 100644 --- a/conda/recipes/libcugraph/build.sh +++ b/conda/recipes/libcugraph/build.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -CMAKE_COMMON_VARIABLES=" -DCMAKE_INSTALL_PREFIX=$PREFIX -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX11_ABI=$BUILD_ABI -DNVG_PLUGIN=True" + +# This assumes the script is executed from the root of the repo directory # show environment printenv @@ -7,10 +8,5 @@ printenv if [ -d .git ]; then git clean -xdf fi -# Use CMake-based build procedure -mkdir -p cpp/build -cd cpp/build -# configure -cmake $CMAKE_COMMON_VARIABLES .. -# build -make -j${PARALLEL_LEVEL} VERBOSE=1 install + +./build.sh libcugraph -v