Refactor travisci tooling for new libcudf and build processes

.travis.yml

@@ -14,14 +14,14 @@ env:
 
 matrix:
   include:
-    - env: CUDA=9.2.148_396.37 PYTHON=3.5 LABEL_MAIN=1 BUILD_CUDF=1 BUILD_LIBGDF=1 BUILD_CFFI=1
+    - env: CUDA=9.2.148_396.37 PYTHON=3.5 LABEL_MAIN=1 BUILD_CUDF=1 BUILD_LIBCUDF=1 BUILD_CFFI=1
     - env: CUDA=9.2.148_396.37 PYTHON=3.6 LABEL_MAIN=1 BUILD_CUDF=1 BUILD_CFFI=1
-    - env: CUDA=10.0.130_410.48 PYTHON=3.5 BUILD_LIBGDF=1 BUILD_CFFI=1
+    - env: CUDA=10.0.130_410.48 PYTHON=3.5 BUILD_LIBCUDF=1 BUILD_CFFI=1
     - env: CUDA=10.0.130_410.48 PYTHON=3.6 BUILD_CFFI=1
 
 before_install:
-  # install build environment for libgdf
-  - source ./travisci/libgdf/install-env-libgdf.sh
+  # install build environment for libcudf
+  - source ./travisci/libcudf/install-env-libcudf.sh
   # install miniconda
   - travis_retry wget --progress=dot:mega https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
   - chmod +x miniconda.sh
@@ -39,15 +39,15 @@ script:
   # check conda versions
   - conda list
   # build ligdf
-  - source ./travisci/libgdf/build_libgdf.sh
-  # build libgdf_cffi
-  - source ./travisci/libgdf/build_libgdf_cffi.sh
-  # upload libgdf and libgdf_cffi
-  - source ./travisci/libgdf/upload.sh
+  - source ./travisci/libcudf/build_libcudf.sh
+  # build libcudf_cffi
+  - source ./travisci/libcudf/build_libcudf_cffi.sh
+  # upload libcudf and libcudf_cffi
+  - source ./travisci/libcudf/upload.sh
   # build cudf
-  - source ./travisci/build_cudf.sh
+  - source ./travisci/cudf/build_cudf.sh
   - flake8
 
 after_success:
   # upload cudf
-  - source ./travisci/upload-anaconda.sh
+  - source ./travisci/cudf/upload-anaconda.sh

conda/environments/builddocs_py35.yml

@@ -1,18 +1,18 @@
 name: builddocs_py35
 channels:
+- mikewendt
 - nvidia
 - rapidsai
-- conda-forge
 - numba
-- mikewendt
+- conda-forge
 - defaults
 dependencies:
 - python=3.5.*
 - pytest
 - cudatoolkit=9.2
 - nvdriver=396.44
-- libgdf=0.2.0.*
-- libgdf_cffi=0.2.0.*
+- libcudf=0.2.0.*
+- libcudf_cffi=0.2.0.*
 - numba>=0.40.0dev
 - nvstrings
 - pandas=0.20.*

conda/environments/testing_py35.yml

@@ -1,9 +1,9 @@
 name: pycudf_testing_py35
 channels:
+- rapidsai
 - nvidia
 - numba
 - conda-forge
-- rapidsai
 - defaults
 dependencies:
 - python=3.5.*

conda/recipes/cudf/build.sh

@@ -1 +1,2 @@
+$PYTHON setup.py build_ext --inplace
 $PYTHON setup.py install

conda/recipes/cudf/meta.yaml

@@ -17,6 +17,12 @@ build:
     - CUDF_BUILD_NO_GPU_TEST
 
 requirements:
+  # use channel:
+  #   - rapidsai
+  #   - nvidia
+  #   - numba
+  #   - conda-forge
+  #   - defaults
   build:
     - python
     - setuptools
@@ -25,8 +31,8 @@ requirements:
   run:
     - python
     - setuptools
-    - libgdf 0.2.0.*
-    - libgdf_cffi 0.2.0.*
+    - libcudf 0.2.0.*
+    - libcudf_cffi 0.2.0.*
     - pandas 0.20.*
     - numba>=0.40.0dev
     - nvstrings

conda/recipes/libgdf/build.sh → conda/recipes/libcudf/build.sh

@@ -10,7 +10,7 @@ fi
 # Cleanup local git
 git clean -xdf
 # Change directory for build process
-cd libgdf
+cd cpp
 # Use CMake-based build procedure
 mkdir build
 cd build

conda/recipes/libgdf/meta.yaml → conda/recipes/libcudf/meta.yaml

@@ -6,7 +6,7 @@
 {% set git_revision_count=environ.get('GIT_DESCRIBE_NUMBER', 0) %}
 {% set cuda_version='.'.join(environ.get('CUDA', 'unknown').split('.')[:2]) %}
 package:
-  name: libgdf
+  name: libcudf
   version: {{ version }}
 
 source:
@@ -22,9 +22,10 @@ build:
 
 requirements:
   # use channel:
-  #   - defaults
-  #   - conda-forge
   #   - rapidsai
+  #   - nvidia
+  #   - conda-forge
+  #   - defaults
   build:
     - cmake 3.12
     - nvstrings
@@ -34,11 +35,18 @@ requirements:
 
 test:
   commands:
-    - test -f $PREFIX/include/gdf/gdf.h    # [unix or osx]
-    - test -f $PREFIX/lib/libgdf.dylib     # [osx]
+    - test -f $PREFIX/include/cudf.h
+    - test -f $PREFIX/include/cudf/convert_types.h
+    - test -f $PREFIX/include/cudf/functions.h
+    - test -f $PREFIX/include/cudf/io_functions.h
+    - test -f $PREFIX/include/cudf/io_types.h
+    - test -f $PREFIX/include/cudf/types.h
+    - test -f $PREFIX/lib/libcudf.so
+    - test -f $PREFIX/lib/librmm.so
+
 
 about:
   home: http://rapids.ai/
   license: Apache-2.0
   # license_file: LICENSE
-  summary: cuDF-libgdf library
+  summary: libcudf library

conda/recipes/libgdf_cffi/build.sh → conda/recipes/libcudf_cffi/build.sh

@@ -10,14 +10,14 @@
 # Cleanup local git
 git clean -xdf
 # Change directory for build process
-cd libgdf
+cd cpp
 # Use CMake-based build procedure
 mkdir build
 cd build
 # configure
 cmake ..
 # build
-make -j$CPU_COUNT copy_python
+make -j$CPU_COUNT python_cffi
 # install
+cd python
 python setup.py install --single-version-externally-managed --record=record.txt
-

conda/recipes/libgdf_cffi/meta.yaml → conda/recipes/libcudf_cffi/meta.yaml

@@ -7,7 +7,7 @@
 {% set cuda_version='.'.join(environ.get('CUDA', 'unknown').split('.')[:2]) %}
 {% set py_version=environ.get('CONDA_PY', 35) %}
 package:
-  name: libgdf_cffi
+  name: libcudf_cffi
   version: {{ version }}
 
 source:
@@ -23,6 +23,9 @@ build:
 
 requirements:
   # use channel:
+  #   - rapidsai
+  #   - nvidia
+  #   - conda-forge
   #   - defaults
   build:
     - cmake
@@ -31,16 +34,16 @@ requirements:
     - setuptools
     - nvstrings
   run:
-    - libgdf {{ version }}.*
+    - libcudf {{ version }}.*
     - python x.x
     - cffi
 
 test:
   commands:
-    - python -c "import libgdf_cffi"
+    - python -c "import libcudf_cffi"
 
 about:
   home: http://rapids.ai/
   license: Apache-2.0
   # license_file: LICENSE
-  summary: cuDF-libgdf_cffi library
+  summary: libcudf_cffi library

travisci/build_cudf.sh → travisci/cudf/build_cudf.sh

@@ -3,5 +3,5 @@ set -e
 if [ "$BUILD_CUDF" == "1" ]; then
   echo "Building cudf"
   export CUDF_BUILD_NO_GPU_TEST=1 
-  travis_retry conda build conda-recipes/cudf -c rapidsai -c nvidia -c numba -c conda-forge -c defaults --python=$PYTHON
+  travis_retry conda build conda/recipes/cudf -c rapidsai -c nvidia -c numba -c conda-forge -c defaults --python=$PYTHON
 fi

travisci/upload-anaconda.sh → travisci/cudf/upload-anaconda.sh

@@ -5,7 +5,7 @@
 set -e
 
 if [ "$BUILD_CUDF" == "1" ]; then
-  export UPLOADFILE=`conda build conda-recipes/cudf -c defaults -c conda-forge -c numba -c rapidsai/label/dev --python=$PYTHON --output`
+  export UPLOADFILE=`conda build conda/recipes/cudf -c rapidsai -c nvidia -c numba -c conda-forge -c defaults --python=$PYTHON --output`
   SOURCE_BRANCH=master
 
   test -e ${UPLOADFILE}

travisci/libcudf/build_libcudf.sh

@@ -0,0 +1,13 @@
+set -e
+
+# Patch tests for CUDA 10 skipping hash_map
+## See https://github.com/rapidsai/libgdf/pull/149
+if [ ${CUDA:0:4} == '10.0' ]; then
+  echo "CUDA 10 detected, removing hash_map tests"
+  sed -i.bak 's/ConfigureTest(HASH_MAP_TEST "${HASH_MAP_TEST_SRC}")/#ConfigureTest(HASH_MAP_TEST "${HASH_MAP_TEST_SRC}")/g' ./cpp/modules/tests/CMakeLists.txt
+fi
+
+if [ "$BUILD_LIBCUDF" == '1' -o "$BUILD_CFFI" == '1' ]; then
+  echo "Building libcudf"
+  travis_retry conda build conda/recipes/libcudf -c rapidsai -c nvidia -c numba -c conda-forge -c defaults
+fi

travisci/libcudf/build_libcudf_cffi.sh

@@ -0,0 +1,13 @@
+set -e
+
+# Patch tests for CUDA 10 skipping hash_map
+## See https://github.com/rapidsai/libgdf/pull/149
+if [ ${CUDA:0:4} == '10.0' ]; then
+  echo "CUDA 10 detected, removing hash_map tests"
+  sed -i.bak 's/ConfigureTest(HASH_MAP_TEST "${HASH_MAP_TEST_SRC}")/#ConfigureTest(HASH_MAP_TEST "${HASH_MAP_TEST_SRC}")/g' ./cpp/modules/tests/CMakeLists.txt
+fi
+
+if [ "$BUILD_CFFI" == '1' ]; then
+  echo "Building cudf_cpp_cffi"
+  travis_retry conda build conda/recipes/libcudf_cffi -c rapidsai -c nvidia -c numba -c conda-forge -c defaults --python=${PYTHON}
+fi

travisci/libgdf/install-env-libgdf.sh → travisci/libcudf/install-env-libcudf.sh

@@ -3,7 +3,7 @@
 # Copyright (c) 2018, NVIDIA CORPORATION.
 set -e
 
-if [ "$BUILD_LIBGDF" == "1" -o "$BUILD_CFFI" == "1" ]; then
+if [ "$BUILD_LIBCUDF" == "1" -o "$BUILD_CFFI" == "1" ]; then
     # install libboost
     sudo apt-get update -q
     sudo apt-get install -y libboost-all-dev
@@ -22,7 +22,7 @@ if [ "$BUILD_LIBGDF" == "1" -o "$BUILD_CFFI" == "1" ]; then
     export CC=/usr/bin/gcc-5
     export CXX=/usr/bin/g++-5
     # install cuda
-    source ./travisci/libgdf/install-cuda-trusty.sh
+    source ./travisci/libcudf/install-cuda-trusty.sh
     # check versions
     $CC --version
     $CXX --version

travisci/libcudf/upload.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Copyright (c) 2018, NVIDIA CORPORATION.
+
+set -e
+
+function upload() {
+    echo "UPLOADFILE = ${UPLOADFILE}"
+    test -e ${UPLOADFILE}
+    source ./travisci/libcudf/upload-anaconda.sh
+}
+
+if [ "$BUILD_LIBCUDF" == "1" ]; then
+    # Upload libcudf
+    export UPLOADFILE=`conda build conda/recipes/libcudf -c defaults -c conda-forge --output`
+    upload
+fi
+
+if [ "$BUILD_CFFI" == "1" ]; then
+    # Upload libcudf_cffi
+    export UPLOADFILE=`conda build conda/recipes/libcudf_cffi -c defaults -c conda-forge --python=${PYTHON} --output`
+    upload
+fi