Merge branch 'branch-0.10' into branch-0.9

CHANGELOG.md

@@ -1,17 +1,36 @@
-# RMM 0.9.0 (Date TBD)
+# RMM 0.10.0 (Date TBD)
+
+## New Features
+
+ - PR #99 Added `device_buffer` class
+ - PR #133 Added `device_scalar` class
+
+## Improvements
+ - PR #123 Remove driver install from ci scripts
+ - PR #131 Use YYMMDD tag in nightly build
+
+## Bug Fixes
+
+ - PR #107 Fix local build generated file ownerships
+ - PR #110 Fix Skip Test Functionality
+ - PR #125 Fixed order of private variables in LogIt
+
+
+# RMM 0.9.0 (21 August 2019)
 
 ## New Features
 
  - PR #96 Added `device_memory_resource` for beginning of overhaul of RMM design
  - PR #103 Add and use unified build script
-
+ 
 ## Improvements
 
-    ...
+ - PR #111 Streamline CUDA_REL environment variable
+ - PR #113 Handle ucp.BufferRegion objects in auto_device
 
 ## Bug Fixes
 
-- PR #110 Fix Skip Test Functionality
+   ...
 
 
 # RMM 0.8.0 (27 June 2019)
@@ -22,7 +41,7 @@
 
 ## Improvements
 
-    ...
+   ...
 
 ## Bug Fixes
 
@@ -42,7 +61,7 @@
 
 - PR #76 Add cudatoolkit conda dependency
 - PR #84 Use latest release version in update-version CI script
-- PR #90 Avoid using c++14 auto return type for thrust_rmm_allocator.h 
+- PR #90 Avoid using c++14 auto return type for thrust_rmm_allocator.h
 
 ## Bug Fixes
 

CMakeLists.txt

@@ -15,7 +15,7 @@
 #=============================================================================
 cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
 
-project(RMM VERSION 0.9.0 LANGUAGES C CXX CUDA)
+project(RMM VERSION 0.10.0 LANGUAGES C CXX CUDA)
 
 ###################################################################################################
 # - build type ------------------------------------------------------------------------------------

README.md

@@ -90,6 +90,7 @@ $ make test
 $ make rmm_python_cffi                              # build CFFI bindings for librmm.so
 $ make rmm_install_python                           # build & install CFFI python bindings. Depends on cffi package from PyPi or Conda
 $ cd python && pytest -v                            # optional, run python tests on low-level python bindings
+```
 
 Done! You are ready to develop for the RMM OSS project.
 
@@ -104,12 +105,16 @@ identifier. This is necessary to enable asynchronous allocation and
 deallocation; however, the default (also known as null) stream (or `0`) can be
 used. For example:
 
+```
+
 ```
 // old
 cudaError_t result = cudaMalloc(&myvar, size_in_bytes) );
 // ...
 cudaError_t result = cudaFree(myvar) );
+```
 
+```
 // new
 rmmError_t result = RMM_ALLOC(&myvar, size_in_bytes, stream_id);
 // ...

ci/cpu/build.sh

@@ -24,6 +24,11 @@ cd $WORKSPACE
 export GIT_DESCRIBE_TAG=`git describe --abbrev=0 --tags`
 export GIT_DESCRIBE_NUMBER=`git rev-list ${GIT_DESCRIBE_TAG}..HEAD --count`
 
+# If nightly build, append current YYMMDD to version
+if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then
+  export VERSION_SUFFIX=`date +%y%m%d`
+fi
+
 ################################################################################
 # SETUP - Check environment
 ################################################################################
@@ -43,21 +48,6 @@ conda list
 # FIX Added to deal with Anancoda SSL verification issues during conda builds
 conda config --set ssl_verify False
 
-################################################################################
-# INSTALL - Install NVIDIA driver
-################################################################################
-
-logger "Install NVIDIA driver for CUDA $CUDA..."
-apt-get update -q
-DRIVER_VER="396.44-1"
-LIBCUDA_VER="396"
-if [ "$CUDA" == "10.0" ]; then
-  DRIVER_VER="410.72-1"
-  LIBCUDA_VER="410"
-fi
-DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-  cuda-drivers=${DRIVER_VER} libcuda1-${LIBCUDA_VER}
-
 ################################################################################
 # BUILD - Conda package builds (conda deps: librmm <- rmm)
 ################################################################################

ci/cpu/librmm/build_librmm.sh

@@ -2,11 +2,7 @@ set -e
 
 if [ "$BUILD_LIBRMM" == '1' ]; then
   echo "Building librmm"
-  CUDA_REL=${CUDA:0:3}
-  if [ "${CUDA:0:2}" == '10' ]; then
-    # CUDA 10 release
-    CUDA_REL=${CUDA:0:4}
-  fi
+  CUDA_REL=${CUDA_VERSION%.*}
 
   conda build conda/recipes/librmm -c nvidia/label/cuda${CUDA_REL} -c rapidsai/label/cuda${CUDA_REL} -c rapidsai-nightly/label/cuda${CUDA_REL} -c conda-forge --python=$PYTHON
 fi

ci/cpu/librmm/upload-anaconda.sh

@@ -8,11 +8,7 @@ if [ "$BUILD_LIBRMM" == '1' ]; then
   export UPLOADFILE=`conda build conda/recipes/librmm -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge --python=$PYTHON --output`
 
   SOURCE_BRANCH=master
-  CUDA_REL=${CUDA:0:3}
-  if [ "${CUDA:0:2}" == '10' ]; then
-    # CUDA 10 release
-    CUDA_REL=${CUDA:0:4}
-  fi
+  CUDA_REL=${CUDA_VERSION%.*}
 
   SOURCE_BRANCH=master
 

ci/local/build.sh

@@ -7,7 +7,7 @@ CPP_BUILD_DIR="build"
 PYTHON_BUILD_DIR="python/build"
 CONTAINER_SHELL_ONLY=0
 
-SHORTHELP="$(basename $0) [-h] [-H] [-s] [-r <repo_dir>] [-i <image_name>]"
+SHORTHELP="$(basename "$0") [-h] [-H] [-s] [-r <repo_dir>] [-i <image_name>]"
 LONGHELP="${SHORTHELP}
 Build and test your local repository using a base gpuCI Docker image
 
@@ -52,9 +52,9 @@ while getopts ":hHr:i:s" option; do
     esac
 done
 
-REPO_PATH_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename ${REPO_PATH})"
-CPP_BUILD_DIR_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename ${REPO_PATH})/${CPP_BUILD_DIR}"
-PYTHON_BUILD_DIR_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename ${REPO_PATH})/${PYTHON_BUILD_DIR}"
+REPO_PATH_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename "${REPO_PATH}")"
+CPP_BUILD_DIR_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename "${REPO_PATH}")/${CPP_BUILD_DIR}"
+PYTHON_BUILD_DIR_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename "${REPO_PATH}")/${PYTHON_BUILD_DIR}"
 
 
 # BASE_CONTAINER_BUILD_DIR is named after the image name, allowing for
@@ -63,42 +63,78 @@ PYTHON_BUILD_DIR_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename ${REPO_PATH
 # running in the container generate build artifacts just as they would
 # in a bare-metal environment, and the host filesystem is able to
 # maintain the host build in BUILD_DIR as well.
-BASE_CONTAINER_BUILD_DIR=${REPO_PATH}/build_$(echo $(basename ${DOCKER_IMAGE})|sed -e 's/:/_/g')
+
+# FIXME: Fix the shellcheck complaints
+# shellcheck disable=SC2001,SC2005
+BASE_CONTAINER_BUILD_DIR=${REPO_PATH}/build_$(echo "$(basename "${DOCKER_IMAGE}")"|sed -e 's/:/_/g')
 CPP_CONTAINER_BUILD_DIR=${BASE_CONTAINER_BUILD_DIR}/cpp
 PYTHON_CONTAINER_BUILD_DIR=${BASE_CONTAINER_BUILD_DIR}/python
 
-
+# Note the prevention of expanding variables. Preventing expansion is critical as
+# expansion in context of calling shell would lead to incorrect result as not
+# all variables are set
 BUILD_SCRIPT="#!/bin/bash
 set -e
 WORKSPACE=${REPO_PATH_IN_CONTAINER}
 PREBUILD_SCRIPT=${REPO_PATH_IN_CONTAINER}/ci/gpu/prebuild.sh
 BUILD_SCRIPT=${REPO_PATH_IN_CONTAINER}/ci/gpu/build.sh
-cd ${WORKSPACE}
+cd \${WORKSPACE}
 if [ -f \${PREBUILD_SCRIPT} ]; then
     source \${PREBUILD_SCRIPT}
 fi
 yes | source \${BUILD_SCRIPT}
 "
 
-if (( ${CONTAINER_SHELL_ONLY} == 0 )); then
+if (( CONTAINER_SHELL_ONLY == 0 )); then
     COMMAND="${CPP_BUILD_DIR_IN_CONTAINER}/build.sh || bash"
 else
     COMMAND="bash"
 fi
 
 # Create the build dir for the container to mount, generate the build script inside of it
-mkdir -p ${BASE_CONTAINER_BUILD_DIR}
-mkdir -p ${CPP_CONTAINER_BUILD_DIR}
-mkdir -p ${PYTHON_CONTAINER_BUILD_DIR}
-echo "${BUILD_SCRIPT}" > ${CPP_CONTAINER_BUILD_DIR}/build.sh
-chmod ugo+x ${CPP_CONTAINER_BUILD_DIR}/build.sh
+mkdir -p "${BASE_CONTAINER_BUILD_DIR}"
+mkdir -p "${CPP_CONTAINER_BUILD_DIR}"
+mkdir -p "${PYTHON_CONTAINER_BUILD_DIR}"
+# Create build directories. This is to ensure correct owner for directories. If
+# directories don't exist there is side effect from docker volume mounting creating build
+# directories owned by root(volume mount point(s))
+mkdir -p "${REPO_PATH}/${CPP_BUILD_DIR}"
+mkdir -p "${REPO_PATH}/${PYTHON_BUILD_DIR}"
+
+echo "${BUILD_SCRIPT}" > "${CPP_CONTAINER_BUILD_DIR}/build.sh"
+chmod ugo+x "${CPP_CONTAINER_BUILD_DIR}/build.sh"
+
+# Mount passwd and group files to docker. This allows docker to resolve username and group
+# avoiding these nags:
+#   * groups: cannot find name for group ID ID
+#   * I have no name!@id:/$
+# For ldap user user information is not present in system /etc/passwd and /etc/group files.
+# Hence we generate dummy files for ldap users which docker uses to resolve username and group
+
+PASSWD_FILE="/etc/passwd"
+GROUP_FILE="/etc/group"
+
+USER_FOUND=$(grep -wc "$(whoami)" < "$PASSWD_FILE")
+if [ "$USER_FOUND" == 0 ]; then
+  echo "Local User not found, LDAP WAR for docker mounts activated. Creating dummy passwd and group"
+  echo "files to allow docker resolve username and group"
+  cp "$PASSWD_FILE" /tmp/passwd
+  PASSWD_FILE="/tmp/passwd"
+  cp "$GROUP_FILE" /tmp/group
+  GROUP_FILE="/tmp/group"
+  echo "$(whoami):x:$(id -u):$(id -g):$(whoami),,,:$HOME:$SHELL" >> "$PASSWD_FILE"
+  echo "$(whoami):x:$(id -g):" >> "$GROUP_FILE"
+fi
+
 
 # Run the generated build script in a container
-docker pull ${DOCKER_IMAGE}
-docker run --runtime=nvidia --rm -it -e NVIDIA_VISIBLE_DEVICES=${NVIDIA_VISIBLE_DEVICES} \
-       --user $(id -u):$(id -g) \
-       -v ${REPO_PATH}:${REPO_PATH_IN_CONTAINER} \
-       -v ${CPP_CONTAINER_BUILD_DIR}:${CPP_BUILD_DIR_IN_CONTAINER} \
-       -v ${PYTHON_CONTAINER_BUILD_DIR}:${PYTHON_BUILD_DIR_IN_CONTAINER} \
+docker pull "${DOCKER_IMAGE}"
+docker run --runtime=nvidia --rm -it -e NVIDIA_VISIBLE_DEVICES="${NVIDIA_VISIBLE_DEVICES}" \
+       --user "$(id -u)":"$(id -g)" \
+       -v "${REPO_PATH}":"${REPO_PATH_IN_CONTAINER}" \
+       -v "${CPP_CONTAINER_BUILD_DIR}":"${CPP_BUILD_DIR_IN_CONTAINER}" \
+       -v "${PYTHON_CONTAINER_BUILD_DIR}":"${PYTHON_BUILD_DIR_IN_CONTAINER}" \
+       -v "$PASSWD_FILE":/etc/passwd:ro \
+       -v "$GROUP_FILE":/etc/group:ro \
        --cap-add=SYS_PTRACE \
-       ${DOCKER_IMAGE} bash -c "${COMMAND}"
+       "${DOCKER_IMAGE}" bash -c "${COMMAND}"

conda/recipes/librmm/meta.yaml

@@ -1,6 +1,6 @@
 # Copyright (c) 2018, NVIDIA CORPORATION.
 
-{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %}
+{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
 {% set git_revision_count=environ.get('GIT_DESCRIBE_NUMBER', 0) %}
 {% set cuda_version='.'.join(environ.get('CUDA_VERSION', '9.2').split('.')[:2]) %}
 package:
@@ -18,6 +18,7 @@ build:
     - CXX
     - CUDAHOSTCXX
     - PARALLEL_LEVEL
+    - VERSION_SUFFIX
   run_exports:
     - {{ pin_subpackage("librmm", max_pin="x.x") }}
 

conda/recipes/rmm/meta.yaml

@@ -1,6 +1,6 @@
 # Copyright (c) 2019, NVIDIA CORPORATION.
 
-{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %}
+{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
 {% set git_revision_count=environ.get('GIT_DESCRIBE_NUMBER', 0) %}
 {% set py_version=environ.get('CONDA_PY', 35) %}
 package:
@@ -15,6 +15,7 @@ build:
   string: py{{ py_version }}_{{ git_revision_count }}
   script_env:
     - RMM_BUILD_NO_GPU_TEST
+    - VERSION_SUFFIX
 
 requirements:
   build:

doxygen/Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = "RMM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 0.9
+PROJECT_NUMBER         = 0.10
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a