Skip to content

Commit

Permalink
Merge branch 'branch-0.19' into fix-uploadoutput
Browse files Browse the repository at this point in the history
  • Loading branch information
dillon-cullinan authored Mar 9, 2021
2 parents 9376788 + 53929eb commit 79272ba
Show file tree
Hide file tree
Showing 119 changed files with 3,314 additions and 1,903 deletions.
8 changes: 4 additions & 4 deletions ci/benchmark/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,10 @@ conda install "rmm=$MINOR_VERSION.*" "cudatoolkit=$CUDA_REL" \
# conda install "your-pkg=1.0.0"

# Install the master version of dask, distributed, and streamz
logger "pip install git+https://github.com/dask/distributed.git@master --upgrade --no-deps"
pip install "git+https://github.com/dask/distributed.git@master" --upgrade --no-deps
logger "pip install git+https://github.com/dask/dask.git@master --upgrade --no-deps"
pip install "git+https://github.com/dask/dask.git@master" --upgrade --no-deps
logger "pip install git+https://github.com/dask/distributed.git@main --upgrade --no-deps"
pip install "git+https://github.com/dask/distributed.git@main" --upgrade --no-deps
logger "pip install git+https://github.com/dask/dask.git@main --upgrade --no-deps"
pip install "git+https://github.com/dask/dask.git@main" --upgrade --no-deps
logger "pip install git+https://github.com/python-streamz/streamz.git --upgrade --no-deps"
pip install "git+https://github.com/python-streamz/streamz.git" --upgrade --no-deps

Expand Down
22 changes: 14 additions & 8 deletions ci/cpu/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ export CUDA_REL=${CUDA_VERSION%.*}
export GPUCI_CONDA_RETRY_MAX=1
export GPUCI_CONDA_RETRY_SLEEP=30

# Use Ninja to build
# Use Ninja to build, setup Conda Build Dir
export CMAKE_GENERATOR="Ninja"
export CONDA_BLD_DIR="${WORKSPACE}/.conda-bld"

# Switch to project root; also root of repo checkout
cd $WORKSPACE
Expand Down Expand Up @@ -63,29 +64,34 @@ if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
CONDA_CHANNEL=""
else
CONDA_BUILD_ARGS="--dirty --no-remove-work-dir"
CONDA_CHANNEL="-c $WORKSPACE/ci/artifacts/cudf/cpu/conda-bld/"
CONDA_CHANNEL="-c $WORKSPACE/ci/artifacts/cudf/cpu/.conda-bld/"
fi

if [ "$BUILD_LIBCUDF" == '1' ]; then
gpuci_logger "Build conda pkg for libcudf"
gpuci_conda_retry build conda/recipes/libcudf $CONDA_BUILD_ARGS
gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcudf $CONDA_BUILD_ARGS
mkdir -p ${CONDA_BLD_DIR}/libcudf/work
cp -r ${CONDA_BLD_DIR}/work/* ${CONDA_BLD_DIR}/libcudf/work


gpuci_logger "Build conda pkg for libcudf_kafka"
gpuci_conda_retry build conda/recipes/libcudf_kafka $CONDA_BUILD_ARGS
gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcudf_kafka $CONDA_BUILD_ARGS
mkdir -p ${CONDA_BLD_DIR}/libcudf_kafka/work
cp -r ${CONDA_BLD_DIR}/work/* ${CONDA_BLD_DIR}/libcudf_kafka/work
fi

if [ "$BUILD_CUDF" == '1' ]; then
gpuci_logger "Build conda pkg for cudf"
gpuci_conda_retry build conda/recipes/cudf --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL
gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/cudf --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL

gpuci_logger "Build conda pkg for dask-cudf"
gpuci_conda_retry build conda/recipes/dask-cudf --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL
gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/dask-cudf --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL

gpuci_logger "Build conda pkg for cudf_kafka"
gpuci_conda_retry build conda/recipes/cudf_kafka --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL
gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/cudf_kafka --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL

gpuci_logger "Build conda pkg for custreamz"
gpuci_conda_retry build conda/recipes/custreamz --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL
gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/custreamz --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL
fi
################################################################################
# UPLOAD - Conda packages
Expand Down
13 changes: 7 additions & 6 deletions ci/cpu/upload.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,13 @@ fi
################################################################################

gpuci_logger "Get conda file output locations"
export LIBCUDF_FILE=`gpuci_conda_retry build conda/recipes/libcudf --output`
export LIBCUDF_KAFKA_FILE=`gpuci_conda_retry build conda/recipes/libcudf_kafka --output`
export CUDF_FILE=`gpuci_conda_retry build conda/recipes/cudf --python=$PYTHON --output`
export DASK_CUDF_FILE=`gpuci_conda_retry build conda/recipes/dask-cudf --python=$PYTHON --output`
export CUDF_KAFKA_FILE=`gpuci_conda_retry build conda/recipes/cudf_kafka --python=$PYTHON --output`
export CUSTREAMZ_FILE=`gpuci_conda_retry build conda/recipes/custreamz --python=$PYTHON --output`

export LIBCUDF_FILE=`gpuci_conda_retry build --no-build-id --croot ${WORKSPACE}/.conda-bld conda/recipes/libcudf --output`
export LIBCUDF_KAFKA_FILE=`gpuci_conda_retry build --no-build-id --croot ${WORKSPACE}/.conda-bld conda/recipes/libcudf_kafka --output`
export CUDF_FILE=`gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/cudf --python=$PYTHON --output`
export DASK_CUDF_FILE=`gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/dask-cudf --python=$PYTHON --output`
export CUDF_KAFKA_FILE=`gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/cudf_kafka --python=$PYTHON --output`
export CUSTREAMZ_FILE=`gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/custreamz --python=$PYTHON --output`

################################################################################
# UPLOAD - Conda packages
Expand Down
17 changes: 9 additions & 8 deletions ci/gpu/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ cd $WORKSPACE

# Determine CUDA release version
export CUDA_REL=${CUDA_VERSION%.*}
export CONDA_ARTIFACT_PATH=${WORKSPACE}/ci/artifacts/cudf/cpu/.conda-bld/

# Parse git describe
export GIT_DESCRIBE_TAG=`git describe --tags`
Expand Down Expand Up @@ -97,11 +98,11 @@ conda config --show-sources
conda list --show-channel-urls

function install_dask {
# Install the master version of dask, distributed, and streamz
gpuci_logger "Install the master version of dask, distributed, and streamz"
# Install the main version of dask, distributed, and streamz
gpuci_logger "Install the main version of dask, distributed, and streamz"
set -x
pip install "git+https://github.com/dask/distributed.git@master" --upgrade --no-deps
pip install "git+https://github.com/dask/dask.git@master" --upgrade --no-deps
pip install "git+https://github.com/dask/distributed.git@main" --upgrade --no-deps
pip install "git+https://github.com/dask/dask.git@main" --upgrade --no-deps
pip install "git+https://github.com/python-streamz/streamz.git" --upgrade --no-deps
set +x
}
Expand Down Expand Up @@ -151,7 +152,7 @@ else
#Project Flash
export LIB_BUILD_DIR="$WORKSPACE/ci/artifacts/cudf/cpu/libcudf_work/cpp/build"
export LD_LIBRARY_PATH="$LIB_BUILD_DIR:$CONDA_PREFIX/lib:$LD_LIBRARY_PATH"

if hasArg --skip-tests; then
gpuci_logger "Skipping Tests"
exit 0
Expand All @@ -170,15 +171,15 @@ else
${gt} --gtest_output=xml:${WORKSPACE}/test-results/
done

CUDF_CONDA_FILE=`find $WORKSPACE/ci/artifacts/cudf/cpu/conda-bld/ -name "libcudf-*.tar.bz2"`
CUDF_CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "libcudf-*.tar.bz2"`
CUDF_CONDA_FILE=`basename "$CUDF_CONDA_FILE" .tar.bz2` #get filename without extension
CUDF_CONDA_FILE=${CUDF_CONDA_FILE//-/=} #convert to conda install
KAFKA_CONDA_FILE=`find $WORKSPACE/ci/artifacts/cudf/cpu/conda-bld/ -name "libcudf_kafka-*.tar.bz2"`
KAFKA_CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "libcudf_kafka-*.tar.bz2"`
KAFKA_CONDA_FILE=`basename "$KAFKA_CONDA_FILE" .tar.bz2` #get filename without extension
KAFKA_CONDA_FILE=${KAFKA_CONDA_FILE//-/=} #convert to conda install

gpuci_logger "Installing $CUDF_CONDA_FILE & $KAFKA_CONDA_FILE"
conda install -c $WORKSPACE/ci/artifacts/cudf/cpu/conda-bld/ "$CUDF_CONDA_FILE" "$KAFKA_CONDA_FILE"
conda install -c ${CONDA_ARTIFACT_PATH} "$CUDF_CONDA_FILE" "$KAFKA_CONDA_FILE"

install_dask

Expand Down
4 changes: 2 additions & 2 deletions conda/environments/cudf_dev_cuda10.1.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ dependencies:
- nvtx>=0.2.1
- cachetools
- pip:
- git+https://github.com/dask/dask.git@master
- git+https://github.com/dask/distributed.git@master
- git+https://github.com/dask/dask.git@main
- git+https://github.com/dask/distributed.git@main
- git+https://github.com/python-streamz/streamz.git
- pyorc
4 changes: 2 additions & 2 deletions conda/environments/cudf_dev_cuda10.2.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ dependencies:
- nvtx>=0.2.1
- cachetools
- pip:
- git+https://github.com/dask/dask.git@master
- git+https://github.com/dask/distributed.git@master
- git+https://github.com/dask/dask.git@main
- git+https://github.com/dask/distributed.git@main
- git+https://github.com/python-streamz/streamz.git
- pyorc
4 changes: 2 additions & 2 deletions conda/environments/cudf_dev_cuda11.0.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ dependencies:
- nvtx>=0.2.1
- cachetools
- pip:
- git+https://github.com/dask/dask.git@master
- git+https://github.com/dask/distributed.git@master
- git+https://github.com/dask/dask.git@main
- git+https://github.com/dask/distributed.git@main
- git+https://github.com/python-streamz/streamz.git
- pyorc
8 changes: 4 additions & 4 deletions conda/recipes/dask-cudf/run_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ function logger() {
}

# Install the latest version of dask and distributed
logger "pip install git+https://github.com/dask/distributed.git@master --upgrade --no-deps"
pip install "git+https://github.com/dask/distributed.git@master" --upgrade --no-deps
logger "pip install git+https://github.com/dask/distributed.git@main --upgrade --no-deps"
pip install "git+https://github.com/dask/distributed.git@main" --upgrade --no-deps

logger "pip install git+https://github.com/dask/dask.git@master --upgrade --no-deps"
pip install "git+https://github.com/dask/dask.git@master" --upgrade --no-deps
logger "pip install git+https://github.com/dask/dask.git@main --upgrade --no-deps"
pip install "git+https://github.com/dask/dask.git@main" --upgrade --no-deps

logger "python -c 'import dask_cudf'"
python -c "import dask_cudf"
1 change: 1 addition & 0 deletions conda/recipes/libcudf/meta.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ test:
- test -f $PREFIX/include/cudf/detail/utilities/alignment.hpp
- test -f $PREFIX/include/cudf/detail/utilities/integer_utils.hpp
- test -f $PREFIX/include/cudf/detail/utilities/int_fastdiv.h
- test -f $PREFIX/include/cudf/detail/utilities/vector_factories.hpp
- test -f $PREFIX/include/cudf/dictionary/detail/concatenate.hpp
- test -f $PREFIX/include/cudf/dictionary/detail/encode.hpp
- test -f $PREFIX/include/cudf/dictionary/detail/merge.hpp
Expand Down
9 changes: 7 additions & 2 deletions cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ option(BUILD_TESTS "Configure CMake to build tests" ON)
option(BUILD_BENCHMARKS "Configure CMake to build (google) benchmarks" OFF)
option(BUILD_SHARED_LIBS "Build cuDF shared libraries" ON)
option(JITIFY_USE_CACHE "Use a file cache for JIT compiled kernels" ON)
option(ARROW_STATIC_LIB "Build and statically link Arrow libraries" OFF)
option(CUDF_USE_ARROW_STATIC "Build and statically link Arrow libraries" OFF)
option(PER_THREAD_DEFAULT_STREAM "Build with per-thread default stream" OFF)
option(DISABLE_DEPRECATION_WARNING "Disable warnings generated from deprecated declarations." OFF)
# Option to enable line info in CUDA device compilation to allow introspection when profiling / memchecking
Expand All @@ -57,7 +57,7 @@ message(VERBOSE "CUDF: Configure CMake to build tests: ${BUILD_TESTS}")
message(VERBOSE "CUDF: Configure CMake to build (google) benchmarks: ${BUILD_BENCHMARKS}")
message(VERBOSE "CUDF: Build cuDF shared libraries: ${BUILD_SHARED_LIBS}")
message(VERBOSE "CUDF: Use a file cache for JIT compiled kernels: ${JITIFY_USE_CACHE}")
message(VERBOSE "CUDF: Build and statically link Arrow libraries: ${ARROW_STATIC_LIB}")
message(VERBOSE "CUDF: Build and statically link Arrow libraries: ${CUDF_USE_ARROW_STATIC}")
message(VERBOSE "CUDF: Build with per-thread default stream: ${PER_THREAD_DEFAULT_STREAM}")
message(VERBOSE "CUDF: Disable warnings generated from deprecated declarations: ${DISABLE_DEPRECATION_WARNING}")
message(VERBOSE "CUDF: Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler: ${CUDA_ENABLE_LINEINFO}")
Expand Down Expand Up @@ -97,6 +97,11 @@ if("$ENV{CONDA_BUILD}" STREQUAL "1")
set(CONDA_INCLUDE_DIRS "$ENV{BUILD_PREFIX}/include" "$ENV{PREFIX}/include")
set(CONDA_LINK_DIRS "$ENV{BUILD_PREFIX}/lib" "$ENV{PREFIX}/lib")
message(VERBOSE "CUDF: Conda build detected, CMAKE_PREFIX_PATH set to: ${CMAKE_PREFIX_PATH}")
elseif(DEFINED ENV{CONDA_PREFIX})
set(CMAKE_PREFIX_PATH "$ENV{CONDA_PREFIX};${CMAKE_PREFIX_PATH}")
set(CONDA_INCLUDE_DIRS "$ENV{CONDA_PREFIX}/include")
set(CONDA_LINK_DIRS "$ENV{CONDA_PREFIX}/lib")
message(VERBOSE "CUDF: Conda environment detected, CMAKE_PREFIX_PATH set to: ${CMAKE_PREFIX_PATH}")
endif()

###################################################################################################
Expand Down
2 changes: 2 additions & 0 deletions cpp/benchmarks/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ ConfigureBench(REDUCTION_BENCH
reduction/anyall_benchmark.cpp
reduction/dictionary_benchmark.cpp
reduction/reduce_benchmark.cpp
reduction/scan_benchmark.cpp
reduction/minmax_benchmark.cpp)

###################################################################################################
Expand Down Expand Up @@ -183,6 +184,7 @@ ConfigureBench(STRINGS_BENCH
string/convert_durations_benchmark.cpp
string/convert_floats_benchmark.cpp
string/copy_benchmark.cpp
string/extract_benchmark.cpp
string/filter_benchmark.cpp
string/find_benchmark.cpp
string/replace_benchmark.cpp
Expand Down
7 changes: 6 additions & 1 deletion cpp/benchmarks/common/generate_benchmark_input.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@

#include <rmm/cuda_stream_view.hpp>
#include <rmm/device_buffer.hpp>
#include <rmm/device_vector.hpp>

#include <future>
#include <memory>
Expand Down Expand Up @@ -411,7 +412,11 @@ std::unique_ptr<cudf::column> create_random_column<cudf::string_view>(data_profi
row += std::max(run_len - 1, 0);
}
}
return cudf::make_strings_column(out_col.chars, out_col.offsets, out_col.null_mask);

rmm::device_vector<char> d_chars(out_col.chars);
rmm::device_vector<cudf::size_type> d_offsets(out_col.offsets);
rmm::device_vector<cudf::bitmask_type> d_null_mask(out_col.null_mask);
return cudf::make_strings_column(d_chars, d_offsets, d_null_mask);
}

template <>
Expand Down
3 changes: 1 addition & 2 deletions cpp/benchmarks/copying/shift_benchmark.cu
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2020, NVIDIA CORPORATION.
* Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -23,7 +23,6 @@

#include <benchmark/benchmark.h>

#include <thrust/device_vector.h>
#include <thrust/execution_policy.h>
#include <thrust/functional.h>
#include <thrust/sequence.h>
Expand Down
63 changes: 63 additions & 0 deletions cpp/benchmarks/reduction/scan_benchmark.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
/*
* Copyright (c) 2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include <benchmark/benchmark.h>
#include <benchmarks/common/generate_benchmark_input.hpp>
#include <benchmarks/fixture/benchmark_fixture.hpp>
#include <benchmarks/synchronization/synchronization.hpp>

#include <cudf/column/column.hpp>
#include <cudf/column/column_view.hpp>
#include <cudf/reduction.hpp>
#include <cudf/table/table.hpp>
#include <cudf/types.hpp>

class ReductionScan : public cudf::benchmark {
};

template <typename type>
static void BM_reduction_scan(benchmark::State& state, bool include_nulls)
{
cudf::size_type const n_rows{(cudf::size_type)state.range(0)};
auto const dtype = cudf::type_to_id<type>();
auto const table = create_random_table({dtype}, 1, row_count{n_rows});
if (!include_nulls) table->get_column(0).set_null_mask(rmm::device_buffer{}, 0);
cudf::column_view input(table->view().column(0));

for (auto _ : state) {
cuda_event_timer timer(state, true);
auto result = cudf::scan(input, cudf::make_min_aggregation(), cudf::scan_type::INCLUSIVE);
}
}

#define SCAN_BENCHMARK_DEFINE(name, type, nulls) \
BENCHMARK_DEFINE_F(ReductionScan, name) \
(::benchmark::State & state) { BM_reduction_scan<type>(state, nulls); } \
BENCHMARK_REGISTER_F(ReductionScan, name) \
->UseManualTime() \
->Arg(10000) /* 10k */ \
->Arg(100000) /* 100k */ \
->Arg(1000000) /* 1M */ \
->Arg(10000000) /* 10M */ \
->Arg(100000000); /* 100M */

SCAN_BENCHMARK_DEFINE(int8_no_nulls, int8_t, false);
SCAN_BENCHMARK_DEFINE(int32_no_nulls, int32_t, false);
SCAN_BENCHMARK_DEFINE(uint64_no_nulls, uint64_t, false);
SCAN_BENCHMARK_DEFINE(float_no_nulls, float, false);
SCAN_BENCHMARK_DEFINE(int16_nulls, int16_t, true);
SCAN_BENCHMARK_DEFINE(uint32_nulls, uint32_t, true);
SCAN_BENCHMARK_DEFINE(double_nulls, double, true);
Loading

0 comments on commit 79272ba

Please sign in to comment.