| PosExplode |
`posexplode_outer`, `posexplode` |
-Given an input array produces a sequence of rows for each value in the array. PosExplode with outer Generate is not supported under GPU runtime. |
+Given an input array produces a sequence of rows for each value in the array. |
None |
project |
input |
diff --git a/integration_tests/README.md b/integration_tests/README.md
index 2e05d2823e2..0934f33f252 100644
--- a/integration_tests/README.md
+++ b/integration_tests/README.md
@@ -107,7 +107,7 @@ individually, so you don't risk running unit tests along with the integration te
http://www.scalatest.org/user_guide/using_the_scalatest_shell
```shell
-spark-shell --jars rapids-4-spark-tests_2.12-0.5.0-tests.jar,rapids-4-spark-udf-examples_2.12-0.5.0,rapids-4-spark-integration-tests_2.12-0.5.0-tests.jar,scalatest_2.12-3.0.5.jar,scalactic_2.12-3.0.5.jar
+spark-shell --jars rapids-4-spark-tests_2.12-0.6.0-SNAPSHOT-tests.jar,rapids-4-spark-udf-examples_2.12-0.6.0-SNAPSHOT,rapids-4-spark-integration-tests_2.12-0.6.0-SNAPSHOT-tests.jar,scalatest_2.12-3.0.5.jar,scalactic_2.12-3.0.5.jar
```
First you import the `scalatest_shell` and tell the tests where they can find the test files you
@@ -128,10 +128,10 @@ durations.run(new com.nvidia.spark.rapids.JoinsSuite)
Most clusters probably will not have the RAPIDS plugin installed in the cluster yet.
If you just want to verify the SQL replacement is working you will need to add the
`rapids-4-spark` and `cudf` jars to your `spark-submit` command. Note the following
-example assumes CUDA 10.1 is being used.
+example assumes CUDA 11.0 is being used.
```
-$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-0.5.0.jar,rapids-4-spark-udf-examples_2.12-0.5.0.jar,cudf-0.19-cuda10-1.jar" ./runtests.py
+$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-0.6.0-SNAPSHOT.jar,rapids-4-spark-udf-examples_2.12-0.6.0-SNAPSHOT.jar,cudf-0.20-SNAPSHOT-cuda11.jar" ./runtests.py
```
You don't have to enable the plugin for this to work, the test framework will do that for you.
@@ -202,10 +202,10 @@ To run cudf_udf tests, need following configuration changes:
* Decrease `spark.rapids.memory.gpu.allocFraction` to reserve enough GPU memory for Python processes in case of out-of-memory.
* Add `spark.rapids.python.concurrentPythonWorkers` and `spark.rapids.python.memory.gpu.allocFraction` to reserve enough GPU memory for Python processes in case of out-of-memory.
-As an example, here is the `spark-submit` command with the cudf_udf parameter on CUDA 10.1:
+As an example, here is the `spark-submit` command with the cudf_udf parameter on CUDA 11.0:
```
-$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-0.5.0.jar,rapids-4-spark-udf-examples_2.12-0.5.0.jar,cudf-0.19-cuda10-1.jar,rapids-4-spark-tests_2.12-0.5.0.jar" --conf spark.rapids.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.concurrentPythonWorkers=2 --py-files "rapids-4-spark_2.12-0.5.0.jar" --conf spark.executorEnv.PYTHONPATH="rapids-4-spark_2.12-0.5.0.jar" ./runtests.py --cudf_udf
+$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-0.6.0-SNAPSHOT.jar,rapids-4-spark-udf-examples_2.12-0.6.0-SNAPSHOT.jar,cudf-0.20-SNAPSHOT-cuda11.jar,rapids-4-spark-tests_2.12-0.6.0-SNAPSHOT.jar" --conf spark.rapids.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.concurrentPythonWorkers=2 --py-files "rapids-4-spark_2.12-0.6.0-SNAPSHOT.jar" --conf spark.executorEnv.PYTHONPATH="rapids-4-spark_2.12-0.6.0-SNAPSHOT.jar" ./runtests.py --cudf_udf
```
## Writing tests
diff --git a/integration_tests/pom.xml b/integration_tests/pom.xml
index cf95ae0b56d..cb68d6de5c2 100644
--- a/integration_tests/pom.xml
+++ b/integration_tests/pom.xml
@@ -22,11 +22,11 @@
com.nvidia
rapids-4-spark-parent
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
com.nvidia
rapids-4-spark-integration-tests_2.12
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/integration_tests/src/main/python/generate_expr_test.py b/integration_tests/src/main/python/generate_expr_test.py
index d3e283b8ae6..e41d49b35f4 100644
--- a/integration_tests/src/main/python/generate_expr_test.py
+++ b/integration_tests/src/main/python/generate_expr_test.py
@@ -69,6 +69,26 @@ def test_explode_nested_array_data(spark_tmp_path, data_gen):
'a', 'explode(b) as c').selectExpr('a', 'explode(c)'),
conf=conf_to_enforce_split_input)
+#sort locally because of https://github.com/NVIDIA/spark-rapids/issues/84
+# After 3.1.0 is the min spark version we can drop this
+@ignore_order(local=True)
+@pytest.mark.parametrize('data_gen', all_gen, ids=idfn)
+def test_explode_outer_array_data(spark_tmp_path, data_gen):
+ data_gen = [int_gen, ArrayGen(data_gen)]
+ assert_gpu_and_cpu_are_equal_collect(
+ lambda spark: two_col_df(spark, *data_gen).selectExpr('a', 'explode_outer(b)'),
+ conf=conf_to_enforce_split_input)
+
+#sort locally because of https://github.com/NVIDIA/spark-rapids/issues/84
+# After 3.1.0 is the min spark version we can drop this
+@ignore_order(local=True)
+@pytest.mark.parametrize('data_gen', all_gen, ids=idfn)
+def test_explode_outer_nested_array_data(spark_tmp_path, data_gen):
+ data_gen = [int_gen, ArrayGen(ArrayGen(data_gen))]
+ assert_gpu_and_cpu_are_equal_collect(
+ lambda spark: two_col_df(spark, *data_gen).selectExpr(
+ 'a', 'explode_outer(b) as c').selectExpr('a', 'explode_outer(c)'),
+ conf=conf_to_enforce_split_input)
#sort locally because of https://github.com/NVIDIA/spark-rapids/issues/84
# After 3.1.0 is the min spark version we can drop this
@@ -108,3 +128,25 @@ def test_posexplode_nested_array_data(spark_tmp_path, data_gen):
lambda spark: two_col_df(spark, *data_gen).selectExpr(
'a', 'posexplode(b) as (pos, c)').selectExpr('a', 'pos', 'posexplode(c)'),
conf=conf_to_enforce_split_input)
+
+#sort locally because of https://github.com/NVIDIA/spark-rapids/issues/84
+# After 3.1.0 is the min spark version we can drop this
+@ignore_order(local=True)
+@pytest.mark.parametrize('data_gen', all_gen, ids=idfn)
+def test_posexplode_outer_array_data(spark_tmp_path, data_gen):
+ data_gen = [int_gen, ArrayGen(data_gen)]
+ assert_gpu_and_cpu_are_equal_collect(
+ lambda spark: two_col_df(spark, *data_gen).selectExpr('a', 'posexplode_outer(b)'),
+ conf=conf_to_enforce_split_input)
+
+#sort locally because of https://github.com/NVIDIA/spark-rapids/issues/84
+# After 3.1.0 is the min spark version we can drop this
+@ignore_order(local=True)
+@pytest.mark.parametrize('data_gen', all_gen, ids=idfn)
+def test_posexplode_nested_outer_array_data(spark_tmp_path, data_gen):
+ data_gen = [int_gen, ArrayGen(ArrayGen(data_gen))]
+ assert_gpu_and_cpu_are_equal_collect(
+ lambda spark: two_col_df(spark, *data_gen).selectExpr(
+ 'a', 'posexplode_outer(b) as (pos, c)').selectExpr(
+ 'a', 'pos', 'posexplode_outer(c)'),
+ conf=conf_to_enforce_split_input)
diff --git a/jenkins/Dockerfile-blossom.integration.centos b/jenkins/Dockerfile-blossom.integration.centos
index 66c00f398d8..6b047b5d84b 100644
--- a/jenkins/Dockerfile-blossom.integration.centos
+++ b/jenkins/Dockerfile-blossom.integration.centos
@@ -17,13 +17,13 @@
###
#
# Arguments:
-# CUDA_VER=10.1, 10.2 or 11.0
+# CUDA_VER=11.0, 11.1 or 11.2.x
# CENTOS_VER=7 or 8
-# CUDF_VER=0.18 or 0.19
+# CUDF_VER=0.20
# URM_URL=
###
-ARG CUDA_VER=10.1
+ARG CUDA_VER=11.0
ARG CENTOS_VER=7
FROM nvidia/cuda:${CUDA_VER}-runtime-centos${CENTOS_VER}
ARG CUDA_VER
@@ -46,7 +46,8 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86
rm -f ~/miniconda.sh
ENV PATH="/opt/conda/bin:$MAVEN_HOME/bin:${PATH}"
# 'pyarrow' and 'pandas' will be installed as the dependencies of cudf below
-RUN conda install -y -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge -c defaults cudf=${CUDF_VER} python=3.8 cudatoolkit=${CUDA_VER} && \
+RUN export CUDA_VER=`echo ${CUDA_VER} | cut -d '.' -f 1,2` && \
+ conda install -y -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge -c defaults cudf=${CUDF_VER} python=3.8 cudatoolkit=${CUDA_VER} && \
conda install -y spacy && python -m spacy download en_core_web_sm && \
conda install -y -c anaconda pytest requests && \
conda install -y -c conda-forge sre_yield && \
diff --git a/jenkins/Dockerfile-blossom.integration.ubuntu b/jenkins/Dockerfile-blossom.integration.ubuntu
index 44f0a111483..f3e986dccb8 100644
--- a/jenkins/Dockerfile-blossom.integration.ubuntu
+++ b/jenkins/Dockerfile-blossom.integration.ubuntu
@@ -19,13 +19,16 @@
# Build the image for rapids-plugin development environment
#
# Arguments:
-# CUDA_VER=10.1, 10.2 or 11.0
-# UBUNTU_VER=18.04 or 20.04
+# CUDA_VER=11.0, 11.1 or 11.2.x
+# UBUNTU_VER=18.04 or 20.04
+# CUDF_VER=0.20
###
ARG CUDA_VER=11.0
ARG UBUNTU_VER=18.04
FROM nvidia/cuda:${CUDA_VER}-runtime-ubuntu${UBUNTU_VER}
+ARG CUDA_VER
+ARG CUDF_VER
# Install jdk-8, jdk-11, maven, docker image
RUN apt-get update -y && \
@@ -44,7 +47,8 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86
rm -f ~/miniconda.sh
ENV PATH="/opt/conda/bin:$MAVEN_HOME/bin:${PATH}"
# 'pyarrow' and 'pandas' will be installed as the dependencies of cudf below
-RUN conda install -y -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge -c defaults cudf=${CUDF_VER} python=3.8 cudatoolkit=${CUDA_VER} && \
+RUN export CUDA_VER=`echo ${CUDA_VER} | cut -d '.' -f 1,2` && \
+ conda install -y -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge -c defaults cudf=${CUDF_VER} python=3.8 cudatoolkit=${CUDA_VER} && \
conda install -y spacy && python -m spacy download en_core_web_sm && \
conda install -y -c anaconda pytest requests && \
conda install -y -c conda-forge sre_yield && \
diff --git a/jenkins/Dockerfile-blossom.ubuntu b/jenkins/Dockerfile-blossom.ubuntu
index ac902997381..2906f8d3b59 100644
--- a/jenkins/Dockerfile-blossom.ubuntu
+++ b/jenkins/Dockerfile-blossom.ubuntu
@@ -19,7 +19,7 @@
# Build the image for rapids-plugin development environment
#
# Arguments:
-# CUDA_VER=10.1, 10.2 or 11.0
+# CUDA_VER=11.0, 11.1 or 11.2.x
# UBUNTU_VER=18.04 or 20.04
###
diff --git a/jenkins/databricks/clusterutils.py b/jenkins/databricks/clusterutils.py
index a48e7cc3f23..dc4407c2b20 100644
--- a/jenkins/databricks/clusterutils.py
+++ b/jenkins/databricks/clusterutils.py
@@ -23,7 +23,7 @@ class ClusterUtils(object):
@staticmethod
def generate_create_templ(sshKey, cluster_name, runtime, idle_timeout,
- num_workers, driver_node_type, worker_node_type, cloud_provider,
+ num_workers, driver_node_type, worker_node_type, cloud_provider, init_scripts,
printLoc=sys.stdout):
timeStr = str(int(time.time()))
uniq_name = cluster_name + "-" + timeStr
@@ -46,13 +46,18 @@ def generate_create_templ(sshKey, cluster_name, runtime, idle_timeout,
templ['driver_node_type_id'] = driver_node_type
templ['ssh_public_keys'] = [ sshKey ]
templ['num_workers'] = num_workers
- templ['init_scripts'] = [
- {
- "dbfs": {
- "destination": "dbfs:/databricks/init_scripts/init_cudf_udf.sh"
- }
- }
- ]
+ if (init_scripts != ''):
+ templ['init_scripts']=[]
+ path_list = init_scripts.split(',')
+ for path in path_list:
+ templ['init_scripts'].append(
+ {
+ 'dbfs' : {
+ 'destination' : path
+ }
+ }
+ )
+
return templ
diff --git a/jenkins/databricks/create.py b/jenkins/databricks/create.py
index a6e14f3e157..f574d2ef2aa 100644
--- a/jenkins/databricks/create.py
+++ b/jenkins/databricks/create.py
@@ -27,21 +27,24 @@ def main():
workspace = 'https://dbc-9ff9942e-a9c4.cloud.databricks.com'
token = ''
sshkey = ''
- cluster_name = 'CI-GPU-databricks-0.5.0-SNAPSHOT'
+ cluster_name = 'CI-GPU-databricks-0.6.0-SNAPSHOT'
idletime = 240
runtime = '7.0.x-gpu-ml-scala2.12'
num_workers = 1
worker_type = 'g4dn.xlarge'
driver_type = 'g4dn.xlarge'
cloud_provider = 'aws'
+ # comma separated init scripts, e.g. dbfs:/foo,dbfs:/bar,...
+ init_scripts = ''
+
try:
- opts, args = getopt.getopt(sys.argv[1:], 'hw:t:k:n:i:r:o:d:e:s:',
+ opts, args = getopt.getopt(sys.argv[1:], 'hw:t:k:n:i:r:o:d:e:s:f:',
['workspace=', 'token=', 'sshkey=', 'clustername=', 'idletime=',
- 'runtime=', 'workertype=', 'drivertype=', 'numworkers=', 'cloudprovider='])
+ 'runtime=', 'workertype=', 'drivertype=', 'numworkers=', 'cloudprovider=', 'initscripts='])
except getopt.GetoptError:
print(
- 'create.py -w -t -k -n -i -r -o -d -e -s ')
+ 'create.py -w -t -k -n -i -r -o -d -e -s -f ')
sys.exit(2)
for opt, arg in opts:
@@ -69,6 +72,8 @@ def main():
num_workers = arg
elif opt in ('-s', '--cloudprovider'):
cloud_provider = arg
+ elif opt in ('-f', '--initscripts'):
+ init_scripts = arg
print('-w is ' + workspace, file=sys.stderr)
print('-k is ' + sshkey, file=sys.stderr)
@@ -79,6 +84,7 @@ def main():
print('-d is ' + driver_type, file=sys.stderr)
print('-e is ' + str(num_workers), file=sys.stderr)
print('-s is ' + cloud_provider, file=sys.stderr)
+ print('-f is ' + init_scripts, file=sys.stderr)
if not sshkey:
print("You must specify an sshkey!", file=sys.stderr)
@@ -89,7 +95,7 @@ def main():
sys.exit(2)
templ = ClusterUtils.generate_create_templ(sshkey, cluster_name, runtime, idletime,
- num_workers, driver_type, worker_type, cloud_provider, printLoc=sys.stderr)
+ num_workers, driver_type, worker_type, cloud_provider, init_scripts, printLoc=sys.stderr)
clusterid = ClusterUtils.create_cluster(workspace, templ, token, printLoc=sys.stderr)
ClusterUtils.wait_for_cluster_start(workspace, clusterid, token, printLoc=sys.stderr)
diff --git a/jenkins/databricks/init_cuda11_runtime.sh b/jenkins/databricks/init_cuda11_runtime.sh
new file mode 100644
index 00000000000..07ae40181d9
--- /dev/null
+++ b/jenkins/databricks/init_cuda11_runtime.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# The init script to install cuda11.0 toolkit
+# Will be automatically pushed into the dbfs:/databricks/init_scripts once it is updated.
+
+wget http://developer.download.nvidia.com/compute/cuda/11.0.2/local_installers/cuda_11.0.2_450.51.05_linux.run
+
+sh cuda_11.0.2_450.51.05_linux.run --toolkit --silent
+
+rm cuda_11.0.2_450.51.05_linux.run
diff --git a/jenkins/databricks/init_cudf_udf.sh b/jenkins/databricks/init_cudf_udf.sh
index 70758a7de91..7c9a8b076c7 100644
--- a/jenkins/databricks/init_cudf_udf.sh
+++ b/jenkins/databricks/init_cudf_udf.sh
@@ -18,7 +18,7 @@
# The initscript to set up environment for the cudf_udf tests on Databrcks
# Will be automatically pushed into the dbfs:/databricks/init_scripts once it is updated.
-CUDF_VER=${CUDF_VER:-0.19}
+CUDF_VER=${CUDF_VER:-0.20}
# Use mamba to install cudf-udf packages to speed up conda resolve time
base=$(conda info --base)
@@ -26,5 +26,5 @@ conda create -y -n mamba -c conda-forge mamba
pip uninstall -y pyarrow
${base}/envs/mamba/bin/mamba remove -y c-ares zstd libprotobuf pandas
${base}/envs/mamba/bin/mamba install -y pyarrow=1.0.1 -c conda-forge
-${base}/envs/mamba/bin/mamba install -y -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge -c defaults cudf=$CUDF_VER cudatoolkit=10.1
+${base}/envs/mamba/bin/mamba install -y -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge -c defaults cudf=$CUDF_VER cudatoolkit=11.0
conda env remove -n mamba
diff --git a/jenkins/printJarVersion.sh b/jenkins/printJarVersion.sh
index f32b40ebf4a..fad4c006b2d 100755
--- a/jenkins/printJarVersion.sh
+++ b/jenkins/printJarVersion.sh
@@ -24,7 +24,7 @@ function print_ver(){
SERVER_ID=$5
# Collect snapshot dependency info only in Jenkins build
- # In dev build, print 'SNAPSHOT' tag without time stamp, e.g.: cudf-0.19-SNAPSHOT.jar
+ # In dev build, print 'SNAPSHOT' tag without time stamp, e.g.: cudf-0.20-SNAPSHOT.jar
if [[ "$VERSION" == *"-SNAPSHOT" && -n "$JENKINS_URL" ]]; then
PREFIX=${VERSION%-SNAPSHOT}
# List the latest SNAPSHOT jar file in the maven repo
diff --git a/jenkins/version-def.sh b/jenkins/version-def.sh
index 738ef022a70..6f11f24fe29 100755
--- a/jenkins/version-def.sh
+++ b/jenkins/version-def.sh
@@ -26,10 +26,10 @@ for VAR in $OVERWRITE_PARAMS;do
done
IFS=$PRE_IFS
-CUDF_VER=${CUDF_VER:-"0.19-SNAPSHOT"}
-CUDA_CLASSIFIER=${CUDA_CLASSIFIER:-"cuda10-1"}
-PROJECT_VER=${PROJECT_VER:-"0.5.0-SNAPSHOT"}
-PROJECT_TEST_VER=${PROJECT_TEST_VER:-"0.5.0-SNAPSHOT"}
+CUDF_VER=${CUDF_VER:-"0.20-SNAPSHOT"}
+CUDA_CLASSIFIER=${CUDA_CLASSIFIER:-"cuda11"}
+PROJECT_VER=${PROJECT_VER:-"0.6.0-SNAPSHOT"}
+PROJECT_TEST_VER=${PROJECT_TEST_VER:-"0.6.0-SNAPSHOT"}
SPARK_VER=${SPARK_VER:-"3.0.0"}
SCALA_BINARY_VER=${SCALA_BINARY_VER:-"2.12"}
SERVER_ID=${SERVER_ID:-"snapshots"}
diff --git a/pom.xml b/pom.xml
index 506e982f14e..cc0683bed1a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -23,7 +23,7 @@
rapids-4-spark-parent
RAPIDS Accelerator for Apache Spark Root Project
The root project of the RAPIDS Accelerator for Apache Spark
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
pom
https://github.com/NVIDIA
@@ -184,18 +184,6 @@
-
- cuda10-1
-
- cuda10-1
-
-
-
- cuda10-2
-
- cuda10-2
-
-
cuda11
@@ -209,8 +197,8 @@
1.8
${spark300.version}
${spark300.version}
- cuda10-1
- 0.19-SNAPSHOT
+ cuda11
+ 0.20-SNAPSHOT
2.12
2.12.8
1.5.8
diff --git a/shims/aggregator/pom.xml b/shims/aggregator/pom.xml
index 98b0453185b..adac79181bf 100644
--- a/shims/aggregator/pom.xml
+++ b/shims/aggregator/pom.xml
@@ -22,7 +22,7 @@
com.nvidia
rapids-4-spark-shims_2.12
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
../pom.xml
com.nvidia
@@ -30,7 +30,7 @@
jar
RAPIDS Accelerator for Apache Spark SQL Plugin Shim Aggregator
The RAPIDS SQL plugin for Apache Spark Shim Aggregator
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/shims/pom.xml b/shims/pom.xml
index bfbf1a1d52b..4f2291c16a0 100644
--- a/shims/pom.xml
+++ b/shims/pom.xml
@@ -22,7 +22,7 @@
com.nvidia
rapids-4-spark-parent
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
../pom.xml
com.nvidia
@@ -30,7 +30,7 @@
pom
RAPIDS Accelerator for Apache Spark SQL Plugin Shims
The RAPIDS SQL plugin for Apache Spark Shims
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/shims/spark300/pom.xml b/shims/spark300/pom.xml
index 0de5b1ddc30..eb412263449 100644
--- a/shims/spark300/pom.xml
+++ b/shims/spark300/pom.xml
@@ -22,14 +22,14 @@
com.nvidia
rapids-4-spark-shims_2.12
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
../pom.xml
com.nvidia
rapids-4-spark-shims-spark300_2.12
RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.0.0 Shim
The RAPIDS SQL plugin for Apache Spark 3.0.0 Shim
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/shims/spark300emr/pom.xml b/shims/spark300emr/pom.xml
index 0e8c1bb0496..3ad48af7f7e 100644
--- a/shims/spark300emr/pom.xml
+++ b/shims/spark300emr/pom.xml
@@ -22,14 +22,14 @@
com.nvidia
rapids-4-spark-shims_2.12
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
../pom.xml
com.nvidia
rapids-4-spark-shims-spark300emr_2.12
RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.0.0 EMR Shim
The RAPIDS SQL plugin for Apache Spark 3.0.0 EMR Shim
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/shims/spark301/pom.xml b/shims/spark301/pom.xml
index 73efc3c8179..98995b8d3d6 100644
--- a/shims/spark301/pom.xml
+++ b/shims/spark301/pom.xml
@@ -22,14 +22,14 @@
com.nvidia
rapids-4-spark-shims_2.12
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
../pom.xml
com.nvidia
rapids-4-spark-shims-spark301_2.12
RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.0.1 Shim
The RAPIDS SQL plugin for Apache Spark 3.0.1 Shim
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/shims/spark301db/pom.xml b/shims/spark301db/pom.xml
index 2364123e5b7..7f5eabda2fc 100644
--- a/shims/spark301db/pom.xml
+++ b/shims/spark301db/pom.xml
@@ -22,14 +22,14 @@
com.nvidia
rapids-4-spark-shims_2.12
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
../pom.xml
com.nvidia
rapids-4-spark-shims-spark301-databricks_2.12
RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.0.1 Databricks Shim
The RAPIDS SQL plugin for Apache Spark 3.0.1 Databricks Shim
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/shims/spark301emr/pom.xml b/shims/spark301emr/pom.xml
index 9df7046e2d5..f13b4c593da 100644
--- a/shims/spark301emr/pom.xml
+++ b/shims/spark301emr/pom.xml
@@ -22,14 +22,14 @@
com.nvidia
rapids-4-spark-shims_2.12
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
../pom.xml
com.nvidia
rapids-4-spark-shims-spark301emr_2.12
RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.0.1 EMR Shim
The RAPIDS SQL plugin for Apache Spark 3.0.1 EMR Shim
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/shims/spark302/pom.xml b/shims/spark302/pom.xml
index 75d2356df62..b58bb3614b0 100644
--- a/shims/spark302/pom.xml
+++ b/shims/spark302/pom.xml
@@ -22,14 +22,14 @@
com.nvidia
rapids-4-spark-shims_2.12
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
../pom.xml
com.nvidia
rapids-4-spark-shims-spark302_2.12
RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.0.2 Shim
The RAPIDS SQL plugin for Apache Spark 3.0.2 Shim
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/shims/spark303/pom.xml b/shims/spark303/pom.xml
index e264e0cc953..08c355c5c5e 100644
--- a/shims/spark303/pom.xml
+++ b/shims/spark303/pom.xml
@@ -22,14 +22,14 @@
com.nvidia
rapids-4-spark-shims_2.12
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
../pom.xml
com.nvidia
rapids-4-spark-shims-spark303_2.12
RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.0.3 Shim
The RAPIDS SQL plugin for Apache Spark 3.0.3 Shim
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/shims/spark311/pom.xml b/shims/spark311/pom.xml
index 4b4358612e3..5c2ba6d3bc5 100644
--- a/shims/spark311/pom.xml
+++ b/shims/spark311/pom.xml
@@ -22,14 +22,14 @@
com.nvidia
rapids-4-spark-shims_2.12
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
../pom.xml
com.nvidia
rapids-4-spark-shims-spark311_2.12
RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.1.1 Shim
The RAPIDS SQL plugin for Apache Spark 3.1.1 Shim
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/shims/spark312/pom.xml b/shims/spark312/pom.xml
index f61ba9b4618..2c6ae6768c4 100644
--- a/shims/spark312/pom.xml
+++ b/shims/spark312/pom.xml
@@ -22,14 +22,14 @@
com.nvidia
rapids-4-spark-shims_2.12
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
../pom.xml
com.nvidia
rapids-4-spark-shims-spark312_2.12
RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.1.2 Shim
The RAPIDS SQL plugin for Apache Spark 3.1.2 Shim
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/shims/spark320/pom.xml b/shims/spark320/pom.xml
index fd893e82315..ab25d316f83 100644
--- a/shims/spark320/pom.xml
+++ b/shims/spark320/pom.xml
@@ -22,14 +22,14 @@
com.nvidia
rapids-4-spark-shims_2.12
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
../pom.xml
com.nvidia
rapids-4-spark-shims-spark320_2.12
RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.2.0 Shim
The RAPIDS SQL plugin for Apache Spark 3.2.0 Shim
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/shuffle-plugin/pom.xml b/shuffle-plugin/pom.xml
index 144786d0ff9..26059242349 100644
--- a/shuffle-plugin/pom.xml
+++ b/shuffle-plugin/pom.xml
@@ -22,14 +22,14 @@
com.nvidia
rapids-4-spark-parent
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
com.nvidia
rapids-4-spark-shuffle_2.12
RAPIDS Accelerator for Apache Spark Shuffle Plugin
Accelerated shuffle plugin for the RAPIDS plugin for Apache Spark
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/sql-plugin/pom.xml b/sql-plugin/pom.xml
index 61496834992..669d20f8fa5 100644
--- a/sql-plugin/pom.xml
+++ b/sql-plugin/pom.xml
@@ -22,13 +22,13 @@
com.nvidia
rapids-4-spark-parent
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
com.nvidia
rapids-4-spark-sql_2.12
RAPIDS Accelerator for Apache Spark SQL Plugin
The RAPIDS SQL plugin for Apache Spark
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuGenerateExec.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuGenerateExec.scala
index 9fdb3b90bd4..13f55290033 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuGenerateExec.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuGenerateExec.scala
@@ -342,8 +342,10 @@ case class GpuExplode(child: Expression) extends GpuExplodeBase {
require(inputBatch.numCols() - 1 == generatorOffset,
"Internal Error GpuExplode supports one and only one input attribute.")
val schema = resultSchema(GpuColumnVector.extractTypes(inputBatch), generatorOffset)
+ val explodeFun = (t: Table) =>
+ if (outer) t.explodeOuter(generatorOffset) else t.explode(generatorOffset)
withResource(GpuColumnVector.from(inputBatch)) { table =>
- withResource(table.explode(generatorOffset)) { exploded =>
+ withResource(explodeFun(table)) { exploded =>
GpuColumnVector.from(exploded, schema)
}
}
@@ -362,8 +364,10 @@ case class GpuPosExplode(child: Expression) extends GpuExplodeBase {
"Internal Error GpuPosExplode supports one and only one input attribute.")
val schema = resultSchema(
GpuColumnVector.extractTypes(inputBatch), generatorOffset, includePos = true)
+ val explodePosFun = (t: Table) =>
+ if (outer) t.explodeOuterPosition(generatorOffset) else t.explodePosition(generatorOffset)
withResource(GpuColumnVector.from(inputBatch)) { table =>
- withResource(table.explodePosition(generatorOffset)) { exploded =>
+ withResource(explodePosFun(table)) { exploded =>
GpuColumnVector.from(exploded, schema)
}
}
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala
index cf7591fa1cc..76d99d1a5ac 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala
@@ -2471,8 +2471,7 @@ object GpuOverrides {
GpuMakeDecimal(child, a.precision, a.scale, a.nullOnOverflow)
}),
expr[Explode](
- "Given an input array produces a sequence of rows for each value in the array. "
- + "Explode with outer Generate is not supported under GPU runtime." ,
+ "Given an input array produces a sequence of rows for each value in the array.",
ExprChecks.unaryProject(
// Here is a walk-around representation, since multi-level nested type is not supported yet.
// related issue: https://github.com/NVIDIA/spark-rapids/issues/1901
@@ -2483,11 +2482,11 @@ object GpuOverrides {
TypeSig.commonCudfTypes + TypeSig.DECIMAL + TypeSig.NULL + TypeSig.ARRAY),
(TypeSig.ARRAY + TypeSig.MAP).nested(TypeSig.all)),
(a, conf, p, r) => new GeneratorExprMeta[Explode](a, conf, p, r) {
+ override val supportOuter: Boolean = true
override def convertToGpu(): GpuExpression = GpuExplode(childExprs.head.convertToGpu())
}),
expr[PosExplode](
- "Given an input array produces a sequence of rows for each value in the array. "
- + "PosExplode with outer Generate is not supported under GPU runtime." ,
+ "Given an input array produces a sequence of rows for each value in the array.",
ExprChecks.unaryProject(
// Here is a walk-around representation, since multi-level nested type is not supported yet.
// related issue: https://github.com/NVIDIA/spark-rapids/issues/1901
@@ -2499,6 +2498,7 @@ object GpuOverrides {
TypeSig.ARRAY.nested(
TypeSig.commonCudfTypes + TypeSig.DECIMAL + TypeSig.NULL + TypeSig.ARRAY)),
(a, conf, p, r) => new GeneratorExprMeta[PosExplode](a, conf, p, r) {
+ override val supportOuter: Boolean = true
override def convertToGpu(): GpuExpression = GpuPosExplode(childExprs.head.convertToGpu())
}),
expr[CollectList](
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala
index 693f1a1a309..1cab4a4b59d 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala
@@ -1064,7 +1064,7 @@ object RapidsConf {
|On startup use: `--conf [conf key]=[conf value]`. For example:
|
|```
- |${SPARK_HOME}/bin/spark --jars 'rapids-4-spark_2.12-0.5.0.jar,cudf-0.19-cuda10-1.jar' \
+ |${SPARK_HOME}/bin/spark --jars 'rapids-4-spark_2.12-0.6.0-SNAPSHOT.jar,cudf-0.20-SNAPSHOT-cuda11.jar' \
|--conf spark.plugins=com.nvidia.spark.SQLPlugin \
|--conf spark.rapids.sql.incompatibleOps.enabled=true
|```
diff --git a/tests-spark310+/pom.xml b/tests-spark310+/pom.xml
index 567a705cc39..1dceaffff5e 100644
--- a/tests-spark310+/pom.xml
+++ b/tests-spark310+/pom.xml
@@ -22,11 +22,11 @@
com.nvidia
rapids-4-spark-parent
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
com.nvidia
rapids-4-spark-tests-next-spark_2.12
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/tests/pom.xml b/tests/pom.xml
index 826efacb78a..9ea6ee5daf0 100644
--- a/tests/pom.xml
+++ b/tests/pom.xml
@@ -22,13 +22,13 @@
com.nvidia
rapids-4-spark-parent
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
com.nvidia
rapids-4-spark-tests_2.12
RAPIDS Accelerator for Apache Spark Tests
RAPIDS plugin for Apache Spark integration tests
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/udf-compiler/pom.xml b/udf-compiler/pom.xml
index 138affddb46..761b0dd62e4 100644
--- a/udf-compiler/pom.xml
+++ b/udf-compiler/pom.xml
@@ -22,13 +22,13 @@
com.nvidia
rapids-4-spark-parent
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
com.nvidia
rapids-4-spark-udf_2.12
RAPIDS Accelerator for Apache Spark Scala UDF Plugin
The RAPIDS Scala UDF plugin for Apache Spark
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
diff --git a/udf-examples/Dockerfile b/udf-examples/Dockerfile
index 265ce849f3c..88f6d6c5ca5 100644
--- a/udf-examples/Dockerfile
+++ b/udf-examples/Dockerfile
@@ -15,63 +15,56 @@
#
# A container that can be used to build UDF native code against libcudf
-ARG CUDA_VERSION=10.1
-ARG CUDA_SHORT_VERSION=${CUDA_VERSION}
+ARG CUDA_VERSION=11.0
ARG LINUX_VERSION=ubuntu18.04
-ARG CUDF_REPO=https://github.com/rapidsai/cudf
-ARG CUDF_TAG=branch-0.19
-ARG CC=7
-ARG CXX=7
FROM nvidia/cuda:${CUDA_VERSION}-devel-${LINUX_VERSION}
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/lib
-# Needed for cudf.concat(), avoids "OSError: library nvvm not found"
-ENV NUMBAPRO_NVVM=/usr/local/cuda/nvvm/lib64/libnvvm.so
-ENV NUMBAPRO_LIBDEVICE=/usr/local/cuda/nvvm/libdevice/
-ENV DEBIAN_FRONTEND=noninteractive
-ARG CC
-ARG CXX
-RUN apt update -y --fix-missing && \
- apt upgrade -y && \
- apt install -y --no-install-recommends \
- git \
- gcc-${CC} \
- g++-${CXX} \
- libboost-all-dev \
- openjdk-8-jdk \
- maven \
- tzdata && \
- apt-get autoremove -y && \
- apt-get clean && \
- rm -rf /var/lib/apt/lists/*
+ARG DEBIAN_FRONTEND=noninteractive
+ARG PARALLEL_LEVEL=10
+ENV PARALLEL_LEVEL=10
-# Install conda
-ADD https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh /miniconda.sh
-RUN sh /miniconda.sh -b -p /conda && /conda/bin/conda update -n base conda
-ENV PATH=${PATH}:/conda/bin
-# Enables "source activate conda"
-SHELL ["/bin/bash", "-c"]
+RUN GCC_VERSION=$(bash -c '\
+CUDA_VERSION=$(nvcc --version | head -n4 | tail -n1 | cut -d" " -f5 | cut -d"," -f1); \
+CUDA_VERSION_MAJOR=$(echo $CUDA_VERSION | tr -d '.' | cut -c 1-2); \
+CUDA_VERSION_MINOR=$(echo $CUDA_VERSION | tr -d '.' | cut -c 3); \
+ if [[ "$CUDA_VERSION_MAJOR" == 9 ]]; then echo "7"; \
+ elif [[ "$CUDA_VERSION_MAJOR" == 10 ]]; then echo "8"; \
+ elif [[ "$CUDA_VERSION_MAJOR" == 11 ]]; then echo "9"; \
+ else echo "10"; \
+ fi') \
+&& apt update -y \
+&& apt install -y software-properties-common \
+&& add-apt-repository -y ppa:git-core/ppa \
+&& add-apt-repository -y ppa:ubuntu-toolchain-r/test \
+&& add-apt-repository ppa:deadsnakes/ppa \
+&& apt update -y \
+&& apt install -y \
+ build-essential git wget \
+ gcc-${GCC_VERSION} g++-${GCC_VERSION} \
+ openjdk-8-jdk maven tzdata \
+ # CMake dependencies
+ curl libssl-dev libcurl4-openssl-dev zlib1g-dev \
+ # cuDF dependencies
+ libboost-filesystem-dev \
+&& apt autoremove -y \
+&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+&& update-alternatives \
+ --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VERSION} 100 \
+# Set gcc-${GCC_VERSION} as the default gcc
+&& update-alternatives --set gcc /usr/bin/gcc-${GCC_VERSION} \
+# Set JDK8 as the default Java
+&& update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
-# Pull the cudf source at the specified branch or tag
-ARG CUDF_TAG
-RUN git clone --recurse-submodules --depth 1 --single-branch --branch ${CUDF_TAG} https://github.com/rapidsai/cudf.git /cudf
+ARG CMAKE_VERSION=3.18.5
-# Create the cudf conda environment
-ARG CUDA_SHORT_VERSION
-RUN conda env create --name cudf --file /cudf/conda/environments/cudf_dev_cuda${CUDA_SHORT_VERSION}.yml
+# Install CMake
+RUN cd /tmp \
+ && curl -fsSLO --compressed "https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.tar.gz" -o /tmp/cmake-$CMAKE_VERSION.tar.gz \
+ && tar -xvzf /tmp/cmake-$CMAKE_VERSION.tar.gz && cd /tmp/cmake-$CMAKE_VERSION \
+ && /tmp/cmake-$CMAKE_VERSION/bootstrap \
+ --system-curl \
+ --parallel=${PARALLEL_LEVEL} \
+ && make install -j${PARALLEL_LEVEL} \
+ && cd /tmp && rm -rf /tmp/cmake-$CMAKE_VERSION*
-# libcudf build/install
-ARG CC
-ARG CXX
-ENV CC=/usr/bin/gcc-${CC}
-ENV CXX=/usr/bin/g++-${CXX}
-RUN source activate cudf && \
- mkdir -p /cudf/cpp/build && \
- cd /cudf/cpp/build && \
- cmake .. -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX} && \
- make -j"$(nproc)" install
-
-# Default to cudf conda environment
-RUN echo "source activate cudf" > /.bash-init.sh
-ENTRYPOINT /bin/bash --init-file /.bash-init.sh
diff --git a/udf-examples/README.md b/udf-examples/README.md
index 532fc5db739..15e258b3f74 100644
--- a/udf-examples/README.md
+++ b/udf-examples/README.md
@@ -18,6 +18,5 @@ can be used to include the native UDF examples in the build, i.e.: specify
The `Dockerfile` in this directory can be used to setup a Docker image that
provides a libcudf build environment. This repository will either need to be
cloned or mounted into a container using that Docker image.
-There are a number of build arguments in the `Dockerfile` to control the
-Linux version, GCC version, CUDA version, and other settings. See the top of
-the `Dockerfile` for details.
+The `Dockerfile` contains build arguments to control the Linux version,
+CUDA version, and other settings. See the top of the `Dockerfile` for details.
diff --git a/udf-examples/pom.xml b/udf-examples/pom.xml
index 64d7d5cb9a4..1d5e698f43d 100644
--- a/udf-examples/pom.xml
+++ b/udf-examples/pom.xml
@@ -22,7 +22,7 @@
com.nvidia
rapids-4-spark-parent
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
com.nvidia
rapids-4-spark-udf-examples_2.12
@@ -30,15 +30,14 @@
Sample implementations of RAPIDS accelerated
user defined functions for use with the RAPIDS Accelerator
for Apache Spark
- 0.5.0-SNAPSHOT
+ 0.6.0-SNAPSHOT
${project.build.directory}/cpp-build
ALL
- ON
ON
- /cudf/cpp/build
+ 10
@@ -116,14 +115,14 @@
-
-
-
-
+
+
+
+
+
diff --git a/udf-examples/src/main/cpp/CMakeLists.txt b/udf-examples/src/main/cpp/CMakeLists.txt
index 17be9e0d6e5..aed7324c249 100755
--- a/udf-examples/src/main/cpp/CMakeLists.txt
+++ b/udf-examples/src/main/cpp/CMakeLists.txt
@@ -13,9 +13,35 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#=============================================================================
-cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
-project(UDFEXAMPLESJNI VERSION 0.5.0 LANGUAGES C CXX CUDA)
+cmake_minimum_required(VERSION 3.18 FATAL_ERROR)
+
+# Use GPU_ARCHS if it is defined
+if(DEFINED GPU_ARCHS)
+ if (GPU_ARCHS STREQUAL "ALL")
+ set(CUDF_BUILD_FOR_ALL_ARCHS TRUE)
+ else()
+ set(CMAKE_CUDA_ARCHITECTURES ${GPU_ARCHS})
+ endif()
+endif()
+
+project(UDFEXAMPLESJNI VERSION 0.6.0 LANGUAGES C CXX CUDA)
+
+option(PER_THREAD_DEFAULT_STREAM "Build with per-thread default stream" OFF)
+option(BUILD_UDF_BENCHMARKS "Build the benchmarks" OFF)
+
+#################################################################################################
+# - CPM -----------------------------------------------------------------------------------------
+
+set(CPM_DOWNLOAD_VERSION 0.27.2)
+set(CPM_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+
+if(NOT (EXISTS ${CPM_DOWNLOAD_LOCATION}))
+ message(STATUS "Downloading CPM.cmake")
+ file(DOWNLOAD https://github.com/TheLartians/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake ${CPM_DOWNLOAD_LOCATION})
+endif()
+
+include(${CPM_DOWNLOAD_LOCATION})
###################################################################################################
# - build type ------------------------------------------------------------------------------------
@@ -23,21 +49,11 @@ project(UDFEXAMPLESJNI VERSION 0.5.0 LANGUAGES C CXX CUDA)
# Set a default build type if none was specified
set(DEFAULT_BUILD_TYPE "Release")
-if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
- message(STATUS "Setting build type to '${DEFAULT_BUILD_TYPE}' since none specified.")
- set(CMAKE_BUILD_TYPE "${DEFAULT_BUILD_TYPE}" CACHE
- STRING "Choose the type of build." FORCE)
- # Set the possible values of build type for cmake-gui
- set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS
- "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
-endif()
-
###################################################################################################
# - compiler options ------------------------------------------------------------------------------
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
set(CMAKE_CXX_STANDARD 14)
-set(CMAKE_C_COMPILER $ENV{CC})
set(CMAKE_CXX_COMPILER $ENV{CXX})
set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -45,11 +61,13 @@ set(CMAKE_CUDA_STANDARD 14)
set(CMAKE_CUDA_STANDARD_REQUIRED ON)
if(CMAKE_COMPILER_IS_GNUCXX)
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-error=deprecated-declarations")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-pragmas")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=deprecated-declarations")
endif(CMAKE_COMPILER_IS_GNUCXX)
if(CMAKE_CUDA_COMPILER_VERSION)
- # Compute the version from CMAKE_CUDA_COMPILER_VERSION
+ # Compute the version. from CMAKE_CUDA_COMPILER_VERSION
string(REGEX REPLACE "([0-9]+)\\.([0-9]+).*" "\\1" CUDA_VERSION_MAJOR ${CMAKE_CUDA_COMPILER_VERSION})
string(REGEX REPLACE "([0-9]+)\\.([0-9]+).*" "\\2" CUDA_VERSION_MINOR ${CMAKE_CUDA_COMPILER_VERSION})
set(CUDA_VERSION "${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}" CACHE STRING "Version of CUDA as computed from nvcc.")
@@ -63,109 +81,28 @@ message(STATUS "CUDA_VERSION: ${CUDA_VERSION}")
# Always set this convenience variable
set(CUDA_VERSION_STRING "${CUDA_VERSION}")
-set(GPU_ARCHS "ALL" CACHE STRING
- "List of GPU architectures (semicolon-separated) to be compiled for. Pass 'ALL' if you want to compile for all supported GPU architectures.")
-
-if("${GPU_ARCHS}" STREQUAL "ALL")
-
- # Check for embedded vs workstation architectures
- if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
- # This is being built for Linux4Tegra or SBSA ARM64
- set(GPU_ARCHS "62")
- if((CUDA_VERSION_MAJOR EQUAL 9) OR (CUDA_VERSION_MAJOR GREATER 9))
- set(GPU_ARCHS "${GPU_ARCHS};72")
- endif()
- if((CUDA_VERSION_MAJOR EQUAL 11) OR (CUDA_VERSION_MAJOR GREATER 11))
- set(GPU_ARCHS "${GPU_ARCHS};75;80")
- endif()
-
- else()
- # This is being built for an x86 or x86_64 architecture
- set(GPU_ARCHS "60")
- if((CUDA_VERSION_MAJOR EQUAL 9) OR (CUDA_VERSION_MAJOR GREATER 9))
- set(GPU_ARCHS "${GPU_ARCHS};70")
- endif()
- if((CUDA_VERSION_MAJOR EQUAL 10) OR (CUDA_VERSION_MAJOR GREATER 10))
- set(GPU_ARCHS "${GPU_ARCHS};75")
- endif()
- if((CUDA_VERSION_MAJOR EQUAL 11) OR (CUDA_VERSION_MAJOR GREATER 11))
- set(GPU_ARCHS "${GPU_ARCHS};80")
- endif()
-
- endif()
-
-endif()
-message("GPU_ARCHS = ${GPU_ARCHS}")
-
-foreach(arch ${GPU_ARCHS})
- set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode=arch=compute_${arch},code=sm_${arch}")
-endforeach()
-
-set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
-
-# set warnings as errors
-set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Werror cross-execution-space-call -Xcompiler -Wall,-Werror,-Wno-error=deprecated-declarations")
-
-# Option to enable line info in CUDA device compilation to allow introspection when profiling / memchecking
-option(CMAKE_CUDA_LINEINFO "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF)
-if (CMAKE_CUDA_LINEINFO)
- set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo")
-endif(CMAKE_CUDA_LINEINFO)
-
-# Debug options
-if(CMAKE_BUILD_TYPE MATCHES Debug)
- message(STATUS "Building with debugging flags")
- set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -G -Xcompiler -rdynamic")
-endif(CMAKE_BUILD_TYPE MATCHES Debug)
-
-option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" ON)
-
-if(CUDA_STATIC_RUNTIME)
- message(STATUS "Enabling static linking of cudart")
- set(CUDART_LIBRARY "cudart_static")
-else()
- message(STATUS "Enabling dynamic linking of cudart")
- set(CUDART_LIBRARY "cudart")
-endif(CUDA_STATIC_RUNTIME)
-
-###################################################################################################
-# - CUDF ------------------------------------------------------------------------------------------
-
-find_path(CUDF_INCLUDE "cudf"
- HINTS "$ENV{CUDF_ROOT}"
- "$ENV{CONDA_PREFIX}/include")
-
-find_library(CUDF_BASE_LIBRARY "cudf_base"
- HINTS "$ENV{CUDF_ROOT}"
- "$ENV{CUDF_ROOT}/lib"
- "$ENV{CONDA_PREFIX}/lib")
-
-###################################################################################################
-# - RMM -------------------------------------------------------------------------------------------
-
-find_path(RMM_INCLUDE "rmm"
- HINTS "$ENV{RMM_ROOT}/include"
- "$ENV{CONDA_PREFIX}/include")
-
-###################################################################################################
-# - Thrust/CUB/libcudacxx ------------------------------------------------------------------------------------
-
-# path to libcudf cmake build directory
-if(NOT DEFINED CUDF_CMAKE_BUILD_DIR)
- set(CUDF_CMAKE_BUILD_DIR "/cudf/cpp/build")
-endif(NOT DEFINED CUDF_CMAKE_BUILD_DIR)
-
-find_path(THRUST_INCLUDE "thrust"
- HINTS "$ENV{CUDF_ROOT}/_deps/thrust-src"
- "${CUDF_CMAKE_BUILD_DIR}/_deps/thrust-src")
-
-find_path(CUB_INCLUDE "cub"
- HINTS "$ENV{CUDF_ROOT}/_deps/thrust-src"
- "${CUDF_CMAKE_BUILD_DIR}/_deps/thrust-src")
-
-find_path(LIBCUDACXX_INCLUDE "cuda"
- HINTS "$ENV{CUDF_ROOT}/_deps/libcudacxx-src/include"
- "${CUDF_CMAKE_BUILD_DIR}/_deps/libcudacxx-src/include")
+set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -w --expt-extended-lambda --expt-relaxed-constexpr")
+
+####################################################################################################
+# - cudf -------------------------------------------------------------------------------------------
+
+# Ensure CUDA runtime is dynamic despite statically linking Arrow in libcudf
+set(CUDA_USE_STATIC_CUDA_RUNTIME OFF)
+
+CPMAddPackage(NAME cudf
+ VERSION "0.19.0"
+ GIT_REPOSITORY https://github.com/rapidsai/cudf.git
+ GIT_TAG branch-0.19
+ GIT_SHALLOW TRUE
+ SOURCE_SUBDIR cpp
+ OPTIONS "BUILD_TESTS OFF"
+ "BUILD_BENCHMARKS OFF"
+ "CUDF_USE_ARROW_STATIC ON"
+ "JITIFY_USE_CACHE ON"
+ "CUDA_STATIC_RUNTIME OFF"
+ "DISABLE_DEPRECATION_WARNING ON"
+ "AUTO_DETECT_CUDA_ARCHITECTURES OFF"
+ )
###################################################################################################
# - find JNI -------------------------------------------------------------------------------------
@@ -177,18 +114,6 @@ else()
message(FATAL_ERROR "JDK with JNI not found, please check your settings.")
endif(JNI_FOUND)
-###################################################################################################
-# - include paths ---------------------------------------------------------------------------------
-
-include_directories("${THRUST_INCLUDE}"
- "${CUB_INCLUDE}"
- "${LIBCUDACXX_INCLUDE}"
- "${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}"
- "${CMAKE_SOURCE_DIR}/src"
- "${CUDF_INCLUDE}"
- "${RMM_INCLUDE}"
- "${JNI_INCLUDE_DIRS}")
-
###################################################################################################
# - library paths ---------------------------------------------------------------------------------
@@ -215,13 +140,15 @@ SET_TARGET_PROPERTIES(udfexamplesjni PROPERTIES BUILD_RPATH "\$ORIGIN")
option(PER_THREAD_DEFAULT_STREAM "Build with per-thread default stream" OFF)
if(PER_THREAD_DEFAULT_STREAM)
message(STATUS "Using per-thread default stream")
- add_compile_definitions(CUDA_API_PER_THREAD_DEFAULT_STREAM)
+ target_compile_definitions(udfexamplesjni PRIVATE CUDA_API_PER_THREAD_DEFAULT_STREAM)
endif(PER_THREAD_DEFAULT_STREAM)
+target_include_directories(udfexamplesjni PRIVATE ${JNI_INCLUDE_DIRS})
+
###################################################################################################
# - rmm logging level -----------------------------------------------------------------------------
-set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.")
+set(RMM_LOGGING_LEVEL "OFF" CACHE STRING "Choose the logging level.")
# Set the possible values of build type for cmake-gui
set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS
"TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF")
@@ -233,4 +160,4 @@ target_compile_definitions(udfexamplesjni
###################################################################################################
# - link libraries --------------------------------------------------------------------------------
-target_link_libraries(udfexamplesjni ${CUDF_BASE_LIBRARY} ${CUDART_LIBRARY} cuda)
+target_link_libraries(udfexamplesjni cudf::cudf)