From 8da9b188811514d9f750d94a51933e4162c2ad98 Mon Sep 17 00:00:00 2001 From: NvTimLiu <50287591+NvTimLiu@users.noreply.github.com> Date: Fri, 16 Apr 2021 09:49:21 +0800 Subject: [PATCH 1/3] Change version to 0.4.2-SNAPSHOT (#2153) Change version to 0.4.2-SNAPSHOT, prepare for v0.4.2 release Signed-off-by: Tim Liu --- api_validation/pom.xml | 6 +++--- dist/pom.xml | 4 ++-- integration_tests/pom.xml | 4 ++-- jenkins/databricks/create.py | 2 +- jenkins/version-def.sh | 6 +++--- pom.xml | 4 ++-- shims/aggregator/pom.xml | 4 ++-- shims/pom.xml | 4 ++-- shims/spark300/pom.xml | 4 ++-- shims/spark300emr/pom.xml | 4 ++-- shims/spark301/pom.xml | 4 ++-- shims/spark301db/pom.xml | 4 ++-- shims/spark301emr/pom.xml | 4 ++-- shims/spark302/pom.xml | 4 ++-- shims/spark311/pom.xml | 4 ++-- shuffle-plugin/pom.xml | 4 ++-- sql-plugin/pom.xml | 4 ++-- tests-spark310+/pom.xml | 4 ++-- tests/pom.xml | 4 ++-- udf-compiler/pom.xml | 4 ++-- udf-examples/pom.xml | 4 ++-- 21 files changed, 43 insertions(+), 43 deletions(-) diff --git a/api_validation/pom.xml b/api_validation/pom.xml index ff2694a40bb..1af94ee7c7b 100644 --- a/api_validation/pom.xml +++ b/api_validation/pom.xml @@ -22,10 +22,10 @@ com.nvidia rapids-4-spark-parent - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT rapids-4-spark-api-validation - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT @@ -78,7 +78,7 @@ com.nvidia rapids-4-spark-shims-aggregator_${scala.binary.version} - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT provided diff --git a/dist/pom.xml b/dist/pom.xml index 2839aeecd76..7f461e43034 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT com.nvidia rapids-4-spark_2.12 RAPIDS Accelerator for Apache Spark Distribution Creates the distribution package of the RAPIDS plugin for Apache Spark - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT diff --git a/integration_tests/pom.xml b/integration_tests/pom.xml index 939835a6c8d..207ff2e1d0a 100644 --- a/integration_tests/pom.xml +++ b/integration_tests/pom.xml @@ -22,11 +22,11 @@ com.nvidia rapids-4-spark-parent - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT com.nvidia rapids-4-spark-integration-tests_2.12 - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT ${spark300.version} diff --git a/jenkins/databricks/create.py b/jenkins/databricks/create.py index dfc4c990d5e..f6b615f014b 100644 --- a/jenkins/databricks/create.py +++ b/jenkins/databricks/create.py @@ -27,7 +27,7 @@ def main(): workspace = 'https://dbc-9ff9942e-a9c4.cloud.databricks.com' token = '' sshkey = '' - cluster_name = 'CI-GPU-databricks-0.4.1-SNAPSHOT' + cluster_name = 'CI-GPU-databricks-0.4.2-SNAPSHOT' idletime = 240 runtime = '7.0.x-gpu-ml-scala2.12' num_workers = 1 diff --git a/jenkins/version-def.sh b/jenkins/version-def.sh index c014b3630e4..ba2997261c9 100755 --- a/jenkins/version-def.sh +++ b/jenkins/version-def.sh @@ -26,10 +26,10 @@ for VAR in $OVERWRITE_PARAMS;do done IFS=$PRE_IFS -CUDF_VER=${CUDF_VER:-"0.18.1"} +CUDF_VER=${CUDF_VER:-"0.18.2"} CUDA_CLASSIFIER=${CUDA_CLASSIFIER:-"cuda10-1"} -PROJECT_VER=${PROJECT_VER:-"0.4.1-SNAPSHOT"} -PROJECT_TEST_VER=${PROJECT_TEST_VER:-"0.4.1-SNAPSHOT"} +PROJECT_VER=${PROJECT_VER:-"0.4.2-SNAPSHOT"} +PROJECT_TEST_VER=${PROJECT_TEST_VER:-"0.4.2-SNAPSHOT"} SPARK_VER=${SPARK_VER:-"3.0.0"} SCALA_BINARY_VER=${SCALA_BINARY_VER:-"2.12"} SERVER_ID=${SERVER_ID:-"snapshots"} diff --git a/pom.xml b/pom.xml index 5fba23b0e28..ebb9130642a 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ rapids-4-spark-parent RAPIDS Accelerator for Apache Spark Root Project The root project of the RAPIDS Accelerator for Apache Spark - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT pom https://github.com/NVIDIA @@ -147,7 +147,7 @@ 1.8 ${spark300.version} cuda10-1 - 0.18.1 + 0.18.2 2.12 2.12.8 1.5.8 diff --git a/shims/aggregator/pom.xml b/shims/aggregator/pom.xml index 7da3529f5cc..71799ac6ab0 100644 --- a/shims/aggregator/pom.xml +++ b/shims/aggregator/pom.xml @@ -22,7 +22,7 @@ com.nvidia rapids-4-spark-shims_2.12 - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT ../pom.xml com.nvidia @@ -30,7 +30,7 @@ jar RAPIDS Accelerator for Apache Spark SQL Plugin Shim Aggregator The RAPIDS SQL plugin for Apache Spark Shim Aggregator - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT diff --git a/shims/pom.xml b/shims/pom.xml index 5c160fe8cde..f203a86a5dc 100644 --- a/shims/pom.xml +++ b/shims/pom.xml @@ -22,7 +22,7 @@ com.nvidia rapids-4-spark-parent - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT ../pom.xml com.nvidia @@ -30,7 +30,7 @@ pom RAPIDS Accelerator for Apache Spark SQL Plugin Shims The RAPIDS SQL plugin for Apache Spark Shims - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT diff --git a/shims/spark300/pom.xml b/shims/spark300/pom.xml index 1ccd315b261..04fd02a933b 100644 --- a/shims/spark300/pom.xml +++ b/shims/spark300/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-shims_2.12 - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT ../pom.xml com.nvidia rapids-4-spark-shims-spark300_2.12 RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.0.0 Shim The RAPIDS SQL plugin for Apache Spark 3.0.0 Shim - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT diff --git a/shims/spark300emr/pom.xml b/shims/spark300emr/pom.xml index 386ed523aa1..eacb421d936 100644 --- a/shims/spark300emr/pom.xml +++ b/shims/spark300emr/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-shims_2.12 - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT ../pom.xml com.nvidia rapids-4-spark-shims-spark300emr_2.12 RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.0.0 EMR Shim The RAPIDS SQL plugin for Apache Spark 3.0.0 EMR Shim - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT diff --git a/shims/spark301/pom.xml b/shims/spark301/pom.xml index 008af268315..480caf91cc2 100644 --- a/shims/spark301/pom.xml +++ b/shims/spark301/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-shims_2.12 - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT ../pom.xml com.nvidia rapids-4-spark-shims-spark301_2.12 RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.0.1 Shim The RAPIDS SQL plugin for Apache Spark 3.0.1 Shim - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT diff --git a/shims/spark301db/pom.xml b/shims/spark301db/pom.xml index 33c67d1a62e..4aa4bf1b89d 100644 --- a/shims/spark301db/pom.xml +++ b/shims/spark301db/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-shims_2.12 - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT ../pom.xml com.nvidia rapids-4-spark-shims-spark301-databricks_2.12 RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.0.1 Databricks Shim The RAPIDS SQL plugin for Apache Spark 3.0.1 Databricks Shim - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT diff --git a/shims/spark301emr/pom.xml b/shims/spark301emr/pom.xml index a1d6e6dd20b..ff1fc7b36e7 100644 --- a/shims/spark301emr/pom.xml +++ b/shims/spark301emr/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-shims_2.12 - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT ../pom.xml com.nvidia rapids-4-spark-shims-spark301emr_2.12 RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.0.1 EMR Shim The RAPIDS SQL plugin for Apache Spark 3.0.1 EMR Shim - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT diff --git a/shims/spark302/pom.xml b/shims/spark302/pom.xml index aa5aa06b6d3..c3f04a59344 100644 --- a/shims/spark302/pom.xml +++ b/shims/spark302/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-shims_2.12 - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT ../pom.xml com.nvidia rapids-4-spark-shims-spark302_2.12 RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.0.2 Shim The RAPIDS SQL plugin for Apache Spark 3.0.2 Shim - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT diff --git a/shims/spark311/pom.xml b/shims/spark311/pom.xml index d0362dd416e..224c0861dae 100644 --- a/shims/spark311/pom.xml +++ b/shims/spark311/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-shims_2.12 - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT ../pom.xml com.nvidia rapids-4-spark-shims-spark311_2.12 RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.1.1 Shim The RAPIDS SQL plugin for Apache Spark 3.1.1 Shim - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT diff --git a/shuffle-plugin/pom.xml b/shuffle-plugin/pom.xml index 26ff8253d15..e945694b0bf 100644 --- a/shuffle-plugin/pom.xml +++ b/shuffle-plugin/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-parent - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT com.nvidia rapids-4-spark-shuffle_2.12 RAPIDS Accelerator for Apache Spark Shuffle Plugin Accelerated shuffle plugin for the RAPIDS plugin for Apache Spark - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT diff --git a/sql-plugin/pom.xml b/sql-plugin/pom.xml index ee865f27102..0a9cedb3ab1 100644 --- a/sql-plugin/pom.xml +++ b/sql-plugin/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT com.nvidia rapids-4-spark-sql_2.12 RAPIDS Accelerator for Apache Spark SQL Plugin The RAPIDS SQL plugin for Apache Spark - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT diff --git a/tests-spark310+/pom.xml b/tests-spark310+/pom.xml index d9f32de4549..5d1acd26f6a 100644 --- a/tests-spark310+/pom.xml +++ b/tests-spark310+/pom.xml @@ -22,11 +22,11 @@ com.nvidia rapids-4-spark-parent - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT com.nvidia rapids-4-spark-tests-next-spark_2.12 - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT ${spark311.version} diff --git a/tests/pom.xml b/tests/pom.xml index 8ffd6e1af64..99b21307853 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT com.nvidia rapids-4-spark-tests_2.12 RAPIDS Accelerator for Apache Spark Tests RAPIDS plugin for Apache Spark integration tests - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT ${spark300.version} diff --git a/udf-compiler/pom.xml b/udf-compiler/pom.xml index 80d3e6ae846..57ff3950bc9 100644 --- a/udf-compiler/pom.xml +++ b/udf-compiler/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT com.nvidia rapids-4-spark-udf_2.12 RAPIDS Accelerator for Apache Spark Scala UDF Plugin The RAPIDS Scala UDF plugin for Apache Spark - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT diff --git a/udf-examples/pom.xml b/udf-examples/pom.xml index 8a4a1cb7796..c3eecc50ce1 100644 --- a/udf-examples/pom.xml +++ b/udf-examples/pom.xml @@ -22,7 +22,7 @@ com.nvidia rapids-4-spark-parent - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT com.nvidia rapids-4-spark-udf-examples_2.12 @@ -30,7 +30,7 @@ Sample implementations of RAPIDS accelerated user defined functions for use with the RAPIDS Accelerator for Apache Spark - 0.4.1-SNAPSHOT + 0.4.2-SNAPSHOT ${project.build.directory}/cpp-build From f562a6960670571c60f6fd65d5761e5fc34638e8 Mon Sep 17 00:00:00 2001 From: NvTimLiu <50287591+NvTimLiu@users.noreply.github.com> Date: Fri, 16 Apr 2021 11:12:13 +0800 Subject: [PATCH 2/3] Update docs for rapids v0.4.2 release [skip ci] (#2155) * Update docs for rapids v0.4.2 release Update docs for rapids v0.4.2 release Update cuDF dependency to v0.18.2 Signed-off-by: Tim Liu * update download and changelog script * update download.md Signed-off-by: Tim Liu * update download.md * update for the indentation * Update download.md * update download.md --- docs/configs.md | 2 +- .../Databricks/generate-init-script.ipynb | 2 +- docs/download.md | 35 +++++++++++++++++++ docs/get-started/Dockerfile.cuda | 4 +-- docs/get-started/getting-started-on-prem.md | 10 +++--- integration_tests/README.md | 6 ++-- scripts/generate-changelog | 8 ++--- .../com/nvidia/spark/rapids/RapidsConf.scala | 2 +- 8 files changed, 52 insertions(+), 17 deletions(-) diff --git a/docs/configs.md b/docs/configs.md index 1ed1c5fbfab..e3a46326490 100644 --- a/docs/configs.md +++ b/docs/configs.md @@ -10,7 +10,7 @@ The following is the list of options that `rapids-plugin-4-spark` supports. On startup use: `--conf [conf key]=[conf value]`. For example: ``` -${SPARK_HOME}/bin/spark --jars 'rapids-4-spark_2.12-0.4.1.jar,cudf-0.18.1-cuda10-1.jar' \ +${SPARK_HOME}/bin/spark --jars 'rapids-4-spark_2.12-0.4.2.jar,cudf-0.18.2-cuda10-1.jar' \ --conf spark.plugins=com.nvidia.spark.SQLPlugin \ --conf spark.rapids.sql.incompatibleOps.enabled=true ``` diff --git a/docs/demo/Databricks/generate-init-script.ipynb b/docs/demo/Databricks/generate-init-script.ipynb index 1c3170471b2..06a5ced7c0a 100644 --- a/docs/demo/Databricks/generate-init-script.ipynb +++ b/docs/demo/Databricks/generate-init-script.ipynb @@ -1 +1 @@ -{"cells":[{"cell_type":"code","source":["dbutils.fs.mkdirs(\"dbfs:/databricks/init_scripts/\")\n \ndbutils.fs.put(\"/databricks/init_scripts/init.sh\",\"\"\"\n#!/bin/bash\nsudo wget -O /databricks/jars/rapids-4-spark_2.12-0.4.1.jar https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/0.4.1/rapids-4-spark_2.12-0.4.1.jar\nsudo wget -O /databricks/jars/cudf-0.18.1-cuda10-1.jar https://repo1.maven.org/maven2/ai/rapids/cudf/0.18.1/cudf-0.18.1-cuda10-1.jar\"\"\", True)"],"metadata":{},"outputs":[],"execution_count":1},{"cell_type":"code","source":["%sh\ncd ../../dbfs/databricks/init_scripts\npwd\nls -ltr\ncat init.sh"],"metadata":{},"outputs":[],"execution_count":2},{"cell_type":"code","source":[""],"metadata":{},"outputs":[],"execution_count":3}],"metadata":{"name":"generate-init-script","notebookId":2645746662301564},"nbformat":4,"nbformat_minor":0} +{"cells":[{"cell_type":"code","source":["dbutils.fs.mkdirs(\"dbfs:/databricks/init_scripts/\")\n \ndbutils.fs.put(\"/databricks/init_scripts/init.sh\",\"\"\"\n#!/bin/bash\nsudo wget -O /databricks/jars/rapids-4-spark_2.12-0.4.2.jar https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/0.4.2/rapids-4-spark_2.12-0.4.2.jar\nsudo wget -O /databricks/jars/cudf-0.18.2-cuda10-1.jar https://repo1.maven.org/maven2/ai/rapids/cudf/0.18.2/cudf-0.18.2-cuda10-1.jar\"\"\", True)"],"metadata":{},"outputs":[],"execution_count":1},{"cell_type":"code","source":["%sh\ncd ../../dbfs/databricks/init_scripts\npwd\nls -ltr\ncat init.sh"],"metadata":{},"outputs":[],"execution_count":2},{"cell_type":"code","source":[""],"metadata":{},"outputs":[],"execution_count":3}],"metadata":{"name":"generate-init-script","notebookId":2645746662301564},"nbformat":4,"nbformat_minor":0} diff --git a/docs/download.md b/docs/download.md index 86d85af4ed0..049f354bd9e 100644 --- a/docs/download.md +++ b/docs/download.md @@ -4,6 +4,41 @@ title: Download nav_order: 3 --- +## Release v0.4.2 + +This is a patch release based on version 0.4.1 with the following change: +* Depend on cuDF 0.18.2 + +The release is supported on Apache Spark 3.0.0, 3.0.1, 3.0.2, 3.1.1, Databricks 7.3 ML LTS and +Google Cloud Platform Dataproc 2.0. + +The list of all supported operations is provided [here](supported_ops.md). + +For a detailed list of changes, please refer to the +[CHANGELOG](https://github.com/NVIDIA/spark-rapids/blob/main/CHANGELOG.md). + +Hardware Requirements: + + GPU Architecture: NVIDIA Pascal™ or better (Tested on V100, T4 and A100 GPU) + +Software Requirements: + + OS: Ubuntu 16.04, Ubuntu 18.04 or CentOS 7 + + CUDA & Nvidia Drivers: 10.1.2 & v418.87+, 10.2 & v440.33+ or 11.0 & v450.36+ + + Apache Spark 3.0, 3.0.1, 3.0.2, 3.1.1, Databricks 7.3 ML LTS Runtime, or GCP Dataproc 2.0 + + Apache Hadoop 2.10+ or 3.1.1+ (3.1.1 for nvidia-docker version 2) + + Python 3.6+, Scala 2.12, Java 8 + +### Download v0.4.2 +* [RAPIDS Spark Package](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/0.4.2/rapids-4-spark_2.12-0.4.2.jar) +* [cuDF 11.0 Package](https://repo1.maven.org/maven2/ai/rapids/cudf/0.18.2/cudf-0.18.2-cuda11.jar) +* [cuDF 10.2 Package](https://repo1.maven.org/maven2/ai/rapids/cudf/0.18.2/cudf-0.18.2-cuda10-2.jar) +* [cuDF 10.1 Package](https://repo1.maven.org/maven2/ai/rapids/cudf/0.18.2/cudf-0.18.2-cuda10-1.jar) + ## Release v0.4.1 This is a patch release based on version 0.4.0 with the following additional fixes: diff --git a/docs/get-started/Dockerfile.cuda b/docs/get-started/Dockerfile.cuda index 6cf5ff08fd4..9e2e90c6af9 100644 --- a/docs/get-started/Dockerfile.cuda +++ b/docs/get-started/Dockerfile.cuda @@ -52,8 +52,8 @@ COPY spark-3.0.1-bin-hadoop3.2/examples /opt/spark/examples COPY spark-3.0.1-bin-hadoop3.2/kubernetes/tests /opt/spark/tests COPY spark-3.0.1-bin-hadoop3.2/data /opt/spark/data -COPY cudf-0.18.1-cuda10-1.jar /opt/sparkRapidsPlugin -COPY rapids-4-spark_2.12-0.4.1.jar /opt/sparkRapidsPlugin +COPY cudf-0.18.2-cuda10-1.jar /opt/sparkRapidsPlugin +COPY rapids-4-spark_2.12-0.4.2.jar /opt/sparkRapidsPlugin COPY getGpusResources.sh /opt/sparkRapidsPlugin RUN mkdir /opt/spark/python diff --git a/docs/get-started/getting-started-on-prem.md b/docs/get-started/getting-started-on-prem.md index cc1887d9d7c..43ff2c71e98 100644 --- a/docs/get-started/getting-started-on-prem.md +++ b/docs/get-started/getting-started-on-prem.md @@ -55,16 +55,16 @@ CUDA and will not run on other versions. The jars use a maven classifier to keep - CUDA 11.0 => classifier cuda11 For example, here is a sample version of the jars and cudf with CUDA 10.1 support: -- cudf-0.18.1-cuda10-1.jar -- rapids-4-spark_2.12-0.4.1.jar +- cudf-0.18.2-cuda10-1.jar +- rapids-4-spark_2.12-0.4.2.jar For simplicity export the location to these jars. This example assumes the sample jars above have been placed in the `/opt/sparkRapidsPlugin` directory: ```shell export SPARK_RAPIDS_DIR=/opt/sparkRapidsPlugin -export SPARK_CUDF_JAR=${SPARK_RAPIDS_DIR}/cudf-0.18.1-cuda10-1.jar -export SPARK_RAPIDS_PLUGIN_JAR=${SPARK_RAPIDS_DIR}/rapids-4-spark_2.12-0.4.1.jar +export SPARK_CUDF_JAR=${SPARK_RAPIDS_DIR}/cudf-0.18.2-cuda10-1.jar +export SPARK_RAPIDS_PLUGIN_JAR=${SPARK_RAPIDS_DIR}/rapids-4-spark_2.12-0.4.2.jar ``` ## Install the GPU Discovery Script @@ -437,7 +437,7 @@ To enable _GPU Scheduling for Pandas UDF_, you need to configure your spark job On Standalone, you need to add ```shell ... - --conf spark.executorEnv.PYTHONPATH=rapids-4-spark_2.12-0.4.1.jar \ + --conf spark.executorEnv.PYTHONPATH=rapids-4-spark_2.12-0.4.2.jar \ --py-files ${SPARK_RAPIDS_PLUGIN_JAR} ``` diff --git a/integration_tests/README.md b/integration_tests/README.md index e5acdbbf9e0..61f5678b46b 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -107,7 +107,7 @@ individually, so you don't risk running unit tests along with the integration te http://www.scalatest.org/user_guide/using_the_scalatest_shell ```shell -spark-shell --jars rapids-4-spark-tests_2.12-0.4.1-tests.jar,rapids-4-spark-udf-examples_2.12-0.4.1,rapids-4-spark-integration-tests_2.12-0.4.1-tests.jar,scalatest_2.12-3.0.5.jar,scalactic_2.12-3.0.5.jar +spark-shell --jars rapids-4-spark-tests_2.12-0.4.2-tests.jar,rapids-4-spark-udf-examples_2.12-0.4.2,rapids-4-spark-integration-tests_2.12-0.4.2-tests.jar,scalatest_2.12-3.0.5.jar,scalactic_2.12-3.0.5.jar ``` First you import the `scalatest_shell` and tell the tests where they can find the test files you @@ -131,7 +131,7 @@ If you just want to verify the SQL replacement is working you will need to add t example assumes CUDA 10.1 is being used. ``` -$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-0.4.1.jar,rapids-4-spark-udf-examples_2.12-0.4.1.jar,cudf-0.18.1-cuda10-1.jar" ./runtests.py +$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-0.4.2.jar,rapids-4-spark-udf-examples_2.12-0.4.2.jar,cudf-0.18.2-cuda10-1.jar" ./runtests.py ``` You don't have to enable the plugin for this to work, the test framework will do that for you. @@ -192,7 +192,7 @@ To run cudf_udf tests, need following configuration changes: As an example, here is the `spark-submit` command with the cudf_udf parameter on CUDA 10.1: ``` -$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-0.4.1.jar,rapids-4-spark-udf-examples_2.12-0.4.1.jar,cudf-0.18.1-cuda10-1.jar,rapids-4-spark-tests_2.12-0.4.1.jar" --conf spark.rapids.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.concurrentPythonWorkers=2 --py-files "rapids-4-spark_2.12-0.4.1.jar" --conf spark.executorEnv.PYTHONPATH="rapids-4-spark_2.12-0.4.1.jar" ./runtests.py --cudf_udf +$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-0.4.2.jar,rapids-4-spark-udf-examples_2.12-0.4.2.jar,cudf-0.18.2-cuda10-1.jar,rapids-4-spark-tests_2.12-0.4.2.jar" --conf spark.rapids.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.concurrentPythonWorkers=2 --py-files "rapids-4-spark_2.12-0.4.2.jar" --conf spark.executorEnv.PYTHONPATH="rapids-4-spark_2.12-0.4.2.jar" ./runtests.py --cudf_udf ``` ## Writing tests diff --git a/scripts/generate-changelog b/scripts/generate-changelog index d6138847d39..cee34aa3420 100755 --- a/scripts/generate-changelog +++ b/scripts/generate-changelog @@ -44,11 +44,11 @@ Github personal access token: https://github.com/settings/tokens, and make you h Usage: cd spark-rapids/ - # generate changelog for release 0.1,0.2,0.3,0.4,0.4.1 - scripts/generate-changelog --token= --releases=0.1,0.2,0.3,0.4,0.4.1 + # generate changelog for release 0.1,0.2,0.3,0.4,0.4.1,0.4.2 + scripts/generate-changelog --token= --releases=0.1,0.2,0.3,0.4,0.4.1,0.4.2 - # generate changelog for release 0.1,0.2,0.3,0.4,0.4.1 to /tmp/CHANGELOG.md - GITHUB_TOKEN= scripts/generate-changelog --releases=0.1,0.2,0.3,0.4,0.4.1 --path=/tmp/CHANGELOG.md + # generate changelog for release 0.1,0.2,0.3,0.4,0.4.1,0.4.2 to /tmp/CHANGELOG.md + GITHUB_TOKEN= scripts/generate-changelog --releases=0.1,0.2,0.3,0.4,0.4.1,0.4.2 --path=/tmp/CHANGELOG.md """ import os import sys diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala index 269ebd7a7b6..b44d4578f3a 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala @@ -916,7 +916,7 @@ object RapidsConf { |On startup use: `--conf [conf key]=[conf value]`. For example: | |``` - |${SPARK_HOME}/bin/spark --jars 'rapids-4-spark_2.12-0.4.1.jar,cudf-0.18.1-cuda10-1.jar' \ + |${SPARK_HOME}/bin/spark --jars 'rapids-4-spark_2.12-0.4.2.jar,cudf-0.18.2-cuda10-1.jar' \ |--conf spark.plugins=com.nvidia.spark.SQLPlugin \ |--conf spark.rapids.sql.incompatibleOps.enabled=true |``` From f42acc270135fc88a87ea250076a89a856c168e5 Mon Sep 17 00:00:00 2001 From: NvTimLiu <50287591+NvTimLiu@users.noreply.github.com> Date: Fri, 16 Apr 2021 11:25:49 +0800 Subject: [PATCH 3/3] Update changelog for v0.4.2 release (#2156) Signed-off-by: Tim Liu --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adfa27ff6d6..1dfc6b7609c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Change log -Generated on 2021-03-23 +Generated on 2021-04-16 + +## Release 0.4.2 + +### PRs +||| +|:---|:---| +|[#2155](https://github.com/NVIDIA/spark-rapids/pull/2155)|Update docs for rapids v0.4.2 release [skip ci]| +|[#2153](https://github.com/NVIDIA/spark-rapids/pull/2153)|Change version to 0.4.2-SNAPSHOT [skip ci]| ## Release 0.4.1