From 8001c3f9236d4a1d5fef3198fb8f004d2bf96628 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Tue, 30 Jun 2020 13:07:12 -0500 Subject: [PATCH 1/2] Update patch to latest --- jenkins/databricks/dbimports.patch | 89 +++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 3 deletions(-) diff --git a/jenkins/databricks/dbimports.patch b/jenkins/databricks/dbimports.patch index d112e85ec7b..316daaab2bf 100644 --- a/jenkins/databricks/dbimports.patch +++ b/jenkins/databricks/dbimports.patch @@ -1,5 +1,88 @@ +diff --git a/jenkins/databricks/dbimports.patch b/jenkins/databricks/dbimports.patch +index d112e85..e69de29 100644 +--- a/jenkins/databricks/dbimports.patch ++++ b/jenkins/databricks/dbimports.patch +@@ -1,78 +0,0 @@ +-diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashJoin.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashJoin.scala +-index e6c3e37..ddd8ca4 100644 +---- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashJoin.scala +-+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashJoin.scala +-@@ -19,8 +19,9 @@ import ai.rapids.cudf.{NvtxColor, Table} +- +- import org.apache.spark.TaskContext +- import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} +-+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight} +- import org.apache.spark.sql.catalyst.plans.{ExistenceJoin, FullOuter, Inner, InnerLike, JoinType, LeftAnti, LeftExistence, LeftOuter, LeftSemi, RightOuter} +--import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight, HashJoin} +-+import org.apache.spark.sql.execution.joins.HashJoin +- import org.apache.spark.sql.execution.metric.SQLMetric +- import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} +- +-diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuShuffledHashJoinExec.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuShuffledHashJoinExec.scala +-index 7ae310b..3ebde77 100644 +---- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuShuffledHashJoinExec.scala +-+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuShuffledHashJoinExec.scala +-@@ -22,10 +22,11 @@ import org.apache.spark.TaskContext +- import org.apache.spark.rdd.RDD +- import org.apache.spark.sql.catalyst.InternalRow +- import org.apache.spark.sql.catalyst.expressions.Expression +-+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} +- import org.apache.spark.sql.catalyst.plans.JoinType +- import org.apache.spark.sql.catalyst.plans.physical.{Distribution, HashClusteredDistribution} +- import org.apache.spark.sql.execution.{BinaryExecNode, SparkPlan} +--import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight, BuildSide, ShuffledHashJoinExec} +-+import org.apache.spark.sql.execution.joins.ShuffledHashJoinExec +- import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} +- import org.apache.spark.sql.vectorized.ColumnarBatch +- +-diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuSortMergeJoinExec.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuSortMergeJoinExec.scala +-index 29ba63d..78febd4 100644 +---- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuSortMergeJoinExec.scala +-+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuSortMergeJoinExec.scala +-@@ -17,8 +17,9 @@ +- package com.nvidia.spark.rapids +- +- import org.apache.spark.internal.Logging +-+import org.apache.spark.sql.catalyst.optimizer.BuildRight +- import org.apache.spark.sql.execution.SortExec +--import org.apache.spark.sql.execution.joins.{BuildRight, SortMergeJoinExec} +-+import org.apache.spark.sql.execution.joins.SortMergeJoinExec +- +- class GpuSortMergeJoinMeta( +- join: SortMergeJoinExec, +-diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala +-index b02182a..1ed13d2 100644 +---- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala +-+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala +-@@ -22,12 +22,13 @@ import com.nvidia.spark.rapids.GpuOverrides.isStringLit +- +- import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, ComplexTypeMergingExpression, Expression, String2TrimExpression, TernaryExpression, UnaryExpression} +- import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateFunction +-+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight} +- import org.apache.spark.sql.catalyst.plans.physical.Partitioning +- import org.apache.spark.sql.connector.read.Scan +- import org.apache.spark.sql.execution.SparkPlan +- import org.apache.spark.sql.execution.command.DataWritingCommand +- import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec +--import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BuildLeft, BuildRight, ShuffledHashJoinExec, SortMergeJoinExec} +-+import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, ShuffledHashJoinExec, SortMergeJoinExec} +- import org.apache.spark.sql.types.{CalendarIntervalType, DataType, DataTypes, StringType} +- +- trait ConfKeysAndIncompat { +-diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala +-index ac444d1..14a8c6e 100644 +---- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala +-+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala +-@@ -22,6 +22,7 @@ import com.nvidia.spark.rapids.GpuMetricNames._ +- import org.apache.spark.rdd.RDD +- import org.apache.spark.sql.catalyst.InternalRow +- import org.apache.spark.sql.catalyst.expressions.Expression +-+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} +- import org.apache.spark.sql.catalyst.plans.JoinType +- import org.apache.spark.sql.catalyst.plans.physical.{BroadcastDistribution, Distribution, UnspecifiedDistribution} +- import org.apache.spark.sql.execution.{BinaryExecNode, SparkPlan} diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashJoin.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashJoin.scala -index e6c3e37..ddd8ca4 100644 +index 94b42dc..cf27f4b 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashJoin.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashJoin.scala @@ -19,8 +19,9 @@ import ai.rapids.cudf.{NvtxColor, Table} @@ -7,7 +90,7 @@ index e6c3e37..ddd8ca4 100644 import org.apache.spark.TaskContext import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} +import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight} - import org.apache.spark.sql.catalyst.plans.{ExistenceJoin, FullOuter, Inner, InnerLike, JoinType, LeftAnti, LeftExistence, LeftOuter, LeftSemi, RightOuter} + import org.apache.spark.sql.catalyst.plans.{ExistenceJoin, FullOuter, InnerLike, JoinType, LeftAnti, LeftExistence, LeftOuter, LeftSemi, RightOuter} -import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight, HashJoin} +import org.apache.spark.sql.execution.joins.HashJoin import org.apache.spark.sql.execution.metric.SQLMetric @@ -46,7 +129,7 @@ index 29ba63d..78febd4 100644 class GpuSortMergeJoinMeta( join: SortMergeJoinExec, diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala -index b02182a..1ed13d2 100644 +index 834ec51..646ccda 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala @@ -22,12 +22,13 @@ import com.nvidia.spark.rapids.GpuOverrides.isStringLit From 62a5b82e76bbd1ceec96604d3537b831490d6e11 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Tue, 30 Jun 2020 13:07:48 -0500 Subject: [PATCH 2/2] remove dbimports from patch --- jenkins/databricks/dbimports.patch | 83 ------------------------------ 1 file changed, 83 deletions(-) diff --git a/jenkins/databricks/dbimports.patch b/jenkins/databricks/dbimports.patch index 316daaab2bf..086093a090e 100644 --- a/jenkins/databricks/dbimports.patch +++ b/jenkins/databricks/dbimports.patch @@ -1,86 +1,3 @@ -diff --git a/jenkins/databricks/dbimports.patch b/jenkins/databricks/dbimports.patch -index d112e85..e69de29 100644 ---- a/jenkins/databricks/dbimports.patch -+++ b/jenkins/databricks/dbimports.patch -@@ -1,78 +0,0 @@ --diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashJoin.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashJoin.scala --index e6c3e37..ddd8ca4 100644 ----- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashJoin.scala --+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashJoin.scala --@@ -19,8 +19,9 @@ import ai.rapids.cudf.{NvtxColor, Table} -- -- import org.apache.spark.TaskContext -- import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} --+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight} -- import org.apache.spark.sql.catalyst.plans.{ExistenceJoin, FullOuter, Inner, InnerLike, JoinType, LeftAnti, LeftExistence, LeftOuter, LeftSemi, RightOuter} ---import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight, HashJoin} --+import org.apache.spark.sql.execution.joins.HashJoin -- import org.apache.spark.sql.execution.metric.SQLMetric -- import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} -- --diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuShuffledHashJoinExec.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuShuffledHashJoinExec.scala --index 7ae310b..3ebde77 100644 ----- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuShuffledHashJoinExec.scala --+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuShuffledHashJoinExec.scala --@@ -22,10 +22,11 @@ import org.apache.spark.TaskContext -- import org.apache.spark.rdd.RDD -- import org.apache.spark.sql.catalyst.InternalRow -- import org.apache.spark.sql.catalyst.expressions.Expression --+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} -- import org.apache.spark.sql.catalyst.plans.JoinType -- import org.apache.spark.sql.catalyst.plans.physical.{Distribution, HashClusteredDistribution} -- import org.apache.spark.sql.execution.{BinaryExecNode, SparkPlan} ---import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight, BuildSide, ShuffledHashJoinExec} --+import org.apache.spark.sql.execution.joins.ShuffledHashJoinExec -- import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} -- import org.apache.spark.sql.vectorized.ColumnarBatch -- --diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuSortMergeJoinExec.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuSortMergeJoinExec.scala --index 29ba63d..78febd4 100644 ----- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuSortMergeJoinExec.scala --+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuSortMergeJoinExec.scala --@@ -17,8 +17,9 @@ -- package com.nvidia.spark.rapids -- -- import org.apache.spark.internal.Logging --+import org.apache.spark.sql.catalyst.optimizer.BuildRight -- import org.apache.spark.sql.execution.SortExec ---import org.apache.spark.sql.execution.joins.{BuildRight, SortMergeJoinExec} --+import org.apache.spark.sql.execution.joins.SortMergeJoinExec -- -- class GpuSortMergeJoinMeta( -- join: SortMergeJoinExec, --diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala --index b02182a..1ed13d2 100644 ----- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala --+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala --@@ -22,12 +22,13 @@ import com.nvidia.spark.rapids.GpuOverrides.isStringLit -- -- import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, ComplexTypeMergingExpression, Expression, String2TrimExpression, TernaryExpression, UnaryExpression} -- import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateFunction --+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight} -- import org.apache.spark.sql.catalyst.plans.physical.Partitioning -- import org.apache.spark.sql.connector.read.Scan -- import org.apache.spark.sql.execution.SparkPlan -- import org.apache.spark.sql.execution.command.DataWritingCommand -- import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec ---import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BuildLeft, BuildRight, ShuffledHashJoinExec, SortMergeJoinExec} --+import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, ShuffledHashJoinExec, SortMergeJoinExec} -- import org.apache.spark.sql.types.{CalendarIntervalType, DataType, DataTypes, StringType} -- -- trait ConfKeysAndIncompat { --diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala --index ac444d1..14a8c6e 100644 ----- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala --+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala --@@ -22,6 +22,7 @@ import com.nvidia.spark.rapids.GpuMetricNames._ -- import org.apache.spark.rdd.RDD -- import org.apache.spark.sql.catalyst.InternalRow -- import org.apache.spark.sql.catalyst.expressions.Expression --+import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} -- import org.apache.spark.sql.catalyst.plans.JoinType -- import org.apache.spark.sql.catalyst.plans.physical.{BroadcastDistribution, Distribution, UnspecifiedDistribution} -- import org.apache.spark.sql.execution.{BinaryExecNode, SparkPlan} diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashJoin.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashJoin.scala index 94b42dc..cf27f4b 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashJoin.scala