Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[auto-merge] branch-0.5 to branch-0.6 [skip ci] [bot] #2212

Merged
merged 1 commit into from
Apr 21, 2021
Merged

[auto-merge] branch-0.5 to branch-0.6 [skip ci] [bot] #2212

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions integration_tests/src/test/scala/com/nvidia/spark/rapids/StringFunctionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,7 @@ class StringOperatorsDiagnostics extends SparkQueryCompareTestSuite {

println("\u001b[1;36mSummary of diffs:\u001b[0m")
println("\u001b[1;36mCodepoint:\u001b[0m ")
for (i <- 0 until fromCpu.length) {
for (i <- fromCpu.indices) {
if (fromCpu(i) != fromGpu(i)) {
val codepoint = TestCodepoints.validCodepointIndices(i)
print(f"$codepoint%5d, ")
Expand All @@ -219,7 +219,7 @@ class StringOperatorsDiagnostics extends SparkQueryCompareTestSuite {
println("\u001b[1;36mDetails:")
println("Codepoint CPU GPU")
println("single -> single mappings\u001b[0m");
for (i <- 0 until fromCpu.length) {
for (i <- fromCpu.indices) {
if (fromCpu(i) != fromGpu(i) && fromCpu(i).getString(0).length == 1) {
val codepoint = TestCodepoints.validCodepointIndices(i)

Expand All @@ -230,7 +230,7 @@ class StringOperatorsDiagnostics extends SparkQueryCompareTestSuite {
}
}
println("\u001b[1;36msingle -> multi mappings\u001b[0m");
for (i <- 0 until fromCpu.length) {
for (i <- fromCpu.indices) {
if (fromCpu(i) != fromGpu(i) && fromCpu(i).getString(0).length > 1) {
var cpu_str = fromCpu(i).getString(0)
var gpu_str = fromGpu(i).getString(0)
Expand Down Expand Up @@ -264,7 +264,7 @@ class StringOperatorsDiagnostics extends SparkQueryCompareTestSuite {

// upper results
val (fromCpuUpper, fromGpuUpper) = generateResults(upper)
for (i <- 0 until fromCpuUpper.length) {
for (i <- fromCpuUpper.indices) {
if (fromCpuUpper(i) != fromGpuUpper(i) && fromGpuUpper(i).getString(0).length == 1) {
val codepoint = TestCodepoints.validCodepointIndices(i)

Expand All @@ -276,7 +276,7 @@ class StringOperatorsDiagnostics extends SparkQueryCompareTestSuite {

// lower results
val (fromCpuLower, fromGpuLower) = generateResults(lower)
for (i <- 0 until fromCpuLower.length) {
for (i <- fromCpuLower.indices) {
if (fromCpuLower(i) != fromGpuLower(i) && fromGpuLower(i).getString(0).length == 1) {
val codepoint = TestCodepoints.validCodepointIndices(i)

Expand Down
9 changes: 4 additions & 5 deletions ...k300/src/main/scala/com/nvidia/spark/rapids/shims/spark300/GpuBroadcastHashJoinExec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,10 @@ class GpuBroadcastHashJoinMeta(

override def tagPlanForGpu(): Unit = {
GpuHashJoin.tagJoin(this, join.joinType, join.leftKeys, join.rightKeys, join.condition)

val Seq(leftChild, rightChild) = childPlans
val buildSide = join.buildSide match {
case BuildLeft => childPlans(0)
case BuildRight => childPlans(1)
case BuildLeft => leftChild
case BuildRight => rightChild
}

if (!buildSide.canThisBeReplaced) {
Expand All @@ -67,8 +67,7 @@ class GpuBroadcastHashJoinMeta(
}

override def convertToGpu(): GpuExec = {
val left = childPlans(0).convertIfNeeded()
val right = childPlans(1).convertIfNeeded()
val Seq(left, right) = childPlans.map(_.convertIfNeeded())
// The broadcast part of this must be a BroadcastExchangeExec
val buildSide = join.buildSide match {
case BuildLeft => left
Expand Down
8 changes: 5 additions & 3 deletions ...rk300/src/main/scala/com/nvidia/spark/rapids/shims/spark300/GpuShuffledHashJoinExec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,16 +56,18 @@ class GpuShuffledHashJoinMeta(
GpuHashJoin.tagJoin(this, join.joinType, join.leftKeys, join.rightKeys, join.condition)
}

override def convertToGpu(): GpuExec =
override def convertToGpu(): GpuExec = {
val Seq(left, right) = childPlans.map(_.convertIfNeeded)
GpuShuffledHashJoinExec(
leftKeys.map(_.convertToGpu()),
rightKeys.map(_.convertToGpu()),
join.joinType,
GpuJoinUtils.getGpuBuildSide(join.buildSide),
condition.map(_.convertToGpu()),
childPlans(0).convertIfNeeded(),
childPlans(1).convertIfNeeded(),
left,
right,
isSkewJoin = false)
}
}

case class GpuShuffledHashJoinExec(
Expand Down
5 changes: 3 additions & 2 deletions ...spark300/src/main/scala/com/nvidia/spark/rapids/shims/spark300/GpuSortMergeJoinExec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,14 +77,15 @@ class GpuSortMergeJoinMeta(
} else {
throw new IllegalStateException(s"Cannot build either side for ${join.joinType} join")
}
val Seq(left, right) = childPlans.map(_.convertIfNeeded())
GpuShuffledHashJoinExec(
leftKeys.map(_.convertToGpu()),
rightKeys.map(_.convertToGpu()),
join.joinType,
GpuJoinUtils.getGpuBuildSide(buildSide),
condition.map(_.convertToGpu()),
childPlans(0).convertIfNeeded(),
childPlans(1).convertIfNeeded(),
left,
right,
join.isSkewJoin)
}

Expand Down
9 changes: 4 additions & 5 deletions ...k301/src/main/scala/com/nvidia/spark/rapids/shims/spark301/GpuBroadcastHashJoinExec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,10 @@ class GpuBroadcastHashJoinMeta(

override def tagPlanForGpu(): Unit = {
GpuHashJoin.tagJoin(this, join.joinType, join.leftKeys, join.rightKeys, join.condition)

val Seq(leftChild, rightChild) = childPlans
val buildSide = join.buildSide match {
case BuildLeft => childPlans(0)
case BuildRight => childPlans(1)
case BuildLeft => leftChild
case BuildRight => rightChild
}

if (!canBuildSideBeReplaced(buildSide)) {
Expand All @@ -65,8 +65,7 @@ class GpuBroadcastHashJoinMeta(
}

override def convertToGpu(): GpuExec = {
val left = childPlans(0).convertIfNeeded()
val right = childPlans(1).convertIfNeeded()
val Seq(left, right) = childPlans.map(_.convertIfNeeded())
// The broadcast part of this must be a BroadcastExchangeExec
val buildSide = join.buildSide match {
case BuildLeft => left
Expand Down
4 changes: 2 additions & 2 deletions shims/spark301/src/main/scala/com/nvidia/spark/rapids/shims/spark301/Spark301Shims.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,15 +68,15 @@ class Spark301Shims extends Spark300Shims {
Seq(ParamCheck("input", TypeSig.commonCudfTypes + TypeSig.NULL, TypeSig.all))),
(a, conf, p, r) => new ExprMeta[First](a, conf, p, r) {
override def convertToGpu(): GpuExpression =
GpuFirst(childExprs(0).convertToGpu(), a.ignoreNulls)
GpuFirst(childExprs.head.convertToGpu(), a.ignoreNulls)
}),
GpuOverrides.expr[Last](
"last aggregate operator",
ExprChecks.aggNotWindow(TypeSig.commonCudfTypes + TypeSig.NULL, TypeSig.all,
Seq(ParamCheck("input", TypeSig.commonCudfTypes + TypeSig.NULL, TypeSig.all))),
(a, conf, p, r) => new ExprMeta[Last](a, conf, p, r) {
override def convertToGpu(): GpuExpression =
GpuLast(childExprs(0).convertToGpu(), a.ignoreNulls)
GpuLast(childExprs.head.convertToGpu(), a.ignoreNulls)
})
).map(r => (r.getClassFor.asSubclass(classOf[Expression]), r)).toMap

Expand Down
9 changes: 4 additions & 5 deletions ...db/src/main/scala/com/nvidia/spark/rapids/shims/spark301db/GpuBroadcastHashJoinExec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,10 @@ class GpuBroadcastHashJoinMeta(

override def tagPlanForGpu(): Unit = {
GpuHashJoin.tagJoin(this, join.joinType, join.leftKeys, join.rightKeys, join.condition)

val Seq(leftChild, rightChild) = childPlans
val buildSide = join.buildSide match {
case BuildLeft => childPlans(0)
case BuildRight => childPlans(1)
case BuildLeft => leftChild
case BuildRight => rightChild
}

if (!canBuildSideBeReplaced(buildSide)) {
Expand All @@ -64,8 +64,7 @@ class GpuBroadcastHashJoinMeta(
}

override def convertToGpu(): GpuExec = {
val left = childPlans(0).convertIfNeeded()
val right = childPlans(1).convertIfNeeded()
val Seq(left, right) = childPlans.map(_.convertIfNeeded())
// The broadcast part of this must be a BroadcastExchangeExec
val buildSide = join.buildSide match {
case BuildLeft => left
Expand Down
8 changes: 5 additions & 3 deletions ...1db/src/main/scala/com/nvidia/spark/rapids/shims/spark301db/GpuShuffledHashJoinExec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,17 @@ class GpuShuffledHashJoinMeta(
GpuHashJoin.tagJoin(this, join.joinType, join.leftKeys, join.rightKeys, join.condition)
}

override def convertToGpu(): GpuExec =
override def convertToGpu(): GpuExec = {
val Seq(leftChild, rightChild) = childPlans.map(_.convertIfNeeded())
GpuShuffledHashJoinExec(
leftKeys.map(_.convertToGpu()),
rightKeys.map(_.convertToGpu()),
join.joinType,
GpuJoinUtils.getGpuBuildSide(join.buildSide),
condition.map(_.convertToGpu()),
childPlans(0).convertIfNeeded(),
childPlans(1).convertIfNeeded())
leftChild,
rightChild)
}
}

case class GpuShuffledHashJoinExec(
Expand Down
5 changes: 3 additions & 2 deletions ...k301db/src/main/scala/com/nvidia/spark/rapids/shims/spark301db/GpuSortMergeJoinExec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,15 @@ class GpuSortMergeJoinMeta(
} else {
throw new IllegalStateException(s"Cannot build either side for ${join.joinType} join")
}
val Seq(leftChild, rightChild) = childPlans.map(_.convertIfNeeded())
GpuShuffledHashJoinExec(
leftKeys.map(_.convertToGpu()),
rightKeys.map(_.convertToGpu()),
join.joinType,
GpuJoinUtils.getGpuBuildSide(buildSide),
condition.map(_.convertToGpu()),
childPlans(0).convertIfNeeded(),
childPlans(1).convertIfNeeded())
leftChild,
rightChild)
}

/**
Expand Down
9 changes: 4 additions & 5 deletions ...k311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuBroadcastHashJoinExec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,10 @@ class GpuBroadcastHashJoinMeta(

override def tagPlanForGpu(): Unit = {
GpuHashJoin.tagJoin(this, join.joinType, join.leftKeys, join.rightKeys, join.condition)

val Seq(leftChild, rightChild) = childPlans
val buildSide = join.buildSide match {
case BuildLeft => childPlans(0)
case BuildRight => childPlans(1)
case BuildLeft => leftChild
case BuildRight => rightChild
}

if (!canBuildSideBeReplaced(buildSide)) {
Expand All @@ -69,8 +69,7 @@ class GpuBroadcastHashJoinMeta(
}

override def convertToGpu(): GpuExec = {
val left = childPlans(0).convertIfNeeded()
val right = childPlans(1).convertIfNeeded()
val Seq(left, right) = childPlans.map(_.convertIfNeeded())
// The broadcast part of this must be a BroadcastExchangeExec
val buildSide = join.buildSide match {
case BuildLeft => left
Expand Down
8 changes: 5 additions & 3 deletions ...rk311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuShuffledHashJoinExec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,18 @@ class GpuShuffledHashJoinMeta(
GpuHashJoin.tagJoin(this, join.joinType, join.leftKeys, join.rightKeys, join.condition)
}

override def convertToGpu(): GpuExec =
override def convertToGpu(): GpuExec = {
val Seq(left, right) = childPlans.map(_.convertIfNeeded())
GpuShuffledHashJoinExec(
leftKeys.map(_.convertToGpu()),
rightKeys.map(_.convertToGpu()),
join.joinType,
GpuJoinUtils.getGpuBuildSide(join.buildSide),
condition.map(_.convertToGpu()),
childPlans(0).convertIfNeeded(),
childPlans(1).convertIfNeeded(),
left,
right,
isSkewJoin = false)
}
}

case class GpuShuffledHashJoinExec(
Expand Down
5 changes: 3 additions & 2 deletions ...spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuSortMergeJoinExec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,15 @@ class GpuSortMergeJoinMeta(
} else {
throw new IllegalStateException(s"Cannot build either side for ${join.joinType} join")
}
val Seq(left, right) = childPlans.map(_.convertIfNeeded())
GpuShuffledHashJoinExec(
leftKeys.map(_.convertToGpu()),
rightKeys.map(_.convertToGpu()),
join.joinType,
GpuJoinUtils.getGpuBuildSide(buildSide),
condition.map(_.convertToGpu()),
childPlans(0).convertIfNeeded(),
childPlans(1).convertIfNeeded(),
left,
right,
join.isSkewJoin)
}

Expand Down
6 changes: 2 additions & 4 deletions shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/Spark311Shims.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,10 +191,8 @@ class Spark311Shims extends Spark301Shims {
}
}
override def convertToGpu(): GpuExpression = {
GpuStringReplace(
childExprs(0).convertToGpu(),
childExprs(1).convertToGpu(),
childExprs(2).convertToGpu())
val Seq(child0, child1, child2) = childExprs.map(_.convertToGpu())
GpuStringReplace(child0, child1, child2)
}
}),
// Spark 3.1.1-specific LEAD expression, using custom OffsetWindowFunctionMeta.
Expand Down
2 changes: 1 addition & 1 deletion sql-plugin/src/main/scala/com/nvidia/spark/rapids/DateUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ object DateUtils {
var sb = new StringBuilder()
var index = 0;
val patterns = new ListBuffer[FormatKeywordToReplace]
format.map(character => {
format.foreach(character => {
// We are checking to see if this char is a part of a previously read pattern
// or start of a new one.
if (sb.isEmpty || sb.last == character) {
Expand Down
31 changes: 16 additions & 15 deletions sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExpressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,46 +272,47 @@ trait GpuTernaryExpression extends TernaryExpression with GpuExpression {
def doColumnar(numRows: Int, val0: Scalar, val1: Scalar, val2: Scalar): ColumnVector

override def columnarEval(batch: ColumnarBatch): Any = {
withResourceIfAllowed(children(0).columnarEval(batch)) { val0 =>
withResourceIfAllowed(children(1).columnarEval(batch)) { val1 =>
withResourceIfAllowed(children(2).columnarEval(batch)) { val2 =>
val Seq(child0, child1, child2) = children
withResourceIfAllowed(child0.columnarEval(batch)) { val0 =>
withResourceIfAllowed(child1.columnarEval(batch)) { val1 =>
withResourceIfAllowed(child2.columnarEval(batch)) { val2 =>
(val0, val1, val2) match {
case (v0: GpuColumnVector, v1: GpuColumnVector, v2: GpuColumnVector) =>
doColumnar(v0, v1, v2)
case (v0, v1: GpuColumnVector, v2: GpuColumnVector) =>
withResource(GpuScalar.from(v0, children(0).dataType)) { scalar0 =>
withResource(GpuScalar.from(v0, child0.dataType)) { scalar0 =>
GpuColumnVector.from(doColumnar(scalar0, v1, v2), dataType)
}
case (v0: GpuColumnVector, v1, v2: GpuColumnVector) =>
withResource(GpuScalar.from(v1, children(1).dataType)) { scalar1 =>
withResource(GpuScalar.from(v1, child1.dataType)) { scalar1 =>
GpuColumnVector.from(doColumnar(v0, scalar1, v2), dataType)
}
case (v0: GpuColumnVector, v1: GpuColumnVector, v2) =>
withResource(GpuScalar.from(v2, children(2).dataType)) { scalar2 =>
withResource(GpuScalar.from(v2, child2.dataType)) { scalar2 =>
GpuColumnVector.from(doColumnar(v0, v1, scalar2), dataType)
}
case (v0, v1, v2: GpuColumnVector) =>
withResource(GpuScalar.from(v0, children(0).dataType)) { scalar0 =>
withResource(GpuScalar.from(v1, children(1).dataType)) { scalar1 =>
withResource(GpuScalar.from(v0, child0.dataType)) { scalar0 =>
withResource(GpuScalar.from(v1, child1.dataType)) { scalar1 =>
GpuColumnVector.from(doColumnar(scalar0, scalar1, v2), dataType)
}
}
case (v0, v1: GpuColumnVector, v2) =>
withResource(GpuScalar.from(v0, children(0).dataType)) { scalar0 =>
withResource(GpuScalar.from(v2, children(2).dataType)) { scalar2 =>
withResource(GpuScalar.from(v0, child0.dataType)) { scalar0 =>
withResource(GpuScalar.from(v2, child2.dataType)) { scalar2 =>
GpuColumnVector.from(doColumnar(scalar0, v1, scalar2), dataType)
}
}
case (v0: GpuColumnVector, v1, v2) =>
withResource(GpuScalar.from(v1, children(1).dataType)) { scalar1 =>
withResource(GpuScalar.from(v2, children(2).dataType)) { scalar2 =>
withResource(GpuScalar.from(v1, child1.dataType)) { scalar1 =>
withResource(GpuScalar.from(v2, child2.dataType)) { scalar2 =>
GpuColumnVector.from(doColumnar(v0, scalar1, scalar2), dataType)
}
}
case (v0, v1, v2) if v0 != null && v1 != null && v2 != null =>
withResource(GpuScalar.from(v0, children(0).dataType)) { v0Scalar =>
withResource(GpuScalar.from(v1, children(1).dataType)) { v1Scalar =>
withResource(GpuScalar.from(v2, children(2).dataType)) { v2Scalar =>
withResource(GpuScalar.from(v0, child0.dataType)) { v0Scalar =>
withResource(GpuScalar.from(v1, child1.dataType)) { v1Scalar =>
withResource(GpuScalar.from(v2, child2.dataType)) { v2Scalar =>
GpuColumnVector.from(doColumnar(batch.numRows(), v0Scalar, v1Scalar, v2Scalar),
dataType)
}
Expand Down
Loading