Skip to content

Commit

Permalink
Address review comments
Browse files Browse the repository at this point in the history
  • Loading branch information
abellina committed Jan 18, 2023
1 parent 88fa116 commit bf06e88
Show file tree
Hide file tree
Showing 7 changed files with 18 additions and 21 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ class RapidsBufferCatalog extends AutoCloseable with Arm {
override val id: RapidsBufferId,
var priority: Long,
spillCallback: SpillCallback)
extends RapidsBufferHandle with Arm {
extends RapidsBufferHandle {

override def setSpillPriority(newPriority: Long): Unit = {
priority = newPriority
Expand Down Expand Up @@ -142,7 +142,9 @@ class RapidsBufferCatalog extends AutoCloseable with Arm {
* returns true, otherwise it returns false.
*
* @param handle handle to stop tracking
* @return
* @return true: if this was the last `RapidsBufferHandle` associated with the
* underlying buffer.
* false: if there are remaining live handles
*/
private def stopTrackingHandle(handle: RapidsBufferHandle): Boolean = {
withResource(acquireBuffer(handle)) { buffer =>
Expand All @@ -159,9 +161,7 @@ class RapidsBufferCatalog extends AutoCloseable with Arm {
null
} else {
val newHandles = handles.filter(h => h != handle).map { h =>
if (h.getSpillPriority > maxPriority) {
maxPriority = h.getSpillPriority
}
maxPriority = maxPriority.max(h.getSpillPriority)
h
}
if (newHandles.isEmpty) {
Expand Down Expand Up @@ -308,6 +308,10 @@ class RapidsBufferCatalog extends AutoCloseable with Arm {
/**
* Remove a buffer handle from the catalog and, if it this was the final handle,
* release the resources of the registered buffers.
*
* @return true: if the buffer for this handle was removed from the spill framework
* (`handle` was the last handle)
* false: if buffer was not removed due to other live handles.
*/
def removeBuffer(handle: RapidsBufferHandle): Boolean = {
// if this is the last handle, remove the buffer
Expand All @@ -324,7 +328,7 @@ class RapidsBufferCatalog extends AutoCloseable with Arm {
def numBuffers: Int = bufferMap.size()

override def close(): Unit = {
bufferIdToHandles.forEach { case (_, handles) =>
bufferIdToHandles.values.forEach { handles =>
handles.foreach(removeBuffer)
}
bufferIdToHandles.clear()
Expand Down Expand Up @@ -396,9 +400,7 @@ object RapidsBufferCatalog extends Logging with Arm {
}

private def closeImpl(): Unit = {
if (singleton != null) {
singleton.close()
}
singleton.close()

if (memoryEventHandler != null) {
// Workaround for shutdown ordering problems where device buffers allocated with this handler
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2020-2022, NVIDIA CORPORATION.
* Copyright (c) 2020-2023, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,10 +194,8 @@ class ShuffleBufferCatalog(
tableMap.remove(id.tableId)
val didRemove = catalog.removeBuffer(bufferIdToHandle.get(id))
if (!didRemove) {
logWarning(s"Unable to remove from underlying storage ${id} when cleaning " +
logWarning(s"Unable to remove $id from underlying storage when cleaning " +
s"shuffle blocks.")
} else {
logWarning(s"Did remove ${id}")
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,16 +58,13 @@ class JustRowsColumnarBatch(numRows: Int, semWait: GpuMetric)
override def numRows(): Int = numRows
override def setSpillPriority(priority: Long): Unit = () // NOOP nothing to spill

private def makeJustRowsBatch(): ColumnarBatch = {
def getColumnarBatch(): ColumnarBatch = {
GpuSemaphore.acquireIfNecessary(TaskContext.get(), semWait)
new ColumnarBatch(Array.empty, numRows)
}

override def close(): Unit = () // NOOP nothing to close
override val sizeInBytes: Long = 0L

def getColumnarBatch(): ColumnarBatch = {
makeJustRowsBatch()
}
}

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019-2022, NVIDIA CORPORATION.
* Copyright (c) 2019-2023, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2020-2022, NVIDIA CORPORATION.
* Copyright (c) 2020-2023, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2020-2022, NVIDIA CORPORATION.
* Copyright (c) 2020-2023, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down

0 comments on commit bf06e88

Please sign in to comment.