Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JNI: Rewrite growBuffersAndRows to accelerate the HostColumnBuilder #10025

Merged
merged 11 commits into from
Jan 31, 2022
24 changes: 15 additions & 9 deletions java/src/main/java/ai/rapids/cudf/ColumnView.java
Original file line number Diff line number Diff line change
Expand Up @@ -3234,7 +3234,7 @@ public final ColumnVector listIndexOf(Scalar key, FindOptions findOption) {
* The index is set to null if one of the following is true:
* 1. The search key row is null.
* 2. The list row is null.
* @param key ColumnView of search keys.
* @param keys ColumnView of search keys.
* @param findOption Whether to find the first index of the key, or the last.
* @return The resultant column of int32 indices
*/
Expand Down Expand Up @@ -3270,6 +3270,17 @@ public final Scalar getScalarElement(int index) {
return new Scalar(getType(), getElement(getNativeView(), index));
}

/**
* Get the number of bytes needed to allocate a validity buffer for the given number of rows.
* According to cudf::bitmask_allocation_size_bytes, the padding boundary for null mask is 64 bytes.
*/
static long getValidityBufferSize(int numRows) {
// number of bytes required = Math.ceil(number of bits / 8)
long actualBytes = ((long) numRows + 7) >> 3;
// padding to the multiplies of the padding boundary(64 bytes)
return ((actualBytes + 63) >> 6) << 6;
}

/////////////////////////////////////////////////////////////////////////////
// INTERNAL/NATIVE ACCESS
/////////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -3686,7 +3697,7 @@ private static native long stringReplaceWithBackrefs(long columnView, String pat
* Native method to find the first (or last) index of each search key in the specified column,
* in each row of a list column.
* @param nativeView the column view handle of the list
* @param scalarColumnHandle handle to the search key column
* @param keyColumnHandle handle to the search key column
* @param isFindFirst Whether to find the first index of the key, or the last.
* @return column handle of the resultant column of int32 indices
*/
Expand Down Expand Up @@ -3866,11 +3877,6 @@ private static native long bitwiseMergeAndSetValidity(long baseHandle, long[] vi
private static native long copyWithBooleanColumnAsValidity(long exemplarViewHandle,
long boolColumnViewHandle) throws CudfException;

/**
* Get the number of bytes needed to allocate a validity buffer for the given number of rows.
*/
static native long getNativeValidPointerSize(int size);

////////
// Native cudf::column_view life cycle and metadata access methods. Life cycle methods
// should typically only be called from the OffHeap inner class.
Expand Down Expand Up @@ -3960,7 +3966,7 @@ static ColumnVector createColumnVector(DType type, int rows, HostMemoryBuffer da
DeviceMemoryBuffer mainValidDevBuff = null;
DeviceMemoryBuffer mainOffsetsDevBuff = null;
if (mainColValid != null) {
long validLen = getNativeValidPointerSize(mainColRows);
long validLen = getValidityBufferSize(mainColRows);
mainValidDevBuff = DeviceMemoryBuffer.allocate(validLen);
mainValidDevBuff.copyFromHostBuffer(mainColValid, 0, validLen);
}
Expand Down Expand Up @@ -4069,7 +4075,7 @@ private static NestedColumnVector createNestedColumnVector(DType type, long rows
data.copyFromHostBuffer(dataBuffer, 0, dataLen);
}
if (validityBuffer != null) {
long validLen = getNativeValidPointerSize((int)rows);
long validLen = getValidityBufferSize((int)rows);
valid = DeviceMemoryBuffer.allocate(validLen);
valid.copyFromHostBuffer(validityBuffer, 0, validLen);
}
Expand Down
4 changes: 2 additions & 2 deletions java/src/main/java/ai/rapids/cudf/DType.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019-2020, NVIDIA CORPORATION.
* Copyright (c) 2019-2022, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -307,7 +307,7 @@ public static DType fromJavaBigDecimal(BigDecimal dec) {
return new DType(DTypeEnum.DECIMAL128, -dec.scale());
}
throw new IllegalArgumentException("Precision " + dec.precision() +
" exceeds max precision cuDF can support " + DECIMAL64_MAX_PRECISION);
" exceeds max precision cuDF can support " + DECIMAL128_MAX_PRECISION);
}

/**
Expand Down
Loading