From f359ec7f9beacf7b8a34a7e30cca3c35eb82f889 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Tue, 5 Apr 2022 13:57:44 -0400 Subject: [PATCH] Support nvComp 2.3 if local, otherwise use nvcomp 2.2 (#10513) This will allow us to utilize new features in nvComp 2.3 when it already has been installed locally, otherwise we fallback to building 2.2 from source. Loccal tests with CUDA 11.5 and 11.6 show no regressions Authors: - Robert Maynard (https://github.com/robertmaynard) - Nghia Truong (https://github.com/ttnghia) - Paul Taylor (https://github.com/trxcllnt) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Jason Lowe (https://github.com/jlowe) URL: https://github.com/rapidsai/cudf/pull/10513 --- cpp/cmake/thirdparty/get_nvcomp.cmake | 20 +-- .../ai/rapids/cudf/nvcomp/LZ4Compressor.java | 126 ----------------- .../rapids/cudf/nvcomp/LZ4Decompressor.java | 118 ---------------- .../java/ai/rapids/cudf/nvcomp/NvcompJni.java | 96 +------------ java/src/main/native/src/NvcompJni.cpp | 127 +----------------- .../ai/rapids/cudf/nvcomp/NvcompTest.java | 98 +------------- 6 files changed, 13 insertions(+), 572 deletions(-) delete mode 100644 java/src/main/java/ai/rapids/cudf/nvcomp/LZ4Compressor.java delete mode 100644 java/src/main/java/ai/rapids/cudf/nvcomp/LZ4Decompressor.java diff --git a/cpp/cmake/thirdparty/get_nvcomp.cmake b/cpp/cmake/thirdparty/get_nvcomp.cmake index c1765408d62..0356725548b 100644 --- a/cpp/cmake/thirdparty/get_nvcomp.cmake +++ b/cpp/cmake/thirdparty/get_nvcomp.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2021, NVIDIA CORPORATION. +# Copyright (c) 2021-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -13,14 +13,15 @@ # ============================================================================= # This function finds nvcomp and sets any additional necessary environment variables. -function(find_and_configure_nvcomp VERSION) - - # Find or install nvcomp +function(find_and_configure_nvcomp VERSION_MIN VERSION_MAX) + # Search for latest version of nvComp + rapids_find_package(nvcomp ${VERSION_MAX} QUIET) + # If latest isn't found, fall back to building oldest support from source rapids_cpm_find( - nvcomp ${VERSION} + nvcomp ${VERSION_MIN} GLOBAL_TARGETS nvcomp::nvcomp CPM_ARGS GITHUB_REPOSITORY NVIDIA/nvcomp - GIT_TAG c435afaf4ba8a8d12f379d688effcb185886cec1 + GIT_TAG v${VERSION_MIN} OPTIONS "BUILD_STATIC ON" "BUILD_TESTS OFF" "BUILD_BENCHMARKS OFF" "BUILD_EXAMPLES OFF" ) @@ -32,9 +33,8 @@ function(find_and_configure_nvcomp VERSION) if(TARGET nvcomp AND PER_THREAD_DEFAULT_STREAM) target_compile_definitions(nvcomp PRIVATE CUDA_API_PER_THREAD_DEFAULT_STREAM) endif() - endfunction() -set(CUDF_MIN_VERSION_nvCOMP 2.1.0) - -find_and_configure_nvcomp(${CUDF_MIN_VERSION_nvCOMP}) +set(CUDF_MIN_VERSION_nvCOMP 2.2.0) +set(CUDF_MAX_VERSION_nvCOMP 2.3.0) +find_and_configure_nvcomp(${CUDF_MIN_VERSION_nvCOMP} ${CUDF_MAX_VERSION_nvCOMP}) diff --git a/java/src/main/java/ai/rapids/cudf/nvcomp/LZ4Compressor.java b/java/src/main/java/ai/rapids/cudf/nvcomp/LZ4Compressor.java deleted file mode 100644 index 67a770f1346..00000000000 --- a/java/src/main/java/ai/rapids/cudf/nvcomp/LZ4Compressor.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ai.rapids.cudf.nvcomp; - -import ai.rapids.cudf.Cuda; -import ai.rapids.cudf.BaseDeviceMemoryBuffer; -import ai.rapids.cudf.DeviceMemoryBuffer; -import ai.rapids.cudf.HostMemoryBuffer; - -/** Single-buffer compressor implementing LZ4 */ -public class LZ4Compressor { - - /** LZ4 compression settings corresponding to a chunk size */ - public static final class Configuration { - private final long metadataBytes; - private final long tempBytes; - private final long maxCompressedBytes; - - Configuration(long metadataBytes, long tempBytes, long maxCompressedBytes) { - this.metadataBytes = metadataBytes; - this.tempBytes = tempBytes; - this.maxCompressedBytes = maxCompressedBytes; - } - - /** Get the size of the metadata information in bytes */ - public long getMetadataBytes() { - return metadataBytes; - } - - /** Get the size of the temporary storage in bytes needed to compress */ - public long getTempBytes() { - return tempBytes; - } - - /** Get the maximum compressed output size in bytes */ - public long getMaxCompressedBytes() { - return maxCompressedBytes; - } - } - - /** - * Get the compression configuration necessary for a particular chunk size. - * @param chunkSize size of an LZ4 chunk in bytes - * @param uncompressedSize total size of the uncompressed data - * @return compression configuration for the specified chunk size - */ - public static Configuration configure(long chunkSize, long uncompressedSize) { - long[] configs = NvcompJni.lz4CompressConfigure(chunkSize, uncompressedSize); - assert configs.length == 3; - return new Configuration(configs[0], configs[1], configs[2]); - } - - /** - * Synchronously compress a buffer with LZ4. - * @param input buffer to compress - * @param inputType type of data within the buffer - * @param chunkSize compression chunk size to use - * @param tempBuffer temporary storage space - * @param output buffer that will contain the compressed result - * @param stream CUDA stream to use - * @return size of the resulting compressed data stored to the output buffer - */ - public static long compress(BaseDeviceMemoryBuffer input, CompressionType inputType, - long chunkSize, BaseDeviceMemoryBuffer tempBuffer, - BaseDeviceMemoryBuffer output, Cuda.Stream stream) { - if (chunkSize <= 0) { - throw new IllegalArgumentException("Illegal chunk size: " + chunkSize); - } - try (DeviceMemoryBuffer devOutputSizeBuffer = DeviceMemoryBuffer.allocate(Long.BYTES); - HostMemoryBuffer hostOutputSizeBuffer = HostMemoryBuffer.allocate(Long.BYTES)) { - compressAsync(devOutputSizeBuffer, input, inputType, chunkSize, tempBuffer, output, stream); - hostOutputSizeBuffer.copyFromDeviceBuffer(devOutputSizeBuffer, stream); - return hostOutputSizeBuffer.getLong(0); - } - } - - /** - * Asynchronously compress a buffer with LZ4. The compressed size output buffer must be pinned - * memory for this operation to be truly asynchronous. Note that the caller must synchronize - * on the specified CUDA stream in order to safely examine the compressed output size! - * @param compressedSizeOutputBuffer device memory where the compressed output size will be stored - * @param input buffer to compress - * @param inputType type of data within the buffer - * @param chunkSize compression chunk size to use - * @param tempBuffer temporary storage space - * @param output buffer that will contain the compressed result - * @param stream CUDA stream to use - */ - public static void compressAsync(DeviceMemoryBuffer compressedSizeOutputBuffer, - BaseDeviceMemoryBuffer input, CompressionType inputType, - long chunkSize, BaseDeviceMemoryBuffer tempBuffer, - BaseDeviceMemoryBuffer output, Cuda.Stream stream) { - if (chunkSize <= 0) { - throw new IllegalArgumentException("Illegal chunk size: " + chunkSize); - } - if (compressedSizeOutputBuffer.getLength() < 8) { - throw new IllegalArgumentException("compressed output size buffer must be able to hold " + - "at least 8 bytes, size is only " + compressedSizeOutputBuffer.getLength()); - } - NvcompJni.lz4CompressAsync( - compressedSizeOutputBuffer.getAddress(), - input.getAddress(), - input.getLength(), - inputType.nativeId, - chunkSize, - tempBuffer.getAddress(), - tempBuffer.getLength(), - output.getAddress(), - output.getLength(), - stream.getStream()); - } -} diff --git a/java/src/main/java/ai/rapids/cudf/nvcomp/LZ4Decompressor.java b/java/src/main/java/ai/rapids/cudf/nvcomp/LZ4Decompressor.java deleted file mode 100644 index 46b3127581b..00000000000 --- a/java/src/main/java/ai/rapids/cudf/nvcomp/LZ4Decompressor.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ai.rapids.cudf.nvcomp; - -import ai.rapids.cudf.BaseDeviceMemoryBuffer; -import ai.rapids.cudf.Cuda; - -/** Single-buffer decompression using LZ4 */ -public class LZ4Decompressor { - - /** - * LZ4 decompression settings corresponding to an LZ4 compressed input. - * NOTE: Each instance must be closed to avoid a native memory leak. - */ - public static final class Configuration implements AutoCloseable { - private final long metadataPtr; - private final long metadataSize; - private final long tempBytes; - private final long uncompressedBytes; - - Configuration(long metadataPtr, long metadataSize, long tempBytes, - long uncompressedBytes) { - this.metadataPtr = metadataPtr; - this.metadataSize = metadataSize; - this.tempBytes = tempBytes; - this.uncompressedBytes = uncompressedBytes; - } - - /** Get the host address of the metadata */ - public long getMetadataPtr() { - return metadataPtr; - } - - /** Get the size of the metadata in bytes */ - public long getMetadataSize() { - return metadataSize; - } - - /** Get the size of the temporary buffer in bytes needed to decompress */ - public long getTempBytes() { - return tempBytes; - } - - /** Get the size of the uncompressed data in bytes */ - public long getUncompressedBytes() { - return uncompressedBytes; - } - - @Override - public void close() { - NvcompJni.lz4DestroyMetadata(metadataPtr); - } - } - - /** - * Determine if a buffer is data compressed with LZ4. - * @param buffer data to examine - * @param stream CUDA stream to use - * @return true if the data is LZ4 compressed - */ - public static boolean isLZ4Data(BaseDeviceMemoryBuffer buffer, Cuda.Stream stream) { - return NvcompJni.isLZ4Data(buffer.getAddress(), buffer.getLength(), stream.getStream()); - } - - /** - * Get the decompression configuration from compressed data. - * NOTE: The resulting configuration object must be closed to avoid a native memory leak. - * @param compressed data that has been compressed by the LZ4 compressor - * @param stream CUDA stream to use - * @return decompression configuration for the specified input - */ - public static Configuration configure(BaseDeviceMemoryBuffer compressed, Cuda.Stream stream) { - long[] configs = NvcompJni.lz4DecompressConfigure(compressed.getAddress(), - compressed.getLength(), stream.getStream()); - assert configs.length == 4; - return new Configuration(configs[0], configs[1], configs[2], configs[3]); - } - - /** - * Asynchronously decompress data compressed with the LZ4 compressor. - * @param compressed buffer containing LZ4-compressed data - * @param config decompression configuration - * @param temp temporary storage buffer - * @param outputBuffer buffer that will be written with the uncompressed output - * @param stream CUDA stream to use - */ - public static void decompressAsync( - BaseDeviceMemoryBuffer compressed, - Configuration config, - BaseDeviceMemoryBuffer temp, - BaseDeviceMemoryBuffer outputBuffer, - Cuda.Stream stream) { - NvcompJni.lz4DecompressAsync( - compressed.getAddress(), - compressed.getLength(), - config.getMetadataPtr(), - config.getMetadataSize(), - temp.getAddress(), - temp.getLength(), - outputBuffer.getAddress(), - outputBuffer.getLength(), - stream.getStream()); - } -} diff --git a/java/src/main/java/ai/rapids/cudf/nvcomp/NvcompJni.java b/java/src/main/java/ai/rapids/cudf/nvcomp/NvcompJni.java index 58f8390d0eb..57094008c08 100644 --- a/java/src/main/java/ai/rapids/cudf/nvcomp/NvcompJni.java +++ b/java/src/main/java/ai/rapids/cudf/nvcomp/NvcompJni.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,100 +24,6 @@ class NvcompJni { NativeDepsLoader.loadNativeDeps(); } - /** - * Determine if data is compressed with the nvcomp LZ4 compressor. - * @param inPtr device address of the compressed data - * @param inSize size of the compressed data in bytes - * @param stream CUDA stream to use - * @return true if the data is compressed with the nvcomp LZ4 compressor - */ - static native boolean isLZ4Data(long inPtr, long inSize, long stream); - - /** - * Determine if the metadata corresponds to data compressed with the nvcomp LZ4 compressor. - * @param metadataPtr address of the metadata object - * @return true if the metadata describes data compressed with the nvcomp LZ4 compressor. - */ - static native boolean isLZ4Metadata(long metadataPtr); - - /** - * Return the LZ4 compression configuration necessary for a particular chunk size. - * @param chunkSize maximum size of an uncompressed chunk in bytes - * @param uncompressedSize total size of the uncompressed data - * @return array of three longs containing metadata size, temp storage size, - * and output buffer size - */ - static native long[] lz4CompressConfigure(long chunkSize, long uncompressedSize); - - /** - * Perform LZ4 compression asynchronously using the specified CUDA stream. - * @param compressedSizeOutputPtr host address of a 64-bit integer to update - * with the resulting compressed size of the - * data. For the operation to be truly - * asynchronous this should point to pinned - * host memory. - * @param inPtr device address of the uncompressed data - * @param inSize size of the uncompressed data in bytes - * @param inputType type of uncompressed data - * @param chunkSize size of an LZ4 chunk in bytes - * @param tempPtr device address of the temporary compression storage buffer - * @param tempSize size of the temporary storage buffer in bytes - * @param outPtr device address of the output buffer - * @param outSize size of the output buffer in bytes - * @param stream CUDA stream to use - */ - static native void lz4CompressAsync( - long compressedSizeOutputPtr, - long inPtr, - long inSize, - int inputType, - long chunkSize, - long tempPtr, - long tempSize, - long outPtr, - long outSize, - long stream); - - /** - * Return the decompression configuration for a compressed input. - * NOTE: The resulting configuration object must be closed to destroy the corresponding - * host-side metadata created by this method to avoid a native memory leak. - * @param inPtr device address of the compressed data - * @param inSize size of the compressed data - * @return array of four longs containing metadata address, metadata size, temp storage size, - * and output buffer size - */ - static native long[] lz4DecompressConfigure(long inPtr, long inSize, long stream); - - /** - * Perform LZ4 decompression asynchronously using the specified CUDA stream. - * @param inPtr device address of the uncompressed data - * @param inSize size of the uncompressed data in bytes - * @param metadataPtr host address of the metadata - * @param metadataSize size of the metadata in bytes - * @param tempPtr device address of the temporary compression storage buffer - * @param tempSize size of the temporary storage buffer in bytes - * @param outPtr device address of the output buffer - * @param outSize size of the output buffer in bytes - * @param stream CUDA stream to use - */ - static native void lz4DecompressAsync( - long inPtr, - long inSize, - long metadataPtr, - long metadataSize, - long tempPtr, - long tempSize, - long outPtr, - long outSize, - long stream); - - /** - * Destroy host-side metadata created by {@link NvcompJni#lz4DecompressConfigure(long, long, long)} - * @param metadataPtr host address of metadata - */ - static native void lz4DestroyMetadata(long metadataPtr); - /** * Get the temporary workspace size required to perform compression of entire LZ4 batch. * @param batchSize number of chunks in the batch diff --git a/java/src/main/native/src/NvcompJni.cpp b/java/src/main/native/src/NvcompJni.cpp index 533654baee1..e616b7f66be 100644 --- a/java/src/main/native/src/NvcompJni.cpp +++ b/java/src/main/native/src/NvcompJni.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -56,131 +56,6 @@ void check_nvcomp_status(JNIEnv *env, nvcompStatus_t status) { extern "C" { -JNIEXPORT jboolean JNICALL Java_ai_rapids_cudf_nvcomp_NvcompJni_isLZ4Data(JNIEnv *env, jclass, - jlong j_in_ptr, - jlong j_in_size, - jlong j_stream) { - try { - cudf::jni::auto_set_device(env); - auto in_ptr = reinterpret_cast(j_in_ptr); - auto in_size = static_cast(j_in_size); - auto stream = reinterpret_cast(j_stream); - return nvcompLZ4IsData(in_ptr, in_size, stream); - } - CATCH_STD(env, 0) -} - -JNIEXPORT jboolean JNICALL Java_ai_rapids_cudf_nvcomp_NvcompJni_isLZ4Metadata(JNIEnv *env, jclass, - jlong metadata_ptr) { - try { - cudf::jni::auto_set_device(env); - return nvcompLZ4IsMetadata(reinterpret_cast(metadata_ptr)); - } - CATCH_STD(env, 0) -} - -JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_nvcomp_NvcompJni_lz4CompressConfigure( - JNIEnv *env, jclass, jlong j_chunk_size, jlong j_uncompressed_size) { - try { - cudf::jni::auto_set_device(env); - nvcompLZ4FormatOpts opts{}; - opts.chunk_size = static_cast(j_chunk_size); - auto uncompressed_size = static_cast(j_uncompressed_size); - std::size_t metadata_bytes = 0; - std::size_t temp_bytes = 0; - std::size_t out_bytes = 0; - auto status = nvcompLZ4CompressConfigure(&opts, NVCOMP_TYPE_CHAR, uncompressed_size, - &metadata_bytes, &temp_bytes, &out_bytes); - check_nvcomp_status(env, status); - cudf::jni::native_jlongArray result(env, 3); - result[0] = static_cast(metadata_bytes); - result[1] = static_cast(temp_bytes); - result[2] = static_cast(out_bytes); - return result.get_jArray(); - } - CATCH_STD(env, 0); -} - -JNIEXPORT void JNICALL Java_ai_rapids_cudf_nvcomp_NvcompJni_lz4CompressAsync( - JNIEnv *env, jclass, jlong j_compressed_size_ptr, jlong j_in_ptr, jlong j_in_size, - jint j_input_type, jlong j_chunk_size, jlong j_temp_ptr, jlong j_temp_size, jlong j_out_ptr, - jlong j_out_size, jlong j_stream) { - try { - cudf::jni::auto_set_device(env); - auto in_ptr = reinterpret_cast(j_in_ptr); - auto in_size = static_cast(j_in_size); - auto comp_type = static_cast(j_input_type); - nvcompLZ4FormatOpts opts{}; - opts.chunk_size = static_cast(j_chunk_size); - auto temp_ptr = reinterpret_cast(j_temp_ptr); - auto temp_size = static_cast(j_temp_size); - auto out_ptr = reinterpret_cast(j_out_ptr); - auto compressed_size_ptr = reinterpret_cast(j_compressed_size_ptr); - auto stream = reinterpret_cast(j_stream); - auto status = nvcompLZ4CompressAsync(&opts, comp_type, in_ptr, in_size, temp_ptr, temp_size, - out_ptr, compressed_size_ptr, stream); - check_nvcomp_status(env, status); - } - CATCH_STD(env, ); -} - -JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_nvcomp_NvcompJni_lz4DecompressConfigure( - JNIEnv *env, jclass, jlong j_input_ptr, jlong j_input_size, jlong j_stream) { - try { - cudf::jni::auto_set_device(env); - auto compressed_ptr = reinterpret_cast(j_input_ptr); - auto compressed_bytes = static_cast(j_input_size); - void *metadata_ptr = nullptr; - std::size_t metadata_bytes = 0; - std::size_t temp_bytes = 0; - std::size_t uncompressed_bytes = 0; - auto stream = reinterpret_cast(j_stream); - auto status = - nvcompLZ4DecompressConfigure(compressed_ptr, compressed_bytes, &metadata_ptr, - &metadata_bytes, &temp_bytes, &uncompressed_bytes, stream); - check_nvcomp_status(env, status); - cudf::jni::native_jlongArray result(env, 4); - result[0] = reinterpret_cast(metadata_ptr); - result[1] = static_cast(metadata_bytes); - result[2] = static_cast(temp_bytes); - result[3] = static_cast(uncompressed_bytes); - return result.get_jArray(); - } - CATCH_STD(env, 0); -} - -JNIEXPORT void JNICALL Java_ai_rapids_cudf_nvcomp_NvcompJni_lz4DecompressAsync( - JNIEnv *env, jclass, jlong j_in_ptr, jlong j_in_size, jlong j_metadata_ptr, - jlong j_metadata_size, jlong j_temp_ptr, jlong j_temp_size, jlong j_out_ptr, jlong j_out_size, - jlong j_stream) { - try { - cudf::jni::auto_set_device(env); - auto compressed_ptr = reinterpret_cast(j_in_ptr); - auto compressed_bytes = static_cast(j_in_size); - auto metadata_ptr = reinterpret_cast(j_metadata_ptr); - auto metadata_bytes = static_cast(j_metadata_size); - auto temp_ptr = reinterpret_cast(j_temp_ptr); - auto temp_bytes = static_cast(j_temp_size); - auto uncompressed_ptr = reinterpret_cast(j_out_ptr); - auto uncompressed_bytes = static_cast(j_out_size); - auto stream = reinterpret_cast(j_stream); - auto status = nvcompLZ4DecompressAsync(compressed_ptr, compressed_bytes, metadata_ptr, - metadata_bytes, temp_ptr, temp_bytes, uncompressed_ptr, - uncompressed_bytes, stream); - check_nvcomp_status(env, status); - } - CATCH_STD(env, ); -} - -JNIEXPORT void JNICALL Java_ai_rapids_cudf_nvcomp_NvcompJni_lz4DestroyMetadata(JNIEnv *env, jclass, - jlong metadata_ptr) { - try { - cudf::jni::auto_set_device(env); - nvcompLZ4DestroyMetadata(reinterpret_cast(metadata_ptr)); - } - CATCH_STD(env, ); -} - JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_nvcomp_NvcompJni_batchedLZ4CompressGetTempSize( JNIEnv *env, jclass, jlong j_batch_size, jlong j_max_chunk_size) { try { diff --git a/java/src/test/java/ai/rapids/cudf/nvcomp/NvcompTest.java b/java/src/test/java/ai/rapids/cudf/nvcomp/NvcompTest.java index c36d241500a..ec14a1cfee6 100644 --- a/java/src/test/java/ai/rapids/cudf/nvcomp/NvcompTest.java +++ b/java/src/test/java/ai/rapids/cudf/nvcomp/NvcompTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,16 +28,6 @@ public class NvcompTest { private static final Logger log = LoggerFactory.getLogger(ColumnVector.class); - @Test - void testLZ4RoundTripViaLZ4DecompressorSync() { - lz4RoundTrip(false); - } - - @Test - void testLZ4RoundTripViaLZ4DecompressorAsync() { - lz4RoundTrip(true); - } - @Test void testBatchedLZ4RoundTripAsync() { final Cuda.Stream stream = Cuda.DEFAULT_STREAM; @@ -134,90 +124,4 @@ private DeviceMemoryBuffer initBatchBuffer(long[] data, int bufferId) { throw new RuntimeException(t); } } - - private void lz4RoundTrip(boolean useAsync) { - final Cuda.Stream stream = Cuda.DEFAULT_STREAM; - final long chunkSize = 64 * 1024; - final int numElements = 10 * 1024 * 1024 + 1; - long[] data = new long[numElements]; - for (int i = 0; i < numElements; ++i) { - data[i] = i; - } - - DeviceMemoryBuffer tempBuffer = null; - DeviceMemoryBuffer compressedBuffer = null; - DeviceMemoryBuffer uncompressedBuffer = null; - try (ColumnVector v = ColumnVector.fromLongs(data)) { - BaseDeviceMemoryBuffer inputBuffer = v.getDeviceBufferFor(BufferType.DATA); - final long uncompressedSize = inputBuffer.getLength(); - log.debug("Uncompressed size is {}", uncompressedSize); - - LZ4Compressor.Configuration compressConf = - LZ4Compressor.configure(chunkSize, uncompressedSize); - Assertions.assertTrue(compressConf.getMetadataBytes() > 0); - log.debug("Using {} temporary space for lz4 compression", compressConf.getTempBytes()); - tempBuffer = DeviceMemoryBuffer.allocate(compressConf.getTempBytes()); - log.debug("lz4 compressed size estimate is {}", compressConf.getMaxCompressedBytes()); - - compressedBuffer = DeviceMemoryBuffer.allocate(compressConf.getMaxCompressedBytes()); - - long startTime = System.nanoTime(); - long compressedSize; - if (useAsync) { - try (DeviceMemoryBuffer devCompressedSizeBuffer = DeviceMemoryBuffer.allocate(8); - HostMemoryBuffer hostCompressedSizeBuffer = HostMemoryBuffer.allocate(8)) { - LZ4Compressor.compressAsync(devCompressedSizeBuffer, inputBuffer, CompressionType.CHAR, - chunkSize, tempBuffer, compressedBuffer, stream); - hostCompressedSizeBuffer.copyFromDeviceBufferAsync(devCompressedSizeBuffer, stream); - stream.sync(); - compressedSize = hostCompressedSizeBuffer.getLong(0); - } - } else { - compressedSize = LZ4Compressor.compress(inputBuffer, CompressionType.CHAR, chunkSize, - tempBuffer, compressedBuffer, stream); - } - double duration = (System.nanoTime() - startTime) / 1000.0; - log.info("Compressed with lz4 to {} in {} us", compressedSize, duration); - - tempBuffer.close(); - tempBuffer = null; - - try (LZ4Decompressor.Configuration decompressConf = - LZ4Decompressor.configure(compressedBuffer, stream)) { - final long tempSize = decompressConf.getTempBytes(); - - log.debug("Using {} temporary space for lz4 compression", tempSize); - tempBuffer = DeviceMemoryBuffer.allocate(tempSize); - - final long outSize = decompressConf.getUncompressedBytes(); - Assertions.assertEquals(inputBuffer.getLength(), outSize); - - uncompressedBuffer = DeviceMemoryBuffer.allocate(outSize); - - LZ4Decompressor.decompressAsync(compressedBuffer, decompressConf, tempBuffer, - uncompressedBuffer, stream); - - try (ColumnVector v2 = new ColumnVector( - DType.INT64, - numElements, - Optional.empty(), - uncompressedBuffer, - null, - null); - HostColumnVector hv2 = v2.copyToHost()) { - uncompressedBuffer = null; - for (int i = 0; i < numElements; ++i) { - long val = hv2.getLong(i); - if (val != i) { - Assertions.fail("Expected " + i + " at " + i + " found " + val); - } - } - } - } - } finally { - closeBuffer(tempBuffer); - closeBuffer(compressedBuffer); - closeBuffer(uncompressedBuffer); - } - } }