From 57afdc87a8c93a0916c1fe165cf200689a0892b8 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Fri, 16 Apr 2021 16:37:14 -0700 Subject: [PATCH] address java code --- .../nvidia/spark/rapids/GpuColumnVector.java | 27 +++++++------------ .../spark/rapids/GpuColumnVectorBase.java | 3 +-- .../spark/rapids/RapidsHostColumnVector.java | 6 +---- .../rapids/RapidsHostColumnVectorCore.java | 5 +--- .../UnsafeRowToColumnarBatchIterator.java | 2 +- .../spark/rapids/udf/hive/URLEncode.java | 2 +- 6 files changed, 14 insertions(+), 31 deletions(-) diff --git a/sql-plugin/src/main/java/com/nvidia/spark/rapids/GpuColumnVector.java b/sql-plugin/src/main/java/com/nvidia/spark/rapids/GpuColumnVector.java index c2cda9e5ab1..a028e5e06db 100644 --- a/sql-plugin/src/main/java/com/nvidia/spark/rapids/GpuColumnVector.java +++ b/sql-plugin/src/main/java/com/nvidia/spark/rapids/GpuColumnVector.java @@ -276,13 +276,8 @@ public static final class GpuArrowColumnarBatchBuilder extends GpuColumnarBatchB /** * A collection of builders for building up columnar data from Arrow data. * @param schema the schema of the batch. - * @param rows the maximum number of rows in this batch. - * @param batch if this is going to copy a ColumnarBatch in a non GPU format that batch - * we are going to copy. If not this may be null. This is used to get an idea - * of how big to allocate buffers that do not necessarily correspond to the - * number of rows. */ - public GpuArrowColumnarBatchBuilder(StructType schema, int rows, ColumnarBatch batch) { + public GpuArrowColumnarBatchBuilder(StructType schema) { fields = schema.fields(); int len = fields.length; builders = new ai.rapids.cudf.ArrowColumnBuilder[len]; @@ -315,9 +310,9 @@ protected ColumnVector buildAndPutOnDevice(int builderIndex) { return gcv; } - public void copyColumnar(ColumnVector cv, int colNum, boolean nullable, int rows) { + public void copyColumnar(ColumnVector cv, int colNum, boolean ignored, int rows) { referenceHolders[colNum].addReferences( - HostColumnarToGpu.arrowColumnarCopy(cv, builder(colNum), nullable, rows) + HostColumnarToGpu.arrowColumnarCopy(cv, builder(colNum), rows) ); } @@ -345,12 +340,8 @@ public static final class GpuColumnarBatchBuilder extends GpuColumnarBatchBuilde * A collection of builders for building up columnar data. * @param schema the schema of the batch. * @param rows the maximum number of rows in this batch. - * @param batch if this is going to copy a ColumnarBatch in a non GPU format that batch - * we are going to copy. If not this may be null. This is used to get an idea - * of how big to allocate buffers that do not necessarily correspond to the - * number of rows. */ - public GpuColumnarBatchBuilder(StructType schema, int rows, ColumnarBatch batch) { + public GpuColumnarBatchBuilder(StructType schema, int rows) { fields = schema.fields(); int len = fields.length; builders = new ai.rapids.cudf.HostColumnVector.ColumnBuilder[len]; @@ -423,7 +414,7 @@ public void close() { } private static final class ArrowBufReferenceHolder { - private List references = new ArrayList<>(); + private final List references = new ArrayList<>(); public void addReferences(List refs) { references.addAll(refs); @@ -495,7 +486,7 @@ public static DType getNonNestedRapidsType(DataType type) { * returning an empty batch from an operator is almost always the wrong thing to do. */ public static ColumnarBatch emptyBatch(StructType schema) { - try (GpuColumnarBatchBuilder builder = new GpuColumnarBatchBuilder(schema, 0, null)) { + try (GpuColumnarBatchBuilder builder = new GpuColumnarBatchBuilder(schema, 0)) { return builder.build(0); } } @@ -514,7 +505,7 @@ public static ColumnarBatch emptyBatch(List format) { * when serializing an empty broadcast table. */ public static HostColumnVector[] emptyHostColumns(StructType schema) { - try (GpuColumnarBatchBuilder builder = new GpuColumnarBatchBuilder(schema, 0, null)) { + try (GpuColumnarBatchBuilder builder = new GpuColumnarBatchBuilder(schema, 0)) { return builder.buildHostColumns(); } } @@ -911,8 +902,8 @@ public static long getTotalDeviceMemoryUsed(ColumnarBatch batch) { public static long getTotalDeviceMemoryUsed(GpuColumnVector[] vectors) { long sum = 0; HashSet found = new HashSet<>(); - for (int i = 0; i < vectors.length; i++) { - ai.rapids.cudf.ColumnVector cv = vectors[i].getBase(); + for (GpuColumnVector vector : vectors) { + ai.rapids.cudf.ColumnVector cv = vector.getBase(); long id = cv.getNativeView(); if (found.add(id)) { sum += cv.getDeviceMemorySize(); diff --git a/sql-plugin/src/main/java/com/nvidia/spark/rapids/GpuColumnVectorBase.java b/sql-plugin/src/main/java/com/nvidia/spark/rapids/GpuColumnVectorBase.java index 85b3e9098c1..406ffaeb7b0 100644 --- a/sql-plugin/src/main/java/com/nvidia/spark/rapids/GpuColumnVectorBase.java +++ b/sql-plugin/src/main/java/com/nvidia/spark/rapids/GpuColumnVectorBase.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ package com.nvidia.spark.rapids; -import ai.rapids.cudf.DType; import org.apache.spark.sql.types.*; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.vectorized.ColumnarArray; diff --git a/sql-plugin/src/main/java/com/nvidia/spark/rapids/RapidsHostColumnVector.java b/sql-plugin/src/main/java/com/nvidia/spark/rapids/RapidsHostColumnVector.java index 22f890ab9d0..024295a0439 100644 --- a/sql-plugin/src/main/java/com/nvidia/spark/rapids/RapidsHostColumnVector.java +++ b/sql-plugin/src/main/java/com/nvidia/spark/rapids/RapidsHostColumnVector.java @@ -1,6 +1,6 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -78,8 +78,4 @@ public final RapidsHostColumnVector incRefCount() { public final ai.rapids.cudf.HostColumnVector getBase() { return cudfCv; } - - public GpuColumnVector copyToDevice() { - return new GpuColumnVector(type, cudfCv.copyToDevice()); - } } diff --git a/sql-plugin/src/main/java/com/nvidia/spark/rapids/RapidsHostColumnVectorCore.java b/sql-plugin/src/main/java/com/nvidia/spark/rapids/RapidsHostColumnVectorCore.java index 7d119adf37a..175214b45b5 100644 --- a/sql-plugin/src/main/java/com/nvidia/spark/rapids/RapidsHostColumnVectorCore.java +++ b/sql-plugin/src/main/java/com/nvidia/spark/rapids/RapidsHostColumnVectorCore.java @@ -1,6 +1,6 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,9 +31,6 @@ import org.apache.spark.sql.vectorized.ColumnarMap; import org.apache.spark.unsafe.types.UTF8String; -import java.math.BigDecimal; -import java.math.RoundingMode; - /** * A GPU accelerated version of the Spark ColumnVector. * Most of the standard Spark APIs should never be called, as they assume that the data diff --git a/sql-plugin/src/main/java/com/nvidia/spark/rapids/UnsafeRowToColumnarBatchIterator.java b/sql-plugin/src/main/java/com/nvidia/spark/rapids/UnsafeRowToColumnarBatchIterator.java index 6e0a45eed77..4aeb963d7c2 100644 --- a/sql-plugin/src/main/java/com/nvidia/spark/rapids/UnsafeRowToColumnarBatchIterator.java +++ b/sql-plugin/src/main/java/com/nvidia/spark/rapids/UnsafeRowToColumnarBatchIterator.java @@ -145,7 +145,7 @@ public ColumnarBatch next() { devColumn = hostColumn.copyToDevice(); } } - try (NvtxRange range = buildRange; + try (NvtxRange ignored = buildRange; ColumnVector cv = devColumn; Table tab = Table.convertFromRows(cv, rapidsTypes)) { return GpuColumnVector.from(tab, outputTypes); diff --git a/udf-examples/src/main/java/com/nvidia/spark/rapids/udf/hive/URLEncode.java b/udf-examples/src/main/java/com/nvidia/spark/rapids/udf/hive/URLEncode.java index 61b05d83309..fbf302440c6 100644 --- a/udf-examples/src/main/java/com/nvidia/spark/rapids/udf/hive/URLEncode.java +++ b/udf-examples/src/main/java/com/nvidia/spark/rapids/udf/hive/URLEncode.java @@ -77,7 +77,7 @@ public Object evaluate(GenericUDF.DeferredObject[] arguments) throws HiveExcepti if (text == null) { return null; } - String encoded = null; + String encoded; try { encoded = URLEncoder.encode(text.toString(), "utf-8") .replace("+", "%20")