Skip to content

Commit

Permalink
address java code
Browse files Browse the repository at this point in the history
  • Loading branch information
gerashegalov committed Apr 16, 2021
1 parent d589867 commit 57afdc8
Show file tree
Hide file tree
Showing 6 changed files with 14 additions and 31 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -276,13 +276,8 @@ public static final class GpuArrowColumnarBatchBuilder extends GpuColumnarBatchB
/**
* A collection of builders for building up columnar data from Arrow data.
* @param schema the schema of the batch.
* @param rows the maximum number of rows in this batch.
* @param batch if this is going to copy a ColumnarBatch in a non GPU format that batch
* we are going to copy. If not this may be null. This is used to get an idea
* of how big to allocate buffers that do not necessarily correspond to the
* number of rows.
*/
public GpuArrowColumnarBatchBuilder(StructType schema, int rows, ColumnarBatch batch) {
public GpuArrowColumnarBatchBuilder(StructType schema) {
fields = schema.fields();
int len = fields.length;
builders = new ai.rapids.cudf.ArrowColumnBuilder[len];
Expand Down Expand Up @@ -315,9 +310,9 @@ protected ColumnVector buildAndPutOnDevice(int builderIndex) {
return gcv;
}

public void copyColumnar(ColumnVector cv, int colNum, boolean nullable, int rows) {
public void copyColumnar(ColumnVector cv, int colNum, boolean ignored, int rows) {
referenceHolders[colNum].addReferences(
HostColumnarToGpu.arrowColumnarCopy(cv, builder(colNum), nullable, rows)
HostColumnarToGpu.arrowColumnarCopy(cv, builder(colNum), rows)
);
}

Expand Down Expand Up @@ -345,12 +340,8 @@ public static final class GpuColumnarBatchBuilder extends GpuColumnarBatchBuilde
* A collection of builders for building up columnar data.
* @param schema the schema of the batch.
* @param rows the maximum number of rows in this batch.
* @param batch if this is going to copy a ColumnarBatch in a non GPU format that batch
* we are going to copy. If not this may be null. This is used to get an idea
* of how big to allocate buffers that do not necessarily correspond to the
* number of rows.
*/
public GpuColumnarBatchBuilder(StructType schema, int rows, ColumnarBatch batch) {
public GpuColumnarBatchBuilder(StructType schema, int rows) {
fields = schema.fields();
int len = fields.length;
builders = new ai.rapids.cudf.HostColumnVector.ColumnBuilder[len];
Expand Down Expand Up @@ -423,7 +414,7 @@ public void close() {
}

private static final class ArrowBufReferenceHolder {
private List<ReferenceManager> references = new ArrayList<>();
private final List<ReferenceManager> references = new ArrayList<>();

public void addReferences(List<ReferenceManager> refs) {
references.addAll(refs);
Expand Down Expand Up @@ -495,7 +486,7 @@ public static DType getNonNestedRapidsType(DataType type) {
* returning an empty batch from an operator is almost always the wrong thing to do.
*/
public static ColumnarBatch emptyBatch(StructType schema) {
try (GpuColumnarBatchBuilder builder = new GpuColumnarBatchBuilder(schema, 0, null)) {
try (GpuColumnarBatchBuilder builder = new GpuColumnarBatchBuilder(schema, 0)) {
return builder.build(0);
}
}
Expand All @@ -514,7 +505,7 @@ public static ColumnarBatch emptyBatch(List<Attribute> format) {
* when serializing an empty broadcast table.
*/
public static HostColumnVector[] emptyHostColumns(StructType schema) {
try (GpuColumnarBatchBuilder builder = new GpuColumnarBatchBuilder(schema, 0, null)) {
try (GpuColumnarBatchBuilder builder = new GpuColumnarBatchBuilder(schema, 0)) {
return builder.buildHostColumns();
}
}
Expand Down Expand Up @@ -911,8 +902,8 @@ public static long getTotalDeviceMemoryUsed(ColumnarBatch batch) {
public static long getTotalDeviceMemoryUsed(GpuColumnVector[] vectors) {
long sum = 0;
HashSet<Long> found = new HashSet<>();
for (int i = 0; i < vectors.length; i++) {
ai.rapids.cudf.ColumnVector cv = vectors[i].getBase();
for (GpuColumnVector vector : vectors) {
ai.rapids.cudf.ColumnVector cv = vector.getBase();
long id = cv.getNativeView();
if (found.add(id)) {
sum += cv.getDeviceMemorySize();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2020, NVIDIA CORPORATION.
* Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -16,7 +16,6 @@

package com.nvidia.spark.rapids;

import ai.rapids.cudf.DType;
import org.apache.spark.sql.types.*;
import org.apache.spark.sql.vectorized.ColumnVector;
import org.apache.spark.sql.vectorized.ColumnarArray;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@

/*
* Copyright (c) 2020, NVIDIA CORPORATION.
* Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -78,8 +78,4 @@ public final RapidsHostColumnVector incRefCount() {
public final ai.rapids.cudf.HostColumnVector getBase() {
return cudfCv;
}

public GpuColumnVector copyToDevice() {
return new GpuColumnVector(type, cudfCv.copyToDevice());
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@

/*
* Copyright (c) 2020, NVIDIA CORPORATION.
* Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -31,9 +31,6 @@
import org.apache.spark.sql.vectorized.ColumnarMap;
import org.apache.spark.unsafe.types.UTF8String;

import java.math.BigDecimal;
import java.math.RoundingMode;

/**
* A GPU accelerated version of the Spark ColumnVector.
* Most of the standard Spark APIs should never be called, as they assume that the data
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ public ColumnarBatch next() {
devColumn = hostColumn.copyToDevice();
}
}
try (NvtxRange range = buildRange;
try (NvtxRange ignored = buildRange;
ColumnVector cv = devColumn;
Table tab = Table.convertFromRows(cv, rapidsTypes)) {
return GpuColumnVector.from(tab, outputTypes);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public Object evaluate(GenericUDF.DeferredObject[] arguments) throws HiveExcepti
if (text == null) {
return null;
}
String encoded = null;
String encoded;
try {
encoded = URLEncoder.encode(text.toString(), "utf-8")
.replace("+", "%20")
Expand Down

0 comments on commit 57afdc8

Please sign in to comment.