[REVIEW] Initial work for decimal type in Java/JNI [skip ci]

java/src/main/java/ai/rapids/cudf/DType.java

@@ -63,6 +63,8 @@ public enum DType {
   DURATION_NANOSECONDS(8, 21, "int64"),
   //DICTIONARY32(4, 22, "NO IDEA"),
 
+  DECIMAL32(4, 25, "decimal32"),
+  DECIMAL64(8, 26, "decimal64"),
   STRING(0, 23, "str"),
   LIST(0, 24, "list"),
   STRUCT(0, 27, "struct");

java/src/main/java/ai/rapids/cudf/DataType.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.rapids.cudf;
+
+import java.io.IOException;
+
+public class DataType {
+
+  DType typeId;
+  int scale;
+   public DataType(DType id) {typeId = id; }
+
+   public DataType(DType id,  int fp_scale) {
+     typeId =id;
+     scale = fp_scale;
+   }
+  private static native long makeNativeType(int typeId, int scale);
+
+  public class NativeDataType implements AutoCloseable {
+    long nativeType;
+    // build native data_type in constructors
+    public NativeDataType(DType id) {
+      this.nativeType = DataType.makeNativeType(id.nativeId, 0);
+    }
+    public NativeDataType(DType id, int decimalScale) {
+      this.nativeType = DataType.makeNativeType(id.nativeId, decimalScale);;
+    }
+    public long returnNativeId() {
+      return this.nativeType;
+    }
+
+    public void close() throws IOException {
+      // call native delete method to clean up native data_type
+      //deleteNativeType
+
+    }
+  }
+
+}

java/src/main/java/ai/rapids/cudf/HostColumnVector.java

@@ -50,6 +50,11 @@ public final class HostColumnVector extends HostColumnVectorCore {
     this(type, rows, nullCount, hostDataBuffer, hostValidityBuffer, null);
   }
 
+  HostColumnVector(ai.rapids.cudf.DataType type, long rows, Optional<Long> nullCount,
+                   HostMemoryBuffer hostDataBuffer, HostMemoryBuffer hostValidityBuffer) {
+    this(type, rows, nullCount, hostDataBuffer, hostValidityBuffer, null);
+  }
+
   /**
    * Create a new column vector with data populated on the host.
    * @param type               the type of the vector
@@ -89,6 +94,21 @@ public final class HostColumnVector extends HostColumnVectorCore {
     incRefCountInternal(true);
   }
 
+  HostColumnVector(ai.rapids.cudf.DataType type, long rows, Optional<Long> nullCount,
+                   HostMemoryBuffer hostDataBuffer, HostMemoryBuffer hostValidityBuffer,
+                   HostMemoryBuffer offsetBuffer) {
+    super(type, rows, nullCount, hostDataBuffer, hostValidityBuffer, offsetBuffer, new ArrayList<>());
+    assert type.typeId != DType.LIST : "This constructor should not be used for list type";
+    if (nullCount.isPresent() && nullCount.get() > 0 && hostValidityBuffer == null) {
+      throw new IllegalStateException("Buffer cannot have a nullCount without a validity buffer");
+    }
+    if (type.typeId != DType.STRING && type.typeId != DType.LIST) {
+      assert offsetBuffer == null : "offsets are only supported for STRING and LIST";
+    }
+    refCount = 0;
+    incRefCountInternal(true);
+  }
+
   /**
    * This is a really ugly API, but it is possible that the lifecycle of a column of
    * data may not have a clear lifecycle thanks to java and GC. This API informs the leak
@@ -384,6 +404,26 @@ public static HostColumnVector fromLongs(long... values) {
     return build(DType.INT64, values.length, (b) -> b.appendArray(values));
   }
 
+
+ /* public static HostColumnVector fromDecimal32(int... values) {
+        return build(DType.DECIMAL32, values.length, (b) -> b.appendArray(values));
+     }
+
+         public static HostColumnVector fromDecimal64(long... values) {
+        return build(DType.DECIMAL64, values.length, (b) -> b.appendArray(values));
+     }
+
+
+        public static HostColumnVector fromBoxedDecimals32(Integer... values) {
+    return build(DType.DECIMAL32, values.length, (b) -> b.appendBoxed(values));
+      }
+
+         public static HostColumnVector fromBoxedDecimals64(Long... values) {
+       return build(DType.DECIMAL64, values.length, (b) -> b.appendBoxed(values));
+  }*/
+
+
+
   /**
    * Create a new vector from the given values.
    * <p>

java/src/main/java/ai/rapids/cudf/HostColumnVectorCore.java

@@ -38,10 +38,12 @@ public class HostColumnVectorCore implements ColumnViewAccess<HostMemoryBuffer>
 
   protected final OffHeapState offHeap;
   protected final DType type;
+  protected long scale;
   protected long rows;
   protected Optional<Long> nullCount;
   protected List<HostColumnVectorCore> children;
 
+
   public HostColumnVectorCore(DType type, long rows,
                               Optional<Long> nullCount, HostMemoryBuffer data, HostMemoryBuffer validity,
                               HostMemoryBuffer offsets, List<HostColumnVectorCore> nestedChildren) {
@@ -53,6 +55,18 @@ public HostColumnVectorCore(DType type, long rows,
     this.children = nestedChildren;
   }
 
+  public HostColumnVectorCore(DataType type, long rows,
+                              Optional<Long> nullCount, HostMemoryBuffer data, HostMemoryBuffer validity,
+                              HostMemoryBuffer offsets, List<HostColumnVectorCore> nestedChildren) {
+    this.offHeap = new OffHeapState(data, validity,  offsets);
+    MemoryCleaner.register(this, offHeap);
+    this.type = type.typeId;
+    this.scale = type.scale;
+    this.rows = rows;
+    this.nullCount = nullCount;
+    this.children = nestedChildren;
+  }
+
   /**
    * Returns the type of this vector.
    */

java/src/main/native/src/ColumnVectorJni.cpp

@@ -1283,15 +1283,15 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnVector_hash(JNIEnv *env,
 ////////
 
 JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnVector_makeCudfColumnView(
-    JNIEnv *env, jobject j_object, jint j_type, jlong j_data, jlong j_data_size, jlong j_offset,
+    JNIEnv *env, jobject j_object, jint j_type, jint scale, jlong j_data, jlong j_data_size, jlong j_offset,
     jlong j_valid, jint j_null_count, jint size, jlongArray j_children) {
 
   JNI_ARG_CHECK(env, (size != 0), "size is 0", 0);
   try {
     using cudf::column_view;
     cudf::jni::auto_set_device(env);
     cudf::type_id n_type = static_cast<cudf::type_id>(j_type);
-    cudf::data_type n_data_type(n_type);
+    cudf::data_type n_data_type(n_type, scale);
 
     std::unique_ptr<cudf::column_view> ret;
     void *data = reinterpret_cast<void *>(j_data);
@@ -1309,25 +1309,25 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnVector_makeCudfColumnView(
       // data is the second child
 
       cudf::size_type *offsets = reinterpret_cast<cudf::size_type *>(j_offset);
-      cudf::column_view offsets_column(cudf::data_type{cudf::type_id::INT32}, size + 1, offsets);
-      cudf::column_view data_column(cudf::data_type{cudf::type_id::INT8}, j_data_size, data);
-      ret.reset(new cudf::column_view(cudf::data_type{cudf::type_id::STRING}, size, nullptr, valid,
+      cudf::column_view offsets_column(cudf::data_type{cudf::type_id::INT32, scale}, size + 1, offsets);
+      cudf::column_view data_column(cudf::data_type{cudf::type_id::INT8, scale}, j_data_size, data);
+      ret.reset(new cudf::column_view(cudf::data_type{cudf::type_id::STRING, scale}, size, nullptr, valid,
                                       j_null_count, 0, {offsets_column, data_column}));
     } else if (n_type == cudf::type_id::LIST) {
       JNI_NULL_CHECK(env, j_offset, "offset is null", 0);
       cudf::jni::native_jpointerArray<cudf::column_view> children(env, j_children);
       JNI_ARG_CHECK(env, (children.size() != 0), "LIST children size is 0", 0);
       cudf::size_type *offsets = reinterpret_cast<cudf::size_type *>(j_offset);
-      cudf::column_view offsets_column(cudf::data_type{cudf::type_id::INT32}, size + 1, offsets);
-      ret.reset(new cudf::column_view(cudf::data_type{cudf::type_id::LIST}, size, nullptr, valid,
+      cudf::column_view offsets_column(cudf::data_type{cudf::type_id::INT32, scale}, size + 1, offsets);
+      ret.reset(new cudf::column_view(cudf::data_type{cudf::type_id::LIST, scale}, size, nullptr, valid,
         j_null_count, 0, {offsets_column, *children[0]}));
    } else if (n_type == cudf::type_id::STRUCT) {
      cudf::jni::native_jpointerArray<cudf::column_view> children(env, j_children);
      std::vector<column_view> children_vector(children.size());
      for (int i = 0; i < children.size(); i++) {
        children_vector[i] = *children[i];
      }
-     ret.reset(new cudf::column_view(cudf::data_type{cudf::type_id::STRUCT}, size, nullptr, valid,
+     ret.reset(new cudf::column_view(cudf::data_type{cudf::type_id::STRUCT, scale}, size, nullptr, valid,
        j_null_count, 0, children_vector));
    } else {
      ret.reset(new cudf::column_view(n_data_type, size, data, valid, j_null_count));
@@ -1350,6 +1350,18 @@ JNIEXPORT jint JNICALL Java_ai_rapids_cudf_ColumnVector_getNativeTypeId(JNIEnv *
   CATCH_STD(env, 0);
 }
 
+JNIEXPORT jint JNICALL Java_ai_rapids_cudf_ColumnVector_getNativeTypeScale(JNIEnv *env,
+                                                                        jobject j_object,
+                                                                        jlong handle) {
+  JNI_NULL_CHECK(env, handle, "native handle is null", 0);
+  try {
+    cudf::jni::auto_set_device(env);
+    cudf::column_view *column = reinterpret_cast<cudf::column_view *>(handle);
+    return static_cast<jint>(column->type().scale());
+  }
+  CATCH_STD(env, 0);
+}
+
 JNIEXPORT jint JNICALL Java_ai_rapids_cudf_ColumnVector_getNativeRowCount(JNIEnv *env,
                                                                           jobject j_object,
                                                                           jlong handle) {
@@ -1570,12 +1582,13 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnVector_getNativeColumnView(JNI
 
 JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnVector_makeEmptyCudfColumn(JNIEnv *env,
                                                                              jobject j_object,
-                                                                             jint j_type) {
+                                                                             jint j_type,
+                                                                             jint scale) {
 
   try {
     cudf::jni::auto_set_device(env);
     cudf::type_id n_type = static_cast<cudf::type_id>(j_type);
-    cudf::data_type n_data_type(n_type);
+    cudf::data_type n_data_type(n_type, scale);
     std::unique_ptr<cudf::column> column(cudf::make_empty_column(n_data_type));
     return reinterpret_cast<jlong>(column.release());
   }

java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java

@@ -704,6 +704,9 @@ void testSequenceOtherTypes() {
   @Test
   void testFromScalarZeroRows() {
     for (DType type : DType.values()) {
+      if (type == DType.DECIMAL32 || type == DType.DECIMAL64) {
+        continue;
+      }
       Scalar s = null;
       try {
         switch (type) {
@@ -794,6 +797,9 @@ void testGetNativeView() {
   void testFromScalar() {
     final int rowCount = 4;
     for (DType type : DType.values()) {
+      if(type == DType.DECIMAL32 || type == DType.DECIMAL64) {
+        continue;
+      }
       Scalar s = null;
       ColumnVector expected = null;
       ColumnVector result = null;
@@ -957,7 +963,8 @@ void testFromScalar() {
   void testFromScalarNull() {
     final int rowCount = 4;
     for (DType type : DType.values()) {
-      if (type == DType.EMPTY || type == DType.LIST || type == DType.STRUCT) {
+      if (type == DType.EMPTY || type == DType.LIST || type == DType.STRUCT
+          || type == DType.DECIMAL32 || type == DType.DECIMAL64) {
         continue;
       }
       try (Scalar s = Scalar.fromNull(type);

java/src/test/java/ai/rapids/cudf/ScalarTest.java

@@ -46,6 +46,9 @@ public void testIncRef() {
   @Test
   public void testNull() {
     for (DType type : DType.values()) {
+      if(type == DType.DECIMAL32 || type == DType.DECIMAL64) {
+        continue;
+      }
       if (!type.isNestedType()) {
         try (Scalar s = Scalar.fromNull(type)) {
           assertEquals(type, s.getType());