Skip to content

Commit

Permalink
Add Kudo & Kudo2 Shuffle. (NVIDIA#6)
Browse files Browse the repository at this point in the history
* Add kudo && kudo2 serializer (NVIDIA#5)

* cudf use allen 0822
  • Loading branch information
liurenjie1024 authored Aug 23, 2024
1 parent 00e607f commit fa00f63
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 1 deletion.
17 changes: 17 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@
<submodule.check.skip>false</submodule.check.skip>
<antrun.version>3.0.0</antrun.version>
<hilbert.version>0.2.2</hilbert.version>
<apache.common.compress.version>1.26.2</apache.common.compress.version>
</properties>

<dependencies>
Expand Down Expand Up @@ -172,6 +173,16 @@
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>${apache.common.compress.version}</version>
</dependency>
<dependency>
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>
<version>1.5.6-4</version> <!-- Use the latest version if available -->
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
Expand All @@ -184,6 +195,12 @@
<version>${hilbert.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>3.26.3</version>
<scope>test</scope>
</dependency>
</dependencies>

<profiles>
Expand Down
2 changes: 1 addition & 1 deletion thirdparty/cudf
Submodule cudf updated 39 files
+10 −3 java/pom.xml
+2 −2 java/src/main/java/ai/rapids/cudf/BitVectorHelper.java
+8 −0 java/src/main/java/ai/rapids/cudf/ColumnVector.java
+62 −2 java/src/main/java/ai/rapids/cudf/ColumnView.java
+8 −0 java/src/main/java/ai/rapids/cudf/HostColumnVector.java
+17 −1 java/src/main/java/ai/rapids/cudf/HostColumnVectorCore.java
+1 −1 java/src/main/java/ai/rapids/cudf/HostMemoryBuffer.java
+1 −1 java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java
+11 −0 java/src/main/java/ai/rapids/cudf/Table.java
+24 −0 java/src/main/java/ai/rapids/cudf/schema/SchemaVisitor.java
+17 −0 java/src/main/java/ai/rapids/cudf/schema/SchemaWithColumnsVisitor.java
+77 −0 java/src/main/java/ai/rapids/cudf/schema/Visitors.java
+978 −0 java/src/main/java/ai/rapids/cudf/serde/KudoSerializer.java
+64 −0 java/src/main/java/ai/rapids/cudf/serde/TableSerializer.java
+162 −0 java/src/main/java/ai/rapids/cudf/serde/kudo2/BufferCompressedDeserializer.java
+156 −0 java/src/main/java/ai/rapids/cudf/serde/kudo2/BufferCompressedSerializer.java
+142 −0 java/src/main/java/ai/rapids/cudf/serde/kudo2/ColumnBatchCompressedSerializer.java
+152 −0 java/src/main/java/ai/rapids/cudf/serde/kudo2/ColumnBatchedCompressedDeserializer.java
+33 −0 java/src/main/java/ai/rapids/cudf/serde/kudo2/CompressionMode.java
+65 −0 java/src/main/java/ai/rapids/cudf/serde/kudo2/DataOutputStreamWriter.java
+36 −0 java/src/main/java/ai/rapids/cudf/serde/kudo2/DataWriter.java
+243 −0 java/src/main/java/ai/rapids/cudf/serde/kudo2/Kudo2Serializer.java
+61 −0 java/src/main/java/ai/rapids/cudf/serde/kudo2/SerializedTable.java
+168 −0 java/src/main/java/ai/rapids/cudf/serde/kudo2/SerializedTableHeader.java
+108 −0 java/src/main/java/ai/rapids/cudf/serde/kudo2/SerializedTableHeaderCalc.java
+34 −0 java/src/main/java/ai/rapids/cudf/serde/kudo2/SliceInfo.java
+54 −0 java/src/main/java/ai/rapids/cudf/serde/kudo2/SlicedValidityBufferInfo.java
+41 −0 java/src/main/java/ai/rapids/cudf/utils/Arms.java
+135 −0 java/src/main/java/ai/rapids/cudf/utils/ByteBufferInputStream.java
+360 −0 java/src/main/java/ai/rapids/cudf/utils/MultiBufferInputStream.java
+159 −0 java/src/main/java/ai/rapids/cudf/utils/SingleBufferInputStream.java
+1 −1 java/src/main/native/CMakeLists.txt
+29 −0 java/src/main/native/src/ColumnVectorJni.cpp
+2 −0 java/src/test/java/ai/rapids/cudf/LargeTableTest.java
+73 −150 java/src/test/java/ai/rapids/cudf/TableTest.java
+96 −0 java/src/test/java/ai/rapids/cudf/TableTestUtils.java
+165 −0 java/src/test/java/ai/rapids/cudf/serde/Kudo2SerializerTest.java
+178 −0 java/src/test/java/ai/rapids/cudf/serde/KudoSerializerTest.java
+1 −1 python/cudf/CMakeLists.txt

0 comments on commit fa00f63

Please sign in to comment.