ORC-1489: Assign a writer id to CUDF

### What changes were proposed in this pull request? This pr is aimed at assigning a writer id to the CUDF. ### Why are the changes needed? This helps to locate the writer of a specific orc file, and it also helps the reader to do some special reads for files created by different writers. ### How was this patch tested? Added UT Closes apache#1594 from guiyanakuang/ORC-1489. Lead-authored-by: zhangyiqun <[email protected]> Co-authored-by: Yiqun Zhang <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
cxzl25 · Jan 11, 2024 · 3f29808 · 3f29808
1 parent 70cb171
commit 3f29808
Show file tree

Hide file tree

Showing 9 changed files with 23 additions and 1 deletion.
diff --git a/c++/include/orc/Common.hh b/c++/include/orc/Common.hh
@@ -70,6 +70,7 @@ namespace orc {
     PRESTO_WRITER = 2,
     SCRITCHLEY_GO = 3,
     TRINO_WRITER = 4,
+    CUDF_WRITER = 5,
     UNKNOWN_WRITER = INT32_MAX
   };
 

diff --git a/c++/src/Common.cc b/c++/src/Common.cc
@@ -82,6 +82,8 @@ namespace orc {
         return "Scritchley Go";
       case TRINO_WRITER:
         return "Trino";
+      case CUDF_WRITER:
+        return "CUDF";
       default: {
         std::ostringstream buffer;
         buffer << "Unknown(" << id << ")";

diff --git a/c++/src/Reader.cc b/c++/src/Reader.cc
@@ -622,7 +622,7 @@ namespace orc {
   WriterId ReaderImpl::getWriterId() const {
     if (footer->has_writer()) {
       uint32_t id = footer->writer();
-      if (id > WriterId::TRINO_WRITER) {
+      if (id > WriterId::CUDF_WRITER) {
         return WriterId::UNKNOWN_WRITER;
       } else {
         return static_cast<WriterId>(id);

diff --git a/java/core/src/java/org/apache/orc/OrcFile.java b/java/core/src/java/org/apache/orc/OrcFile.java
@@ -132,6 +132,7 @@ public enum WriterImplementation {
     PRESTO(2),   // Presto writer
     SCRITCHLEY_GO(3), // Go writer from https://github.com/scritchley/orc
     TRINO(4),   // Trino writer
+    CUDF(5),    // CUDF writer
     UNKNOWN(Integer.MAX_VALUE);
 
     private final int id;
@@ -189,6 +190,9 @@ public enum WriterVersion {
     // Trino Writer
     TRINO_ORIGINAL(WriterImplementation.TRINO, 6),
 
+    // CUDF Writer
+    CUDF_ORIGINAL(WriterImplementation.CUDF, 6),
+
     // Don't use any magic numbers here except for the below:
     FUTURE(WriterImplementation.UNKNOWN, Integer.MAX_VALUE); // a version from a future writer
 

diff --git a/java/core/src/java/org/apache/orc/OrcUtils.java b/java/core/src/java/org/apache/orc/OrcUtils.java
@@ -414,6 +414,9 @@ public static String getSoftwareVersion(int writer,
       case 4:
         base = "Trino";
         break;
+      case 5:
+        base = "CUDF";
+        break;
       default:
         base = String.format("Unknown(%d)", writer);
         break;

diff --git a/java/core/src/test/org/apache/orc/TestVectorOrcFile.java b/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
@@ -3624,6 +3624,8 @@ public void testWriterVersion(Version fileFormat) throws Exception {
         OrcFile.WriterImplementation.from(2));
     assertEquals(OrcFile.WriterImplementation.TRINO,
             OrcFile.WriterImplementation.from(4));
+    assertEquals(OrcFile.WriterImplementation.CUDF,
+        OrcFile.WriterImplementation.from(5));
     assertEquals(OrcFile.WriterImplementation.UNKNOWN,
         OrcFile.WriterImplementation.from(99));
 
@@ -3642,6 +3644,8 @@ public void testWriterVersion(Version fileFormat) throws Exception {
         OrcFile.WriterVersion.from(OrcFile.WriterImplementation.PRESTO, 6));
     assertEquals(OrcFile.WriterVersion.TRINO_ORIGINAL,
             OrcFile.WriterVersion.from(OrcFile.WriterImplementation.TRINO, 6));
+    assertEquals(OrcFile.WriterVersion.CUDF_ORIGINAL,
+        OrcFile.WriterVersion.from(OrcFile.WriterImplementation.CUDF, 6));
     assertEquals(OrcFile.WriterVersion.FUTURE,
         OrcFile.WriterVersion.from(OrcFile.WriterImplementation.UNKNOWN, 0));
 
@@ -3660,6 +3664,8 @@ public void testWriterVersion(Version fileFormat) throws Exception {
         OrcFile.WriterVersion.PRESTO_ORIGINAL));
     assertTrue(OrcFile.WriterVersion.HIVE_12055.includes(
         OrcFile.WriterVersion.TRINO_ORIGINAL));
+    assertTrue(OrcFile.WriterVersion.HIVE_12055.includes(
+        OrcFile.WriterVersion.CUDF_ORIGINAL));
   }
 
   @ParameterizedTest

diff --git a/proto/orc_proto.proto b/proto/orc_proto.proto
@@ -367,6 +367,7 @@ message Footer {
   // 2 = Presto
   // 3 = Scritchley Go from https://github.com/scritchley/orc
   // 4 = Trino
+  // 5 = CUDF
   optional uint32 writer = 9;
 
   // information about the encryption in this file
@@ -432,6 +433,9 @@ message PostScript {
   // Version of the Trino writer:
   //   6 = original
   //
+  // Version of the CUDF writer:
+  //   6 = original
+  //
   optional uint32 writerVersion = 6;
 
   // the number of bytes in the encrypted stripe statistics

diff --git a/site/specification/ORCv1.md b/site/specification/ORCv1.md
@@ -136,6 +136,7 @@ message Footer {
  // 2 = Presto
  // 3 = Scritchley Go from https://github.com/scritchley/orc
  // 4 = Trino
+ // 5 = CUDF
  optional uint32 writer = 9;
  // information about the encryption in this file
  optional Encryption encryption = 10;

diff --git a/site/specification/ORCv2.md b/site/specification/ORCv2.md
@@ -156,6 +156,7 @@ message Footer {
  // 2 = Presto
  // 3 = Scritchley Go from https://github.com/scritchley/orc
  // 4 = Trino
+ // 5 = CUDF
  optional uint32 writer = 9;
  // information about the encryption in this file
  optional Encryption encryption = 10;