Add native OpenX JSON format

trinodb · Mar 14, 2023 · a297801 · a297801
1 parent 0b2e189
commit a297801
Show file tree

Hide file tree

Showing 27 changed files with 4,753 additions and 7 deletions.
diff --git a/lib/trino-hive-formats/pom.xml b/lib/trino-hive-formats/pom.xml
@@ -125,12 +125,43 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>io.starburst.openjson</groupId>
+            <artifactId>openjson</artifactId>
+            <version>1.8-e.10</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>io.starburst.openx.data</groupId>
+            <artifactId>json-serde</artifactId>
+            <version>1.3.9-e.10</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-serde</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-exec</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
         <dependency>
             <groupId>it.unimi.dsi</groupId>
             <artifactId>fastutil</artifactId>
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.12.0</version>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.assertj</groupId>
             <artifactId>assertj-core</artifactId>

diff --git a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/HiveFormatUtils.java b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/HiveFormatUtils.java
@@ -89,6 +89,7 @@ public final class HiveFormatUtils
             .toFormatter();
 
     private static final DateTimeFormatter DEFAULT_TIMESTAMP_PARSER = new DateTimeFormatterBuilder()
+            .parseCaseInsensitive()
             .parseLenient()
             // Date part
             .appendValue(YEAR, 1, 10, SignStyle.NORMAL)
@@ -97,7 +98,8 @@ public final class HiveFormatUtils
             .appendLiteral('-')
             .appendValue(DAY_OF_MONTH, 1, 2, SignStyle.NORMAL)
             // Time part
-            .optionalStart().appendLiteral(" ")
+            .optionalStart()
+            .appendLiteral(" ")
             .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NORMAL)
             .appendLiteral(':')
             .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NORMAL)
@@ -108,6 +110,13 @@ public final class HiveFormatUtils
             .toFormatter()
             .withResolverStyle(LENIENT);
 
+    private static final DateTimeFormatter ISO_TIMESTAMP_PARSER = new DateTimeFormatterBuilder()
+            .parseCaseInsensitive()
+            .parseLenient()
+            .append(DateTimeFormatter.ISO_LOCAL_DATE_TIME)
+            .toFormatter()
+            .withResolverStyle(LENIENT);
+
     private static final DateTimeFormatter TIMESTAMP_FORMATTER = new DateTimeFormatterBuilder()
             // Date part
             .appendValue(YEAR, 1, 10, SignStyle.NORMAL)
@@ -116,7 +125,8 @@ public final class HiveFormatUtils
             .appendLiteral('-')
             .appendValue(DAY_OF_MONTH, 2, 2, SignStyle.NORMAL)
             // Time part
-            .optionalStart().appendLiteral(" ")
+            .optionalStart()
+            .appendLiteral(" ")
             .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NORMAL)
             .appendLiteral(':')
             .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NORMAL)
@@ -159,13 +169,18 @@ public static void writeDecimal(String value, DecimalType decimalType, BlockBuil
     public static BigDecimal parseDecimal(String value, DecimalType decimalType)
     {
         try {
-            return new BigDecimal(value).setScale(DecimalConversions.intScale(decimalType.getScale()), HALF_UP);
+            return scaleDecimal(new BigDecimal(value), decimalType);
         }
         catch (NumberFormatException e) {
             throw new NumberFormatException(format("Cannot convert '%s' to %s. Value is not a number.", value, decimalType));
         }
     }
 
+    public static BigDecimal scaleDecimal(BigDecimal bigDecimal, DecimalType decimalType)
+    {
+        return bigDecimal.setScale(DecimalConversions.intScale(decimalType.getScale()), HALF_UP);
+    }
+
     public static Function<String, DecodedTimestamp> createTimestampParser(List<String> timestampFormats)
     {
         requireNonNull(timestampFormats, "timestampFormats is null");
@@ -229,7 +244,7 @@ public static DecodedTimestamp parseHiveTimestamp(String value)
         }
         catch (DateTimeParseException e) {
             // Try ISO-8601 format
-            localDateTime = LocalDateTime.parse(value);
+            localDateTime = LocalDateTime.parse(value, ISO_TIMESTAMP_PARSER);
         }
         return new DecodedTimestamp(localDateTime.toEpochSecond(ZoneOffset.UTC), localDateTime.getNano());
     }

diff --git a/...hive-formats/src/main/java/io/trino/hive/formats/line/openxjson/InvalidJsonException.java b/...hive-formats/src/main/java/io/trino/hive/formats/line/openxjson/InvalidJsonException.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.hive.formats.line.openxjson;
+
+import java.io.IOException;
+
+class InvalidJsonException
+        extends IOException
+{
+    public InvalidJsonException(String message, int position, String json)
+    {
+        super("%s at character %d of JSON: %s".formatted(message, position, json));
+    }
+}