From dbd3e0fb5237f334ffa2e9bf4ce6841030a63cc1 Mon Sep 17 00:00:00 2001 From: praveenkrishna Date: Thu, 22 Jun 2023 11:56:09 +0530 Subject: [PATCH] Treat precision as NANOSECONDS for timestamp to be coerced This will be irrespective of the precision configured or specified as session property. --- .../plugin/hive/coercions/CoercionUtils.java | 8 +- .../hive/coercions/TimestampCoercer.java | 23 ----- .../plugin/hive/orc/OrcPageSourceFactory.java | 3 +- .../plugin/hive/orc/OrcTypeTranslator.java | 13 +-- .../hive/coercions/TestTimestampCoercer.java | 86 +++++++------------ .../product/hive/BaseTestHiveCoercion.java | 51 +++++++---- .../TestHiveCoercionOnPartitionedTable.java | 7 +- .../TestHiveCoercionOnUnpartitionedTable.java | 10 ++- 8 files changed, 84 insertions(+), 117 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/CoercionUtils.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/CoercionUtils.java index abbe99be9b4e..d6e7ef6ee5ae 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/CoercionUtils.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/CoercionUtils.java @@ -61,6 +61,7 @@ import static io.trino.spi.block.ColumnarRow.toColumnarRow; import static io.trino.spi.type.DoubleType.DOUBLE; import static io.trino.spi.type.RealType.REAL; +import static io.trino.spi.type.TimestampType.TIMESTAMP_NANOS; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -135,11 +136,8 @@ public static Type createTypeFromCoercer(TypeManager typeManager, HiveType fromH if (fromType == REAL && toType instanceof DecimalType toDecimalType) { return Optional.of(createRealToDecimalCoercer(toDecimalType)); } - if (fromType instanceof TimestampType timestampType && toType instanceof VarcharType varcharType) { - if (timestampType.isShort()) { - return Optional.of(new TimestampCoercer.ShortTimestampToVarcharCoercer(timestampType, varcharType)); - } - return Optional.of(new TimestampCoercer.LongTimestampToVarcharCoercer(timestampType, varcharType)); + if (fromType instanceof TimestampType && toType instanceof VarcharType varcharType) { + return Optional.of(new TimestampCoercer.LongTimestampToVarcharCoercer(TIMESTAMP_NANOS, varcharType)); } if ((fromType instanceof ArrayType) && (toType instanceof ArrayType)) { return createCoercerForList( diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/TimestampCoercer.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/TimestampCoercer.java index 2e15793365e5..b44f5295f100 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/TimestampCoercer.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/TimestampCoercer.java @@ -48,29 +48,6 @@ public final class TimestampCoercer private TimestampCoercer() {} - public static class ShortTimestampToVarcharCoercer - extends TypeCoercer - { - public ShortTimestampToVarcharCoercer(TimestampType fromType, VarcharType toType) - { - super(fromType, toType); - } - - @Override - protected void applyCoercedValue(BlockBuilder blockBuilder, Block block, int position) - { - long epochMicros = fromType.getLong(block, position); - long epochSecond = floorDiv(epochMicros, MICROSECONDS_PER_SECOND); - int nanoFraction = floorMod(epochMicros, MICROSECONDS_PER_SECOND) * NANOSECONDS_PER_MICROSECOND; - toType.writeSlice( - blockBuilder, - truncateToLength( - Slices.utf8Slice( - LOCAL_DATE_TIME.format(LocalDateTime.ofEpochSecond(epochSecond, nanoFraction, UTC))), - toType)); - } - } - public static class LongTimestampToVarcharCoercer extends TypeCoercer { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSourceFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSourceFactory.java index 28c61abf528c..439599f87b51 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSourceFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSourceFactory.java @@ -89,7 +89,6 @@ import static io.trino.plugin.hive.HiveSessionProperties.getOrcMaxReadBlockSize; import static io.trino.plugin.hive.HiveSessionProperties.getOrcStreamBufferSize; import static io.trino.plugin.hive.HiveSessionProperties.getOrcTinyStripeThreshold; -import static io.trino.plugin.hive.HiveSessionProperties.getTimestampPrecision; import static io.trino.plugin.hive.HiveSessionProperties.isOrcBloomFiltersEnabled; import static io.trino.plugin.hive.HiveSessionProperties.isOrcNestedLazy; import static io.trino.plugin.hive.HiveSessionProperties.isUseOrcColumnNames; @@ -364,7 +363,7 @@ else if (column.getBaseHiveColumnIndex() < fileColumns.size()) { Type readType = column.getType(); if (orcColumn != null) { int sourceIndex = fileReadColumns.size(); - Optional> coercer = createCoercer(orcColumn.getColumnType(), readType, getTimestampPrecision(session)); + Optional> coercer = createCoercer(orcColumn.getColumnType(), readType); if (coercer.isPresent()) { fileReadTypes.add(coercer.get().getFromType()); columnAdaptations.add(ColumnAdaptation.coercedColumn(sourceIndex, coercer.get())); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java index edfe07f869d8..77028d6f4643 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java @@ -14,31 +14,24 @@ package io.trino.plugin.hive.orc; import io.trino.orc.metadata.OrcType.OrcTypeKind; -import io.trino.plugin.hive.HiveTimestampPrecision; import io.trino.plugin.hive.coercions.TimestampCoercer.LongTimestampToVarcharCoercer; -import io.trino.plugin.hive.coercions.TimestampCoercer.ShortTimestampToVarcharCoercer; import io.trino.plugin.hive.coercions.TypeCoercer; -import io.trino.spi.type.TimestampType; import io.trino.spi.type.Type; import io.trino.spi.type.VarcharType; import java.util.Optional; import static io.trino.orc.metadata.OrcType.OrcTypeKind.TIMESTAMP; -import static io.trino.spi.type.TimestampType.createTimestampType; +import static io.trino.spi.type.TimestampType.TIMESTAMP_NANOS; public final class OrcTypeTranslator { private OrcTypeTranslator() {} - public static Optional> createCoercer(OrcTypeKind fromOrcType, Type toTrinoType, HiveTimestampPrecision timestampPrecision) + public static Optional> createCoercer(OrcTypeKind fromOrcType, Type toTrinoType) { if (fromOrcType == TIMESTAMP && toTrinoType instanceof VarcharType varcharType) { - TimestampType timestampType = createTimestampType(timestampPrecision.getPrecision()); - if (timestampType.isShort()) { - return Optional.of(new ShortTimestampToVarcharCoercer(timestampType, varcharType)); - } - return Optional.of(new LongTimestampToVarcharCoercer(timestampType, varcharType)); + return Optional.of(new LongTimestampToVarcharCoercer(TIMESTAMP_NANOS, varcharType)); } return Optional.empty(); } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/coercions/TestTimestampCoercer.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/coercions/TestTimestampCoercer.java index bed42fa22ce1..deabbe331ee6 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/coercions/TestTimestampCoercer.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/coercions/TestTimestampCoercer.java @@ -26,13 +26,11 @@ import java.time.LocalDateTime; -import static io.trino.plugin.hive.HiveTimestampPrecision.MICROSECONDS; import static io.trino.plugin.hive.HiveTimestampPrecision.NANOSECONDS; import static io.trino.plugin.hive.HiveType.toHiveType; import static io.trino.plugin.hive.coercions.CoercionUtils.createCoercer; import static io.trino.spi.predicate.Utils.blockToNativeValue; import static io.trino.spi.predicate.Utils.nativeValueToBlock; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MICROS; import static io.trino.spi.type.TimestampType.TIMESTAMP_PICOS; import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; import static io.trino.spi.type.VarcharType.createVarcharType; @@ -44,15 +42,7 @@ public class TestTimestampCoercer { @Test(dataProvider = "timestampValuesProvider") - public void testShortTimestampToVarchar(String timestampValue, String hiveTimestampValue) - { - LocalDateTime localDateTime = LocalDateTime.parse(timestampValue); - SqlTimestamp timestamp = SqlTimestamp.fromSeconds(TIMESTAMP_MICROS.getPrecision(), localDateTime.toEpochSecond(UTC), localDateTime.get(NANO_OF_SECOND)); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createUnboundedVarcharType(), hiveTimestampValue); - } - - @Test(dataProvider = "timestampValuesProvider") - public void testLongTimestampToVarchar(String timestampValue, String hiveTimestampValue) + public void testTimestampToVarchar(String timestampValue, String hiveTimestampValue) { LocalDateTime localDateTime = LocalDateTime.parse(timestampValue); SqlTimestamp timestamp = SqlTimestamp.fromSeconds(TIMESTAMP_PICOS.getPrecision(), localDateTime.toEpochSecond(UTC), localDateTime.get(NANO_OF_SECOND)); @@ -60,47 +50,40 @@ public void testLongTimestampToVarchar(String timestampValue, String hiveTimesta } @Test - public void testShortTimestampToSmallerVarchar() - { - LocalDateTime localDateTime = LocalDateTime.parse("2023-04-11T05:16:12.345678"); - SqlTimestamp timestamp = SqlTimestamp.fromSeconds(TIMESTAMP_MICROS.getPrecision(), localDateTime.toEpochSecond(UTC), localDateTime.get(NANO_OF_SECOND)); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(1), "2"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(2), "20"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(3), "202"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(4), "2023"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(5), "2023-"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(6), "2023-0"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(7), "2023-04"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(8), "2023-04-"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(9), "2023-04-1"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(10), "2023-04-11"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(11), "2023-04-11 "); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(12), "2023-04-11 0"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(13), "2023-04-11 05"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(14), "2023-04-11 05:"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(15), "2023-04-11 05:1"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(16), "2023-04-11 05:16"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(17), "2023-04-11 05:16:"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(18), "2023-04-11 05:16:1"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(19), "2023-04-11 05:16:12"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(20), "2023-04-11 05:16:12."); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(21), "2023-04-11 05:16:12.3"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(22), "2023-04-11 05:16:12.34"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(23), "2023-04-11 05:16:12.345"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(24), "2023-04-11 05:16:12.3456"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(25), "2023-04-11 05:16:12.34567"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(26), "2023-04-11 05:16:12.345678"); - assertShortTimestampToVarcharCoercions(TIMESTAMP_MICROS, timestamp.getEpochMicros(), createVarcharType(27), "2023-04-11 05:16:12.345678"); - } - - @Test - public void testLongTimestampToSmallerVarchar() + public void testTimestampToSmallerVarchar() { LocalDateTime localDateTime = LocalDateTime.parse("2023-04-11T05:16:12.345678876"); SqlTimestamp timestamp = SqlTimestamp.fromSeconds(TIMESTAMP_PICOS.getPrecision(), localDateTime.toEpochSecond(UTC), localDateTime.get(NANO_OF_SECOND)); - assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, new LongTimestamp(timestamp.getEpochMicros(), timestamp.getPicosOfMicros()), createVarcharType(27), "2023-04-11 05:16:12.3456788"); - assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, new LongTimestamp(timestamp.getEpochMicros(), timestamp.getPicosOfMicros()), createVarcharType(28), "2023-04-11 05:16:12.34567887"); - assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, new LongTimestamp(timestamp.getEpochMicros(), timestamp.getPicosOfMicros()), createVarcharType(29), "2023-04-11 05:16:12.345678876"); + LongTimestamp longTimestamp = new LongTimestamp(timestamp.getEpochMicros(), timestamp.getPicosOfMicros()); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(1), "2"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(2), "20"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(3), "202"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(4), "2023"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(5), "2023-"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(6), "2023-0"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(7), "2023-04"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(8), "2023-04-"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(9), "2023-04-1"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(10), "2023-04-11"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(11), "2023-04-11 "); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(12), "2023-04-11 0"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(13), "2023-04-11 05"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(14), "2023-04-11 05:"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(15), "2023-04-11 05:1"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(16), "2023-04-11 05:16"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(17), "2023-04-11 05:16:"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(18), "2023-04-11 05:16:1"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(19), "2023-04-11 05:16:12"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(20), "2023-04-11 05:16:12."); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(21), "2023-04-11 05:16:12.3"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(22), "2023-04-11 05:16:12.34"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(23), "2023-04-11 05:16:12.345"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(24), "2023-04-11 05:16:12.3456"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(25), "2023-04-11 05:16:12.34567"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(26), "2023-04-11 05:16:12.345678"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(27), "2023-04-11 05:16:12.3456788"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(28), "2023-04-11 05:16:12.34567887"); + assertLongTimestampToVarcharCoercions(TIMESTAMP_PICOS, longTimestamp, createVarcharType(29), "2023-04-11 05:16:12.345678876"); } @DataProvider @@ -129,11 +112,6 @@ public Object[][] timestampValuesProvider() }; } - public static void assertShortTimestampToVarcharCoercions(TimestampType fromType, Long valueToBeCoerced, VarcharType toType, String expectedValue) - { - assertCoercions(fromType, valueToBeCoerced, toType, Slices.utf8Slice(expectedValue), MICROSECONDS); - } - public static void assertLongTimestampToVarcharCoercions(TimestampType fromType, LongTimestamp valueToBeCoerced, VarcharType toType, String expectedValue) { assertCoercions(fromType, valueToBeCoerced, toType, Slices.utf8Slice(expectedValue), NANOSECONDS); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java index e0cf2f4e710d..a47442365ecd 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; +import com.google.common.collect.Streams; import io.trino.jdbc.TrinoArray; import io.trino.plugin.hive.HiveTimestampPrecision; import io.trino.tempto.assertions.QueryAssert.Row; @@ -382,9 +383,10 @@ protected void doTestHiveCoercionWithDifferentTimestampPrecision(HiveTableDefini """ INSERT INTO %s SELECT - (CAST(ROW (timestamp_value, -1) AS ROW(keep TIMESTAMP(9), si2i SMALLINT))), - ARRAY [CAST(ROW (timestamp_value, -1) AS ROW (keep TIMESTAMP(9), si2i SMALLINT))], - MAP (ARRAY [2], ARRAY [CAST(ROW (timestamp_value, -1) AS ROW (keep TIMESTAMP(9), si2i SMALLINT))]), + (CAST(ROW (timestamp_value, -1, timestamp_value) AS ROW(keep TIMESTAMP(9), si2i SMALLINT, timestamp2string TIMESTAMP(9)))), + ARRAY [CAST(ROW (timestamp_value, -1, timestamp_value) AS ROW (keep TIMESTAMP(9), si2i SMALLINT, timestamp2string TIMESTAMP(9)))], + MAP (ARRAY [2], ARRAY [CAST(ROW (timestamp_value, -1, timestamp_value) AS ROW (keep TIMESTAMP(9), si2i SMALLINT, timestamp2string TIMESTAMP(9)))]), + timestamp_value, 1 FROM (VALUES (TIMESTAMP '2121-07-15 15:30:12.123499'), @@ -395,22 +397,25 @@ protected void doTestHiveCoercionWithDifferentTimestampPrecision(HiveTableDefini (TIMESTAMP '2121-07-15 15:30:12.123500001')) AS t (timestamp_value) """.formatted(tableName)); - onHive().executeQuery(format("ALTER TABLE %s CHANGE COLUMN timestamp_row_to_row timestamp_row_to_row struct", tableName)); - onHive().executeQuery(format("ALTER TABLE %s CHANGE COLUMN timestamp_list_to_list timestamp_list_to_list array>", tableName)); - onHive().executeQuery(format("ALTER TABLE %s CHANGE COLUMN timestamp_map_to_map timestamp_map_to_map map>", tableName)); + onHive().executeQuery(format("ALTER TABLE %s CHANGE COLUMN timestamp_row_to_row timestamp_row_to_row struct", tableName)); + onHive().executeQuery(format("ALTER TABLE %s CHANGE COLUMN timestamp_list_to_list timestamp_list_to_list array>", tableName)); + onHive().executeQuery(format("ALTER TABLE %s CHANGE COLUMN timestamp_map_to_map timestamp_map_to_map map>", tableName)); + onHive().executeQuery(format("ALTER TABLE %s CHANGE COLUMN timestamp_to_string timestamp_to_string string", tableName)); for (HiveTimestampPrecision hiveTimestampPrecision : HiveTimestampPrecision.values()) { setHiveTimestampPrecision(hiveTimestampPrecision); assertThat(onTrino().executeQuery("SHOW COLUMNS FROM " + tableName).project(1, 2)).containsExactlyInOrder( - row("timestamp_row_to_row", "row(keep timestamp(%d), si2i integer)".formatted(hiveTimestampPrecision.getPrecision())), - row("timestamp_list_to_list", "array(row(keep timestamp(%d), si2i integer))".formatted(hiveTimestampPrecision.getPrecision())), - row("timestamp_map_to_map", "map(integer, row(keep timestamp(%d), si2i integer))".formatted(hiveTimestampPrecision.getPrecision())), + row("timestamp_row_to_row", "row(keep timestamp(%d), si2i integer, timestamp2string varchar)".formatted(hiveTimestampPrecision.getPrecision())), + row("timestamp_list_to_list", "array(row(keep timestamp(%d), si2i integer, timestamp2string varchar))".formatted(hiveTimestampPrecision.getPrecision())), + row("timestamp_map_to_map", "map(integer, row(keep timestamp(%d), si2i integer, timestamp2string varchar))".formatted(hiveTimestampPrecision.getPrecision())), + row("timestamp_to_string", "varchar"), row("id", "bigint")); List allColumns = ImmutableList.of( "timestamp_row_to_row", "timestamp_list_to_list", "timestamp_map_to_map", + "timestamp_to_string", "id"); // For Trino, remove unsupported columns @@ -434,14 +439,21 @@ protected void doTestHiveCoercionWithDifferentTimestampPrecision(HiveTableDefini protected Map> expectedRowsForEngineProvider(Engine engine, HiveTimestampPrecision timestampPrecision) { + List timestampAsString = ImmutableList.of( + "2121-07-15 15:30:12.123499", + "2121-07-15 15:30:12.1235", + "2121-07-15 15:30:12.123501", + "2121-07-15 15:30:12.123499999", + "2121-07-15 15:30:12.1235", + "2121-07-15 15:30:12.123500001"); if (engine == Engine.HIVE) { List baseData = ImmutableList.of( - "{\"keep\":\"2121-07-15 15:30:12.123499\",\"si2i\":-1}", - "{\"keep\":\"2121-07-15 15:30:12.1235\",\"si2i\":-1}", - "{\"keep\":\"2121-07-15 15:30:12.123501\",\"si2i\":-1}", - "{\"keep\":\"2121-07-15 15:30:12.123499999\",\"si2i\":-1}", - "{\"keep\":\"2121-07-15 15:30:12.1235\",\"si2i\":-1}", - "{\"keep\":\"2121-07-15 15:30:12.123500001\",\"si2i\":-1}"); + "{\"keep\":\"2121-07-15 15:30:12.123499\",\"si2i\":-1,\"timestamp2string\":\"2121-07-15 15:30:12.123499\"}", + "{\"keep\":\"2121-07-15 15:30:12.1235\",\"si2i\":-1,\"timestamp2string\":\"2121-07-15 15:30:12.1235\"}", + "{\"keep\":\"2121-07-15 15:30:12.123501\",\"si2i\":-1,\"timestamp2string\":\"2121-07-15 15:30:12.123501\"}", + "{\"keep\":\"2121-07-15 15:30:12.123499999\",\"si2i\":-1,\"timestamp2string\":\"2121-07-15 15:30:12.123499999\"}", + "{\"keep\":\"2121-07-15 15:30:12.1235\",\"si2i\":-1,\"timestamp2string\":\"2121-07-15 15:30:12.1235\"}", + "{\"keep\":\"2121-07-15 15:30:12.123500001\",\"si2i\":-1,\"timestamp2string\":\"2121-07-15 15:30:12.123500001\"}"); return ImmutableMap.>builder() .put("timestamp_row_to_row", baseData) .put("timestamp_list_to_list", baseData.stream() @@ -451,6 +463,7 @@ protected Map> expectedRowsForEngineProvider(Engine engine, .put("timestamp_map_to_map", baseData.stream() .map("{2:%s}"::formatted) .collect(toImmutableList())) + .put("timestamp_to_string", timestampAsString) .put("id", nCopies(6, 1)) .buildOrThrow(); } @@ -479,10 +492,13 @@ protected Map> expectedRowsForEngineProvider(Engine engine, Timestamp.valueOf("2121-07-15 15:30:12.123500001")); }; - List baseData = timestampValue.stream() - .map(timestamp -> rowBuilder() + List baseData = Streams.zip( + timestampValue.stream(), + timestampAsString.stream(), + (timestamp, timestampCoerced) -> rowBuilder() .addField("keep", timestamp) .addField("si2i", -1) + .addField("timestamp2string", timestampCoerced) .build()) .collect(toImmutableList()); @@ -494,6 +510,7 @@ protected Map> expectedRowsForEngineProvider(Engine engine, .put("timestamp_map_to_map", baseData.stream() .map(entry -> ImmutableMap.of(2, entry)) .collect(toImmutableList())) + .put("timestamp_to_string", timestampAsString) .put("id", nCopies(6, 1)) .buildOrThrow(); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnPartitionedTable.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnPartitionedTable.java index 7ad6c749cad0..ea7cae009979 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnPartitionedTable.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnPartitionedTable.java @@ -135,9 +135,10 @@ private static HiveTableDefinition.HiveTableDefinitionBuilder tableDefinitionFor return HiveTableDefinition.builder(tableName) .setCreateTableDDLTemplate("" + "CREATE TABLE %NAME%(" + - " timestamp_row_to_row STRUCT, " + - " timestamp_list_to_list ARRAY>, " + - " timestamp_map_to_map MAP>" + + " timestamp_row_to_row STRUCT, " + + " timestamp_list_to_list ARRAY>, " + + " timestamp_map_to_map MAP>," + + " timestamp_to_string TIMESTAMP" + ") " + "PARTITIONED BY (id BIGINT) " + rowFormat.map(s -> format("ROW FORMAT %s ", s)).orElse("") + diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnUnpartitionedTable.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnUnpartitionedTable.java index 68adf1a33e60..82ec2347b987 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnUnpartitionedTable.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnUnpartitionedTable.java @@ -90,9 +90,10 @@ private static HiveTableDefinition.HiveTableDefinitionBuilder tableDefinitionFor return HiveTableDefinition.builder(tableName) .setCreateTableDDLTemplate(""" CREATE TABLE %NAME%( - timestamp_row_to_row STRUCT, - timestamp_list_to_list ARRAY>, - timestamp_map_to_map MAP>, + timestamp_row_to_row STRUCT, + timestamp_list_to_list ARRAY>, + timestamp_map_to_map MAP>, + timestamp_to_string TIMESTAMP, id BIGINT) STORED AS\s""" + fileFormat); } @@ -146,6 +147,9 @@ protected Map expectedExceptionsWithTrinoContext() .put(columnContext("orc", "long_decimal_to_varchar"), "Cannot read SQL type 'varchar' from ORC stream '.long_decimal_to_varchar' of type DECIMAL") .put(columnContext("orc", "short_decimal_to_bounded_varchar"), "Cannot read SQL type 'varchar(30)' from ORC stream '.short_decimal_to_bounded_varchar' of type DECIMAL") .put(columnContext("orc", "long_decimal_to_bounded_varchar"), "Cannot read SQL type 'varchar(30)' from ORC stream '.long_decimal_to_bounded_varchar' of type DECIMAL") + .put(columnContext("orc", "timestamp_row_to_row"), "Cannot read SQL type 'varchar' from ORC stream '.timestamp_row_to_row.timestamp2string' of type TIMESTAMP with attributes {}") + .put(columnContext("orc", "timestamp_list_to_list"), "Cannot read SQL type 'varchar' from ORC stream '.timestamp_row_to_row.timestamp2string' of type TIMESTAMP with attributes {}") + .put(columnContext("orc", "timestamp_map_to_map"), "Cannot read SQL type 'varchar' from ORC stream '.timestamp_row_to_row.timestamp2string' of type TIMESTAMP with attributes {}") .buildOrThrow(); } }