Skip to content

Commit

Permalink
Add char->varchar type coercion in Delta CTAS
Browse files Browse the repository at this point in the history
Similar to #21515, extend type coercion support to char type.
  • Loading branch information
SemionPar authored and ebyhr committed Jun 5, 2024
1 parent f7e45ac commit 6a0e30f
Show file tree
Hide file tree
Showing 3 changed files with 137 additions and 15 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@
import io.trino.spi.statistics.TableStatistics;
import io.trino.spi.statistics.TableStatisticsMetadata;
import io.trino.spi.type.ArrayType;
import io.trino.spi.type.CharType;
import io.trino.spi.type.DecimalType;
import io.trino.spi.type.FixedWidthType;
import io.trino.spi.type.HyperLogLogType;
Expand Down Expand Up @@ -663,6 +664,9 @@ private Type coerceType(Type type)
if (type instanceof TimestampType) {
return TIMESTAMP_MICROS;
}
if (type instanceof CharType) {
return VARCHAR;
}
if (type instanceof ArrayType arrayType) {
return new ArrayType(coerceType(arrayType.getElementType()));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -291,14 +291,6 @@ public void testDropSchemaExternalFiles()

protected abstract String bucketUrl();

@Test
public void testCharTypeIsNotSupported()
{
String tableName = "test_char_type_not_supported" + randomNameSuffix();
assertQueryFails("CREATE TABLE " + tableName + " (a int, b CHAR(5)) WITH (location = '" + getLocationForTable(bucketName, tableName) + "')",
"Unsupported type: char\\(5\\)");
}

@Test
public void testCreateTableInNonexistentSchemaFails()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -215,8 +215,9 @@ private static String transactionConflictErrors()
protected Optional<DataMappingTestSetup> filterCaseSensitiveDataMappingTestData(DataMappingTestSetup dataMappingTestSetup)
{
String typeName = dataMappingTestSetup.getTrinoTypeName();
if (typeName.equals("char(1)")) {
return Optional.of(dataMappingTestSetup.asUnsupported());
if (typeName.equals("char(3)")) {
// Use explicitly padded literal in char mapping test due to whitespace padding on coercion to varchar
return Optional.of(new DataMappingTestSetup(typeName, "'ab '", dataMappingTestSetup.getHighValueLiteral()));
}
return Optional.of(dataMappingTestSetup);
}
Expand All @@ -227,10 +228,13 @@ protected Optional<DataMappingTestSetup> filterDataMappingSmokeTestData(DataMapp
String typeName = dataMappingTestSetup.getTrinoTypeName();
if (typeName.equals("time") ||
typeName.equals("time(6)") ||
typeName.equals("timestamp(6) with time zone") ||
typeName.equals("char(3)")) {
typeName.equals("timestamp(6) with time zone")) {
return Optional.of(dataMappingTestSetup.asUnsupported());
}
if (typeName.equals("char(3)")) {
// Use explicitly padded literal in char mapping test due to whitespace padding on coercion to varchar
return Optional.of(new DataMappingTestSetup(typeName, "'ab '", dataMappingTestSetup.getHighValueLiteral()));
}
return Optional.of(dataMappingTestSetup);
}

Expand Down Expand Up @@ -551,9 +555,23 @@ public void testRenameColumnName()
@Override
public void testCharVarcharComparison()
{
// Delta Lake doesn't have a char type
assertThatThrownBy(super::testCharVarcharComparison)
.hasStackTraceContaining("Unsupported type: char(3)");
// with char->varchar coercion on table creation, this is essentially varchar/varchar comparison
try (TestTable table = new TestTable(
getQueryRunner()::execute,
"test_char_varchar",
"(k, v) AS VALUES" +
" (-1, CAST(NULL AS CHAR(3))), " +
" (3, CAST(' ' AS CHAR(3)))," +
" (6, CAST('x ' AS CHAR(3)))")) {
// varchar of length shorter than column's length
assertThat(query("SELECT k, v FROM " + table.getName() + " WHERE v = CAST(' ' AS varchar(2))")).returnsEmptyResult();
// varchar of length longer than column's length
assertThat(query("SELECT k, v FROM " + table.getName() + " WHERE v = CAST(' ' AS varchar(4))")).returnsEmptyResult();
// value that's not all-spaces
assertThat(query("SELECT k, v FROM " + table.getName() + " WHERE v = CAST('x ' AS varchar(2))")).returnsEmptyResult();
// exact match
assertQuery("SELECT k, v FROM " + table.getName() + " WHERE v = CAST(' ' AS varchar(3))", "VALUES (3, ' ')");
}
}

@Test
Expand Down Expand Up @@ -3960,6 +3978,12 @@ public void testTypeCoercionOnCreateTable()
testTimestampCoercionOnCreateTable("TIMESTAMP '1969-12-31 23:59:59.9999995'", "TIMESTAMP '1970-01-01 00:00:00.000000'");
testTimestampCoercionOnCreateTable("TIMESTAMP '1969-12-31 23:59:59.999999499999'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testTimestampCoercionOnCreateTable("TIMESTAMP '1969-12-31 23:59:59.9999994'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testCharCoercionOnCreateTable("CHAR 'ab '", "'ab '");
testCharCoercionOnCreateTable("CHAR 'A'", "'A'");
testCharCoercionOnCreateTable("CHAR 'é'", "'é'");
testCharCoercionOnCreateTable("CHAR 'A '", "'A '");
testCharCoercionOnCreateTable("CHAR ' A'", "' A'");
testCharCoercionOnCreateTable("CHAR 'ABc'", "'ABc'");
}

private void testTimestampCoercionOnCreateTable(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
Expand All @@ -3975,6 +3999,18 @@ private void testTimestampCoercionOnCreateTable(@Language("SQL") String actualVa
}
}

private void testCharCoercionOnCreateTable(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
{
try (TestTable testTable = new TestTable(
getQueryRunner()::execute,
"test_char_coercion_on_create_table",
"(vch VARCHAR)")) {
assertUpdate("INSERT INTO " + testTable.getName() + " VALUES (" + actualValue + ")", 1);
assertThat(getColumnType(testTable.getName(), "vch")).isEqualTo("varchar");
assertQuery("SELECT * FROM " + testTable.getName(), "VALUES " + expectedValue);
}
}

@Test
public void testTypeCoercionOnCreateTableAsSelect()
{
Expand Down Expand Up @@ -4004,6 +4040,12 @@ public void testTypeCoercionOnCreateTableAsSelect()
testTimestampCoercionOnCreateTableAsSelect("TIMESTAMP '1969-12-31 23:59:59.9999995'", "TIMESTAMP '1970-01-01 00:00:00.000000'");
testTimestampCoercionOnCreateTableAsSelect("TIMESTAMP '1969-12-31 23:59:59.999999499999'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testTimestampCoercionOnCreateTableAsSelect("TIMESTAMP '1969-12-31 23:59:59.9999994'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testCharCoercionOnCreateTableAsSelect("CHAR 'ab '", "'ab '");
testCharCoercionOnCreateTableAsSelect("CHAR 'A'", "'A'");
testCharCoercionOnCreateTableAsSelect("CHAR 'é'", "'é'");
testCharCoercionOnCreateTableAsSelect("CHAR 'A '", "'A '");
testCharCoercionOnCreateTableAsSelect("CHAR ' A'", "' A'");
testCharCoercionOnCreateTableAsSelect("CHAR 'ABc'", "'ABc'");
}

private void testTimestampCoercionOnCreateTableAsSelect(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
Expand All @@ -4018,6 +4060,17 @@ private void testTimestampCoercionOnCreateTableAsSelect(@Language("SQL") String
}
}

private void testCharCoercionOnCreateTableAsSelect(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
{
try (TestTable testTable = new TestTable(
getQueryRunner()::execute,
"test_char_coercion_on_create_table_as_select",
"AS SELECT %s col".formatted(actualValue))) {
assertThat(getColumnType(testTable.getName(), "col")).isEqualTo("varchar");
assertQuery("SELECT * FROM " + testTable.getName(), "VALUES " + expectedValue);
}
}

@Test
public void testTypeCoercionOnCreateTableAsSelectWithNoData()
{
Expand Down Expand Up @@ -4047,6 +4100,12 @@ public void testTypeCoercionOnCreateTableAsSelectWithNoData()
testTimestampCoercionOnCreateTableAsSelectWithNoData("TIMESTAMP '1969-12-31 23:59:59.9999995'");
testTimestampCoercionOnCreateTableAsSelectWithNoData("TIMESTAMP '1969-12-31 23:59:59.999999499999'");
testTimestampCoercionOnCreateTableAsSelectWithNoData("TIMESTAMP '1969-12-31 23:59:59.9999994'");
testCharCoercionOnCreateTableAsSelectWithNoData("CHAR 'ab '");
testCharCoercionOnCreateTableAsSelectWithNoData("CHAR 'A'");
testCharCoercionOnCreateTableAsSelectWithNoData("CHAR 'é'");
testCharCoercionOnCreateTableAsSelectWithNoData("CHAR 'A '");
testCharCoercionOnCreateTableAsSelectWithNoData("CHAR ' A'");
testCharCoercionOnCreateTableAsSelectWithNoData("CHAR 'ABc'");
}

private void testTimestampCoercionOnCreateTableAsSelectWithNoData(@Language("SQL") String actualValue)
Expand All @@ -4060,6 +4119,16 @@ private void testTimestampCoercionOnCreateTableAsSelectWithNoData(@Language("SQL
}
}

private void testCharCoercionOnCreateTableAsSelectWithNoData(@Language("SQL") String actualValue)
{
try (TestTable testTable = new TestTable(
getQueryRunner()::execute,
"test_char_coercion_on_create_table_as_select_with_no_data",
"AS SELECT %s col WITH NO DATA".formatted(actualValue))) {
assertThat(getColumnType(testTable.getName(), "col")).isEqualTo("varchar");
}
}

@Test
public void testTypeCoercionOnCreateTableAsWithRowType()
{
Expand Down Expand Up @@ -4089,6 +4158,13 @@ public void testTypeCoercionOnCreateTableAsWithRowType()
testTimestampCoercionOnCreateTableAsWithRowType("TIMESTAMP '1969-12-31 23:59:59.9999995'", "TIMESTAMP '1970-01-01 00:00:00.000000'");
testTimestampCoercionOnCreateTableAsWithRowType("TIMESTAMP '1969-12-31 23:59:59.999999499999'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testTimestampCoercionOnCreateTableAsWithRowType("TIMESTAMP '1969-12-31 23:59:59.9999994'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testCharCoercionOnCreateTableAsWithRowType("CHAR 'ab '", "CHAR(3)", "'ab '");
testCharCoercionOnCreateTableAsWithRowType("CHAR 'A'", "CHAR(3)", "'A '");
testCharCoercionOnCreateTableAsWithRowType("CHAR 'A'", "CHAR(1)", "'A'");
testCharCoercionOnCreateTableAsWithRowType("CHAR 'é'", "CHAR(3)", "'é '");
testCharCoercionOnCreateTableAsWithRowType("CHAR 'A '", "CHAR(3)", "'A '");
testCharCoercionOnCreateTableAsWithRowType("CHAR ' A'", "CHAR(3)", "' A '");
testCharCoercionOnCreateTableAsWithRowType("CHAR 'ABc'", "CHAR(3)", "'ABc'");
}

private void testTimestampCoercionOnCreateTableAsWithRowType(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
Expand All @@ -4105,6 +4181,19 @@ private void testTimestampCoercionOnCreateTableAsWithRowType(@Language("SQL") St
}
}

private void testCharCoercionOnCreateTableAsWithRowType(@Language("SQL") String actualValue, @Language("SQL") String actualTypeLiteral, @Language("SQL") String expectedValue)
{
try (TestTable testTable = new TestTable(
getQueryRunner()::execute,
"test_char_coercion_on_create_table_as_with_row_type",
"AS SELECT CAST(row(%s) AS row(value %s)) col".formatted(actualValue, actualTypeLiteral))) {
assertThat(getColumnType(testTable.getName(), "col")).isEqualTo("row(value varchar)");
assertThat(query("SELECT col.value FROM " + testTable.getName()))
.skippingTypesCheck()
.matches("VALUES " + expectedValue);
}
}

@Test
public void testTypeCoercionOnCreateTableAsWithArrayType()
{
Expand Down Expand Up @@ -4134,6 +4223,12 @@ public void testTypeCoercionOnCreateTableAsWithArrayType()
testTimestampCoercionOnCreateTableAsWithArrayType("TIMESTAMP '1969-12-31 23:59:59.9999995'", "TIMESTAMP '1970-01-01 00:00:00.000000'");
testTimestampCoercionOnCreateTableAsWithArrayType("TIMESTAMP '1969-12-31 23:59:59.999999499999'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testTimestampCoercionOnCreateTableAsWithArrayType("TIMESTAMP '1969-12-31 23:59:59.9999994'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testCharCoercionOnCreateTableAsWithArrayType("CHAR 'ab '", "'ab '");
testCharCoercionOnCreateTableAsWithArrayType("CHAR 'A'", "'A'");
testCharCoercionOnCreateTableAsWithArrayType("CHAR 'é'", "'é'");
testCharCoercionOnCreateTableAsWithArrayType("CHAR 'A '", "'A '");
testCharCoercionOnCreateTableAsWithArrayType("CHAR ' A'", "' A'");
testCharCoercionOnCreateTableAsWithArrayType("CHAR 'ABc'", "'ABc'");
}

private void testTimestampCoercionOnCreateTableAsWithArrayType(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
Expand All @@ -4149,6 +4244,18 @@ private void testTimestampCoercionOnCreateTableAsWithArrayType(@Language("SQL")
assertTimestampNtzFeature(testTable.getName());
}
}
private void testCharCoercionOnCreateTableAsWithArrayType(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
{
try (TestTable testTable = new TestTable(
getQueryRunner()::execute,
"test_char_coercion_on_create_table_as_with_array_type",
"AS SELECT array[%s] col".formatted(actualValue))) {
assertThat(getColumnType(testTable.getName(), "col")).isEqualTo("array(varchar)");
assertThat(query("SELECT col[1] FROM " + testTable.getName()))
.skippingTypesCheck()
.matches("VALUES " + expectedValue);
}
}

@Test
public void testTypeCoercionOnCreateTableAsWithMapType()
Expand Down Expand Up @@ -4179,6 +4286,12 @@ public void testTypeCoercionOnCreateTableAsWithMapType()
testTimestampCoercionOnCreateTableAsWithMapType("TIMESTAMP '1969-12-31 23:59:59.9999995'", "TIMESTAMP '1970-01-01 00:00:00.000000'");
testTimestampCoercionOnCreateTableAsWithMapType("TIMESTAMP '1969-12-31 23:59:59.999999499999'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testTimestampCoercionOnCreateTableAsWithMapType("TIMESTAMP '1969-12-31 23:59:59.9999994'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testCharCoercionOnCreateTableAsWithMapType("CHAR 'ab '", "'ab '");
testCharCoercionOnCreateTableAsWithMapType("CHAR 'A'", "'A'");
testCharCoercionOnCreateTableAsWithMapType("CHAR 'é'", "'é'");
testCharCoercionOnCreateTableAsWithMapType("CHAR 'A '", "'A '");
testCharCoercionOnCreateTableAsWithMapType("CHAR ' A'", "' A'");
testCharCoercionOnCreateTableAsWithMapType("CHAR 'ABc'", "'ABc'");
}

private void testTimestampCoercionOnCreateTableAsWithMapType(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
Expand All @@ -4195,6 +4308,19 @@ private void testTimestampCoercionOnCreateTableAsWithMapType(@Language("SQL") St
}
}

private void testCharCoercionOnCreateTableAsWithMapType(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
{
try (TestTable testTable = new TestTable(
getQueryRunner()::execute,
"test_char_coercion_on_create_table_as_with_map_type",
"AS SELECT map(array[%1$s], array[%1$s]) col".formatted(actualValue))) {
assertThat(getColumnType(testTable.getName(), "col")).isEqualTo("map(varchar, varchar)");
assertThat(query("SELECT * FROM " + testTable.getName()))
.skippingTypesCheck()
.matches("SELECT map(array[%1$s], array[%1$s])".formatted(expectedValue));
}
}

private void assertTimestampNtzFeature(String tableName)
{
assertThat(query("SELECT * FROM \"" + tableName + "$properties\""))
Expand Down

0 comments on commit 6a0e30f

Please sign in to comment.