Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extend automatic type coercion support in Delta table creation to char #22049

Merged
merged 2 commits into from
Jun 5, 2024
Merged

Extend automatic type coercion support in Delta table creation to char #22049

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3482e5c
Generify type coercion test names
SemionPar Apr 29, 2024
8d782f1
Add char->varchar type coercion in Delta CTAS
SemionPar Apr 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@
import io.trino.spi.statistics.TableStatistics;
import io.trino.spi.statistics.TableStatisticsMetadata;
import io.trino.spi.type.ArrayType;
import io.trino.spi.type.CharType;
import io.trino.spi.type.DecimalType;
import io.trino.spi.type.FixedWidthType;
import io.trino.spi.type.HyperLogLogType;
Expand Down Expand Up @@ -663,6 +664,9 @@ private Type coerceType(Type type)
if (type instanceof TimestampType) {
return TIMESTAMP_MICROS;
}
if (type instanceof CharType) {
return VARCHAR;
}
if (type instanceof ArrayType arrayType) {
return new ArrayType(coerceType(arrayType.getElementType()));
}
Expand Down
8 changes: 0 additions & 8 deletions ...o-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeConnectorSmokeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,14 +291,6 @@ public void testDropSchemaExternalFiles()

protected abstract String bucketUrl();

@Test
public void testCharTypeIsNotSupported()
{
String tableName = "test_char_type_not_supported" + randomNameSuffix();
assertQueryFails("CREATE TABLE " + tableName + " (a int, b CHAR(5)) WITH (location = '" + getLocationForTable(bucketName, tableName) + "')",
"Unsupported type: char\\(5\\)");
}

@Test
public void testCreateTableInNonexistentSchemaFails()
{
Expand Down
152 changes: 139 additions & 13 deletions .../trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConnectorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,8 +215,9 @@ private static String transactionConflictErrors()
protected Optional<DataMappingTestSetup> filterCaseSensitiveDataMappingTestData(DataMappingTestSetup dataMappingTestSetup)
{
String typeName = dataMappingTestSetup.getTrinoTypeName();
if (typeName.equals("char(1)")) {
return Optional.of(dataMappingTestSetup.asUnsupported());
if (typeName.equals("char(3)")) {
// Use explicitly padded literal in char mapping test due to whitespace padding on coercion to varchar
return Optional.of(new DataMappingTestSetup(typeName, "'ab '", dataMappingTestSetup.getHighValueLiteral()));
}
return Optional.of(dataMappingTestSetup);
}
Expand All @@ -227,10 +228,13 @@ protected Optional<DataMappingTestSetup> filterDataMappingSmokeTestData(DataMapp
String typeName = dataMappingTestSetup.getTrinoTypeName();
if (typeName.equals("time") ||
typeName.equals("time(6)") ||
typeName.equals("timestamp(6) with time zone") ||
typeName.equals("char(3)")) {
typeName.equals("timestamp(6) with time zone")) {
return Optional.of(dataMappingTestSetup.asUnsupported());
}
if (typeName.equals("char(3)")) {
// Use explicitly padded literal in char mapping test due to whitespace padding on coercion to varchar
return Optional.of(new DataMappingTestSetup(typeName, "'ab '", dataMappingTestSetup.getHighValueLiteral()));
}
return Optional.of(dataMappingTestSetup);
}

Expand Down Expand Up @@ -551,9 +555,23 @@ public void testRenameColumnName()
@Override
public void testCharVarcharComparison()
{
// Delta Lake doesn't have a char type
assertThatThrownBy(super::testCharVarcharComparison)
.hasStackTraceContaining("Unsupported type: char(3)");
// with char->varchar coercion on table creation, this is essentially varchar/varchar comparison
try (TestTable table = new TestTable(
getQueryRunner()::execute,
"test_char_varchar",
"(k, v) AS VALUES" +
" (-1, CAST(NULL AS CHAR(3))), " +
" (3, CAST(' ' AS CHAR(3)))," +
" (6, CAST('x ' AS CHAR(3)))")) {
// varchar of length shorter than column's length
assertThat(query("SELECT k, v FROM " + table.getName() + " WHERE v = CAST(' ' AS varchar(2))")).returnsEmptyResult();
// varchar of length longer than column's length
assertThat(query("SELECT k, v FROM " + table.getName() + " WHERE v = CAST(' ' AS varchar(4))")).returnsEmptyResult();
// value that's not all-spaces
assertThat(query("SELECT k, v FROM " + table.getName() + " WHERE v = CAST('x ' AS varchar(2))")).returnsEmptyResult();
// exact match
assertQuery("SELECT k, v FROM " + table.getName() + " WHERE v = CAST(' ' AS varchar(3))", "VALUES (3, ' ')");
}
}

@Test
Expand Down Expand Up @@ -3932,7 +3950,7 @@ public void testQueriesWithoutCheckpointFiltering()
}

@Test
public void testTimestampCoercionOnCreateTable()
public void testTypeCoercionOnCreateTable()
{
testTimestampCoercionOnCreateTable("TIMESTAMP '1970-01-01 00:00:00'", "TIMESTAMP '1970-01-01 00:00:00.000000'");
testTimestampCoercionOnCreateTable("TIMESTAMP '1970-01-01 00:00:00.9'", "TIMESTAMP '1970-01-01 00:00:00.900000'");
Expand Down Expand Up @@ -3960,6 +3978,12 @@ public void testTimestampCoercionOnCreateTable()
testTimestampCoercionOnCreateTable("TIMESTAMP '1969-12-31 23:59:59.9999995'", "TIMESTAMP '1970-01-01 00:00:00.000000'");
testTimestampCoercionOnCreateTable("TIMESTAMP '1969-12-31 23:59:59.999999499999'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testTimestampCoercionOnCreateTable("TIMESTAMP '1969-12-31 23:59:59.9999994'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testCharCoercionOnCreateTable("CHAR 'ab '", "'ab '");
testCharCoercionOnCreateTable("CHAR 'A'", "'A'");
testCharCoercionOnCreateTable("CHAR 'é'", "'é'");
testCharCoercionOnCreateTable("CHAR 'A '", "'A '");
testCharCoercionOnCreateTable("CHAR ' A'", "' A'");
testCharCoercionOnCreateTable("CHAR 'ABc'", "'ABc'");
}

private void testTimestampCoercionOnCreateTable(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
Expand All @@ -3975,8 +3999,20 @@ private void testTimestampCoercionOnCreateTable(@Language("SQL") String actualVa
}
}

private void testCharCoercionOnCreateTable(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
{
try (TestTable testTable = new TestTable(
getQueryRunner()::execute,
"test_char_coercion_on_create_table",
"(vch VARCHAR)")) {
SemionPar marked this conversation as resolved.
Show resolved Hide resolved
assertUpdate("INSERT INTO " + testTable.getName() + " VALUES (" + actualValue + ")", 1);
assertThat(getColumnType(testTable.getName(), "vch")).isEqualTo("varchar");
assertQuery("SELECT * FROM " + testTable.getName(), "VALUES " + expectedValue);
}
}

@Test
public void testTimestampCoercionOnCreateTableAsSelect()
public void testTypeCoercionOnCreateTableAsSelect()
{
testTimestampCoercionOnCreateTableAsSelect("TIMESTAMP '1970-01-01 00:00:00'", "TIMESTAMP '1970-01-01 00:00:00.000000'");
testTimestampCoercionOnCreateTableAsSelect("TIMESTAMP '1970-01-01 00:00:00.9'", "TIMESTAMP '1970-01-01 00:00:00.900000'");
Expand Down Expand Up @@ -4004,6 +4040,12 @@ public void testTimestampCoercionOnCreateTableAsSelect()
testTimestampCoercionOnCreateTableAsSelect("TIMESTAMP '1969-12-31 23:59:59.9999995'", "TIMESTAMP '1970-01-01 00:00:00.000000'");
testTimestampCoercionOnCreateTableAsSelect("TIMESTAMP '1969-12-31 23:59:59.999999499999'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testTimestampCoercionOnCreateTableAsSelect("TIMESTAMP '1969-12-31 23:59:59.9999994'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testCharCoercionOnCreateTableAsSelect("CHAR 'ab '", "'ab '");
testCharCoercionOnCreateTableAsSelect("CHAR 'A'", "'A'");
testCharCoercionOnCreateTableAsSelect("CHAR 'é'", "'é'");
testCharCoercionOnCreateTableAsSelect("CHAR 'A '", "'A '");
testCharCoercionOnCreateTableAsSelect("CHAR ' A'", "' A'");
testCharCoercionOnCreateTableAsSelect("CHAR 'ABc'", "'ABc'");
}

private void testTimestampCoercionOnCreateTableAsSelect(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
Expand All @@ -4018,8 +4060,19 @@ private void testTimestampCoercionOnCreateTableAsSelect(@Language("SQL") String
}
}

private void testCharCoercionOnCreateTableAsSelect(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
{
try (TestTable testTable = new TestTable(
getQueryRunner()::execute,
"test_char_coercion_on_create_table_as_select",
"AS SELECT %s col".formatted(actualValue))) {
assertThat(getColumnType(testTable.getName(), "col")).isEqualTo("varchar");
assertQuery("SELECT * FROM " + testTable.getName(), "VALUES " + expectedValue);
}
}

@Test
public void testTimestampCoercionOnCreateTableAsSelectWithNoData()
public void testTypeCoercionOnCreateTableAsSelectWithNoData()
{
testTimestampCoercionOnCreateTableAsSelectWithNoData("TIMESTAMP '1970-01-01 00:00:00'");
testTimestampCoercionOnCreateTableAsSelectWithNoData("TIMESTAMP '1970-01-01 00:00:00.9'");
Expand Down Expand Up @@ -4047,6 +4100,12 @@ public void testTimestampCoercionOnCreateTableAsSelectWithNoData()
testTimestampCoercionOnCreateTableAsSelectWithNoData("TIMESTAMP '1969-12-31 23:59:59.9999995'");
testTimestampCoercionOnCreateTableAsSelectWithNoData("TIMESTAMP '1969-12-31 23:59:59.999999499999'");
testTimestampCoercionOnCreateTableAsSelectWithNoData("TIMESTAMP '1969-12-31 23:59:59.9999994'");
testCharCoercionOnCreateTableAsSelectWithNoData("CHAR 'ab '");
testCharCoercionOnCreateTableAsSelectWithNoData("CHAR 'A'");
testCharCoercionOnCreateTableAsSelectWithNoData("CHAR 'é'");
testCharCoercionOnCreateTableAsSelectWithNoData("CHAR 'A '");
testCharCoercionOnCreateTableAsSelectWithNoData("CHAR ' A'");
testCharCoercionOnCreateTableAsSelectWithNoData("CHAR 'ABc'");
}

private void testTimestampCoercionOnCreateTableAsSelectWithNoData(@Language("SQL") String actualValue)
Expand All @@ -4060,8 +4119,18 @@ private void testTimestampCoercionOnCreateTableAsSelectWithNoData(@Language("SQL
}
}

private void testCharCoercionOnCreateTableAsSelectWithNoData(@Language("SQL") String actualValue)
{
try (TestTable testTable = new TestTable(
getQueryRunner()::execute,
"test_char_coercion_on_create_table_as_select_with_no_data",
"AS SELECT %s col WITH NO DATA".formatted(actualValue))) {
assertThat(getColumnType(testTable.getName(), "col")).isEqualTo("varchar");
}
}

@Test
public void testTimestampCoercionOnCreateTableAsWithRowType()
public void testTypeCoercionOnCreateTableAsWithRowType()
{
testTimestampCoercionOnCreateTableAsWithRowType("TIMESTAMP '1970-01-01 00:00:00'", "TIMESTAMP '1970-01-01 00:00:00'");
testTimestampCoercionOnCreateTableAsWithRowType("TIMESTAMP '1970-01-01 00:00:00.9'", "TIMESTAMP '1970-01-01 00:00:00.9'");
Expand Down Expand Up @@ -4089,6 +4158,13 @@ public void testTimestampCoercionOnCreateTableAsWithRowType()
testTimestampCoercionOnCreateTableAsWithRowType("TIMESTAMP '1969-12-31 23:59:59.9999995'", "TIMESTAMP '1970-01-01 00:00:00.000000'");
testTimestampCoercionOnCreateTableAsWithRowType("TIMESTAMP '1969-12-31 23:59:59.999999499999'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testTimestampCoercionOnCreateTableAsWithRowType("TIMESTAMP '1969-12-31 23:59:59.9999994'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testCharCoercionOnCreateTableAsWithRowType("CHAR 'ab '", "CHAR(3)", "'ab '");
testCharCoercionOnCreateTableAsWithRowType("CHAR 'A'", "CHAR(3)", "'A '");
testCharCoercionOnCreateTableAsWithRowType("CHAR 'A'", "CHAR(1)", "'A'");
testCharCoercionOnCreateTableAsWithRowType("CHAR 'é'", "CHAR(3)", "'é '");
testCharCoercionOnCreateTableAsWithRowType("CHAR 'A '", "CHAR(3)", "'A '");
testCharCoercionOnCreateTableAsWithRowType("CHAR ' A'", "CHAR(3)", "' A '");
testCharCoercionOnCreateTableAsWithRowType("CHAR 'ABc'", "CHAR(3)", "'ABc'");
}

private void testTimestampCoercionOnCreateTableAsWithRowType(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
Expand All @@ -4105,8 +4181,21 @@ private void testTimestampCoercionOnCreateTableAsWithRowType(@Language("SQL") St
}
}

private void testCharCoercionOnCreateTableAsWithRowType(@Language("SQL") String actualValue, @Language("SQL") String actualTypeLiteral, @Language("SQL") String expectedValue)
{
try (TestTable testTable = new TestTable(
getQueryRunner()::execute,
"test_char_coercion_on_create_table_as_with_row_type",
"AS SELECT CAST(row(%s) AS row(value %s)) col".formatted(actualValue, actualTypeLiteral))) {
assertThat(getColumnType(testTable.getName(), "col")).isEqualTo("row(value varchar)");
assertThat(query("SELECT col.value FROM " + testTable.getName()))
.skippingTypesCheck()
.matches("VALUES " + expectedValue);
}
}

@Test
public void testTimestampCoercionOnCreateTableAsWithArrayType()
public void testTypeCoercionOnCreateTableAsWithArrayType()
{
testTimestampCoercionOnCreateTableAsWithArrayType("TIMESTAMP '1970-01-01 00:00:00'", "TIMESTAMP '1970-01-01 00:00:00'");
testTimestampCoercionOnCreateTableAsWithArrayType("TIMESTAMP '1970-01-01 00:00:00.9'", "TIMESTAMP '1970-01-01 00:00:00.9'");
Expand Down Expand Up @@ -4134,6 +4223,12 @@ public void testTimestampCoercionOnCreateTableAsWithArrayType()
testTimestampCoercionOnCreateTableAsWithArrayType("TIMESTAMP '1969-12-31 23:59:59.9999995'", "TIMESTAMP '1970-01-01 00:00:00.000000'");
testTimestampCoercionOnCreateTableAsWithArrayType("TIMESTAMP '1969-12-31 23:59:59.999999499999'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testTimestampCoercionOnCreateTableAsWithArrayType("TIMESTAMP '1969-12-31 23:59:59.9999994'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testCharCoercionOnCreateTableAsWithArrayType("CHAR 'ab '", "'ab '");
testCharCoercionOnCreateTableAsWithArrayType("CHAR 'A'", "'A'");
testCharCoercionOnCreateTableAsWithArrayType("CHAR 'é'", "'é'");
testCharCoercionOnCreateTableAsWithArrayType("CHAR 'A '", "'A '");
testCharCoercionOnCreateTableAsWithArrayType("CHAR ' A'", "' A'");
testCharCoercionOnCreateTableAsWithArrayType("CHAR 'ABc'", "'ABc'");
}

private void testTimestampCoercionOnCreateTableAsWithArrayType(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
Expand All @@ -4149,9 +4244,21 @@ private void testTimestampCoercionOnCreateTableAsWithArrayType(@Language("SQL")
assertTimestampNtzFeature(testTable.getName());
}
}
private void testCharCoercionOnCreateTableAsWithArrayType(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
{
try (TestTable testTable = new TestTable(
getQueryRunner()::execute,
"test_char_coercion_on_create_table_as_with_array_type",
"AS SELECT array[%s] col".formatted(actualValue))) {
assertThat(getColumnType(testTable.getName(), "col")).isEqualTo("array(varchar)");
assertThat(query("SELECT col[1] FROM " + testTable.getName()))
.skippingTypesCheck()
.matches("VALUES " + expectedValue);
}
}

@Test
public void testTimestampCoercionOnCreateTableAsWithMapType()
public void testTypeCoercionOnCreateTableAsWithMapType()
{
testTimestampCoercionOnCreateTableAsWithMapType("TIMESTAMP '1970-01-01 00:00:00'", "TIMESTAMP '1970-01-01 00:00:00'");
testTimestampCoercionOnCreateTableAsWithMapType("TIMESTAMP '1970-01-01 00:00:00.9'", "TIMESTAMP '1970-01-01 00:00:00.9'");
Expand Down Expand Up @@ -4179,6 +4286,12 @@ public void testTimestampCoercionOnCreateTableAsWithMapType()
testTimestampCoercionOnCreateTableAsWithMapType("TIMESTAMP '1969-12-31 23:59:59.9999995'", "TIMESTAMP '1970-01-01 00:00:00.000000'");
testTimestampCoercionOnCreateTableAsWithMapType("TIMESTAMP '1969-12-31 23:59:59.999999499999'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testTimestampCoercionOnCreateTableAsWithMapType("TIMESTAMP '1969-12-31 23:59:59.9999994'", "TIMESTAMP '1969-12-31 23:59:59.999999'");
testCharCoercionOnCreateTableAsWithMapType("CHAR 'ab '", "'ab '");
testCharCoercionOnCreateTableAsWithMapType("CHAR 'A'", "'A'");
testCharCoercionOnCreateTableAsWithMapType("CHAR 'é'", "'é'");
testCharCoercionOnCreateTableAsWithMapType("CHAR 'A '", "'A '");
testCharCoercionOnCreateTableAsWithMapType("CHAR ' A'", "' A'");
testCharCoercionOnCreateTableAsWithMapType("CHAR 'ABc'", "'ABc'");
}

private void testTimestampCoercionOnCreateTableAsWithMapType(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
Expand All @@ -4195,6 +4308,19 @@ private void testTimestampCoercionOnCreateTableAsWithMapType(@Language("SQL") St
}
}

private void testCharCoercionOnCreateTableAsWithMapType(@Language("SQL") String actualValue, @Language("SQL") String expectedValue)
{
try (TestTable testTable = new TestTable(
getQueryRunner()::execute,
"test_char_coercion_on_create_table_as_with_map_type",
"AS SELECT map(array[%1$s], array[%1$s]) col".formatted(actualValue))) {
assertThat(getColumnType(testTable.getName(), "col")).isEqualTo("map(varchar, varchar)");
assertThat(query("SELECT * FROM " + testTable.getName()))
.skippingTypesCheck()
.matches("SELECT map(array[%1$s], array[%1$s])".formatted(expectedValue));
}
}

private void assertTimestampNtzFeature(String tableName)
{
assertThat(query("SELECT * FROM \"" + tableName + "$properties\""))
Expand Down