Skip to content

Commit

Permalink
Support writing NUMERIC and BIGNUMERIC types in BigQuery
Browse files Browse the repository at this point in the history
  • Loading branch information
@vlad-lyutenko @ebyhr
vlad-lyutenko authored and ebyhr committed Feb 18, 2023
1 parent a940794 commit 8cfa24c
Show file tree
Hide file tree
Showing 4 changed files with 74 additions and 3 deletions.
3 changes: 3 additions & 0 deletions docs/src/main/sphinx/connector/bigquery.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,9 @@ to the following table:
- ``INT64``
- ``INT``, ``SMALLINT``, ``INTEGER``, ``BIGINT``, ``TINYINT``, and
``BYTEINT`` are aliases for ``INT64`` in BigQuery.
* - ``DECIMAL(P,S)``
- ``NUMERIC``
- The default precision and scale of ``NUMERIC`` is ``(38, 9)``.
* - ``VARCHAR``
- ``STRING``
-
Expand Down
7 changes: 6 additions & 1 deletion plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryTypeUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import io.trino.spi.TrinoException;
import io.trino.spi.block.Block;
import io.trino.spi.type.ArrayType;
import io.trino.spi.type.DecimalType;
import io.trino.spi.type.RowType;
import io.trino.spi.type.Type;
import io.trino.spi.type.VarcharType;
Expand All @@ -38,6 +39,7 @@
import static io.trino.spi.type.BigintType.BIGINT;
import static io.trino.spi.type.BooleanType.BOOLEAN;
import static io.trino.spi.type.DateType.DATE;
import static io.trino.spi.type.Decimals.readBigDecimal;
import static io.trino.spi.type.DoubleType.DOUBLE;
import static io.trino.spi.type.IntegerType.INTEGER;
import static io.trino.spi.type.SmallintType.SMALLINT;
Expand Down Expand Up @@ -66,7 +68,7 @@ public static Object readNativeValue(Type type, Block block, int position)
return null;
}

// TODO https://github.com/trinodb/trino/issues/13741 Add support for decimal, time, timestamp with time zone, geography, map type
// TODO https://github.com/trinodb/trino/issues/13741 Add support for time, timestamp with time zone, geography, map type
if (type.equals(BOOLEAN)) {
return type.getBoolean(block, position);
}
Expand All @@ -85,6 +87,9 @@ public static Object readNativeValue(Type type, Block block, int position)
if (type.equals(DOUBLE)) {
return type.getDouble(block, position);
}
if (type instanceof DecimalType) {
return readBigDecimal((DecimalType) type, block, position).toString();
}
if (type instanceof VarcharType) {
return type.getSlice(block, position).toStringUtf8();
}
Expand Down
2 changes: 0 additions & 2 deletions plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BaseBigQueryConnectorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,8 +223,6 @@ protected Optional<DataMappingTestSetup> filterDataMappingSmokeTestData(DataMapp
switch (dataMappingTestSetup.getTrinoTypeName()) {
case "real":
case "char(3)":
case "decimal(5,3)":
case "decimal(15,3)":
case "time":
case "time(3)":
case "time(6)":
Expand Down
65 changes: 65 additions & 0 deletions plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BaseBigQueryTypeMapping.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import io.trino.spi.type.RowType.Field;
import io.trino.testing.AbstractTestQueryFramework;
import io.trino.testing.datatype.CreateAndInsertDataSetup;
import io.trino.testing.datatype.CreateAndTrinoInsertDataSetup;
import io.trino.testing.datatype.CreateAsSelectDataSetup;
import io.trino.testing.datatype.DataSetup;
import io.trino.testing.datatype.SqlDataTypeTest;
Expand Down Expand Up @@ -234,6 +235,36 @@ public void testNumericMapping()
.execute(getQueryRunner(), bigqueryCreateAndInsert("test.numeric"));
}

@Test
public void testNumericWriteMapping()
{
// Max precision is 29 when the scale is 0 in BigQuery
// Valid scale range is between 0 and 9 in BigQuery

SqlDataTypeTest.create()
.addRoundTrip("NUMERIC(3, 0)", "CAST(193 AS DECIMAL(3, 0))", createDecimalType(3, 0), "CAST(193 AS DECIMAL(3, 0))")
.addRoundTrip("NUMERIC(3, 0)", "CAST(19 AS DECIMAL(3, 0))", createDecimalType(3, 0), "CAST(19 AS DECIMAL(3, 0))")
.addRoundTrip("NUMERIC(3, 0)", "CAST(-193 AS DECIMAL(3, 0))", createDecimalType(3, 0), "CAST(-193 AS DECIMAL(3, 0))")
.addRoundTrip("NUMERIC(3, 1)", "CAST(10.0 AS DECIMAL(3, 1))", createDecimalType(3, 1), "CAST(10.0 AS DECIMAL(3, 1))")
.addRoundTrip("NUMERIC(3, 1)", "CAST(10.1 AS DECIMAL(3, 1))", createDecimalType(3, 1), "CAST(10.1 AS DECIMAL(3, 1))")
.addRoundTrip("NUMERIC(3, 1)", "CAST(-10.1 AS DECIMAL(3, 1))", createDecimalType(3, 1), "CAST(-10.1 AS DECIMAL(3, 1))")
.addRoundTrip("NUMERIC(4, 2)", "CAST(2 AS DECIMAL(4, 2))", createDecimalType(4, 2), "CAST(2 AS DECIMAL(4, 2))")
.addRoundTrip("NUMERIC(4, 2)", "CAST(2.3 AS DECIMAL(4, 2))", createDecimalType(4, 2), "CAST(2.3 AS DECIMAL(4, 2))")
.addRoundTrip("NUMERIC(24, 2)", "CAST(2 AS DECIMAL(24, 2))", createDecimalType(24, 2), "CAST(2 AS DECIMAL(24, 2))")
.addRoundTrip("NUMERIC(24, 2)", "CAST(2.3 AS DECIMAL(24, 2))", createDecimalType(24, 2), "CAST(2.3 AS DECIMAL(24, 2))")
.addRoundTrip("NUMERIC(24, 2)", "CAST(123456789.3 AS DECIMAL(24, 2))", createDecimalType(24, 2), "CAST(123456789.3 AS DECIMAL(24, 2))")
.addRoundTrip("NUMERIC(24, 4)", "CAST(12345678901234567890.31 AS DECIMAL(24, 4))", createDecimalType(24, 4), "CAST(12345678901234567890.31 AS DECIMAL(24, 4))")
.addRoundTrip("NUMERIC(29, 0)", "CAST('27182818284590452353602874713' AS DECIMAL(29, 0))", createDecimalType(29, 0), "CAST('27182818284590452353602874713' AS DECIMAL(29, 0))")
.addRoundTrip("NUMERIC(29, 0)", "CAST('-27182818284590452353602874713' AS DECIMAL(29, 0))", createDecimalType(29, 0), "CAST('-27182818284590452353602874713' AS DECIMAL(29, 0))")
.addRoundTrip("NUMERIC(30, 5)", "CAST(3141592653589793238462643.38327 AS DECIMAL(30, 5))", createDecimalType(30, 5), "CAST(3141592653589793238462643.38327 AS DECIMAL(30, 5))")
.addRoundTrip("NUMERIC(30, 5)", "CAST(-3141592653589793238462643.38327 AS DECIMAL(30, 5))", createDecimalType(30, 5), "CAST(-3141592653589793238462643.38327 AS DECIMAL(30, 5))")
.addRoundTrip("NUMERIC(38, 9)", "CAST(100000000020000000001234567.123456789 AS DECIMAL(38, 9))", createDecimalType(38, 9), "CAST(100000000020000000001234567.123456789 AS DECIMAL(38, 9))")
.addRoundTrip("NUMERIC(38, 9)", "CAST(-100000000020000000001234567.123456789 AS DECIMAL(38, 9))", createDecimalType(38, 9), "CAST(-100000000020000000001234567.123456789 AS DECIMAL(38, 9))")
.addRoundTrip("NUMERIC(10, 3)", "CAST(NULL AS DECIMAL(10, 3))", createDecimalType(10, 3), "CAST(NULL AS DECIMAL(10, 3))")
.addRoundTrip("NUMERIC(38, 9)", "CAST(NULL AS DECIMAL(38, 9))", createDecimalType(38, 9), "CAST(NULL AS DECIMAL(38, 9))")
.execute(getQueryRunner(), bigqueryCreateAndTrinoInsert("test.writenumeric"));
}

@Test
public void testNumericMappingView()
{
Expand Down Expand Up @@ -307,6 +338,35 @@ public void testBigNumericMapping()
// TODO (https://github.com/trinodb/trino/pull/12210) Add support for bigquery type in views
}

@Test
public void testBigNumericWriteMapping()
{
SqlDataTypeTest.create()
.addRoundTrip("BIGNUMERIC(3, 0)", "CAST(193 AS DECIMAL(3, 0))", createDecimalType(3, 0), "CAST(193 AS DECIMAL(3, 0))")
.addRoundTrip("BIGNUMERIC(3, 0)", "CAST(19 AS DECIMAL(3, 0))", createDecimalType(3, 0), "CAST(19 AS DECIMAL(3, 0))")
.addRoundTrip("BIGNUMERIC(3, 0)", "CAST(-193 AS DECIMAL(3, 0))", createDecimalType(3, 0), "CAST(-193 AS DECIMAL(3, 0))")
.addRoundTrip("BIGNUMERIC(3, 1)", "CAST(10.0 AS DECIMAL(3, 1))", createDecimalType(3, 1), "CAST(10.0 AS DECIMAL(3, 1))")
.addRoundTrip("BIGNUMERIC(3, 1)", "CAST(10.1 AS DECIMAL(3, 1))", createDecimalType(3, 1), "CAST(10.1 AS DECIMAL(3, 1))")
.addRoundTrip("BIGNUMERIC(3, 1)", "CAST(-10.1 AS DECIMAL(3, 1))", createDecimalType(3, 1), "CAST(-10.1 AS DECIMAL(3, 1))")
.addRoundTrip("BIGNUMERIC(4, 2)", "CAST(2 AS DECIMAL(4, 2))", createDecimalType(4, 2), "CAST(2 AS DECIMAL(4, 2))")
.addRoundTrip("BIGNUMERIC(4, 2)", "CAST(2.3 AS DECIMAL(4, 2))", createDecimalType(4, 2), "CAST(2.3 AS DECIMAL(4, 2))")
.addRoundTrip("BIGNUMERIC(24, 2)", "CAST(2 AS DECIMAL(24, 2))", createDecimalType(24, 2), "CAST(2 AS DECIMAL(24, 2))")
.addRoundTrip("BIGNUMERIC(24, 2)", "CAST(2.3 AS DECIMAL(24, 2))", createDecimalType(24, 2), "CAST(2.3 AS DECIMAL(24, 2))")
.addRoundTrip("BIGNUMERIC(24, 2)", "CAST(123456789.3 AS DECIMAL(24, 2))", createDecimalType(24, 2), "CAST(123456789.3 AS DECIMAL(24, 2))")
.addRoundTrip("BIGNUMERIC(24, 4)", "CAST(12345678901234567890.31 AS DECIMAL(24, 4))", createDecimalType(24, 4), "CAST(12345678901234567890.31 AS DECIMAL(24, 4))")
.addRoundTrip("BIGNUMERIC(29, 0)", "CAST('27182818284590452353602874713' AS DECIMAL(29, 0))", createDecimalType(29, 0), "CAST('27182818284590452353602874713' AS DECIMAL(29, 0))")
.addRoundTrip("BIGNUMERIC(29, 0)", "CAST('-27182818284590452353602874713' AS DECIMAL(29, 0))", createDecimalType(29, 0), "CAST('-27182818284590452353602874713' AS DECIMAL(29, 0))")
.addRoundTrip("BIGNUMERIC(30, 5)", "CAST(3141592653589793238462643.38327 AS DECIMAL(30, 5))", createDecimalType(30, 5), "CAST(3141592653589793238462643.38327 AS DECIMAL(30, 5))")
.addRoundTrip("BIGNUMERIC(30, 5)", "CAST(-3141592653589793238462643.38327 AS DECIMAL(30, 5))", createDecimalType(30, 5), "CAST(-3141592653589793238462643.38327 AS DECIMAL(30, 5))")
.addRoundTrip("BIGNUMERIC(38, 9)", "CAST(100000000020000000001234567.123456789 AS DECIMAL(38, 9))", createDecimalType(38, 9), "CAST(100000000020000000001234567.123456789 AS DECIMAL(38, 9))")
.addRoundTrip("BIGNUMERIC(38, 9)", "CAST(-100000000020000000001234567.123456789 AS DECIMAL(38, 9))", createDecimalType(38, 9), "CAST(-100000000020000000001234567.123456789 AS DECIMAL(38, 9))")
.addRoundTrip("BIGNUMERIC(10, 3)", "CAST(NULL AS DECIMAL(10, 3))", createDecimalType(10, 3), "CAST(NULL AS DECIMAL(10, 3))")
.addRoundTrip("BIGNUMERIC(38, 9)", "CAST(NULL AS DECIMAL(38, 9))", createDecimalType(38, 9), "CAST(NULL AS DECIMAL(38, 9))")
.addRoundTrip("BIGNUMERIC(1)", "CAST(1 AS DECIMAL(1, 0))", createDecimalType(1, 0), "CAST(1 AS DECIMAL(1, 0))")
.addRoundTrip("BIGNUMERIC(1)", "CAST(-1 AS DECIMAL(1, 0))", createDecimalType(1, 0), "CAST(-1 AS DECIMAL(1, 0))")
.execute(getQueryRunner(), bigqueryCreateAndTrinoInsert("test.writebignumeric"));
}

@Test
public void testUnsupportedBigNumericMappingView()
{
Expand Down Expand Up @@ -651,6 +711,11 @@ private DataSetup bigqueryCreateAndInsert(String tableNamePrefix)
return new CreateAndInsertDataSetup(getBigQuerySqlExecutor(), tableNamePrefix);
}

private DataSetup bigqueryCreateAndTrinoInsert(String tableNamePrefix)
{
return new CreateAndTrinoInsertDataSetup(getBigQuerySqlExecutor(), new TrinoSqlExecutor(getQueryRunner()), tableNamePrefix);
}

private DataSetup bigqueryViewCreateAndInsert(String tableNamePrefix)
{
return new BigQueryViewCreateAndInsertDataSetup(getBigQuerySqlExecutor(), tableNamePrefix);
Expand Down

0 comments on commit 8cfa24c

Please sign in to comment.