From aa37505f2363fcd248db0d0f1c9abb6dcc663556 Mon Sep 17 00:00:00 2001 From: Clearvive Date: Thu, 25 Jan 2024 19:20:30 +0800 Subject: [PATCH] [#1707] feat(mysql): Support mysql index. --- .../gravitino/rel/indexes/Indexes.java | 13 ++ .../catalog/jdbc/JdbcCatalogOperations.java | 6 +- .../gravitino/catalog/jdbc/JdbcTable.java | 1 + .../jdbc/operation/JdbcTableOperations.java | 9 +- .../jdbc/operation/TableOperation.java | 5 +- .../jdbc/operation/SqliteTableOperations.java | 4 +- .../operation/TestJdbcTableOperations.java | 3 +- .../mysql/operation/MysqlTableOperations.java | 102 +++++++++++++++- .../operation/PostgreSqlTableOperations.java | 4 +- .../client/TestRelationalCatalog.java | 8 +- .../gravitino/dto/rel/TableDTO.java | 9 +- .../gravitino/dto/rel/indexes/IndexDTO.java | 9 +- .../dto/requests/TableCreateRequest.java | 14 ++- .../gravitino/dto/util/DTOConverters.java | 16 ++- .../datastrato/gravitino/json/JsonUtils.java | 77 +++++++----- .../test/catalog/jdbc/TestJdbcAbstractIT.java | 24 ++++ .../catalog/jdbc/mysql/CatalogMysqlIT.java | 113 ++++++++++++++++++ .../jdbc/mysql/TestMysqlTableOperations.java | 65 ++++++---- .../TestPostgreSqlTableOperations.java | 33 +++-- .../integration/test/util/AbstractIT.java | 49 ++++++++ .../server/web/rest/TableOperations.java | 3 +- .../server/web/rest/TestTableOperations.java | 10 +- 22 files changed, 483 insertions(+), 94 deletions(-) diff --git a/api/src/main/java/com/datastrato/gravitino/rel/indexes/Indexes.java b/api/src/main/java/com/datastrato/gravitino/rel/indexes/Indexes.java index 8e7d5a1395c..b1164378548 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/indexes/Indexes.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/indexes/Indexes.java @@ -9,6 +9,9 @@ public class Indexes { public static final Index[] EMPTY_INDEXES = new Index[0]; + /** MySQL does not support setting the name of the primary key, so the default name is used. */ + public static final String DEFAULT_MYSQL_PRIMARY_KEY_NAME = "PRIMARY"; + /** * Create a unique index on columns. Like unique (a) or unique (a, b), for complex like unique * @@ -20,6 +23,16 @@ public static Index unique(String name, String[][] fieldNames) { return of(Index.IndexType.UNIQUE_KEY, name, fieldNames); } + /** + * To create a MySQL primary key, you need to use the default primary key name. + * + * @param fieldNames The field names under the table contained in the index. + * @return The primary key index + */ + public static Index createMysqlPrimaryKey(String[][] fieldNames) { + return primary(DEFAULT_MYSQL_PRIMARY_KEY_NAME, fieldNames); + } + /** * Create a primary index on columns. Like primary (a), for complex like primary * diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java index 0d05f91990d..38f77070413 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java @@ -291,6 +291,7 @@ public Table loadTable(NameIdentifier tableIdent) throws NoSuchTableException { // Remove id from comment .withComment(StringIdentifier.removeIdFromComment(load.comment())) .withProperties(properties) + .withIndexes(load.index()) .build(); } @@ -364,7 +365,6 @@ public Table createTable( SortOrder[] sortOrders, Index[] indexes) throws NoSuchSchemaException, TableAlreadyExistsException { - Preconditions.checkArgument(indexes.length == 0, "Jdbc-catalog does not support indexes"); Preconditions.checkArgument( null == distribution || distribution == Distributions.NONE, "jdbc-catalog does not support distribution"); @@ -399,7 +399,8 @@ public Table createTable( jdbcColumns, StringIdentifier.addToComment(identifier, comment), resultProperties, - partitioning); + partitioning, + indexes); return new JdbcTable.Builder() .withAuditInfo( @@ -412,6 +413,7 @@ public Table createTable( .withComment(comment) .withProperties(jdbcTablePropertiesMetadata.convertFromJdbcProperties(resultProperties)) .withPartitioning(partitioning) + .withIndexes(indexes) .build(); } diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcTable.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcTable.java index 73092668571..7eec9792a63 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcTable.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcTable.java @@ -34,6 +34,7 @@ protected JdbcTable internalBuild() { jdbcTable.columns = columns; jdbcTable.partitioning = partitioning; jdbcTable.sortOrders = sortOrders; + jdbcTable.indexes = indexes; return jdbcTable; } } diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java index 8c6c7fffcec..ebaa725c616 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java @@ -15,6 +15,7 @@ import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.indexes.Index; import com.google.common.collect.Lists; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -63,13 +64,14 @@ public void create( JdbcColumn[] columns, String comment, Map properties, - Transform[] partitioning) + Transform[] partitioning, + Index[] indexes) throws TableAlreadyExistsException { LOG.info("Attempting to create table {} in database {}", tableName, databaseName); try (Connection connection = getConnection(databaseName)) { JdbcConnectorUtils.executeUpdate( connection, - generateCreateTableSql(tableName, columns, comment, properties, partitioning)); + generateCreateTableSql(tableName, columns, comment, properties, partitioning, indexes)); LOG.info("Created table {} in database {}", tableName, databaseName); } catch (final SQLException se) { throw this.exceptionMapper.toGravitinoException(se); @@ -215,7 +217,8 @@ protected abstract String generateCreateTableSql( JdbcColumn[] columns, String comment, Map properties, - Transform[] partitioning); + Transform[] partitioning, + Index[] indexes); protected abstract String generateRenameTableSql(String oldTableName, String newTableName); diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/TableOperation.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/TableOperation.java index 71633a4b832..b6c832992ad 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/TableOperation.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/TableOperation.java @@ -14,6 +14,7 @@ import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.indexes.Index; import java.util.List; import java.util.Map; import javax.sql.DataSource; @@ -41,6 +42,7 @@ void initialize( * @param comment The comment of the table. * @param properties The properties of the table. * @param partitioning The partitioning of the table. + * @param indexes The indexes of the table. */ void create( String databaseName, @@ -48,7 +50,8 @@ void create( JdbcColumn[] columns, String comment, Map properties, - Transform[] partitioning) + Transform[] partitioning, + Index[] indexes) throws TableAlreadyExistsException; /** diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/SqliteTableOperations.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/SqliteTableOperations.java index 1e28692e5be..320a736df70 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/SqliteTableOperations.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/SqliteTableOperations.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.catalog.jdbc.JdbcColumn; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.indexes.Index; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collections; @@ -20,7 +21,8 @@ protected String generateCreateTableSql( JdbcColumn[] columns, String comment, Map properties, - Transform[] partitioning) { + Transform[] partitioning, + Index[] indexes) { StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("CREATE TABLE ").append(tableName).append(" ("); diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java index 1e3608641b7..6c1a63f4886 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java @@ -15,6 +15,7 @@ import com.datastrato.gravitino.exceptions.NoSuchTableException; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.indexes.Indexes; import com.datastrato.gravitino.rel.types.Type; import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.Maps; @@ -121,7 +122,7 @@ public void testOperationTable() { Assertions.assertDoesNotThrow( () -> JDBC_TABLE_OPERATIONS.create( - DATABASE_NAME, table1, jdbcColumns, null, properties, null)); + DATABASE_NAME, table1, jdbcColumns, null, properties, null, Indexes.EMPTY_INDEXES)); // list table. List allTables = JDBC_TABLE_OPERATIONS.listTables(DATABASE_NAME); diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java index 2f5c73ab559..38511065087 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java @@ -19,6 +19,8 @@ import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.indexes.Index; +import com.datastrato.gravitino.rel.indexes.Indexes; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.PreparedStatement; @@ -29,8 +31,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; @@ -87,6 +92,9 @@ public JdbcTable load(String databaseName, String tableName) throws NoSuchTableE comment = tableProperties.getOrDefault(COMMENT, comment); } + // 4.Get index information + List indexes = getIndexes(databaseName, tableName, metaData); + return new JdbcTable.Builder() .withName(tableName) .withColumns(jdbcColumns.toArray(new JdbcColumn[0])) @@ -102,6 +110,7 @@ public JdbcTable load(String databaseName, String tableName) throws NoSuchTableE } } })) + .withIndexes(indexes.toArray(new Index[0])) .withAuditInfo(AuditInfo.EMPTY) .build(); } catch (SQLException e) { @@ -109,6 +118,63 @@ public JdbcTable load(String databaseName, String tableName) throws NoSuchTableE } } + private List getIndexes(String databaseName, String tableName, DatabaseMetaData metaData) + throws SQLException { + List indexes = new ArrayList<>(); + // 1.Get primary key information + Map> primaryKeyGroupByName = new HashMap<>(); + ResultSet primaryKeys = metaData.getPrimaryKeys(databaseName, null, tableName); + while (primaryKeys.next()) { + String columnName = primaryKeys.getString("COLUMN_NAME"); + primaryKeyGroupByName.compute( + primaryKeys.getString("PK_NAME"), + (k, v) -> { + if (v == null) { + v = new HashSet<>(); + } + v.add(columnName); + return v; + }); + } + for (Map.Entry> entry : primaryKeyGroupByName.entrySet()) { + indexes.add( + Indexes.primary( + entry.getKey(), convertIndexFieldNames(entry.getValue().toArray(new String[0])))); + } + + // 2.Get unique key information + Map> indexGroupByName = new HashMap<>(); + ResultSet indexInfo = metaData.getIndexInfo(databaseName, null, tableName, false, false); + while (indexInfo.next()) { + String indexName = indexInfo.getString("INDEX_NAME"); + if (!indexInfo.getBoolean("NON_UNIQUE") + && !StringUtils.equalsIgnoreCase(Indexes.DEFAULT_MYSQL_PRIMARY_KEY_NAME, indexName)) { + String columnName = indexInfo.getString("COLUMN_NAME"); + indexGroupByName.compute( + indexName, + (k, v) -> { + if (v == null) { + v = new HashSet<>(); + } + v.add(columnName); + return v; + }); + } + } + for (Map.Entry> entry : indexGroupByName.entrySet()) { + indexes.add( + Indexes.unique( + entry.getKey(), convertIndexFieldNames(entry.getValue().toArray(new String[0])))); + } + return indexes; + } + + private String[][] convertIndexFieldNames(String[] fieldNames) { + return Arrays.stream(fieldNames) + .map(colName -> new String[] {colName}) + .toArray(String[][]::new); + } + private Map loadTablePropertiesFromSql(Connection connection, String tableName) throws SQLException { try (PreparedStatement statement = connection.prepareStatement("SHOW TABLE STATUS LIKE ?")) { @@ -178,7 +244,8 @@ protected String generateCreateTableSql( JdbcColumn[] columns, String comment, Map properties, - Transform[] partitioning) { + Transform[] partitioning, + Index[] indexes) { if (ArrayUtils.isNotEmpty(partitioning)) { throw new UnsupportedOperationException("Currently we do not support Partitioning in mysql"); } @@ -201,6 +268,39 @@ protected String generateCreateTableSql( sqlBuilder.append(",\n"); } } + + for (Index index : indexes) { + String fieldStr = + Arrays.stream(index.fieldNames()) + .map( + colNames -> { + if (colNames.length > 1) { + throw new IllegalArgumentException( + "Index does not support complex fields in MySQL"); + } + return colNames[0]; + }) + .collect(Collectors.joining(", ")); + sqlBuilder.append(",\n"); + switch (index.type()) { + case PRIMARY_KEY: + if (null != index.name() && !"PRIMARY".equals(index.name().toUpperCase(Locale.ROOT))) { + throw new IllegalArgumentException("Primary key name must be PRIMARY in MySQL"); + } + sqlBuilder.append("CONSTRAINT ").append("PRIMARY KEY (").append(fieldStr).append(")"); + break; + case UNIQUE_KEY: + sqlBuilder.append("CONSTRAINT "); + if (null != index.name()) { + sqlBuilder.append(index.name()); + } + sqlBuilder.append(" UNIQUE (").append(fieldStr).append(")"); + break; + default: + throw new IllegalArgumentException("MySQL doesn't support index : " + index.type()); + } + } + sqlBuilder.append("\n)"); // Add table comment if specified diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java index 9cc9f976b74..fff44aa3019 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java @@ -16,6 +16,7 @@ import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.indexes.Index; import com.datastrato.gravitino.rel.types.Types; import com.google.common.base.Preconditions; import java.sql.Connection; @@ -225,7 +226,8 @@ protected String generateCreateTableSql( JdbcColumn[] columns, String comment, Map properties, - Transform[] partitioning) { + Transform[] partitioning, + Index[] indexes) { if (ArrayUtils.isNotEmpty(partitioning)) { throw new UnsupportedOperationException( "Currently we do not support Partitioning in PostgreSQL"); diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java index 782e310ce8f..75eadcbc049 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalCatalog.java @@ -26,6 +26,7 @@ import com.datastrato.gravitino.dto.rel.expressions.FieldReferenceDTO; import com.datastrato.gravitino.dto.rel.expressions.FunctionArg; import com.datastrato.gravitino.dto.rel.expressions.LiteralDTO; +import com.datastrato.gravitino.dto.rel.indexes.IndexDTO; import com.datastrato.gravitino.dto.rel.partitioning.DayPartitioningDTO; import com.datastrato.gravitino.dto.rel.partitioning.IdentityPartitioningDTO; import com.datastrato.gravitino.dto.rel.partitioning.Partitioning; @@ -55,7 +56,6 @@ import com.datastrato.gravitino.rel.expressions.distributions.Strategy; import com.datastrato.gravitino.rel.expressions.sorts.SortDirection; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; -import com.datastrato.gravitino.rel.indexes.Indexes; import com.datastrato.gravitino.rel.types.Type; import com.datastrato.gravitino.rel.types.Types; import com.fasterxml.jackson.core.JsonProcessingException; @@ -362,7 +362,7 @@ public void testCreateTable() throws JsonProcessingException { sortOrderDTOs, DistributionDTO.NONE, EMPTY_PARTITIONING, - Indexes.EMPTY_INDEXES); + IndexDTO.EMPTY_INDEXES); TableResponse resp = new TableResponse(expectedTable); buildMockResource(Method.POST, tablePath, req, resp, SC_OK); @@ -481,7 +481,7 @@ public void testCreatePartitionedTable() throws JsonProcessingException { SortOrderDTO.EMPTY_SORT, DistributionDTO.NONE, EMPTY_PARTITIONING, - Indexes.EMPTY_INDEXES); + IndexDTO.EMPTY_INDEXES); TableResponse resp = new TableResponse(expectedTable); buildMockResource(Method.POST, tablePath, req, resp, SC_OK); @@ -514,7 +514,7 @@ public void testCreatePartitionedTable() throws JsonProcessingException { SortOrderDTO.EMPTY_SORT, DistributionDTO.NONE, partitioning, - Indexes.EMPTY_INDEXES); + IndexDTO.EMPTY_INDEXES); resp = new TableResponse(expectedTable); buildMockResource(Method.POST, tablePath, req, resp, SC_OK); diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java index f2e5f3cbf6d..cb893ef63f2 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.dto.rel; import com.datastrato.gravitino.dto.AuditDTO; +import com.datastrato.gravitino.dto.rel.indexes.IndexDTO; import com.datastrato.gravitino.dto.rel.partitioning.Partitioning; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Table; @@ -44,7 +45,7 @@ public class TableDTO implements Table { private Partitioning[] partitioning; @JsonProperty("indexes") - private Index[] indexes; + private IndexDTO[] indexes; private TableDTO() {} @@ -68,7 +69,7 @@ private TableDTO( Partitioning[] partitioning, DistributionDTO distribution, SortOrderDTO[] sortOrderDTOs, - Index[] indexes) { + IndexDTO[] indexes) { this.name = name; this.comment = comment; this.columns = columns; @@ -148,7 +149,7 @@ public static class Builder { protected SortOrderDTO[] sortOrderDTOs; protected DistributionDTO distributionDTO; protected Partitioning[] Partitioning; - protected Index[] indexes; + protected IndexDTO[] indexes; public Builder() {} @@ -222,7 +223,7 @@ public S withPartitioning(Partitioning[] Partitioning) { return (S) this; } - public S withIndex(Index[] indexes) { + public S withIndex(IndexDTO[] indexes) { this.indexes = indexes; return (S) this; } diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/IndexDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/IndexDTO.java index 1297b599046..1d64d983223 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/IndexDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/IndexDTO.java @@ -4,12 +4,18 @@ */ package com.datastrato.gravitino.dto.rel.indexes; +import com.datastrato.gravitino.json.JsonUtils; import com.datastrato.gravitino.rel.indexes.Index; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.base.Preconditions; -import org.apache.commons.lang3.StringUtils; +@JsonSerialize(using = JsonUtils.IndexSerializer.class) +@JsonDeserialize(using = JsonUtils.IndexDeserializer.class) public class IndexDTO implements Index { + public static final IndexDTO[] EMPTY_INDEXES = new IndexDTO[0]; + private IndexType indexType; private String name; private String[][] fieldNames; @@ -67,7 +73,6 @@ public S withFieldNames(String[][] fieldNames) { public IndexDTO build() { Preconditions.checkArgument(indexType != null, "Index type cannot be null"); - Preconditions.checkArgument(StringUtils.isNotBlank(name), "Index name cannot be blank"); Preconditions.checkArgument( fieldNames != null && fieldNames.length > 0, "The index must be set with corresponding column names"); diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TableCreateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TableCreateRequest.java index 2e0571ac5f4..e7636a592fd 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TableCreateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TableCreateRequest.java @@ -8,8 +8,8 @@ import com.datastrato.gravitino.dto.rel.DistributionDTO; import com.datastrato.gravitino.dto.rel.SortOrderDTO; import com.datastrato.gravitino.dto.rel.expressions.FunctionArg; +import com.datastrato.gravitino.dto.rel.indexes.IndexDTO; import com.datastrato.gravitino.dto.rel.partitioning.Partitioning; -import com.datastrato.gravitino.rel.indexes.Index; import com.datastrato.gravitino.rest.RESTRequest; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; @@ -60,7 +60,7 @@ public class TableCreateRequest implements RESTRequest { @Nullable @JsonProperty("indexes") - private final Index[] indexes; + private final IndexDTO[] indexes; public TableCreateRequest() { this(null, null, null, null, null, null, null, null); @@ -74,7 +74,7 @@ public TableCreateRequest( @Nullable SortOrderDTO[] sortOrders, @Nullable DistributionDTO distribution, @Nullable Partitioning[] partitioning, - @Nullable Index[] indexes) { + @Nullable IndexDTO[] indexes) { this.name = name; this.columns = columns; this.comment = comment; @@ -119,7 +119,13 @@ public void validate() throws IllegalArgumentException { + autoIncrementColsStr); if (indexes != null && indexes.length > 0) { - throw new UnsupportedOperationException("Support for indexing is currently not implemented"); + Arrays.stream(indexes) + .forEach( + index -> { + Preconditions.checkArgument(index.type() != null, "Index type cannot be null"); + Preconditions.checkArgument( + index.fieldNames().length > 0, "Index field names cannot be null"); + }); } } } diff --git a/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java b/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java index 2632ae587de..4b398df7494 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java @@ -49,6 +49,7 @@ import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.expressions.transforms.Transforms; import com.datastrato.gravitino.rel.indexes.Index; +import com.datastrato.gravitino.rel.indexes.Indexes; import java.util.Arrays; import org.apache.commons.lang3.ArrayUtils; @@ -120,6 +121,7 @@ public static TableDTO toDTO(Table table) { .withDistribution(toDTO(table.distribution())) .withAudit(toDTO(table.auditInfo())) .withPartitioning(toDTOs(table.partitioning())) + .withIndex(toDTOs(table.index())) .build(); } @@ -261,7 +263,7 @@ public static Partitioning[] toDTOs(Transform[] transforms) { public static IndexDTO[] toDTOs(Index[] indexes) { if (ArrayUtils.isEmpty(indexes)) { - return new IndexDTO[0]; + return IndexDTO.EMPTY_INDEXES; } return Arrays.stream(indexes).map(DTOConverters::toDTO).toArray(IndexDTO[]::new); } @@ -302,6 +304,18 @@ public static Expression fromFunctionArg(FunctionArg arg) { } } + public static Index fromDTO(IndexDTO indexDTO) { + return Indexes.of(indexDTO.type(), indexDTO.name(), indexDTO.fieldNames()); + } + + public static Index[] fromDTOs(IndexDTO[] indexDTOS) { + if (ArrayUtils.isEmpty(indexDTOS)) { + return Indexes.EMPTY_INDEXES; + } + + return Arrays.stream(indexDTOS).map(DTOConverters::fromDTO).toArray(Index[]::new); + } + public static SortOrder fromDTO(SortOrderDTO sortOrderDTO) { return SortOrders.of( fromFunctionArg(sortOrderDTO.sortTerm()), diff --git a/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java b/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java index 024f61b8f77..0d7f542aa13 100644 --- a/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java +++ b/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java @@ -23,10 +23,6 @@ import com.datastrato.gravitino.dto.rel.partitioning.RangePartitioningDTO; import com.datastrato.gravitino.dto.rel.partitioning.TruncatePartitioningDTO; import com.datastrato.gravitino.dto.rel.partitioning.YearPartitioningDTO; -import com.datastrato.gravitino.dto.rel.partitions.IdentityPartitionDTO; -import com.datastrato.gravitino.dto.rel.partitions.ListPartitionDTO; -import com.datastrato.gravitino.dto.rel.partitions.PartitionDTO; -import com.datastrato.gravitino.dto.rel.partitions.RangePartitionDTO; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.Expression; @@ -58,7 +54,6 @@ import java.io.IOException; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.slf4j.Logger; @@ -86,6 +81,10 @@ public class JsonUtils { private static final String SORT_TERM = "sortTerm"; private static final String DIRECTION = "direction"; private static final String NULL_ORDERING = "nullOrdering"; + + private static final String INDEX_TYPE = "indexType"; + private static final String INDEX_NAME = "name"; + private static final String INDEX_FIELD_NAMES = "fieldNames"; private static final String NUMBER = "number"; private static final String TYPE = "type"; private static final String STRUCT = "struct"; @@ -369,29 +368,6 @@ public static String getString(String property, JsonNode node) { return convertToString(property, pNode); } - private static String getStringOrNull(String property, JsonNode node) { - if (!node.has(property) || node.get(property).isNull()) { - return null; - } - - return getString(property, node); - } - - private static Map getStringMapOrNull(String property, JsonNode node) { - if (!node.has(property) || node.get(property).isNull()) { - return null; - } - - JsonNode propertiesNode = node.get(property); - Iterator> fieldsIterator = propertiesNode.fields(); - Map properties = Maps.newHashMap(); - while (fieldsIterator.hasNext()) { - Map.Entry field = fieldsIterator.next(); - properties.put(field.getKey(), field.getValue().asText()); - } - return properties; - } - private static String convertToString(String property, JsonNode pNode) { Preconditions.checkArgument( pNode != null && !pNode.isNull() && pNode.isTextual(), @@ -1096,4 +1072,49 @@ public PartitionDTO deserialize(JsonParser p, DeserializationContext ctxt) throw } } } + + public static class IndexSerializer extends JsonSerializer { + @Override + public void serialize(Index value, JsonGenerator gen, SerializerProvider serializers) + throws IOException { + gen.writeStartObject(); + gen.writeStringField(INDEX_TYPE, value.type().name().toUpperCase(Locale.ROOT)); + if (null != value.name()) { + gen.writeStringField(INDEX_NAME, value.name()); + } + gen.writeArrayFieldStart(INDEX_FIELD_NAMES); + for (String[] fieldName : value.fieldNames()) { + gen.writeArray(fieldName, 0, fieldName.length); + } + gen.writeEndArray(); + gen.writeEndObject(); + } + } + + public static class IndexDeserializer extends JsonDeserializer { + @Override + public Index deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + JsonNode node = p.getCodec().readTree(p); + Preconditions.checkArgument( + node != null && !node.isNull() && node.isObject(), + "Cannot parse index from invalid JSON: %s", + node); + + IndexDTO.Builder builder = new IndexDTO.Builder(); + Preconditions.checkArgument( + node.has(INDEX_TYPE), "Cannot parse index from missing name: %s", node); + String indexType = getString(INDEX_TYPE, node); + builder.withIndexType(Index.IndexType.valueOf(indexType.toUpperCase(Locale.ROOT))); + if (node.has(INDEX_NAME)) { + builder.withName(getString(INDEX_NAME, node)); + } + Preconditions.checkArgument( + node.has(INDEX_FIELD_NAMES), "Cannot parse index from missing field names: %s", node); + List fieldNames = Lists.newArrayList(); + node.get(INDEX_FIELD_NAMES) + .forEach(field -> fieldNames.add(getStringArray((ArrayNode) field))); + builder.withFieldNames(fieldNames.toArray(new String[0][0])); + return builder.build(); + } + } } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestJdbcAbstractIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestJdbcAbstractIT.java index 91af2c24dc8..ff7938bee05 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestJdbcAbstractIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestJdbcAbstractIT.java @@ -14,10 +14,15 @@ import com.datastrato.gravitino.catalog.jdbc.utils.DataSourceUtils; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; +import com.datastrato.gravitino.rel.indexes.Index; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import javax.sql.DataSource; +import org.apache.commons.lang3.ArrayUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.testcontainers.containers.JdbcDatabaseContainer; @@ -75,6 +80,7 @@ protected static void assertionsTableInfo( String tableComment, List columns, Map properties, + Index[] indexes, JdbcTable table) { Assertions.assertEquals(tableName, table.name()); Assertions.assertEquals(tableComment, table.comment()); @@ -96,6 +102,24 @@ protected static void assertionsTableInfo( for (Map.Entry entry : properties.entrySet()) { Assertions.assertEquals(entry.getValue(), table.properties().get(entry.getKey())); } + if (ArrayUtils.isNotEmpty(indexes)) { + Assertions.assertEquals(indexes.length, table.index().length); + + Map indexByName = + Arrays.stream(indexes).collect(Collectors.toMap(Index::name, index -> index)); + + for (int i = 0; i < table.index().length; i++) { + Assertions.assertTrue(indexByName.containsKey(table.index()[i].name())); + Assertions.assertEquals( + indexByName.get(table.index()[i].name()).type(), table.index()[i].type()); + for (int j = 0; j < table.index()[i].fieldNames().length; j++) { + Set colNames = + Arrays.stream(indexByName.get(table.index()[i].name()).fieldNames()[j]) + .collect(Collectors.toSet()); + colNames.containsAll(Arrays.asList(table.index()[i].fieldNames()[j])); + } + } + } } @AfterAll diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java index 9b4aabeeeaf..283d379dddd 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java @@ -32,6 +32,8 @@ import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.expressions.transforms.Transforms; +import com.datastrato.gravitino.rel.indexes.Index; +import com.datastrato.gravitino.rel.indexes.Indexes; import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; @@ -569,4 +571,115 @@ void testDropMySQLDatabase() { .asSchemas() .loadSchema(NameIdentifier.of(metalakeName, catalogName, schemaName))); } + + @Test + void testCreateTableIndex() { + ColumnDTO col1 = + new ColumnDTO.Builder() + .withName("col_1") + .withDataType(Types.LongType.get()) + .withComment("id") + .withNullable(false) + .build(); + ColumnDTO col2 = + new ColumnDTO.Builder() + .withName("col_2") + .withDataType(Types.ByteType.get()) + .withNullable(false) + .withComment("yes") + .build(); + ColumnDTO col3 = + new ColumnDTO.Builder() + .withName("col_3") + .withDataType(Types.DateType.get()) + .withComment("comment") + .build(); + ColumnDTO col4 = + new ColumnDTO.Builder() + .withName("col_4") + .withDataType(Types.VarCharType.of(255)) + .withComment("code") + .build(); + ColumnDTO col5 = + new ColumnDTO.Builder() + .withName("col_5") + .withDataType(Types.VarCharType.of(255)) + .withComment("config") + .build(); + ColumnDTO[] newColumns = new ColumnDTO[] {col1, col2, col3, col4, col5}; + + Index[] indexes = + new Index[] { + Indexes.createMysqlPrimaryKey(new String[][] {{"col_1"}, {"col_2"}}), + Indexes.unique("u1_key", new String[][] {{"col_2"}, {"col_3"}}), + Indexes.unique("u2_key", new String[][] {{"col_3"}, {"col_4"}}), + Indexes.unique("u3_key", new String[][] {{"col_5"}, {"col_4"}}), + Indexes.unique("u4_key", new String[][] {{"col_2"}, {"col_3"}, {"col_4"}}), + Indexes.unique("u5_key", new String[][] {{"col_2"}, {"col_3"}, {"col_4"}}), + Indexes.unique("u6_key", new String[][] {{"col_1"}, {"col_2"}, {"col_3"}, {"col_4"}}), + }; + + NameIdentifier tableIdentifier = + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName); + + Map properties = createProperties(); + TableCatalog tableCatalog = catalog.asTableCatalog(); + Table createdTable = + tableCatalog.createTable( + tableIdentifier, + newColumns, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + indexes); + assertionsTableInfo( + tableName, table_comment, Arrays.asList(newColumns), properties, indexes, createdTable); + Table table = tableCatalog.loadTable(tableIdentifier); + assertionsTableInfo( + tableName, table_comment, Arrays.asList(newColumns), properties, indexes, table); + + IllegalArgumentException illegalArgumentException = + assertThrows( + IllegalArgumentException.class, + () -> { + tableCatalog.createTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, "test_failed"), + newColumns, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + new Index[] { + Indexes.createMysqlPrimaryKey(new String[][] {{"col_1", "col_2"}}), + Indexes.unique("u1_key", new String[][] {{"col_2", "col_3"}}) + }); + }); + Assertions.assertTrue( + StringUtils.contains( + illegalArgumentException.getMessage(), + "Index does not support complex fields in MySQL")); + + table = + tableCatalog.createTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, "test_null_key"), + newColumns, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + new Index[] { + Indexes.of( + Index.IndexType.UNIQUE_KEY, + null, + new String[][] {{"col_1"}, {"col_3"}, {"col_4"}}), + Indexes.of(Index.IndexType.UNIQUE_KEY, null, new String[][] {{"col_4"}}), + }); + Assertions.assertEquals(2, table.index().length); + Assertions.assertNotNull(table.index()[0].name()); + Assertions.assertNotNull(table.index()[1].name()); + } } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/TestMysqlTableOperations.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/TestMysqlTableOperations.java index d4e26f29697..2a4610f0374 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/TestMysqlTableOperations.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/TestMysqlTableOperations.java @@ -13,6 +13,8 @@ import com.datastrato.gravitino.exceptions.NoSuchTableException; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.indexes.Index; +import com.datastrato.gravitino.rel.indexes.Indexes; import com.datastrato.gravitino.rel.types.Type; import com.datastrato.gravitino.rel.types.Types; import java.util.ArrayList; @@ -63,8 +65,9 @@ public void testOperationTable() { .withNullable(false) .build()); Map properties = new HashMap<>(); - properties.put(MYSQL_ENGINE_KEY, "InnoDB"); properties.put(MYSQL_AUTO_INCREMENT_OFFSET_KEY, "10"); + + Index[] indexes = new Index[] {Indexes.unique("test", new String[][] {{"col_1"}, {"col_2"}})}; // create table TABLE_OPERATIONS.create( TEST_DB_NAME, @@ -72,7 +75,8 @@ public void testOperationTable() { columns.toArray(new JdbcColumn[0]), tableComment, properties, - null); + null, + indexes); // list table List tables = TABLE_OPERATIONS.listTables(TEST_DB_NAME); @@ -80,7 +84,7 @@ public void testOperationTable() { // load table JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); - assertionsTableInfo(tableName, tableComment, columns, properties, load); + assertionsTableInfo(tableName, tableComment, columns, properties, indexes, load); // rename table String newName = "new_table"; @@ -103,8 +107,8 @@ public void testOperationTable() { newColumn.dataType(), newColumn.comment(), TableChange.ColumnPosition.after("col_1")), - TableChange.setProperty(MYSQL_ENGINE_KEY, "MyISAM")); - properties.put(MYSQL_ENGINE_KEY, "MyISAM"); + TableChange.setProperty(MYSQL_ENGINE_KEY, "InnoDB")); + properties.put(MYSQL_ENGINE_KEY, "InnoDB"); load = TABLE_OPERATIONS.load(TEST_DB_NAME, newName); List alterColumns = new ArrayList() { @@ -116,7 +120,7 @@ public void testOperationTable() { add(columns.get(3)); } }; - assertionsTableInfo(newName, tableComment, alterColumns, properties, load); + assertionsTableInfo(newName, tableComment, alterColumns, properties, indexes, load); // Detect unsupported properties GravitinoRuntimeException gravitinoRuntimeException = @@ -133,7 +137,7 @@ public void testOperationTable() { TABLE_OPERATIONS.alterTable( TEST_DB_NAME, newName, TableChange.deleteColumn(new String[] {newColumn.name()}, true)); load = TABLE_OPERATIONS.load(TEST_DB_NAME, newName); - assertionsTableInfo(newName, tableComment, columns, properties, load); + assertionsTableInfo(newName, tableComment, columns, properties, indexes, load); IllegalArgumentException illegalArgumentException = Assertions.assertThrows( @@ -184,6 +188,11 @@ public void testAlterTable() { columns.add(col_2); Map properties = new HashMap<>(); + Index[] indexes = + new Index[] { + Indexes.createMysqlPrimaryKey(new String[][] {{"col_1"}, {"col_2"}}), + Indexes.unique("uk_2", new String[][] {{"col_1"}, {"col_2"}}) + }; // create table TABLE_OPERATIONS.create( TEST_DB_NAME, @@ -191,9 +200,10 @@ public void testAlterTable() { columns.toArray(new JdbcColumn[0]), tableComment, properties, - null); + null, + indexes); JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); - assertionsTableInfo(tableName, tableComment, columns, properties, load); + assertionsTableInfo(tableName, tableComment, columns, properties, indexes, load); TABLE_OPERATIONS.alterTable( TEST_DB_NAME, @@ -215,7 +225,7 @@ public void testAlterTable() { .build(); columns.add(col_1); columns.add(col_2); - assertionsTableInfo(tableName, tableComment, columns, properties, load); + assertionsTableInfo(tableName, tableComment, columns, properties, indexes, load); String newComment = "new_comment"; // update table comment and column comment @@ -249,7 +259,7 @@ public void testAlterTable() { .build(); columns.add(col_1); columns.add(col_2); - assertionsTableInfo(tableName, tableComment, columns, properties, load); + assertionsTableInfo(tableName, tableComment, columns, properties, indexes, load); String newColName_1 = "new_col_1"; String newColName_2 = "new_col_2"; @@ -285,7 +295,7 @@ public void testAlterTable() { .build(); columns.add(col_1); columns.add(col_2); - assertionsTableInfo(tableName, tableComment, columns, properties, load); + assertionsTableInfo(tableName, tableComment, columns, properties, indexes, load); newComment = "txt3"; String newCol2Comment = "xxx"; @@ -322,7 +332,7 @@ public void testAlterTable() { .build(); columns.add( new JdbcColumn.Builder().withName("col_3").withType(VARCHAR).withComment("txt3").build()); - assertionsTableInfo(tableName, newComment, columns, properties, load); + assertionsTableInfo(tableName, newComment, columns, properties, indexes, load); TABLE_OPERATIONS.alterTable( TEST_DB_NAME, @@ -344,7 +354,7 @@ public void testAlterTable() { .build()); columns.add(col_2); - assertionsTableInfo(tableName, newComment, columns, properties, load); + assertionsTableInfo(tableName, newComment, columns, properties, indexes, load); } @Test @@ -389,6 +399,11 @@ public void testCreateAndLoadTable() { .build()); Map properties = new HashMap<>(); + Index[] indexes = + new Index[] { + Indexes.createMysqlPrimaryKey(new String[][] {{"col_2"}}), + Indexes.unique("uk_col_4", new String[][] {{"col_4"}}) + }; // create table TABLE_OPERATIONS.create( TEST_DB_NAME, @@ -396,10 +411,11 @@ public void testCreateAndLoadTable() { columns.toArray(new JdbcColumn[0]), tableComment, properties, - null); + null, + indexes); JdbcTable loaded = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); - assertionsTableInfo(tableName, tableComment, columns, properties, loaded); + assertionsTableInfo(tableName, tableComment, columns, properties, indexes, loaded); } @Test @@ -493,10 +509,11 @@ public void testCreateAllTypeTable() { columns.toArray(new JdbcColumn[0]), tableComment, Collections.emptyMap(), - null); + null, + Indexes.EMPTY_INDEXES); JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); - assertionsTableInfo(tableName, tableComment, columns, Collections.emptyMap(), load); + assertionsTableInfo(tableName, tableComment, columns, Collections.emptyMap(), null, load); } @Test @@ -533,7 +550,8 @@ public void testCreateNotSupportTypeTable() { columns.toArray(new JdbcColumn[0]), tableComment, Collections.emptyMap(), - null)); + null, + Indexes.EMPTY_INDEXES)); Assertions.assertTrue( illegalArgumentException .getMessage() @@ -559,7 +577,8 @@ public void testCreateMultipleTables() { }, "test_comment", null, - null); + null, + Indexes.EMPTY_INDEXES); String testDb = "test_db_2"; @@ -583,7 +602,8 @@ public void testCreateMultipleTables() { }, "test_comment", null, - null); + null, + Indexes.EMPTY_INDEXES); tables = TABLE_OPERATIONS.listTables(TEST_DB_NAME); Assertions.assertFalse(tables.contains(test_table_2)); @@ -605,7 +625,8 @@ public void testLoadTableDefaultProperties() { }, "test_comment", null, - null); + null, + Indexes.EMPTY_INDEXES); JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME, test_table_1); Assertions.assertEquals("InnoDB", load.properties().get(MYSQL_ENGINE_KEY)); } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java index a98e727d369..035fad28a4d 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java @@ -15,6 +15,7 @@ import com.datastrato.gravitino.exceptions.NoSuchTableException; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.indexes.Indexes; import com.datastrato.gravitino.rel.types.Type; import com.datastrato.gravitino.rel.types.Types; import java.sql.Connection; @@ -75,7 +76,8 @@ public void testOperationTable() { columns.toArray(new JdbcColumn[0]), tableComment, properties, - null); + null, + Indexes.EMPTY_INDEXES); // list table List tables = TABLE_OPERATIONS.listTables(TEST_DB_NAME); @@ -83,7 +85,7 @@ public void testOperationTable() { // load table JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); - assertionsTableInfo(tableName, tableComment, columns, properties, load); + assertionsTableInfo(tableName, tableComment, columns, properties, null, load); // rename table String newName = "new_table"; @@ -126,7 +128,7 @@ public void testOperationTable() { .build()); alterColumns.add(columns.get(3)); alterColumns.add(newColumn); - assertionsTableInfo(newName, tableComment, alterColumns, properties, load); + assertionsTableInfo(newName, tableComment, alterColumns, properties, null, load); TABLE_OPERATIONS.alterTable( TEST_DB_NAME, @@ -152,7 +154,7 @@ public void testOperationTable() { .build()); alterColumns.add(columns.get(3)); alterColumns.add(newColumn); - assertionsTableInfo(newName, tableComment, alterColumns, properties, load); + assertionsTableInfo(newName, tableComment, alterColumns, properties, null, load); // alter column Nullability TABLE_OPERATIONS.alterTable( @@ -176,7 +178,7 @@ public void testOperationTable() { .build()); alterColumns.add(columns.get(3)); alterColumns.add(newColumn); - assertionsTableInfo(newName, tableComment, alterColumns, properties, load); + assertionsTableInfo(newName, tableComment, alterColumns, properties, null, load); // delete column TABLE_OPERATIONS.alterTable( @@ -184,7 +186,7 @@ public void testOperationTable() { load = TABLE_OPERATIONS.load(TEST_DB_NAME, newName); alterColumns.remove(newColumn); - assertionsTableInfo(newName, tableComment, alterColumns, properties, load); + assertionsTableInfo(newName, tableComment, alterColumns, properties, null, load); IllegalArgumentException illegalArgumentException = Assertions.assertThrows( @@ -303,10 +305,11 @@ public void testCreateAllTypeTable() { columns.toArray(new JdbcColumn[0]), tableComment, Collections.emptyMap(), - null); + null, + Indexes.EMPTY_INDEXES); JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); - assertionsTableInfo(tableName, tableComment, columns, Collections.emptyMap(), load); + assertionsTableInfo(tableName, tableComment, columns, Collections.emptyMap(), null, load); } @Test @@ -352,7 +355,8 @@ public void testCreateMultipleTable() throws SQLException { }, null, null, - null); + null, + Indexes.EMPTY_INDEXES); List tableNames = TABLE_OPERATIONS.listTables(TEST_DB_NAME); Assertions.assertFalse(tableNames.contains(table_1)); @@ -379,7 +383,8 @@ public void testCreateMultipleTable() throws SQLException { }, null, null, - null); + null, + Indexes.EMPTY_INDEXES); tableNames = postgreSqlTableOperations.listTables(TEST_DB_NAME); Assertions.assertFalse(tableNames.contains(table_2)); @@ -429,7 +434,8 @@ public void testCreateAutoIncrementTable() { columns.toArray(new JdbcColumn[0]), tableComment, properties, - null); + null, + Indexes.EMPTY_INDEXES); // list table List tables = TABLE_OPERATIONS.listTables(TEST_DB_NAME); @@ -437,7 +443,7 @@ public void testCreateAutoIncrementTable() { // load table JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME, tableName); - assertionsTableInfo(tableName, tableComment, columns, properties, load); + assertionsTableInfo(tableName, tableComment, columns, properties, null, load); columns.clear(); columns.add( @@ -467,7 +473,8 @@ public void testCreateAutoIncrementTable() { columns.toArray(new JdbcColumn[0]), tableComment, properties, - null)); + null, + Indexes.EMPTY_INDEXES)); Assertions.assertTrue( StringUtils.contains(illegalArgumentException.getMessage(), "Unsupported auto-increment")); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java index f2a6dcfe08e..26b7b3b49c9 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java @@ -12,6 +12,9 @@ import com.datastrato.gravitino.client.GravitinoClient; import com.datastrato.gravitino.integration.test.MiniGravitino; import com.datastrato.gravitino.integration.test.MiniGravitinoContext; +import com.datastrato.gravitino.rel.Column; +import com.datastrato.gravitino.rel.Table; +import com.datastrato.gravitino.rel.indexes.Index; import com.datastrato.gravitino.server.GravitinoServer; import com.datastrato.gravitino.server.ServerConfig; import com.datastrato.gravitino.server.auth.OAuthConfig; @@ -25,11 +28,17 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.Enumeration; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.ArrayUtils; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.Logger; @@ -197,4 +206,44 @@ protected String readGitCommitIdFromGitFile() { return ""; } } + + protected static void assertionsTableInfo( + String tableName, + String tableComment, + List columns, + Map properties, + Index[] indexes, + Table table) { + Assertions.assertEquals(tableName, table.name()); + Assertions.assertEquals(tableComment, table.comment()); + Assertions.assertEquals(columns.size(), table.columns().length); + for (int i = 0; i < columns.size(); i++) { + Assertions.assertEquals(columns.get(i).name(), table.columns()[i].name()); + Assertions.assertEquals(columns.get(i).dataType(), table.columns()[i].dataType()); + Assertions.assertEquals(columns.get(i).nullable(), table.columns()[i].nullable()); + Assertions.assertEquals(columns.get(i).comment(), table.columns()[i].comment()); + Assertions.assertEquals(columns.get(i).autoIncrement(), table.columns()[i].autoIncrement()); + } + for (Map.Entry entry : properties.entrySet()) { + Assertions.assertEquals(entry.getValue(), table.properties().get(entry.getKey())); + } + if (ArrayUtils.isNotEmpty(indexes)) { + Assertions.assertEquals(indexes.length, table.index().length); + + Map indexByName = + Arrays.stream(indexes).collect(Collectors.toMap(Index::name, index -> index)); + + for (int i = 0; i < table.index().length; i++) { + Assertions.assertTrue(indexByName.containsKey(table.index()[i].name())); + Assertions.assertEquals( + indexByName.get(table.index()[i].name()).type(), table.index()[i].type()); + for (int j = 0; j < table.index()[i].fieldNames().length; j++) { + Set colNames = + Arrays.stream(indexByName.get(table.index()[i].name()).fieldNames()[j]) + .collect(Collectors.toSet()); + colNames.containsAll(Arrays.asList(table.index()[i].fieldNames()[j])); + } + } + } + } } diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java index f11714d2ac6..ee075d9474d 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java @@ -100,7 +100,8 @@ public Response createTable( request.getProperties(), fromDTOs(request.getPartitioning()), fromDTO(request.getDistribution()), - fromDTOs(request.getSortOrders())); + fromDTOs(request.getSortOrders()), + fromDTOs(request.getIndexes())); return Utils.ok(new TableResponse(DTOConverters.toDTO(table))); }); diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java index cf1faeca439..453209e7ff2 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java @@ -18,6 +18,7 @@ import com.datastrato.gravitino.dto.rel.SortOrderDTO; import com.datastrato.gravitino.dto.rel.TableDTO; import com.datastrato.gravitino.dto.rel.expressions.FieldReferenceDTO; +import com.datastrato.gravitino.dto.rel.indexes.IndexDTO; import com.datastrato.gravitino.dto.rel.partitioning.IdentityPartitioningDTO; import com.datastrato.gravitino.dto.rel.partitioning.Partitioning; import com.datastrato.gravitino.dto.requests.TableCreateRequest; @@ -41,7 +42,6 @@ import com.datastrato.gravitino.rel.expressions.sorts.SortDirection; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.transforms.Transform; -import com.datastrato.gravitino.rel.indexes.Indexes; import com.datastrato.gravitino.rel.types.Type; import com.datastrato.gravitino.rel.types.Types; import com.datastrato.gravitino.rest.RESTUtils; @@ -199,7 +199,7 @@ public void testCreateTable() { sortOrderDTOs, distributionDTO, Partitioning.EMPTY_PARTITIONING, - Indexes.EMPTY_INDEXES); + IndexDTO.EMPTY_INDEXES); Response resp = target(tablePath(metalake, catalog, schema)) @@ -240,7 +240,7 @@ public void testCreateTable() { sortOrderDTOs, distributionDTO, Partitioning.EMPTY_PARTITIONING, - Indexes.EMPTY_INDEXES); + IndexDTO.EMPTY_INDEXES); resp = target(tablePath(metalake, catalog, schema)) @@ -332,7 +332,7 @@ public void testCreatePartitionedTable() { SortOrderDTO.EMPTY_SORT, DistributionDTO.NONE, partitioning, - Indexes.EMPTY_INDEXES); + IndexDTO.EMPTY_INDEXES); Response resp = target(tablePath(metalake, catalog, schema)) @@ -381,7 +381,7 @@ public void testCreatePartitionedTable() { SortOrderDTO.EMPTY_SORT, null, new Partitioning[] {errorPartition}, - Indexes.EMPTY_INDEXES); + IndexDTO.EMPTY_INDEXES); resp = target(tablePath(metalake, catalog, schema)) .request(MediaType.APPLICATION_JSON_TYPE)