From 0fcea0682bdb4155fbbe655ad2d2d3a18d1dbdd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hubert=20=C5=81ojek?= Date: Tue, 25 Apr 2023 17:13:37 +0200 Subject: [PATCH 1/2] Fix returned schema prefixes count When there is more than 100 schemas InformationSchemaMetadata#calculatePrefixesWithSchemaName should return default prefix instead of all schemas --- .../informationschema/InformationSchemaMetadata.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaMetadata.java b/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaMetadata.java index 405096565041..f7423aeaa5de 100644 --- a/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaMetadata.java +++ b/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaMetadata.java @@ -247,9 +247,15 @@ private Set calculatePrefixesWithSchemaName( } Session session = ((FullConnectorSession) connectorSession).getSession(); - return listSchemaNames(session) + Set schemaPrefixes = listSchemaNames(session) .filter(prefix -> predicate.get().test(schemaAsFixedValues(prefix.getSchemaName().get()))) .collect(toImmutableSet()); + if (schemaPrefixes.size() > MAX_PREFIXES_COUNT) { + // in case of high number of prefixes it is better to populate all data and then filter + // TODO this may cause re-running the above filtering upon next applyFilter + return defaultPrefixes(catalogName); + } + return schemaPrefixes; } private Set calculatePrefixesWithTableName( From 33f8815ed6ce1e46390d406a8c2e638e49fe38d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hubert=20=C5=81ojek?= Date: Tue, 25 Apr 2023 18:55:34 +0200 Subject: [PATCH 2/2] Add tests to verify HMS request count for larger schema/table amount --- .../InformationSchemaMetadata.java | 4 +- .../CountingAccessHiveMetastore.java | 11 +- ...astoreMetadataQueriesAccessOperations.java | 511 ++++++++++++++++++ 3 files changed, 523 insertions(+), 3 deletions(-) create mode 100644 plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreMetadataQueriesAccessOperations.java diff --git a/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaMetadata.java b/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaMetadata.java index f7423aeaa5de..38b80322bb0c 100644 --- a/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaMetadata.java +++ b/core/trino-main/src/main/java/io/trino/connector/informationschema/InformationSchemaMetadata.java @@ -13,6 +13,7 @@ */ package io.trino.connector.informationschema; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -79,7 +80,8 @@ public class InformationSchemaMetadata private static final InformationSchemaColumnHandle TABLE_NAME_COLUMN_HANDLE = new InformationSchemaColumnHandle("table_name"); private static final InformationSchemaColumnHandle ROLE_NAME_COLUMN_HANDLE = new InformationSchemaColumnHandle("role_name"); private static final InformationSchemaColumnHandle GRANTEE_COLUMN_HANDLE = new InformationSchemaColumnHandle("grantee"); - private static final int MAX_PREFIXES_COUNT = 100; + @VisibleForTesting + public static final int MAX_PREFIXES_COUNT = 100; private final String catalogName; private final Metadata metadata; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/CountingAccessHiveMetastore.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/CountingAccessHiveMetastore.java index d89f5cac6103..60dba758ea17 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/CountingAccessHiveMetastore.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/CountingAccessHiveMetastore.java @@ -33,6 +33,9 @@ import java.util.Set; import java.util.function.Function; +import static io.trino.plugin.hive.metastore.CountingAccessHiveMetastore.Methods.GET_ALL_TABLES_FROM_DATABASE; +import static io.trino.plugin.hive.metastore.CountingAccessHiveMetastore.Methods.GET_ALL_VIEWS_FROM_DATABASE; + @ThreadSafe public class CountingAccessHiveMetastore implements HiveMetastore @@ -44,8 +47,10 @@ public enum Methods GET_ALL_DATABASES, GET_DATABASE, GET_TABLE, + GET_ALL_TABLES_FROM_DATABASE, GET_TABLE_WITH_PARAMETER, GET_TABLE_STATISTICS, + GET_ALL_VIEWS_FROM_DATABASE, UPDATE_TABLE_STATISTICS, ADD_PARTITIONS, GET_PARTITION_NAMES_BY_FILTER, @@ -113,7 +118,8 @@ public List getTablesWithParameter(String databaseName, String parameter @Override public List getAllViews(String databaseName) { - throw new UnsupportedOperationException(); + methodInvocations.add(GET_ALL_VIEWS_FROM_DATABASE); + return delegate.getAllViews(databaseName); } @Override @@ -341,6 +347,7 @@ public void updatePartitionStatistics(Table table, Map getAllTables(String databaseName) { - throw new UnsupportedOperationException(); + methodInvocations.add(GET_ALL_TABLES_FROM_DATABASE); + return delegate.getAllTables(databaseName); } } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreMetadataQueriesAccessOperations.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreMetadataQueriesAccessOperations.java new file mode 100644 index 000000000000..aa7904267f85 --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/thrift/TestHiveMetastoreMetadataQueriesAccessOperations.java @@ -0,0 +1,511 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.metastore.thrift; + +import com.google.common.collect.HashMultiset; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultiset; +import com.google.common.collect.Multiset; +import io.trino.plugin.hive.HiveType; +import io.trino.plugin.hive.TestingHivePlugin; +import io.trino.plugin.hive.metastore.Column; +import io.trino.plugin.hive.metastore.CountingAccessHiveMetastore; +import io.trino.plugin.hive.metastore.CountingAccessHiveMetastore.Methods; +import io.trino.plugin.hive.metastore.CountingAccessHiveMetastoreUtil; +import io.trino.plugin.hive.metastore.Database; +import io.trino.plugin.hive.metastore.Table; +import io.trino.plugin.hive.metastore.file.FileHiveMetastore; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.QueryRunner; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.Test; + +import java.nio.file.Path; +import java.util.Optional; + +import static io.trino.connector.informationschema.InformationSchemaMetadata.MAX_PREFIXES_COUNT; +import static io.trino.plugin.hive.HiveStorageFormat.ORC; +import static io.trino.plugin.hive.TableType.MANAGED_TABLE; +import static io.trino.plugin.hive.metastore.CountingAccessHiveMetastore.Methods.GET_ALL_DATABASES; +import static io.trino.plugin.hive.metastore.CountingAccessHiveMetastore.Methods.GET_ALL_TABLES_FROM_DATABASE; +import static io.trino.plugin.hive.metastore.CountingAccessHiveMetastore.Methods.GET_ALL_VIEWS_FROM_DATABASE; +import static io.trino.plugin.hive.metastore.CountingAccessHiveMetastore.Methods.GET_TABLE; +import static io.trino.plugin.hive.metastore.PrincipalPrivileges.NO_PRIVILEGES; +import static io.trino.plugin.hive.metastore.StorageFormat.fromHiveStorageFormat; +import static io.trino.plugin.hive.metastore.file.FileHiveMetastore.createTestingFileHiveMetastore; +import static io.trino.testing.TestingSession.testSessionBuilder; +import static org.assertj.core.api.Assertions.assertThat; + +@Test(singleThreaded = true) // metastore invocation counters shares mutable state so can't be run from many threads simultaneously +public class TestHiveMetastoreMetadataQueriesAccessOperations + extends AbstractTestQueryFramework +{ + private static final int TEST_SCHEMAS_COUNT = MAX_PREFIXES_COUNT + 1; + private static final int TEST_TABLES_IN_SCHEMA_COUNT = MAX_PREFIXES_COUNT + 3; + private static final int TEST_ALL_TABLES_COUNT = TEST_SCHEMAS_COUNT * TEST_TABLES_IN_SCHEMA_COUNT; + + private CountingAccessHiveMetastore metastore; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + DistributedQueryRunner queryRunner = DistributedQueryRunner.builder( + testSessionBuilder() + .setCatalog("hive") + .setSchema(Optional.empty()) + .build()) + .build(); + + Path baseDir = queryRunner.getCoordinator().getBaseDataDir().resolve("hive"); + FileHiveMetastore fileHiveMetastore = createTestingFileHiveMetastore(baseDir.toFile()); + metastore = new CountingAccessHiveMetastore(fileHiveMetastore); + + queryRunner.installPlugin(new TestingHivePlugin(metastore)); + queryRunner.createCatalog("hive", "hive", ImmutableMap.of()); + + for (int schemaNumber = 0; schemaNumber < TEST_SCHEMAS_COUNT; schemaNumber++) { + String schemaName = "test_schema_%d".formatted(schemaNumber); + Path schemaPath = baseDir.resolve(schemaName); + fileHiveMetastore.createDatabase(prepareSchema(schemaName)); + for (int tableNumber = 0; tableNumber < TEST_TABLES_IN_SCHEMA_COUNT; tableNumber++) { + String tableName = "test_table_%d".formatted(tableNumber); + Path tablePath = schemaPath.resolve(tableName); + fileHiveMetastore.createTable(prepareTable(schemaName, tableName, tablePath), NO_PRIVILEGES); + } + } + return queryRunner; + } + + @Test + public void testSelectSchemasWithoutPredicate() + { + assertMetastoreInvocations("SELECT * FROM information_schema.schemata", ImmutableMultiset.of(GET_ALL_DATABASES)); + assertMetastoreInvocations("SELECT * FROM system.jdbc.schemas", ImmutableMultiset.of(GET_ALL_DATABASES)); + } + + @Test + public void testSelectSchemasWithFilterByInformationSchema() + { + assertMetastoreInvocations("SELECT * FROM information_schema.schemata WHERE schema_name = 'information_schema'", ImmutableMultiset.of(GET_ALL_DATABASES)); + assertMetastoreInvocations("SELECT * FROM system.jdbc.schemas WHERE table_schem = 'information_schema'", ImmutableMultiset.of(GET_ALL_DATABASES)); + } + + @Test + public void testSelectSchemasWithLikeOverSchemaName() + { + assertMetastoreInvocations("SELECT * FROM information_schema.schemata WHERE schema_name LIKE 'test%'", ImmutableMultiset.of(GET_ALL_DATABASES)); + assertMetastoreInvocations("SELECT * FROM system.jdbc.schemas WHERE table_schem LIKE 'test%'", ImmutableMultiset.of(GET_ALL_DATABASES)); + } + + @Test + public void testSelectTablesWithoutPredicate() + { + assertMetastoreInvocations("SELECT * FROM information_schema.tables", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.tables", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + } + + @Test + public void testSelectTablesWithFilterByInformationSchema() + { + assertMetastoreInvocations("SELECT * FROM information_schema.tables WHERE table_schema = 'information_schema'", ImmutableMultiset.of()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.tables WHERE table_schem = 'information_schema'", ImmutableMultiset.of()); + } + + @Test + public void testSelectTablesWithFilterBySchema() + { + assertMetastoreInvocations("SELECT * FROM information_schema.tables WHERE table_schema = 'test_schema_0'", + ImmutableMultiset.builder() + .add(GET_ALL_TABLES_FROM_DATABASE) + .add(GET_ALL_VIEWS_FROM_DATABASE) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.tables WHERE table_schem = 'test_schema_0'", + ImmutableMultiset.builder() + .add(GET_ALL_TABLES_FROM_DATABASE) + .add(GET_ALL_VIEWS_FROM_DATABASE) + .build()); + } + + @Test + public void testSelectTablesWithLikeOverSchema() + { + assertMetastoreInvocations("SELECT * FROM information_schema.tables WHERE table_schema LIKE 'test%'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.tables WHERE table_schem LIKE 'test%'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + } + + @Test + public void testSelectTablesWithFilterByTableName() + { + assertMetastoreInvocations("SELECT * FROM information_schema.tables WHERE table_name = 'test_table_0'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.tables WHERE table_name = 'test_table_0'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.tables WHERE table_name LIKE 'test\\_table\\_0' ESCAPE '\\'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.tables WHERE table_name LIKE 'test_table_0' ESCAPE '\\'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + } + + @Test + public void testSelectTablesWithLikeOverTableName() + { + assertMetastoreInvocations("SELECT * FROM information_schema.tables WHERE table_name LIKE 'test%'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.tables WHERE table_name LIKE 'test%'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + } + + @Test + public void testSelectViewsWithoutPredicate() + { + assertMetastoreInvocations("SELECT * FROM information_schema.views", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.tables WHERE table_type = 'VIEW'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + } + + @Test + public void testSelectViewsWithFilterByInformationSchema() + { + assertMetastoreInvocations("SELECT * FROM information_schema.views WHERE table_schema = 'information_schema'", ImmutableMultiset.of()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.tables WHERE table_type = 'VIEW' AND table_schem = 'information_schema'", ImmutableMultiset.of()); + } + + @Test + public void testSelectViewsWithFilterBySchema() + { + assertMetastoreInvocations("SELECT * FROM information_schema.views WHERE table_schema = 'test_schema_0'", ImmutableMultiset.of(GET_ALL_VIEWS_FROM_DATABASE)); + assertMetastoreInvocations("SELECT * FROM system.jdbc.tables WHERE table_type = 'VIEW' AND table_schem = 'test_schema_0'", + ImmutableMultiset.builder() + .add(GET_ALL_TABLES_FROM_DATABASE) + .add(GET_ALL_VIEWS_FROM_DATABASE) + .build()); + } + + @Test + public void testSelectViewsWithLikeOverSchema() + { + assertMetastoreInvocations("SELECT * FROM information_schema.views WHERE table_schema LIKE 'test%'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.tables WHERE table_type = 'VIEW' AND table_schem LIKE 'test%'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + } + + @Test + public void testSelectViewsWithFilterByTableName() + { + assertMetastoreInvocations("SELECT * FROM information_schema.views WHERE table_name = 'test_table_0'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.tables WHERE table_type = 'VIEW' AND table_name = 'test_table_0'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + } + + @Test + public void testSelectViewsWithLikeOverTableName() + { + assertMetastoreInvocations("SELECT * FROM information_schema.views WHERE table_name LIKE 'test%'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.tables WHERE table_type = 'VIEW' AND table_name LIKE 'test%'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + } + + @Test + public void testSelectColumnsWithoutPredicate() + { + assertMetastoreInvocations("SELECT * FROM information_schema.columns", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_TABLE, TEST_ALL_TABLES_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.columns", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_TABLE, TEST_ALL_TABLES_COUNT) + .build()); + } + + @Test + public void testSelectColumnsFilterByInformationSchema() + { + assertMetastoreInvocations("SELECT * FROM information_schema.columns WHERE table_schema = 'information_schema'", ImmutableMultiset.of()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.columns WHERE table_schem = 'information_schema'", ImmutableMultiset.of()); + } + + @Test + public void testSelectColumnsFilterBySchema() + { + assertMetastoreInvocations("SELECT * FROM information_schema.columns WHERE table_schema = 'test_schema_0'", + ImmutableMultiset.builder() + .add(GET_ALL_TABLES_FROM_DATABASE) + .add(GET_ALL_VIEWS_FROM_DATABASE) + .addCopies(GET_TABLE, TEST_TABLES_IN_SCHEMA_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.columns WHERE table_schem = 'test_schema_0'", + ImmutableMultiset.builder() + .add(GET_ALL_TABLES_FROM_DATABASE) + .add(GET_ALL_VIEWS_FROM_DATABASE) + .addCopies(GET_TABLE, TEST_TABLES_IN_SCHEMA_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.columns WHERE table_schem LIKE 'test\\_schema\\_0' ESCAPE '\\'", + ImmutableMultiset.builder() + .add(GET_ALL_TABLES_FROM_DATABASE) + .add(GET_ALL_VIEWS_FROM_DATABASE) + .addCopies(GET_TABLE, TEST_TABLES_IN_SCHEMA_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.columns WHERE table_schem LIKE 'test_schema_0' ESCAPE '\\'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .add(GET_ALL_TABLES_FROM_DATABASE) + .add(GET_ALL_VIEWS_FROM_DATABASE) + .addCopies(GET_TABLE, TEST_TABLES_IN_SCHEMA_COUNT) + .build()); + } + + @Test + public void testSelectColumnsWithLikeOverSchema() + { + assertMetastoreInvocations("SELECT * FROM information_schema.columns WHERE table_schema LIKE 'test%'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_TABLE, TEST_ALL_TABLES_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.columns WHERE table_schem LIKE 'test%'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_TABLE, TEST_ALL_TABLES_COUNT) + .build()); + } + + @Test + public void testSelectColumnsFilterByTableName() + { + metastore.resetCounters(); + computeActual("SELECT * FROM information_schema.columns WHERE table_name = 'test_table_0'"); + Multiset invocations = metastore.getMethodInvocations(); + + assertThat(invocations.count(GET_TABLE)).as("GET_TABLE invocations") + // some lengthy explanatory comment why variable count + // TODO switch to assertMetastoreInvocations when GET_TABLE invocation count becomes deterministic + // TODO this is horribly lot, why, o why we do those redirect-related calls in ... even if redirects not enabled?????????!!?!1 + .isBetween(TEST_ALL_TABLES_COUNT + 85, TEST_ALL_TABLES_COUNT + 95); + invocations = HashMultiset.create(invocations); + invocations.elementSet().remove(GET_TABLE); + + assertThat(invocations).as("invocations except of GET_TABLE") + .isEqualTo(ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .build()); + + assertMetastoreInvocations("SELECT * FROM system.jdbc.columns WHERE table_name = 'test_table_0'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_TABLE, TEST_SCHEMAS_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.columns WHERE table_name LIKE 'test\\_table\\_0' ESCAPE '\\'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_TABLE, TEST_SCHEMAS_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.columns WHERE table_name LIKE 'test_table_0' ESCAPE '\\'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_TABLE, TEST_SCHEMAS_COUNT) + .build()); + } + + @Test + public void testSelectColumnsWithLikeOverTableName() + { + assertMetastoreInvocations("SELECT * FROM information_schema.columns WHERE table_name LIKE 'test%'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_TABLE, TEST_ALL_TABLES_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.columns WHERE table_name LIKE 'test%'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_TABLE, TEST_ALL_TABLES_COUNT) + .build()); + } + + @Test + public void testSelectColumnsFilterByColumn() + { + assertMetastoreInvocations("SELECT * FROM information_schema.columns WHERE column_name = 'name'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_TABLE, TEST_ALL_TABLES_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.columns WHERE column_name = 'name'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_TABLE, TEST_ALL_TABLES_COUNT) + .build()); + } + + @Test + public void testSelectColumnsWithLikeOverColumn() + { + assertMetastoreInvocations("SELECT * FROM information_schema.columns WHERE column_name LIKE 'n%'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_TABLE, TEST_ALL_TABLES_COUNT) + .build()); + assertMetastoreInvocations("SELECT * FROM system.jdbc.columns WHERE column_name LIKE 'n%'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .addCopies(GET_ALL_TABLES_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_ALL_VIEWS_FROM_DATABASE, TEST_SCHEMAS_COUNT) + .addCopies(GET_TABLE, TEST_ALL_TABLES_COUNT) + .build()); + } + + @Test + public void testSelectColumnsFilterByTableAndSchema() + { + assertMetastoreInvocations("SELECT * FROM information_schema.columns WHERE table_schema = 'test_schema_0' AND table_name = 'test_table_0'", ImmutableMultiset.of(GET_TABLE)); + assertMetastoreInvocations("SELECT * FROM system.jdbc.columns WHERE table_schem = 'test_schema_0' AND table_name = 'test_table_0'", ImmutableMultiset.of(GET_TABLE)); + assertMetastoreInvocations("SELECT * FROM system.jdbc.columns WHERE table_schem LIKE 'test\\_schema\\_0' ESCAPE '\\' AND table_name LIKE 'test\\_table\\_0' ESCAPE '\\'", ImmutableMultiset.of(GET_TABLE)); + assertMetastoreInvocations("SELECT * FROM system.jdbc.columns WHERE table_schem LIKE 'test_schema_0' ESCAPE '\\' AND table_name LIKE 'test_table_0' ESCAPE '\\'", + ImmutableMultiset.builder() + .add(GET_ALL_DATABASES) + .add(GET_ALL_TABLES_FROM_DATABASE) + .add(GET_TABLE) + .build()); + } + + private void assertMetastoreInvocations(@Language("SQL") String query, Multiset expectedInvocations) + { + CountingAccessHiveMetastoreUtil.assertMetastoreInvocations(metastore, getQueryRunner(), getQueryRunner().getDefaultSession(), query, expectedInvocations); + } + + private static Database prepareSchema(String schemaName) + { + return Database.builder() + .setDatabaseName(schemaName) + .setOwnerName(Optional.empty()) + .setOwnerType(Optional.empty()) + .build(); + } + + private static Table prepareTable(String schemaName, String tableName, Path tablePath) + { + return Table.builder() + .setDatabaseName(schemaName) + .setTableName(tableName) + .setDataColumns(ImmutableList.of( + new Column("id", HiveType.HIVE_INT, Optional.empty()), + new Column("name", HiveType.HIVE_STRING, Optional.empty()))) + .setOwner(Optional.empty()) + .setTableType(MANAGED_TABLE.name()) + .withStorage(storage -> + storage.setStorageFormat(fromHiveStorageFormat(ORC)) + .setLocation(Optional.of(tablePath.toUri().toString()))) + .build(); + } +}