Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow fetching relation names with their types (v2) #19863

Merged
merged 5 commits into from
Nov 24, 2023
Merged

Allow fetching relation names with their types (v2) #19863

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
349285a
Test Iceberg information_schema.tables cost with Glue
findepi Nov 23, 2023
428353c
Add getRelationTypes for listing relations with their types
findepi Nov 20, 2023
59c70e5
Implement getRelationTypes for Hive
findepi Nov 20, 2023
15e6ff4
Implement getRelationTypes for Iceberg
findepi Nov 23, 2023
131c790
Rename test class to match tested class name
findepi Nov 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 13 additions & 22 deletions client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1116,8 +1116,7 @@ public void testGetTablesMetadataCalls()
databaseMetaData -> databaseMetaData.getTables(null, null, null, null),
list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")),
ImmutableMultiset.<String>builder()
.add("ConnectorMetadata.listViews")
.add("ConnectorMetadata.listTables")
.add("ConnectorMetadata.getRelationTypes")
.build());

// Equality predicate on catalog name
Expand All @@ -1127,8 +1126,7 @@ public void testGetTablesMetadataCalls()
databaseMetaData -> databaseMetaData.getTables(COUNTING_CATALOG, null, null, null),
list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")),
ImmutableMultiset.<String>builder()
.add("ConnectorMetadata.listViews")
.add("ConnectorMetadata.listTables")
.add("ConnectorMetadata.getRelationTypes")
.build());

// Equality predicate on schema name
Expand All @@ -1142,8 +1140,7 @@ public void testGetTablesMetadataCalls()
.map(schemaTableName -> list(COUNTING_CATALOG, schemaTableName.getSchemaName(), schemaTableName.getTableName(), "TABLE"))
.collect(toImmutableList()),
ImmutableMultiset.<String>builder()
.add("ConnectorMetadata.listViews(schema=test_schema1)")
.add("ConnectorMetadata.listTables(schema=test_schema1)")
.add("ConnectorMetadata.getRelationTypes(schema=test_schema1)")
.build());

// LIKE predicate on schema name
Expand All @@ -1157,8 +1154,7 @@ public void testGetTablesMetadataCalls()
.map(schemaTableName -> list(COUNTING_CATALOG, schemaTableName.getSchemaName(), schemaTableName.getTableName(), "TABLE"))
.collect(toImmutableList()),
ImmutableMultiset.<String>builder()
.add("ConnectorMetadata.listViews")
.add("ConnectorMetadata.listTables")
.add("ConnectorMetadata.getRelationTypes")
.build());

// Equality predicate on table name
Expand All @@ -1171,8 +1167,7 @@ public void testGetTablesMetadataCalls()
list(COUNTING_CATALOG, "test_schema1", "test_table1", "TABLE"),
list(COUNTING_CATALOG, "test_schema2", "test_table1", "TABLE")),
ImmutableMultiset.<String>builder()
.add("ConnectorMetadata.listViews")
.add("ConnectorMetadata.listTables")
.add("ConnectorMetadata.getRelationTypes")
.build());

// LIKE predicate on table name
Expand All @@ -1185,8 +1180,7 @@ public void testGetTablesMetadataCalls()
list(COUNTING_CATALOG, "test_schema1", "test_table1", "TABLE"),
list(COUNTING_CATALOG, "test_schema2", "test_table1", "TABLE")),
ImmutableMultiset.<String>builder()
.add("ConnectorMetadata.listViews")
.add("ConnectorMetadata.listTables")
.add("ConnectorMetadata.getRelationTypes")
.build());

// Equality predicate on schema name and table name
Expand All @@ -1197,8 +1191,8 @@ public void testGetTablesMetadataCalls()
list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")),
list(list(COUNTING_CATALOG, "test_schema1", "test_table1", "TABLE")),
ImmutableMultiset.<String>builder()
.addCopies("ConnectorMetadata.getSystemTable(schema=test_schema1, table=test_table1)", 5)
.addCopies("ConnectorMetadata.getView(schema=test_schema1, table=test_table1)", 2)
.addCopies("ConnectorMetadata.getSystemTable(schema=test_schema1, table=test_table1)", 4)
.add("ConnectorMetadata.getView(schema=test_schema1, table=test_table1)")
.add("ConnectorMetadata.getMaterializedView(schema=test_schema1, table=test_table1)")
.add("ConnectorMetadata.redirectTable(schema=test_schema1, table=test_table1)")
.add("ConnectorMetadata.getTableHandle(schema=test_schema1, table=test_table1)")
Expand All @@ -1212,8 +1206,7 @@ public void testGetTablesMetadataCalls()
list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")),
list(list(COUNTING_CATALOG, "test_schema1", "test_table1", "TABLE")),
ImmutableMultiset.<String>builder()
.add("ConnectorMetadata.listViews")
.add("ConnectorMetadata.listTables")
.add("ConnectorMetadata.getRelationTypes")
.build());

// catalog does not exist
Expand Down Expand Up @@ -1530,8 +1523,7 @@ private void testAssumeLiteralMetadataCalls(String escapeLiteralParameter)
.map(schemaTableName -> list(COUNTING_CATALOG, schemaTableName.getSchemaName(), schemaTableName.getTableName(), "TABLE"))
.collect(toImmutableList()),
ImmutableMultiset.<String>builder()
.add("ConnectorMetadata.listViews(schema=test_schema1)")
.add("ConnectorMetadata.listTables(schema=test_schema1)")
.add("ConnectorMetadata.getRelationTypes(schema=test_schema1)")
.build());

// getTables's schema and table name patterns treated as literals
Expand All @@ -1542,8 +1534,8 @@ private void testAssumeLiteralMetadataCalls(String escapeLiteralParameter)
list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")),
list(list(COUNTING_CATALOG, "test_schema1", "test_table1", "TABLE")),
ImmutableMultiset.<String>builder()
.addCopies("ConnectorMetadata.getSystemTable(schema=test_schema1, table=test_table1)", 5)
.addCopies("ConnectorMetadata.getView(schema=test_schema1, table=test_table1)", 2)
.addCopies("ConnectorMetadata.getSystemTable(schema=test_schema1, table=test_table1)", 4)
.add("ConnectorMetadata.getView(schema=test_schema1, table=test_table1)")
.add("ConnectorMetadata.getMaterializedView(schema=test_schema1, table=test_table1)")
.add("ConnectorMetadata.redirectTable(schema=test_schema1, table=test_table1)")
.add("ConnectorMetadata.getTableHandle(schema=test_schema1, table=test_table1)")
Expand All @@ -1557,8 +1549,7 @@ private void testAssumeLiteralMetadataCalls(String escapeLiteralParameter)
list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")),
list(),
ImmutableMultiset.<String>builder()
.add("ConnectorMetadata.listViews(schema=test_schema_)")
.add("ConnectorMetadata.listTables(schema=test_schema_)")
.add("ConnectorMetadata.getRelationTypes(schema=test_schema_)")
.build());

// getColumns's schema and table name patterns treated as literals
Expand Down
26 changes: 18 additions & 8 deletions ...-main/src/main/java/io/trino/connector/informationschema/InformationSchemaPageSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import io.trino.spi.connector.ColumnHandle;
import io.trino.spi.connector.ColumnMetadata;
import io.trino.spi.connector.ConnectorPageSource;
import io.trino.spi.connector.RelationType;
import io.trino.spi.connector.SchemaTableName;
import io.trino.spi.security.AccessDeniedException;
import io.trino.spi.security.GrantInfo;
Expand All @@ -37,6 +38,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.OptionalLong;
import java.util.Queue;
Expand All @@ -52,12 +54,12 @@
import static io.trino.SystemSessionProperties.isOmitDateTimeTypePrecision;
import static io.trino.connector.informationschema.InformationSchemaMetadata.defaultPrefixes;
import static io.trino.connector.informationschema.InformationSchemaMetadata.isTablesEnumeratingTable;
import static io.trino.metadata.MetadataListing.getRelationTypes;
import static io.trino.metadata.MetadataListing.getViews;
import static io.trino.metadata.MetadataListing.listSchemas;
import static io.trino.metadata.MetadataListing.listTableColumns;
import static io.trino.metadata.MetadataListing.listTablePrivileges;
import static io.trino.metadata.MetadataListing.listTables;
import static io.trino.metadata.MetadataListing.listViews;
import static io.trino.spi.security.PrincipalType.USER;
import static io.trino.spi.type.TypeUtils.writeNativeValue;
import static io.trino.type.TypeUtils.getDisplayLabel;
Expand Down Expand Up @@ -245,7 +247,7 @@ private void buildPages()

private void addColumnsRecords(QualifiedTablePrefix prefix)
{
for (Map.Entry<SchemaTableName, List<ColumnMetadata>> entry : listTableColumns(session, metadata, accessControl, prefix).entrySet()) {
for (Entry<SchemaTableName, List<ColumnMetadata>> entry : listTableColumns(session, metadata, accessControl, prefix).entrySet()) {
SchemaTableName tableName = entry.getKey();
long ordinalPosition = 1;

Expand Down Expand Up @@ -275,16 +277,24 @@ private void addColumnsRecords(QualifiedTablePrefix prefix)

private void addTablesRecords(QualifiedTablePrefix prefix)
{
Set<SchemaTableName> tables = listTables(session, metadata, accessControl, prefix);
boolean needsTableType = requiredColumns.contains("table_type");
Set<SchemaTableName> views = Set.of();
Set<SchemaTableName> relations;
Set<SchemaTableName> views;
if (needsTableType) {
// TODO introduce a dedicated method for getting relations with their type from the connector, instead of calling (potentially much more expensive) getViews
views = listViews(session, metadata, accessControl, prefix);
Map<SchemaTableName, RelationType> relationTypes = getRelationTypes(session, metadata, accessControl, prefix);
relations = relationTypes.keySet();
views = relationTypes.entrySet().stream()
.filter(entry -> entry.getValue() == RelationType.VIEW)
.map(Entry::getKey)
.collect(toImmutableSet());
}
else {
relations = listTables(session, metadata, accessControl, prefix);
views = Set.of();
}
// TODO (https://github.com/trinodb/trino/issues/8207) define a type for materialized views

for (SchemaTableName name : tables) {
for (SchemaTableName name : relations) {
String type = null;
if (needsTableType) {
// if table and view names overlap, the view wins
Expand All @@ -304,7 +314,7 @@ private void addTablesRecords(QualifiedTablePrefix prefix)

private void addViewsRecords(QualifiedTablePrefix prefix)
{
for (Map.Entry<SchemaTableName, ViewInfo> entry : getViews(session, metadata, accessControl, prefix).entrySet()) {
for (Entry<SchemaTableName, ViewInfo> entry : getViews(session, metadata, accessControl, prefix).entrySet()) {
addRecord(
prefix.getCatalogName(),
entry.getKey().getSchemaName(),
Expand Down
12 changes: 5 additions & 7 deletions core/trino-main/src/main/java/io/trino/connector/system/jdbc/TableJdbcTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,19 +26,18 @@
import io.trino.spi.connector.InMemoryRecordSet;
import io.trino.spi.connector.InMemoryRecordSet.Builder;
import io.trino.spi.connector.RecordCursor;
import io.trino.spi.connector.RelationType;
import io.trino.spi.connector.SchemaTableName;
import io.trino.spi.predicate.Domain;
import io.trino.spi.predicate.TupleDomain;

import java.util.Optional;
import java.util.Set;

import static io.trino.connector.system.jdbc.FilterUtil.isImpossibleObjectName;
import static io.trino.connector.system.jdbc.FilterUtil.tablePrefix;
import static io.trino.connector.system.jdbc.FilterUtil.tryGetSingleVarcharValue;
import static io.trino.metadata.MetadataListing.getRelationTypes;
import static io.trino.metadata.MetadataListing.listCatalogNames;
import static io.trino.metadata.MetadataListing.listTables;
import static io.trino.metadata.MetadataListing.listViews;
import static io.trino.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder;
import static io.trino.spi.type.VarcharType.VARCHAR;
import static java.util.Objects.requireNonNull;
Expand Down Expand Up @@ -104,13 +103,12 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect
for (String catalog : listCatalogNames(session, metadata, accessControl, catalogDomain)) {
QualifiedTablePrefix prefix = tablePrefix(catalog, schemaFilter, tableFilter);

Set<SchemaTableName> views = listViews(session, metadata, accessControl, prefix);
for (SchemaTableName name : listTables(session, metadata, accessControl, prefix)) {
boolean isView = views.contains(name);
getRelationTypes(session, metadata, accessControl, prefix).forEach((name, type) -> {
boolean isView = type == RelationType.VIEW;
if ((includeTables && !isView) || (includeViews && isView)) {
table.addRow(tableRow(catalog, name, isView ? "VIEW" : "TABLE"));
}
}
});
}
return table.build().cursor();
}
Expand Down
7 changes: 7 additions & 0 deletions core/trino-main/src/main/java/io/trino/metadata/Metadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import io.trino.spi.connector.MaterializedViewFreshness;
import io.trino.spi.connector.ProjectionApplicationResult;
import io.trino.spi.connector.RelationCommentMetadata;
import io.trino.spi.connector.RelationType;
import io.trino.spi.connector.RowChangeParadigm;
import io.trino.spi.connector.SampleApplicationResult;
import io.trino.spi.connector.SampleType;
Expand Down Expand Up @@ -164,6 +165,12 @@ Optional<TableExecuteHandle> getTableHandleForExecute(
*/
List<QualifiedObjectName> listTables(Session session, QualifiedTablePrefix prefix);

/**
* Get the relation names that match the specified table prefix (never null).
* This includes all relations (e.g. tables, views, materialized views).
*/
Map<SchemaTableName, RelationType> getRelationTypes(Session session, QualifiedTablePrefix prefix);

/**
* Gets all of the columns on the specified table, or an empty map if the columns cannot be enumerated.
*
Expand Down
25 changes: 17 additions & 8 deletions core/trino-main/src/main/java/io/trino/metadata/MetadataListing.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import io.trino.spi.TrinoException;
import io.trino.spi.connector.CatalogHandle;
import io.trino.spi.connector.ColumnMetadata;
import io.trino.spi.connector.RelationType;
import io.trino.spi.connector.SchemaTableName;
import io.trino.spi.connector.TableColumnsMetadata;
import io.trino.spi.predicate.Domain;
Expand All @@ -46,6 +47,7 @@
import static io.trino.connector.system.jdbc.FilterUtil.tryGetSingleVarcharValue;
import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
import static io.trino.spi.StandardErrorCode.TABLE_REDIRECTION_ERROR;
import static java.util.function.Function.identity;

public final class MetadataListing
{
Expand Down Expand Up @@ -133,22 +135,29 @@ private static Set<SchemaTableName> doListTables(Session session, Metadata metad
return accessControl.filterTables(session.toSecurityContext(), prefix.getCatalogName(), tableNames);
}

public static Set<SchemaTableName> listViews(Session session, Metadata metadata, AccessControl accessControl, QualifiedTablePrefix prefix)
public static Map<SchemaTableName, RelationType> getRelationTypes(Session session, Metadata metadata, AccessControl accessControl, QualifiedTablePrefix prefix)
{
try {
return doListViews(session, metadata, accessControl, prefix);
return doGetRelationTypes(session, metadata, accessControl, prefix);
}
catch (RuntimeException exception) {
throw handleListingException(exception, "views", prefix.getCatalogName());
throw handleListingException(exception, "tables", prefix.getCatalogName());
}
}

private static Set<SchemaTableName> doListViews(Session session, Metadata metadata, AccessControl accessControl, QualifiedTablePrefix prefix)
private static Map<SchemaTableName, RelationType> doGetRelationTypes(Session session, Metadata metadata, AccessControl accessControl, QualifiedTablePrefix prefix)
{
Set<SchemaTableName> tableNames = metadata.listViews(session, prefix).stream()
.map(QualifiedObjectName::asSchemaTableName)
.collect(toImmutableSet());
return accessControl.filterTables(session.toSecurityContext(), prefix.getCatalogName(), tableNames);
Map<SchemaTableName, RelationType> relationTypes = metadata.getRelationTypes(session, prefix);

// Table listing operation only involves getting table names, but not any metadata. So redirected tables are not
// handled any differently. The target table or catalog are not involved. Thus the following filter is only called
// for the source catalog on source table names.
Set<SchemaTableName> accessibleNames = accessControl.filterTables(session.toSecurityContext(), prefix.getCatalogName(), relationTypes.keySet());
if (accessibleNames.equals(relationTypes.keySet())) {
return relationTypes;
}
return accessibleNames.stream()
.collect(toImmutableMap(identity(), relationTypes::get));
}

public static Map<SchemaTableName, ViewInfo> getViews(Session session, Metadata metadata, AccessControl accessControl, QualifiedTablePrefix prefix)
Expand Down
Loading