Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use explicit values for MaxResults in all Glue APIs #20189

Merged
merged 1 commit into from
Dec 28, 2023
Merged

Use explicit values for MaxResults in all Glue APIs #20189

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,9 @@ public class GlueHiveMetastore
private static final int BATCH_CREATE_PARTITION_MAX_PAGE_SIZE = 100;
private static final int BATCH_UPDATE_PARTITION_MAX_PAGE_SIZE = 100;
private static final int AWS_GLUE_GET_PARTITIONS_MAX_RESULTS = 1000;
private static final int AWS_GLUE_GET_DATABASES_MAX_RESULTS = 100;
private static final int AWS_GLUE_GET_FUNCTIONS_MAX_RESULTS = 100;
private static final int AWS_GLUE_GET_TABLES_MAX_RESULTS = 100;
private static final Comparator<Iterable<String>> PARTITION_VALUE_COMPARATOR = lexicographical(String.CASE_INSENSITIVE_ORDER);
private static final Predicate<com.amazonaws.services.glue.model.Table> SOME_KIND_OF_VIEW_FILTER = table -> VIRTUAL_VIEW.name().equals(getTableTypeNullable(table));
private static final RetryPolicy<?> CONCURRENT_MODIFICATION_EXCEPTION_RETRY_POLICY = RetryPolicy.builder()
Expand Down Expand Up @@ -253,7 +256,8 @@ public List<String> getAllDatabases()
try {
List<String> databaseNames = getPaginatedResults(
glueClient::getDatabases,
new GetDatabasesRequest(),
new GetDatabasesRequest()
.withMaxResults(AWS_GLUE_GET_DATABASES_MAX_RESULTS),
GetDatabasesRequest::setNextToken,
GetDatabasesResult::getNextToken,
stats.getGetDatabases())
Expand Down Expand Up @@ -1291,7 +1295,8 @@ private Collection<LanguageFunction> getFunctionsByPattern(String databaseName,
glueClient::getUserDefinedFunctions,
new GetUserDefinedFunctionsRequest()
.withDatabaseName(databaseName)
.withPattern(functionNamePattern),
.withPattern(functionNamePattern)
.withMaxResults(AWS_GLUE_GET_FUNCTIONS_MAX_RESULTS),
GetUserDefinedFunctionsRequest::setNextToken,
GetUserDefinedFunctionsResult::getNextToken,
stats.getGetUserDefinedFunctions())
Expand Down Expand Up @@ -1370,7 +1375,8 @@ private Stream<com.amazonaws.services.glue.model.Table> getGlueTables(String dat
return getPaginatedResults(
glueClient::getTables,
new GetTablesRequest()
.withDatabaseName(databaseName),
.withDatabaseName(databaseName)
.withMaxResults(AWS_GLUE_GET_TABLES_MAX_RESULTS),
GetTablesRequest::setNextToken,
GetTablesResult::getNextToken,
stats.getGetTables())
Expand Down