Skip to content

Commit

Permalink
Simplify Delta Lake query runner
Browse files Browse the repository at this point in the history
  • Loading branch information
ebyhr committed Apr 1, 2022
1 parent eb8fbbf commit a3b07e1
Show file tree
Hide file tree
Showing 3 changed files with 3 additions and 44 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -58,12 +58,6 @@ public static DistributedQueryRunner createDeltaLakeQueryRunner()
return createDeltaLakeQueryRunner(ImmutableMap.of(), ImmutableMap.of());
}

public static DistributedQueryRunner createDeltaLakeQueryRunner(Map<String, String> connectorProperties)
throws Exception
{
return createDeltaLakeQueryRunner(ImmutableMap.of(), connectorProperties);
}

public static DistributedQueryRunner createDeltaLakeQueryRunner(Map<String, String> extraProperties, Map<String, String> connectorProperties)
throws Exception
{
Expand Down Expand Up @@ -126,25 +120,6 @@ public static DistributedQueryRunner createS3DeltaLakeQueryRunner(
additionalSetup);
}

public static QueryRunner createAbfsDeltaLakeQueryRunner(
String catalogName,
String schemaName,
TestingHadoop testingHadoop)
throws Exception
{
return createAbfsDeltaLakeQueryRunner(catalogName, schemaName, ImmutableMap.of(), ImmutableMap.of(), testingHadoop);
}

public static QueryRunner createAbfsDeltaLakeQueryRunner(
String catalogName,
String schemaName,
Map<String, String> connectorProperties,
TestingHadoop testingHadoop)
throws Exception
{
return createAbfsDeltaLakeQueryRunner(catalogName, schemaName, ImmutableMap.of(), connectorProperties, testingHadoop);
}

public static QueryRunner createAbfsDeltaLakeQueryRunner(
String catalogName,
String schemaName,
Expand All @@ -156,29 +131,13 @@ public static QueryRunner createAbfsDeltaLakeQueryRunner(
return createDockerizedDeltaLakeQueryRunner(
catalogName,
schemaName,
ImmutableMap.of(),
extraProperties,
ImmutableMap.<String, String>builder()
.put("hive.azure.abfs-storage-account", requiredNonEmptySystemProperty("hive.hadoop2.azure-abfs-account"))
.put("hive.azure.abfs-access-key", requiredNonEmptySystemProperty("hive.hadoop2.azure-abfs-access-key"))
.putAll(connectorProperties)
.buildOrThrow(),
testingHadoop);
}

public static DistributedQueryRunner createDockerizedDeltaLakeQueryRunner(
String catalogName,
String schemaName,
Map<String, String> extraProperties,
Map<String, String> connectorProperties,
TestingHadoop testingHadoop)
throws Exception
{
return createDockerizedDeltaLakeQueryRunner(
catalogName,
schemaName,
ImmutableMap.of(),
extraProperties,
connectorProperties,
testingHadoop,
queryRunner -> {});
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ DockerizedDataLake createDockerizedDataLake()
QueryRunner createDeltaLakeQueryRunner(Map<String, String> connectorProperties)
throws Exception
{
return createAbfsDeltaLakeQueryRunner(DELTA_CATALOG, SCHEMA, connectorProperties, dockerizedDataLake.getTestingHadoop());
return createAbfsDeltaLakeQueryRunner(DELTA_CATALOG, SCHEMA, ImmutableMap.of(), connectorProperties, dockerizedDataLake.getTestingHadoop());
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ protected QueryRunner createQueryRunner()
ImmutableMap.of(hadoopCoreSiteXmlTempFile.toString(), "/etc/hadoop/conf/core-site.xml")));
testingHadoop = dockerizedDataLake.getTestingHadoop();

return createAbfsDeltaLakeQueryRunner(DELTA_CATALOG, SCHEMA_NAME, testingHadoop);
return createAbfsDeltaLakeQueryRunner(DELTA_CATALOG, SCHEMA_NAME, ImmutableMap.of(), ImmutableMap.of(), testingHadoop);
}

private Path createHadoopCoreSiteXmlTempFileWithAbfsSettings()
Expand Down

0 comments on commit a3b07e1

Please sign in to comment.