Skip to content

Commit

Permalink
fixup! Fetch Redshift query results unloaded to S3
Browse files Browse the repository at this point in the history
  • Loading branch information
mayankvadariya authored and ebyhr committed Dec 5, 2024
1 parent 18fa63c commit 6f2b682
Showing 1 changed file with 15 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,9 @@
import io.trino.spi.connector.FixedSplitSource;
import io.trino.spi.connector.SchemaTableName;
import io.trino.spi.predicate.TupleDomain;
import io.trino.testing.AbstractTestQueries;
import io.trino.testing.QueryRunner;
import io.trino.testing.TestingConnectorSession;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.parallel.Execution;
Expand Down Expand Up @@ -78,16 +79,22 @@
@TestInstance(PER_CLASS)
@Execution(CONCURRENT)
public class TestRedshiftUnload
extends AbstractTestQueries
{
private RedshiftConfig redshiftJdbcConfig;
private RedshiftConfig redshiftUnloadConfig;
private RedshiftSplitManager redshiftSplitManager;

@BeforeAll
public void setup()
@Override
protected QueryRunner createQueryRunner()
throws Exception
{
redshiftJdbcConfig = new RedshiftConfig().setUnloadLocation(S3_UNLOAD_ROOT).setUnloadIamRole(IAM_ROLE);
redshiftUnloadConfig = new RedshiftConfig().setUnloadLocation(S3_UNLOAD_ROOT).setUnloadIamRole(IAM_ROLE);
redshiftSplitManager = createSplitManager();
return RedshiftQueryRunner.builder()
// NOTE this can cause tests to time-out if larger tables like
// lineitem and orders need to be re-created.
.setInitialTables(REQUIRED_TPCH_TABLES)
.build();
}

private RedshiftSplitManager createSplitManager()
Expand All @@ -96,7 +103,7 @@ private RedshiftSplitManager createSplitManager()
.build();
RedshiftClient redshiftClient = new RedshiftClient(
new BaseJdbcConfig().setConnectionUrl(JDBC_URL),
redshiftJdbcConfig,
redshiftUnloadConfig,
driverConnectionFactory,
new JdbcStatisticsConfig(),
new DefaultQueryBuilder(NONE),
Expand All @@ -108,7 +115,7 @@ private RedshiftSplitManager createSplitManager()
new DefaultQueryBuilder(NONE),
NONE,
new JdbcSplitManager(redshiftClient),
redshiftJdbcConfig,
redshiftUnloadConfig,
new S3FileSystemFactory(OpenTelemetry.noop(), new S3FileSystemConfig()
.setAwsAccessKey(AWS_ACCESS_KEY)
.setAwsSecretKey(AWS_SECRET_KEY)
Expand Down Expand Up @@ -178,7 +185,7 @@ private ConnectorSession createUnloadSession()
.setPropertyMetadata(
Stream.concat(
new JdbcDynamicFilteringSessionProperties(new JdbcDynamicFilteringConfig()).getSessionProperties().stream(),
new RedshiftSessionProperties(redshiftJdbcConfig).getSessionProperties().stream())
new RedshiftSessionProperties(redshiftUnloadConfig).getSessionProperties().stream())
.toList())
.build();
}
Expand Down

0 comments on commit 6f2b682

Please sign in to comment.