-
Notifications
You must be signed in to change notification settings - Fork 3.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add CI infrastructure for
trino-redshift
Use an ephemeral/throw-away Redshift cluster for running integration tests on `trino-redshift` connector. Once the tests are run, the testing Redshift cluster is being reclaimed. The Redshift cluster is publicly accessible in order to be accessible from the general purpose Github runners.
- Loading branch information
1 parent
3106f8d
commit fe5a925
Showing
8 changed files
with
291 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -uo pipefail | ||
|
|
||
| REDSHIFT_SCRIPTS_DIR="${BASH_SOURCE%/*}" | ||
|
|
||
| if [[ ! -f "${REDSHIFT_SCRIPTS_DIR}/.cluster-identifier" ]]; then | ||
| echo "Missing file ${REDSHIFT_SCRIPTS_DIR}/.cluster-identifier" | ||
| exit 0 | ||
| fi | ||
|
|
||
| REDSHIFT_CLUSTER_IDENTIFIER=$(cat $REDSHIFT_SCRIPTS_DIR/.cluster-identifier) | ||
|
|
||
| echo "Deleting Amazon Redshift cluster $REDSHIFT_CLUSTER_IDENTIFIER" | ||
| REDSHIFT_DELETE_CLUSTER_OUTPUT=$(aws redshift delete-cluster --cluster-identifier $REDSHIFT_CLUSTER_IDENTIFIER --skip-final-cluster-snapshot) | ||
|
|
||
| if [ -z "${REDSHIFT_DELETE_CLUSTER_OUTPUT}" ]; then | ||
| echo ${REDSHIFT_DELETE_CLUSTER_OUTPUT} | ||
| # Don't fail the build because of cleanup issues | ||
| exit 0 | ||
| fi | ||
|
|
||
| echo "Waiting for the Amazon Redshift cluster $REDSHIFT_CLUSTER_IDENTIFIER to be deleted" | ||
| aws redshift wait cluster-deleted \ | ||
| --cluster-identifier $REDSHIFT_CLUSTER_IDENTIFIER | ||
| if [ "$?" -ne 0 ] | ||
| then | ||
| echo "Amazon Redshift cluster $REDSHIFT_CLUSTER_IDENTIFIER deletion has timed out" | ||
| else | ||
| echo "Amazon Redshift cluster $REDSHIFT_CLUSTER_IDENTIFIER has been deleted" | ||
| fi | ||
|
|
||
| rm -f ${REDSHIFT_SCRIPTS_DIR}/.cluster-identifier | ||
| exit 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| REDSHIFT_SCRIPTS_DIR="${BASH_SOURCE%/*}" | ||
|
|
||
| # Redshift requires passwords containing at least a digit, a lower case letter and a upper case letter. | ||
| # Having no warranty that openssl will output a string following the above mentioned password policy, | ||
| # add explicitly the string 'Red1!' to the password | ||
| REDSHIFT_PASSWORD="$(openssl rand -base64 16 | tr -dc 'a-zA-Z0-9')Red1!" | ||
|
|
||
| REDSHIFT_CLUSTER_IDENTIFIER=trino-redshift-ci-cluster-$(openssl rand -hex 8) | ||
|
|
||
| REDSHIFT_CLUSTER_TTL=$(date -u -d "+2 hours" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v "+2H" +"%Y-%m-%dT%H:%M:%SZ") | ||
|
|
||
| echo "Creating the Amazon Redshift cluster ${REDSHIFT_CLUSTER_IDENTIFIER} on the region ${AWS_REGION}." | ||
| REDSHIFT_CREATE_CLUSTER_OUTPUT=$(aws redshift create-cluster \ | ||
| --db-name testdb \ | ||
| --region ${AWS_REGION} \ | ||
| --node-type dc2.large \ | ||
| --number-of-nodes 1 \ | ||
| --master-username admin \ | ||
| --master-user-password ${REDSHIFT_PASSWORD} \ | ||
| --cluster-identifier ${REDSHIFT_CLUSTER_IDENTIFIER} \ | ||
| --cluster-subnet-group-name ${REDSHIFT_SUBNET_GROUP_NAME} \ | ||
| --cluster-type single-node\ | ||
| --vpc-security-group-ids "${REDSHIFT_VPC_SECURITY_GROUP_IDS}" \ | ||
| --iam-roles ${REDSHIFT_IAM_ROLES} \ | ||
| --automated-snapshot-retention-period 0 \ | ||
| --publicly-accessible \ | ||
| --tags Key=cloud,Value=aws Key=environment,Value=test Key=project,Value=trino-redshift Key=ttl,Value=${REDSHIFT_CLUSTER_TTL}) | ||
|
|
||
| if [ -z "${REDSHIFT_CREATE_CLUSTER_OUTPUT}" ]; then | ||
| # Only show errors | ||
| echo ${REDSHIFT_CREATE_CLUSTER_OUTPUT} | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo ${REDSHIFT_CLUSTER_IDENTIFIER} > ${REDSHIFT_SCRIPTS_DIR}/.cluster-identifier | ||
| echo "Waiting for the Amazon Redshift cluster ${REDSHIFT_CLUSTER_IDENTIFIER} on the region ${AWS_REGION} to be available." | ||
|
|
||
| # Wait for the cluster to become available | ||
| aws redshift wait cluster-available \ | ||
| --cluster-identifier ${REDSHIFT_CLUSTER_IDENTIFIER} | ||
|
|
||
| echo "The Amazon Redshift cluster ${REDSHIFT_CLUSTER_IDENTIFIER} on the region ${AWS_REGION} is available for queries." | ||
|
|
||
| REDSHIFT_CLUSTER_DESCRIPTION=$(aws redshift describe-clusters --cluster-identifier ${REDSHIFT_CLUSTER_IDENTIFIER}) | ||
|
|
||
| export REDSHIFT_ENDPOINT=$(echo ${REDSHIFT_CLUSTER_DESCRIPTION} | jq -r '.Clusters[0].Endpoint.Address' ) | ||
| export REDSHIFT_PORT=$(echo ${REDSHIFT_CLUSTER_DESCRIPTION} | jq -r '.Clusters[0].Endpoint.Port' ) | ||
| export REDSHIFT_CLUSTER_DATABASE_NAME=$(echo ${REDSHIFT_CLUSTER_DESCRIPTION} | jq -r '.Clusters[0].DBName' ) | ||
| export REDSHIFT_USER=$(echo ${REDSHIFT_CLUSTER_DESCRIPTION} | jq -r '.Clusters[0].MasterUsername' ) | ||
| export REDSHIFT_PASSWORD |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
49 changes: 49 additions & 0 deletions
49
...trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftConnectorSmokeTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| /* | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package io.trino.plugin.redshift; | ||
|
|
||
| import com.google.common.collect.ImmutableMap; | ||
| import io.trino.plugin.jdbc.BaseJdbcConnectorSmokeTest; | ||
| import io.trino.testing.QueryRunner; | ||
| import io.trino.testing.TestingConnectorBehavior; | ||
|
|
||
| import static io.trino.plugin.redshift.RedshiftQueryRunner.createRedshiftQueryRunner; | ||
| import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_RENAME_TABLE_ACROSS_SCHEMAS; | ||
|
|
||
| public class TestRedshiftConnectorSmokeTest | ||
| extends BaseJdbcConnectorSmokeTest | ||
| { | ||
| @Override | ||
| @SuppressWarnings("DuplicateBranchesInSwitch") // options here are grouped per-feature | ||
| protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) | ||
| { | ||
| switch (connectorBehavior) { | ||
| case SUPPORTS_RENAME_TABLE_ACROSS_SCHEMAS: | ||
| return false; | ||
|
|
||
| default: | ||
| return super.hasBehavior(connectorBehavior); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| protected QueryRunner createQueryRunner() | ||
| throws Exception | ||
| { | ||
| return createRedshiftQueryRunner( | ||
| ImmutableMap.of(), | ||
| ImmutableMap.of(), | ||
| REQUIRED_TPCH_TABLES); | ||
| } | ||
| } |