diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java index 545e8aff58969..2c7a3dddd5833 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java @@ -5,7 +5,6 @@ package io.airbyte.integrations.destination.bigquery; import static io.airbyte.integrations.destination.bigquery.helpers.LoggerHelper.getJobErrorMessage; -import static java.util.Objects.isNull; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -172,9 +171,9 @@ public static String getDatasetId(final JsonNode config) { String projectId = config.get(BigQueryConsts.CONFIG_PROJECT_ID).asText(); if (!(projectId.equals(projectIdPart))) { throw new IllegalArgumentException(String.format( - "Project ID included in Dataset ID must match Project ID field's value: Project ID is `%s`, but you specified `%s` in Dataset ID", - projectId, - projectIdPart)); + "Project ID included in Dataset ID must match Project ID field's value: Project ID is `%s`, but you specified `%s` in Dataset ID", + projectId, + projectIdPart)); } } // if colonIndex is -1, then this returns the entire string diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java index 70c7e9dd1627e..9edfa8f2ba0bc 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java @@ -385,6 +385,7 @@ private boolean isTablePartitioned(final BigQuery bigquery, final Dataset datase } private static class DatasetIdResetter { + private Consumer consumer; DatasetIdResetter(Consumer consumer) { @@ -394,20 +395,20 @@ private static class DatasetIdResetter { public void accept(JsonNode config) { consumer.accept(config); } + } private static Stream datasetIdResetterProvider() { // parameterized test with two dataset-id patterns: `dataset_id` and `project-id:dataset_id` return Stream.of( - Arguments.arguments(new DatasetIdResetter(config -> {})), - Arguments.arguments(new DatasetIdResetter( - config -> { - String projectId = ((ObjectNode) config).get(BigQueryConsts.CONFIG_PROJECT_ID).asText(); - String datasetId = ((ObjectNode) config).get(BigQueryConsts.CONFIG_DATASET_ID).asText(); - ((ObjectNode) config).put(BigQueryConsts.CONFIG_DATASET_ID, - String.format("%s:%s", projectId, datasetId)); - } - )) - ); + Arguments.arguments(new DatasetIdResetter(config -> {})), + Arguments.arguments(new DatasetIdResetter( + config -> { + String projectId = ((ObjectNode) config).get(BigQueryConsts.CONFIG_PROJECT_ID).asText(); + String datasetId = ((ObjectNode) config).get(BigQueryConsts.CONFIG_DATASET_ID).asText(); + ((ObjectNode) config).put(BigQueryConsts.CONFIG_DATASET_ID, + String.format("%s:%s", projectId, datasetId)); + }))); } + } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java index 586e0cf7ce747..f0538e443a624 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java @@ -26,7 +26,7 @@ public void init() { .put(BigQueryConsts.CONFIG_CREDS, "test_secret") .put(BigQueryConsts.CONFIG_DATASET_LOCATION, "US"); } - + @ParameterizedTest @MethodSource("validBigQueryIdProvider") public void testGetDatasetIdSuccess(String projectId, String datasetId, String expected) throws Exception { @@ -66,4 +66,5 @@ private static Stream invalidBigQueryIdProvider() { Arguments.arguments("my-project", "your-project:my_dataset", "Project ID included in Dataset ID must match Project ID field's value: Project ID is `my-project`, but you specified `your-project` in Dataset ID")); } + } diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3CopyConfig.java b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3CopyConfig.java index bfce8529dfceb..6b13de6f73c5f 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3CopyConfig.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3CopyConfig.java @@ -22,4 +22,9 @@ public static boolean shouldPurgeStagingData(final JsonNode config) { } } + public static S3CopyConfig getS3CopyConfig(final JsonNode config) { + return new S3CopyConfig(S3CopyConfig.shouldPurgeStagingData(config), + S3DestinationConfig.getS3DestinationConfig(config)); + } + } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftCopyS3Destination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftCopyS3Destination.java index 1b5249d0ed7fa..8b8212bb8cf7e 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftCopyS3Destination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftCopyS3Destination.java @@ -43,7 +43,7 @@ public AirbyteMessageConsumer getConsumer(final JsonNode config, getDatabase(config), getSqlOperations(), getNameTransformer(), - new S3CopyConfig(S3CopyConfig.shouldPurgeStagingData(config), getS3DestinationConfig(config)), + S3CopyConfig.getS3CopyConfig(config), catalog, new RedshiftStreamCopierFactory(), getConfiguredSchema(config)); diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeCopyS3Destination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeCopyS3Destination.java index 64b018c74973b..53fd764e1227a 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeCopyS3Destination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeCopyS3Destination.java @@ -29,7 +29,7 @@ public AirbyteMessageConsumer getConsumer(final JsonNode config, getDatabase(config), getSqlOperations(), getNameTransformer(), - getS3CopyConfig(config), + S3CopyConfig.getS3CopyConfig(config.get("loading_method")), catalog, new SnowflakeS3StreamCopierFactory(), getConfiguredSchema(config)); @@ -59,10 +59,6 @@ private String getConfiguredSchema(final JsonNode config) { return config.get("schema").asText(); } - private S3CopyConfig getS3CopyConfig(final JsonNode config) { - return new S3CopyConfig(S3CopyConfig.shouldPurgeStagingData(config), getS3DestinationConfig(config)); - } - private S3DestinationConfig getS3DestinationConfig(final JsonNode config) { final JsonNode loadingMethod = config.get("loading_method"); return S3DestinationConfig.getS3DestinationConfig(loadingMethod);