From 06e7c201dbdd8fe37d308b0bad2b1684e85e1dc7 Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Thu, 3 Mar 2022 14:38:41 +0100 Subject: [PATCH] [adhoc] Prepare aws2 ClientConfiguration for json serialization and cleanup AWS Module (#16894) * [adhoc] Prepare aws2 ClientConfiguration and related classes for json serialization and cleanup AWS Module --- .../java/io/amazon-web-services2/build.gradle | 1 + .../io/aws2/common/ClientConfiguration.java | 95 +++++++------------ .../aws2/common/HttpClientConfiguration.java | 23 ++++- .../io/aws2/common/RetryConfiguration.java | 43 ++++++++- .../beam/sdk/io/aws2/options/AwsModule.java | 80 ++-------------- .../beam/sdk/io/aws2/s3/SSECustomerKey.java | 6 ++ .../aws2/common/ClientConfigurationTest.java | 31 +++++- .../common/HttpClientConfigurationTest.java | 49 ++++++++++ .../aws2/common/RetryConfigurationTest.java | 22 +++++ .../sdk/io/aws2/options/AwsModuleTest.java | 29 ------ .../sdk/io/aws2/s3/SSECustomerKeyTest.java | 17 +++- 11 files changed, 229 insertions(+), 167 deletions(-) create mode 100644 sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfigurationTest.java diff --git a/sdks/java/io/amazon-web-services2/build.gradle b/sdks/java/io/amazon-web-services2/build.gradle index dceb4c41bed9..817b7b48e24c 100644 --- a/sdks/java/io/amazon-web-services2/build.gradle +++ b/sdks/java/io/amazon-web-services2/build.gradle @@ -30,6 +30,7 @@ ext.summary = "IO library to read and write Amazon Web Services services from Be dependencies { implementation library.java.vendored_guava_26_0_jre + implementation library.java.error_prone_annotations implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.aws_java_sdk2_apache_client implementation library.java.aws_java_sdk2_netty_client diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientConfiguration.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientConfiguration.java index 4371d7509999..9ee8eb277ddc 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientConfiguration.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientConfiguration.java @@ -19,19 +19,21 @@ import static org.apache.beam.sdk.io.aws2.options.AwsSerializableUtils.deserializeAwsCredentialsProvider; import static org.apache.beam.sdk.io.aws2.options.AwsSerializableUtils.serializeAwsCredentialsProvider; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import com.google.auto.value.AutoValue; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; +import com.google.auto.value.extension.memoized.Memoized; import java.io.Serializable; import java.net.URI; import java.util.function.Consumer; import javax.annotation.Nullable; import org.apache.beam.sdk.io.aws2.options.AwsOptions; import org.checkerframework.dataflow.qual.Pure; -import software.amazon.awssdk.auth.credentials.AwsCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.regions.Region; @@ -47,18 +49,28 @@ * uses a backoff strategy with equal jitter for computing the delay before the next retry. */ @AutoValue +@JsonInclude(value = JsonInclude.Include.NON_EMPTY) +@JsonDeserialize(builder = ClientConfiguration.Builder.class) public abstract class ClientConfiguration implements Serializable { /** * Optional {@link AwsCredentialsProvider}. If set, this overwrites the default in {@link * AwsOptions#getAwsCredentialsProvider()}. */ - public abstract @Nullable @Pure AwsCredentialsProvider credentialsProvider(); + @JsonProperty + @Memoized + public @Nullable @Pure AwsCredentialsProvider credentialsProvider() { + return credentialsProviderAsJson() != null + ? deserializeAwsCredentialsProvider(credentialsProviderAsJson()) + : null; + } /** * Optional {@link Region}. If set, this overwrites the default in {@link * AwsOptions#getAwsRegion()}. */ + @JsonProperty + @Memoized public @Nullable @Pure Region region() { return regionId() != null ? Region.of(regionId()) : null; } @@ -67,20 +79,24 @@ public abstract class ClientConfiguration implements Serializable { * Optional service endpoint to use AWS compatible services instead, e.g. for testing. If set, * this overwrites the default in {@link AwsOptions#getEndpoint()}. */ + @JsonProperty public abstract @Nullable @Pure URI endpoint(); /** * Optional {@link RetryConfiguration} for AWS clients. If unset, retry behavior will be unchanged * and use SDK defaults. */ + @JsonProperty public abstract @Nullable @Pure RetryConfiguration retry(); abstract @Nullable @Pure String regionId(); + abstract @Nullable @Pure String credentialsProviderAsJson(); + public abstract Builder toBuilder(); public static Builder builder() { - return new AutoValue_ClientConfiguration.Builder(); + return Builder.builder(); } public static ClientConfiguration create( @@ -93,12 +109,20 @@ public static ClientConfiguration create( } @AutoValue.Builder + @JsonPOJOBuilder(withPrefix = "") public abstract static class Builder { + @JsonCreator + static Builder builder() { + return new AutoValue_ClientConfiguration.Builder(); + } + /** * Optional {@link AwsCredentialsProvider}. If set, this overwrites the default in {@link * AwsOptions#getAwsCredentialsProvider()}. */ - public abstract Builder credentialsProvider(AwsCredentialsProvider credentialsProvider); + public Builder credentialsProvider(AwsCredentialsProvider credentialsProvider) { + return credentialsProviderAsJson(serializeAwsCredentialsProvider(credentialsProvider)); + } /** * Optional {@link Region}. If set, this overwrites the default in {@link @@ -118,6 +142,7 @@ public Builder region(Region region) { * Optional {@link RetryConfiguration} for AWS clients. If unset, retry behavior will be * unchanged and use SDK defaults. */ + @JsonSetter public abstract Builder retry(RetryConfiguration retry); /** @@ -132,58 +157,8 @@ public Builder retry(Consumer retry) { abstract Builder regionId(String region); - abstract AwsCredentialsProvider credentialsProvider(); - - abstract ClientConfiguration autoBuild(); + abstract Builder credentialsProviderAsJson(String credentialsProvider); - public ClientConfiguration build() { - if (credentialsProvider() != null) { - credentialsProvider(new SerializableAwsCredentialsProvider(credentialsProvider())); - } - return autoBuild(); - } - } - - /** Internal serializable {@link AwsCredentialsProvider}. */ - private static class SerializableAwsCredentialsProvider - implements AwsCredentialsProvider, Serializable { - private transient AwsCredentialsProvider provider; - private String serializedProvider; - - SerializableAwsCredentialsProvider(AwsCredentialsProvider provider) { - this.provider = checkNotNull(provider, "AwsCredentialsProvider cannot be null"); - this.serializedProvider = serializeAwsCredentialsProvider(provider); - } - - private void writeObject(ObjectOutputStream out) throws IOException { - out.writeUTF(serializedProvider); - } - - private void readObject(ObjectInputStream in) throws IOException { - serializedProvider = in.readUTF(); - provider = deserializeAwsCredentialsProvider(serializedProvider); - } - - @Override - public AwsCredentials resolveCredentials() { - return provider.resolveCredentials(); - } - - @Override - public boolean equals(@Nullable Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SerializableAwsCredentialsProvider that = (SerializableAwsCredentialsProvider) o; - return serializedProvider.equals(that.serializedProvider); - } - - @Override - public int hashCode() { - return serializedProvider.hashCode(); - } + public abstract ClientConfiguration build(); } } diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfiguration.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfiguration.java index 3b9cb1d9f936..65b882a95391 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfiguration.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfiguration.java @@ -17,6 +17,11 @@ */ package org.apache.beam.sdk.io.aws2.common; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import com.google.auto.value.AutoValue; import java.io.Serializable; import javax.annotation.Nullable; @@ -32,11 +37,14 @@ * HTTP Configuration */ @AutoValue +@JsonInclude(value = JsonInclude.Include.NON_EMPTY) +@JsonDeserialize(builder = HttpClientConfiguration.Builder.class) public abstract class HttpClientConfiguration implements Serializable { /** * Milliseconds to wait when acquiring a connection from the pool before giving up and timing out. */ + @JsonProperty public abstract @Nullable @Pure Integer connectionAcquisitionTimeout(); /** @@ -45,12 +53,14 @@ public abstract class HttpClientConfiguration implements Serializable { *

This will never close a connection that is currently in use, so long-lived connections may * remain open longer than this time. */ + @JsonProperty public abstract @Nullable @Pure Integer connectionMaxIdleTime(); /** * Milliseconds to wait when initially establishing a connection before giving up and timing out. * A duration of 0 means infinity, and is not recommended. */ + @JsonProperty public abstract @Nullable @Pure Integer connectionTimeout(); /** @@ -60,12 +70,14 @@ public abstract class HttpClientConfiguration implements Serializable { *

This will never close a connection that is currently in use, so long-lived connections may * remain open longer than this time. */ + @JsonProperty public abstract @Nullable @Pure Integer connectionTimeToLive(); /** * Milliseconds to wait for data to be transferred over an established, open connection before the * connection is timed out. A duration of 0 means infinity, and is not recommended. */ + @JsonProperty public abstract @Nullable @Pure Integer socketTimeout(); /** @@ -75,6 +87,7 @@ public abstract class HttpClientConfiguration implements Serializable { *

Note: Read timeout is only supported for async clients and ignored otherwise. Use {@link * #socketTimeout()} instead. */ + @JsonProperty public abstract @Nullable @Pure Integer readTimeout(); /** @@ -84,6 +97,7 @@ public abstract class HttpClientConfiguration implements Serializable { *

Note: Write timeout is only supported for async clients and ignored otherwise. Use {@link * #socketTimeout()} instead. */ + @JsonProperty public abstract @Nullable @Pure Integer writeTimeout(); /** @@ -94,14 +108,21 @@ public abstract class HttpClientConfiguration implements Serializable { * concurrent requests. When using HTTP/2 the number of connections that will be used depends on * the max streams allowed per connection. */ + @JsonProperty public abstract @Nullable @Pure Integer maxConnections(); public static Builder builder() { - return new AutoValue_HttpClientConfiguration.Builder(); + return Builder.builder(); } @AutoValue.Builder + @JsonPOJOBuilder(withPrefix = "") public abstract static class Builder { + @JsonCreator + static Builder builder() { + return new AutoValue_HttpClientConfiguration.Builder(); + } + /** * Milliseconds to wait when acquiring a connection from the pool before giving up and timing * out. diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/RetryConfiguration.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/RetryConfiguration.java index 1c226010e9e9..6ca342927a89 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/RetryConfiguration.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/RetryConfiguration.java @@ -20,6 +20,13 @@ import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; import static org.joda.time.Duration.ZERO; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.util.StdConverter; import com.google.auto.value.AutoValue; import java.io.Serializable; import javax.annotation.Nullable; @@ -36,31 +43,51 @@ * SdkDefaultRetrySetting} for further details. */ @AutoValue +@JsonInclude(value = JsonInclude.Include.NON_EMPTY) +@JsonDeserialize(builder = RetryConfiguration.Builder.class) public abstract class RetryConfiguration implements Serializable { private static final java.time.Duration BASE_BACKOFF = java.time.Duration.ofMillis(100); private static final java.time.Duration THROTTLED_BASE_BACKOFF = java.time.Duration.ofSeconds(1); private static final java.time.Duration MAX_BACKOFF = java.time.Duration.ofSeconds(20); + @JsonProperty public abstract @Pure int numRetries(); + @JsonProperty + @JsonSerialize(converter = DurationToMillis.class) public abstract @Nullable @Pure Duration baseBackoff(); + @JsonProperty + @JsonSerialize(converter = DurationToMillis.class) public abstract @Nullable @Pure Duration throttledBaseBackoff(); + @JsonProperty + @JsonSerialize(converter = DurationToMillis.class) public abstract @Nullable @Pure Duration maxBackoff(); + public abstract RetryConfiguration.Builder toBuilder(); + public static Builder builder() { - return new AutoValue_RetryConfiguration.Builder(); + return Builder.builder(); } @AutoValue.Builder + @JsonPOJOBuilder(withPrefix = "") public abstract static class Builder { + @JsonCreator + static Builder builder() { + return new AutoValue_RetryConfiguration.Builder(); + } + public abstract Builder numRetries(int numRetries); + @JsonDeserialize(converter = MillisToDuration.class) public abstract Builder baseBackoff(Duration baseBackoff); + @JsonDeserialize(converter = MillisToDuration.class) public abstract Builder throttledBaseBackoff(Duration baseBackoff); + @JsonDeserialize(converter = MillisToDuration.class) public abstract Builder maxBackoff(Duration maxBackoff); abstract RetryConfiguration autoBuild(); @@ -115,4 +142,18 @@ RetryPolicy toClientRetryPolicy() { private @Nullable static java.time.Duration toJava(@Nullable Duration duration) { return duration == null ? null : java.time.Duration.ofMillis(duration.getMillis()); } + + static class DurationToMillis extends StdConverter { + @Override + public Long convert(Duration duration) { + return duration.getMillis(); + } + } + + static class MillisToDuration extends StdConverter { + @Override + public Duration convert(Long millis) { + return Duration.millis(millis); + } + } } diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java index ff54a18fcc46..0f8b138d0b95 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java @@ -20,6 +20,7 @@ import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull; import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.core.JsonGenerator; @@ -42,14 +43,10 @@ import com.fasterxml.jackson.databind.util.NameTransformer; import com.google.auto.service.AutoService; import java.io.IOException; -import java.net.URI; -import java.util.Map; import java.util.function.Supplier; import org.apache.beam.repackaged.core.org.apache.commons.lang3.reflect.FieldUtils; import org.apache.beam.sdk.annotations.Experimental; import org.apache.beam.sdk.annotations.Experimental.Kind; -import org.apache.beam.sdk.io.aws2.common.HttpClientConfiguration; -import org.apache.beam.sdk.io.aws2.s3.SSECustomerKey; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.NonNull; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; @@ -67,7 +64,6 @@ import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; -import software.amazon.awssdk.utils.AttributeMap; /** * A Jackson {@link Module} that registers a {@link JsonSerializer} and {@link JsonDeserializer} for @@ -88,15 +84,11 @@ public AwsModule() { public void setupModule(SetupContext cxt) { cxt.setMixInAnnotations(AwsCredentialsProvider.class, AwsCredentialsProviderMixin.class); cxt.setMixInAnnotations(ProxyConfiguration.class, ProxyConfigurationMixin.class); - cxt.setMixInAnnotations(HttpClientConfiguration.class, HttpClientConfigurationMixin.class); cxt.setMixInAnnotations( - HttpClientConfiguration.Builder.class, HttpClientConfigurationMixin.Builder.class); - cxt.setMixInAnnotations(SSECustomerKey.class, SSECustomerKeyMixin.class); - cxt.setMixInAnnotations(SSECustomerKey.Builder.class, SSECustomerKeyMixin.Builder.class); + ProxyConfiguration.Builder.class, ProxyConfigurationMixin.Builder.class); cxt.setMixInAnnotations(Region.class, RegionMixin.class); - addValueInstantiator(HttpClientConfiguration.Builder.class, HttpClientConfiguration::builder); - + addValueInstantiator(ProxyConfiguration.Builder.class, ProxyConfiguration::builder); super.setupModule(cxt); } @@ -266,71 +258,11 @@ private Object readField(AwsCredentialsProvider provider, String fieldName) thro } /** A mixin to add Jackson annotations to {@link ProxyConfiguration}. */ - @JsonDeserialize(using = ProxyConfigurationDeserializer.class) - @JsonSerialize(using = ProxyConfigurationSerializer.class) - private static class ProxyConfigurationMixin {} - - private static class ProxyConfigurationDeserializer extends JsonDeserializer { - @Override - public ProxyConfiguration deserialize(JsonParser jsonParser, DeserializationContext context) - throws IOException { - Map asMap = - checkNotNull( - jsonParser.readValueAs(new TypeReference>() {}), - "Serialized ProxyConfiguration is null"); - - ProxyConfiguration.Builder builder = ProxyConfiguration.builder(); - final String endpoint = asMap.get("endpoint"); - if (endpoint != null) { - builder.endpoint(URI.create(endpoint)); - } - final String username = asMap.get("username"); - if (username != null) { - builder.username(username); - } - final String password = asMap.get("password"); - if (password != null) { - builder.password(password); - } - // defaults to FALSE / disabled - Boolean useSystemPropertyValues = Boolean.valueOf(asMap.get("useSystemPropertyValues")); - return builder.useSystemPropertyValues(useSystemPropertyValues).build(); - } - } - - private static class ProxyConfigurationSerializer extends JsonSerializer { - @Override - public void serialize( - ProxyConfiguration proxyConfiguration, - JsonGenerator jsonGenerator, - SerializerProvider serializer) - throws IOException { - // proxyConfiguration.endpoint() is private so we have to build it manually. - final String endpoint = - proxyConfiguration.scheme() - + "://" - + proxyConfiguration.host() - + ":" - + proxyConfiguration.port(); - jsonGenerator.writeStartObject(); - jsonGenerator.writeStringField("endpoint", endpoint); - jsonGenerator.writeStringField("username", proxyConfiguration.username()); - jsonGenerator.writeStringField("password", proxyConfiguration.password()); - jsonGenerator.writeEndObject(); - } - } - - /** A mixin to add Jackson annotations to {@link AttributeMap}. */ - @JsonDeserialize(builder = HttpClientConfiguration.Builder.class) + @JsonDeserialize(builder = ProxyConfiguration.Builder.class) @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) + @JsonIgnoreProperties(value = {"host", "port", "scheme"}) @JsonInclude(value = JsonInclude.Include.NON_EMPTY) - private static class HttpClientConfigurationMixin { - @JsonPOJOBuilder(withPrefix = "") - static class Builder {} - } - - @JsonDeserialize(builder = SSECustomerKey.Builder.class) - private static class SSECustomerKeyMixin { + private static class ProxyConfigurationMixin { @JsonPOJOBuilder(withPrefix = "") static class Builder {} } diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKey.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKey.java index 64ee4aa0347a..8e0ea423378f 100644 --- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKey.java +++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKey.java @@ -19,12 +19,17 @@ import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import java.nio.charset.StandardCharsets; import java.util.Base64; import org.apache.commons.codec.digest.DigestUtils; import org.checkerframework.checker.nullness.qual.Nullable; /** Customer provided key for use with Amazon S3 server-side encryption. */ +@JsonInclude(value = JsonInclude.Include.NON_EMPTY) +@JsonDeserialize(builder = SSECustomerKey.Builder.class) public class SSECustomerKey { private final @Nullable String key; @@ -63,6 +68,7 @@ public static Builder builder() { return new Builder(); } + @JsonPOJOBuilder(withPrefix = "") public static class Builder { private @Nullable String key; diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/ClientConfigurationTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/ClientConfigurationTest.java index 93615f513a33..0c2c91b1f8c4 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/ClientConfigurationTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/ClientConfigurationTest.java @@ -22,16 +22,18 @@ import static org.assertj.core.api.Assertions.assertThat; import java.net.URI; +import org.apache.beam.sdk.io.aws2.options.SerializationTestUtil; import org.junit.Test; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; public class ClientConfigurationTest { @Test - public void testSerialization() { + public void testJavaSerialization() { AwsCredentialsProvider credentials = StaticCredentialsProvider.create(AwsBasicCredentials.create("key", "secret")); @@ -50,4 +52,31 @@ public void testSerialization() { assertThat(deserializedConfig).isEqualTo(config); } + + @Test + public void testJsonSerialization() { + ClientConfiguration config = ClientConfiguration.builder().build(); + assertThat(jsonSerializeDeserialize(config)).isEqualTo(config); + + config = config.toBuilder().region(Region.US_WEST_1).build(); + assertThat(jsonSerializeDeserialize(config)).isEqualTo(config); + + config = config.toBuilder().credentialsProvider(DefaultCredentialsProvider.create()).build(); + assertThat(jsonSerializeDeserialize(config)).isEqualTo(config); + + AwsBasicCredentials credentials = AwsBasicCredentials.create("key", "secret"); + StaticCredentialsProvider credentialsProvider = StaticCredentialsProvider.create(credentials); + config = config.toBuilder().credentialsProvider(credentialsProvider).build(); + assertThat(jsonSerializeDeserialize(config)).isEqualTo(config); + + config = config.toBuilder().endpoint(URI.create("https://localhost:8080")).build(); + assertThat(jsonSerializeDeserialize(config)).isEqualTo(config); + + config = config.toBuilder().retry(r -> r.numRetries(10)).build(); + assertThat(jsonSerializeDeserialize(config)).isEqualTo(config); + } + + private ClientConfiguration jsonSerializeDeserialize(ClientConfiguration obj) { + return SerializationTestUtil.serializeDeserialize(ClientConfiguration.class, obj); + } } diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfigurationTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfigurationTest.java new file mode 100644 index 000000000000..b3147ac4e433 --- /dev/null +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfigurationTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.aws2.common; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.beam.sdk.io.aws2.options.SerializationTestUtil; +import org.junit.Test; + +public class HttpClientConfigurationTest { + @Test + public void testJsonSerialization() { + HttpClientConfiguration expected = HttpClientConfiguration.builder().build(); + assertThat(serializeAndDeserialize(expected)).isEqualTo(expected); + + expected = + HttpClientConfiguration.builder() + .connectionAcquisitionTimeout(100) + .connectionMaxIdleTime(200) + .connectionTimeout(300) + .connectionTimeToLive(400) + .socketTimeout(500) + .readTimeout(600) + .writeTimeout(700) + .maxConnections(10) + .build(); + + assertThat(serializeAndDeserialize(expected)).isEqualTo(expected); + } + + private HttpClientConfiguration serializeAndDeserialize(HttpClientConfiguration obj) { + return SerializationTestUtil.serializeDeserialize(HttpClientConfiguration.class, obj); + } +} diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/RetryConfigurationTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/RetryConfigurationTest.java index 7ec86cbc45f1..dca607dbb60e 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/RetryConfigurationTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/RetryConfigurationTest.java @@ -17,9 +17,12 @@ */ package org.apache.beam.sdk.io.aws2.common; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.joda.time.Duration.ZERO; +import org.apache.beam.sdk.io.aws2.options.SerializationTestUtil; +import org.joda.time.Duration; import org.junit.Test; public class RetryConfigurationTest { @@ -47,4 +50,23 @@ public void verifyMaxBackoffLargerZero() { assertThatThrownBy(() -> RetryConfiguration.builder().numRetries(1).maxBackoff(ZERO).build()) .hasMessage("maxBackoff must be greater than 0"); } + + @Test + public void testJsonSerialization() { + RetryConfiguration config = RetryConfiguration.builder().numRetries(10).build(); + assertThat(jsonSerializeDeserialize(config)).isEqualTo(config); + + config = config.toBuilder().maxBackoff(Duration.millis(1000)).build(); + assertThat(jsonSerializeDeserialize(config)).isEqualTo(config); + + config = config.toBuilder().baseBackoff(Duration.millis(200)).build(); + assertThat(jsonSerializeDeserialize(config)).isEqualTo(config); + + config = config.toBuilder().throttledBaseBackoff(Duration.millis(100)).build(); + assertThat(jsonSerializeDeserialize(config)).isEqualTo(config); + } + + private RetryConfiguration jsonSerializeDeserialize(RetryConfiguration obj) { + return SerializationTestUtil.serializeDeserialize(RetryConfiguration.class, obj); + } } diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/options/AwsModuleTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/options/AwsModuleTest.java index 2876b90a6590..b294c3faf711 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/options/AwsModuleTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/options/AwsModuleTest.java @@ -33,8 +33,6 @@ import java.util.List; import java.util.Properties; import java.util.function.Supplier; -import org.apache.beam.sdk.io.aws2.common.HttpClientConfiguration; -import org.apache.beam.sdk.io.aws2.s3.SSECustomerKey; import org.apache.beam.sdk.util.ThrowingSupplier; import org.apache.beam.sdk.util.common.ReflectHelpers; import org.hamcrest.MatcherAssert; @@ -151,33 +149,6 @@ public void testProxyConfigurationSerializationDeserialization() throws Exceptio assertEquals("password", deserializedProxyConfiguration.password()); } - @Test - public void testHttpClientConfigurationSerializationDeserialization() throws Exception { - HttpClientConfiguration expected = - HttpClientConfiguration.builder() - .connectionAcquisitionTimeout(100) - .connectionMaxIdleTime(200) - .connectionTimeout(300) - .connectionTimeToLive(400) - .socketTimeout(500) - .readTimeout(600) - .writeTimeout(700) - .maxConnections(10) - .build(); - - assertThat(serializeAndDeserialize(expected)).isEqualTo(expected); - } - - @Test - public void testSSECustomerKeySerializationDeserialization() throws Exception { - // default key created by S3Options.SSECustomerKeyFactory - SSECustomerKey emptyKey = SSECustomerKey.builder().build(); - assertThat(serializeAndDeserialize(emptyKey)).isEqualToComparingFieldByField(emptyKey); - - SSECustomerKey key = SSECustomerKey.builder().key("key").algorithm("algo").md5("md5").build(); - assertThat(serializeAndDeserialize(key)).isEqualToComparingFieldByField(key); - } - private T withSystemPropertyOverrides(Properties overrides, ThrowingSupplier fun) throws Exception { Properties systemProps = System.getProperties(); diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKeyTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKeyTest.java index ac395520ac95..943a3fded9ac 100644 --- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKeyTest.java +++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKeyTest.java @@ -17,9 +17,11 @@ */ package org.apache.beam.sdk.io.aws2.s3; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; +import org.apache.beam.sdk.io.aws2.options.SerializationTestUtil; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -30,7 +32,6 @@ "nullness" // TODO(https://issues.apache.org/jira/browse/BEAM-10402) }) public class SSECustomerKeyTest { - @Test public void testBuild() { assertThrows( @@ -54,4 +55,18 @@ public void buildWithArgs(String key, String algorithm, String md5, String encod assertEquals(algorithm, sseCustomerKey.getAlgorithm()); assertEquals(encodedMD5, sseCustomerKey.getMD5()); } + + @Test + public void testJsonSerializeDeserialize() { + // default key created by S3Options.SSECustomerKeyFactory + SSECustomerKey emptyKey = SSECustomerKey.builder().build(); + assertThat(jsonSerializeDeserialize(emptyKey)).isEqualToComparingFieldByField(emptyKey); + + SSECustomerKey key = SSECustomerKey.builder().key("key").algorithm("algo").md5("md5").build(); + assertThat(jsonSerializeDeserialize(key)).isEqualToComparingFieldByField(key); + } + + private SSECustomerKey jsonSerializeDeserialize(SSECustomerKey key) { + return SerializationTestUtil.serializeDeserialize(SSECustomerKey.class, key); + } }