From 5560ba46c6a948ee34b64bc52929026b1cc65723 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 21 Feb 2023 15:15:01 -0500 Subject: [PATCH] Opensearch repository-s3 plugin cannot read ServiceAccount token (#6390) * Opensearch repository-s3 plugin cannot read ServiceAccount token Signed-off-by: Andriy Redko * Address code review comments Signed-off-by: Andriy Redko --------- Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + .../s3/S3BlobStoreRepositoryTests.java | 5 +- .../repositories/s3/S3ClientSettings.java | 24 +++-- .../repositories/s3/S3RepositoryPlugin.java | 13 ++- .../opensearch/repositories/s3/S3Service.java | 11 ++- .../s3/AwsS3ServiceImplTests.java | 23 +++-- .../repositories/s3/ConfigPathSupport.java | 22 +++++ .../s3/RepositoryCredentialsTests.java | 9 +- .../s3/S3BlobContainerRetriesTests.java | 6 +- .../s3/S3ClientSettingsTests.java | 97 +++++++++++++------ .../repositories/s3/S3RepositoryTests.java | 9 +- .../repositories/s3/S3ServiceTests.java | 9 +- 12 files changed, 160 insertions(+), 69 deletions(-) create mode 100644 plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/ConfigPathSupport.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d64624421432..e5c60bdc99db9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -129,6 +129,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix weighted shard routing state across search requests([#6004](https://github.com/opensearch-project/OpenSearch/pull/6004)) - [Segment Replication] Fix bug where inaccurate sequence numbers are sent during replication ([#6122](https://github.com/opensearch-project/OpenSearch/pull/6122)) - Enable numeric sort optimisation for few numerical sort types ([#6321](https://github.com/opensearch-project/OpenSearch/pull/6321)) +- Fix Opensearch repository-s3 plugin cannot read ServiceAccount token ([#6390](https://github.com/opensearch-project/OpenSearch/pull/6390) ### Security diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java index 954e35289ece9..9dc1b1106e062 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -58,6 +58,7 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -149,8 +150,8 @@ protected Settings nodeSettings(int nodeOrdinal) { */ public static class TestS3RepositoryPlugin extends S3RepositoryPlugin { - public TestS3RepositoryPlugin(final Settings settings) { - super(settings); + public TestS3RepositoryPlugin(final Settings settings, final Path configPath) { + super(settings, configPath); } @Override diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java index 1f9af5314f30d..a20cc156545ce 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java @@ -35,6 +35,8 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import org.opensearch.common.Strings; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.io.PathUtils; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.SecureSetting; import org.opensearch.common.settings.SecureString; @@ -46,6 +48,7 @@ import java.net.InetAddress; import java.net.UnknownHostException; +import java.nio.file.Path; import java.util.Collections; import java.util.HashMap; import java.util.Locale; @@ -348,16 +351,16 @@ S3ClientSettings refine(Settings repositorySettings) { * * Note this will always at least return a client named "default". */ - static Map load(Settings settings) { + static Map load(final Settings settings, final Path configPath) { final Set clientNames = settings.getGroups(PREFIX).keySet(); final Map clients = new HashMap<>(); for (final String clientName : clientNames) { - clients.put(clientName, getClientSettings(settings, clientName)); + clients.put(clientName, getClientSettings(settings, clientName, configPath)); } if (clients.containsKey("default") == false) { // this won't find any settings under the default client, // but it will pull all the fallback static settings - clients.put("default", getClientSettings(settings, "default")); + clients.put("default", getClientSettings(settings, "default", configPath)); } return Collections.unmodifiableMap(clients); } @@ -425,8 +428,17 @@ private static S3BasicCredentials loadCredentials(Settings settings, String clie } } - private static IrsaCredentials loadIrsaCredentials(Settings settings, String clientName) { + @SuppressForbidden(reason = "PathUtils#get") + private static IrsaCredentials loadIrsaCredentials(Settings settings, String clientName, Path configPath) { String identityTokenFile = getConfigValue(settings, clientName, IDENTITY_TOKEN_FILE_SETTING); + if (identityTokenFile.length() != 0) { + final Path identityTokenFilePath = PathUtils.get(identityTokenFile); + // If the path is not absolute, resolve it relatively to config path + if (!identityTokenFilePath.isAbsolute()) { + identityTokenFile = PathUtils.get(new Path[] { configPath }, identityTokenFile).toString(); + } + } + try ( SecureString roleArn = getConfigValue(settings, clientName, ROLE_ARN_SETTING); SecureString roleSessionName = getConfigValue(settings, clientName, ROLE_SESSION_NAME_SETTING) @@ -441,11 +453,11 @@ private static IrsaCredentials loadIrsaCredentials(Settings settings, String cli // pkg private for tests /** Parse settings for a single client. */ - static S3ClientSettings getClientSettings(final Settings settings, final String clientName) { + static S3ClientSettings getClientSettings(final Settings settings, final String clientName, final Path configPath) { final Protocol awsProtocol = getConfigValue(settings, clientName, PROTOCOL_SETTING); return new S3ClientSettings( S3ClientSettings.loadCredentials(settings, clientName), - S3ClientSettings.loadIrsaCredentials(settings, clientName), + S3ClientSettings.loadIrsaCredentials(settings, clientName, configPath), getConfigValue(settings, clientName, ENDPOINT_SETTING), awsProtocol, Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()), diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java index 58be2c90fd3bb..a52e05d94fa95 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java @@ -47,6 +47,7 @@ import org.opensearch.repositories.Repository; import java.io.IOException; +import java.nio.file.Path; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Arrays; @@ -77,15 +78,17 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo } protected final S3Service service; + private final Path configPath; - public S3RepositoryPlugin(final Settings settings) { - this(settings, new S3Service()); + public S3RepositoryPlugin(final Settings settings, final Path configPath) { + this(settings, configPath, new S3Service(configPath)); } - S3RepositoryPlugin(final Settings settings, final S3Service service) { + S3RepositoryPlugin(final Settings settings, final Path configPath, final S3Service service) { this.service = Objects.requireNonNull(service, "S3 service must not be null"); + this.configPath = configPath; // eagerly load client settings so that secure settings are read - final Map clientsSettings = S3ClientSettings.load(settings); + final Map clientsSettings = S3ClientSettings.load(settings, configPath); this.service.refreshAndClearCache(clientsSettings); } @@ -142,7 +145,7 @@ public List> getSettings() { @Override public void reload(Settings settings) { // secure settings should be readable - final Map clientsSettings = S3ClientSettings.load(settings); + final Map clientsSettings = S3ClientSettings.load(settings, configPath); service.refreshAndClearCache(clientsSettings); } diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java index 930af6f8a9799..ac74857737107 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java @@ -72,6 +72,7 @@ import java.net.PasswordAuthentication; import java.net.Proxy; import java.net.Socket; +import java.nio.file.Path; import java.security.SecureRandom; import java.util.Map; @@ -90,9 +91,7 @@ class S3Service implements Closeable { /** * Client settings calculated from static configuration and settings in the keystore. */ - private volatile Map staticClientSettings = MapBuilder.newMapBuilder() - .put("default", S3ClientSettings.getClientSettings(Settings.EMPTY, "default")) - .immutableMap(); + private volatile Map staticClientSettings; /** * Client settings derived from those in {@link #staticClientSettings} by combining them with settings @@ -100,6 +99,12 @@ class S3Service implements Closeable { */ private volatile Map derivedClientSettings = emptyMap(); + S3Service(final Path configPath) { + staticClientSettings = MapBuilder.newMapBuilder() + .put("default", S3ClientSettings.getClientSettings(Settings.EMPTY, "default", configPath)) + .immutableMap(); + } + /** * Refreshes the settings for the AmazonS3 clients and clears the cache of * existing clients. New clients will be build using these new settings. Old diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java index 76bd5d303e5fb..47c77b909c12c 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java @@ -55,7 +55,7 @@ import static org.opensearch.repositories.s3.S3ClientSettings.PROTOCOL_SETTING; import static org.opensearch.repositories.s3.S3ClientSettings.PROXY_TYPE_SETTING; -public class AwsS3ServiceImplTests extends OpenSearchTestCase { +public class AwsS3ServiceImplTests extends OpenSearchTestCase implements ConfigPathSupport { @AfterClass public static void shutdownIdleConnectionReaper() { // created by default STS client @@ -64,7 +64,7 @@ public static void shutdownIdleConnectionReaper() { public void testAWSCredentialsDefaultToInstanceProviders() { final String inexistentClientName = randomAlphaOfLength(8).toLowerCase(Locale.ROOT); - final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(Settings.EMPTY, inexistentClientName); + final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(Settings.EMPTY, inexistentClientName, configPath()); final AWSCredentialsProvider credentialsProvider = S3Service.buildCredentials(logger, clientSettings); assertThat(credentialsProvider, instanceOf(S3Service.PrivilegedInstanceProfileCredentialsProvider.class)); } @@ -79,7 +79,7 @@ public void testAWSCredentialsFromKeystore() { secureSettings.setString("s3.client." + clientName + ".secret_key", clientName + "_aws_secret_key"); } final Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - final Map allClientsSettings = S3ClientSettings.load(settings); + final Map allClientsSettings = S3ClientSettings.load(settings, configPath()); // no less, no more assertThat(allClientsSettings.size(), is(clientsCount + 1)); // including default for (int i = 0; i < clientsCount; i++) { @@ -114,7 +114,7 @@ public void testCredentialsAndIrsaWithIdentityTokenFileCredentialsFromKeystore() plainSettings.put("s3.client." + clientName + ".identity_token_file", clientName + "_identity_token_file"); } final Settings settings = Settings.builder().loadFromMap(plainSettings).setSecureSettings(secureSettings).build(); - final Map allClientsSettings = S3ClientSettings.load(settings); + final Map allClientsSettings = S3ClientSettings.load(settings, configPath()); // no less, no more assertThat(allClientsSettings.size(), is(clientsCount + 1)); // including default for (int i = 0; i < clientsCount; i++) { @@ -148,7 +148,7 @@ public void testCredentialsAndIrsaCredentialsFromKeystore() throws IOException { plainSettings.put("s3.client." + clientName + ".region", "us-east1"); } final Settings settings = Settings.builder().loadFromMap(plainSettings).setSecureSettings(secureSettings).build(); - final Map allClientsSettings = S3ClientSettings.load(settings); + final Map allClientsSettings = S3ClientSettings.load(settings, configPath()); // no less, no more assertThat(allClientsSettings.size(), is(clientsCount + 1)); // including default for (int i = 0; i < clientsCount; i++) { @@ -175,7 +175,7 @@ public void testIrsaCredentialsFromKeystore() throws IOException { secureSettings.setString("s3.client." + clientName + ".role_session_name", clientName + "_role_session_name"); } final Settings settings = Settings.builder().loadFromMap(plainSettings).setSecureSettings(secureSettings).build(); - final Map allClientsSettings = S3ClientSettings.load(settings); + final Map allClientsSettings = S3ClientSettings.load(settings, configPath()); // no less, no more assertThat(allClientsSettings.size(), is(clientsCount + 1)); // including default for (int i = 0; i < clientsCount; i++) { @@ -198,7 +198,7 @@ public void testSetDefaultCredential() { secureSettings.setString("s3.client.default.access_key", awsAccessKey); secureSettings.setString("s3.client.default.secret_key", awsSecretKey); final Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - final Map allClientsSettings = S3ClientSettings.load(settings); + final Map allClientsSettings = S3ClientSettings.load(settings, configPath()); assertThat(allClientsSettings.size(), is(1)); // test default exists and is an Instance provider final S3ClientSettings defaultClientSettings = allClientsSettings.get("default"); @@ -218,7 +218,7 @@ public void testCredentialsIncomplete() { secureSettings.setString("s3.client." + clientName + ".secret_key", "aws_secret_key"); } final Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - final Exception e = expectThrows(IllegalArgumentException.class, () -> S3ClientSettings.load(settings)); + final Exception e = expectThrows(IllegalArgumentException.class, () -> S3ClientSettings.load(settings, configPath())); if (missingOrMissing) { assertThat(e.getMessage(), containsString("Missing secret key for s3 client [" + clientName + "]")); } else { @@ -308,7 +308,7 @@ public void testSocksProxyConfiguration() throws IOException { .put("s3.client.default.read_timeout", "10s") .build(); - final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, "default"); + final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, "default", configPath()); final ClientConfiguration configuration = S3Service.buildConfiguration(clientSettings); assertEquals(Protocol.HTTPS, configuration.getProtocol()); @@ -342,7 +342,7 @@ private void launchAWSConfigurationTest( int expectedReadTimeout ) { - final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, "default"); + final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, "default", configPath()); final ClientConfiguration configuration = S3Service.buildConfiguration(clientSettings); assertThat(configuration.getResponseMetadataCacheSize(), is(0)); @@ -363,8 +363,7 @@ public void testEndpointSetting() { private void assertEndpoint(Settings repositorySettings, Settings settings, String expectedEndpoint) { final String configName = S3Repository.CLIENT_NAME.get(repositorySettings); - final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, configName); + final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, configName, configPath()); assertThat(clientSettings.endpoint, is(expectedEndpoint)); } - } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/ConfigPathSupport.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/ConfigPathSupport.java new file mode 100644 index 0000000000000..890dd245c67fd --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/ConfigPathSupport.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3; + +import org.opensearch.common.io.PathUtils; + +import java.nio.file.Path; + +/** + * The trait that adds the config path to the test cases + */ +interface ConfigPathSupport { + default Path configPath() { + return PathUtils.get("config"); + } +} diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java index 47a190d3831bc..d3148f89e8a80 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java @@ -57,6 +57,7 @@ import org.opensearch.test.OpenSearchSingleNodeTestCase; import org.opensearch.test.rest.FakeRestRequest; +import java.nio.file.Path; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Collection; @@ -284,8 +285,8 @@ private void createRepository(final String name, final Settings repositorySettin */ public static final class ProxyS3RepositoryPlugin extends S3RepositoryPlugin { - public ProxyS3RepositoryPlugin(Settings settings) { - super(settings, new ProxyS3Service()); + public ProxyS3RepositoryPlugin(Settings settings, Path configPath) { + super(settings, configPath, new ProxyS3Service(configPath)); } @Override @@ -316,6 +317,10 @@ public static final class ProxyS3Service extends S3Service { private static final Logger logger = LogManager.getLogger(ProxyS3Service.class); + ProxyS3Service(final Path configPath) { + super(configPath); + } + @Override AmazonS3WithCredentials buildClient(final S3ClientSettings clientSettings) { final AmazonS3WithCredentials client = super.buildClient(clientSettings); diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java index 4f4ec0afcf9f4..b847890f1ea89 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -81,13 +81,13 @@ * This class tests how a {@link S3BlobContainer} and its underlying AWS S3 client are retrying requests when reading or writing blobs. */ @SuppressForbidden(reason = "use a http server") -public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTestCase { +public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTestCase implements ConfigPathSupport { private S3Service service; @Before public void setUp() throws Exception { - service = new S3Service(); + service = new S3Service(configPath()); super.setUp(); } @@ -140,7 +140,7 @@ protected BlobContainer createBlobContainer( secureSettings.setString(S3ClientSettings.ACCESS_KEY_SETTING.getConcreteSettingForNamespace(clientName).getKey(), "access"); secureSettings.setString(S3ClientSettings.SECRET_KEY_SETTING.getConcreteSettingForNamespace(clientName).getKey(), "secret"); clientSettings.setSecureSettings(secureSettings); - service.refreshAndClearCache(S3ClientSettings.load(clientSettings.build())); + service.refreshAndClearCache(S3ClientSettings.load(clientSettings.build(), configPath())); final RepositoryMetadata repositoryMetadata = new RepositoryMetadata( "repository", diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java index a86ed3af17476..1f25023bf8bd9 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java @@ -35,6 +35,7 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.services.s3.AmazonS3Client; + import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; @@ -51,9 +52,9 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -public class S3ClientSettingsTests extends OpenSearchTestCase { +public class S3ClientSettingsTests extends OpenSearchTestCase implements ConfigPathSupport { public void testThereIsADefaultClientByDefault() { - final Map settings = S3ClientSettings.load(Settings.EMPTY); + final Map settings = S3ClientSettings.load(Settings.EMPTY, configPath()); assertThat(settings.keySet(), contains("default")); final S3ClientSettings defaultSettings = settings.get("default"); @@ -68,7 +69,8 @@ public void testThereIsADefaultClientByDefault() { public void testDefaultClientSettingsCanBeSet() { final Map settings = S3ClientSettings.load( - Settings.builder().put("s3.client.default.max_retries", 10).build() + Settings.builder().put("s3.client.default.max_retries", 10).build(), + configPath() ); assertThat(settings.keySet(), contains("default")); @@ -78,7 +80,8 @@ public void testDefaultClientSettingsCanBeSet() { public void testNondefaultClientCreatedBySettingItsSettings() { final Map settings = S3ClientSettings.load( - Settings.builder().put("s3.client.another_client.max_retries", 10).build() + Settings.builder().put("s3.client.another_client.max_retries", 10).build(), + configPath() ); assertThat(settings.keySet(), contains("default", "another_client")); @@ -94,7 +97,7 @@ public void testRejectionOfLoneAccessKey() { secureSettings.setString("s3.client.default.access_key", "aws_key"); final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()) + () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build(), configPath()) ); assertThat(e.getMessage(), is("Missing secret key for s3 client [default]")); } @@ -104,7 +107,7 @@ public void testRejectionOfLoneSecretKey() { secureSettings.setString("s3.client.default.secret_key", "aws_key"); final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()) + () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build(), configPath()) ); assertThat(e.getMessage(), is("Missing access key for s3 client [default]")); } @@ -114,18 +117,19 @@ public void testRejectionOfLoneSessionToken() { secureSettings.setString("s3.client.default.session_token", "aws_key"); final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()) + () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build(), configPath()) ); assertThat(e.getMessage(), is("Missing access key and secret key for s3 client [default]")); } public void testIrsaCredentialsTypeWithIdentityTokenFile() { final Map settings = S3ClientSettings.load( - Settings.builder().put("s3.client.default.identity_token_file", "file").build() + Settings.builder().put("s3.client.default.identity_token_file", "file").build(), + configPath() ); final S3ClientSettings defaultSettings = settings.get("default"); final S3ClientSettings.IrsaCredentials credentials = defaultSettings.irsaCredentials; - assertThat(credentials.getIdentityTokenFile(), is("file")); + assertThat(credentials.getIdentityTokenFile(), is("config/file")); assertThat(credentials.getRoleArn(), is(nullValue())); assertThat(credentials.getRoleSessionName(), startsWith("s3-sdk-java-")); } @@ -133,7 +137,10 @@ public void testIrsaCredentialsTypeWithIdentityTokenFile() { public void testIrsaCredentialsTypeRoleArn() { final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("s3.client.default.role_arn", "role"); - final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + final Map settings = S3ClientSettings.load( + Settings.builder().setSecureSettings(secureSettings).build(), + configPath() + ); final S3ClientSettings defaultSettings = settings.get("default"); final S3ClientSettings.IrsaCredentials credentials = defaultSettings.irsaCredentials; assertThat(credentials.getRoleArn(), is("role")); @@ -144,23 +151,42 @@ public void testIrsaCredentialsTypeWithRoleArnAndRoleSessionName() { final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("s3.client.default.role_arn", "role"); secureSettings.setString("s3.client.default.role_session_name", "session"); - final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + final Map settings = S3ClientSettings.load( + Settings.builder().setSecureSettings(secureSettings).build(), + configPath() + ); final S3ClientSettings defaultSettings = settings.get("default"); final S3ClientSettings.IrsaCredentials credentials = defaultSettings.irsaCredentials; assertThat(credentials.getRoleArn(), is("role")); assertThat(credentials.getRoleSessionName(), is("session")); } - public void testIrsaCredentialsTypeWithRoleArnAndRoleSessionNameAndIdentityTokeFile() { + public void testIrsaCredentialsTypeWithRoleArnAndRoleSessionNameAndIdentityTokeFileRelative() { final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("s3.client.default.role_arn", "role"); secureSettings.setString("s3.client.default.role_session_name", "session"); final Map settings = S3ClientSettings.load( - Settings.builder().setSecureSettings(secureSettings).put("s3.client.default.identity_token_file", "file").build() + Settings.builder().setSecureSettings(secureSettings).put("s3.client.default.identity_token_file", "file").build(), + configPath() ); final S3ClientSettings defaultSettings = settings.get("default"); final S3ClientSettings.IrsaCredentials credentials = defaultSettings.irsaCredentials; - assertThat(credentials.getIdentityTokenFile(), is("file")); + assertThat(credentials.getIdentityTokenFile(), is("config/file")); + assertThat(credentials.getRoleArn(), is("role")); + assertThat(credentials.getRoleSessionName(), is("session")); + } + + public void testIrsaCredentialsTypeWithRoleArnAndRoleSessionNameAndIdentityTokeFileAbsolute() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.role_arn", "role"); + secureSettings.setString("s3.client.default.role_session_name", "session"); + final Map settings = S3ClientSettings.load( + Settings.builder().setSecureSettings(secureSettings).put("s3.client.default.identity_token_file", "/file").build(), + configPath() + ); + final S3ClientSettings defaultSettings = settings.get("default"); + final S3ClientSettings.IrsaCredentials credentials = defaultSettings.irsaCredentials; + assertThat(credentials.getIdentityTokenFile(), is("/file")); assertThat(credentials.getRoleArn(), is("role")); assertThat(credentials.getRoleSessionName(), is("session")); } @@ -169,7 +195,10 @@ public void testCredentialsTypeWithAccessKeyAndSecretKey() { final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("s3.client.default.access_key", "access_key"); secureSettings.setString("s3.client.default.secret_key", "secret_key"); - final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + final Map settings = S3ClientSettings.load( + Settings.builder().setSecureSettings(secureSettings).build(), + configPath() + ); final S3ClientSettings defaultSettings = settings.get("default"); S3BasicCredentials credentials = defaultSettings.credentials; assertThat(credentials.getAWSAccessKeyId(), is("access_key")); @@ -181,7 +210,10 @@ public void testCredentialsTypeWithAccessKeyAndSecretKeyAndSessionToken() { secureSettings.setString("s3.client.default.access_key", "access_key"); secureSettings.setString("s3.client.default.secret_key", "secret_key"); secureSettings.setString("s3.client.default.session_token", "session_token"); - final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + final Map settings = S3ClientSettings.load( + Settings.builder().setSecureSettings(secureSettings).build(), + configPath() + ); final S3ClientSettings defaultSettings = settings.get("default"); S3BasicSessionCredentials credentials = (S3BasicSessionCredentials) defaultSettings.credentials; assertThat(credentials.getAWSAccessKeyId(), is("access_key")); @@ -194,8 +226,10 @@ public void testRefineWithRepoSettings() { secureSettings.setString("s3.client.default.access_key", "access_key"); secureSettings.setString("s3.client.default.secret_key", "secret_key"); secureSettings.setString("s3.client.default.session_token", "session_token"); - final S3ClientSettings baseSettings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()) - .get("default"); + final S3ClientSettings baseSettings = S3ClientSettings.load( + Settings.builder().setSecureSettings(secureSettings).build(), + configPath() + ).get("default"); { final S3ClientSettings refinedSettings = baseSettings.refine(Settings.EMPTY); @@ -224,7 +258,8 @@ public void testRefineWithRepoSettings() { public void testPathStyleAccessCanBeSet() { final Map settings = S3ClientSettings.load( - Settings.builder().put("s3.client.other.path_style_access", true).build() + Settings.builder().put("s3.client.other.path_style_access", true).build(), + configPath() ); assertThat(settings.get("default").pathStyleAccess, is(false)); assertThat(settings.get("other").pathStyleAccess, is(true)); @@ -232,7 +267,8 @@ public void testPathStyleAccessCanBeSet() { public void testUseChunkedEncodingCanBeSet() { final Map settings = S3ClientSettings.load( - Settings.builder().put("s3.client.other.disable_chunked_encoding", true).build() + Settings.builder().put("s3.client.other.disable_chunked_encoding", true).build(), + configPath() ); assertThat(settings.get("default").disableChunkedEncoding, is(false)); assertThat(settings.get("other").disableChunkedEncoding, is(true)); @@ -241,11 +277,12 @@ public void testUseChunkedEncodingCanBeSet() { public void testRegionCanBeSet() { final String region = randomAlphaOfLength(5); final Map settings = S3ClientSettings.load( - Settings.builder().put("s3.client.other.region", region).build() + Settings.builder().put("s3.client.other.region", region).build(), + configPath() ); assertThat(settings.get("default").region, is("")); assertThat(settings.get("other").region, is(region)); - try (S3Service s3Service = new S3Service()) { + try (S3Service s3Service = new S3Service(configPath())) { AmazonS3Client other = (AmazonS3Client) s3Service.buildClient(settings.get("other")).client(); assertThat(other.getSignerRegionOverride(), is(region)); } @@ -254,7 +291,8 @@ public void testRegionCanBeSet() { public void testSignerOverrideCanBeSet() { final String signerOverride = randomAlphaOfLength(5); final Map settings = S3ClientSettings.load( - Settings.builder().put("s3.client.other.signer_override", signerOverride).build() + Settings.builder().put("s3.client.other.signer_override", signerOverride).build(), + configPath() ); assertThat(settings.get("default").region, is("")); assertThat(settings.get("other").signerOverride, is(signerOverride)); @@ -281,7 +319,7 @@ public void testSetProxySettings() throws Exception { .setSecureSettings(secureSettings) .build(); - final S3ClientSettings s3ClientSettings = S3ClientSettings.load(settings).get("default"); + final S3ClientSettings s3ClientSettings = S3ClientSettings.load(settings, configPath()).get("default"); assertEquals(ProxySettings.ProxyType.valueOf(proxyType.toUpperCase(Locale.ROOT)), s3ClientSettings.proxySettings.getType()); assertEquals(new InetSocketAddress(InetAddress.getByName("127.0.0.10"), port), s3ClientSettings.proxySettings.getAddress()); @@ -295,7 +333,7 @@ public void testProxyWrongHost() { .put("s3.client.default.proxy.host", "thisisnotavalidhostorwehavebeensuperunlucky") .put("s3.client.default.proxy.port", 8080) .build(); - final SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings)); + final SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings, configPath())); assertEquals("S3 proxy host is unknown.", e.getMessage()); } @@ -305,7 +343,7 @@ public void testProxyTypeNotSet() { .put("s3.client.default.proxy.port", 8080) .build(); - SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(hostPortSettings)); + SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(hostPortSettings, configPath())); assertEquals("S3 proxy port or host or username or password have been set but proxy type is not defined.", e.getMessage()); final MockSecureSettings secureSettings = new MockSecureSettings(); @@ -313,7 +351,7 @@ public void testProxyTypeNotSet() { secureSettings.setString("s3.client.default.proxy.password", "bbbb"); final Settings usernamePasswordSettings = Settings.builder().setSecureSettings(secureSettings).build(); - e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(usernamePasswordSettings)); + e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(usernamePasswordSettings, configPath())); assertEquals("S3 proxy port or host or username or password have been set but proxy type is not defined.", e.getMessage()); } @@ -322,7 +360,7 @@ public void testProxyHostNotSet() { .put("s3.client.default.proxy.port", 8080) .put("s3.client.default.proxy.type", randomFrom("socks", "http", "https")) .build(); - final SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings)); + final SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings, configPath())); assertEquals("S3 proxy type has been set but proxy host or port is not defined.", e.getMessage()); } @@ -333,7 +371,6 @@ public void testSocksDoesNotSupportForHttpProtocol() { .put("s3.client.default.protocol", "http") .put("s3.client.default.proxy.type", "socks") .build(); - expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings)); + expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings, configPath())); } - } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java index 26fb29904510c..673ee36f01eb7 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java @@ -45,6 +45,7 @@ import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.Matchers; +import java.nio.file.Path; import java.util.Map; import static org.hamcrest.Matchers.containsString; @@ -52,7 +53,7 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; -public class S3RepositoryTests extends OpenSearchTestCase { +public class S3RepositoryTests extends OpenSearchTestCase implements ConfigPathSupport { private static class DummyS3Client extends AbstractAmazonS3 { @@ -63,6 +64,10 @@ public void shutdown() { } private static class DummyS3Service extends S3Service { + DummyS3Service(final Path configPath) { + super(configPath); + } + @Override public AmazonS3Reference client(RepositoryMetadata repositoryMetadata) { return new AmazonS3Reference(new DummyS3Client()); @@ -139,7 +144,7 @@ private S3Repository createS3Repo(RepositoryMetadata metadata) { return new S3Repository( metadata, NamedXContentRegistry.EMPTY, - new DummyS3Service(), + new DummyS3Service(configPath()), BlobStoreTestUtil.mockClusterService(), new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) ) { diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ServiceTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ServiceTests.java index 71e42907ab997..9f6c5e51309c5 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ServiceTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ServiceTests.java @@ -38,10 +38,10 @@ import java.util.Map; -public class S3ServiceTests extends OpenSearchTestCase { +public class S3ServiceTests extends OpenSearchTestCase implements ConfigPathSupport { public void testCachedClientsAreReleased() { - final S3Service s3Service = new S3Service(); + final S3Service s3Service = new S3Service(configPath()); final Settings settings = Settings.builder().put("endpoint", "http://first").build(); final RepositoryMetadata metadata1 = new RepositoryMetadata("first", "s3", settings); final RepositoryMetadata metadata2 = new RepositoryMetadata("second", "s3", settings); @@ -63,9 +63,10 @@ public void testCachedClientsWithCredentialsAreReleased() { final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("s3.client.default.role_arn", "role"); final Map defaults = S3ClientSettings.load( - Settings.builder().setSecureSettings(secureSettings).put("s3.client.default.identity_token_file", "file").build() + Settings.builder().setSecureSettings(secureSettings).put("s3.client.default.identity_token_file", "file").build(), + configPath() ); - final S3Service s3Service = new S3Service(); + final S3Service s3Service = new S3Service(configPath()); s3Service.refreshAndClearCache(defaults); final Settings settings = Settings.builder().put("endpoint", "http://first").put("region", "us-east-2").build(); final RepositoryMetadata metadata1 = new RepositoryMetadata("first", "s3", settings);