diff --git a/CHANGELOG.md b/CHANGELOG.md index bd9f12877a213..3d88385d5f4e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -127,6 +127,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix weighted shard routing state across search requests([#6004](https://github.com/opensearch-project/OpenSearch/pull/6004)) - [Segment Replication] Fix bug where inaccurate sequence numbers are sent during replication ([#6122](https://github.com/opensearch-project/OpenSearch/pull/6122)) - Enable numeric sort optimisation for few numerical sort types ([#6321](https://github.com/opensearch-project/OpenSearch/pull/6321)) +- Fix Opensearch repository-s3 plugin cannot read ServiceAccount token ([#6390](https://github.com/opensearch-project/OpenSearch/pull/6390) ### Security diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java index 3c3f2887469b3..71606b1c0237a 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -58,6 +58,7 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -149,8 +150,8 @@ protected Settings nodeSettings(int nodeOrdinal) { */ public static class TestS3RepositoryPlugin extends S3RepositoryPlugin { - public TestS3RepositoryPlugin(final Settings settings) { - super(settings); + public TestS3RepositoryPlugin(final Settings settings, final Path configPath) { + super(settings, configPath); } @Override diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java index 1f9af5314f30d..a20cc156545ce 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java @@ -35,6 +35,8 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import org.opensearch.common.Strings; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.io.PathUtils; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.SecureSetting; import org.opensearch.common.settings.SecureString; @@ -46,6 +48,7 @@ import java.net.InetAddress; import java.net.UnknownHostException; +import java.nio.file.Path; import java.util.Collections; import java.util.HashMap; import java.util.Locale; @@ -348,16 +351,16 @@ S3ClientSettings refine(Settings repositorySettings) { * * Note this will always at least return a client named "default". */ - static Map load(Settings settings) { + static Map load(final Settings settings, final Path configPath) { final Set clientNames = settings.getGroups(PREFIX).keySet(); final Map clients = new HashMap<>(); for (final String clientName : clientNames) { - clients.put(clientName, getClientSettings(settings, clientName)); + clients.put(clientName, getClientSettings(settings, clientName, configPath)); } if (clients.containsKey("default") == false) { // this won't find any settings under the default client, // but it will pull all the fallback static settings - clients.put("default", getClientSettings(settings, "default")); + clients.put("default", getClientSettings(settings, "default", configPath)); } return Collections.unmodifiableMap(clients); } @@ -425,8 +428,17 @@ private static S3BasicCredentials loadCredentials(Settings settings, String clie } } - private static IrsaCredentials loadIrsaCredentials(Settings settings, String clientName) { + @SuppressForbidden(reason = "PathUtils#get") + private static IrsaCredentials loadIrsaCredentials(Settings settings, String clientName, Path configPath) { String identityTokenFile = getConfigValue(settings, clientName, IDENTITY_TOKEN_FILE_SETTING); + if (identityTokenFile.length() != 0) { + final Path identityTokenFilePath = PathUtils.get(identityTokenFile); + // If the path is not absolute, resolve it relatively to config path + if (!identityTokenFilePath.isAbsolute()) { + identityTokenFile = PathUtils.get(new Path[] { configPath }, identityTokenFile).toString(); + } + } + try ( SecureString roleArn = getConfigValue(settings, clientName, ROLE_ARN_SETTING); SecureString roleSessionName = getConfigValue(settings, clientName, ROLE_SESSION_NAME_SETTING) @@ -441,11 +453,11 @@ private static IrsaCredentials loadIrsaCredentials(Settings settings, String cli // pkg private for tests /** Parse settings for a single client. */ - static S3ClientSettings getClientSettings(final Settings settings, final String clientName) { + static S3ClientSettings getClientSettings(final Settings settings, final String clientName, final Path configPath) { final Protocol awsProtocol = getConfigValue(settings, clientName, PROTOCOL_SETTING); return new S3ClientSettings( S3ClientSettings.loadCredentials(settings, clientName), - S3ClientSettings.loadIrsaCredentials(settings, clientName), + S3ClientSettings.loadIrsaCredentials(settings, clientName, configPath), getConfigValue(settings, clientName, ENDPOINT_SETTING), awsProtocol, Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()), diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java index e1ea31dc53d1e..271e8f161dbd1 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java @@ -47,6 +47,7 @@ import org.opensearch.repositories.Repository; import java.io.IOException; +import java.nio.file.Path; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Arrays; @@ -77,15 +78,17 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo } protected final S3Service service; + private final Path configPath; - public S3RepositoryPlugin(final Settings settings) { - this(settings, new S3Service()); + public S3RepositoryPlugin(final Settings settings, final Path configPath) { + this(settings, configPath, new S3Service(configPath)); } - S3RepositoryPlugin(final Settings settings, final S3Service service) { + S3RepositoryPlugin(final Settings settings, final Path configPath, final S3Service service) { this.service = Objects.requireNonNull(service, "S3 service must not be null"); + this.configPath = configPath; // eagerly load client settings so that secure settings are read - final Map clientsSettings = S3ClientSettings.load(settings); + final Map clientsSettings = S3ClientSettings.load(settings, configPath); this.service.refreshAndClearCache(clientsSettings); } @@ -142,7 +145,7 @@ public List> getSettings() { @Override public void reload(Settings settings) { // secure settings should be readable - final Map clientsSettings = S3ClientSettings.load(settings); + final Map clientsSettings = S3ClientSettings.load(settings, configPath); service.refreshAndClearCache(clientsSettings); } diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java index 930af6f8a9799..ac74857737107 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java @@ -72,6 +72,7 @@ import java.net.PasswordAuthentication; import java.net.Proxy; import java.net.Socket; +import java.nio.file.Path; import java.security.SecureRandom; import java.util.Map; @@ -90,9 +91,7 @@ class S3Service implements Closeable { /** * Client settings calculated from static configuration and settings in the keystore. */ - private volatile Map staticClientSettings = MapBuilder.newMapBuilder() - .put("default", S3ClientSettings.getClientSettings(Settings.EMPTY, "default")) - .immutableMap(); + private volatile Map staticClientSettings; /** * Client settings derived from those in {@link #staticClientSettings} by combining them with settings @@ -100,6 +99,12 @@ class S3Service implements Closeable { */ private volatile Map derivedClientSettings = emptyMap(); + S3Service(final Path configPath) { + staticClientSettings = MapBuilder.newMapBuilder() + .put("default", S3ClientSettings.getClientSettings(Settings.EMPTY, "default", configPath)) + .immutableMap(); + } + /** * Refreshes the settings for the AmazonS3 clients and clears the cache of * existing clients. New clients will be build using these new settings. Old diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java index 76bd5d303e5fb..7670c940046d4 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java @@ -39,12 +39,14 @@ import com.amazonaws.http.IdleConnectionReaper; import org.junit.AfterClass; +import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; import org.opensearch.test.OpenSearchTestCase; import java.io.Closeable; import java.io.IOException; +import java.nio.file.Path; import java.util.HashMap; import java.util.Locale; import java.util.Map; @@ -64,7 +66,7 @@ public static void shutdownIdleConnectionReaper() { public void testAWSCredentialsDefaultToInstanceProviders() { final String inexistentClientName = randomAlphaOfLength(8).toLowerCase(Locale.ROOT); - final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(Settings.EMPTY, inexistentClientName); + final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(Settings.EMPTY, inexistentClientName, configPath()); final AWSCredentialsProvider credentialsProvider = S3Service.buildCredentials(logger, clientSettings); assertThat(credentialsProvider, instanceOf(S3Service.PrivilegedInstanceProfileCredentialsProvider.class)); } @@ -79,7 +81,7 @@ public void testAWSCredentialsFromKeystore() { secureSettings.setString("s3.client." + clientName + ".secret_key", clientName + "_aws_secret_key"); } final Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - final Map allClientsSettings = S3ClientSettings.load(settings); + final Map allClientsSettings = S3ClientSettings.load(settings, configPath()); // no less, no more assertThat(allClientsSettings.size(), is(clientsCount + 1)); // including default for (int i = 0; i < clientsCount; i++) { @@ -114,7 +116,7 @@ public void testCredentialsAndIrsaWithIdentityTokenFileCredentialsFromKeystore() plainSettings.put("s3.client." + clientName + ".identity_token_file", clientName + "_identity_token_file"); } final Settings settings = Settings.builder().loadFromMap(plainSettings).setSecureSettings(secureSettings).build(); - final Map allClientsSettings = S3ClientSettings.load(settings); + final Map allClientsSettings = S3ClientSettings.load(settings, configPath()); // no less, no more assertThat(allClientsSettings.size(), is(clientsCount + 1)); // including default for (int i = 0; i < clientsCount; i++) { @@ -148,7 +150,7 @@ public void testCredentialsAndIrsaCredentialsFromKeystore() throws IOException { plainSettings.put("s3.client." + clientName + ".region", "us-east1"); } final Settings settings = Settings.builder().loadFromMap(plainSettings).setSecureSettings(secureSettings).build(); - final Map allClientsSettings = S3ClientSettings.load(settings); + final Map allClientsSettings = S3ClientSettings.load(settings, configPath()); // no less, no more assertThat(allClientsSettings.size(), is(clientsCount + 1)); // including default for (int i = 0; i < clientsCount; i++) { @@ -175,7 +177,7 @@ public void testIrsaCredentialsFromKeystore() throws IOException { secureSettings.setString("s3.client." + clientName + ".role_session_name", clientName + "_role_session_name"); } final Settings settings = Settings.builder().loadFromMap(plainSettings).setSecureSettings(secureSettings).build(); - final Map allClientsSettings = S3ClientSettings.load(settings); + final Map allClientsSettings = S3ClientSettings.load(settings, configPath()); // no less, no more assertThat(allClientsSettings.size(), is(clientsCount + 1)); // including default for (int i = 0; i < clientsCount; i++) { @@ -198,7 +200,7 @@ public void testSetDefaultCredential() { secureSettings.setString("s3.client.default.access_key", awsAccessKey); secureSettings.setString("s3.client.default.secret_key", awsSecretKey); final Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - final Map allClientsSettings = S3ClientSettings.load(settings); + final Map allClientsSettings = S3ClientSettings.load(settings, configPath()); assertThat(allClientsSettings.size(), is(1)); // test default exists and is an Instance provider final S3ClientSettings defaultClientSettings = allClientsSettings.get("default"); @@ -218,7 +220,7 @@ public void testCredentialsIncomplete() { secureSettings.setString("s3.client." + clientName + ".secret_key", "aws_secret_key"); } final Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - final Exception e = expectThrows(IllegalArgumentException.class, () -> S3ClientSettings.load(settings)); + final Exception e = expectThrows(IllegalArgumentException.class, () -> S3ClientSettings.load(settings, configPath())); if (missingOrMissing) { assertThat(e.getMessage(), containsString("Missing secret key for s3 client [" + clientName + "]")); } else { @@ -308,7 +310,7 @@ public void testSocksProxyConfiguration() throws IOException { .put("s3.client.default.read_timeout", "10s") .build(); - final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, "default"); + final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, "default", configPath()); final ClientConfiguration configuration = S3Service.buildConfiguration(clientSettings); assertEquals(Protocol.HTTPS, configuration.getProtocol()); @@ -342,7 +344,7 @@ private void launchAWSConfigurationTest( int expectedReadTimeout ) { - final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, "default"); + final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, "default", configPath()); final ClientConfiguration configuration = S3Service.buildConfiguration(clientSettings); assertThat(configuration.getResponseMetadataCacheSize(), is(0)); @@ -363,8 +365,11 @@ public void testEndpointSetting() { private void assertEndpoint(Settings repositorySettings, Settings settings, String expectedEndpoint) { final String configName = S3Repository.CLIENT_NAME.get(repositorySettings); - final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, configName); + final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, configName, configPath()); assertThat(clientSettings.endpoint, is(expectedEndpoint)); } + private Path configPath() { + return PathUtils.get("config"); + } } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java index a30b36cdd659c..04a7006d86ed9 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java @@ -57,6 +57,7 @@ import org.opensearch.test.OpenSearchSingleNodeTestCase; import org.opensearch.test.rest.FakeRestRequest; +import java.nio.file.Path; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Collection; @@ -284,8 +285,8 @@ private void createRepository(final String name, final Settings repositorySettin */ public static final class ProxyS3RepositoryPlugin extends S3RepositoryPlugin { - public ProxyS3RepositoryPlugin(Settings settings) { - super(settings, new ProxyS3Service()); + public ProxyS3RepositoryPlugin(Settings settings, Path configPath) { + super(settings, configPath, new ProxyS3Service(configPath)); } @Override @@ -316,6 +317,10 @@ public static final class ProxyS3Service extends S3Service { private static final Logger logger = LogManager.getLogger(ProxyS3Service.class); + ProxyS3Service(final Path configPath) { + super(configPath); + } + @Override AmazonS3WithCredentials buildClient(final S3ClientSettings clientSettings) { final AmazonS3WithCredentials client = super.buildClient(clientSettings); diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java index 4f4ec0afcf9f4..241fc7c550da4 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -41,6 +41,7 @@ import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.Streams; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.lucene.store.InputStreamIndexInput; @@ -63,6 +64,7 @@ import java.net.InetSocketAddress; import java.net.SocketTimeoutException; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.Locale; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; @@ -87,7 +89,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes @Before public void setUp() throws Exception { - service = new S3Service(); + service = new S3Service(configPath()); super.setUp(); } @@ -140,7 +142,7 @@ protected BlobContainer createBlobContainer( secureSettings.setString(S3ClientSettings.ACCESS_KEY_SETTING.getConcreteSettingForNamespace(clientName).getKey(), "access"); secureSettings.setString(S3ClientSettings.SECRET_KEY_SETTING.getConcreteSettingForNamespace(clientName).getKey(), "secret"); clientSettings.setSecureSettings(secureSettings); - service.refreshAndClearCache(S3ClientSettings.load(clientSettings.build())); + service.refreshAndClearCache(S3ClientSettings.load(clientSettings.build(), configPath())); final RepositoryMetadata repositoryMetadata = new RepositoryMetadata( "repository", @@ -401,4 +403,8 @@ public void close() throws IOException { } } } + + private Path configPath() { + return PathUtils.get("config"); + } } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java index a86ed3af17476..7e76a656c2d1d 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java @@ -35,6 +35,8 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.services.s3.AmazonS3Client; + +import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; @@ -42,6 +44,7 @@ import java.net.InetAddress; import java.net.InetSocketAddress; +import java.nio.file.Path; import java.util.Locale; import java.util.Map; @@ -53,7 +56,7 @@ public class S3ClientSettingsTests extends OpenSearchTestCase { public void testThereIsADefaultClientByDefault() { - final Map settings = S3ClientSettings.load(Settings.EMPTY); + final Map settings = S3ClientSettings.load(Settings.EMPTY, configPath()); assertThat(settings.keySet(), contains("default")); final S3ClientSettings defaultSettings = settings.get("default"); @@ -68,7 +71,8 @@ public void testThereIsADefaultClientByDefault() { public void testDefaultClientSettingsCanBeSet() { final Map settings = S3ClientSettings.load( - Settings.builder().put("s3.client.default.max_retries", 10).build() + Settings.builder().put("s3.client.default.max_retries", 10).build(), + configPath() ); assertThat(settings.keySet(), contains("default")); @@ -78,7 +82,8 @@ public void testDefaultClientSettingsCanBeSet() { public void testNondefaultClientCreatedBySettingItsSettings() { final Map settings = S3ClientSettings.load( - Settings.builder().put("s3.client.another_client.max_retries", 10).build() + Settings.builder().put("s3.client.another_client.max_retries", 10).build(), + configPath() ); assertThat(settings.keySet(), contains("default", "another_client")); @@ -94,7 +99,7 @@ public void testRejectionOfLoneAccessKey() { secureSettings.setString("s3.client.default.access_key", "aws_key"); final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()) + () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build(), configPath()) ); assertThat(e.getMessage(), is("Missing secret key for s3 client [default]")); } @@ -104,7 +109,7 @@ public void testRejectionOfLoneSecretKey() { secureSettings.setString("s3.client.default.secret_key", "aws_key"); final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()) + () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build(), configPath()) ); assertThat(e.getMessage(), is("Missing access key for s3 client [default]")); } @@ -114,18 +119,19 @@ public void testRejectionOfLoneSessionToken() { secureSettings.setString("s3.client.default.session_token", "aws_key"); final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()) + () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build(), configPath()) ); assertThat(e.getMessage(), is("Missing access key and secret key for s3 client [default]")); } public void testIrsaCredentialsTypeWithIdentityTokenFile() { final Map settings = S3ClientSettings.load( - Settings.builder().put("s3.client.default.identity_token_file", "file").build() + Settings.builder().put("s3.client.default.identity_token_file", "file").build(), + configPath() ); final S3ClientSettings defaultSettings = settings.get("default"); final S3ClientSettings.IrsaCredentials credentials = defaultSettings.irsaCredentials; - assertThat(credentials.getIdentityTokenFile(), is("file")); + assertThat(credentials.getIdentityTokenFile(), is("config/file")); assertThat(credentials.getRoleArn(), is(nullValue())); assertThat(credentials.getRoleSessionName(), startsWith("s3-sdk-java-")); } @@ -133,7 +139,10 @@ public void testIrsaCredentialsTypeWithIdentityTokenFile() { public void testIrsaCredentialsTypeRoleArn() { final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("s3.client.default.role_arn", "role"); - final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + final Map settings = S3ClientSettings.load( + Settings.builder().setSecureSettings(secureSettings).build(), + configPath() + ); final S3ClientSettings defaultSettings = settings.get("default"); final S3ClientSettings.IrsaCredentials credentials = defaultSettings.irsaCredentials; assertThat(credentials.getRoleArn(), is("role")); @@ -144,23 +153,42 @@ public void testIrsaCredentialsTypeWithRoleArnAndRoleSessionName() { final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("s3.client.default.role_arn", "role"); secureSettings.setString("s3.client.default.role_session_name", "session"); - final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + final Map settings = S3ClientSettings.load( + Settings.builder().setSecureSettings(secureSettings).build(), + configPath() + ); final S3ClientSettings defaultSettings = settings.get("default"); final S3ClientSettings.IrsaCredentials credentials = defaultSettings.irsaCredentials; assertThat(credentials.getRoleArn(), is("role")); assertThat(credentials.getRoleSessionName(), is("session")); } - public void testIrsaCredentialsTypeWithRoleArnAndRoleSessionNameAndIdentityTokeFile() { + public void testIrsaCredentialsTypeWithRoleArnAndRoleSessionNameAndIdentityTokeFileRelative() { final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("s3.client.default.role_arn", "role"); secureSettings.setString("s3.client.default.role_session_name", "session"); final Map settings = S3ClientSettings.load( - Settings.builder().setSecureSettings(secureSettings).put("s3.client.default.identity_token_file", "file").build() + Settings.builder().setSecureSettings(secureSettings).put("s3.client.default.identity_token_file", "file").build(), + configPath() ); final S3ClientSettings defaultSettings = settings.get("default"); final S3ClientSettings.IrsaCredentials credentials = defaultSettings.irsaCredentials; - assertThat(credentials.getIdentityTokenFile(), is("file")); + assertThat(credentials.getIdentityTokenFile(), is("config/file")); + assertThat(credentials.getRoleArn(), is("role")); + assertThat(credentials.getRoleSessionName(), is("session")); + } + + public void testIrsaCredentialsTypeWithRoleArnAndRoleSessionNameAndIdentityTokeFileAbsolute() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.role_arn", "role"); + secureSettings.setString("s3.client.default.role_session_name", "session"); + final Map settings = S3ClientSettings.load( + Settings.builder().setSecureSettings(secureSettings).put("s3.client.default.identity_token_file", "/file").build(), + configPath() + ); + final S3ClientSettings defaultSettings = settings.get("default"); + final S3ClientSettings.IrsaCredentials credentials = defaultSettings.irsaCredentials; + assertThat(credentials.getIdentityTokenFile(), is("/file")); assertThat(credentials.getRoleArn(), is("role")); assertThat(credentials.getRoleSessionName(), is("session")); } @@ -169,7 +197,10 @@ public void testCredentialsTypeWithAccessKeyAndSecretKey() { final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("s3.client.default.access_key", "access_key"); secureSettings.setString("s3.client.default.secret_key", "secret_key"); - final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + final Map settings = S3ClientSettings.load( + Settings.builder().setSecureSettings(secureSettings).build(), + configPath() + ); final S3ClientSettings defaultSettings = settings.get("default"); S3BasicCredentials credentials = defaultSettings.credentials; assertThat(credentials.getAWSAccessKeyId(), is("access_key")); @@ -181,7 +212,10 @@ public void testCredentialsTypeWithAccessKeyAndSecretKeyAndSessionToken() { secureSettings.setString("s3.client.default.access_key", "access_key"); secureSettings.setString("s3.client.default.secret_key", "secret_key"); secureSettings.setString("s3.client.default.session_token", "session_token"); - final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + final Map settings = S3ClientSettings.load( + Settings.builder().setSecureSettings(secureSettings).build(), + configPath() + ); final S3ClientSettings defaultSettings = settings.get("default"); S3BasicSessionCredentials credentials = (S3BasicSessionCredentials) defaultSettings.credentials; assertThat(credentials.getAWSAccessKeyId(), is("access_key")); @@ -194,8 +228,10 @@ public void testRefineWithRepoSettings() { secureSettings.setString("s3.client.default.access_key", "access_key"); secureSettings.setString("s3.client.default.secret_key", "secret_key"); secureSettings.setString("s3.client.default.session_token", "session_token"); - final S3ClientSettings baseSettings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()) - .get("default"); + final S3ClientSettings baseSettings = S3ClientSettings.load( + Settings.builder().setSecureSettings(secureSettings).build(), + configPath() + ).get("default"); { final S3ClientSettings refinedSettings = baseSettings.refine(Settings.EMPTY); @@ -224,7 +260,8 @@ public void testRefineWithRepoSettings() { public void testPathStyleAccessCanBeSet() { final Map settings = S3ClientSettings.load( - Settings.builder().put("s3.client.other.path_style_access", true).build() + Settings.builder().put("s3.client.other.path_style_access", true).build(), + configPath() ); assertThat(settings.get("default").pathStyleAccess, is(false)); assertThat(settings.get("other").pathStyleAccess, is(true)); @@ -232,7 +269,8 @@ public void testPathStyleAccessCanBeSet() { public void testUseChunkedEncodingCanBeSet() { final Map settings = S3ClientSettings.load( - Settings.builder().put("s3.client.other.disable_chunked_encoding", true).build() + Settings.builder().put("s3.client.other.disable_chunked_encoding", true).build(), + configPath() ); assertThat(settings.get("default").disableChunkedEncoding, is(false)); assertThat(settings.get("other").disableChunkedEncoding, is(true)); @@ -241,11 +279,12 @@ public void testUseChunkedEncodingCanBeSet() { public void testRegionCanBeSet() { final String region = randomAlphaOfLength(5); final Map settings = S3ClientSettings.load( - Settings.builder().put("s3.client.other.region", region).build() + Settings.builder().put("s3.client.other.region", region).build(), + configPath() ); assertThat(settings.get("default").region, is("")); assertThat(settings.get("other").region, is(region)); - try (S3Service s3Service = new S3Service()) { + try (S3Service s3Service = new S3Service(configPath())) { AmazonS3Client other = (AmazonS3Client) s3Service.buildClient(settings.get("other")).client(); assertThat(other.getSignerRegionOverride(), is(region)); } @@ -254,7 +293,8 @@ public void testRegionCanBeSet() { public void testSignerOverrideCanBeSet() { final String signerOverride = randomAlphaOfLength(5); final Map settings = S3ClientSettings.load( - Settings.builder().put("s3.client.other.signer_override", signerOverride).build() + Settings.builder().put("s3.client.other.signer_override", signerOverride).build(), + configPath() ); assertThat(settings.get("default").region, is("")); assertThat(settings.get("other").signerOverride, is(signerOverride)); @@ -281,7 +321,7 @@ public void testSetProxySettings() throws Exception { .setSecureSettings(secureSettings) .build(); - final S3ClientSettings s3ClientSettings = S3ClientSettings.load(settings).get("default"); + final S3ClientSettings s3ClientSettings = S3ClientSettings.load(settings, configPath()).get("default"); assertEquals(ProxySettings.ProxyType.valueOf(proxyType.toUpperCase(Locale.ROOT)), s3ClientSettings.proxySettings.getType()); assertEquals(new InetSocketAddress(InetAddress.getByName("127.0.0.10"), port), s3ClientSettings.proxySettings.getAddress()); @@ -295,7 +335,7 @@ public void testProxyWrongHost() { .put("s3.client.default.proxy.host", "thisisnotavalidhostorwehavebeensuperunlucky") .put("s3.client.default.proxy.port", 8080) .build(); - final SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings)); + final SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings, configPath())); assertEquals("S3 proxy host is unknown.", e.getMessage()); } @@ -305,7 +345,7 @@ public void testProxyTypeNotSet() { .put("s3.client.default.proxy.port", 8080) .build(); - SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(hostPortSettings)); + SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(hostPortSettings, configPath())); assertEquals("S3 proxy port or host or username or password have been set but proxy type is not defined.", e.getMessage()); final MockSecureSettings secureSettings = new MockSecureSettings(); @@ -313,7 +353,7 @@ public void testProxyTypeNotSet() { secureSettings.setString("s3.client.default.proxy.password", "bbbb"); final Settings usernamePasswordSettings = Settings.builder().setSecureSettings(secureSettings).build(); - e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(usernamePasswordSettings)); + e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(usernamePasswordSettings, configPath())); assertEquals("S3 proxy port or host or username or password have been set but proxy type is not defined.", e.getMessage()); } @@ -322,7 +362,7 @@ public void testProxyHostNotSet() { .put("s3.client.default.proxy.port", 8080) .put("s3.client.default.proxy.type", randomFrom("socks", "http", "https")) .build(); - final SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings)); + final SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings, configPath())); assertEquals("S3 proxy type has been set but proxy host or port is not defined.", e.getMessage()); } @@ -333,7 +373,10 @@ public void testSocksDoesNotSupportForHttpProtocol() { .put("s3.client.default.protocol", "http") .put("s3.client.default.proxy.type", "socks") .build(); - expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings)); + expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings, configPath())); } + private Path configPath() { + return PathUtils.get("config"); + } } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java index da1f5d71b4da5..4372167bef730 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java @@ -34,6 +34,7 @@ import com.amazonaws.services.s3.AbstractAmazonS3; import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; @@ -45,6 +46,7 @@ import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.Matchers; +import java.nio.file.Path; import java.util.Map; import static org.hamcrest.Matchers.containsString; @@ -63,6 +65,10 @@ public void shutdown() { } private static class DummyS3Service extends S3Service { + DummyS3Service(final Path configPath) { + super(configPath); + } + @Override public AmazonS3Reference client(RepositoryMetadata repositoryMetadata) { return new AmazonS3Reference(new DummyS3Client()); @@ -139,7 +145,7 @@ private S3Repository createS3Repo(RepositoryMetadata metadata) { return new S3Repository( metadata, NamedXContentRegistry.EMPTY, - new DummyS3Service(), + new DummyS3Service(configPath()), BlobStoreTestUtil.mockClusterService(), new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) ) { @@ -149,4 +155,8 @@ protected void assertSnapshotOrGenericThread() { } }; } + + private Path configPath() { + return PathUtils.get("config"); + } } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ServiceTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ServiceTests.java index 71e42907ab997..e8a1b6a9ea0cc 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ServiceTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ServiceTests.java @@ -32,16 +32,18 @@ package org.opensearch.repositories.s3; import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; import org.opensearch.test.OpenSearchTestCase; +import java.nio.file.Path; import java.util.Map; public class S3ServiceTests extends OpenSearchTestCase { public void testCachedClientsAreReleased() { - final S3Service s3Service = new S3Service(); + final S3Service s3Service = new S3Service(configPath()); final Settings settings = Settings.builder().put("endpoint", "http://first").build(); final RepositoryMetadata metadata1 = new RepositoryMetadata("first", "s3", settings); final RepositoryMetadata metadata2 = new RepositoryMetadata("second", "s3", settings); @@ -63,9 +65,10 @@ public void testCachedClientsWithCredentialsAreReleased() { final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("s3.client.default.role_arn", "role"); final Map defaults = S3ClientSettings.load( - Settings.builder().setSecureSettings(secureSettings).put("s3.client.default.identity_token_file", "file").build() + Settings.builder().setSecureSettings(secureSettings).put("s3.client.default.identity_token_file", "file").build(), + configPath() ); - final S3Service s3Service = new S3Service(); + final S3Service s3Service = new S3Service(configPath()); s3Service.refreshAndClearCache(defaults); final Settings settings = Settings.builder().put("endpoint", "http://first").put("region", "us-east-2").build(); final RepositoryMetadata metadata1 = new RepositoryMetadata("first", "s3", settings); @@ -83,4 +86,8 @@ public void testCachedClientsWithCredentialsAreReleased() { final S3ClientSettings clientSettingsReloaded = s3Service.settings(metadata1); assertNotSame(clientSettings, clientSettingsReloaded); } + + private Path configPath() { + return PathUtils.get("config"); + } }