From ee8dae30530861c6a9709e93d093c5c9ba81aca1 Mon Sep 17 00:00:00 2001 From: Vikas Bansal <43470111+vikasvb90@users.noreply.github.com> Date: Thu, 7 Sep 2023 02:44:04 +0530 Subject: [PATCH 01/10] Remvoving ESDK changes from core (#9833) Signed-off-by: Vikas Bansal <43470111+vikasvb90@users.noreply.github.com> --- distribution/tools/plugin-cli/build.gradle | 4 - .../common/crypto/CryptoHandler.java | 3 +- libs/encryption-sdk/build.gradle | 47 ---- .../aws-encryption-sdk-java-2.4.0.jar.sha1 | 1 - .../aws-encryption-sdk-java-NOTICE.txt | 11 - .../src/forbidden/crypto-signatures.txt | 13 -- .../src/forbidden/crypto-test-signatures.txt | 13 -- .../opensearch/encryption/CryptoManager.java | 33 --- .../encryption/CryptoManagerFactory.java | 127 ----------- .../encryption/NoOpCryptoHandler.java | 128 ----------- .../opensearch/encryption/TrimmingStream.java | 119 ----------- .../keyprovider/CryptoMasterKey.java | 84 -------- .../encryption/keyprovider/package-info.java | 12 -- .../opensearch/encryption/package-info.java | 12 -- .../encryption/CryptoManagerFactoryTests.java | 92 -------- .../encryption/MockKeyProvider.java | 109 ---------- .../encryption/NoOpCryptoHandlerTests.java | 95 -------- .../encryption/TrimmingStreamTests.java | 125 ----------- .../src/test/resources/encrypted_key | 1 - .../resources/raw_content_for_crypto_test | 25 --- .../encryption-sdk/src/test/resources/raw_key | 1 - plugins/crypto-kms/build.gradle | 6 +- .../org/opensearch/crypto/kms/KmsService.java | 5 +- .../licenses/commons-lang3-3.13.0.jar.sha1 | 0 .../licenses/commons-lang3-LICENSE.txt | 0 .../licenses/commons-lang3-NOTICE.txt | 0 .../licenses/commons-lang3-3.13.0.jar.sha1 | 1 + .../licenses/commons-lang3-LICENSE.txt | 5 +- .../licenses/commons-lang3-NOTICE.txt | 5 + .../licenses/commons-lang3-3.13.0.jar.sha1 | 1 + .../licenses/commons-lang3-LICENSE.txt | 202 ++++++++++++++++++ .../licenses/commons-lang3-NOTICE.txt | 5 + server/build.gradle | 1 - .../common/blobstore/EncryptedBlobStore.java | 24 +-- ...gistry.java => CryptoHandlerRegistry.java} | 93 ++++---- .../main/java/org/opensearch/node/Node.java | 9 +- .../org/opensearch/plugins/CryptoPlugin.java | 37 ++++ ...s.java => CryptoHandlerRegistryTests.java} | 74 ++++--- .../RepositoriesServiceTests.java | 177 ++++++--------- 39 files changed, 439 insertions(+), 1261 deletions(-) delete mode 100644 libs/encryption-sdk/build.gradle delete mode 100644 libs/encryption-sdk/licenses/aws-encryption-sdk-java-2.4.0.jar.sha1 delete mode 100644 libs/encryption-sdk/licenses/aws-encryption-sdk-java-NOTICE.txt delete mode 100644 libs/encryption-sdk/src/forbidden/crypto-signatures.txt delete mode 100644 libs/encryption-sdk/src/forbidden/crypto-test-signatures.txt delete mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/CryptoManager.java delete mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/CryptoManagerFactory.java delete mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/NoOpCryptoHandler.java delete mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/TrimmingStream.java delete mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/keyprovider/CryptoMasterKey.java delete mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/keyprovider/package-info.java delete mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/package-info.java delete mode 100644 libs/encryption-sdk/src/test/java/org/opensearch/encryption/CryptoManagerFactoryTests.java delete mode 100644 libs/encryption-sdk/src/test/java/org/opensearch/encryption/MockKeyProvider.java delete mode 100644 libs/encryption-sdk/src/test/java/org/opensearch/encryption/NoOpCryptoHandlerTests.java delete mode 100644 libs/encryption-sdk/src/test/java/org/opensearch/encryption/TrimmingStreamTests.java delete mode 100644 libs/encryption-sdk/src/test/resources/encrypted_key delete mode 100644 libs/encryption-sdk/src/test/resources/raw_content_for_crypto_test delete mode 100644 libs/encryption-sdk/src/test/resources/raw_key rename {libs/encryption-sdk => plugins/ingest-attachment}/licenses/commons-lang3-3.13.0.jar.sha1 (100%) rename {libs/encryption-sdk => plugins/ingest-attachment}/licenses/commons-lang3-LICENSE.txt (100%) rename {libs/encryption-sdk => plugins/ingest-attachment}/licenses/commons-lang3-NOTICE.txt (100%) create mode 100644 plugins/repository-azure/licenses/commons-lang3-3.13.0.jar.sha1 rename libs/encryption-sdk/licenses/aws-encryption-sdk-java-LICENSE.txt => plugins/repository-azure/licenses/commons-lang3-LICENSE.txt (99%) create mode 100644 plugins/repository-azure/licenses/commons-lang3-NOTICE.txt create mode 100644 plugins/repository-hdfs/licenses/commons-lang3-3.13.0.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/commons-lang3-LICENSE.txt create mode 100644 plugins/repository-hdfs/licenses/commons-lang3-NOTICE.txt rename server/src/main/java/org/opensearch/crypto/{CryptoManagerRegistry.java => CryptoHandlerRegistry.java} (58%) create mode 100644 server/src/main/java/org/opensearch/plugins/CryptoPlugin.java rename server/src/test/java/org/opensearch/crypto/{CryptoManagerRegistryTests.java => CryptoHandlerRegistryTests.java} (72%) diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index d78c16d17de72..5103999428814 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -48,10 +48,6 @@ dependencies { implementation 'org.apache.commons:commons-compress:1.23.0' } -configurations.implementation { - exclude group: 'org.bouncycastle', module: 'bcprov-jdk15to18' -} - tasks.named("dependencyLicenses").configure { mapping from: /bc.*/, to: 'bouncycastle' } diff --git a/libs/common/src/main/java/org/opensearch/common/crypto/CryptoHandler.java b/libs/common/src/main/java/org/opensearch/common/crypto/CryptoHandler.java index f8fd983543117..37322310c7287 100644 --- a/libs/common/src/main/java/org/opensearch/common/crypto/CryptoHandler.java +++ b/libs/common/src/main/java/org/opensearch/common/crypto/CryptoHandler.java @@ -11,6 +11,7 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.io.InputStreamContainer; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; @@ -22,7 +23,7 @@ * U - Parsed Encryption Metadata / CryptoContext */ @ExperimentalApi -public interface CryptoHandler { +public interface CryptoHandler extends Closeable { /** * To initialise or create a new crypto metadata to be used in encryption. This is needed to set the context before diff --git a/libs/encryption-sdk/build.gradle b/libs/encryption-sdk/build.gradle deleted file mode 100644 index c87394f8d9cb9..0000000000000 --- a/libs/encryption-sdk/build.gradle +++ /dev/null @@ -1,47 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -apply plugin: 'opensearch.build' -apply plugin: 'opensearch.publish' - -forbiddenApis.ignoreFailures = false - -thirdPartyAudit.enabled = false -forbiddenApisTest.ignoreFailures = true -testingConventions.enabled = false - -dependencies { - // Common crypto classes - api project(':libs:opensearch-common') - - // Encryption - implementation "com.amazonaws:aws-encryption-sdk-java:2.4.0" - implementation "org.apache.commons:commons-lang3:${versions.commonslang}" - - //Tests - testImplementation "junit:junit:${versions.junit}" - testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" - testImplementation(project(":test:framework")) { - exclude group: 'org.opensearch', module: 'opensearch-encryption-sdk' - } - - compileOnly 'com.google.code.findbugs:annotations:3.0.1' -} - -tasks.named('forbiddenApisMain').configure { - // Only enable limited check because AD code has too many violations. - replaceSignatureFiles 'jdk-signatures' - signaturesFiles += files('src/forbidden/crypto-signatures.txt') -} - -// Encryption SDK files have missing java docs so disabling for the lib. -tasks.named('missingJavadoc').configure { - enabled = false -} - -forbiddenApisTest.setSignaturesFiles(files('src/forbidden/crypto-test-signatures.txt')) diff --git a/libs/encryption-sdk/licenses/aws-encryption-sdk-java-2.4.0.jar.sha1 b/libs/encryption-sdk/licenses/aws-encryption-sdk-java-2.4.0.jar.sha1 deleted file mode 100644 index 504b4a423a975..0000000000000 --- a/libs/encryption-sdk/licenses/aws-encryption-sdk-java-2.4.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -98943eda1dc05bb01f4f5405e115b08dc541afbf \ No newline at end of file diff --git a/libs/encryption-sdk/licenses/aws-encryption-sdk-java-NOTICE.txt b/libs/encryption-sdk/licenses/aws-encryption-sdk-java-NOTICE.txt deleted file mode 100644 index e32695955374a..0000000000000 --- a/libs/encryption-sdk/licenses/aws-encryption-sdk-java-NOTICE.txt +++ /dev/null @@ -1,11 +0,0 @@ -AWS Encryption SDK -Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. - -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: - --Cryptographic functions from Bouncy Castle Crypto APIs for Java - Copyright -2000-2013 The Legion of the Bouncy Castle - -The licenses for these third party components are included in LICENSE.txt diff --git a/libs/encryption-sdk/src/forbidden/crypto-signatures.txt b/libs/encryption-sdk/src/forbidden/crypto-signatures.txt deleted file mode 100644 index 3699186679924..0000000000000 --- a/libs/encryption-sdk/src/forbidden/crypto-signatures.txt +++ /dev/null @@ -1,13 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -@defaultMessage use format with Locale -java.lang.String#format(java.lang.String,java.lang.Object[]) \ No newline at end of file diff --git a/libs/encryption-sdk/src/forbidden/crypto-test-signatures.txt b/libs/encryption-sdk/src/forbidden/crypto-test-signatures.txt deleted file mode 100644 index 3699186679924..0000000000000 --- a/libs/encryption-sdk/src/forbidden/crypto-test-signatures.txt +++ /dev/null @@ -1,13 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -@defaultMessage use format with Locale -java.lang.String#format(java.lang.String,java.lang.Object[]) \ No newline at end of file diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/CryptoManager.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/CryptoManager.java deleted file mode 100644 index 8e1fc8570d552..0000000000000 --- a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/CryptoManager.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.encryption; - -import org.opensearch.common.crypto.CryptoHandler; -import org.opensearch.common.util.concurrent.RefCounted; - -/** - * Crypto plugin interface used for encryption and decryption. - */ -public interface CryptoManager extends RefCounted { - - /** - * @return key provider type - */ - String type(); - - /** - * @return key provider name - */ - String name(); - - /** - * @return Crypto provider for encrypting or decrypting raw content. - */ - CryptoHandler getCryptoProvider(); -} diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/CryptoManagerFactory.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/CryptoManagerFactory.java deleted file mode 100644 index e1dc9291ed1a6..0000000000000 --- a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/CryptoManagerFactory.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.encryption; - -import org.opensearch.common.crypto.CryptoHandler; -import org.opensearch.common.crypto.MasterKeyProvider; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.AbstractRefCounted; -import org.opensearch.encryption.keyprovider.CryptoMasterKey; - -import java.security.SecureRandom; -import java.util.concurrent.TimeUnit; - -import com.amazonaws.encryptionsdk.CryptoAlgorithm; -import com.amazonaws.encryptionsdk.caching.CachingCryptoMaterialsManager; -import com.amazonaws.encryptionsdk.caching.LocalCryptoMaterialsCache; - -public class CryptoManagerFactory { - - private final int dataKeyCacheSize; - private final String algorithm; - - // - Cache TTL and Jitter is used to decide the Crypto Cache TTL. - // - Random number between: (TTL Jitter, TTL - Jitter) - private final long dataKeyCacheTTL; - private static final long dataKeyCacheJitter = TimeUnit.MINUTES.toMillis(30); // - 30 minutes - - public CryptoManagerFactory(String algorithm, TimeValue keyRefreshInterval, int keyCacheSize) { - this.dataKeyCacheSize = keyCacheSize; - validateAndGetAlgorithmId(algorithm); - this.algorithm = algorithm; - dataKeyCacheTTL = keyRefreshInterval.getMillis(); - } - - private String validateAndGetAlgorithmId(String algorithm) { - // Supporting only 256 bit algorithm - switch (algorithm) { - case "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY": - return CryptoAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY.getDataKeyAlgo(); - case "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384": - return CryptoAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384.getDataKeyAlgo(); - default: - throw new IllegalArgumentException("Unsupported algorithm: " + algorithm); - } - } - - public CryptoManager getOrCreateCryptoManager( - MasterKeyProvider keyProvider, - String keyProviderName, - String keyProviderType, - Runnable onClose - ) { - CachingCryptoMaterialsManager materialsManager = createMaterialsManager( - keyProvider, - keyProviderName, - validateAndGetAlgorithmId(algorithm) - ); - CryptoHandler cryptoHandler = createCryptoProvider(algorithm, materialsManager, keyProvider); - return createCryptoManager(cryptoHandler, keyProviderType, keyProviderName, onClose); - } - - // package private for tests - CryptoHandler createCryptoProvider( - String algorithm, - CachingCryptoMaterialsManager materialsManager, - MasterKeyProvider masterKeyProvider - ) { - return new NoOpCryptoHandler(); - } - - // Package private for tests - CachingCryptoMaterialsManager createMaterialsManager(MasterKeyProvider masterKeyProvider, String keyProviderName, String algorithm) { - SecureRandom r = new SecureRandom(); - long low = dataKeyCacheTTL - dataKeyCacheJitter; - long high = dataKeyCacheTTL + dataKeyCacheJitter; - long masterKeyCacheTTL = r.nextInt((int) (high - low)) + low; - - CryptoMasterKey cryptoMasterKey = new CryptoMasterKey(masterKeyProvider, keyProviderName, algorithm); - return CachingCryptoMaterialsManager.newBuilder() - .withMasterKeyProvider(cryptoMasterKey) - .withCache(new LocalCryptoMaterialsCache(dataKeyCacheSize)) - .withMaxAge(masterKeyCacheTTL, TimeUnit.MILLISECONDS) - .build(); - } - - // package private for tests - CryptoManager createCryptoManager( - CryptoHandler cryptoHandler, - String keyProviderType, - String keyProviderName, - Runnable onClose - ) { - return new CryptoManagerImpl(keyProviderName, keyProviderType) { - @Override - protected void closeInternal() { - onClose.run(); - } - - @Override - public String type() { - return keyProviderType; - } - - @Override - public String name() { - return keyProviderName; - } - - @Override - public CryptoHandler getCryptoProvider() { - return cryptoHandler; - } - }; - } - - private static abstract class CryptoManagerImpl extends AbstractRefCounted implements CryptoManager { - public CryptoManagerImpl(String keyProviderName, String keyProviderType) { - super(keyProviderName + "-" + keyProviderType); - } - } -} diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/NoOpCryptoHandler.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/NoOpCryptoHandler.java deleted file mode 100644 index d6b23ed08c6b0..0000000000000 --- a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/NoOpCryptoHandler.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.encryption; - -import org.opensearch.common.crypto.CryptoHandler; -import org.opensearch.common.crypto.DecryptedRangedStreamProvider; -import org.opensearch.common.crypto.EncryptedHeaderContentSupplier; -import org.opensearch.common.io.InputStreamContainer; - -import java.io.IOException; -import java.io.InputStream; - -public class NoOpCryptoHandler implements CryptoHandler { - - /** - * No op - Initialises metadata store used in encryption. - * @return crypto metadata object constructed with encryption metadata like data key pair, encryption algorithm, etc. - */ - public Object initEncryptionMetadata() { - return new Object(); - } - - /** - * No op content size adjustment of length of a partial content used in partial encryption. - * - * @param cryptoContextObj stateful object for a request consisting of materials required in encryption. - * @param streamSize Size of the stream to be adjusted. - * @return Adjusted size of the stream. - */ - public long adjustContentSizeForPartialEncryption(Object cryptoContextObj, long streamSize) { - return streamSize; - } - - /** - * No op - Estimate length of the encrypted stream. - * - * @param cryptoMetadataObj crypto metadata instance - * @param contentLength Size of the raw content - * @return Calculated size of the encrypted stream for the provided raw stream. - */ - public long estimateEncryptedLengthOfEntireContent(Object cryptoMetadataObj, long contentLength) { - return contentLength; - } - - /** - * No op length estimation for a given content length. - * - * @param cryptoMetadataObj crypto metadata instance - * @param contentLength Size of the encrypted content - * @return Calculated size of the encrypted stream for the provided raw stream. - */ - public long estimateDecryptedLength(Object cryptoMetadataObj, long contentLength) { - return contentLength; - } - - /** - * No op encrypting stream wrapper. - * - * @param cryptoContextObj consists encryption metadata. - * @param stream Raw InputStream to encrypt - * @return encrypting stream wrapped around raw InputStream. - */ - public InputStreamContainer createEncryptingStream(Object cryptoContextObj, InputStreamContainer stream) { - return stream; - } - - /** - * No op encrypting stream provider for a part of content. - * - * @param cryptoContextObj stateful object for a request consisting of materials required in encryption. - * @param stream raw stream for which encrypted stream has to be created. - * @param totalStreams Number of streams being used for the entire content. - * @param streamIdx Index of the current stream. - * @return Encrypted stream for the provided raw stream. - */ - public InputStreamContainer createEncryptingStreamOfPart( - Object cryptoContextObj, - InputStreamContainer stream, - int totalStreams, - int streamIdx - ) { - return stream; - } - - /** - * - * @param encryptedHeaderContentSupplier Supplier used to fetch bytes from source for header creation - * @return parsed encryption metadata object - * @throws IOException if content fetch for header creation fails - */ - public Object loadEncryptionMetadata(EncryptedHeaderContentSupplier encryptedHeaderContentSupplier) throws IOException { - return new Object(); - } - - /** - * No op decrypting stream provider. - * - * @param encryptedStream to be decrypted. - * @return Decrypting wrapper stream - */ - public InputStream createDecryptingStream(InputStream encryptedStream) { - return encryptedStream; - } - - /** - * No Op decrypted stream range provider - * - * @param cryptoContext crypto metadata instance consisting of encryption metadata used in encryption. - * @param startPosOfRawContent starting position in the raw/decrypted content - * @param endPosOfRawContent ending position in the raw/decrypted content - * @return stream provider for decrypted stream for the specified range of content including adjusted range - */ - public DecryptedRangedStreamProvider createDecryptingStreamOfRange( - Object cryptoContext, - long startPosOfRawContent, - long endPosOfRawContent - ) { - long[] range = { startPosOfRawContent, endPosOfRawContent }; - return new DecryptedRangedStreamProvider(range, (encryptedStream) -> encryptedStream); - } - -} diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/TrimmingStream.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/TrimmingStream.java deleted file mode 100644 index d6640bbe5e79e..0000000000000 --- a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/TrimmingStream.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.encryption; - -import java.io.IOException; -import java.io.InputStream; - -/** - * Trims content from a given source range to a target range. - */ -public class TrimmingStream extends InputStream { - - private final long sourceStart; - private final long sourceEnd; - private final long targetStart; - private final long targetEnd; - private final InputStream in; - - private long offsetFromStart = 0; - - public TrimmingStream(long sourceStart, long sourceEnd, long targetStart, long targetEnd, InputStream in) { - if (sourceStart < 0 - || targetStart < 0 - || targetEnd < 0 - || targetStart > targetEnd - || sourceStart > targetStart - || sourceEnd < targetEnd) { - throw new IllegalArgumentException("Invalid arguments to the bounded stream"); - } - - this.sourceStart = sourceStart; - this.sourceEnd = sourceEnd; - this.targetStart = targetStart; - this.targetEnd = targetEnd; - this.in = in; - } - - private void skipBytesOutsideBounds() throws IOException { - long relativeOffset = offsetFromStart + sourceStart; - - if (relativeOffset < targetStart) { - skipBytes(relativeOffset, targetStart); - } - - if (relativeOffset > targetEnd) { - skipBytes(relativeOffset, sourceEnd + 1); - } - } - - private void skipBytes(long offset, long end) throws IOException { - long bytesToSkip = end - offset; - while (bytesToSkip > 0) { - long skipped = skip(bytesToSkip); - if (skipped <= 0) { - // End of stream or unable to skip further - break; - } - bytesToSkip -= skipped; - } - } - - @Override - public int read() throws IOException { - skipBytesOutsideBounds(); - if (offsetFromStart + sourceStart > targetEnd) { - return -1; - } - int b = in.read(); - if (b != -1) { - offsetFromStart++; - } - // This call is made again to ensure that source stream is fully consumed when it reaches end of target range. - skipBytesOutsideBounds(); - return b; - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - skipBytesOutsideBounds(); - if (offsetFromStart + sourceStart > targetEnd) { - return -1; - } - len = (int) Math.min(len, targetEnd - offsetFromStart - sourceStart + 1); - int bytesRead = in.read(b, off, len); - if (bytesRead != -1) { - offsetFromStart += bytesRead; - } - // This call is made again to ensure that source stream is fully consumed when it reaches end of target range. - skipBytesOutsideBounds(); - return bytesRead; - } - - /** - * Skips specified number of bytes of input. - * @param n the number of bytes to skip - * @return the actual number of bytes skipped - * @throws IOException if an I/O error has occurred - */ - public long skip(long n) throws IOException { - byte[] buf = new byte[512]; - long total = 0; - while (total < n) { - long len = n - total; - len = in.read(buf, 0, len < buf.length ? (int) len : buf.length); - if (len == -1) { - return total; - } - offsetFromStart += len; - total += len; - } - return total; - } -} diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/keyprovider/CryptoMasterKey.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/keyprovider/CryptoMasterKey.java deleted file mode 100644 index 6f014c9b4d99b..0000000000000 --- a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/keyprovider/CryptoMasterKey.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.encryption.keyprovider; - -import org.opensearch.common.crypto.DataKeyPair; -import org.opensearch.common.crypto.MasterKeyProvider; - -import javax.crypto.spec.SecretKeySpec; - -import java.io.Closeable; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Collection; -import java.util.Map; - -import com.amazonaws.encryptionsdk.CryptoAlgorithm; -import com.amazonaws.encryptionsdk.DataKey; -import com.amazonaws.encryptionsdk.EncryptedDataKey; -import com.amazonaws.encryptionsdk.MasterKey; -import com.amazonaws.encryptionsdk.exception.AwsCryptoException; - -public class CryptoMasterKey extends MasterKey implements Closeable { - private final MasterKeyProvider keyProvider; - private final String keyProviderName; - private final String cryptoAlgorithm; - - public CryptoMasterKey(MasterKeyProvider keyProvider, String keyProviderName, String cryptoAlgorithm) { - this.keyProvider = keyProvider; - this.keyProviderName = keyProviderName; - this.cryptoAlgorithm = cryptoAlgorithm; - } - - @Override - public String getProviderId() { - return keyProviderName; - } - - @Override - public String getKeyId() { - return keyProvider.getKeyId(); - } - - @Override - public DataKey generateDataKey(CryptoAlgorithm algorithm, Map encryptionContext) { - DataKeyPair dataKeyPairResponse = keyProvider.generateDataPair(); - final SecretKeySpec key = new SecretKeySpec(dataKeyPairResponse.getRawKey(), cryptoAlgorithm); - return new DataKey<>(key, dataKeyPairResponse.getEncryptedKey(), getKeyId().getBytes(StandardCharsets.UTF_8), this); - } - - @Override - public DataKey encryptDataKey(CryptoAlgorithm algorithm, Map encryptionContext, DataKey dataKey) { - throw new UnsupportedOperationException("Multiple data-key encryption is not supported."); - } - - @Override - public DataKey decryptDataKey( - CryptoAlgorithm algorithm, - Collection encryptedDataKeys, - Map encryptionContext - ) throws AwsCryptoException { - if (encryptedDataKeys == null || encryptedDataKeys.isEmpty()) { - throw new IllegalArgumentException("No encrypted data key passed for decryption."); - } - EncryptedDataKey encryptedDataKey = encryptedDataKeys.iterator().next(); - final String keyId = new String(encryptedDataKey.getProviderInformation(), StandardCharsets.UTF_8); - if (!this.getKeyId().equals(keyId)) { - throw new IllegalArgumentException("Invalid provider info present in encrypted key."); - } - - byte[] encryptedKey = encryptedDataKey.getEncryptedDataKey(); - byte[] rawKey = keyProvider.decryptKey(encryptedKey); - return new DataKey<>(new SecretKeySpec(rawKey, cryptoAlgorithm), encryptedKey, keyId.getBytes(StandardCharsets.UTF_8), this); - } - - @Override - public void close() throws IOException { - keyProvider.close(); - } -} diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/keyprovider/package-info.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/keyprovider/package-info.java deleted file mode 100644 index 611b095a54250..0000000000000 --- a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/keyprovider/package-info.java +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/** - * Key provider package specific to encryption sdk - */ -package org.opensearch.encryption.keyprovider; diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/package-info.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/package-info.java deleted file mode 100644 index 1fa008797ce87..0000000000000 --- a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/package-info.java +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/** - * Crypto plugin to for encryption and decryption use cases. - */ -package org.opensearch.encryption; diff --git a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/CryptoManagerFactoryTests.java b/libs/encryption-sdk/src/test/java/org/opensearch/encryption/CryptoManagerFactoryTests.java deleted file mode 100644 index fb5c477232bc4..0000000000000 --- a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/CryptoManagerFactoryTests.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.encryption; - -import org.opensearch.common.crypto.CryptoHandler; -import org.opensearch.common.crypto.MasterKeyProvider; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.test.OpenSearchTestCase; -import org.junit.Before; - -import java.util.Collections; - -import com.amazonaws.encryptionsdk.caching.CachingCryptoMaterialsManager; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class CryptoManagerFactoryTests extends OpenSearchTestCase { - - private CryptoManagerFactory cryptoManagerFactory; - - @Before - public void setup() { - cryptoManagerFactory = new CryptoManagerFactory( - "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", - TimeValue.timeValueDays(2), - 10 - ); - } - - public void testGetOrCreateCryptoManager() { - MasterKeyProvider mockKeyProvider = mock(MasterKeyProvider.class); - when(mockKeyProvider.getEncryptionContext()).thenReturn(Collections.emptyMap()); - - CryptoManager cryptoManager = cryptoManagerFactory.getOrCreateCryptoManager( - mockKeyProvider, - "keyProviderName", - "keyProviderType", - () -> {} - ); - - assertNotNull(cryptoManager); - } - - public void testCreateCryptoProvider() { - CachingCryptoMaterialsManager mockMaterialsManager = mock(CachingCryptoMaterialsManager.class); - MasterKeyProvider mockKeyProvider = mock(MasterKeyProvider.class); - when(mockKeyProvider.getEncryptionContext()).thenReturn(Collections.emptyMap()); - - CryptoHandler cryptoHandler = cryptoManagerFactory.createCryptoProvider( - "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", - mockMaterialsManager, - mockKeyProvider - ); - - assertNotNull(cryptoHandler); - } - - public void testCreateMaterialsManager() { - MasterKeyProvider mockKeyProvider = mock(MasterKeyProvider.class); - when(mockKeyProvider.getEncryptionContext()).thenReturn(Collections.emptyMap()); - - CachingCryptoMaterialsManager materialsManager = cryptoManagerFactory.createMaterialsManager( - mockKeyProvider, - "keyProviderName", - "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384" - ); - - assertNotNull(materialsManager); - } - - public void testCreateCryptoManager() { - CryptoHandler mockCryptoHandler = mock(CryptoHandler.class); - CryptoManager cryptoManager = cryptoManagerFactory.createCryptoManager( - mockCryptoHandler, - "keyProviderName", - "keyProviderType", - null - ); - assertNotNull(cryptoManager); - } - - public void testUnsupportedAlgorithm() { - expectThrows(IllegalArgumentException.class, () -> new CryptoManagerFactory("Unsupported_algo", TimeValue.timeValueDays(2), 10)); - } -} diff --git a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/MockKeyProvider.java b/libs/encryption-sdk/src/test/java/org/opensearch/encryption/MockKeyProvider.java deleted file mode 100644 index a5e74534ef32b..0000000000000 --- a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/MockKeyProvider.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.encryption; - -import javax.crypto.spec.SecretKeySpec; - -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.Collection; -import java.util.Map; - -import com.amazonaws.encryptionsdk.CryptoAlgorithm; -import com.amazonaws.encryptionsdk.DataKey; -import com.amazonaws.encryptionsdk.MasterKey; -import com.amazonaws.encryptionsdk.exception.AwsCryptoException; -import com.amazonaws.encryptionsdk.exception.UnsupportedProviderException; - -@SuppressWarnings({ "rawtypes", "unchecked" }) -public class MockKeyProvider extends MasterKey { - - private static final String keyId = "test-key-id"; - - public static byte[] loadFile(String file) { - byte[] content; - try { - InputStream in = MockKeyProvider.class.getResourceAsStream(file); - StringBuilder stringBuilder = new StringBuilder(); - BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); - for (String line; (line = bufferedReader.readLine()) != null;) { - stringBuilder.append(line); - } - content = stringBuilder.toString().getBytes(StandardCharsets.UTF_8); - } catch (Exception e) { - throw new IllegalArgumentException("File " + file + " cannot be read correctly."); - } - String text = new String(content, StandardCharsets.UTF_8); - - String[] byteValues = text.substring(1, text.length() - 1).split(","); - byte[] bytes = new byte[byteValues.length]; - - for (int i = 0, len = bytes.length; i < len; i++) { - bytes[i] = Byte.parseByte(byteValues[i].trim()); - } - - return bytes; - } - - private static final byte[] rawKey = loadFile("/raw_key"); - private static final byte[] encryptedKey = loadFile("/encrypted_key"); - - @Override - public String getProviderId() { - return "sample-provider-id"; - } - - @Override - public String getKeyId() { - return "Sample-key-id"; - } - - @Override - public DataKey encryptDataKey(CryptoAlgorithm algorithm, Map encryptionContext, DataKey dataKey) { - throw new UnsupportedOperationException("Multiple data-key encryption is not supported."); - } - - @Override - public DataKey generateDataKey(CryptoAlgorithm algorithm, Map encryptionContext) { - final SecretKeySpec key = new SecretKeySpec(rawKey, algorithm.getDataKeyAlgo()); - return new DataKey(key, encryptedKey, getKeyId().getBytes(StandardCharsets.UTF_8), this); - } - - @Override - public DataKey decryptDataKey(CryptoAlgorithm algorithm, Collection collection, Map encryptionContext) - throws UnsupportedProviderException, AwsCryptoException { - return new DataKey<>( - new SecretKeySpec(rawKey, algorithm.getDataKeyAlgo()), - encryptedKey, - keyId.getBytes(StandardCharsets.UTF_8), - this - ); - } - - static class DataKeyPair { - private final byte[] rawKey; - private final byte[] encryptedKey; - - public DataKeyPair(byte[] rawKey, byte[] encryptedKey) { - this.rawKey = rawKey; - this.encryptedKey = encryptedKey; - } - - public byte[] getRawKey() { - return this.rawKey; - } - - public byte[] getEncryptedKey() { - return this.encryptedKey; - } - } - -} diff --git a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/NoOpCryptoHandlerTests.java b/libs/encryption-sdk/src/test/java/org/opensearch/encryption/NoOpCryptoHandlerTests.java deleted file mode 100644 index 5e3836fd10988..0000000000000 --- a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/NoOpCryptoHandlerTests.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.encryption; - -import org.opensearch.common.crypto.DecryptedRangedStreamProvider; -import org.opensearch.common.crypto.EncryptedHeaderContentSupplier; -import org.opensearch.common.io.InputStreamContainer; -import org.opensearch.test.OpenSearchTestCase; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; - -public class NoOpCryptoHandlerTests extends OpenSearchTestCase { - - public void testInitEncryptionMetadata() { - NoOpCryptoHandler cryptoProvider = new NoOpCryptoHandler(); - Object encryptionMetadata = cryptoProvider.initEncryptionMetadata(); - assertNotNull(encryptionMetadata); - } - - public void testAdjustContentSizeForPartialEncryption() { - NoOpCryptoHandler cryptoProvider = new NoOpCryptoHandler(); - long originalSize = 1000L; - long adjustedSize = cryptoProvider.adjustContentSizeForPartialEncryption(new Object(), originalSize); - assertEquals(originalSize, adjustedSize); - } - - public void testEstimateEncryptedLengthOfEntireContent() { - NoOpCryptoHandler cryptoProvider = new NoOpCryptoHandler(); - long originalSize = 2000L; - long estimatedSize = cryptoProvider.estimateEncryptedLengthOfEntireContent(new Object(), originalSize); - assertEquals(originalSize, estimatedSize); - } - - public void testEstimateDecryptedLength() { - NoOpCryptoHandler cryptoProvider = new NoOpCryptoHandler(); - long originalSize = 1500L; - long estimatedSize = cryptoProvider.estimateDecryptedLength(new Object(), originalSize); - assertEquals(originalSize, estimatedSize); - } - - public void testCreateEncryptingStream() { - NoOpCryptoHandler cryptoProvider = new NoOpCryptoHandler(); - InputStreamContainer inputStream = randomStream(); - InputStreamContainer encryptedStream = cryptoProvider.createEncryptingStream(new Object(), inputStream); - assertEquals(inputStream, encryptedStream); - } - - public void testCreateEncryptingStreamOfPart() { - NoOpCryptoHandler cryptoProvider = new NoOpCryptoHandler(); - InputStreamContainer inputStream = randomStream(); - InputStreamContainer encryptedStream = cryptoProvider.createEncryptingStreamOfPart(new Object(), inputStream, 2, 1); - assertEquals(inputStream, encryptedStream); - } - - private InputStreamContainer randomStream() { - byte[] bytes = randomAlphaOfLength(10).getBytes(); - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); - int offset = randomIntBetween(0, bytes.length - 1); - return new InputStreamContainer(byteArrayInputStream, bytes.length, offset); - } - - public void testLoadEncryptionMetadata() throws IOException { - NoOpCryptoHandler cryptoProvider = new NoOpCryptoHandler(); - EncryptedHeaderContentSupplier supplier = (start, length) -> { throw new UnsupportedOperationException("Not implemented"); }; - Object encryptionMetadata = cryptoProvider.loadEncryptionMetadata(supplier); - assertNotNull(encryptionMetadata); - } - - public void testCreateDecryptingStream() { - NoOpCryptoHandler cryptoProvider = new NoOpCryptoHandler(); - InputStream encryptedStream = randomStream().getInputStream(); - InputStream decryptedStream = cryptoProvider.createDecryptingStream(encryptedStream); - assertEquals(encryptedStream, decryptedStream); - } - - public void testCreateDecryptingStreamOfRange() { - NoOpCryptoHandler cryptoProvider = new NoOpCryptoHandler(); - Object cryptoContext = new Object(); - long startPos = 0L; - long endPos = 100L; - DecryptedRangedStreamProvider streamProvider = cryptoProvider.createDecryptingStreamOfRange(cryptoContext, startPos, endPos); - assertNotNull(streamProvider); - InputStream stream = randomStream().getInputStream(); - InputStream decryptedStream = streamProvider.getDecryptedStreamProvider().apply(stream); // Replace with your encrypted input stream - assertEquals(stream, decryptedStream); - } -} diff --git a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/TrimmingStreamTests.java b/libs/encryption-sdk/src/test/java/org/opensearch/encryption/TrimmingStreamTests.java deleted file mode 100644 index f0d957d81e1e1..0000000000000 --- a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/TrimmingStreamTests.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.encryption; - -import org.opensearch.test.OpenSearchTestCase; - -import java.io.ByteArrayInputStream; -import java.io.IOException; - -public class TrimmingStreamTests extends OpenSearchTestCase { - - static class ReadCountInputStreamTest extends ByteArrayInputStream { - - public ReadCountInputStreamTest(byte[] buf) { - super(buf); - } - - public int getPos() { - return pos; - } - } - - public void testReadInRange() throws IOException { - byte[] data = generateRandomData(100); - ReadCountInputStreamTest input = new ReadCountInputStreamTest(data); - - long sourceStart = generateRandomValue(0, 80); - long sourceEnd = generateRandomValue(sourceStart, 99); - long targetStart = generateRandomValue(sourceStart, sourceEnd); - long targetEnd = generateRandomValue(targetStart, sourceEnd); - - TrimmingStream trimmingStream = new TrimmingStream(sourceStart, sourceEnd, targetStart, targetEnd, input); - - byte[] result = new byte[(int) (sourceEnd - sourceStart + 1)]; - int bytesRead = trimmingStream.read(result, 0, result.length); - - long expectedBytesRead = targetEnd - targetStart + 1; - assertEquals(expectedBytesRead, bytesRead); - assertEquals(sourceEnd - sourceStart + 1, input.getPos()); - } - - public void testReadOutsideRange() throws IOException { - byte[] data = generateRandomData(100); - ReadCountInputStreamTest input = new ReadCountInputStreamTest(data); - - long sourceStart = generateRandomValue(0, 80); - long sourceEnd = generateRandomValue(sourceStart, 99); - long targetStart = generateRandomValue(sourceStart, sourceEnd); - long targetEnd = generateRandomValue(targetStart, sourceEnd); - - TrimmingStream trimmingStream = new TrimmingStream(sourceStart, sourceEnd, targetStart, targetEnd, input); - - byte[] result = new byte[(int) (targetEnd - targetStart + 1)]; - int bytesRead = trimmingStream.read(result, 0, result.length); - - long expectedBytesRead = targetEnd - targetStart + 1; - assertEquals(expectedBytesRead, bytesRead); - assertEquals(sourceEnd - sourceStart + 1, input.getPos()); - - // Try to read more bytes, should return -1 (end of stream) - int additionalBytesRead = trimmingStream.read(result, 0, 50); - assertEquals(-1, additionalBytesRead); - assertEquals(sourceEnd - sourceStart + 1, input.getPos()); - } - - public void testSingleByteReadInRange() throws IOException { - byte[] data = generateRandomData(100); - ReadCountInputStreamTest input = new ReadCountInputStreamTest(data); - - long sourceStart = generateRandomValue(0, 80); - long sourceEnd = generateRandomValue(sourceStart, 99); - long targetStart = generateRandomValue(sourceStart, sourceEnd); - long targetEnd = generateRandomValue(targetStart, sourceEnd); - - TrimmingStream trimmingStream = new TrimmingStream(sourceStart, sourceEnd, targetStart, targetEnd, input); - - int bytesRead = 0; - int value; - while ((value = trimmingStream.read()) != -1) { - bytesRead++; - } - - long expectedBytesRead = targetEnd - targetStart + 1; - assertEquals(expectedBytesRead, bytesRead); - assertEquals(sourceEnd - sourceStart + 1, input.getPos()); - } - - public void testInvalidInputs() { - assertThrows(IllegalArgumentException.class, () -> new TrimmingStream(-10, 60, 20, 40, new ByteArrayInputStream(new byte[100]))); - assertThrows(IllegalArgumentException.class, () -> new TrimmingStream(10, 60, 40, 20, new ByteArrayInputStream(new byte[100]))); - } - - public void testSourceSameAsTarget() throws IOException { - byte[] data = generateRandomData(100); - ReadCountInputStreamTest input = new ReadCountInputStreamTest(data); - - long sourceStart = generateRandomValue(0, 80); - long sourceEnd = generateRandomValue(sourceStart, 99); - TrimmingStream trimmingStream = new TrimmingStream(sourceStart, sourceEnd, sourceStart, sourceEnd, input); - - byte[] result = new byte[(int) (sourceEnd - sourceStart + 1)]; - int bytesRead = trimmingStream.read(result, 0, result.length); - - assertEquals(sourceEnd - sourceStart + 1, bytesRead); - assertEquals(sourceEnd - sourceStart + 1, input.getPos()); - } - - private byte[] generateRandomData(int length) { - byte[] data = new byte[length]; - for (int i = 0; i < length; i++) { - data[i] = (byte) (Math.random() * 256 - 128); - } - return data; - } - - private long generateRandomValue(long min, long max) { - return min + (long) (Math.random() * (max - min + 1)); - } -} diff --git a/libs/encryption-sdk/src/test/resources/encrypted_key b/libs/encryption-sdk/src/test/resources/encrypted_key deleted file mode 100644 index da4e503581585..0000000000000 --- a/libs/encryption-sdk/src/test/resources/encrypted_key +++ /dev/null @@ -1 +0,0 @@ -[1, 2, 1, 0, 120, -96, 18, 71, -6, 90, -126, -39, -16, 94, -113, -46, 71, 85, 35, -66, -117, -108, -59, 88, -81, 64, -118, -74, -102, 50, 103, 16, -76, 23, 19, 20, 67, 1, -11, 55, -3, 32, -89, -16, 1, -40, 59, 76, -2, -61, -49, -97, 34, 14, 0, 0, 0, 126, 48, 124, 6, 9, 42, -122, 72, -122, -9, 13, 1, 7, 6, -96, 111, 48, 109, 2, 1, 0, 48, 104, 6, 9, 42, -122, 72, -122, -9, 13, 1, 7, 1, 48, 30, 6, 9, 96, -122, 72, 1, 101, 3, 4, 1, 46, 48, 17, 4, 12, -63, 67, 37, -51, 85, 75, 7, -64, -78, 52, 102, 26, 2, 1, 16, -128, 59, -98, -123, 100, 125, -37, 102, -87, -71, 74, 68, 54, 56, -32, 77, 127, -86, -125, -17, 45, 75, -98, 54, -52, -15, -56, -47, -88, -12, -128, 113, -5, -18, -14, 127, 114, -9, 47, -112, -38, 39, 2, -89, 117, 64, -2, 47, -81, 52, 27, -118, 37, 79, -64, 58, -3, 10, -115, 122, 124] \ No newline at end of file diff --git a/libs/encryption-sdk/src/test/resources/raw_content_for_crypto_test b/libs/encryption-sdk/src/test/resources/raw_content_for_crypto_test deleted file mode 100644 index c93b6161ac8d6..0000000000000 --- a/libs/encryption-sdk/src/test/resources/raw_content_for_crypto_test +++ /dev/null @@ -1,25 +0,0 @@ -ewogICJmaWxlSW5mb3MiOiBbCiAgICB7CiAgICAgICJuYW1lIjogIl80LmZubSIsCiAgICAgICJyZW1vdGVfc -GF0aCI6ICIyYzYwMzNmNmZlZTY0NTY1YTU3YzQzZWVmZThmY2QzMS9kdW1teS1jb2xsZWN0aW9uMi9kMDRmYz -AyZi0wMDQ0LTRhYmYtYjgzMy0xMGE0YTA5M2VkNTcvMC8wL2luZGljZXMvMSIsCiAgICAgICJzaXplIjogOTQz -CiAgICB9LAogICAgewogICAgICAibmFtZSI6ICJfMl9MdWNlbmU4MF8wLmR2ZCIsCiAgICAgICJyZW1vdGVfcGF -0aCI6ICIyYzYwMzNmNmZlZTY0NTY1YTU3YzQzZWVmZThmY2QzMS9kdW1teS1jb2xsZWN0aW9uMi9kMDRmYzAyZi0wMDQ0LTRhYmYtYjg -zMy0xMGE0YTA5M2VkNTcvMC8wL2luZGljZXMvMSIsCiAgICAgICJzaXplIjogMzU1CiAgICB9CiAgXQp9 -ewogICJja3BfZmlsZSI6IHsKICAgICJuYW1lIjogInRyYW5zbG9nLTguY2twIiwKICAgICJyZW1vdGVfcGF0aCI6ICIyYz -YwMzNmNmZlZTY0NTY1YTU3YzQzZWVmZThmY2QzMS9kdW1teS1jb2xsZWN0aW9uMi9kMDRmYzAyZi0wMDQ0LTRhYmYtYjgzMy0 -xMGE0YTA5M2VkNTcvMC8wL3RyYW5zbG9nLzEiLAogICAgInNpemUiOiAwCiAgfSwKICAidGxvZ192ZXJzaW9uIjogewogICAgIjg -iOiAiMmM2MDMzZjZmZWU2NDU2NWE1N2M0M2VlZmU4ZmNkMzEvZHVtbXktY29sbGVjdGlvbjIvZDA0ZmMwMmYtMDA0NC00YWJmLWI4MzMtMT -BhNGEwOTNlZDU3LzAvMC90cmFuc2xvZy8xIgogIH0KfQ== -ewogICJmaWxlSW5mb3MiOiBbCiAgICB7CiAgICAgICJuYW1lIjogIl80LmZubSIsCiAgICAgICJyZW1vdGVfcGF0aCI6ICIyYzYwMzNmNmZl -ZTY0NTY1YTU3YzQzZWVmZThmY2QzMS9kdW1teS1jb2xsZWN0aW9uMi9kMDRmYzAyZi0wMDQ0LTRhYmYtYjgzMy0xMGE0YTA5M2VkNTcvMC8wL2luZG -ljZXMvMSIsCiAgICAgICJzaXplIjogOTQzCiAgICB9LAogICAgewogICAgICAibmFtZSI6ICJfNC5mZHQiLAogICAgICAicmVtb3RlX3BhdGgiOiAi -MmM2MDMzZjZmZWU2NDU2NWE1N2M0M2VlZmU4ZmNkMzEvZHVtbXktY29sbGVjdGlvbjIvZDA0ZmMwMmYtMDA0NC00YWJmLWI4MzMtMTBhNGEwOTNlZDU3 -LzAvMC9pbmRpY2VzLzEiLAogICAgICAic2l6ZSI6IDQ1MTMKICAgIH0sCiAgICB7CiAgICAgICJuYW1lIjogInNlZ21lbnRzX2MiLAogICAgICAicmVtb3R -lX3BhdGgiOiAiMmM2MDMzZjZmZWU2NDU2NWE1N2M0M2VlZmU4ZmNkMzEvZHVtbXktY29sbGVjdGlvbjIvZDA0ZmMwMmYtMDA0NC00YWJmLWI4MzM -tMTBhNGEwOTNlZDU3LzAvMC9pbmRpY2VzLzEiLAogICAgICAic2l6ZSI6IDM1NQogICAgfQogIF0KfQ== -ewogICJja3BfZmlsZSI6IHsKICAgICJuYW1lIjogInRyYW5zbG9nLTcuY2twIiwKICAgICJyZW1vdGVfcGF0aCI6ICIyYzYwMzNmNmZlZ -TY0NTY1YTU3YzQzZWVmZThmY2QzMS9kdW1teS1jb2xsZWN0aW9uMi9kMDRmYzAyZi0wMDQ0LTRhYmYtYjgzMy0xMGE0YTA5M2VkNTcvMC8wL3RyY -W5zbG9nLzEiLAogICAgInNpemUiOiAwCiAgfSwKICAidGxvZ192ZXJzaW9uIjogewogICAgIjYiOiAiMmM2MDMzZjZmZWU2NDU2NWE1N2M0M2VlZ -mU4ZmNkMzEvZHVtbXktY29sbGVjdGlvbjIvZDA0ZmMwMmYtMDA0NC00YWJmLWI4MzMtMTBhNGEwOTNlZDU3LzAvMC90cmFuc2xvZy8xIiwKICAgICI3Ijo -gIjJjNjAzM2Y2ZmVlNjQ1NjVhNTdjNDNlZWZlOGZjZDMxL2R1bW15LWNvbGxlY3Rpb24yL2QwNGZjMDJmLTAwNDQtNGFiZi1iODMzLTEwYTRhMDkzZW -Q1Ny8wLzAvdHJhbnNsb2cvMSIKICB9Cn0= - diff --git a/libs/encryption-sdk/src/test/resources/raw_key b/libs/encryption-sdk/src/test/resources/raw_key deleted file mode 100644 index 3c4f8b54cbb6a..0000000000000 --- a/libs/encryption-sdk/src/test/resources/raw_key +++ /dev/null @@ -1 +0,0 @@ -[57, 59, -48, -8, -44, 9, -78, 16, 106, -80, 66, -41, 66, 43, -88, 7, 47, -23, -16, -43, 99, 104, -8, -74, 46, -117, -111, -41, -39, -69, 5, 117] \ No newline at end of file diff --git a/plugins/crypto-kms/build.gradle b/plugins/crypto-kms/build.gradle index fa76118a43d92..c4a8609b6df48 100644 --- a/plugins/crypto-kms/build.gradle +++ b/plugins/crypto-kms/build.gradle @@ -26,8 +26,6 @@ ext { } dependencies { - api project(':libs:opensearch-encryption-sdk') - api "software.amazon.awssdk:sdk-core:${versions.aws}" api "software.amazon.awssdk:aws-core:${versions.aws}" api "software.amazon.awssdk:utils:${versions.aws}" @@ -56,6 +54,10 @@ dependencies { api "org.reactivestreams:reactive-streams:${versions.reactivestreams}" } +//testClusters.all { +// module ':modules:crypto' +//} + tasks.named("dependencyLicenses").configure { mapping from: /jackson-.*/, to: 'jackson' mapping from: /jaxb-.*/, to: 'jaxb' diff --git a/plugins/crypto-kms/src/main/java/org/opensearch/crypto/kms/KmsService.java b/plugins/crypto-kms/src/main/java/org/opensearch/crypto/kms/KmsService.java index 87f6cfbb254c6..108c88bd3bf80 100644 --- a/plugins/crypto-kms/src/main/java/org/opensearch/crypto/kms/KmsService.java +++ b/plugins/crypto-kms/src/main/java/org/opensearch/crypto/kms/KmsService.java @@ -242,12 +242,13 @@ static void setDefaultAwsProfilePath() { } public MasterKeyProvider createMasterKeyProvider(CryptoMetadata cryptoMetadata) { - String keyArn = KEY_ARN_SETTING.get(cryptoMetadata.settings()); + Settings cryptoSettings = Settings.builder().put(cryptoMetadata.settings()).normalizePrefix("kms.").build(); + String keyArn = KEY_ARN_SETTING.get(cryptoSettings); if (!Strings.hasText(keyArn)) { throw new IllegalArgumentException("Missing key_arn setting"); } - String kmsEncCtx = ENC_CTX_SETTING.get(cryptoMetadata.settings()); + String kmsEncCtx = ENC_CTX_SETTING.get(cryptoSettings); Map encCtx; if (Strings.hasText(kmsEncCtx)) { try { diff --git a/libs/encryption-sdk/licenses/commons-lang3-3.13.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-lang3-3.13.0.jar.sha1 similarity index 100% rename from libs/encryption-sdk/licenses/commons-lang3-3.13.0.jar.sha1 rename to plugins/ingest-attachment/licenses/commons-lang3-3.13.0.jar.sha1 diff --git a/libs/encryption-sdk/licenses/commons-lang3-LICENSE.txt b/plugins/ingest-attachment/licenses/commons-lang3-LICENSE.txt similarity index 100% rename from libs/encryption-sdk/licenses/commons-lang3-LICENSE.txt rename to plugins/ingest-attachment/licenses/commons-lang3-LICENSE.txt diff --git a/libs/encryption-sdk/licenses/commons-lang3-NOTICE.txt b/plugins/ingest-attachment/licenses/commons-lang3-NOTICE.txt similarity index 100% rename from libs/encryption-sdk/licenses/commons-lang3-NOTICE.txt rename to plugins/ingest-attachment/licenses/commons-lang3-NOTICE.txt diff --git a/plugins/repository-azure/licenses/commons-lang3-3.13.0.jar.sha1 b/plugins/repository-azure/licenses/commons-lang3-3.13.0.jar.sha1 new file mode 100644 index 0000000000000..d0c2f2486ee1f --- /dev/null +++ b/plugins/repository-azure/licenses/commons-lang3-3.13.0.jar.sha1 @@ -0,0 +1 @@ +b7263237aa89c1f99b327197c41d0669707a462e \ No newline at end of file diff --git a/libs/encryption-sdk/licenses/aws-encryption-sdk-java-LICENSE.txt b/plugins/repository-azure/licenses/commons-lang3-LICENSE.txt similarity index 99% rename from libs/encryption-sdk/licenses/aws-encryption-sdk-java-LICENSE.txt rename to plugins/repository-azure/licenses/commons-lang3-LICENSE.txt index 8dada3edaf50d..d645695673349 100644 --- a/libs/encryption-sdk/licenses/aws-encryption-sdk-java-LICENSE.txt +++ b/plugins/repository-azure/licenses/commons-lang3-LICENSE.txt @@ -1,3 +1,4 @@ + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -178,7 +179,7 @@ APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -186,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/plugins/repository-azure/licenses/commons-lang3-NOTICE.txt b/plugins/repository-azure/licenses/commons-lang3-NOTICE.txt new file mode 100644 index 0000000000000..13a3140897472 --- /dev/null +++ b/plugins/repository-azure/licenses/commons-lang3-NOTICE.txt @@ -0,0 +1,5 @@ +Apache Commons Lang +Copyright 2001-2019 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/plugins/repository-hdfs/licenses/commons-lang3-3.13.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-lang3-3.13.0.jar.sha1 new file mode 100644 index 0000000000000..d0c2f2486ee1f --- /dev/null +++ b/plugins/repository-hdfs/licenses/commons-lang3-3.13.0.jar.sha1 @@ -0,0 +1 @@ +b7263237aa89c1f99b327197c41d0669707a462e \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/commons-lang3-LICENSE.txt b/plugins/repository-hdfs/licenses/commons-lang3-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/repository-hdfs/licenses/commons-lang3-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/repository-hdfs/licenses/commons-lang3-NOTICE.txt b/plugins/repository-hdfs/licenses/commons-lang3-NOTICE.txt new file mode 100644 index 0000000000000..13a3140897472 --- /dev/null +++ b/plugins/repository-hdfs/licenses/commons-lang3-NOTICE.txt @@ -0,0 +1,5 @@ +Apache Commons Lang +Copyright 2001-2019 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/server/build.gradle b/server/build.gradle index af56032897ee1..f6db3d53a0dcc 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -106,7 +106,6 @@ dependencies { api project(':libs:opensearch-x-content') api project(":libs:opensearch-geo") api project(":libs:opensearch-telemetry") - api project(":libs:opensearch-encryption-sdk") compileOnly project(':libs:opensearch-plugin-classloader') diff --git a/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobStore.java b/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobStore.java index 5365b5a9d8bdf..4d2d69e473438 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobStore.java +++ b/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobStore.java @@ -9,9 +9,9 @@ package org.opensearch.common.blobstore; import org.opensearch.cluster.metadata.CryptoMetadata; -import org.opensearch.crypto.CryptoManagerRegistry; +import org.opensearch.common.crypto.CryptoHandler; +import org.opensearch.crypto.CryptoHandlerRegistry; import org.opensearch.crypto.CryptoRegistryException; -import org.opensearch.encryption.CryptoManager; import java.io.IOException; import java.util.Map; @@ -25,7 +25,7 @@ public class EncryptedBlobStore implements BlobStore { private final BlobStore blobStore; - private final CryptoManager cryptoManager; + private final CryptoHandler cryptoHandler; /** * Constructs an EncryptedBlobStore that wraps the provided BlobStore with encryption capabilities based on the @@ -36,17 +36,16 @@ public class EncryptedBlobStore implements BlobStore { * @throws CryptoRegistryException If the CryptoManager is not found during encrypted BlobStore creation. */ public EncryptedBlobStore(BlobStore blobStore, CryptoMetadata cryptoMetadata) { - CryptoManagerRegistry cryptoManagerRegistry = CryptoManagerRegistry.getInstance(); - assert cryptoManagerRegistry != null : "CryptoManagerRegistry is not initialized"; - this.cryptoManager = cryptoManagerRegistry.fetchCryptoManager(cryptoMetadata); - if (cryptoManager == null) { + CryptoHandlerRegistry cryptoHandlerRegistry = CryptoHandlerRegistry.getInstance(); + assert cryptoHandlerRegistry != null : "CryptoManagerRegistry is not initialized"; + this.cryptoHandler = cryptoHandlerRegistry.fetchCryptoHandler(cryptoMetadata); + if (cryptoHandler == null) { throw new CryptoRegistryException( cryptoMetadata.keyProviderName(), cryptoMetadata.keyProviderType(), "Crypto manager not found during encrypted blob store creation." ); } - this.cryptoManager.incRef(); this.blobStore = blobStore; } @@ -61,12 +60,9 @@ public EncryptedBlobStore(BlobStore blobStore, CryptoMetadata cryptoMetadata) { public BlobContainer blobContainer(BlobPath path) { BlobContainer blobContainer = blobStore.blobContainer(path); if (blobContainer instanceof AsyncMultiStreamBlobContainer) { - return new AsyncMultiStreamEncryptedBlobContainer<>( - (AsyncMultiStreamBlobContainer) blobContainer, - cryptoManager.getCryptoProvider() - ); + return new AsyncMultiStreamEncryptedBlobContainer<>((AsyncMultiStreamBlobContainer) blobContainer, cryptoHandler); } - return new EncryptedBlobContainer<>(blobContainer, cryptoManager.getCryptoProvider()); + return new EncryptedBlobContainer<>(blobContainer, cryptoHandler); } /** @@ -87,7 +83,7 @@ public Map stats() { */ @Override public void close() throws IOException { - cryptoManager.decRef(); + cryptoHandler.close(); blobStore.close(); } diff --git a/server/src/main/java/org/opensearch/crypto/CryptoManagerRegistry.java b/server/src/main/java/org/opensearch/crypto/CryptoHandlerRegistry.java similarity index 58% rename from server/src/main/java/org/opensearch/crypto/CryptoManagerRegistry.java rename to server/src/main/java/org/opensearch/crypto/CryptoHandlerRegistry.java index c6a8b56a8d8c8..0a14331be35f7 100644 --- a/server/src/main/java/org/opensearch/crypto/CryptoManagerRegistry.java +++ b/server/src/main/java/org/opensearch/crypto/CryptoHandlerRegistry.java @@ -13,12 +13,11 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.cluster.metadata.CryptoMetadata; import org.opensearch.common.SetOnce; +import org.opensearch.common.crypto.CryptoHandler; import org.opensearch.common.crypto.MasterKeyProvider; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.encryption.CryptoManager; -import org.opensearch.encryption.CryptoManagerFactory; import org.opensearch.plugins.CryptoKeyProviderPlugin; +import org.opensearch.plugins.CryptoPlugin; import java.util.HashMap; import java.util.List; @@ -27,19 +26,19 @@ /** * During node bootstrap, installed key provider extensions responsible for generating data keys are loaded. - * Crypto factories against the respective extensions are cached. A crypto factory is used to register crypto - * manager against an {@link org.opensearch.common.blobstore.EncryptedBlobStore} + * Crypto factories against the respective KP plugins are cached. A crypto factory is used to register crypto + * handler against an {@link org.opensearch.common.blobstore.EncryptedBlobStore} */ -public class CryptoManagerRegistry { - private static final Logger logger = LogManager.getLogger(CryptoManagerRegistry.class); +public class CryptoHandlerRegistry { + private static final Logger logger = LogManager.getLogger(CryptoHandlerRegistry.class); // Package private for tests SetOnce> registry = new SetOnce<>(); // Package private for tests - SetOnce cryptoManagerFactory = new SetOnce(); - private final Map registeredCryptoManagers = new HashMap<>(); + SetOnce cryptoHandlerPlugin = new SetOnce<>(); + private final Map registeredCryptoHandlers = new HashMap<>(); - private static volatile CryptoManagerRegistry instance; + private static volatile CryptoHandlerRegistry instance; private static final Object lock = new Object(); /** @@ -48,22 +47,38 @@ public class CryptoManagerRegistry { * @param cryptoPlugins The list of installed crypto key provider plugins. * @param settings Crypto settings. */ - protected CryptoManagerRegistry(List cryptoPlugins, Settings settings) { - cryptoManagerFactory.set(new CryptoManagerFactory("ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY", TimeValue.timeValueDays(2), 500)); - registry.set(loadCryptoFactories(cryptoPlugins)); + protected CryptoHandlerRegistry( + List cryptoPlugins, + List cryptoKeyProviderPlugins, + Settings settings + ) { + if (cryptoPlugins == null || cryptoPlugins.size() == 0) { + return; + } + if (cryptoPlugins.size() > 1) { + // We can remove this to support multiple implementations in future if needed. + throw new IllegalStateException("More than 1 implementation of crypto plugin found."); + } + + cryptoHandlerPlugin.set(cryptoPlugins.get(0)); + registry.set(loadCryptoFactories(cryptoKeyProviderPlugins)); } - public static CryptoManagerRegistry getInstance() { + public static CryptoHandlerRegistry getInstance() { return instance; } - public static CryptoManagerRegistry initRegistry(List cryptoPlugins, Settings settings) { - CryptoManagerRegistry curInstance = instance; + public static CryptoHandlerRegistry initRegistry( + List cryptoPlugins, + List cryptoKeyProviderPlugins, + Settings settings + ) { + CryptoHandlerRegistry curInstance = instance; if (curInstance == null) { synchronized (lock) { curInstance = instance; if (curInstance == null) { - instance = curInstance = new CryptoManagerRegistry(cryptoPlugins, settings); + instance = curInstance = new CryptoHandlerRegistry(cryptoPlugins, cryptoKeyProviderPlugins, settings); } } } @@ -71,23 +86,23 @@ public static CryptoManagerRegistry initRegistry(List c } // For tests - protected Map loadCryptoFactories(List cryptoPlugins) { + protected Map loadCryptoFactories(List cryptoKPPlugins) { Map cryptoFactories = new HashMap<>(); - for (CryptoKeyProviderPlugin cryptoPlugin : cryptoPlugins) { - if (cryptoFactories.containsKey(cryptoPlugin.type())) { - throw new IllegalArgumentException("Crypto plugin key provider type [" + cryptoPlugin.type() + "] is already registered"); + for (CryptoKeyProviderPlugin cryptoKPPlugin : cryptoKPPlugins) { + if (cryptoFactories.containsKey(cryptoKPPlugin.type())) { + throw new IllegalArgumentException("Crypto plugin key provider type [" + cryptoKPPlugin.type() + "] is already registered"); } - cryptoFactories.put(cryptoPlugin.type(), cryptoPlugin); + cryptoFactories.put(cryptoKPPlugin.type(), cryptoKPPlugin); } return Map.copyOf(cryptoFactories); } /** - * Retrieves the crypto factory associated with the given key provider type (extension id). + * Retrieves the crypto factory associated with the given key provider type . * - * @param keyProviderType The unique extension type for which the factory is to be fetched. - * @return The crypto factory used to create {@link CryptoManager} + * @param keyProviderType The unique provider type for which the factory is to be fetched. + * @return The crypto factory used to create {@link CryptoHandler} * instances in a {@link org.opensearch.common.blobstore.EncryptedBlobStore}. * @throws IllegalStateException If the crypto registry is not yet loaded. */ @@ -106,26 +121,26 @@ public CryptoKeyProviderPlugin getCryptoKeyProviderPlugin(String keyProviderType * @return The crypto manager for performing encrypt/decrypt operations. * @throws CryptoRegistryException If the key provider is not installed or there is an error during crypto manager creation. */ - public CryptoManager fetchCryptoManager(CryptoMetadata cryptoMetadata) { - CryptoManager cryptoManager = registeredCryptoManagers.get(cryptoMetadata); - if (cryptoManager == null) { - synchronized (registeredCryptoManagers) { - cryptoManager = registeredCryptoManagers.get(cryptoMetadata); - if (cryptoManager == null) { + public CryptoHandler fetchCryptoHandler(CryptoMetadata cryptoMetadata) { + CryptoHandler cryptoHandler = registeredCryptoHandlers.get(cryptoMetadata); + if (cryptoHandler == null) { + synchronized (registeredCryptoHandlers) { + cryptoHandler = registeredCryptoHandlers.get(cryptoMetadata); + if (cryptoHandler == null) { Runnable onClose = () -> { - synchronized (registeredCryptoManagers) { - registeredCryptoManagers.remove(cryptoMetadata); + synchronized (registeredCryptoHandlers) { + registeredCryptoHandlers.remove(cryptoMetadata); } }; - cryptoManager = createCryptoManager(cryptoMetadata, onClose); - registeredCryptoManagers.put(cryptoMetadata, cryptoManager); + cryptoHandler = createCryptoHandler(cryptoMetadata, onClose); + registeredCryptoHandlers.put(cryptoMetadata, cryptoHandler); } } } - return cryptoManager; + return cryptoHandler; } - private CryptoManager createCryptoManager(CryptoMetadata cryptoMetadata, Runnable onClose) { + private CryptoHandler createCryptoHandler(CryptoMetadata cryptoMetadata, Runnable onClose) { logger.debug("creating crypto client [{}][{}]", cryptoMetadata.keyProviderType(), cryptoMetadata.keyProviderName()); CryptoKeyProviderPlugin keyProviderPlugin = getCryptoKeyProviderPlugin(cryptoMetadata.keyProviderType()); if (keyProviderPlugin == null) { @@ -134,8 +149,8 @@ private CryptoManager createCryptoManager(CryptoMetadata cryptoMetadata, Runnabl try { MasterKeyProvider masterKeyProvider = keyProviderPlugin.createKeyProvider(cryptoMetadata); - return Objects.requireNonNull(cryptoManagerFactory.get()) - .getOrCreateCryptoManager(masterKeyProvider, cryptoMetadata.keyProviderName(), cryptoMetadata.keyProviderType(), onClose); + return Objects.requireNonNull(cryptoHandlerPlugin.get()) + .getOrCreateCryptoHandler(masterKeyProvider, cryptoMetadata.keyProviderName(), cryptoMetadata.keyProviderType(), onClose); } catch (Exception e) { logger.warn( diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 837b0e3795fc0..e2365adb440cc 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -115,7 +115,7 @@ import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.crypto.CryptoManagerRegistry; +import org.opensearch.crypto.CryptoHandlerRegistry; import org.opensearch.discovery.Discovery; import org.opensearch.discovery.DiscoveryModule; import org.opensearch.env.Environment; @@ -175,6 +175,7 @@ import org.opensearch.plugins.CircuitBreakerPlugin; import org.opensearch.plugins.ClusterPlugin; import org.opensearch.plugins.CryptoKeyProviderPlugin; +import org.opensearch.plugins.CryptoPlugin; import org.opensearch.plugins.DiscoveryPlugin; import org.opensearch.plugins.EnginePlugin; import org.opensearch.plugins.ExtensionAwarePlugin; @@ -939,7 +940,11 @@ protected Node( xContentRegistry, recoverySettings ); - CryptoManagerRegistry.initRegistry(pluginsService.filterPlugins(CryptoKeyProviderPlugin.class), settings); + CryptoHandlerRegistry.initRegistry( + pluginsService.filterPlugins(CryptoPlugin.class), + pluginsService.filterPlugins(CryptoKeyProviderPlugin.class), + settings + ); RepositoriesService repositoryService = repositoriesModule.getRepositoryService(); repositoriesServiceReference.set(repositoryService); SnapshotsService snapshotsService = new SnapshotsService( diff --git a/server/src/main/java/org/opensearch/plugins/CryptoPlugin.java b/server/src/main/java/org/opensearch/plugins/CryptoPlugin.java new file mode 100644 index 0000000000000..ad348d07e23d3 --- /dev/null +++ b/server/src/main/java/org/opensearch/plugins/CryptoPlugin.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugins; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.crypto.CryptoHandler; +import org.opensearch.common.crypto.MasterKeyProvider; + +/** + * Crypto plugin to provide encryption and decryption support. + * @opensearch.api + */ +@ExperimentalApi +public interface CryptoPlugin { + + /** + * To create a crypto handler for handling encryption and decryption ops. + * @param keyProvider key provider instance to provide keys used in encrypting data. + * @param keyProviderName Name of key provider to distinguish between multiple instances created with different + * configurations of same keyProviderType. + * @param keyProviderType Unique type of key provider to distinguish between different key provider implementations. + * @param onClose Closes key provider or other clean up operations on close. + * @return crypto handler instance. + */ + CryptoHandler getOrCreateCryptoHandler( + MasterKeyProvider keyProvider, + String keyProviderName, + String keyProviderType, + Runnable onClose + ); +} diff --git a/server/src/test/java/org/opensearch/crypto/CryptoManagerRegistryTests.java b/server/src/test/java/org/opensearch/crypto/CryptoHandlerRegistryTests.java similarity index 72% rename from server/src/test/java/org/opensearch/crypto/CryptoManagerRegistryTests.java rename to server/src/test/java/org/opensearch/crypto/CryptoHandlerRegistryTests.java index f6c8f71bd653c..93a7b3d3eb4b9 100644 --- a/server/src/test/java/org/opensearch/crypto/CryptoManagerRegistryTests.java +++ b/server/src/test/java/org/opensearch/crypto/CryptoHandlerRegistryTests.java @@ -9,10 +9,11 @@ package org.opensearch.crypto; import org.opensearch.cluster.metadata.CryptoMetadata; +import org.opensearch.common.crypto.CryptoHandler; import org.opensearch.common.crypto.MasterKeyProvider; import org.opensearch.common.settings.Settings; -import org.opensearch.encryption.CryptoManager; import org.opensearch.plugins.CryptoKeyProviderPlugin; +import org.opensearch.plugins.CryptoPlugin; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; @@ -24,30 +25,30 @@ import org.mockito.ArgumentMatchers; import org.mockito.Mockito; -public class CryptoManagerRegistryTests extends OpenSearchTestCase { +public class CryptoHandlerRegistryTests extends OpenSearchTestCase { - private TestCryptoManagerRegistry cryptoManagerRegistry; + private TestCryptoHandlerRegistry cryptoManagerRegistry; private String pluginTypeWithCreationFailure; private CryptoKeyProviderPlugin cryptoPlugin1; private CryptoKeyProviderPlugin cryptoPlugin2; @Before public void setup() { - List cryptoPlugins = new ArrayList<>(); + List cryptoKPPlugins = new ArrayList<>(); CryptoKeyProviderPlugin cryptoPlugin1 = Mockito.mock(CryptoKeyProviderPlugin.class); String pluginType1 = UUID.randomUUID().toString(); Mockito.when(cryptoPlugin1.type()).thenReturn(pluginType1); MasterKeyProvider masterKeyProvider1 = Mockito.mock(MasterKeyProvider.class); Mockito.when(cryptoPlugin1.createKeyProvider(ArgumentMatchers.any())).thenReturn(masterKeyProvider1); this.cryptoPlugin1 = cryptoPlugin1; - cryptoPlugins.add(cryptoPlugin1); + cryptoKPPlugins.add(cryptoPlugin1); CryptoKeyProviderPlugin cryptoPlugin2 = Mockito.mock(CryptoKeyProviderPlugin.class); String pluginType2 = UUID.randomUUID().toString(); Mockito.when(cryptoPlugin2.type()).thenReturn(pluginType2); MasterKeyProvider masterKeyProvider2 = Mockito.mock(MasterKeyProvider.class); Mockito.when(cryptoPlugin2.createKeyProvider(ArgumentMatchers.any())).thenReturn(masterKeyProvider2); - cryptoPlugins.add(cryptoPlugin2); + cryptoKPPlugins.add(cryptoPlugin2); this.cryptoPlugin2 = cryptoPlugin2; CryptoKeyProviderPlugin cryptoPluginCreationFailure = Mockito.mock(CryptoKeyProviderPlugin.class); @@ -55,20 +56,33 @@ public void setup() { Mockito.when(cryptoPluginCreationFailure.type()).thenReturn(pluginTypeWithCreationFailure); Mockito.when(cryptoPluginCreationFailure.createKeyProvider(ArgumentMatchers.any())) .thenThrow(new RuntimeException("Injected failure")); - cryptoPlugins.add(cryptoPluginCreationFailure); + cryptoKPPlugins.add(cryptoPluginCreationFailure); + + cryptoManagerRegistry = new TestCryptoHandlerRegistry(new TestCryptoPlugin(), cryptoKPPlugins, Settings.EMPTY); + } + + static class TestCryptoPlugin implements CryptoPlugin { - cryptoManagerRegistry = new TestCryptoManagerRegistry(cryptoPlugins, Settings.EMPTY); + @Override + public CryptoHandler getOrCreateCryptoHandler( + MasterKeyProvider keyProvider, + String keyProviderName, + String keyProviderType, + Runnable onClose + ) { + return Mockito.mock(CryptoHandler.class); + } } - static class TestCryptoManagerRegistry extends CryptoManagerRegistry { + static class TestCryptoHandlerRegistry extends CryptoHandlerRegistry { - protected TestCryptoManagerRegistry(List cryptoPlugins, Settings settings) { - super(cryptoPlugins, settings); + protected TestCryptoHandlerRegistry(CryptoPlugin cryptoPlugin, List cryptoPlugins, Settings settings) { + super(List.of(cryptoPlugin), cryptoPlugins, settings); } @Override - public Map loadCryptoFactories(List cryptoPlugins) { - return super.loadCryptoFactories(cryptoPlugins); + public Map loadCryptoFactories(List cryptoKPPlugins) { + return super.loadCryptoFactories(cryptoKPPlugins); } } @@ -115,44 +129,40 @@ public void testCryptoManagerMissing() { String pluginName = UUID.randomUUID().toString(); String pluginType = UUID.randomUUID().toString(); CryptoMetadata cryptoMetadata = new CryptoMetadata(pluginName, pluginType, Settings.EMPTY); - expectThrows(CryptoRegistryException.class, () -> cryptoManagerRegistry.fetchCryptoManager(cryptoMetadata)); + expectThrows(CryptoRegistryException.class, () -> cryptoManagerRegistry.fetchCryptoHandler(cryptoMetadata)); } public void testCryptoManagerCreationFailure() { String pluginName = UUID.randomUUID().toString(); CryptoMetadata cryptoMetadata = new CryptoMetadata(pluginName, pluginTypeWithCreationFailure, Settings.EMPTY); - expectThrows(CryptoRegistryException.class, () -> cryptoManagerRegistry.fetchCryptoManager(cryptoMetadata)); + expectThrows(CryptoRegistryException.class, () -> cryptoManagerRegistry.fetchCryptoHandler(cryptoMetadata)); } public void testCryptoManagerCreationSuccess() { String pluginName1 = UUID.randomUUID().toString(); CryptoMetadata cryptoMetadata = new CryptoMetadata(pluginName1, cryptoPlugin1.type(), Settings.EMPTY); - CryptoManager createdCryptoManager1 = cryptoManagerRegistry.fetchCryptoManager(cryptoMetadata); - assertNotNull(createdCryptoManager1); - assertEquals(cryptoPlugin1.type(), createdCryptoManager1.type()); - assertEquals(cryptoMetadata.keyProviderName(), createdCryptoManager1.name()); - assertEquals(cryptoMetadata.keyProviderType(), createdCryptoManager1.type()); + CryptoHandler cryptoHandler = cryptoManagerRegistry.fetchCryptoHandler(cryptoMetadata); + assertNotNull(cryptoHandler); String pluginName2 = UUID.randomUUID().toString(); - CryptoManager createdCryptoManager2 = cryptoManagerRegistry.fetchCryptoManager( + CryptoHandler cryptoHandler2 = cryptoManagerRegistry.fetchCryptoHandler( new CryptoMetadata(pluginName2, cryptoPlugin2.type(), Settings.EMPTY) ); - assertNotNull(createdCryptoManager2); - assertEquals(pluginName2, createdCryptoManager2.name()); - assertEquals(cryptoPlugin2.type(), createdCryptoManager2.type()); - CryptoManager createdCryptoManager3 = cryptoManagerRegistry.fetchCryptoManager( + assertNotNull(cryptoHandler2); + CryptoHandler cryptoHandler3 = cryptoManagerRegistry.fetchCryptoHandler( new CryptoMetadata(pluginName1, cryptoPlugin1.type(), Settings.EMPTY) ); - assertNotNull(createdCryptoManager3); - assertEquals(createdCryptoManager1, createdCryptoManager3); + assertNotNull(cryptoHandler3); + assertEquals(cryptoHandler, cryptoHandler3); + assertNotEquals(cryptoHandler2, cryptoHandler); - CryptoManager createdCryptoMgrNewType = cryptoManagerRegistry.fetchCryptoManager( + CryptoHandler cryptoHandlerNewType = cryptoManagerRegistry.fetchCryptoHandler( new CryptoMetadata(pluginName1, cryptoPlugin2.type(), Settings.EMPTY) ); - assertNotNull(createdCryptoMgrNewType); - assertNotEquals(createdCryptoManager1, createdCryptoMgrNewType); - assertNotEquals(createdCryptoManager2, createdCryptoMgrNewType); - assertNotEquals(createdCryptoManager3, createdCryptoMgrNewType); + assertNotNull(cryptoHandlerNewType); + assertNotEquals(cryptoHandler, cryptoHandlerNewType); + assertNotEquals(cryptoHandler2, cryptoHandlerNewType); + assertNotEquals(cryptoHandler3, cryptoHandlerNewType); } } diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java index b20cb323e144f..d14858a4441b5 100644 --- a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java @@ -57,6 +57,7 @@ import org.opensearch.common.crypto.CryptoHandler; import org.opensearch.common.crypto.DecryptedRangedStreamProvider; import org.opensearch.common.crypto.EncryptedHeaderContentSupplier; +import org.opensearch.common.crypto.MasterKeyProvider; import org.opensearch.common.io.InputStreamContainer; import org.opensearch.common.lifecycle.Lifecycle; import org.opensearch.common.lifecycle.LifecycleListener; @@ -65,7 +66,6 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.encryption.CryptoManager; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; @@ -73,6 +73,7 @@ import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RecoveryState; +import org.opensearch.plugins.CryptoPlugin; import org.opensearch.repositories.blobstore.MeteredBlobStoreRepository; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInfo; @@ -89,7 +90,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Function; @@ -102,6 +102,8 @@ public class RepositoriesServiceTests extends OpenSearchTestCase { private RepositoriesService repositoriesService; + private final String kpTypeA = "kp-type-a"; + private final String kpTypeB = "kp-type-b"; @Override public void setUp() throws Exception { @@ -229,28 +231,28 @@ public void testWithSameKeyProviderNames() { "repoName", MeteredRepositoryTypeA.TYPE, keyProviderName, - TestCryptoManagerTypeA.TYPE + kpTypeA ); repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", clusterStateWithRepoTypeA, emptyState())); assertThat(repositoriesService.repositoriesStats().size(), equalTo(1)); MeteredRepositoryTypeA repository = (MeteredRepositoryTypeA) repositoriesService.repository("repoName"); assertNotNull(repository); - assertNotNull(repository.cryptoManager); - assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + assertNotNull(repository.cryptoHandler); + assertEquals(kpTypeA, repository.cryptoHandler.kpType); ClusterState clusterStateWithRepoTypeB = createClusterStateWithKeyProvider( "repoName", MeteredRepositoryTypeB.TYPE, keyProviderName, - TestCryptoManagerTypeB.TYPE + kpTypeA ); repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", clusterStateWithRepoTypeB, emptyState())); assertThat(repositoriesService.repositoriesStats().size(), equalTo(2)); MeteredRepositoryTypeB repositoryB = (MeteredRepositoryTypeB) repositoriesService.repository("repoName"); assertNotNull(repositoryB); - assertNotNull(repository.cryptoManager); - assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + assertNotNull(repository.cryptoHandler); + assertEquals(kpTypeA, repository.cryptoHandler.kpType); } public void testCryptoManagersUnchangedWithSameCryptoMetadata() { @@ -259,21 +261,21 @@ public void testCryptoManagersUnchangedWithSameCryptoMetadata() { "repoName", MeteredRepositoryTypeA.TYPE, keyProviderName, - TestCryptoManagerTypeA.TYPE + kpTypeA ); repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", clusterStateWithRepoTypeA, emptyState())); assertThat(repositoriesService.repositoriesStats().size(), equalTo(1)); MeteredRepositoryTypeA repository = (MeteredRepositoryTypeA) repositoriesService.repository("repoName"); assertNotNull(repository); - assertNotNull(repository.cryptoManager); - assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + assertNotNull(repository.cryptoHandler); + assertEquals(kpTypeA, repository.cryptoHandler.kpType); repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", clusterStateWithRepoTypeA, emptyState())); assertThat(repositoriesService.repositoriesStats().size(), equalTo(1)); repository = (MeteredRepositoryTypeA) repositoriesService.repository("repoName"); assertNotNull(repository); - assertNotNull(repository.cryptoManager); - assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + assertNotNull(repository.cryptoHandler); + assertEquals(kpTypeA, repository.cryptoHandler.kpType); } public void testRepositoryUpdateWithDifferentCryptoMetadata() { @@ -283,7 +285,7 @@ public void testRepositoryUpdateWithDifferentCryptoMetadata() { "repoName", MeteredRepositoryTypeA.TYPE, keyProviderName, - TestCryptoManagerTypeA.TYPE + kpTypeA ); ClusterService clusterService = mock(ClusterService.class); @@ -303,13 +305,13 @@ public void testRepositoryUpdateWithDifferentCryptoMetadata() { assertThat(repositoriesService.repositoriesStats().size(), equalTo(1)); MeteredRepositoryTypeA repository = (MeteredRepositoryTypeA) repositoriesService.repository("repoName"); assertNotNull(repository); - assertNotNull(repository.cryptoManager); - assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + assertNotNull(repository.cryptoHandler); + assertEquals(kpTypeA, repository.cryptoHandler.kpType); expectThrows(IllegalArgumentException.class, () -> repositoriesService.registerRepository(request, null)); CryptoSettings cryptoSettings = new CryptoSettings(keyProviderName); - cryptoSettings.keyProviderType(TestCryptoManagerTypeA.TYPE); + cryptoSettings.keyProviderType(kpTypeA); cryptoSettings.settings(Settings.builder().put("key-1", "val-1")); request.cryptoSettings(cryptoSettings); expectThrows(IllegalArgumentException.class, () -> repositoriesService.registerRepository(request, null)); @@ -320,7 +322,7 @@ public void testRepositoryUpdateWithDifferentCryptoMetadata() { cryptoSettings.keyProviderName(keyProviderName); - cryptoSettings.keyProviderType(TestCryptoManagerTypeA.TYPE); + assertEquals(kpTypeA, repository.cryptoHandler.kpType); repositoriesService.registerRepository(request, null); } @@ -332,7 +334,7 @@ public void testCryptoManagerClusterStateChanges() { String keyProviderName = "kp-name-1"; String repoName = "repoName"; - String keyProviderType = TestCryptoManagerTypeA.TYPE; + String keyProviderType = kpTypeA; Settings.Builder settings = Settings.builder(); PutRepositoryRequest request = createPutRepositoryEncryptedRequest( repoName, @@ -353,12 +355,12 @@ public void testCryptoManagerClusterStateChanges() { ); repositoriesService.registerRepository(request, null); MeteredRepositoryTypeA repository = (MeteredRepositoryTypeA) repositoriesService.repository(repoName); - assertNotNull(repository.cryptoManager); - assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + assertNotNull(repository.cryptoHandler); + assertEquals(kpTypeA, repository.cryptoHandler.kpType); assertTrue(verified.get()); // No change - keyProviderType = TestCryptoManagerTypeA.TYPE; + keyProviderType = kpTypeA; settings = Settings.builder(); request = createPutRepositoryEncryptedRequest(repoName, MeteredRepositoryTypeA.TYPE, keyProviderName, settings, keyProviderType); verified.set(false); @@ -374,13 +376,13 @@ public void testCryptoManagerClusterStateChanges() { repositoriesService.registerRepository(request, null); repository = (MeteredRepositoryTypeA) repositoriesService.repository(repoName); - assertNotNull(repository.cryptoManager); - assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + assertNotNull(repository.cryptoHandler); + assertEquals(kpTypeA, repository.cryptoHandler.kpType); assertTrue(verified.get()); // Same crypto client in new repo repoName = "repoName-2"; - keyProviderType = TestCryptoManagerTypeA.TYPE; + keyProviderType = kpTypeA; settings = Settings.builder(); request = createPutRepositoryEncryptedRequest(repoName, MeteredRepositoryTypeA.TYPE, keyProviderName, settings, keyProviderType); verified.set(false); @@ -395,13 +397,13 @@ public void testCryptoManagerClusterStateChanges() { ); repositoriesService.registerRepository(request, null); repository = (MeteredRepositoryTypeA) repositoriesService.repository(repoName); - assertNotNull(repository.cryptoManager); - assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeA); + assertNotNull(repository.cryptoHandler); + assertEquals(kpTypeA, repository.cryptoHandler.kpType); assertTrue(verified.get()); // Different crypto client in new repo repoName = "repoName-3"; - keyProviderType = TestCryptoManagerTypeB.TYPE; + keyProviderType = kpTypeB; settings = Settings.builder(); request = createPutRepositoryEncryptedRequest(repoName, MeteredRepositoryTypeA.TYPE, keyProviderName, settings, keyProviderType); verified.set(false); @@ -416,8 +418,8 @@ public void testCryptoManagerClusterStateChanges() { ); repositoriesService.registerRepository(request, null); repository = (MeteredRepositoryTypeA) repositoriesService.repository(repoName); - assertNotNull(repository.cryptoManager); - assertTrue(repository.cryptoManager instanceof TestCryptoManagerTypeB); + assertNotNull(repository.cryptoHandler); + assertEquals(kpTypeB, repository.cryptoHandler.kpType); assertTrue(verified.get()); } @@ -530,6 +532,13 @@ private void assertThrowsOnRegister(String repoName) { } private static class TestCryptoProvider implements CryptoHandler { + final String kpName; + final String kpType; + + public TestCryptoProvider(String kpName, String kpType) { + this.kpName = kpName; + this.kpType = kpType; + } @Override public Object initEncryptionMetadata() { @@ -584,75 +593,27 @@ public DecryptedRangedStreamProvider createDecryptingStreamOfRange( public long estimateDecryptedLength(Object cryptoContext, long contentLength) { return 0; } - } - - private static abstract class TestCryptoManager implements CryptoManager { - private final String name; - private final AtomicInteger ref; - - private final CryptoHandler cryptoProvider; - - public TestCryptoManager(Settings settings, String keyProviderName) { - this.name = keyProviderName; - this.ref = new AtomicInteger(1); - this.cryptoProvider = new TestCryptoProvider(); - } @Override - public void incRef() { - ref.incrementAndGet(); - } + public void close() throws IOException { - @Override - public boolean tryIncRef() { - ref.incrementAndGet(); - return true; - } - - @Override - public boolean decRef() { - ref.decrementAndGet(); - return true; } - - public int getReferenceCount() { - return ref.get(); - } - - @Override - public String name() { - return name; - } - - public CryptoHandler getCryptoProvider() { - return cryptoProvider; - } - } - - private static class TestCryptoManagerTypeA extends TestCryptoManager { - public static final String TYPE = "type-A"; - - public TestCryptoManagerTypeA(Settings settings, String keyProviderName) { - super(settings, keyProviderName); - } - - @Override - public String type() { - return TYPE; - } - } - private static class TestCryptoManagerTypeB extends TestCryptoManager { - public static final String TYPE = "type-B"; + private static abstract class TestCryptoHandler implements CryptoPlugin { + private final Settings settings; - public TestCryptoManagerTypeB(Settings settings, String keyProviderName) { - super(settings, keyProviderName); + public TestCryptoHandler(Settings settings) { + this.settings = settings; } - @Override - public String type() { - return TYPE; + public CryptoHandler getOrCreateCryptoHandler( + MasterKeyProvider keyProvider, + String keyProviderName, + String keyProviderType, + Runnable onClose + ) { + return new TestCryptoProvider(keyProviderName, keyProviderType); } } @@ -885,7 +846,7 @@ public void close() { private static class MeteredRepositoryTypeA extends MeteredBlobStoreRepository { private static final String TYPE = "type-a"; private static final RepositoryStats STATS = new RepositoryStats(Map.of("GET", 10L)); - private final CryptoManager cryptoManager; + private final TestCryptoProvider cryptoHandler; private MeteredRepositoryTypeA(RepositoryMetadata metadata, ClusterService clusterService) { super( @@ -898,18 +859,12 @@ private MeteredRepositoryTypeA(RepositoryMetadata metadata, ClusterService clust ); if (metadata.cryptoMetadata() != null) { - switch (metadata.cryptoMetadata().keyProviderType()) { - case TestCryptoManagerTypeA.TYPE: - cryptoManager = new TestCryptoManagerTypeA(null, metadata.cryptoMetadata().keyProviderName()); - break; - case TestCryptoManagerTypeB.TYPE: - cryptoManager = new TestCryptoManagerTypeB(null, metadata.cryptoMetadata().keyProviderName()); - break; - default: - cryptoManager = null; - } + cryptoHandler = new TestCryptoProvider( + metadata.cryptoMetadata().keyProviderName(), + metadata.cryptoMetadata().keyProviderType() + ); } else { - cryptoManager = null; + cryptoHandler = null; } } @@ -932,7 +887,7 @@ public BlobPath basePath() { private static class MeteredRepositoryTypeB extends MeteredBlobStoreRepository { private static final String TYPE = "type-b"; private static final RepositoryStats STATS = new RepositoryStats(Map.of("LIST", 20L)); - private final CryptoManager cryptoManager; + private final TestCryptoProvider cryptoHandler; private MeteredRepositoryTypeB(RepositoryMetadata metadata, ClusterService clusterService) { super( @@ -945,18 +900,12 @@ private MeteredRepositoryTypeB(RepositoryMetadata metadata, ClusterService clust ); if (metadata.cryptoMetadata() != null) { - switch (metadata.cryptoMetadata().keyProviderType()) { - case TestCryptoManagerTypeA.TYPE: - cryptoManager = new TestCryptoManagerTypeA(null, metadata.cryptoMetadata().keyProviderName()); - break; - case TestCryptoManagerTypeB.TYPE: - cryptoManager = new TestCryptoManagerTypeB(null, metadata.cryptoMetadata().keyProviderName()); - break; - default: - cryptoManager = null; - } + cryptoHandler = new TestCryptoProvider( + metadata.cryptoMetadata().keyProviderName(), + metadata.cryptoMetadata().keyProviderType() + ); } else { - cryptoManager = null; + cryptoHandler = null; } } From e4a1841e23dadd3351216383fdc16bf56a0b783d Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Wed, 6 Sep 2023 14:17:04 -0700 Subject: [PATCH 02/10] Mute flaky test ReadContextListenerTests.testReadContextListenerFailure (#9842) --- .../blobstore/stream/read/listener/ReadContextListenerTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java index 936e2170bed6c..ed08a436d3ca0 100644 --- a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java +++ b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java @@ -70,6 +70,7 @@ public void testReadContextListener() throws InterruptedException, IOException { assertEquals(NUMBER_OF_PARTS * PART_SIZE, Files.size(fileLocation)); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9776") public void testReadContextListenerFailure() throws Exception { Path fileLocation = path.resolve(UUID.randomUUID().toString()); List blobPartStreams = initializeBlobPartStreams(); From 305c8a88a641356a1cab6123bd2cf78d835d3387 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Thu, 7 Sep 2023 10:23:50 +0530 Subject: [PATCH 03/10] [Remote Store] Muting RemoteStoreRefreshListenerTests.testReplicaPromotion (#9810) * Muting RemoteStoreRefreshListenerTests.testReplicaPromotion Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> * Retrigger integs Signed-off-by: shourya035 <114977491+shourya035@users.noreply.github.com> * Retrigger integs Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --------- Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Signed-off-by: shourya035 <114977491+shourya035@users.noreply.github.com> --- .../opensearch/index/shard/RemoteStoreRefreshListenerTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 188e371ae223e..c713ccdddd66a 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -261,6 +261,7 @@ public void testReplica() throws IOException { } } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9773") public void testReplicaPromotion() throws IOException, InterruptedException { setup(false, 3); remoteStoreRefreshListener.afterRefresh(true); From 3f18e0c346ea5f4fab35199619f33f8b28eb014d Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Thu, 7 Sep 2023 00:35:52 -0700 Subject: [PATCH 04/10] Bump 2.x version to 2.11 (#9856) Signed-off-by: Kunal Kotwani --- .ci/bwcVersions | 1 + libs/core/src/main/java/org/opensearch/Version.java | 1 + 2 files changed, 2 insertions(+) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 8cfd636b9fd2e..6b86da2c91261 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -23,3 +23,4 @@ BWC_VERSION: - "2.9.0" - "2.9.1" - "2.10.0" + - "2.11.0" diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index b05a069ba971c..32f4ca0317907 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -94,6 +94,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_9_1 = new Version(2090199, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_10_0 = new Version(2100099, org.apache.lucene.util.Version.LUCENE_9_7_0); + public static final Version V_2_11_0 = new Version(2110099, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_8_0); public static final Version CURRENT = V_3_0_0; From 951f051580a9feef37affd409d5892f50e9585a4 Mon Sep 17 00:00:00 2001 From: Harish Bhakuni Date: Thu, 7 Sep 2023 01:39:21 -0700 Subject: [PATCH 05/10] [Snapshot Interop] Fix Flakiness in Snapshot Interop Code (#9795) Signed-off-by: Harish Bhakuni Co-authored-by: Harish Bhakuni --- .../remotestore/RemoteRestoreSnapshotIT.java | 118 +++++++----------- .../BlobStoreRepositoryHelperTests.java | 24 +++- .../BlobStoreRepositoryRemoteIndexTests.java | 103 +++++++-------- .../AbstractSnapshotIntegTestCase.java | 9 +- 4 files changed, 120 insertions(+), 134 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index 346e9d12410b7..16264253b4000 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -10,7 +10,6 @@ import org.opensearch.action.DocWriteResponse; import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; -import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.opensearch.action.admin.indices.get.GetIndexRequest; import org.opensearch.action.admin.indices.get.GetIndexResponse; @@ -26,6 +25,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.snapshots.AbstractSnapshotIntegTestCase; +import org.opensearch.snapshots.SnapshotInfo; import org.opensearch.snapshots.SnapshotState; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; @@ -34,7 +34,9 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.concurrent.ExecutionException; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; @@ -129,32 +131,21 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut internalCluster().startDataOnlyNode(); logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(true) - .setIndices(indexName1, indexName2) - .get(); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat( - createSnapshotResponse.getSnapshotInfo().successfulShards(), - equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) - ); - assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + + SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1, indexName2))); + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); updateRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, false)); - CreateSnapshotResponse createSnapshotResponse2 = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName2) - .setWaitForCompletion(true) - .setIndices(indexName1, indexName2) - .get(); - assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat( - createSnapshotResponse2.getSnapshotInfo().successfulShards(), - equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()) + SnapshotInfo snapshotInfo2 = createSnapshot( + snapshotRepoName, + snapshotName2, + new ArrayList<>(Arrays.asList(indexName1, indexName2)) ); - assertThat(createSnapshotResponse2.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo2.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo2.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo2.successfulShards(), equalTo(snapshotInfo2.totalShards())); DeleteResponse deleteResponse = client().prepareDelete(indexName1, "0").execute().actionGet(); assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED); @@ -258,7 +249,6 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1 + 2); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9326") public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); String primary = internalCluster().startDataOnlyNode(); @@ -289,32 +279,24 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { internalCluster().startDataOnlyNode(); logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(true) - .setIndices(indexName1, indexName2) - .get(); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat( - createSnapshotResponse.getSnapshotInfo().successfulShards(), - equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) + SnapshotInfo snapshotInfo1 = createSnapshot( + snapshotRepoName, + snapshotName1, + new ArrayList<>(Arrays.asList(indexName1, indexName2)) ); - assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo1.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards())); + assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS)); updateRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, false)); - CreateSnapshotResponse createSnapshotResponse2 = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName2) - .setWaitForCompletion(true) - .setIndices(indexName1, indexName2) - .get(); - assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat( - createSnapshotResponse2.getSnapshotInfo().successfulShards(), - equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()) + SnapshotInfo snapshotInfo2 = createSnapshot( + snapshotRepoName, + snapshotName2, + new ArrayList<>(Arrays.asList(indexName1, indexName2)) ); - assertThat(createSnapshotResponse2.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo2.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo2.successfulShards(), equalTo(snapshotInfo2.totalShards())); + assertThat(snapshotInfo2.state(), equalTo(SnapshotState.SUCCESS)); DeleteResponse deleteResponse = client().prepareDelete(indexName1, "0").execute().actionGet(); assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED); @@ -342,6 +324,10 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { ensureGreen(indexName1, restoredIndexName2); assertDocsPresentInIndex(client, indexName1, numDocsInIndex1); assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2); + // indexing some new docs and validating + indexDocuments(client, indexName1, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(indexName1); + assertDocsPresentInIndex(client, indexName1, numDocsInIndex1 + 2); // deleting data for restoredIndexName1 and restoring from remote store. internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); @@ -356,9 +342,9 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { ensureGreen(indexName1); assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1); // indexing some new docs and validating - indexDocuments(client, indexName1, numDocsInIndex1, numDocsInIndex1 + 2); + indexDocuments(client, indexName1, numDocsInIndex1 + 2, numDocsInIndex1 + 4); ensureGreen(indexName1); - assertDocsPresentInIndex(client, indexName1, numDocsInIndex1 + 2); + assertDocsPresentInIndex(client, indexName1, numDocsInIndex1 + 4); } public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException { @@ -392,18 +378,14 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException internalCluster().startDataOnlyNode(); logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(true) - .setIndices(indexName1, indexName2) - .get(); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat( - createSnapshotResponse.getSnapshotInfo().successfulShards(), - equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) + SnapshotInfo snapshotInfo1 = createSnapshot( + snapshotRepoName, + snapshotName1, + new ArrayList<>(Arrays.asList(indexName1, indexName2)) ); - assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo1.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards())); + assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS)); Settings remoteStoreIndexSettings = Settings.builder() .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo2Name) @@ -479,18 +461,10 @@ public void testRestoreShallowSnapshotRepositoryOverriden() throws ExecutionExce ensureGreen(indexName1); logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(true) - .setIndices(indexName1) - .get(); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat( - createSnapshotResponse.getSnapshotInfo().successfulShards(), - equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) - ); - assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + SnapshotInfo snapshotInfo1 = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(List.of(indexName1))); + assertThat(snapshotInfo1.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards())); + assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS)); createRepository(BASE_REMOTE_REPO, "fs", absolutePath2); diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java index a24fd04d3d4f6..4e60cddd14d73 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java @@ -8,6 +8,7 @@ package org.opensearch.repositories.blobstore; +import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; @@ -15,6 +16,7 @@ import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.index.IndexModule; @@ -29,6 +31,8 @@ import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.fs.FsRepository; +import org.opensearch.snapshots.SnapshotInfo; +import org.opensearch.snapshots.SnapshotState; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchSingleNodeTestCase; @@ -36,9 +40,12 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; public class BlobStoreRepositoryHelperTests extends OpenSearchSingleNodeTestCase { @@ -122,13 +129,28 @@ protected Settings getRemoteStoreBackedIndexSettings() { .build(); } + protected SnapshotInfo createSnapshot(String repositoryName, String snapshot, List indices) { + logger.info("--> creating snapshot [{}] of {} in [{}]", snapshot, indices, repositoryName); + + final CreateSnapshotResponse response = client().admin() + .cluster() + .prepareCreateSnapshot(repositoryName, snapshot) + .setIndices(indices.toArray(Strings.EMPTY_ARRAY)) + .setWaitForCompletion(true) + .get(); + SnapshotInfo snapshotInfo = response.getSnapshotInfo(); + assertThat(snapshotInfo.state(), is(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.failedShards(), equalTo(0)); + return snapshotInfo; + } + protected void indexDocuments(Client client, String indexName) { int numDocs = randomIntBetween(10, 20); for (int i = 0; i < numDocs; i++) { String id = Integer.toString(i); client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get(); } - client.admin().indices().prepareFlush(indexName).get(); } protected IndexSettings getIndexSettings(String indexName) { diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index 0f24d60993f2f..0221661e6c03b 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -33,7 +33,6 @@ package org.opensearch.repositories.blobstore; import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; -import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.common.settings.Settings; @@ -48,11 +47,13 @@ import org.opensearch.repositories.RepositoryData; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.snapshots.SnapshotId; +import org.opensearch.snapshots.SnapshotInfo; import org.opensearch.test.FeatureFlagSetter; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Locale; @@ -137,13 +138,12 @@ public void testRetrieveShallowCopySnapshotCase1() throws IOException { indexDocuments(client, remoteStoreIndexName); logger.info("--> create first snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-1") - .setWaitForCompletion(true) - .setIndices(indexName, remoteStoreIndexName) - .get(); - final SnapshotId snapshotId1 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + SnapshotInfo snapshotInfo = createSnapshot( + snapshotRepositoryName, + "test-snap-1", + new ArrayList<>(Arrays.asList(indexName, remoteStoreIndexName)) + ); + final SnapshotId snapshotId1 = snapshotInfo.snapshotId(); String[] lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); assert (lockFiles.length == 0) : "there should be no lock files present in directory, but found " + Arrays.toString(lockFiles); @@ -154,13 +154,12 @@ public void testRetrieveShallowCopySnapshotCase1() throws IOException { .build(); updateRepository(client, snapshotRepositoryName, snapshotRepoSettingsForShallowCopy); - createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-2") - .setWaitForCompletion(true) - .setIndices(indexName, remoteStoreIndexName) - .get(); - final SnapshotId snapshotId2 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + snapshotInfo = createSnapshot( + snapshotRepositoryName, + "test-snap-2", + new ArrayList<>(Arrays.asList(indexName, remoteStoreIndexName)) + ); + final SnapshotId snapshotId2 = snapshotInfo.snapshotId(); lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); assert (lockFiles.length == 1) : "there should be only one lock file, but found " + Arrays.toString(lockFiles); @@ -168,13 +167,12 @@ public void testRetrieveShallowCopySnapshotCase1() throws IOException { logger.info("--> create another normal snapshot"); updateRepository(client, snapshotRepositoryName, snapshotRepoSettings); - createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-3") - .setWaitForCompletion(true) - .setIndices(indexName, remoteStoreIndexName) - .get(); - final SnapshotId snapshotId3 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + snapshotInfo = createSnapshot( + snapshotRepositoryName, + "test-snap-3", + new ArrayList<>(Arrays.asList(indexName, remoteStoreIndexName)) + ); + final SnapshotId snapshotId3 = snapshotInfo.snapshotId(); lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); assert (lockFiles.length == 1) : "there should be only one lock file, but found " + Arrays.toString(lockFiles); @@ -230,13 +228,8 @@ public void testGetRemoteStoreShallowCopyShardMetadata() throws IOException { .build(); updateRepository(client, snapshotRepositoryName, snapshotRepoSettingsForShallowCopy); - CreateSnapshotResponse createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-2") - .setWaitForCompletion(true) - .setIndices(remoteStoreIndexName) - .get(); - final SnapshotId snapshotId = createSnapshotResponse.getSnapshotInfo().snapshotId(); + SnapshotInfo snapshotInfo = createSnapshot(snapshotRepositoryName, "test-snap-2", new ArrayList<>(List.of(remoteStoreIndexName))); + final SnapshotId snapshotId = snapshotInfo.snapshotId(); String[] lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); assert (lockFiles.length == 1) : "there should be only one lock file, but found " + Arrays.toString(lockFiles); @@ -307,26 +300,24 @@ public void testRetrieveShallowCopySnapshotCase2() throws IOException { assertTrue(updatedRepositoryMetadata.settings().getAsBoolean(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), false)); - CreateSnapshotResponse createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-1") - .setWaitForCompletion(true) - .setIndices(indexName, remoteStoreIndexName) - .get(); - final SnapshotId snapshotId1 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + SnapshotInfo snapshotInfo = createSnapshot( + snapshotRepositoryName, + "test-snap-1", + new ArrayList<>(Arrays.asList(indexName, remoteStoreIndexName)) + ); + final SnapshotId snapshotId1 = snapshotInfo.snapshotId(); String[] lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); assert (lockFiles.length == 1) : "lock files are " + Arrays.toString(lockFiles); assert lockFiles[0].endsWith(snapshotId1.getUUID() + ".lock"); logger.info("--> create second remote index shallow snapshot"); - createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-2") - .setWaitForCompletion(true) - .setIndices(indexName, remoteStoreIndexName) - .get(); - final SnapshotId snapshotId2 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + snapshotInfo = createSnapshot( + snapshotRepositoryName, + "test-snap-2", + new ArrayList<>(Arrays.asList(indexName, remoteStoreIndexName)) + ); + final SnapshotId snapshotId2 = snapshotInfo.snapshotId(); lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); assert (lockFiles.length == 2) : "lock files are " + Arrays.toString(lockFiles); @@ -335,13 +326,12 @@ public void testRetrieveShallowCopySnapshotCase2() throws IOException { assert lockFiles[0].contains(snapshotId.getUUID()) || lockFiles[1].contains(snapshotId.getUUID()); } logger.info("--> create third remote index shallow snapshot"); - createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-3") - .setWaitForCompletion(true) - .setIndices(indexName, remoteStoreIndexName) - .get(); - final SnapshotId snapshotId3 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + snapshotInfo = createSnapshot( + snapshotRepositoryName, + "test-snap-3", + new ArrayList<>(Arrays.asList(indexName, remoteStoreIndexName)) + ); + final SnapshotId snapshotId3 = snapshotInfo.snapshotId(); lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); assert (lockFiles.length == 3); @@ -353,13 +343,12 @@ public void testRetrieveShallowCopySnapshotCase2() throws IOException { } logger.info("--> create normal snapshot"); createRepository(client, snapshotRepositoryName, snapshotRepoSettings); - createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-4") - .setWaitForCompletion(true) - .setIndices(indexName, remoteStoreIndexName) - .get(); - final SnapshotId snapshotId4 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + snapshotInfo = createSnapshot( + snapshotRepositoryName, + "test-snap-4", + new ArrayList<>(Arrays.asList(indexName, remoteStoreIndexName)) + ); + final SnapshotId snapshotId4 = snapshotInfo.snapshotId(); lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); assert (lockFiles.length == 3) : "lock files are " + Arrays.toString(lockFiles); diff --git a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java index a17d71957167a..1bb1e44a8a600 100644 --- a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -480,11 +480,12 @@ protected String initWithSnapshotVersion(String repoName, Path repoPath, Version protected SnapshotInfo createFullSnapshot(String repoName, String snapshotName) { logger.info("--> creating full snapshot [{}] in [{}]", snapshotName, repoName); - CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(repoName, snapshotName) - .setIncludeGlobalState(true) + final CreateSnapshotResponse response = client().admin() + .cluster() + .prepareCreateSnapshot(repoName, snapshotName) .setWaitForCompletion(true) .get(); - final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); + final SnapshotInfo snapshotInfo = response.getSnapshotInfo(); assertThat(snapshotInfo.successfulShards(), is(snapshotInfo.totalShards())); assertThat(snapshotInfo.state(), is(SnapshotState.SUCCESS)); return snapshotInfo; @@ -498,8 +499,8 @@ protected SnapshotInfo createSnapshot(String repositoryName, String snapshot, Li .setIndices(indices.toArray(Strings.EMPTY_ARRAY)) .setWaitForCompletion(true) .get(); + SnapshotInfo snapshotInfo = response.getSnapshotInfo(); - final SnapshotInfo snapshotInfo = response.getSnapshotInfo(); assertThat(snapshotInfo.state(), is(SnapshotState.SUCCESS)); assertThat(snapshotInfo.successfulShards(), greaterThan(0)); assertThat(snapshotInfo.failedShards(), equalTo(0)); From 199123d9d62a726dfcbf8e0530dff265844fe91f Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Thu, 7 Sep 2023 16:35:03 +0530 Subject: [PATCH 06/10] Decrease store refcount on any failure to create NRTReplicationEngine (#9796) * Decrease store refcount on any failure to create NRTReplicationEngine Signed-off-by: Sachin Kale * Add unit tests Signed-off-by: Sachin Kale * Fix spotless Signed-off-by: Sachin Kale --------- Signed-off-by: Sachin Kale Co-authored-by: Sachin Kale --- .../index/engine/NRTReplicationEngine.java | 14 ++++++++++-- .../engine/NRTReplicationEngineTests.java | 22 +++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java index b529dfbe13bf4..570a2b186841a 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java @@ -24,6 +24,7 @@ import org.opensearch.index.seqno.SeqNoStats; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.translog.Translog; +import org.opensearch.index.translog.TranslogCorruptedException; import org.opensearch.index.translog.TranslogDeletionPolicy; import org.opensearch.index.translog.TranslogException; import org.opensearch.index.translog.TranslogManager; @@ -71,6 +72,7 @@ public NRTReplicationEngine(EngineConfig engineConfig) { store.incRef(); NRTReplicationReaderManager readerManager = null; WriteOnlyTranslogManager translogManagerRef = null; + boolean success = false; try { this.replicaFileTracker = new ReplicaFileTracker(store::deleteQuiet); this.lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo(); @@ -124,9 +126,17 @@ public void onAfterTranslogSync() { engineConfig.getPrimaryModeSupplier() ); this.translogManager = translogManagerRef; - } catch (IOException e) { - IOUtils.closeWhileHandlingException(store::decRef, readerManager, translogManagerRef); + success = true; + } catch (IOException | TranslogCorruptedException e) { throw new EngineCreationFailureException(shardId, "failed to create engine", e); + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(readerManager, translogManagerRef); + if (isClosed.get() == false) { + // failure, we need to dec the store reference + store.decRef(); + } + } } } diff --git a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java index 22eb5195af507..7b587e9a83d2d 100644 --- a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java @@ -75,6 +75,28 @@ public void testCreateEngine() throws IOException { } } + public void testCreateEngineWithException() throws IOException { + final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); + final Store nrtEngineStore = createStore(INDEX_SETTINGS, newDirectory()); + try { + // Passing null translogPath to induce failure + final EngineConfig replicaConfig = config( + defaultSettings, + nrtEngineStore, + null, + NoMergePolicy.INSTANCE, + null, + null, + globalCheckpoint::get + ); + new NRTReplicationEngine(replicaConfig); + } catch (Exception e) { + // Ignore as engine creation will fail + } + assertEquals(1, nrtEngineStore.refCount()); + nrtEngineStore.close(); + } + public void testEngineWritesOpsToTranslog() throws Exception { final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); From 91f28d4f3885e6223875ab047fd9b769dc4fa7de Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Thu, 7 Sep 2023 16:57:31 +0530 Subject: [PATCH 07/10] Mute org.opensearch.index.codec.MultiCodecMergeIT.testForceMergeMultipleCodecs (#9873) Signed-off-by: Sachin Kale Co-authored-by: Sachin Kale --- .../java/org/opensearch/index/codec/MultiCodecMergeIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/custom-codecs/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java b/plugins/custom-codecs/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java index 82633513a2f50..fda9327c79dc1 100644 --- a/plugins/custom-codecs/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java +++ b/plugins/custom-codecs/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java @@ -49,6 +49,7 @@ protected Collection> nodePlugins() { return Collections.singletonList(CustomCodecPlugin.class); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9872") public void testForceMergeMultipleCodecs() throws ExecutionException, InterruptedException { Map codecMap = Map.of( From e6a7202874ecba347117ccbfc0d220c197ff0635 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Thu, 7 Sep 2023 17:45:05 +0530 Subject: [PATCH 08/10] [Remote Store] Update version check for Remote Translog Store upload flow stats in nodes stats API (#9754) Signed-off-by: Bhumika Saini --- .../java/org/opensearch/index/translog/TranslogStats.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogStats.java b/server/src/main/java/org/opensearch/index/translog/TranslogStats.java index 25bf40e129626..a4699cea671a0 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogStats.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogStats.java @@ -71,9 +71,7 @@ public TranslogStats(StreamInput in) throws IOException { uncommittedOperations = in.readVInt(); uncommittedSizeInBytes = in.readVLong(); earliestLastModifiedAge = in.readVLong(); - // TODO: remoteTranslogStats = in.getVersion().onOrAfter(Version.V_2_10_0) ? in.readOptionalWriteable(RemoteTranslogStats::new) : - // new RemoteTranslogStats(); - remoteTranslogStats = in.getVersion().onOrAfter(Version.CURRENT) + remoteTranslogStats = in.getVersion().onOrAfter(Version.V_2_10_0) ? in.readOptionalWriteable(RemoteTranslogStats::new) : new RemoteTranslogStats(); } @@ -184,8 +182,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(uncommittedOperations); out.writeVLong(uncommittedSizeInBytes); out.writeVLong(earliestLastModifiedAge); - // TODO: if (out.getVersion().onOrAfter(Version.V_2_10_0)) { - if (out.getVersion().onOrAfter(Version.CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_2_10_0)) { out.writeOptionalWriteable(remoteTranslogStats); } } From 7760074148d4cd81ee15920fe34b5ac2bbaa41d5 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Thu, 7 Sep 2023 17:58:53 +0530 Subject: [PATCH 09/10] Removing asserts based on translog stats as it is not applicable for remote store (#9878) Signed-off-by: Gaurav Bafna --- .../org/opensearch/index/shard/RemoteIndexShardTests.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java index 0739174fc3afd..bcda7b941aaee 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java @@ -81,22 +81,18 @@ protected ReplicationGroup getReplicationGroup(int numberOfReplicas) throws IOEx return createGroup(numberOfReplicas, settings, indexMapping, new NRTReplicationEngineFactory(), createTempDir()); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9624") public void testNRTReplicaWithRemoteStorePromotedAsPrimaryRefreshRefresh() throws Exception { testNRTReplicaWithRemoteStorePromotedAsPrimary(false, false); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9624") public void testNRTReplicaWithRemoteStorePromotedAsPrimaryRefreshCommit() throws Exception { testNRTReplicaWithRemoteStorePromotedAsPrimary(false, true); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9624") public void testNRTReplicaWithRemoteStorePromotedAsPrimaryCommitRefresh() throws Exception { testNRTReplicaWithRemoteStorePromotedAsPrimary(true, false); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9624") public void testNRTReplicaWithRemoteStorePromotedAsPrimaryCommitCommit() throws Exception { testNRTReplicaWithRemoteStorePromotedAsPrimary(true, true); } @@ -156,10 +152,6 @@ public void testNRTReplicaWithRemoteStorePromotedAsPrimary(boolean performFlushF assertEquals(InternalEngine.class, nextPrimary.getEngine().getClass()); assertDocCounts(nextPrimary, totalDocs, totalDocs); - // As we are downloading segments from remote segment store on failover, there should not be - // any operations replayed from translog - assertEquals(prevOperationCount, nextPrimary.translogStats().estimatedNumberOfOperations()); - // refresh and push segments to our other replica. nextPrimary.refresh("test"); From d58943def83e6ce8a582f315a2255997303e019e Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Thu, 7 Sep 2023 18:03:39 +0530 Subject: [PATCH 10/10] [Remote Cluster State] Move remote cluster state related settings to node attrs (#9811) --------- Signed-off-by: bansvaru --- ...emoteStoreMockRepositoryIntegTestCase.java | 14 ++++ .../RemoteStoreBaseIntegTestCase.java | 16 +++- .../RemoteStoreClusterStateRestoreIT.java | 7 +- .../coordination/CoordinationState.java | 4 +- .../common/settings/ClusterSettings.java | 1 - .../opensearch/gateway/GatewayMetaState.java | 4 +- .../remote/RemoteClusterStateService.java | 23 ++--- .../main/java/org/opensearch/node/Node.java | 3 +- .../remotestore/RemoteStoreNodeAttribute.java | 7 ++ .../coordination/CoordinationStateTests.java | 24 +++++- .../coordination/JoinTaskExecutorTests.java | 63 ++++++++++---- .../GatewayMetaStatePersistedStateTests.java | 29 ++++++- .../RemoteClusterStateServiceTests.java | 24 +++++- .../node/RemoteStoreNodeAttributeTests.java | 83 ++++++++++++------- .../BlobStoreRepositoryRemoteIndexTests.java | 6 ++ .../gateway/MockGatewayMetaState.java | 3 +- 16 files changed, 230 insertions(+), 81 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java index 73f5278c175a2..a5017144ce890 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java @@ -31,6 +31,7 @@ import java.util.stream.Collectors; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; @@ -91,6 +92,16 @@ public Settings buildRemoteStoreNodeAttributes(Path repoLocation, double ioFailu "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, TRANSLOG_REPOSITORY_NAME ); + String stateRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + REPOSITORY_NAME + ); + String stateRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + REPOSITORY_NAME + ); return Settings.builder() .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME) @@ -104,6 +115,9 @@ public Settings buildRemoteStoreNodeAttributes(Path repoLocation, double ioFailu .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, TRANSLOG_REPOSITORY_NAME) .put(translogRepoTypeAttributeKey, "mock") .put(translogRepoSettingsAttributeKeyPrefix + "location", repoLocation) + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME) + .put(stateRepoTypeAttributeKey, "mock") + .put(stateRepoSettingsAttributeKeyPrefix + "location", repoLocation) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 7173fda89505c..1660201dcb307 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -49,6 +49,7 @@ import java.util.stream.Collectors; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; @@ -222,6 +223,16 @@ public static Settings buildRemoteStoreNodeAttributes( "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, translogRepoName ); + String stateRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + segmentRepoName + ); + String stateRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + segmentRepoName + ); Settings.Builder settings = Settings.builder() .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) @@ -229,7 +240,10 @@ public static Settings buildRemoteStoreNodeAttributes( .put(segmentRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName) .put(translogRepoTypeAttributeKey, FsRepository.TYPE) - .put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath); + .put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath) + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) + .put(stateRepoTypeAttributeKey, FsRepository.TYPE) + .put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath); if (withRateLimiterAttributes) { settings.put(segmentRepoSettingsAttributeKeyPrefix + "compress", randomBoolean()) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index cbead232e1262..5e92bb195680b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -23,7 +23,6 @@ import java.util.concurrent.ExecutionException; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; -import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_REPOSITORY_SETTING; import static org.opensearch.indices.ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE; import static org.opensearch.indices.ShardLimitValidator.SETTING_MAX_SHARDS_PER_CLUSTER_KEY; @@ -32,11 +31,7 @@ public class RemoteStoreClusterStateRestoreIT extends BaseRemoteStoreRestoreIT { @Override protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) - .put(REMOTE_CLUSTER_STATE_REPOSITORY_SETTING.getKey(), REPOSITORY_NAME) - .build(); + return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true).build(); } private void addNewNodes(int dataNodeCount, int clusterManagerNodeCount) { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java index fa32618b96ef3..a339852e6ed8d 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java @@ -52,7 +52,7 @@ import java.util.Set; import static org.opensearch.cluster.coordination.Coordinator.ZEN1_BWC_TERM; -import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** * The core class of the cluster state coordination algorithm, directly implementing the @@ -101,7 +101,7 @@ public CoordinationState( .getLastAcceptedState() .getLastAcceptedConfiguration(); this.publishVotes = new VoteCollection(); - this.isRemoteStateEnabled = REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings); + this.isRemoteStateEnabled = isRemoteStoreClusterStateEnabled(settings); } public long getCurrentTerm() { diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index b8fe322234140..8c04e01a75fb4 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -670,7 +670,6 @@ public void apply(Settings value, Settings current, Settings previous) { // Remote cluster state settings RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING, - RemoteClusterStateService.REMOTE_CLUSTER_STATE_REPOSITORY_SETTING, RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING ) ) diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java index 4df18e4014c3d..ac6297598ecf5 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java @@ -84,7 +84,7 @@ import java.util.function.UnaryOperator; import static org.opensearch.common.util.concurrent.OpenSearchExecutors.daemonThreadFactory; -import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** * Loads (and maybe upgrades) cluster metadata at startup, and persistently stores cluster metadata for future restarts. @@ -165,7 +165,7 @@ public void start( if (DiscoveryNode.isClusterManagerNode(settings)) { persistedState = new LucenePersistedState(persistedClusterStateService, currentTerm, clusterState); - if (REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) == true) { + if (isRemoteStoreClusterStateEnabled(settings)) { remotePersistedState = new RemotePersistedState(remoteClusterStateService); } } else { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 2c38bec4802d7..d2f6fd8ebe228 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -28,6 +28,8 @@ import org.opensearch.core.index.Index; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.node.Node; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -52,6 +54,7 @@ import java.util.stream.Collectors; import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** * A Service which provides APIs to upload and download cluster metadata from remote store. @@ -80,23 +83,13 @@ public class RemoteClusterStateService implements Closeable { /** * Used to specify if cluster state metadata should be published to remote store */ - // TODO The remote state enabled and repository settings should be read from node attributes. - // Dependent on https://github.com/opensearch-project/OpenSearch/pull/9105/ public static final Setting REMOTE_CLUSTER_STATE_ENABLED_SETTING = Setting.boolSetting( "cluster.remote_store.state.enabled", false, Property.NodeScope, Property.Final ); - /** - * Used to specify default repo to use for cluster state metadata upload - */ - public static final Setting REMOTE_CLUSTER_STATE_REPOSITORY_SETTING = Setting.simpleString( - "cluster.remote_store.state.repository", - "", - Property.NodeScope, - Property.Final - ); + private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class); public static final String DELIMITER = "__"; @@ -115,7 +108,7 @@ public RemoteClusterStateService( ClusterSettings clusterSettings, LongSupplier relativeTimeNanosSupplier ) { - assert REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) == true : "Remote cluster state is not enabled"; + assert isRemoteStoreClusterStateEnabled(settings) : "Remote cluster state is not enabled"; this.nodeId = nodeId; this.repositoriesService = repositoriesService; this.settings = settings; @@ -384,8 +377,10 @@ void ensureRepositorySet() { if (blobStoreRepository != null) { return; } - assert REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) == true : "Remote cluster state is not enabled"; - final String remoteStoreRepo = REMOTE_CLUSTER_STATE_REPOSITORY_SETTING.get(settings); + assert isRemoteStoreClusterStateEnabled(settings) : "Remote cluster state is not enabled"; + final String remoteStoreRepo = settings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY + ); assert remoteStoreRepo != null : "Remote Cluster State repository is not configured"; final Repository repository = repositoriesService.get().repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index e2365adb440cc..d34d4ed3bed3e 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -272,6 +272,7 @@ import static org.opensearch.env.NodeEnvironment.collectFileCacheDataPath; import static org.opensearch.index.ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreAttributePresent; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** * A node represent a node within a cluster ({@code cluster.name}). The {@link #client()} can be used @@ -681,7 +682,7 @@ protected Node( threadPool::relativeTimeInMillis ); final RemoteClusterStateService remoteClusterStateService; - if (RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) == true) { + if (isRemoteStoreClusterStateEnabled(settings)) { remoteClusterStateService = new RemoteClusterStateService( nodeEnvironment.nodeId(), repositoriesServiceReference::get, diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index 9cb171790dbe2..7b2a6c34d3db6 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -13,6 +13,7 @@ import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.Settings; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.node.Node; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -36,6 +37,7 @@ public class RemoteStoreNodeAttribute { public static final String REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX = "remote_store"; public static final String REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.segment.repository"; public static final String REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.translog.repository"; + public static final String REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.state.repository"; public static final String REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT = "remote_store.repository.%s.type"; public static final String REMOTE_STORE_REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT = "remote_store.repository.%s." + CryptoMetadata.CRYPTO_METADATA_KEY; @@ -134,6 +136,7 @@ private RepositoriesMetadata buildRepositoriesMetadata(DiscoveryNode node) { repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY)); repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY)); + repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)); for (String repositoryName : repositoryNames) { repositoryMetadataList.add(buildRepositoryMetadata(node, repositoryName)); @@ -146,6 +149,10 @@ public static boolean isRemoteStoreAttributePresent(Settings settings) { return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX).isEmpty() == false; } + public static boolean isRemoteStoreClusterStateEnabled(Settings settings) { + return RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) && isRemoteStoreAttributePresent(settings); + } + public RepositoriesMetadata getRepositoriesMetadata() { return this.repositoriesMetadata; } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java index 02fae835d7cbf..ef56d70d6153c 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java @@ -48,12 +48,14 @@ import org.opensearch.gateway.GatewayMetaState.RemotePersistedState; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteClusterStateService; +import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.EqualsHashCodeTestUtils; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; import java.io.IOException; import java.util.Collections; +import java.util.Locale; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -62,6 +64,9 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -937,9 +942,26 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); persistedStateRegistry.addPersistedState(PersistedStateType.LOCAL, ps1); persistedStateRegistry.addPersistedState(PersistedStateType.REMOTE, new RemotePersistedState(remoteClusterStateService)); - final Settings settings = Settings.builder() + + String randomRepoName = "randomRepoName"; + String stateRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + randomRepoName + ); + String stateRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + randomRepoName + ); + + Settings settings = Settings.builder() + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, randomRepoName) + .put(stateRepoTypeAttributeKey, FsRepository.TYPE) + .put(stateRepoSettingsAttributeKeyPrefix + "location", "randomRepoPath") .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) .build(); + final CoordinationState coordinationState = createCoordinationState(persistedStateRegistry, node1, settings); coordinationState.handlePrePublish(clusterState); Mockito.verifyNoInteractions(remoteClusterStateService); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index e0844fd521d3f..5952cc1bcaac2 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -67,6 +67,7 @@ import java.util.Map; import java.util.stream.Collectors; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; @@ -437,7 +438,8 @@ public void testPreventJoinClusterWithRemoteStoreNodeWithDifferentAttributesJoin for (Map.Entry nodeAttribute : existingNodeAttributes.entrySet()) { if (nodeAttribute.getKey() != REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY - && nodeAttribute.getKey() != REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY) { + && nodeAttribute.getKey() != REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY + && nodeAttribute.getKey() != REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY) { remoteStoreNodeAttributes.put(nodeAttribute.getKey(), nodeAttribute.getValue() + "-new"); validateAttributes(remoteStoreNodeAttributes, currentState, existingNode); remoteStoreNodeAttributes.put(nodeAttribute.getKey(), nodeAttribute.getValue()); @@ -465,6 +467,13 @@ public void testPreventJoinClusterWithRemoteStoreNodeWithDifferentNameAttributes } else if (REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO + "new"); validateAttributes(remoteStoreNodeAttributes, currentState, existingNode); + } else if (REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { + Map remoteStoreNodeAttributes = remoteStoreNodeAttributes( + SEGMENT_REPO, + TRANSLOG_REPO, + CLUSTER_STATE_REPO + "new" + ); + validateAttributes(remoteStoreNodeAttributes, currentState, existingNode); } } } @@ -560,7 +569,7 @@ public void testUpdatesClusterStateWithSingleNodeCluster() throws Exception { assertThat(result.executionResults.entrySet(), hasSize(1)); final ClusterStateTaskExecutor.TaskResult taskResult = result.executionResults.values().iterator().next(); assertTrue(taskResult.isSuccess()); - validateRepositoryMetadata(result.resultingState, clusterManagerNode, 2); + validateRepositoryMetadata(result.resultingState, clusterManagerNode, 3); } public void testUpdatesClusterStateWithMultiNodeCluster() throws Exception { @@ -623,7 +632,7 @@ public void testUpdatesClusterStateWithMultiNodeCluster() throws Exception { assertThat(result.executionResults.entrySet(), hasSize(1)); final ClusterStateTaskExecutor.TaskResult taskResult = result.executionResults.values().iterator().next(); assertTrue(taskResult.isSuccess()); - validateRepositoryMetadata(result.resultingState, clusterManagerNode, 2); + validateRepositoryMetadata(result.resultingState, clusterManagerNode, 3); } public void testUpdatesClusterStateWithSingleNodeClusterAndSameRepository() throws Exception { @@ -668,7 +677,7 @@ public void testUpdatesClusterStateWithSingleNodeClusterAndSameRepository() thro assertThat(result.executionResults.entrySet(), hasSize(1)); final ClusterStateTaskExecutor.TaskResult taskResult = result.executionResults.values().iterator().next(); assertTrue(taskResult.isSuccess()); - validateRepositoryMetadata(result.resultingState, clusterManagerNode, 1); + validateRepositoryMetadata(result.resultingState, clusterManagerNode, 2); } public void testUpdatesClusterStateWithMultiNodeClusterAndSameRepository() throws Exception { @@ -729,7 +738,7 @@ public void testUpdatesClusterStateWithMultiNodeClusterAndSameRepository() throw assertThat(result.executionResults.entrySet(), hasSize(1)); final ClusterStateTaskExecutor.TaskResult taskResult = result.executionResults.values().iterator().next(); assertTrue(taskResult.isSuccess()); - validateRepositoryMetadata(result.resultingState, clusterManagerNode, 1); + validateRepositoryMetadata(result.resultingState, clusterManagerNode, 2); } private void validateRepositoryMetadata(ClusterState updatedState, DiscoveryNode existingNode, int expectedRepositories) @@ -737,14 +746,17 @@ private void validateRepositoryMetadata(ClusterState updatedState, DiscoveryNode final RepositoriesMetadata repositoriesMetadata = updatedState.metadata().custom(RepositoriesMetadata.TYPE); assertTrue(repositoriesMetadata.repositories().size() == expectedRepositories); - if (repositoriesMetadata.repositories().size() == 2) { + if (repositoriesMetadata.repositories().size() == 2 || repositoriesMetadata.repositories().size() == 3) { final RepositoryMetadata segmentRepositoryMetadata = buildRepositoryMetadata(existingNode, SEGMENT_REPO); final RepositoryMetadata translogRepositoryMetadata = buildRepositoryMetadata(existingNode, TRANSLOG_REPO); for (RepositoryMetadata repositoryMetadata : repositoriesMetadata.repositories()) { if (repositoryMetadata.name().equals(segmentRepositoryMetadata.name())) { assertTrue(segmentRepositoryMetadata.equalsIgnoreGenerations(repositoryMetadata)); - } else if (repositoryMetadata.name().equals(segmentRepositoryMetadata.name())) { + } else if (repositoryMetadata.name().equals(translogRepositoryMetadata.name())) { assertTrue(translogRepositoryMetadata.equalsIgnoreGenerations(repositoryMetadata)); + } else if (repositoriesMetadata.repositories().size() == 3) { + final RepositoryMetadata clusterStateRepoMetadata = buildRepositoryMetadata(existingNode, CLUSTER_STATE_REPO); + assertTrue(clusterStateRepoMetadata.equalsIgnoreGenerations(repositoryMetadata)); } } } else if (repositoriesMetadata.repositories().size() == 1) { @@ -768,9 +780,14 @@ private DiscoveryNode newDiscoveryNode(Map attributes) { private static final String SEGMENT_REPO = "segment-repo"; private static final String TRANSLOG_REPO = "translog-repo"; + private static final String CLUSTER_STATE_REPO = "cluster-state-repo"; private static final String COMMON_REPO = "remote-repo"; private Map remoteStoreNodeAttributes(String segmentRepoName, String translogRepoName) { + return remoteStoreNodeAttributes(segmentRepoName, translogRepoName, CLUSTER_STATE_REPO); + } + + private Map remoteStoreNodeAttributes(String segmentRepoName, String translogRepoName, String clusterStateRepo) { String segmentRepositoryTypeAttributeKey = String.format( Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, @@ -791,6 +808,16 @@ private Map remoteStoreNodeAttributes(String segmentRepoName, St REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, translogRepoName ); + String clusterStateRepositoryTypeAttributeKey = String.format( + Locale.getDefault(), + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + clusterStateRepo + ); + String clusterStateRepositorySettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + clusterStateRepo + ); return new HashMap<>() { { @@ -802,6 +829,10 @@ private Map remoteStoreNodeAttributes(String segmentRepoName, St putIfAbsent(translogRepositoryTypeAttributeKey, "s3"); putIfAbsent(translogRepositorySettingsAttributeKeyPrefix + "bucket", "translog_bucket"); putIfAbsent(translogRepositorySettingsAttributeKeyPrefix + "base_path", "/translog/path"); + put(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, clusterStateRepo); + putIfAbsent(clusterStateRepositoryTypeAttributeKey, "s3"); + putIfAbsent(clusterStateRepositorySettingsAttributeKeyPrefix + "bucket", "state_bucket"); + putIfAbsent(clusterStateRepositorySettingsAttributeKeyPrefix + "base_path", "/state/path"); } }; } @@ -812,16 +843,14 @@ private void validateAttributes(Map remoteStoreNodeAttributes, C IllegalStateException.class, () -> JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()) ); - assertTrue( - e.getMessage() - .equals( - "a remote store node [" - + joiningNode - + "] is trying to join a remote store cluster with incompatible node attributes in " - + "comparison with existing node [" - + existingNode - + "]" - ) + assertEquals( + e.getMessage(), + "a remote store node [" + + joiningNode + + "] is trying to join a remote store cluster with incompatible node attributes in " + + "comparison with existing node [" + + existingNode + + "]" ); } diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java index c293768151fa4..98a9c0ded2d9d 100644 --- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -67,6 +67,7 @@ import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.node.Node; import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -79,12 +80,17 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import org.mockito.Mockito; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; import static org.opensearch.test.NodeRoles.nonClusterManagerNode; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -445,7 +451,7 @@ public void testDataOnlyNodePersistence() throws Exception { () -> 0L ); Supplier remoteClusterStateServiceSupplier = () -> { - if (RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) == true) { + if (isRemoteStoreClusterStateEnabled(settings)) { return new RemoteClusterStateService( nodeEnvironment.nodeId(), () -> new RepositoriesService( @@ -757,11 +763,26 @@ public void testGatewayForRemoteState() throws IOException { try { gateway = new MockGatewayMetaState(localNode, bigArrays); final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); - final Settings settingWithRemoteStateEnabled = Settings.builder() - .put(settings) + + String stateRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + "randomRepoName" + ); + String stateRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + "randomRepoName" + ); + + Settings settings = Settings.builder() + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "randomRepoName") + .put(stateRepoTypeAttributeKey, FsRepository.TYPE) + .put(stateRepoSettingsAttributeKeyPrefix + "location", "randomRepoPath") .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) .build(); - gateway.start(settingWithRemoteStateEnabled, nodeEnvironment, xContentRegistry(), persistedStateRegistry); + gateway.start(settings, nodeEnvironment, xContentRegistry(), persistedStateRegistry); + final CoordinationState.PersistedState persistedState = gateway.getPersistedState(); assertThat(persistedState, instanceOf(GatewayMetaState.LucenePersistedState.class)); assertThat( diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 1f4c32b59f183..da1f97f7042f1 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -38,6 +38,7 @@ import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryMissingException; import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; import org.junit.Assert; @@ -51,11 +52,15 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.Supplier; import org.mockito.ArgumentCaptor; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -79,10 +84,25 @@ public void setup() { repositoriesServiceSupplier = mock(Supplier.class); repositoriesService = mock(RepositoriesService.class); when(repositoriesServiceSupplier.get()).thenReturn(repositoriesService); - final Settings settings = Settings.builder() + + String stateRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + "remote_store_repository" + ); + String stateRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + "remote_store_repository" + ); + + Settings settings = Settings.builder() + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") + .put(stateRepoTypeAttributeKey, FsRepository.TYPE) + .put(stateRepoSettingsAttributeKeyPrefix + "location", "randomRepoPath") .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) - .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_REPOSITORY_SETTING.getKey(), "remote_store_repository") .build(); + blobStoreRepository = mock(BlobStoreRepository.class); blobStore = mock(BlobStore.class); when(blobStoreRepository.blobStore()).thenReturn(blobStore); diff --git a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java index 5206064273334..c4ba271d27ae9 100644 --- a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java +++ b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java @@ -19,9 +19,17 @@ import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.Locale; import java.util.Map; import static java.util.Collections.emptySet; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_CRYPTO_SETTINGS_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; public class RemoteStoreNodeAttributeTests extends OpenSearchTestCase { @@ -29,24 +37,31 @@ public class RemoteStoreNodeAttributeTests extends OpenSearchTestCase { static private final String REGION = "us-east-1"; public void testCryptoMetadata() throws UnknownHostException { + String repoName = "remote-store-A"; + String repoTypeSettingKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, repoName); + String repoSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, repoName); + String repoCryptoMetadataKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT, repoName); + String repoCryptoMetadataSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_CRYPTO_SETTINGS_PREFIX, repoName); Map attr = Map.of( - "remote_store.segment.repository", - "remote-store-A", - "remote_store.translog.repository", - "remote-store-A", - "remote_store.repository.remote-store-A.type", + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + repoTypeSettingKey, "s3", - "remote_store.repository.remote-store-A.settings.bucket", + repoSettingsKey, "abc", - "remote_store.repository.remote-store-A.settings.base_path", + repoSettingsKey + "base_path", "xyz", - "remote_store.repository.remote-store-A.crypto_metadata.key_provider_name", + repoCryptoMetadataKey + ".key_provider_name", "store-test", - "remote_store.repository.remote-store-A.crypto_metadata.key_provider_type", + repoCryptoMetadataKey + ".key_provider_type", "aws-kms", - "remote_store.repository.remote-store-A.crypto_metadata.settings.region", + repoCryptoMetadataSettingsKey + ".region", REGION, - "remote_store.repository.remote-store-A.crypto_metadata.settings.key_arn", + repoCryptoMetadataSettingsKey + ".key_arn", KEY_ARN ); DiscoveryNode node = new DiscoveryNode( @@ -68,22 +83,27 @@ public void testCryptoMetadata() throws UnknownHostException { } public void testInvalidCryptoMetadata() throws UnknownHostException { + String repoName = "remote-store-A"; + String repoTypeSettingKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, repoName); + String repoSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, repoName); + String repoCryptoMetadataKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT, repoName); + String repoCryptoMetadataSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_CRYPTO_SETTINGS_PREFIX, repoName); Map attr = Map.of( - "remote_store.segment.repository", - "remote-store-A", - "remote_store.translog.repository", - "remote-store-A", - "remote_store.repository.remote-store-A.type", + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + repoTypeSettingKey, "s3", - "remote_store.repository.remote-store-A.settings.bucket", + repoSettingsKey, "abc", - "remote_store.repository.remote-store-A.settings.base_path", + repoSettingsKey + "base_path", "xyz", - "remote_store.repository.remote-store-A.crypto_metadata.key_provider_name", - "store-test", - "remote_store.repository.remote-store-A.crypto_metadata.settings.region", + repoCryptoMetadataSettingsKey + ".region", REGION, - "remote_store.repository.remote-store-A.crypto_metadata.settings.key_arn", + repoCryptoMetadataSettingsKey + ".key_arn", KEY_ARN ); DiscoveryNode node = new DiscoveryNode( @@ -98,16 +118,21 @@ public void testInvalidCryptoMetadata() throws UnknownHostException { } public void testNoCryptoMetadata() throws UnknownHostException { + String repoName = "remote-store-A"; + String repoTypeSettingKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, repoName); + String repoSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, repoName); Map attr = Map.of( - "remote_store.segment.repository", - "remote-store-A", - "remote_store.translog.repository", - "remote-store-A", - "remote_store.repository.remote-store-A.type", + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + repoTypeSettingKey, "s3", - "remote_store.repository.remote-store-A.settings.bucket", + repoSettingsKey, "abc", - "remote_store.repository.remote-store-A.settings.base_path", + repoSettingsKey + "base_path", "xyz" ); DiscoveryNode node = new DiscoveryNode( diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index 0221661e6c03b..a820c17d188d6 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -39,6 +39,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.index.shard.ShardId; import org.opensearch.env.Environment; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.index.IndexSettings; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.indices.replication.common.ReplicationType; @@ -60,6 +61,7 @@ import java.util.stream.Collectors; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; @@ -107,6 +109,10 @@ private Settings buildRemoteStoreNodeAttributes(String repoName, Path repoPath) .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) .put(repoTypeAttributeKey, FsRepository.TYPE) .put(repoSettingsAttributeKeyPrefix + "location", repoPath) + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) + .put(repoTypeAttributeKey, FsRepository.TYPE) + .put(repoSettingsAttributeKeyPrefix + "location", repoPath) + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) .build(); } diff --git a/test/framework/src/main/java/org/opensearch/gateway/MockGatewayMetaState.java b/test/framework/src/main/java/org/opensearch/gateway/MockGatewayMetaState.java index 14d646cf84a0a..bf59e9ff06cbe 100644 --- a/test/framework/src/main/java/org/opensearch/gateway/MockGatewayMetaState.java +++ b/test/framework/src/main/java/org/opensearch/gateway/MockGatewayMetaState.java @@ -55,6 +55,7 @@ import java.util.Collections; import java.util.function.Supplier; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -108,7 +109,7 @@ public void start( throw new AssertionError(e); } Supplier remoteClusterStateServiceSupplier = () -> { - if (RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) == true) { + if (isRemoteStoreClusterStateEnabled(settings)) { return new RemoteClusterStateService( nodeEnvironment.nodeId(), () -> new RepositoriesService(