Merge remote-tracking branch 'upstream/main' into remote-state-stats

CHANGELOG.md

@@ -95,6 +95,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add search query categorizer ([#10255](https://github.com/opensearch-project/OpenSearch/pull/10255))
 - Introduce ConcurrentQueryProfiler to profile query using concurrent segment search path and support concurrency during rewrite and create weight ([10352](https://github.com/opensearch-project/OpenSearch/pull/10352))
 - [Remote cluster state] Make index and global metadata upload timeout dynamic cluster settings ([#10814](https://github.com/opensearch-project/OpenSearch/pull/10814))
+- Added cluster setting cluster.restrict.index.replication_type to restrict setting of index setting replication type ([#10866](https://github.com/opensearch-project/OpenSearch/pull/10866))
 - Add cluster state stats ([#10670](https://github.com/opensearch-project/OpenSearch/pull/10670))
 
 ### Dependencies
@@ -108,6 +109,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.codehaus.woodstox:stax2-api` from 4.2.1 to 4.2.2 ([#10639](https://github.com/opensearch-project/OpenSearch/pull/10639))
 - Bump `com.google.http-client:google-http-client` from 1.43.2 to 1.43.3 ([#10635](https://github.com/opensearch-project/OpenSearch/pull/10635))
 - Bump `com.squareup.okio:okio` from 3.5.0 to 3.6.0 ([#10637](https://github.com/opensearch-project/OpenSearch/pull/10637))
+- Bump `org.apache.logging.log4j:log4j-core` from 2.20.0 to 2.21.0 ([#10858](https://github.com/opensearch-project/OpenSearch/pull/10858))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))

buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle

@@ -15,8 +15,9 @@ plugins {
 repositories {
   mavenCentral()
 }
+
 dependencies {
-  implementation "org.apache.logging.log4j:log4j-core:2.20.0"
+  implementation "org.apache.logging.log4j:log4j-core:2.21.0"
 }
 
 ["0.0.1", "0.0.2"].forEach { v ->

buildSrc/version.properties

@@ -14,7 +14,7 @@ jackson_databind  = 2.15.2
 snakeyaml         = 2.1
 icu4j             = 70.1
 supercsv          = 2.4.0
-log4j             = 2.20.0
+log4j             = 2.21.0
 slf4j             = 1.7.36
 asm               = 9.6
 jettison          = 1.5.4

libs/core/licenses/log4j-api-2.21.0.jar.sha1

@@ -0,0 +1 @@
+760192f2b69eacf4a4afc78e5a1d7a8de054fcbd

plugins/crypto-kms/licenses/log4j-1.2-api-2.21.0.jar.sha1

@@ -0,0 +1 @@
+12bad3819a9570807f3c97315930699584c12152

plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.21.0.jar.sha1

@@ -0,0 +1 @@
+12bad3819a9570807f3c97315930699584c12152

plugins/discovery-ec2/licenses/log4j-1.2-api-2.21.0.jar.sha1

@@ -0,0 +1 @@
+12bad3819a9570807f3c97315930699584c12152

plugins/discovery-gce/licenses/log4j-1.2-api-2.21.0.jar.sha1

@@ -0,0 +1 @@
+12bad3819a9570807f3c97315930699584c12152

plugins/repository-gcs/licenses/log4j-1.2-api-2.21.0.jar.sha1

@@ -0,0 +1 @@
+12bad3819a9570807f3c97315930699584c12152

plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.21.0.jar.sha1

@@ -0,0 +1 @@
+911fdb5b1a1df36719c579ecc6f2957b88bce1ab

plugins/repository-s3/licenses/log4j-1.2-api-2.21.0.jar.sha1

@@ -0,0 +1 @@
+12bad3819a9570807f3c97315930699584c12152

qa/os/build.gradle

@@ -70,6 +70,11 @@ tasks.dependenciesInfo.enabled = false
 
 tasks.thirdPartyAudit.ignoreMissingClasses()
 
+tasks.thirdPartyAudit.ignoreViolations(
+  'org.apache.logging.log4j.core.util.internal.UnsafeUtil',
+  'org.apache.logging.log4j.core.util.internal.UnsafeUtil$1'
+)
+
 tasks.register('destructivePackagingTest') {
   dependsOn 'destructiveDistroTest'
 }

server/build.gradle

@@ -364,7 +364,9 @@ tasks.named("thirdPartyAudit").configure {
             'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
             'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
             'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
-            'com.google.protobuf.UnsafeUtil$MemoryAccessor'
+            'com.google.protobuf.UnsafeUtil$MemoryAccessor',
+            'org.apache.logging.log4j.core.util.internal.UnsafeUtil',
+            'org.apache.logging.log4j.core.util.internal.UnsafeUtil$1'
     )
 }
 

server/licenses/log4j-api-2.21.0.jar.sha1

@@ -0,0 +1 @@
+760192f2b69eacf4a4afc78e5a1d7a8de054fcbd

server/licenses/log4j-core-2.21.0.jar.sha1

@@ -0,0 +1 @@
+122e1a9e0603cc9eae07b0846a6ff01f2454bc49

server/licenses/log4j-jul-2.21.0.jar.sha1

@@ -0,0 +1 @@
+f0da61113f4a47654677e6a98b1e13ca7de2483d

server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java

@@ -19,6 +19,7 @@
 import org.opensearch.test.OpenSearchIntegTestCase;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
+import static org.opensearch.indices.IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING;
 import static org.opensearch.indices.IndicesService.CLUSTER_SETTING_REPLICATION_TYPE;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
@@ -123,4 +124,30 @@ public void testIndexReplicationSettingOverridesDocRepClusterSetting() throws Ex
         assertEquals(indicesService.indexService(anotherIndex).getIndexSettings().isSegRepEnabled(), false);
     }
 
+    public void testIndexReplicationTypeWhenRestrictSettingTrue() {
+        testRestrictIndexReplicationTypeSetting(true, randomFrom(ReplicationType.values()));
+    }
+
+    public void testIndexReplicationTypeWhenRestrictSettingFalse() {
+        testRestrictIndexReplicationTypeSetting(false, randomFrom(ReplicationType.values()));
+    }
+
+    private void testRestrictIndexReplicationTypeSetting(boolean setRestrict, ReplicationType replicationType) {
+        String expectedExceptionMsg =
+            "Validation Failed: 1: index setting [index.replication.type] is not allowed to be set as [cluster.restrict.index.replication_type=true];";
+        String clusterManagerName = internalCluster().startNode(
+            Settings.builder().put(CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING.getKey(), setRestrict).build()
+        );
+        internalCluster().startDataOnlyNodes(1);
+
+        // Test create index fails
+        Settings indexSettings = Settings.builder().put(indexSettings()).put(SETTING_REPLICATION_TYPE, replicationType).build();
+        if (setRestrict) {
+            IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> createIndex(INDEX_NAME, indexSettings));
+            assertEquals(expectedExceptionMsg, exception.getMessage());
+        } else {
+            createIndex(INDEX_NAME, indexSettings);
+        }
+    }
+
 }

server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java

@@ -10,39 +10,33 @@
 
 import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
 import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreResponse;
-import org.opensearch.action.admin.indices.get.GetIndexRequest;
-import org.opensearch.action.admin.indices.get.GetIndexResponse;
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.cluster.health.ClusterHealthStatus;
-import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.repositories.Repository;
-import org.opensearch.test.CorruptionUtils;
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
 import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.greaterThan;
 
-@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 0)
 public class RemoteStoreRestoreIT extends BaseRemoteStoreRestoreIT {
 
     /**
@@ -467,30 +461,5 @@ public void testRateLimitedRemoteDownloads() throws Exception {
         }
     }
 
-    public void testRestoreCorruptSegmentShouldFail() throws IOException, ExecutionException, InterruptedException {
-        prepareCluster(1, 3, INDEX_NAME, 0, 1);
-        indexData(randomIntBetween(3, 4), true, INDEX_NAME);
-
-        GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest()).get();
-        String indexUUID = getIndexResponse.getSettings().get(INDEX_NAME).get(IndexMetadata.SETTING_INDEX_UUID);
-
-        logger.info("--> Corrupting segment files in remote segment store");
-        Path path = segmentRepoPath.resolve(indexUUID).resolve("0").resolve("segments").resolve("data");
-        try (Stream<Path> dataPath = Files.list(path)) {
-            CorruptionUtils.corruptFile(random(), dataPath.toArray(Path[]::new));
-        }
-
-        logger.info("--> Stop primary");
-        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME)));
-
-        logger.info("--> Close and restore the index");
-        client().admin()
-            .cluster()
-            .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME).waitForCompletion(true), PlainActionFuture.newFuture());
-
-        logger.info("--> Check for index status, should be red due to corruption");
-        ensureRed(INDEX_NAME);
-    }
-
     // TODO: Restore flow - index aliases
 }

server/src/main/java/org/opensearch/action/search/SearchRequest.java

@@ -256,7 +256,7 @@ public SearchRequest(StreamInput in) throws IOException {
         if (in.getVersion().onOrAfter(Version.V_2_7_0)) {
             pipeline = in.readOptionalString();
         }
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             phaseTook = in.readOptionalBoolean();
         }
     }
@@ -290,7 +290,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_7_0)) {
             out.writeOptionalString(pipeline);
         }
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeOptionalBoolean(phaseTook);
         }
     }

server/src/main/java/org/opensearch/action/search/SearchResponse.java

@@ -116,7 +116,7 @@ public SearchResponse(StreamInput in) throws IOException {
         clusters = new Clusters(in);
         scrollId = in.readOptionalString();
         tookInMillis = in.readVLong();
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             phaseTook = in.readOptionalWriteable(PhaseTook::new);
         } else {
             phaseTook = null;
@@ -557,7 +557,7 @@ public void writeTo(StreamOutput out) throws IOException {
         clusters.writeTo(out);
         out.writeOptionalString(scrollId);
         out.writeVLong(tookInMillis);
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeOptionalWriteable(phaseTook);
         }
         out.writeVInt(skippedShards);

server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

@@ -1252,6 +1252,7 @@ List<String> getIndexSettingsValidationErrors(
         if (forbidPrivateIndexSettings) {
             validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, indexScopedSettings));
         }
+        validateIndexReplicationTypeSettings(settings, clusterService.getClusterSettings()).ifPresent(validationErrors::add);
         if (indexName.isEmpty() || indexName.get().charAt(0) != '.') {
             // Apply aware replica balance validation only to non system indices
             int replicaCount = settings.getAsInt(
@@ -1306,6 +1307,24 @@ private static List<String> validateIndexCustomPath(Settings settings, @Nullable
         return validationErrors;
     }
 
+    /**
+     * Validates {@code index.replication.type} is not set if {@code cluster.restrict.index.replication_type} is set to true.
+     *
+     * @param requestSettings settings passed in during index create request
+     * @param clusterSettings cluster setting
+     */
+    private static Optional<String> validateIndexReplicationTypeSettings(Settings requestSettings, ClusterSettings clusterSettings) {
+        if (requestSettings.hasValue(SETTING_REPLICATION_TYPE)
+            && clusterSettings.get(IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING)) {
+            return Optional.of(
+                "index setting [index.replication.type] is not allowed to be set as ["
+                    + IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING.getKey()
+                    + "=true]"
+            );
+        }
+        return Optional.empty();
+    }
+
     /**
      * Validates the settings and mappings for shrinking an index.
      *

server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

@@ -690,7 +690,8 @@ public void apply(Settings value, Settings current, Settings previous) {
                 AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE,
                 CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE,
                 CPUBasedAdmissionControllerSettings.INDEXING_CPU_USAGE_LIMIT,
-                CPUBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT
+                CPUBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT,
+                IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING
             )
         )
     );

server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java

@@ -262,7 +262,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException {
         this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new));
         this.previousClusterUUID = in.readString();
         this.clusterUUIDCommitted = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             this.codecVersion = in.readInt();
             this.globalMetadataFileName = in.readString();
         } else {
@@ -316,7 +316,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeCollection(indices);
         out.writeString(previousClusterUUID);
         out.writeBoolean(clusterUUIDCommitted);
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeInt(codecVersion);
             out.writeString(globalMetadataFileName);
         }

server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java

@@ -290,10 +290,6 @@ public void setWrittenByMajor(int writtenByMajor) {
                 );
             }
         }
-
-        public int getWrittenByMajor() {
-            return writtenByMajor;
-        }
     }
 
     /**