Merge branch 'main' into move-ext-rest-request

CHANGELOG.md

@@ -92,6 +92,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Adds ExtensionsManager.lookupExtensionSettingsById ([#7466](https://github.com/opensearch-project/OpenSearch/pull/7466))
 - SegRep with Remote: Add hook for publishing checkpoint notifications after segment upload to remote store ([#7394](https://github.com/opensearch-project/OpenSearch/pull/7394))
 - Provide mechanism to configure XContent parsing constraints (after update to Jackson 2.15.0 and above) ([#7550](https://github.com/opensearch-project/OpenSearch/pull/7550))
+- Support to clear filecache using clear indices cache API ([#7498](https://github.com/opensearch-project/OpenSearch/pull/7498))
 - Create NamedRoute to map extension routes to a shortened name ([#6870](https://github.com/opensearch-project/OpenSearch/pull/6870))
 
 ### Dependencies
@@ -115,17 +116,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.google.guava:guava` from 30.1.1-jre to 31.1-jre (#7565)
 - Bump `com.azure:azure-storage-common` from 12.20.0 to 12.21.0 (#7566)
 - Bump `org.apache.commons:commons-compress` from 1.22 to 1.23.0 (#7563)
+- Bump `jackson` from 2.15.0 to 2.15.1 ([#7603](https://github.com/opensearch-project/OpenSearch/pull/7603))
 
 ### Changed
 - Enable `./gradlew build` on MacOS by disabling bcw tests ([#7303](https://github.com/opensearch-project/OpenSearch/pull/7303))
 - Moved concurrent-search from sandbox plugin to server module behind feature flag ([#7203](https://github.com/opensearch-project/OpenSearch/pull/7203))
 - Allow access to indices cache clear APIs for read only indexes ([#7303](https://github.com/opensearch-project/OpenSearch/pull/7303))
+- Changed concurrent-search threadpool type to be resizable and support task resource tracking ([#7502](https://github.com/opensearch-project/OpenSearch/pull/7502))
 
 ### Deprecated
 
 ### Removed
 
 ### Fixed
+- Add more index blocks check for resize APIs ([#6774](https://github.com/opensearch-project/OpenSearch/pull/6774))
 - Replaces ZipInputStream with ZipFile to fix Zip Slip vulnerability ([#7230](https://github.com/opensearch-project/OpenSearch/pull/7230))
 - Add missing validation/parsing of SearchBackpressureMode of SearchBackpressureSettings ([#7541](https://github.com/opensearch-project/OpenSearch/pull/7541))
 

buildSrc/version.properties

@@ -8,8 +8,8 @@ bundled_jdk = 19.0.2+7
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.15.0
-jackson_databind  = 2.15.0
+jackson           = 2.15.1
+jackson_databind  = 2.15.1
 snakeyaml         = 2.0
 icu4j             = 70.1
 supercsv          = 2.4.0

client/rest-high-level/src/main/java/org/opensearch/client/IndicesRequestConverters.java

@@ -261,6 +261,7 @@ static Request clearCache(ClearIndicesCacheRequest clearIndicesCacheRequest) {
         parameters.withIndicesOptions(clearIndicesCacheRequest.indicesOptions());
         parameters.putParam("query", Boolean.toString(clearIndicesCacheRequest.queryCache()));
         parameters.putParam("fielddata", Boolean.toString(clearIndicesCacheRequest.fieldDataCache()));
+        parameters.putParam("file", Boolean.toString(clearIndicesCacheRequest.fileCache()));
         parameters.putParam("request", Boolean.toString(clearIndicesCacheRequest.requestCache()));
         parameters.putParam("fields", String.join(",", clearIndicesCacheRequest.fields()));
         request.addParameters(parameters.asMap());

client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java

@@ -596,6 +596,10 @@ public void testClearCache() {
             clearIndicesCacheRequest.fields(RequestConvertersTests.randomIndicesNames(1, 5));
             expectedParams.put("fields", String.join(",", clearIndicesCacheRequest.fields()));
         }
+        if (OpenSearchTestCase.randomBoolean()) {
+            clearIndicesCacheRequest.fileCache(OpenSearchTestCase.randomBoolean());
+        }
+        expectedParams.put("file", Boolean.toString(clearIndicesCacheRequest.fileCache()));
 
         Request request = IndicesRequestConverters.clearCache(clearIndicesCacheRequest);
         StringJoiner endpoint = new StringJoiner("/", "/", "");

client/rest-high-level/src/test/java/org/opensearch/client/RestHighLevelClientTests.java

@@ -875,6 +875,7 @@ public void testApiNamingConventions() throws Exception {
             "nodes.reload_secure_settings",
             "search_shards",
             "remote_store.restore",
+            "remote_store.stats",
             "cluster.put_weighted_routing",
             "cluster.get_weighted_routing",
             "cluster.delete_weighted_routing",

client/sniffer/licenses/jackson-core-2.15.1.jar.sha1

@@ -0,0 +1 @@
+241c054ba8503de092a12acad9f083dd39935cc0

codecov.yml

@@ -1,6 +1,9 @@
 codecov:
   require_ci_to_pass: yes
 
+ignore:
+  - "test"
+
 coverage:
   precision: 2
   round: down

distribution/tools/upgrade-cli/licenses/jackson-annotations-2.15.1.jar.sha1

@@ -0,0 +1 @@
+092a90d3739e970e03b5971839e4fe51f13c1fa3

distribution/tools/upgrade-cli/licenses/jackson-databind-2.15.1.jar.sha1

@@ -0,0 +1 @@
+ac9ba74d208faf356e4719a49e59c6ea9237c01d

libs/core/licenses/jackson-core-2.15.1.jar.sha1

@@ -0,0 +1 @@
+241c054ba8503de092a12acad9f083dd39935cc0

libs/x-content/licenses/jackson-core-2.15.1.jar.sha1

@@ -0,0 +1 @@
+241c054ba8503de092a12acad9f083dd39935cc0

libs/x-content/licenses/jackson-dataformat-cbor-2.15.1.jar.sha1

@@ -0,0 +1 @@
+c1c88d6050bbfbfb040e1036e1cfb8f82e00df7b

libs/x-content/licenses/jackson-dataformat-smile-2.15.1.jar.sha1

@@ -0,0 +1 @@
+46d5f70e4ebb2e4c329701c45226a78978601e04

libs/x-content/licenses/jackson-dataformat-yaml-2.15.1.jar.sha1

@@ -0,0 +1 @@
+97a9c5867fbacd44a1791c2d0a00bd4f3f061b77

modules/ingest-geoip/licenses/jackson-annotations-2.15.1.jar.sha1

@@ -0,0 +1 @@
+092a90d3739e970e03b5971839e4fe51f13c1fa3

modules/ingest-geoip/licenses/jackson-databind-2.15.1.jar.sha1

@@ -0,0 +1 @@
+ac9ba74d208faf356e4719a49e59c6ea9237c01d

plugins/discovery-ec2/licenses/jackson-annotations-2.15.1.jar.sha1

@@ -0,0 +1 @@
+092a90d3739e970e03b5971839e4fe51f13c1fa3

plugins/discovery-ec2/licenses/jackson-databind-2.15.1.jar.sha1

@@ -0,0 +1 @@
+ac9ba74d208faf356e4719a49e59c6ea9237c01d

plugins/repository-azure/licenses/jackson-annotations-2.15.1.jar.sha1

@@ -0,0 +1 @@
+092a90d3739e970e03b5971839e4fe51f13c1fa3

plugins/repository-azure/licenses/jackson-databind-2.15.1.jar.sha1

@@ -0,0 +1 @@
+ac9ba74d208faf356e4719a49e59c6ea9237c01d

plugins/repository-azure/licenses/jackson-dataformat-xml-2.15.1.jar.sha1

@@ -0,0 +1 @@
+66e5b7284865d1175dd63d0ffc0385f8810e0810

plugins/repository-azure/licenses/jackson-datatype-jsr310-2.15.1.jar.sha1

@@ -0,0 +1 @@
+21300ba63a6408bbd3505af4287d1b9a911a9718

plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.15.1.jar.sha1

@@ -0,0 +1 @@
+ab2078805370360d32c77f1ba46dc9098f84daa6

plugins/repository-s3/licenses/jackson-annotations-2.15.1.jar.sha1

@@ -0,0 +1 @@
+092a90d3739e970e03b5971839e4fe51f13c1fa3

plugins/repository-s3/licenses/jackson-databind-2.15.1.jar.sha1

@@ -0,0 +1 @@
+ac9ba74d208faf356e4719a49e59c6ea9237c01d

rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json

@@ -67,6 +67,10 @@
       "request":{
         "type":"boolean",
         "description":"Clear request cache"
+      },
+      "file":{
+        "type":"boolean",
+        "description":"Clear filecache"
       }
     }
   }

rest-api-spec/src/main/resources/rest-api-spec/api/remote_store.stats.json

@@ -0,0 +1,48 @@
+{
+  "remote_store.stats":{
+    "documentation":{
+      "url": "https://opensearch.org/docs/latest/tuning-your-cluster/availability-and-recovery/remote",
+      "description":"Stats for remote store."
+    },
+    "stability":"experimental",
+    "url":{
+      "paths":[
+        {
+          "path":"/_remotestore/stats/{index}",
+          "methods":[
+            "GET"
+          ],
+          "parts":{
+            "index":{
+              "type":"string",
+              "description": "Index name to fetch stats"
+            }
+          }
+        },
+        {
+          "path":"/_remotestore/stats/{index}/{shard_id}",
+          "methods":[
+            "GET"
+          ],
+          "parts":{
+            "index":{
+              "type":"string",
+              "description":"Index name to fetch stats"
+            },
+            "shard_id":{
+              "type":"string",
+              "description":"Specific shard to fetch stats"
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+      "timeout":{
+        "type":"time",
+        "default":"10s",
+        "description":"Max time each individual stats request should take."
+      }
+    }
+  }
+}

rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/10_basic.yml

@@ -112,3 +112,99 @@ setup:
           settings:
             index.number_of_replicas: 0
             index.number_of_shards: 6
+
+---
+"Returns error if target index's metadata write is blocked":
+
+  - skip:
+      version: " - 2.9.99"
+      reason: "only available in 3.0+"
+
+  # block source index's write operations
+  - do:
+      indices.put_settings:
+        index: source
+        body:
+          index.blocks.write: true
+          index.number_of_replicas: 0
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+        index: source
+
+  # set `index.blocks.read_only` to `true` for target index
+  - do:
+      catch: /action_request_validation_exception/
+      indices.clone:
+        index: "source"
+        target: "new_cloned_index"
+        wait_for_active_shards: 1
+        cluster_manager_timeout: 10s
+        body:
+          settings:
+            index.number_of_replicas: 0
+            index.blocks.read_only: true
+
+  # set `index.blocks.metadata` to `true` for target index
+  - do:
+      catch: /action_request_validation_exception/
+      indices.clone:
+        index: "source"
+        target: "new_cloned_index"
+        wait_for_active_shards: 1
+        cluster_manager_timeout: 10s
+        body:
+          settings:
+            index.number_of_replicas: 0
+            index.blocks.metadata: true
+
+  # set source index's setting `index.blocks.read_only` to `true`
+  - do:
+      indices.put_settings:
+        index: source
+        body:
+          index.blocks.read_only: true
+
+  - do:
+      catch: /illegal_argument_exception/
+      indices.clone:
+        index: "source"
+        target: "new_cloned_index"
+        wait_for_active_shards: 1
+        cluster_manager_timeout: 10s
+        body:
+          settings:
+            index.number_of_replicas: 0
+
+  # overwrite the source index's setting, everything is fine
+  - do:
+      indices.clone:
+        index: "source"
+        target: "new_cloned_index"
+        wait_for_active_shards: 1
+        cluster_manager_timeout: 10s
+        body:
+          settings:
+            index.number_of_replicas: 0
+            index.blocks.read_only: null
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+
+  - do:
+      get:
+        index: new_cloned_index
+        id:    "1"
+
+  - match: { _index:   new_cloned_index }
+  - match: { _id:      "1"     }
+  - match: { _source:  { foo: "hello world" } }
+
+  # clear the source index's read_only blocks because it will block deleting index
+  - do:
+      indices.put_settings:
+        index: source
+        body:
+          index.blocks.read_only: null