[8.15] Emit deprecation warning when executing one of the rollup APIs (…

…#113042)

* [8.15] Emit deprecation warning when executing one of the rollup APIs

Backporting #112807 to 8.15 branch.
Currently none of the rollup APIs deprecation warning. Only the reference docs indicate that rollup has been deprecated.

Relates to #112690

* Fixed compile errors after cherry-picking from main branch.

* fix issue after merging

* added some more allowed warnings

docs/build.gradle

@@ -852,6 +852,9 @@ buildRestTests.setups['library'] = '''
 
 '''
 buildRestTests.setups['sensor_rollup_job'] = '''
+  - requires:
+      test_runner_features: [ "allowed_warnings" ]
+
   - do:
       indices.create:
         index: dummy-rollup-index
@@ -882,9 +885,10 @@ buildRestTests.setups['sensor_rollup_job'] = '''
               node:
                 type: keyword
   - do:
-      raw:
-        method: PUT
-        path: _rollup/job/sensor
+      allowed_warnings:
+        - "The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information."
+      rollup.put_job:
+        id: sensor
         body:  >
             {
                 "index_pattern": "sensor-*",
@@ -914,6 +918,9 @@ buildRestTests.setups['sensor_rollup_job'] = '''
             }
 '''
 buildRestTests.setups['sensor_started_rollup_job'] = '''
+  - requires:
+      test_runner_features: [ "allowed_warnings" ]
+
   - do:
       indices.create:
         index: dummy-rollup-index
@@ -963,9 +970,10 @@ buildRestTests.setups['sensor_started_rollup_job'] = '''
           {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"}
 
   - do:
-      raw:
-        method: PUT
-        path: _rollup/job/sensor
+      allowed_warnings:
+        - "The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information."
+      rollup.put_job:
+        id: sensor
         body:  >
             {
                 "index_pattern": "sensor-*",
@@ -994,9 +1002,10 @@ buildRestTests.setups['sensor_started_rollup_job'] = '''
                 ]
             }
   - do:
-      raw:
-        method: POST
-        path: _rollup/job/sensor/_start
+      allowed_warnings:
+        - "The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information."
+      rollup.start_job:
+        id: sensor
 '''
 
 buildRestTests.setups['sensor_index'] = '''

docs/reference/rollup/apis/delete-job.asciidoc

@@ -80,6 +80,7 @@ If we have a rollup job named `sensor`, it can be deleted with:
 DELETE _rollup/job/sensor
 --------------------------------------------------
 // TEST[setup:sensor_rollup_job]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 Which will return the response:
 

docs/reference/rollup/apis/get-job.asciidoc

@@ -89,6 +89,7 @@ job can be retrieved with:
 GET _rollup/job/sensor
 --------------------------------------------------
 // TEST[setup:sensor_rollup_job]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 The API yields the following response:
 
@@ -192,6 +193,7 @@ PUT _rollup/job/sensor2 <1>
 GET _rollup/job/_all <2>
 --------------------------------------------------
 // TEST[setup:sensor_rollup_job]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 <1> We create a second job with name `sensor2`
 <2> Then request all jobs by using `_all` in the GetJobs API
 

docs/reference/rollup/apis/put-job.asciidoc

@@ -281,6 +281,7 @@ PUT _rollup/job/sensor
 }
 --------------------------------------------------
 // TEST[setup:sensor_index]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 <1> This configuration enables date histograms to be used on the `timestamp`
 field and `terms` aggregations to be used on the `node` field.
 <2> This configuration defines metrics over two fields: `temperature` and

docs/reference/rollup/apis/rollup-caps.asciidoc

@@ -83,6 +83,7 @@ PUT _rollup/job/sensor
 }
 --------------------------------------------------
 // TEST[setup:sensor_index]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 We can then retrieve the rollup capabilities of that index pattern (`sensor-*`)
 via the following command:
@@ -92,6 +93,7 @@ via the following command:
 GET _rollup/data/sensor-*
 --------------------------------------------------
 // TEST[continued]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 Which will yield the following response:
 
@@ -164,6 +166,7 @@ We could also retrieve the same information with a request to `_all`:
 GET _rollup/data/_all
 --------------------------------------------------
 // TEST[continued]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 But note that if we use the concrete index name (`sensor-1`), we'll retrieve no
 rollup capabilities:
@@ -173,6 +176,7 @@ rollup capabilities:
 GET _rollup/data/sensor-1
 --------------------------------------------------
 // TEST[continued]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 [source,console-result]
 ----

docs/reference/rollup/apis/rollup-index-caps.asciidoc

@@ -80,6 +80,7 @@ PUT _rollup/job/sensor
 }
 --------------------------------------------------
 // TEST[setup:sensor_index]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 If at a later date, we'd like to determine what jobs and capabilities were
 stored in the `sensor_rollup` index, we can use the get rollup index API:
@@ -89,6 +90,7 @@ stored in the `sensor_rollup` index, we can use the get rollup index API:
 GET /sensor_rollup/_rollup/data
 --------------------------------------------------
 // TEST[continued]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 Note how we are requesting the concrete rollup index name (`sensor_rollup`) as
 the first part of the URL. This will yield the following response:
@@ -164,3 +166,4 @@ instead of explicit indices:
 GET /*_rollup/_rollup/data
 --------------------------------------------------
 // TEST[continued]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]

docs/reference/rollup/apis/rollup-search.asciidoc

@@ -105,6 +105,7 @@ PUT _rollup/job/sensor
 }
 --------------------------------------------------
 // TEST[setup:sensor_index]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 This rolls up the `sensor-*` pattern and stores the results in `sensor_rollup`.
 To search this rolled up data, we need to use the `_rollup_search` endpoint.
@@ -127,6 +128,7 @@ GET /sensor_rollup/_rollup_search
 --------------------------------------------------
 // TEST[setup:sensor_prefab_data]
 // TEST[s/_rollup_search/_rollup_search?filter_path=took,timed_out,terminated_early,_shards,hits,aggregations/]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 The query is targeting the `sensor_rollup` data, since this contains the rollup
 data as configured in the job. A `max` aggregation has been used on the
@@ -182,6 +184,7 @@ GET sensor_rollup/_rollup_search
 --------------------------------------------------
 // TEST[continued]
 // TEST[catch:/illegal_argument_exception/]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 [source,console-result]
 ----
@@ -225,6 +228,7 @@ GET sensor-1,sensor_rollup/_rollup_search <1>
 --------------------------------------------------
 // TEST[continued]
 // TEST[s/_rollup_search/_rollup_search?filter_path=took,timed_out,terminated_early,_shards,hits,aggregations/]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 <1> Note the URI now searches `sensor-1` and `sensor_rollup` at the same time
 
 When the search is executed, the rollup search endpoint does two things:

docs/reference/rollup/apis/start-job.asciidoc

@@ -51,6 +51,7 @@ If we have already created a {rollup-job} named `sensor`, it can be started with
 POST _rollup/job/sensor/_start
 --------------------------------------------------
 // TEST[setup:sensor_rollup_job]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 Which will return the response:
 
@@ -59,4 +60,4 @@ Which will return the response:
 {
   "started": true
 }
-----
+----

docs/reference/rollup/apis/stop-job.asciidoc

@@ -76,6 +76,7 @@ the indexer has fully stopped. This is accomplished with the
 POST _rollup/job/sensor/_stop?wait_for_completion=true&timeout=10s
 --------------------------------------------------
 // TEST[setup:sensor_started_rollup_job]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 The parameter blocks the API call from returning until either the job has moved
 to `STOPPED` or the specified time has elapsed. If the specified time elapses

docs/reference/rollup/migrating-to-downsampling.asciidoc

@@ -51,6 +51,7 @@ PUT _rollup/job/sensor
 }
 --------------------------------------------------
 // TEST[setup:sensor_index]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 The equivalent <<tsds,time series data stream (TSDS)>> setup that uses downsampling via DSL:
 

docs/reference/rollup/rollup-getting-started.asciidoc

@@ -64,6 +64,7 @@ PUT _rollup/job/sensor
 }
 --------------------------------------------------
 // TEST[setup:sensor_index]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 We give the job the ID of "sensor" (in the url: `PUT _rollup/job/sensor`), and tell it to rollup the index pattern `"sensor-*"`.
 This job will find and rollup any index that matches that pattern. Rollup summaries are then stored in the `"sensor_rollup"` index.
@@ -143,6 +144,7 @@ To start the job, execute this command:
 POST _rollup/job/sensor/_start
 --------------------------------------------------
 // TEST[setup:sensor_rollup_job]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 [discrete]
 ==== Searching the rolled results
@@ -167,6 +169,7 @@ GET /sensor_rollup/_rollup_search
 }
 --------------------------------------------------
 // TEST[setup:sensor_prefab_data]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 It's a simple aggregation that calculates the maximum of the `temperature` field. But you'll notice that it is being sent to the `sensor_rollup`
 index instead of the raw `sensor-*` indices. And you'll also notice that it is using the `_rollup_search` endpoint. Otherwise the syntax
@@ -198,6 +201,7 @@ If you were to execute that query, you'd receive a result that looks like a norm
 ----
 // TESTRESPONSE[s/"took" : 102/"took" : $body.$_path/]
 // TESTRESPONSE[s/"_shards" : \.\.\. /"_shards" : $body.$_path/]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 The only notable difference is that Rollup search results have zero `hits`, because we aren't really searching the original, live data any
 more. Otherwise it's identical syntax.
@@ -244,6 +248,7 @@ GET /sensor_rollup/_rollup_search
 }
 --------------------------------------------------
 // TEST[setup:sensor_prefab_data]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 Which returns a corresponding response:
 

docs/reference/rollup/rollup-search-limitations.asciidoc

@@ -56,6 +56,7 @@ GET sensor_rollup/_rollup_search
 --------------------------------------------------
 // TEST[setup:sensor_prefab_data]
 // TEST[catch:/illegal_argument_exception/]
+// TEST[warning:The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.]
 
 The response will tell you that the field and aggregation were not possible, because no rollup jobs were found which contained them:
 

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -157,6 +157,20 @@ public abstract class ESRestTestCase extends ESTestCase {
 
     private static final Pattern SEMANTIC_VERSION_PATTERN = Pattern.compile("^(\\d+\\.\\d+\\.\\d+)\\D?.*");
 
+    private static final String EXPECTED_ROLLUP_WARNING_MESSAGE =
+        "The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.";
+    public static final RequestOptions.Builder ROLLUP_REQUESTS_OPTIONS = RequestOptions.DEFAULT.toBuilder().setWarningsHandler(warnings -> {
+        // Either no warning, because of bwc integration test OR
+        // the expected warning, because on current version
+        if (warnings.isEmpty()) {
+            return false;
+        } else if (warnings.size() == 1 && EXPECTED_ROLLUP_WARNING_MESSAGE.equals(warnings.get(0))) {
+            return false;
+        } else {
+            return true;
+        }
+    });
+
     /**
      * Convert the entity from a {@link Response} into a map of maps.
      * Consumes the underlying HttpEntity, releasing any resources it may be holding.
@@ -1278,7 +1292,9 @@ private static void wipeClusterSettings() throws IOException {
     private void wipeRollupJobs() throws IOException {
         final Response response;
         try {
-            response = adminClient().performRequest(new Request("GET", "/_rollup/job/_all"));
+            var request = new Request("GET", "/_rollup/job/_all");
+            request.setOptions(ROLLUP_REQUESTS_OPTIONS);
+            response = adminClient().performRequest(request);
         } catch (ResponseException e) {
             // If we don't see the rollup endpoint (possibly because of running against an older ES version) we just bail
             if (e.getResponse().getStatusLine().getStatusCode() == RestStatus.NOT_FOUND.getStatus()) {
@@ -1298,6 +1314,7 @@ private void wipeRollupJobs() throws IOException {
             @SuppressWarnings("unchecked")
             String jobId = (String) ((Map<String, Object>) jobConfig.get("config")).get("id");
             Request request = new Request("POST", "/_rollup/job/" + jobId + "/_stop");
+            request.setOptions(ROLLUP_REQUESTS_OPTIONS);
             setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND);
             request.addParameter("wait_for_completion", "true");
             request.addParameter("timeout", "10s");
@@ -1309,6 +1326,7 @@ private void wipeRollupJobs() throws IOException {
             @SuppressWarnings("unchecked")
             String jobId = (String) ((Map<String, Object>) jobConfig.get("config")).get("id");
             Request request = new Request("DELETE", "/_rollup/job/" + jobId);
+            request.setOptions(ROLLUP_REQUESTS_OPTIONS);
             setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); // 404s imply someone was racing us to delete this
             logger.debug("deleting rollup job [{}]", jobId);
             adminClient().performRequest(request);

x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java

@@ -1272,6 +1272,7 @@ public void testLookbackOnlyGivenAggregationsWithHistogramAndRollupIndex() throw
 
         String rollupJobId = "rollup-" + jobId;
         Request createRollupRequest = new Request("PUT", "/_rollup/job/" + rollupJobId);
+        createRollupRequest.setOptions(ROLLUP_REQUESTS_OPTIONS);
         createRollupRequest.setJsonEntity("""
             {
             "index_pattern": "airline-data-aggs",
@@ -1299,18 +1300,26 @@ public void testLookbackOnlyGivenAggregationsWithHistogramAndRollupIndex() throw
                 ]
             }""");
         client().performRequest(createRollupRequest);
-        client().performRequest(new Request("POST", "/_rollup/job/" + rollupJobId + "/_start"));
+        var startRolupRequest = new Request("POST", "/_rollup/job/" + rollupJobId + "/_start");
+        startRolupRequest.setOptions(ROLLUP_REQUESTS_OPTIONS);
+        client().performRequest(startRolupRequest);
 
         assertBusy(() -> {
-            Response getRollup = client().performRequest(new Request("GET", "/_rollup/job/" + rollupJobId));
+            var getRollupRequest = new Request("GET", "/_rollup/job/" + rollupJobId);
+            getRollupRequest.setOptions(ROLLUP_REQUESTS_OPTIONS);
+            Response getRollup = client().performRequest(getRollupRequest);
             String body = EntityUtils.toString(getRollup.getEntity());
             assertThat(body, containsString("\"job_state\":\"started\""));
             assertThat(body, containsString("\"rollups_indexed\":4"));
         }, 60, TimeUnit.SECONDS);
 
-        client().performRequest(new Request("POST", "/_rollup/job/" + rollupJobId + "/_stop"));
+        var stopRollupRequest = new Request("POST", "/_rollup/job/" + rollupJobId + "/_stop");
+        stopRollupRequest.setOptions(ROLLUP_REQUESTS_OPTIONS);
+        client().performRequest(stopRollupRequest);
         assertBusy(() -> {
-            Response getRollup = client().performRequest(new Request("GET", "/_rollup/job/" + rollupJobId));
+            var getRollupRequest = new Request("GET", "/_rollup/job/" + rollupJobId);
+            getRollupRequest.setOptions(ROLLUP_REQUESTS_OPTIONS);
+            Response getRollup = client().performRequest(getRollupRequest);
             assertThat(EntityUtils.toString(getRollup.getEntity()), containsString("\"job_state\":\"stopped\""));
         }, 60, TimeUnit.SECONDS);
 
@@ -1827,6 +1836,7 @@ private Response createJobAndDataFeed(String jobId, String datafeedId) throws IO
 
         String rollupJobId = "rollup-" + jobId;
         Request createRollupRequest = new Request("PUT", "/_rollup/job/" + rollupJobId);
+        createRollupRequest.setOptions(ROLLUP_REQUESTS_OPTIONS);
         createRollupRequest.setJsonEntity("""
             {
             "index_pattern": "airline-data-aggs",

x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java

@@ -70,6 +70,10 @@
 
 public class Rollup extends Plugin implements ActionPlugin, PersistentTaskPlugin {
 
+    public static final String DEPRECATION_MESSAGE =
+        "The rollup functionality will be removed in in Elasticsearch 9.0. See docs for more information.";
+    public static final String DEPRECATION_KEY = "rollup_removal";
+
     // Introduced in ES version 6.3
     public static final int ROLLUP_VERSION_V1 = 1;
     // Introduced in ES Version 6.4