Merge branch 'master' into long_sort_optimization

docs/build.gradle

@@ -1146,3 +1146,42 @@ buildRestTests.setups['kibana_sample_data_ecommerce'] = '''
               number_of_shards: 1
               number_of_replicas: 0
 '''
+buildRestTests.setups['setup_logdata'] = '''
+  - do:
+        indices.create:
+          index: logdata
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 1
+            mappings:
+              properties:
+                grade:
+                  type: byte
+  - do:
+        bulk:
+          index: logdata
+          refresh: true
+          body: |
+            {"index":{}}
+            {"grade": 100, "weight": 2}
+            {"index":{}}
+            {"grade": 50, "weight": 3}
+'''
+buildRestTests.setups['logdata_job'] = buildRestTests.setups['setup_logdata'] + '''
+  - do:
+      ml.put_data_frame_analytics:
+        id: "loganalytics"
+        body:  >
+          {
+            "source": { 
+              "index": "logdata"
+              },
+            "dest": {
+              "index": "logdata_out"
+              },
+            "analysis": {
+              "outlier_detection": {}
+            }
+          }
+'''

docs/reference/data-frames/apis/preview-transform.asciidoc

@@ -30,6 +30,15 @@ Previews a {dataframe-transform}.
 {stack-ov}/security-privileges.html[Security privileges] and
 {stack-ov}/built-in-roles.html[Built-in roles].
 
+[discrete]
+[[preview-data-frame-transform-desc]]
+==== {api-description-title}
+
+This API generates a preview of the results that you will get when you run the
+<<put-data-frame-transform,create {dataframe-transforms} API>> with the same
+configuration. It returns a maximum of 100 results. The calculations are based
+on all the current data in the source index. 
+
 [discrete]
 [[preview-data-frame-transform-request-body]]
 ==== {api-request-body-title}
@@ -40,6 +49,15 @@ Previews a {dataframe-transform}.
 `pivot` (Required)::
   (object) Defines the pivot function `group by` fields and the aggregation to
   reduce the data. See <<data-frame-transform-pivot>>.
+
+[discrete]
+[[preview-data-frame-transform-response]]
+==== {api-response-body-title}
+
+`preview`::
+  (array) An array of documents. In particular, they are the JSON
+  representation of the documents that would be created in the destination index
+  by the {dataframe-transform}. 
 
 [discrete]
 ==== {api-examples-title}

docs/reference/ml/apis/delete-dfanalytics.asciidoc

@@ -0,0 +1,52 @@
+[role="xpack"]
+[testenv="platinum"]
+[[delete-dfanalytics]]
+=== Delete {dfanalytics-jobs} API
+[subs="attributes"]
+++++
+<titleabbrev>Delete {dfanalytics-jobs}</titleabbrev>
+++++
+
+experimental[]
+
+Deletes an existing {dfanalytics-job}.
+
+[[ml-delete-dfanalytics-request]]
+==== {api-request-title}
+
+`DELETE _ml/data_frame/analytics/<data_frame_analytics_id>`
+
+[[ml-delete-dfanalytics-prereq]]
+==== {api-prereq-title}
+
+* You must have `machine_learning_admin` built-in role to use this API. For more 
+information, see {stack-ov}/security-privileges.html[Security privileges] and 
+{stack-ov}/built-in-roles.html[Built-in roles].
+
+[[ml-delete-dfanalytics-path-params]]
+==== {api-path-parms-title}
+
+`<data_frame_analytics_id>` (Required)::
+  (string) Identifier for the {dfanalytics-job} you want to delete.
+
+[[ml-delete-dfanalytics-example]]
+==== {api-examples-title}
+
+The following example deletes the `loganalytics` {dfanalytics-job}:
+
+[source,js]
+--------------------------------------------------
+DELETE _ml/data_frame/analytics/loganalytics
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:TBD]
+
+The API returns the following result:
+
+[source,js]
+----
+{
+  "acknowledged" : true
+}
+----
+// TESTRESPONSE

docs/reference/ml/apis/evaluate-dfanalytics.asciidoc

@@ -0,0 +1,105 @@
+[role="xpack"]
+[testenv="platinum"]
+[[evaluate-dfanalytics]]
+=== Evaluate {dfanalytics} API
+
+[subs="attributes"]
+++++
+<titleabbrev>Evaluate {dfanalytics}</titleabbrev>
+++++
+
+experimental[]
+
+Evaluates the executed analysis on an index that is already annotated with a 
+field that contains the results of the analytics (the `ground truth`) for each 
+{dataframe} row. Evaluation is typically done via calculating a set of metrics 
+that capture various aspects of the quality of the results over the data for 
+which we have the `ground truth`. For different types of analyses different 
+metrics are suitable. This API packages together commonly used metrics for 
+various analyses.
+
+[[ml-evaluate-dfanalytics-request]]
+==== {api-request-title}
+
+`POST _ml/data_frame/_evaluate`
+
+[[ml-evaluate-dfanalytics-prereq]]
+==== {api-prereq-title}
+
+* You must have `monitor_ml` privilege to use this API. For more 
+information, see {stack-ov}/security-privileges.html[Security privileges] and 
+{stack-ov}/built-in-roles.html[Built-in roles].
+
+[[ml-evaluate-dfanalytics-request-body]]
+==== {api-request-body-title}
+
+`index` (Required)::
+  (object) Defines the `index` in which the evaluation will be performed.
+
+`evaluation` (Required)::
+  (object) Defines the type of evaluation you want to perform. For example: 
+  `binary_soft_classification`.
+  See Evaluate API resources.
+
+[[ml-evaluate-dfanalytics-example]]
+==== {api-examples-title}
+
+[source,js]
+--------------------------------------------------
+POST _ml/data_frame/_evaluate
+{
+  "index": "my_analytics_dest_index",
+  "evaluation": {
+    "binary_soft_classification": {
+      "actual_field": "is_outlier",
+      "predicted_probability_field": "ml.outlier_score"
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:TBD]
+
+The API returns the following results:
+
+[source,js]
+----
+{
+  "binary_soft_classification": {
+    "auc_roc": {
+      "score": 0.92584757746414444
+    },
+    "confusion_matrix": {
+      "0.25": {
+          "tp": 5,
+          "fp": 9,
+          "tn": 204,
+          "fn": 5
+      },
+      "0.5": {
+          "tp": 1,
+          "fp": 5,
+          "tn": 208,
+          "fn": 9
+      },
+      "0.75": {
+          "tp": 0,
+          "fp": 4,
+          "tn": 209,
+          "fn": 10
+      }
+    },
+    "precision": {
+        "0.25": 0.35714285714285715,
+        "0.5": 0.16666666666666666,
+        "0.75": 0
+    },
+    "recall": {
+        "0.25": 0.5,
+        "0.5": 0.1,
+        "0.75": 0
+    }
+  }
+}
+----
+// TESTRESPONSE

docs/reference/ml/apis/get-dfanalytics-stats.asciidoc

@@ -0,0 +1,91 @@
+[role="xpack"]
+[testenv="platinum"]
+[[get-dfanalytics-stats]]
+=== Get {dfanalytics-jobs} statistics API
+[subs="attributes"]
+++++
+<titleabbrev>Get {dfanalytics-jobs} stats</titleabbrev>
+++++
+
+experimental[]
+
+Retrieves usage information for {dfanalytics-jobs}.
+
+[[ml-get-dfanalytics-stats-request]]
+==== {api-request-title}
+
+`GET _ml/data_frame/analytics/<data_frame_analytics_id>/_stats` +
+
+`GET _ml/data_frame/analytics/<data_frame_analytics_id>,<data_frame_analytics_id>/_stats` +
+
+`GET _ml/data_frame/analytics/_stats` +
+
+`GET _ml/data_frame/analytics/_all/_stats` +
+
+`GET _ml/data_frame/analytics/*/_stats`
+
+[[ml-get-dfanalytics-stats-prereq]]
+==== {api-prereq-title}
+
+* You must have `monitor_ml` privilege to use this API. For more 
+information, see {stack-ov}/security-privileges.html[Security privileges] and 
+{stack-ov}/built-in-roles.html[Built-in roles].
+
+[[ml-get-dfanalytics-stats-path-params]]
+==== {api-path-parms-title}
+
+`<data_frame_analytics_id>` (Optional)::
+  (string) Identifier for the {dfanalytics-job}. If you do not specify one of 
+  these options, the API returns information for the first hundred
+  {dfanalytics-jobs}.
+
+`allow_no_match` (Optional)
+  (boolean) If `false` and the `data_frame_analytics_id` does not match any 
+  {dfanalytics-job} an error will be returned. The default value is `true`.
+
+[[ml-get-dfanalytics-stats-query-params]]
+==== {api-query-parms-title}
+
+`from` (Optional)::
+  (integer) Skips the specified number of {dfanalytics-jobs}. The default value 
+  is `0`.
+
+`size` (Optional)::
+  (integer) Specifies the maximum number of {dfanalytics-jobs} to obtain. The 
+  default value is `100`.
+
+[discrete]
+[[ml-get-dfanalytics-stats-response-body]]
+==== {api-response-body-title}
+
+The API returns the following information:
+
+`data_frame_analytics`::
+  (array) An array of statistics objects for {dfanalytics-jobs}, which are
+  sorted by the `id` value in ascending order.
+
+[[ml-get-dfanalytics-stats-example]]
+==== {api-examples-title}
+
+[source,js]
+--------------------------------------------------
+GET _ml/data_frame/analytics/loganalytics/_stats
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:TBD]
+
+The API returns the following results:
+
+[source,js]
+----
+{
+    "count": 1,
+    "data_frame_analytics": [
+        {
+            "id": "loganalytics",
+            "state": "stopped"
+        }
+    ]
+}
+----
+// TESTRESPONSE