Merge branch 'master' into ml-transforms-api-tests

docs/development/core/server/kibana-plugin-core-server.md

@@ -123,7 +123,7 @@ The plugin integrates with the core system via lifecycle events: `setup`<!-- -->
 |  [LoggerFactory](./kibana-plugin-core-server.loggerfactory.md) | The single purpose of <code>LoggerFactory</code> interface is to define a way to retrieve a context-based logger instance. |
 |  [LoggingServiceSetup](./kibana-plugin-core-server.loggingservicesetup.md) | Provides APIs to plugins for customizing the plugin's logger. |
 |  [LogMeta](./kibana-plugin-core-server.logmeta.md) | Contextual metadata |
-|  [MetricsServiceSetup](./kibana-plugin-core-server.metricsservicesetup.md) |  |
+|  [MetricsServiceSetup](./kibana-plugin-core-server.metricsservicesetup.md) | APIs to retrieves metrics gathered and exposed by the core platform. |
 |  [NodesVersionCompatibility](./kibana-plugin-core-server.nodesversioncompatibility.md) |  |
 |  [OnPostAuthToolkit](./kibana-plugin-core-server.onpostauthtoolkit.md) | A tool set defining an outcome of OnPostAuth interceptor for incoming request. |
 |  [OnPreAuthToolkit](./kibana-plugin-core-server.onpreauthtoolkit.md) | A tool set defining an outcome of OnPreAuth interceptor for incoming request. |

docs/development/core/server/kibana-plugin-core-server.metricsservicesetup.collectioninterval.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [MetricsServiceSetup](./kibana-plugin-core-server.metricsservicesetup.md) &gt; [collectionInterval](./kibana-plugin-core-server.metricsservicesetup.collectioninterval.md)
+
+## MetricsServiceSetup.collectionInterval property
+
+Interval metrics are collected in milliseconds
+
+<b>Signature:</b>
+
+```typescript
+readonly collectionInterval: number;
+```

docs/development/core/server/kibana-plugin-core-server.metricsservicesetup.getopsmetrics_.md

@@ -0,0 +1,24 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [MetricsServiceSetup](./kibana-plugin-core-server.metricsservicesetup.md) &gt; [getOpsMetrics$](./kibana-plugin-core-server.metricsservicesetup.getopsmetrics_.md)
+
+## MetricsServiceSetup.getOpsMetrics$ property
+
+Retrieve an observable emitting the [OpsMetrics](./kibana-plugin-core-server.opsmetrics.md) gathered. The observable will emit an initial value during core's `start` phase, and a new value every fixed interval of time, based on the `opts.interval` configuration property.
+
+<b>Signature:</b>
+
+```typescript
+getOpsMetrics$: () => Observable<OpsMetrics>;
+```
+
+## Example
+
+
+```ts
+core.metrics.getOpsMetrics$().subscribe(metrics => {
+  // do something with the metrics
+})
+
+```
+

docs/development/core/server/kibana-plugin-core-server.metricsservicesetup.md

@@ -4,8 +4,18 @@
 
 ## MetricsServiceSetup interface
 
+APIs to retrieves metrics gathered and exposed by the core platform.
+
 <b>Signature:</b>
 
 ```typescript
 export interface MetricsServiceSetup 
 ```
+
+## Properties
+
+|  Property | Type | Description |
+|  --- | --- | --- |
+|  [collectionInterval](./kibana-plugin-core-server.metricsservicesetup.collectioninterval.md) | <code>number</code> | Interval metrics are collected in milliseconds |
+|  [getOpsMetrics$](./kibana-plugin-core-server.metricsservicesetup.getopsmetrics_.md) | <code>() =&gt; Observable&lt;OpsMetrics&gt;</code> | Retrieve an observable emitting the [OpsMetrics](./kibana-plugin-core-server.opsmetrics.md) gathered. The observable will emit an initial value during core's <code>start</code> phase, and a new value every fixed interval of time, based on the <code>opts.interval</code> configuration property. |
+

docs/development/core/server/kibana-plugin-core-server.opsmetrics.collected_at.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [OpsMetrics](./kibana-plugin-core-server.opsmetrics.md) &gt; [collected\_at](./kibana-plugin-core-server.opsmetrics.collected_at.md)
+
+## OpsMetrics.collected\_at property
+
+Time metrics were recorded at.
+
+<b>Signature:</b>
+
+```typescript
+collected_at: Date;
+```

docs/development/core/server/kibana-plugin-core-server.opsmetrics.md

@@ -16,6 +16,7 @@ export interface OpsMetrics
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [collected\_at](./kibana-plugin-core-server.opsmetrics.collected_at.md) | <code>Date</code> | Time metrics were recorded at. |
 |  [concurrent\_connections](./kibana-plugin-core-server.opsmetrics.concurrent_connections.md) | <code>OpsServerMetrics['concurrent_connections']</code> | number of current concurrent connections to the server |
 |  [os](./kibana-plugin-core-server.opsmetrics.os.md) | <code>OpsOsMetrics</code> | OS related metrics |
 |  [process](./kibana-plugin-core-server.opsmetrics.process.md) | <code>OpsProcessMetrics</code> | Process related metrics |

docs/development/core/server/kibana-plugin-core-server.opsosmetrics.cpu.md

@@ -0,0 +1,22 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [OpsOsMetrics](./kibana-plugin-core-server.opsosmetrics.md) &gt; [cpu](./kibana-plugin-core-server.opsosmetrics.cpu.md)
+
+## OpsOsMetrics.cpu property
+
+cpu cgroup metrics, undefined when not running in a cgroup
+
+<b>Signature:</b>
+
+```typescript
+cpu?: {
+        control_group: string;
+        cfs_period_micros: number;
+        cfs_quota_micros: number;
+        stat: {
+            number_of_elapsed_periods: number;
+            number_of_times_throttled: number;
+            time_throttled_nanos: number;
+        };
+    };
+```

docs/development/core/server/kibana-plugin-core-server.opsosmetrics.cpuacct.md

@@ -0,0 +1,16 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [OpsOsMetrics](./kibana-plugin-core-server.opsosmetrics.md) &gt; [cpuacct](./kibana-plugin-core-server.opsosmetrics.cpuacct.md)
+
+## OpsOsMetrics.cpuacct property
+
+cpu accounting metrics, undefined when not running in a cgroup
+
+<b>Signature:</b>
+
+```typescript
+cpuacct?: {
+        control_group: string;
+        usage_nanos: number;
+    };
+```

docs/development/core/server/kibana-plugin-core-server.opsosmetrics.md

@@ -16,6 +16,8 @@ export interface OpsOsMetrics
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [cpu](./kibana-plugin-core-server.opsosmetrics.cpu.md) | <code>{</code><br/><code>        control_group: string;</code><br/><code>        cfs_period_micros: number;</code><br/><code>        cfs_quota_micros: number;</code><br/><code>        stat: {</code><br/><code>            number_of_elapsed_periods: number;</code><br/><code>            number_of_times_throttled: number;</code><br/><code>            time_throttled_nanos: number;</code><br/><code>        };</code><br/><code>    }</code> | cpu cgroup metrics, undefined when not running in a cgroup |
+|  [cpuacct](./kibana-plugin-core-server.opsosmetrics.cpuacct.md) | <code>{</code><br/><code>        control_group: string;</code><br/><code>        usage_nanos: number;</code><br/><code>    }</code> | cpu accounting metrics, undefined when not running in a cgroup |
 |  [distro](./kibana-plugin-core-server.opsosmetrics.distro.md) | <code>string</code> | The os distrib. Only present for linux platforms |
 |  [distroRelease](./kibana-plugin-core-server.opsosmetrics.distrorelease.md) | <code>string</code> | The os distrib release, prefixed by the os distrib. Only present for linux platforms |
 |  [load](./kibana-plugin-core-server.opsosmetrics.load.md) | <code>{</code><br/><code>        '1m': number;</code><br/><code>        '5m': number;</code><br/><code>        '15m': number;</code><br/><code>    }</code> | cpu load metrics |

docs/development/core/server/kibana-plugin-core-server.statusservicesetup.dependencies_.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [StatusServiceSetup](./kibana-plugin-core-server.statusservicesetup.md) &gt; [dependencies$](./kibana-plugin-core-server.statusservicesetup.dependencies_.md)
+
+## StatusServiceSetup.dependencies$ property
+
+Current status for all plugins this plugin depends on. Each key of the `Record` is a plugin id.
+
+<b>Signature:</b>
+
+```typescript
+dependencies$: Observable<Record<string, ServiceStatus>>;
+```

docs/development/core/server/kibana-plugin-core-server.statusservicesetup.derivedstatus_.md

@@ -0,0 +1,20 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [StatusServiceSetup](./kibana-plugin-core-server.statusservicesetup.md) &gt; [derivedStatus$](./kibana-plugin-core-server.statusservicesetup.derivedstatus_.md)
+
+## StatusServiceSetup.derivedStatus$ property
+
+The status of this plugin as derived from its dependencies.
+
+<b>Signature:</b>
+
+```typescript
+derivedStatus$: Observable<ServiceStatus>;
+```
+
+## Remarks
+
+By default, plugins inherit this derived status from their dependencies. Calling  overrides this default status.
+
+This may emit multliple times for a single status change event as propagates through the dependency tree
+

docs/development/core/server/kibana-plugin-core-server.statusservicesetup.md

@@ -12,10 +12,73 @@ API for accessing status of Core and this plugin's dependencies as well as for c
 export interface StatusServiceSetup 
 ```
 
+## Remarks
+
+By default, a plugin inherits it's current status from the most severe status level of any Core services and any plugins that it depends on. This default status is available on the  API.
+
+Plugins may customize their status calculation by calling the  API with an Observable. Within this Observable, a plugin may choose to only depend on the status of some of its dependencies, to ignore severe status levels of particular Core services they are not concerned with, or to make its status dependent on other external services.
+
+## Example 1
+
+Customize a plugin's status to only depend on the status of SavedObjects:
+
+```ts
+core.status.set(
+  core.status.core$.pipe(
+.   map((coreStatus) => {
+      return coreStatus.savedObjects;
+    }) ;
+  );
+);
+
+```
+
+## Example 2
+
+Customize a plugin's status to include an external service:
+
+```ts
+const externalStatus$ = interval(1000).pipe(
+  switchMap(async () => {
+    const resp = await fetch(`https://myexternaldep.com/_healthz`);
+    const body = await resp.json();
+    if (body.ok) {
+      return of({ level: ServiceStatusLevels.available, summary: 'External Service is up'});
+    } else {
+      return of({ level: ServiceStatusLevels.available, summary: 'External Service is unavailable'});
+    }
+  }),
+  catchError((error) => {
+    of({ level: ServiceStatusLevels.unavailable, summary: `External Service is down`, meta: { error }})
+  })
+);
+
+core.status.set(
+  combineLatest([core.status.derivedStatus$, externalStatus$]).pipe(
+    map(([derivedStatus, externalStatus]) => {
+      if (externalStatus.level > derivedStatus) {
+        return externalStatus;
+      } else {
+        return derivedStatus;
+      }
+    })
+  )
+);
+
+```
+
 ## Properties
 
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [core$](./kibana-plugin-core-server.statusservicesetup.core_.md) | <code>Observable&lt;CoreStatus&gt;</code> | Current status for all Core services. |
+|  [dependencies$](./kibana-plugin-core-server.statusservicesetup.dependencies_.md) | <code>Observable&lt;Record&lt;string, ServiceStatus&gt;&gt;</code> | Current status for all plugins this plugin depends on. Each key of the <code>Record</code> is a plugin id. |
+|  [derivedStatus$](./kibana-plugin-core-server.statusservicesetup.derivedstatus_.md) | <code>Observable&lt;ServiceStatus&gt;</code> | The status of this plugin as derived from its dependencies. |
 |  [overall$](./kibana-plugin-core-server.statusservicesetup.overall_.md) | <code>Observable&lt;ServiceStatus&gt;</code> | Overall system status for all of Kibana. |
 
+## Methods
+
+|  Method | Description |
+|  --- | --- |
+|  [set(status$)](./kibana-plugin-core-server.statusservicesetup.set.md) | Allows a plugin to specify a custom status dependent on its own criteria. Completely overrides the default inherited status. |
+

docs/development/core/server/kibana-plugin-core-server.statusservicesetup.set.md

@@ -0,0 +1,28 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [StatusServiceSetup](./kibana-plugin-core-server.statusservicesetup.md) &gt; [set](./kibana-plugin-core-server.statusservicesetup.set.md)
+
+## StatusServiceSetup.set() method
+
+Allows a plugin to specify a custom status dependent on its own criteria. Completely overrides the default inherited status.
+
+<b>Signature:</b>
+
+```typescript
+set(status$: Observable<ServiceStatus>): void;
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  status$ | <code>Observable&lt;ServiceStatus&gt;</code> |  |
+
+<b>Returns:</b>
+
+`void`
+
+## Remarks
+
+See the [StatusServiceSetup.derivedStatus$](./kibana-plugin-core-server.statusservicesetup.derivedstatus_.md) API for leveraging the default status calculation that is provided by Core.
+

docs/setup/settings.asciidoc

@@ -20,12 +20,12 @@ which may cause a delay before pages start being served.
 Set to `false` to disable Console. *Default: `true`*
 
 | `cpu.cgroup.path.override:`
- | Override for cgroup cpu path when mounted in a
-manner that is inconsistent with `/proc/self/cgroup`.
+ | *deprecated* This setting has been renamed to `ops.cGroupOverrides.cpuPath`
+and the old name will no longer be supported as of 8.0.
 
 | `cpuacct.cgroup.path.override:`
- | Override for cgroup cpuacct path when mounted
-in a manner that is inconsistent with `/proc/self/cgroup`.
+ | *deprecated* This setting has been renamed to `ops.cGroupOverrides.cpuAcctPath`
+and the old name will no longer be supported as of 8.0.
 
 | `csp.rules:`
  | A https://w3c.github.io/webappsec-csp/[content-security-policy] template
@@ -438,6 +438,14 @@ not saved in {es}. *Default: `data`*
  | Set the interval in milliseconds to sample
 system and process performance metrics. The minimum value is 100. *Default: `5000`*
 
+| `ops.cGroupOverrides.cpuPath:`
+ | Override for cgroup cpu path when mounted in a
+manner that is inconsistent with `/proc/self/cgroup`.
+
+| `ops.cGroupOverrides.cpuAcctPath:`
+ | Override for cgroup cpuacct path when mounted
+in a manner that is inconsistent with `/proc/self/cgroup`.
+
 | `server.basePath:`
  | Enables you to specify a path to mount {kib} at if you are
 running behind a proxy. Use the `server.rewriteBasePath` setting to tell {kib}

docs/user/reporting/reporting-troubleshooting.asciidoc

@@ -7,6 +7,7 @@
 
 Having trouble? Here are solutions to common problems you might encounter while using Reporting.
 
+* <<reporting-diagnostics>>
 * <<reporting-troubleshooting-system-dependencies>>
 * <<reporting-troubleshooting-text-incorrect>>
 * <<reporting-troubleshooting-missing-data>>
@@ -15,6 +16,11 @@ Having trouble? Here are solutions to common problems you might encounter while
 * <<reporting-troubleshooting-puppeteer-debug-logs>>
 * <<reporting-troubleshooting-system-requirements>>
 
+[float]
+[[reporting-diagnostics]]
+=== Reporting Diagnostics
+Reporting comes with a built-in utility to try to automatically find common issues. When Kibana is running, navigate to the Report Listing page, and click the "Run reporting diagnostics..." button. This will open up a diagnostic tool that checks various parts of the Kibana deployment to come up with any relevant recommendations.
+
 [float]
 [[reporting-troubleshooting-system-dependencies]]
 === System dependencies

rfcs/text/0010_service_status.md

@@ -137,7 +137,7 @@ interface StatusSetup {
    * Current status for all dependencies of the current plugin.
    * Each key of the `Record` is a plugin id.
    */
-  plugins$: Observable<Record<string, ServiceStatus>>;
+  dependencies$: Observable<Record<string, ServiceStatus>>;
 
   /**
    * The status of this plugin as derived from its dependencies.

src/cli/cluster/worker.ts

@@ -21,7 +21,7 @@ import _ from 'lodash';
 import cluster from 'cluster';
 import { EventEmitter } from 'events';
 
-import { BinderFor } from '../../legacy/utils/binder_for';
+import { BinderFor } from './binder_for';
 import { fromRoot } from '../../core/server/utils';
 
 const cliPath = fromRoot('src/cli');

src/cli_keystore/add.js

@@ -18,7 +18,7 @@
  */
 
 import { Logger } from '../cli_plugin/lib/logger';
-import { confirm, question } from '../legacy/server/utils';
+import { confirm, question } from './utils';
 import { createPromiseFromStreams, createConcatStream } from '../core/server/utils';
 
 /**

src/cli_keystore/add.test.js

@@ -42,7 +42,7 @@ import { PassThrough } from 'stream';
 import { Keystore } from '../legacy/server/keystore';
 import { add } from './add';
 import { Logger } from '../cli_plugin/lib/logger';
-import * as prompt from '../legacy/server/utils/prompt';
+import * as prompt from './utils/prompt';
 
 describe('Kibana keystore', () => {
   describe('add', () => {

src/cli_keystore/create.js

@@ -18,7 +18,7 @@
  */
 
 import { Logger } from '../cli_plugin/lib/logger';
-import { confirm } from '../legacy/server/utils';
+import { confirm } from './utils';
 
 export async function create(keystore, command, options) {
   const logger = new Logger(options);

src/cli_keystore/create.test.js

@@ -41,7 +41,7 @@ import sinon from 'sinon';
 import { Keystore } from '../legacy/server/keystore';
 import { create } from './create';
 import { Logger } from '../cli_plugin/lib/logger';
-import * as prompt from '../legacy/server/utils/prompt';
+import * as prompt from './utils/prompt';
 
 describe('Kibana keystore', () => {
   describe('create', () => {

src/core/public/core_app/status/lib/load_status.test.ts

@@ -57,6 +57,7 @@ const mockedResponse: StatusResponse = {
     ],
   },
   metrics: {
+    collected_at: new Date('2020-01-01 01:00:00'),
     collection_interval_in_millis: 1000,
     os: {
       platform: 'darwin' as const,