Merge branch 'master' into nls/tsr-obs-apm

.ci/Jenkinsfile_security_cypress

@@ -10,7 +10,8 @@ kibanaPipeline(timeoutMinutes: 180) {
   ) {
     catchError {
       withEnv([
-        'CI_PARALLEL_PROCESS_NUMBER=1'
+        'CI_PARALLEL_PROCESS_NUMBER=1',
+        'IGNORE_SHIP_CI_STATS_ERROR=true',
       ]) {
         def job = 'xpack-securityCypress'
 

.ci/es-snapshots/Jenkinsfile_verify_es

@@ -26,7 +26,10 @@ kibanaPipeline(timeoutMinutes: 150) {
       message: "[${SNAPSHOT_VERSION}] ES Snapshot Verification Failure",
     ) {
       retryable.enable(2)
-      withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) {
+      withEnv([
+        "ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}",
+        'IGNORE_SHIP_CI_STATS_ERROR=true',
+      ]) {
         parallel([
           'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
           'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [

dev_docs/tutorials/building_a_plugin.mdx

@@ -0,0 +1,226 @@
+---
+id: kibDevTutorialBuildAPlugin
+slug: /kibana-dev-docs/tutorials/build-a-plugin
+title: Kibana plugin tutorial
+summary: Anatomy of a Kibana plugin and how to build one
+date: 2021-02-05
+tags: ['kibana','onboarding', 'dev', 'tutorials']
+---
+
+Prereading material:
+
+- <DocLink id="kibPlatformIntro" />
+
+## The anatomy of a plugin
+
+Plugins are defined as classes and present themselves to Kibana through a simple wrapper function. A plugin can have browser-side code, server-side code,
+or both. There is no architectural difference between a plugin in the browser and a plugin on the server. In both places, you describe your plugin similarly, 
+and you interact with Core and other plugins in the same way.
+
+The basic file structure of a Kibana plugin named demo that has both client-side and server-side code would be:
+
+```
+plugins/
+  demo
+    kibana.json [1]
+    public
+      index.ts  [2]
+      plugin.ts [3]
+    server
+      index.ts  [4]
+      plugin.ts [5]
+    common
+      index.ts  [6]
+```
+
+### [1] kibana.json 
+
+`kibana.json` is a static manifest file that is used to identify the plugin and to specify if this plugin has server-side code, browser-side code, or both:
+
+```
+{
+  "id": "demo",
+  "version": "kibana",
+  "server": true,
+  "ui": true
+}
+```
+
+### [2] public/index.ts
+
+`public/index.ts` is the entry point into the client-side code of this plugin. It must export a function named plugin, which will receive a standard set of
+ core capabilities as an argument. It should return an instance of its plugin class for Kibana to load.
+
+```
+import type { PluginInitializerContext } from 'kibana/server';
+import { DemoPlugin } from './plugin';
+
+export function plugin(initializerContext: PluginInitializerContext) {
+  return new DemoPlugin(initializerContext);
+}
+```
+
+### [3] public/plugin.ts
+
+`public/plugin.ts` is the client-side plugin definition itself. Technically speaking, it does not need to be a class or even a separate file from the entry
+ point, but all plugins at Elastic should be consistent in this way.
+
+
+ ```ts
+import type { Plugin, PluginInitializerContext, CoreSetup, CoreStart } from 'kibana/server';
+
+export class DemoPlugin implements Plugin {
+  constructor(initializerContext: PluginInitializerContext) {}
+
+  public setup(core: CoreSetup) {
+    // called when plugin is setting up during Kibana's startup sequence
+  }
+
+  public start(core: CoreStart) {
+    // called after all plugins are set up
+  }
+
+  public stop() {
+    // called when plugin is torn down during Kibana's shutdown sequence
+  }
+}
+ ```
+
+
+### [4] server/index.ts 
+
+`server/index.ts` is the entry-point into the server-side code of this plugin. It is identical in almost every way to the client-side entry-point:
+
+### [5] server/plugin.ts
+
+`server/plugin.ts` is the server-side plugin definition. The shape of this plugin is the same as it’s client-side counter-part:
+
+```ts
+import type { Plugin, PluginInitializerContext, CoreSetup, CoreStart } from 'kibana/server';
+
+export class DemoPlugin implements Plugin {
+  constructor(initializerContext: PluginInitializerContext) {}
+
+  public setup(core: CoreSetup) {
+    // called when plugin is setting up during Kibana's startup sequence
+  }
+
+  public start(core: CoreStart) {
+    // called after all plugins are set up
+  }
+
+  public stop() {
+    // called when plugin is torn down during Kibana's shutdown sequence
+  }
+}
+```
+
+Kibana does not impose any technical restrictions on how the the internals of a plugin are architected, though there are certain 
+considerations related to how plugins integrate with core APIs and APIs exposed by other plugins that may greatly impact how they are built.
+
+### [6] common/index.ts
+
+`common/index.ts` is the entry-point into code that can be used both server-side or client side.
+
+## How plugin's interact with each other, and Core
+
+The lifecycle-specific contracts exposed by core services are always passed as the first argument to the equivalent lifecycle function in a plugin. 
+For example, the core http service exposes a function createRouter to all plugin setup functions. To use this function to register an HTTP route handler, 
+a plugin just accesses it off of the first argument:
+
+```ts
+import type { CoreSetup } from 'kibana/server';
+
+export class DemoPlugin {
+  public setup(core: CoreSetup) {
+    const router = core.http.createRouter();
+    // handler is called when '/path' resource is requested with `GET` method
+    router.get({ path: '/path', validate: false }, (context, req, res) => res.ok({ content: 'ok' }));
+  }
+}
+```
+
+Unlike core, capabilities exposed by plugins are not automatically injected into all plugins.
+Instead, if a plugin wishes to use the public interface provided by another plugin, it must first declare that plugin as a
+ dependency in it’s kibana.json manifest file.
+
+** foobar plugin.ts: **
+
+```
+import type { Plugin } from 'kibana/server';
+export interface FoobarPluginSetup { [1]
+  getFoo(): string;
+}
+
+export interface FoobarPluginStart { [1]
+  getBar(): string;
+}
+
+export class MyPlugin implements Plugin<FoobarPluginSetup, FoobarPluginStart> {
+  public setup(): FoobarPluginSetup {
+    return {
+      getFoo() {
+        return 'foo';
+      },
+    };
+  }
+
+  public start(): FoobarPluginStart {
+    return {
+      getBar() {
+        return 'bar';
+      },
+    };
+  }
+}
+```
+[1]  We highly encourage plugin authors to explicitly declare public interfaces for their plugins.
+
+
+** demo kibana.json**
+
+```
+{
+  "id": "demo",
+  "requiredPlugins": ["foobar"],
+  "server": true,
+  "ui": true
+}
+```
+
+With that specified in the plugin manifest, the appropriate interfaces are then available via the second argument of setup and/or start:
+
+```ts
+import type { CoreSetup, CoreStart } from 'kibana/server';
+import type { FoobarPluginSetup, FoobarPluginStart } from '../../foobar/server';
+
+interface DemoSetupPlugins {  [1]
+  foobar: FoobarPluginSetup;
+}
+
+interface DemoStartPlugins {
+  foobar: FoobarPluginStart;
+}
+
+export class DemoPlugin {
+  public setup(core: CoreSetup, plugins: DemoSetupPlugins) { [2]
+    const { foobar } = plugins;
+    foobar.getFoo(); // 'foo'
+    foobar.getBar(); // throws because getBar does not exist
+  }
+
+  public start(core: CoreStart, plugins: DemoStartPlugins) { [3]
+    const { foobar } = plugins;
+    foobar.getFoo(); // throws because getFoo does not exist
+    foobar.getBar(); // 'bar'
+  }
+
+  public stop() {}
+}
+```
+
+[1] The interface for plugin’s dependencies must be manually composed. You can do this by importing the appropriate type from the plugin and constructing an interface where the property name is the plugin’s ID.
+
+[2] These manually constructed types should then be used to specify the type of the second argument to the plugin.
+
+[3] Notice that the type for the setup and start lifecycles are different. Plugin lifecycle functions can only access the APIs that are exposed during that lifecycle.

docs/developer/plugin-list.asciidoc

@@ -443,10 +443,6 @@ using the CURL scripts in the scripts folder.
 |Visualize geo data from Elasticsearch or 3rd party geo-services.
 
 
-|{kib-repo}blob/{branch}/x-pack/plugins/maps_file_upload/README.md[mapsFileUpload]
-|Deprecated - plugin targeted for removal and will get merged into file_upload plugin
-
-
 |{kib-repo}blob/{branch}/x-pack/plugins/maps_legacy_licensing/README.md[mapsLegacyLicensing]
 |This plugin provides access to the detailed tile map services from Elastic.
 

docs/migration/migrate_8_0.asciidoc

@@ -52,7 +52,18 @@ for example, `logstash-*`.
 ==== Default logging timezone is now the system's timezone
 *Details:* In prior releases the timezone used in logs defaulted to UTC.  We now use the host machine's timezone by default.
 
-*Impact:* To restore the previous behavior, in kibana.yml set `logging.timezone: UTC`.
+*Impact:* To restore the previous behavior, in kibana.yml use the pattern layout, with a date modifier:
+[source,yaml]
+-------------------
+logging:
+  appenders:
+    console:
+      kind: console
+      layout:
+        kind: pattern
+        pattern: "%date{ISO8601_TZ}{UTC}"
+-------------------
+See https://github.com/elastic/kibana/pull/90368 for more details.
 
 [float]
 ==== Responses are never logged by default

docs/setup/settings.asciidoc

@@ -309,7 +309,8 @@ suppress all logging output. *Default: `false`*
  | Set to the canonical time zone ID
 (for example, `America/Los_Angeles`) to log events using that time zone.
 For possible values, refer to
-https://en.wikipedia.org/wiki/List_of_tz_database_time_zones[database time zones]. *Default: `UTC`*
+https://en.wikipedia.org/wiki/List_of_tz_database_time_zones[database time zones].
+When not set, log events use the host timezone
 
 | [[logging-verbose]] `logging.verbose:` {ess-icon}
  | Set to `true` to log all events, including system usage information and all

package.json

@@ -97,7 +97,7 @@
     "@babel/runtime": "^7.12.5",
     "@elastic/datemath": "link:packages/elastic-datemath",
     "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary",
-    "@elastic/ems-client": "7.11.0",
+    "@elastic/ems-client": "7.12.0",
     "@elastic/eui": "31.4.0",
     "@elastic/filesaver": "1.1.2",
     "@elastic/good": "^9.0.1-kibana3",

packages/kbn-dev-utils/src/ci_stats_reporter/ship_ci_stats_cli.ts

@@ -22,10 +22,18 @@ export function shipCiStatsCli() {
         throw createFlagError('expected --metrics to be a string');
       }
 
+      const maybeFail = (message: string) => {
+        const error = createFailError(message);
+        if (process.env.IGNORE_SHIP_CI_STATS_ERROR === 'true') {
+          error.exitCode = 0;
+        }
+        return error;
+      };
+
       const reporter = CiStatsReporter.fromEnv(log);
 
       if (!reporter.isEnabled()) {
-        throw createFailError('unable to initilize the CI Stats reporter');
+        throw maybeFail('unable to initilize the CI Stats reporter');
       }
 
       for (const path of metricPaths) {
@@ -35,7 +43,7 @@ export function shipCiStatsCli() {
         if (await reporter.metrics(JSON.parse(json))) {
           log.success('shipped metrics from', path);
         } else {
-          throw createFailError('failed to ship metrics');
+          throw maybeFail('failed to ship metrics');
         }
       }
     },

packages/kbn-optimizer/limits.yml

@@ -104,4 +104,4 @@ pageLoadAssetSize:
   presentationUtil: 28545
   spacesOss: 18817
   osquery: 107090
-  mapsFileUpload: 23775
+  fileUpload: 25664

src/core/server/config/deprecation/core_deprecations.test.ts

@@ -298,4 +298,24 @@ describe('core deprecations', () => {
       expect(messages).toEqual([]);
     });
   });
+
+  describe('logging.timezone', () => {
+    it('warns when ops events are used', () => {
+      const { messages } = applyCoreDeprecations({
+        logging: { timezone: 'GMT' },
+      });
+      expect(messages).toMatchInlineSnapshot(`
+        Array [
+          "\\"logging.timezone\\" has been deprecated and will be removed in 8.0. To set the timezone moving forward, please add a timezone date modifier to the log pattern in your logging configuration. For more details, see https://github.com/elastic/kibana/blob/master/src/core/server/logging/README.md",
+        ]
+      `);
+    });
+
+    it('does not warn when other events are configured', () => {
+      const { messages } = applyCoreDeprecations({
+        logging: { events: { log: '*' } },
+      });
+      expect(messages).toEqual([]);
+    });
+  });
 });

src/core/server/config/deprecation/core_deprecations.ts

@@ -127,6 +127,18 @@ const requestLoggingEventDeprecation: ConfigDeprecation = (settings, fromPath, l
   return settings;
 };
 
+const timezoneLoggingDeprecation: ConfigDeprecation = (settings, fromPath, log) => {
+  if (has(settings, 'logging.timezone')) {
+    log(
+      '"logging.timezone" has been deprecated and will be removed ' +
+        'in 8.0. To set the timezone moving forward, please add a timezone date modifier to the log pattern ' +
+        'in your logging configuration. For more details, see ' +
+        'https://github.com/elastic/kibana/blob/master/src/core/server/logging/README.md'
+    );
+  }
+  return settings;
+};
+
 export const coreDeprecationProvider: ConfigDeprecationProvider = ({ rename, unusedFromRoot }) => [
   unusedFromRoot('savedObjects.indexCheckTimeout'),
   unusedFromRoot('server.xsrf.token'),
@@ -163,4 +175,5 @@ export const coreDeprecationProvider: ConfigDeprecationProvider = ({ rename, unu
   mapManifestServiceUrlDeprecation,
   opsLoggingEventDeprecation,
   requestLoggingEventDeprecation,
+  timezoneLoggingDeprecation,
 ];