Merge branch 'master' into docs/sample-data

Jenkinsfile

@@ -6,7 +6,7 @@ kibanaLibrary.load()
 kibanaPipeline(timeoutMinutes: 210, checkPrChanges: true, setCommitStatus: true) {
   slackNotifications.onFailure(disabled: !params.NOTIFY_ON_FAILURE) {
     githubPr.withDefaultPrComments {
-      ciStats.trackBuild(requireSuccess: githubPr.isPr()) {
+      ciStats.trackBuild(requireSuccess: githubPr.isTrackedBranchPr()) {
         catchError {
           retryable.enable()
           kibanaPipeline.allCiTasks()

docs/user/security/authentication/index.asciidoc

@@ -7,6 +7,7 @@
 
 {kib} supports the following authentication mechanisms:
 
+- <<multiple-authentication-providers>>
 - <<basic-authentication>>
 - <<token-authentication>>
 - <<pki-authentication>>
@@ -16,15 +17,20 @@
 - <<anonymous-authentication>>
 - <<http-authentication>>
 
-Enable multiple authentication mechanisms at the same time specifying a prioritized list of the authentication _providers_ (typically of various types) in the configuration. Providers are consulted in ascending order. Make sure each configured provider has a unique name (e.g. `basic1` or `saml1` in the configuration example) and `order` setting. In the event that two or more providers have the same name or `order`, {kib} will fail to start.
+For an introduction to {kib}'s security features, including the login process, refer to <<tutorial-secure-access-to-kibana>>.
+
+[[multiple-authentication-providers]]
+==== Multiple authentication providers
+
+Enable multiple authentication mechanisms at the same time by specifying a prioritized list of the authentication _providers_ (typically of various types) in the configuration. Providers are consulted in ascending order. Make sure each configured provider has a unique name (e.g. `basic1` or `saml1` in the configuration example) and `order` setting. In the event that two or more providers have the same name or `order`, {kib} will fail to start.
 
 When two or more providers are configured, you can choose the provider you want to use on the Login Selector UI. The order the providers appear is determined by the `order` setting. The appearance of the specific provider entry can be customized with the `description`, `hint`, and `icon` settings.
 
 TIP: To provide login instructions to users, use the `xpack.security.loginHelp` setting, which supports Markdown format. When you specify the `xpack.security.loginHelp` setting, the Login Selector UI displays a `Need help?` link that lets users access login help information.
 
 If you don't want a specific provider to show up at the Login Selector UI (e.g. to only support third-party initiated login) you can hide it with `showInSelector` setting set to `false`. However, in this case, the provider is presented in the provider chain and may be consulted during authentication based on its `order`. To disable the provider, use the `enabled` setting.
 
-TIP: The Login Selector UI can also be disabled or enabled with `xpack.security.authc.selector.enabled` setting. 
+TIP: The Login Selector UI can also be disabled or enabled with `xpack.security.authc.selector.enabled` setting.
 
 Here is how your `kibana.yml` and Login Selector UI can look like if you deal with multiple authentication providers:
 
@@ -292,9 +298,9 @@ xpack.security.authc.providers:
     order: 1
 -----------------------------------------------
 
-IMPORTANT: {kib} uses SPNEGO, which wraps the Kerberos protocol for use with HTTP, extending it to web applications. 
+IMPORTANT: {kib} uses SPNEGO, which wraps the Kerberos protocol for use with HTTP, extending it to web applications.
 At the end of the Kerberos handshake, {kib} forwards the service ticket to {es}, then {es} unpacks the service ticket and responds with an access and refresh token, which are used for subsequent authentication.
-On every {es} node that {kib} connects to, the keytab file should always contain the HTTP service principal for the {kib} host. 
+On every {es} node that {kib} connects to, the keytab file should always contain the HTTP service principal for the {kib} host.
 The HTTP service principal name must have the `HTTP/[email protected]` format.
 
 
@@ -386,7 +392,7 @@ xpack.security.authc.providers:
 [[anonymous-access-and-embedding]]
 ===== Anonymous access and embedding
 
-One of the most popular use cases for anonymous access is when you embed {kib} into other applications and don't want to force your users to log in to view it. 
+One of the most popular use cases for anonymous access is when you embed {kib} into other applications and don't want to force your users to log in to view it.
 If you configured {kib} to use anonymous access as the sole authentication mechanism, you don't need to do anything special while embedding {kib}.
 
 If you have multiple authentication providers enabled, and you want to automatically log in anonymous users when embedding dashboards and visualizations:

src/core/server/saved_objects/migrations/core/document_migrator.test.ts

@@ -11,6 +11,7 @@ import { set } from '@elastic/safer-lodash-set';
 import _ from 'lodash';
 import { SavedObjectUnsanitizedDoc } from '../../serialization';
 import { DocumentMigrator } from './document_migrator';
+import { TransformSavedObjectDocumentError } from './transform_saved_object_document_error';
 import { loggingSystemMock } from '../../../logging/logging_system.mock';
 import { SavedObjectsType } from '../../types';
 import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry';
@@ -724,6 +725,12 @@ describe('DocumentMigrator', () => {
 
     it('logs the original error and throws a transform error if a document transform fails', () => {
       const log = mockLogger;
+      const failedDoc = {
+        id: 'smelly',
+        type: 'dog',
+        attributes: {},
+        migrationVersion: {},
+      };
       const migrator = new DocumentMigrator({
         ...testOpts(),
         typeRegistry: createRegistry({
@@ -737,12 +744,6 @@ describe('DocumentMigrator', () => {
         log,
       });
       migrator.prepareMigrations();
-      const failedDoc = {
-        id: 'smelly',
-        type: 'dog',
-        attributes: {},
-        migrationVersion: {},
-      };
       try {
         migrator.migrate(_.cloneDeep(failedDoc));
         expect('Did not throw').toEqual('But it should have!');
@@ -751,6 +752,7 @@ describe('DocumentMigrator', () => {
           "Failed to transform document smelly. Transform: dog:1.2.3
           Doc: {\\"id\\":\\"smelly\\",\\"type\\":\\"dog\\",\\"attributes\\":{},\\"migrationVersion\\":{}}"
         `);
+        expect(error).toBeInstanceOf(TransformSavedObjectDocumentError);
         expect(loggingSystemMock.collect(mockLoggerFactory).error[0][0]).toMatchInlineSnapshot(
           `[Error: Dang diggity!]`
         );

src/core/server/saved_objects/migrations/core/document_migrator.ts

@@ -62,6 +62,7 @@ import {
   SavedObjectsType,
 } from '../../types';
 import { MigrationLogger } from './migration_logger';
+import { TransformSavedObjectDocumentError } from '.';
 import { ISavedObjectTypeRegistry } from '../../saved_objects_type_registry';
 import { SavedObjectMigrationFn, SavedObjectMigrationMap } from '../types';
 import { DEFAULT_NAMESPACE_STRING } from '../../service/lib/utils';
@@ -679,9 +680,15 @@ function wrapWithTry(
       const failedTransform = `${type.name}:${version}`;
       const failedDoc = JSON.stringify(doc);
       log.error(error);
-
-      throw new Error(
-        `Failed to transform document ${doc?.id}. Transform: ${failedTransform}\nDoc: ${failedDoc}`
+      // To make debugging failed migrations easier, we add items needed to convert the
+      // saved object id to the full raw id (the id only contains the uuid part) and the full error itself
+      throw new TransformSavedObjectDocumentError(
+        doc.id,
+        doc.type,
+        doc.namespace,
+        failedTransform,
+        failedDoc,
+        error
       );
     }
   };

src/core/server/saved_objects/migrations/core/index.ts

@@ -15,3 +15,9 @@ export type { MigrationResult, MigrationStatus } from './migration_coordinator';
 export { createMigrationEsClient } from './migration_es_client';
 export type { MigrationEsClient } from './migration_es_client';
 export { excludeUnusedTypesQuery } from './elastic_index';
+export { TransformSavedObjectDocumentError } from './transform_saved_object_document_error';
+export type {
+  DocumentsTransformFailed,
+  DocumentsTransformSuccess,
+  TransformErrorObjects,
+} from './migrate_raw_docs';

src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts

@@ -7,10 +7,17 @@
  */
 
 import { set } from '@elastic/safer-lodash-set';
+import * as Either from 'fp-ts/lib/Either';
 import _ from 'lodash';
 import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry';
 import { SavedObjectsSerializer } from '../../serialization';
-import { migrateRawDocs } from './migrate_raw_docs';
+import {
+  DocumentsTransformFailed,
+  DocumentsTransformSuccess,
+  migrateRawDocs,
+  migrateRawDocsSafely,
+} from './migrate_raw_docs';
+import { TransformSavedObjectDocumentError } from './transform_saved_object_document_error';
 
 describe('migrateRawDocs', () => {
   test('converts raw docs to saved objects', async () => {
@@ -120,3 +127,156 @@ describe('migrateRawDocs', () => {
     ).rejects.toThrowErrorMatchingInlineSnapshot(`"error during transform"`);
   });
 });
+
+describe('migrateRawDocsSafely', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('converts raw docs to saved objects', async () => {
+    const transform = jest.fn<any, any>((doc: any) => [
+      set(_.cloneDeep(doc), 'attributes.name', 'HOI!'),
+    ]);
+    const task = migrateRawDocsSafely(
+      new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
+      transform,
+      [
+        { _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } },
+        { _id: 'c:d', _source: { type: 'c', c: { name: 'DDD' } } },
+      ]
+    );
+    const result = (await task()) as Either.Right<DocumentsTransformSuccess>;
+    expect(result._tag).toEqual('Right');
+    expect(result.right.processedDocs).toEqual([
+      {
+        _id: 'a:b',
+        _source: { type: 'a', a: { name: 'HOI!' }, migrationVersion: {}, references: [] },
+      },
+      {
+        _id: 'c:d',
+        _source: { type: 'c', c: { name: 'HOI!' }, migrationVersion: {}, references: [] },
+      },
+    ]);
+
+    const obj1 = {
+      id: 'b',
+      type: 'a',
+      attributes: { name: 'AAA' },
+      migrationVersion: {},
+      references: [],
+    };
+    const obj2 = {
+      id: 'd',
+      type: 'c',
+      attributes: { name: 'DDD' },
+      migrationVersion: {},
+      references: [],
+    };
+    expect(transform).toHaveBeenCalledTimes(2);
+    expect(transform).toHaveBeenNthCalledWith(1, obj1);
+    expect(transform).toHaveBeenNthCalledWith(2, obj2);
+  });
+
+  test('returns a `left` tag when encountering a corrupt saved object document', async () => {
+    const transform = jest.fn<any, any>((doc: any) => [
+      set(_.cloneDeep(doc), 'attributes.name', 'TADA'),
+    ]);
+    const task = migrateRawDocsSafely(
+      new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
+      transform,
+      [
+        { _id: 'foo:b', _source: { type: 'a', a: { name: 'AAA' } } },
+        { _id: 'c:d', _source: { type: 'c', c: { name: 'DDD' } } },
+      ]
+    );
+    const result = (await task()) as Either.Left<DocumentsTransformFailed>;
+    expect(transform).toHaveBeenCalledTimes(1);
+    expect(result._tag).toEqual('Left');
+    expect(Object.keys(result.left)).toEqual(['type', 'corruptDocumentIds', 'transformErrors']);
+    expect(result.left.corruptDocumentIds.length).toEqual(1);
+    expect(result.left.transformErrors.length).toEqual(0);
+  });
+
+  test('handles when one document is transformed into multiple documents', async () => {
+    const transform = jest.fn<any, any>((doc: any) => [
+      set(_.cloneDeep(doc), 'attributes.name', 'HOI!'),
+      { id: 'bar', type: 'foo', attributes: { name: 'baz' } },
+    ]);
+    const task = migrateRawDocsSafely(
+      new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
+      transform,
+      [{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }]
+    );
+    const result = (await task()) as Either.Right<DocumentsTransformSuccess>;
+    expect(result._tag).toEqual('Right');
+    expect(result.right.processedDocs).toEqual([
+      {
+        _id: 'a:b',
+        _source: { type: 'a', a: { name: 'HOI!' }, migrationVersion: {}, references: [] },
+      },
+      {
+        _id: 'foo:bar',
+        _source: { type: 'foo', foo: { name: 'baz' }, references: [] },
+      },
+    ]);
+
+    const obj = {
+      id: 'b',
+      type: 'a',
+      attributes: { name: 'AAA' },
+      migrationVersion: {},
+      references: [],
+    };
+    expect(transform).toHaveBeenCalledTimes(1);
+    expect(transform).toHaveBeenCalledWith(obj);
+  });
+
+  test('instance of Either.left containing transform errors when the transform function throws a TransformSavedObjectDocument error', async () => {
+    const transform = jest.fn<any, any>((doc: any) => {
+      throw new TransformSavedObjectDocumentError(
+        `${doc.id}`,
+        `${doc.type}`,
+        `${doc.namespace}`,
+        `${doc.type}1.2.3`,
+        JSON.stringify(doc),
+        new Error('error during transform')
+      );
+    });
+    const task = migrateRawDocsSafely(
+      new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
+      transform,
+      [{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }] // this is the raw doc
+    );
+    const result = (await task()) as Either.Left<DocumentsTransformFailed>;
+    expect(transform).toHaveBeenCalledTimes(1);
+    expect(result._tag).toEqual('Left');
+    expect(result.left.corruptDocumentIds.length).toEqual(0);
+    expect(result.left.transformErrors.length).toEqual(1);
+    expect(result.left.transformErrors[0].err.message).toMatchInlineSnapshot(`
+      "Failed to transform document b. Transform: a1.2.3
+      Doc: {\\"type\\":\\"a\\",\\"id\\":\\"b\\",\\"attributes\\":{\\"name\\":\\"AAA\\"},\\"references\\":[],\\"migrationVersion\\":{}}"
+    `);
+  });
+
+  test("instance of Either.left containing errors when the transform function throws an error that isn't a TransformSavedObjectDocument error", async () => {
+    const transform = jest.fn<any, any>((doc: any) => {
+      throw new Error('error during transform');
+    });
+    const task = migrateRawDocsSafely(
+      new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
+      transform,
+      [{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }] // this is the raw doc
+    );
+    const result = (await task()) as Either.Left<DocumentsTransformFailed>;
+    expect(transform).toHaveBeenCalledTimes(1);
+    expect(result._tag).toEqual('Left');
+    expect(result.left.corruptDocumentIds.length).toEqual(0);
+    expect(result.left.transformErrors.length).toEqual(1);
+    expect(result.left.transformErrors[0]).toMatchInlineSnapshot(`
+      Object {
+        "err": [Error: error during transform],
+        "rawId": "a:b",
+      }
+    `);
+  });
+});