From ae2e0766f9bc109a9200c75d055e095f7324cc4b Mon Sep 17 00:00:00 2001
From: Yara Tercero
Date: Mon, 3 May 2021 13:34:16 -0700
Subject: [PATCH 001/106] [Security Solution][Timeline] - update flakey
timeline cypress tests (#99097)
###Summary
Removes some anti-patterns I introduced in my last PR and makes use of Cypress' chainable pattern that will hopefully help with the flakiness.
---
.../integration/timelines/notes_tab.spec.ts | 22 ++++++++----
.../timelines/open_timeline.spec.ts | 7 ++--
.../cypress/tasks/timeline.ts | 35 ++++++++++++-------
3 files changed, 43 insertions(+), 21 deletions(-)
diff --git a/x-pack/plugins/security_solution/cypress/integration/timelines/notes_tab.spec.ts b/x-pack/plugins/security_solution/cypress/integration/timelines/notes_tab.spec.ts
index 72f82aa54b7d5..fdc003039afbc 100644
--- a/x-pack/plugins/security_solution/cypress/integration/timelines/notes_tab.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/timelines/notes_tab.spec.ts
@@ -26,12 +26,22 @@ describe('Timeline notes tab', () => {
loginAndWaitForPageWithoutDateRange(TIMELINES_URL);
waitForTimelinesPanelToBeLoaded();
- createTimeline(timeline).then((response) => {
- timelineId = response.body.data.persistTimeline.timeline.savedObjectId;
- waitForTimelinesPanelToBeLoaded();
- openTimelineById(timelineId!);
- addNotesToTimeline(timeline.notes);
- });
+ createTimeline(timeline)
+ .then((response) => {
+ if (response.body.data.persistTimeline.timeline.savedObjectId == null) {
+ cy.log('"createTimeline" did not return expected response');
+ }
+ timelineId = response.body.data.persistTimeline.timeline.savedObjectId;
+ waitForTimelinesPanelToBeLoaded();
+ })
+ .then(() => {
+ // TODO: It would be great to add response validation to avoid such things like
+ // the bang below and to more easily understand where failures are coming from -
+ // client vs server side
+ openTimelineById(timelineId!).then(() => {
+ addNotesToTimeline(timeline.notes);
+ });
+ });
});
after(() => {
diff --git a/x-pack/plugins/security_solution/cypress/integration/timelines/open_timeline.spec.ts b/x-pack/plugins/security_solution/cypress/integration/timelines/open_timeline.spec.ts
index 93c8d7265478e..7bf0409d91039 100644
--- a/x-pack/plugins/security_solution/cypress/integration/timelines/open_timeline.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/timelines/open_timeline.spec.ts
@@ -48,9 +48,10 @@ describe('Open timeline', () => {
addNoteToTimeline(note, timelineId!).should((response) => {
expect(response.status).to.equal(200);
waitForTimelinesPanelToBeLoaded();
- openTimelineById(timelineId!);
- pinFirstEvent();
- markAsFavorite();
+ openTimelineById(timelineId!).then(() => {
+ pinFirstEvent();
+ markAsFavorite();
+ });
});
});
});
diff --git a/x-pack/plugins/security_solution/cypress/tasks/timeline.ts b/x-pack/plugins/security_solution/cypress/tasks/timeline.ts
index 1474098b76281..a7842a7d98e83 100644
--- a/x-pack/plugins/security_solution/cypress/tasks/timeline.ts
+++ b/x-pack/plugins/security_solution/cypress/tasks/timeline.ts
@@ -89,13 +89,14 @@ export const addNameAndDescriptionToTimeline = (timeline: Timeline) => {
cy.get(TIMELINE_TITLE_INPUT).should('not.exist');
};
-export const goToNotesTab = () => {
+export const goToNotesTab = (): Cypress.Chainable> => {
cy.root()
.pipe(($el) => {
$el.find(NOTES_TAB_BUTTON).trigger('click');
return $el.find(NOTES_TEXT_AREA);
})
.should('be.visible');
+ return cy.root().find(NOTES_TAB_BUTTON);
};
export const getNotePreviewByNoteId = (noteId: string) => {
@@ -112,15 +113,16 @@ export const goToQueryTab = () => {
};
export const addNotesToTimeline = (notes: string) => {
- cy.wait(150);
- goToNotesTab();
- cy.get(NOTES_TEXT_AREA).type(notes);
- cy.root()
- .pipe(($el) => {
- $el.find(ADD_NOTE_BUTTON).trigger('click');
- return $el.find(NOTES_TAB_BUTTON).find('.euiBadge');
- })
- .should('have.text', '1');
+ goToNotesTab().then(() => {
+ cy.get(NOTES_TEXT_AREA).type(notes);
+ cy.root()
+ .pipe(($el) => {
+ $el.find(ADD_NOTE_BUTTON).trigger('click');
+ return $el.find(NOTES_TAB_BUTTON).find('.euiBadge');
+ })
+ .should('have.text', '1');
+ });
+
goToQueryTab();
goToNotesTab();
};
@@ -136,7 +138,7 @@ export const addFilter = (filter: TimelineFilter) => {
cy.get(SAVE_FILTER_BTN).click();
};
-export const addDataProvider = (filter: TimelineFilter) => {
+export const addDataProvider = (filter: TimelineFilter): Cypress.Chainable> => {
cy.get(TIMELINE_ADD_FIELD_BUTTON).click();
cy.get(TIMELINE_DATA_PROVIDER_FIELD).type(`${filter.field}{downarrow}{enter}`);
cy.get(TIMELINE_DATA_PROVIDER_OPERATOR).type(filter.operator);
@@ -231,13 +233,22 @@ export const openTimelineTemplateFromSettings = (id: string) => {
cy.get(TIMELINE_TITLE_BY_ID(id)).click({ force: true });
};
-export const openTimelineById = (timelineId: string) => {
+export const openTimelineById = (timelineId: string): Cypress.Chainable> => {
+ // Why are we checking for null if it is typed to 'string'? We don't currently validate the timeline response
+ // so technically we cannot guarantee that we will have the id. Changing the type to 'string | null' results in
+ // a lot of other changes being needed that would be best as part of a cleanup. Added a log, to give a dev a clue
+ // as to whether it's failing client or server side.
+ if (timelineId == null) {
+ cy.log('"timelineId" is null or undefined');
+ }
+
cy.root()
.pipe(($el) => {
$el.find(TIMELINE_TITLE_BY_ID(timelineId)).trigger('click');
return $el.find(QUERY_TAB_BUTTON).find('.euiBadge');
})
.should('be.visible');
+ return cy.root().find(TIMELINE_TITLE_BY_ID(timelineId));
};
export const pinFirstEvent = () => {
From 292b6d2638640db84c308916e7fba34e3a716d07 Mon Sep 17 00:00:00 2001
From: Constance
Date: Mon, 3 May 2021 14:29:36 -0700
Subject: [PATCH 002/106] [Enterprise Search] Refactor shared
SchemaAddFieldModal component (#99096)
* Move SchemaAddFieldModal to its own folder
+ misc test cleanup - remove unnecessary jest.spyOn in individual tests, it's already in the beforeEach
* i18n fixes
- Move constants to subfolder
- Fix various i18n IDs/var names - modal titles is not a fieldNote title
- Fix an i18n string that should be a FormattedMessage (could have grammar issues otherwise)
- Add missing i18n strings - labels (shared, AS will also use these as column headers) & placeholder
- Import order
* Move formatFieldName util to its own file
- simplify leading/trailing trimming regex
- add unit tests - primarily for documenting regexes & providing examples
* Refactor modal form submission
- See https://elastic.github.io/eui/#/layout/modal#forms-in-a-modal for documentation - form should be submitted via ID & EuiModalFooter form={id}
* Misc props cleanup
- Move optional props to bottom of type list
- Remove unnecessary props (default EUI behavior)
- Data test subj cleanup
- Add missing `disabled` passed prop, add unit test for disableForm prop
---
.../schema/add_field_modal/constants.tsx | 44 +++++
.../index.test.tsx} | 31 ++--
.../shared/schema/add_field_modal/index.tsx | 141 ++++++++++++++++
.../schema/add_field_modal/utils.test.ts | 26 +++
.../shared/schema/add_field_modal/utils.ts | 13 ++
.../applications/shared/schema/constants.ts | 39 +----
.../applications/shared/schema/index.ts | 2 +-
.../shared/schema/schema_add_field_modal.tsx | 152 ------------------
.../translations/translations/ja-JP.json | 8 +-
.../translations/translations/zh-CN.json | 8 +-
10 files changed, 257 insertions(+), 207 deletions(-)
create mode 100644 x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/constants.tsx
rename x-pack/plugins/enterprise_search/public/applications/shared/schema/{schema_add_field_modal.test.tsx => add_field_modal/index.test.tsx} (74%)
create mode 100644 x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/index.tsx
create mode 100644 x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/utils.test.ts
create mode 100644 x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/utils.ts
delete mode 100644 x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_add_field_modal.tsx
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/constants.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/constants.tsx
new file mode 100644
index 0000000000000..3cc43532eda73
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/constants.tsx
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n/react';
+
+export const FORM_ID = 'schemaAddFieldForm';
+
+export const ADD_FIELD_MODAL_TITLE = i18n.translate(
+ 'xpack.enterpriseSearch.schema.addFieldModal.title',
+ { defaultMessage: 'Add a new field' }
+);
+export const ADD_FIELD_MODAL_DESCRIPTION = i18n.translate(
+ 'xpack.enterpriseSearch.schema.addFieldModal.description',
+ { defaultMessage: 'Once added, a field cannot be removed from your schema.' }
+);
+export const ADD_FIELD_BUTTON = i18n.translate(
+ 'xpack.enterpriseSearch.schema.addFieldModal.addFieldButtonLabel',
+ { defaultMessage: 'Add field' }
+);
+
+export const FIELD_NAME_PLACEHOLDER = i18n.translate(
+ 'xpack.enterpriseSearch.schema.addFieldModal.fieldNamePlaceholder',
+ { defaultMessage: 'Enter a field name' }
+);
+export const FIELD_NAME_CORRECT_NOTE = i18n.translate(
+ 'xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.correct',
+ { defaultMessage: 'Field names can only contain lowercase letters, numbers, and underscores' }
+);
+export const FIELD_NAME_CORRECTED_NOTE = (correctedName: string) => (
+ {correctedName},
+ }}
+ />
+);
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_add_field_modal.test.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/index.test.tsx
similarity index 74%
rename from x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_add_field_modal.test.tsx
rename to x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/index.test.tsx
index 12bc61d723919..186560c694d24 100644
--- a/x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_add_field_modal.test.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/index.test.tsx
@@ -9,12 +9,13 @@ import React from 'react';
import { shallow, mount } from 'enzyme';
-import { EuiFieldText, EuiModal } from '@elastic/eui';
+import { EuiForm, EuiFieldText, EuiModal } from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n/react';
-import { FIELD_NAME_CORRECTED_PREFIX } from './constants';
-import { SchemaType } from './types';
+import { SchemaFieldTypeSelect } from '../index';
+import { SchemaType } from '../types';
-import { SchemaFieldTypeSelect, SchemaAddFieldModal } from './';
+import { SchemaAddFieldModal } from './';
describe('SchemaAddFieldModal', () => {
const addNewField = jest.fn();
@@ -53,8 +54,15 @@ describe('SchemaAddFieldModal', () => {
expect(setState).toHaveBeenCalledWith(false);
});
+ it('passes disabled state', () => {
+ const wrapper = shallow();
+
+ expect(wrapper.find('[data-test-subj="SchemaAddFieldNameField"]').prop('disabled')).toBe(true);
+ expect(wrapper.find('[data-test-subj="SchemaSelect"]').prop('disabled')).toBe(true);
+ expect(wrapper.find('[data-test-subj="SchemaAddFieldButton"]').prop('disabled')).toBe(true);
+ });
+
it('handles input change - with non-formatted name', () => {
- jest.spyOn(React, 'useState').mockImplementationOnce(setStateMock);
const wrapper = shallow();
const input = wrapper.find(EuiFieldText);
input.simulate('change', { currentTarget: { value: 'foobar' } });
@@ -65,16 +73,14 @@ describe('SchemaAddFieldModal', () => {
});
it('handles input change - with formatted name', () => {
- jest.spyOn(React, 'useState').mockImplementationOnce(setStateMock);
const wrapper = shallow();
const input = wrapper.find(EuiFieldText);
input.simulate('change', { currentTarget: { value: 'foo-bar' } });
- expect(wrapper.find('[data-test-subj="SchemaAddFieldNameRow"]').prop('helpText')).toEqual(
-
- {FIELD_NAME_CORRECTED_PREFIX} foo_bar
-
- );
+ const helpText = wrapper
+ .find('[data-test-subj="SchemaAddFieldNameRow"]')
+ .prop('helpText') as React.ReactElement;
+ expect(helpText.type).toEqual(FormattedMessage);
});
it('handles field type select change', () => {
@@ -87,10 +93,9 @@ describe('SchemaAddFieldModal', () => {
});
it('handles form submission', () => {
- jest.spyOn(React, 'useState').mockImplementationOnce(setStateMock);
const wrapper = shallow();
const preventDefault = jest.fn();
- wrapper.find('form').simulate('submit', { preventDefault });
+ wrapper.find(EuiForm).simulate('submit', { preventDefault });
expect(addNewField).toHaveBeenCalled();
expect(setState).toHaveBeenCalled();
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/index.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/index.tsx
new file mode 100644
index 0000000000000..902417d02665e
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/index.tsx
@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { ChangeEvent, FormEvent, useEffect, useState } from 'react';
+
+import {
+ EuiButton,
+ EuiButtonEmpty,
+ EuiFieldText,
+ EuiFlexGroup,
+ EuiFlexItem,
+ EuiForm,
+ EuiFormRow,
+ EuiModal,
+ EuiModalBody,
+ EuiModalFooter,
+ EuiModalHeader,
+ EuiModalHeaderTitle,
+ EuiSpacer,
+} from '@elastic/eui';
+
+import { CANCEL_BUTTON_LABEL } from '../../constants';
+import { FIELD_NAME, FIELD_TYPE } from '../constants';
+import { SchemaFieldTypeSelect } from '../index';
+import { SchemaType } from '../types';
+
+import {
+ ADD_FIELD_MODAL_TITLE,
+ ADD_FIELD_MODAL_DESCRIPTION,
+ ADD_FIELD_BUTTON,
+ FORM_ID,
+ FIELD_NAME_PLACEHOLDER,
+ FIELD_NAME_CORRECT_NOTE,
+ FIELD_NAME_CORRECTED_NOTE,
+} from './constants';
+import { formatFieldName } from './utils';
+
+interface Props {
+ addNewField(fieldName: string, newFieldType: string): void;
+ closeAddFieldModal(): void;
+ disableForm?: boolean;
+ addFieldFormErrors?: string[] | null;
+}
+
+export const SchemaAddFieldModal: React.FC = ({
+ addNewField,
+ addFieldFormErrors,
+ closeAddFieldModal,
+ disableForm,
+}) => {
+ const [loading, setLoading] = useState(false);
+ const [newFieldType, updateNewFieldType] = useState(SchemaType.Text);
+ const [formattedFieldName, setFormattedFieldName] = useState('');
+ const [rawFieldName, setRawFieldName] = useState('');
+
+ useEffect(() => {
+ if (addFieldFormErrors) setLoading(false);
+ }, [addFieldFormErrors]);
+
+ const handleChange = ({ currentTarget: { value } }: ChangeEvent) => {
+ setRawFieldName(value);
+ setFormattedFieldName(formatFieldName(value));
+ };
+
+ const submitForm = (e: FormEvent) => {
+ e.preventDefault();
+ addNewField(formattedFieldName, newFieldType);
+ setLoading(true);
+ };
+
+ const fieldNameNote =
+ rawFieldName !== formattedFieldName
+ ? FIELD_NAME_CORRECTED_NOTE(formattedFieldName)
+ : FIELD_NAME_CORRECT_NOTE;
+
+ return (
+
+
+ {ADD_FIELD_MODAL_TITLE}
+
+
+ {ADD_FIELD_MODAL_DESCRIPTION}
+
+
+
+
+
+
+
+
+
+
+ updateNewFieldType(type)}
+ disabled={disableForm}
+ data-test-subj="SchemaSelect"
+ />
+
+
+
+
+
+
+ {CANCEL_BUTTON_LABEL}
+
+ {ADD_FIELD_BUTTON}
+
+
+
+ );
+};
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/utils.test.ts b/x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/utils.test.ts
new file mode 100644
index 0000000000000..917478cd0aa03
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/utils.test.ts
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { formatFieldName } from './utils';
+
+describe('formatFieldName', () => {
+ it('removes leading and trailing spaces', () => {
+ expect(formatFieldName(' helloworld ')).toEqual('helloworld');
+ });
+
+ it('converts all special characters to underscores', () => {
+ expect(formatFieldName('hello!#@$123---world')).toEqual('hello_123_world');
+ });
+
+ it('strips leading and trailing special characters/underscores', () => {
+ expect(formatFieldName('!!helloworld__')).toEqual('helloworld');
+ });
+
+ it('lowercases any caps', () => {
+ expect(formatFieldName('HELLO_WORLD')).toEqual('hello_world');
+ });
+});
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/utils.ts b/x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/utils.ts
new file mode 100644
index 0000000000000..6242a337871b1
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/schema/add_field_modal/utils.ts
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const formatFieldName = (rawName: string) =>
+ rawName
+ .trim()
+ .replace(/[^a-zA-Z0-9]+/g, '_')
+ .replace(/^(_+)|(_+)$/g, '')
+ .toLowerCase();
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/constants.ts b/x-pack/plugins/enterprise_search/public/applications/shared/schema/constants.ts
index 3791626f54398..e85050aa22ef1 100644
--- a/x-pack/plugins/enterprise_search/public/applications/shared/schema/constants.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/schema/constants.ts
@@ -7,40 +7,13 @@
import { i18n } from '@kbn/i18n';
-export const FIELD_NAME_CORRECT_NOTE = i18n.translate(
- 'xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.correct',
- {
- defaultMessage: 'Field names can only contain lowercase letters, numbers, and underscores',
- }
-);
-
-export const FIELD_NAME_CORRECTED_PREFIX = i18n.translate(
- 'xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.corrected',
- {
- defaultMessage: 'The field will be named',
- }
-);
-
-export const FIELD_NAME_MODAL_TITLE = i18n.translate(
- 'xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.title',
- {
- defaultMessage: 'Add a New Field',
- }
-);
+export const FIELD_NAME = i18n.translate('xpack.enterpriseSearch.schema.fieldNameLabel', {
+ defaultMessage: 'Field name',
+});
-export const FIELD_NAME_MODAL_DESCRIPTION = i18n.translate(
- 'xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.description',
- {
- defaultMessage: 'Once added, a field cannot be removed from your schema.',
- }
-);
-
-export const FIELD_NAME_MODAL_ADD_FIELD = i18n.translate(
- 'xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.addField',
- {
- defaultMessage: 'Add field',
- }
-);
+export const FIELD_TYPE = i18n.translate('xpack.enterpriseSearch.schema.fieldTypeLabel', {
+ defaultMessage: 'Field type',
+});
export const ERROR_TABLE_ID_HEADER = i18n.translate(
'xpack.enterpriseSearch.schema.errorsTable.heading.id',
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/index.ts b/x-pack/plugins/enterprise_search/public/applications/shared/schema/index.ts
index b47d7ccabb57e..657ddef64b7ca 100644
--- a/x-pack/plugins/enterprise_search/public/applications/shared/schema/index.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/schema/index.ts
@@ -5,6 +5,6 @@
* 2.0.
*/
-export { SchemaAddFieldModal } from './schema_add_field_modal';
+export { SchemaAddFieldModal } from './add_field_modal';
export { SchemaFieldTypeSelect } from './field_type_select';
export { SchemaErrorsCallout } from './errors_callout';
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_add_field_modal.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_add_field_modal.tsx
deleted file mode 100644
index e6f7bffc2d83f..0000000000000
--- a/x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_add_field_modal.tsx
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import React, { ChangeEvent, FormEvent, useEffect, useState } from 'react';
-
-import {
- EuiButton,
- EuiButtonEmpty,
- EuiFieldText,
- EuiFlexGroup,
- EuiFlexItem,
- EuiForm,
- EuiFormRow,
- EuiModal,
- EuiModalBody,
- EuiModalFooter,
- EuiModalHeader,
- EuiModalHeaderTitle,
- EuiSpacer,
-} from '@elastic/eui';
-
-import { CANCEL_BUTTON_LABEL } from '../constants';
-
-import {
- FIELD_NAME_CORRECT_NOTE,
- FIELD_NAME_CORRECTED_PREFIX,
- FIELD_NAME_MODAL_TITLE,
- FIELD_NAME_MODAL_DESCRIPTION,
- FIELD_NAME_MODAL_ADD_FIELD,
-} from './constants';
-import { SchemaType } from './types';
-
-import { SchemaFieldTypeSelect } from './';
-
-interface ISchemaAddFieldModalProps {
- disableForm?: boolean;
- addFieldFormErrors?: string[] | null;
- addNewField(fieldName: string, newFieldType: string): void;
- closeAddFieldModal(): void;
-}
-
-export const SchemaAddFieldModal: React.FC = ({
- addNewField,
- addFieldFormErrors,
- closeAddFieldModal,
- disableForm,
-}) => {
- const [loading, setLoading] = useState(false);
- const [newFieldType, updateNewFieldType] = useState(SchemaType.Text);
- const [formattedFieldName, setFormattedFieldName] = useState('');
- const [rawFieldName, setRawFieldName] = useState('');
-
- useEffect(() => {
- if (addFieldFormErrors) setLoading(false);
- }, [addFieldFormErrors]);
-
- const handleChange = ({ currentTarget: { value } }: ChangeEvent) => {
- setRawFieldName(value);
- setFormattedFieldName(formatFieldName(value));
- };
-
- const submitForm = (e: FormEvent) => {
- e.preventDefault();
- addNewField(formattedFieldName, newFieldType);
- setLoading(true);
- };
-
- const fieldNameNote =
- rawFieldName !== formattedFieldName ? (
- <>
- {FIELD_NAME_CORRECTED_PREFIX} {formattedFieldName}
- >
- ) : (
- FIELD_NAME_CORRECT_NOTE
- );
-
- return (
-
-
-
- );
-};
-
-const formatFieldName = (rawName: string) =>
- rawName
- .trim()
- .replace(/[^a-zA-Z0-9]+/g, '_')
- .replace(/^[^a-zA-Z0-9]+/, '')
- .replace(/[^a-zA-Z0-9]+$/, '')
- .toLowerCase();
diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json
index a3fef7f6bbf09..a20cf5a94312a 100644
--- a/x-pack/plugins/translations/translations/ja-JP.json
+++ b/x-pack/plugins/translations/translations/ja-JP.json
@@ -7619,11 +7619,11 @@
"xpack.enterpriseSearch.overview.subheading": "開始する製品を選択します。",
"xpack.enterpriseSearch.productName": "エンタープライズサーチ",
"xpack.enterpriseSearch.readOnlyMode.warning": "エンタープライズ サーチは読み取り専用モードです。作成、編集、削除などの変更を実行できません。",
- "xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.addField": "フィールドの追加",
+ "xpack.enterpriseSearch.schema.addFieldModal.addFieldButtonLabel": "フィールドの追加",
+ "xpack.enterpriseSearch.schema.addFieldModal.description": "追加すると、フィールドはスキーマから削除されます。",
+ "xpack.enterpriseSearch.schema.addFieldModal.title": "新しいフィールドを追加",
"xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.correct": "フィールド名には、小文字、数字、アンダースコアのみを使用できます。",
- "xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.corrected": "フィールド名が変更されます",
- "xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.description": "追加すると、フィールドはスキーマから削除されます。",
- "xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.title": "新しいフィールドを追加",
+ "xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.corrected": "フィールド名が変更されます {correctedName}",
"xpack.enterpriseSearch.schema.errorsCallout.buttonLabel": "エラーを表示",
"xpack.enterpriseSearch.schema.errorsTable.control.review": "見直し",
"xpack.enterpriseSearch.schema.errorsTable.heading.error": "エラー",
diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json
index 413983c7c8f83..c4c18a18d37e2 100644
--- a/x-pack/plugins/translations/translations/zh-CN.json
+++ b/x-pack/plugins/translations/translations/zh-CN.json
@@ -7689,11 +7689,11 @@
"xpack.enterpriseSearch.overview.subheading": "选择产品开始使用。",
"xpack.enterpriseSearch.productName": "企业搜索",
"xpack.enterpriseSearch.readOnlyMode.warning": "企业搜索处于只读模式。您将无法执行更改,例如创建、编辑或删除。",
- "xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.addField": "添加字段",
+ "xpack.enterpriseSearch.schema.addFieldModal.addFieldButtonLabel": "添加字段",
+ "xpack.enterpriseSearch.schema.addFieldModal.description": "字段添加后,将无法从架构中删除。",
+ "xpack.enterpriseSearch.schema.addFieldModal.title": "添加新字段",
"xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.correct": "字段名称只能包含小写字母、数字和下划线",
- "xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.corrected": "该字段将被命名",
- "xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.description": "字段添加后,将无法从架构中删除。",
- "xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.title": "添加新字段",
+ "xpack.enterpriseSearch.schema.addFieldModal.fieldNameNote.corrected": "该字段将被命名 {correctedName}",
"xpack.enterpriseSearch.schema.errorsCallout.buttonLabel": "查看错误",
"xpack.enterpriseSearch.schema.errorsTable.control.review": "复查",
"xpack.enterpriseSearch.schema.errorsTable.heading.error": "错误",
From 58b3c1bf130c0fcba55ae24a32f211006e6507f0 Mon Sep 17 00:00:00 2001
From: Aaron Caldwell
Date: Mon, 3 May 2021 15:30:19 -0600
Subject: [PATCH 003/106] Fix malformed geo alerts call to `transformResults`
(#98094)
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
.../alert_types/geo_containment/alert_type.ts | 1 +
.../geo_containment/geo_containment.ts | 6 +-
.../tests/es_sample_response.json | 383 +++++++++-------
.../tests/es_sample_response_shapes.json | 428 ++++++++++++++++++
.../es_sample_response_with_nesting.json | 308 ++++++-------
.../tests/geo_containment.test.ts | 315 ++++++++++---
6 files changed, 1068 insertions(+), 373 deletions(-)
create mode 100644 x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/es_sample_response_shapes.json
diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts
index 6015881270405..e3d379f2869b9 100644
--- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts
+++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts
@@ -120,6 +120,7 @@ export interface GeoContainmentParams extends AlertTypeParams {
export interface GeoContainmentState extends AlertTypeState {
shapesFilters: Record;
shapesIdsNamesMap: Record;
+ prevLocationMap: Record;
}
export interface GeoContainmentInstanceState extends AlertInstanceState {
location: number[];
diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts
index 15a6564395c16..754af920b009e 100644
--- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts
+++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts
@@ -16,6 +16,7 @@ import {
GeoContainmentInstanceState,
GeoContainmentAlertType,
GeoContainmentInstanceContext,
+ GeoContainmentState,
} from './alert_type';
export type LatestEntityLocation = GeoContainmentInstanceState;
@@ -141,7 +142,7 @@ export const getGeoContainmentExecutor = (log: Logger): GeoContainmentAlertType[
params,
alertId,
state,
- }) {
+ }): Promise {
const { shapesFilters, shapesIdsNamesMap } = state.shapesFilters
? state
: await getShapesFilters(
@@ -176,8 +177,7 @@ export const getGeoContainmentExecutor = (log: Logger): GeoContainmentAlertType[
}
const currLocationMap: Map = transformResults(
- // @ts-expect-error body doesn't exist on currentIntervalResults
- currentIntervalResults?.body,
+ currentIntervalResults,
params.dateField,
params.geoField
);
diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/es_sample_response.json b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/es_sample_response.json
index 70edbd09aa5a1..b5751e527df4d 100644
--- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/es_sample_response.json
+++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/es_sample_response.json
@@ -1,167 +1,246 @@
{
- "took" : 2760,
- "timed_out" : false,
- "_shards" : {
- "total" : 1,
- "successful" : 1,
- "skipped" : 0,
- "failed" : 0
- },
- "hits" : {
- "total" : {
- "value" : 10000,
- "relation" : "gte"
+ "body": {
+ "took":194,
+ "timed_out":false,
+ "_shards":{
+ "total":1,
+ "successful":1,
+ "skipped":0,
+ "failed":0
},
- "max_score" : 0.0,
- "hits" : []
- },
- "aggregations" : {
- "shapes" : {
- "meta" : { },
- "buckets" : {
- "0DrJu3QB6yyY-xQxv6Ip" : {
- "doc_count" : 1047,
- "entitySplit" : {
- "doc_count_error_upper_bound" : 0,
- "sum_other_doc_count" : 957,
- "buckets" : [
- {
- "key" : "936",
- "doc_count" : 9,
- "entityHits" : {
- "hits" : {
- "total" : {
- "value" : 9,
- "relation" : "eq"
- },
- "max_score" : null,
- "hits" : [
- {
- "_index" : "flight_tracks",
- "_id" : "N-ng1XQB6yyY-xQxnGSM",
- "_score" : null,
- "fields" : {
- "@timestamp" : [
- "2020-09-28T18:01:41.190Z"
- ],
- "location" : [
- "40.62806099653244, -82.8814151789993"
- ],
- "entity_id" : [
- "936"
+ "hits":{
+ "total":{
+ "value":18,
+ "relation":"eq"
+ },
+ "max_score":null,
+ "hits":[
+
+ ]
+ },
+ "aggregations":{
+ "shapes":{
+ "meta":{
+
+ },
+ "buckets":{
+ "k1ATGXkBsFLYN2Tj6AAk":{
+ "doc_count":0,
+ "entitySplit":{
+ "doc_count_error_upper_bound":0,
+ "sum_other_doc_count":0,
+ "buckets":[
+
+ ]
+ }
+ },
+ "kFATGXkBsFLYN2Tj6AAk":{
+ "doc_count":2,
+ "entitySplit":{
+ "doc_count_error_upper_bound":0,
+ "sum_other_doc_count":0,
+ "buckets":[
+ {
+ "key":"0",
+ "doc_count":1,
+ "entityHits":{
+ "hits":{
+ "total":{
+ "value":1,
+ "relation":"eq"
+ },
+ "max_score":null,
+ "hits":[
+ {
+ "_index":"tracks",
+ "_id":"ZVBoGXkBsFLYN2Tj1wmV",
+ "_score":null,
+ "fields":{
+ "@timestamp":[
+ "2021-04-28T16:56:11.923Z"
+ ],
+ "location":[
+ "40.751759740523994, -73.99018926545978"
+ ],
+ "entity_id":[
+ "0"
+ ]
+ },
+ "sort":[
+ 1619628971923
]
- },
- "sort" : [
- 1601316101190
- ]
- }
- ]
+ }
+ ]
+ }
}
- }
- },
- {
- "key" : "AAL2019",
- "doc_count" : 9,
- "entityHits" : {
- "hits" : {
- "total" : {
- "value" : 9,
- "relation" : "eq"
- },
- "max_score" : null,
- "hits" : [
- {
- "_index" : "flight_tracks",
- "_id" : "iOng1XQB6yyY-xQxnGSM",
- "_score" : null,
- "fields" : {
- "@timestamp" : [
- "2020-09-28T18:01:41.191Z"
- ],
- "location" : [
- "39.006176185794175, -82.22068064846098"
- ],
- "entity_id" : [
- "AAL2019"
+ },
+ {
+ "key":"1",
+ "doc_count":1,
+ "entityHits":{
+ "hits":{
+ "total":{
+ "value":1,
+ "relation":"eq"
+ },
+ "max_score":null,
+ "hits":[
+ {
+ "_index":"tracks",
+ "_id":"ZlBoGXkBsFLYN2Tj1wmV",
+ "_score":null,
+ "fields":{
+ "@timestamp":[
+ "2021-04-28T16:56:11.923Z"
+ ],
+ "location":[
+ "40.75449890457094, -73.99561604484916"
+ ],
+ "entity_id":[
+ "1"
+ ]
+ },
+ "sort":[
+ 1619628971923
]
- },
- "sort" : [
- 1601316101191
- ]
- }
- ]
+ }
+ ]
+ }
}
}
- },
- {
- "key" : "AAL2323",
- "doc_count" : 9,
- "entityHits" : {
- "hits" : {
- "total" : {
- "value" : 9,
- "relation" : "eq"
- },
- "max_score" : null,
- "hits" : [
- {
- "_index" : "flight_tracks",
- "_id" : "n-ng1XQB6yyY-xQxnGSM",
- "_score" : null,
- "fields" : {
- "@timestamp" : [
- "2020-09-28T18:01:41.191Z"
- ],
- "location" : [
- "41.6677269525826, -84.71324851736426"
- ],
- "entity_id" : [
- "AAL2323"
+ ]
+ }
+ },
+ "kVATGXkBsFLYN2Tj6AAk":{
+ "doc_count":0,
+ "entitySplit":{
+ "doc_count_error_upper_bound":0,
+ "sum_other_doc_count":0,
+ "buckets":[
+
+ ]
+ }
+ },
+ "lVATGXkBsFLYN2Tj6AAk":{
+ "doc_count":0,
+ "entitySplit":{
+ "doc_count_error_upper_bound":0,
+ "sum_other_doc_count":0,
+ "buckets":[
+
+ ]
+ }
+ },
+ "other":{
+ "doc_count":15,
+ "entitySplit":{
+ "doc_count_error_upper_bound":0,
+ "sum_other_doc_count":0,
+ "buckets":[
+ {
+ "key":"2",
+ "doc_count":6,
+ "entityHits":{
+ "hits":{
+ "total":{
+ "value":6,
+ "relation":"eq"
+ },
+ "max_score":null,
+ "hits":[
+ {
+ "_index":"tracks",
+ "_id":"Z1BoGXkBsFLYN2Tj1wmV",
+ "_score":null,
+ "fields":{
+ "@timestamp":[
+ "2021-04-28T16:56:11.923Z"
+ ],
+ "location":[
+ "40.7667087810114, -73.98662586696446"
+ ],
+ "entity_id":[
+ "2"
+ ]
+ },
+ "sort":[
+ 1619628971923
]
- },
- "sort" : [
- 1601316101191
- ]
- }
- ]
+ }
+ ]
+ }
}
- }
- },
- {
- "key" : "ABD5250",
- "doc_count" : 9,
- "entityHits" : {
- "hits" : {
- "total" : {
- "value" : 9,
- "relation" : "eq"
- },
- "max_score" : null,
- "hits" : [
- {
- "_index" : "flight_tracks",
- "_id" : "GOng1XQB6yyY-xQxnGWM",
- "_score" : null,
- "fields" : {
- "@timestamp" : [
- "2020-09-28T18:01:41.192Z"
- ],
- "location" : [
- "39.07997465226799, 6.073727197945118"
- ],
- "entity_id" : [
- "ABD5250"
+ },
+ {
+ "key":"1",
+ "doc_count":5,
+ "entityHits":{
+ "hits":{
+ "total":{
+ "value":5,
+ "relation":"eq"
+ },
+ "max_score":null,
+ "hits":[
+ {
+ "_index":"tracks",
+ "_id":"Y1BoGXkBsFLYN2TjsAlp",
+ "_score":null,
+ "fields":{
+ "@timestamp":[
+ "2021-04-28T16:56:01.896Z"
+ ],
+ "location":[
+ "40.755913141183555, -73.99459345266223"
+ ],
+ "entity_id":[
+ "1"
+ ]
+ },
+ "sort":[
+ 1619628961896
+ ]
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key":"0",
+ "doc_count":4,
+ "entityHits":{
+ "hits":{
+ "total":{
+ "value":4,
+ "relation":"eq"
+ },
+ "max_score":null,
+ "hits":[
+ {
+ "_index":"tracks",
+ "_id":"YlBoGXkBsFLYN2TjsAlp",
+ "_score":null,
+ "fields":{
+ "@timestamp":[
+ "2021-04-28T16:56:01.896Z"
+ ],
+ "location":[
+ "40.7506317878142, -73.98968475870788"
+ ],
+ "entity_id":[
+ "0"
+ ]
+ },
+ "sort":[
+ 1619628961896
]
- },
- "sort" : [
- 1601316101192
- ]
- }
- ]
+ }
+ ]
+ }
}
}
- }
- ]
+ ]
+ }
}
}
}
diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/es_sample_response_shapes.json b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/es_sample_response_shapes.json
new file mode 100644
index 0000000000000..dfe5a8b7f6ec1
--- /dev/null
+++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/es_sample_response_shapes.json
@@ -0,0 +1,428 @@
+{
+ "body": {
+ "took":1,
+ "timed_out":false,
+ "_shards":{
+ "total":1,
+ "successful":1,
+ "skipped":0,
+ "failed":0
+ },
+ "hits":{
+ "total":{
+ "value":11,
+ "relation":"eq"
+ },
+ "max_score":1,
+ "hits":[
+ {
+ "_index":"manhattan_boundaries",
+ "_id":"waFXH3kBi9P-_6qn8c8A",
+ "_score":1,
+ "_source":{
+ "coordinates":{
+ "type":"Polygon",
+ "coordinates":[
+ [
+ [
+ -73.96772861480713,
+ 40.76060200607076
+ ],
+ [
+ -73.96805047988892,
+ 40.7601631739201
+ ],
+ [
+ -73.96732091903687,
+ 40.7598706175435
+ ],
+ [
+ -73.96693468093872,
+ 40.760471982031845
+ ],
+ [
+ -73.96759986877441,
+ 40.76078078870895
+ ],
+ [
+ -73.96772861480713,
+ 40.76060200607076
+ ]
+ ]
+ ]
+ }
+ }
+ },
+ {
+ "_index":"manhattan_boundaries",
+ "_id":"wqFXH3kBi9P-_6qn8c8A",
+ "_score":1,
+ "_source":{
+ "coordinates":{
+ "type":"Polygon",
+ "coordinates":[
+ [
+ [
+ -73.97641897201538,
+ 40.75618104606283
+ ],
+ [
+ -73.97865056991577,
+ 40.75371038152863
+ ],
+ [
+ -73.97770643234252,
+ 40.75323899431278
+ ],
+ [
+ -73.9788007736206,
+ 40.75187357799962
+ ],
+ [
+ -73.97671937942503,
+ 40.751060816881505
+ ],
+ [
+ -73.97500276565552,
+ 40.75377540019266
+ ],
+ [
+ -73.97639751434325,
+ 40.75460438258571
+ ],
+ [
+ -73.9755392074585,
+ 40.755985996937774
+ ],
+ [
+ -73.97641897201538,
+ 40.75618104606283
+ ]
+ ]
+ ]
+ }
+ }
+ },
+ {
+ "_index":"manhattan_boundaries",
+ "_id":"w6FXH3kBi9P-_6qn8c8A",
+ "_score":1,
+ "_source":{
+ "coordinates":{
+ "type":"Polygon",
+ "coordinates":[
+ [
+ [
+ -73.98592472076415,
+ 40.75957805987928
+ ],
+ [
+ -73.98695468902588,
+ 40.75566091379097
+ ],
+ [
+ -73.98573160171509,
+ 40.75553088008716
+ ],
+ [
+ -73.98465871810913,
+ 40.75946428710659
+ ],
+ [
+ -73.98592472076415,
+ 40.75957805987928
+ ]
+ ]
+ ]
+ }
+ }
+ },
+ {
+ "_index":"manhattan_boundaries",
+ "_id":"xKFXH3kBi9P-_6qn8c8A",
+ "_score":1,
+ "_source":{
+ "coordinates":{
+ "type":"Polygon",
+ "coordinates":[
+ [
+ [
+ -73.9894437789917,
+ 40.75161349552273
+ ],
+ [
+ -73.98914337158203,
+ 40.75206863918968
+ ],
+ [
+ -73.99575233459473,
+ 40.75486445336327
+ ],
+ [
+ -73.99819850921631,
+ 40.75148345390278
+ ],
+ [
+ -73.9914608001709,
+ 40.74881754464601
+ ],
+ [
+ -73.9894437789917,
+ 40.75161349552273
+ ]
+ ]
+ ]
+ }
+ }
+ },
+ {
+ "_index":"manhattan_boundaries",
+ "_id":"xaFXH3kBi9P-_6qn8c8A",
+ "_score":1,
+ "_source":{
+ "coordinates":{
+ "type":"Polygon",
+ "coordinates":[
+ [
+ [
+ -73.96914482116699,
+ 40.75874913950493
+ ],
+ [
+ -73.96946668624878,
+ 40.758229027325804
+ ],
+ [
+ -73.96856546401978,
+ 40.75793646243674
+ ],
+ [
+ -73.96824359893799,
+ 40.75845657690492
+ ],
+ [
+ -73.96914482116699,
+ 40.75874913950493
+ ]
+ ]
+ ]
+ }
+ }
+ },
+ {
+ "_index":"manhattan_boundaries",
+ "_id":"xqFXH3kBi9P-_6qn8c8A",
+ "_score":1,
+ "_source":{
+ "coordinates":{
+ "type":"Polygon",
+ "coordinates":[
+ [
+ [
+ -73.96953105926514,
+ 40.7581640130173
+ ],
+ [
+ -73.9699387550354,
+ 40.75749761268889
+ ],
+ [
+ -73.96923065185547,
+ 40.75728631362887
+ ],
+ [
+ -73.96862983703613,
+ 40.757920208794026
+ ],
+ [
+ -73.96953105926514,
+ 40.7581640130173
+ ]
+ ]
+ ]
+ }
+ }
+ },
+ {
+ "_index":"manhattan_boundaries",
+ "_id":"x6FXH3kBi9P-_6qn8c8A",
+ "_score":1,
+ "_source":{
+ "coordinates":{
+ "type":"Polygon",
+ "coordinates":[
+ [
+ [
+ -73.97045373916626,
+ 40.75679869785023
+ ],
+ [
+ -73.97079706192015,
+ 40.75629482445485
+ ],
+ [
+ -73.96998167037964,
+ 40.756051013376364
+ ],
+ [
+ -73.96961688995361,
+ 40.756554888619675
+ ],
+ [
+ -73.97045373916626,
+ 40.75679869785023
+ ]
+ ]
+ ]
+ }
+ }
+ },
+ {
+ "_index":"manhattan_boundaries",
+ "_id":"yKFXH3kBi9P-_6qn8c8A",
+ "_score":1,
+ "_source":{
+ "coordinates":{
+ "type":"Polygon",
+ "coordinates":[
+ [
+ [
+ -73.98412227630615,
+ 40.75479943576424
+ ],
+ [
+ -73.98498058319092,
+ 40.75351532515499
+ ],
+ [
+ -73.98191213607788,
+ 40.75219867966512
+ ],
+ [
+ -73.9808177947998,
+ 40.75340154200611
+ ],
+ [
+ -73.98412227630615,
+ 40.75479943576424
+ ]
+ ]
+ ]
+ }
+ }
+ },
+ {
+ "_index":"manhattan_boundaries",
+ "_id":"yaFXH3kBi9P-_6qn8c8A",
+ "_score":1,
+ "_source":{
+ "coordinates":{
+ "type":"Polygon",
+ "coordinates":[
+ [
+ [
+ -73.99725437164307,
+ 40.74498104863726
+ ],
+ [
+ -74.00386333465576,
+ 40.736136757139285
+ ],
+ [
+ -73.99703979492188,
+ 40.73334015558748
+ ],
+ [
+ -73.9897871017456,
+ 40.74153451605774
+ ],
+ [
+ -73.99725437164307,
+ 40.74498104863726
+ ]
+ ]
+ ]
+ }
+ }
+ },
+ {
+ "_index":"manhattan_boundaries",
+ "_id":"yqFXH3kBi9P-_6qn8c8A",
+ "_score":1,
+ "_source":{
+ "coordinates":{
+ "type":"Polygon",
+ "coordinates":[
+ [
+ [
+ -73.98830652236938,
+ 40.75075196505171
+ ],
+ [
+ -73.9885640144348,
+ 40.74759834321152
+ ],
+ [
+ -73.98761987686157,
+ 40.747582087041366
+ ],
+ [
+ -73.98751258850098,
+ 40.74816730666263
+ ],
+ [
+ -73.98807048797607,
+ 40.74826484276548
+ ],
+ [
+ -73.9875340461731,
+ 40.75075196505171
+ ],
+ [
+ -73.98830652236938,
+ 40.75075196505171
+ ]
+ ]
+ ]
+ }
+ }
+ },
+ {
+ "_index":"manhattan_boundaries",
+ "_id":"y6FXH3kBi9P-_6qn8c8A",
+ "_score":1,
+ "_source":{
+ "coordinates":{
+ "type":"Polygon",
+ "coordinates":[
+ [
+ [
+ -73.9824914932251,
+ 40.7467692734681
+ ],
+ [
+ -73.98356437683105,
+ 40.7452411570555
+ ],
+ [
+ -73.9813756942749,
+ 40.74446082874893
+ ],
+ [
+ -73.98030281066895,
+ 40.745696344339564
+ ],
+ [
+ -73.9824914932251,
+ 40.7467692734681
+ ]
+ ]
+ ]
+ }
+ }
+ }
+ ]
+ }
+ }
+}
+
diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/es_sample_response_with_nesting.json b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/es_sample_response_with_nesting.json
index a4b7b6872b341..9baf58465c38e 100644
--- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/es_sample_response_with_nesting.json
+++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/es_sample_response_with_nesting.json
@@ -1,167 +1,169 @@
{
- "took" : 2760,
- "timed_out" : false,
- "_shards" : {
- "total" : 1,
- "successful" : 1,
- "skipped" : 0,
- "failed" : 0
- },
- "hits" : {
- "total" : {
- "value" : 10000,
- "relation" : "gte"
+ "body": {
+ "took" : 2760,
+ "timed_out" : false,
+ "_shards" : {
+ "total" : 1,
+ "successful" : 1,
+ "skipped" : 0,
+ "failed" : 0
},
- "max_score" : 0.0,
- "hits" : []
- },
- "aggregations" : {
- "shapes" : {
- "meta" : { },
- "buckets" : {
- "0DrJu3QB6yyY-xQxv6Ip" : {
- "doc_count" : 1047,
- "entitySplit" : {
- "doc_count_error_upper_bound" : 0,
- "sum_other_doc_count" : 957,
- "buckets" : [
- {
- "key" : "936",
- "doc_count" : 9,
- "entityHits" : {
- "hits" : {
- "total" : {
- "value" : 9,
- "relation" : "eq"
- },
- "max_score" : null,
- "hits" : [
- {
- "_index" : "flight_tracks",
- "_id" : "N-ng1XQB6yyY-xQxnGSM",
- "_score" : null,
- "fields" : {
- "time_data.@timestamp" : [
- "2020-09-28T18:01:41.190Z"
- ],
- "geo.coords.location" : [
- "40.62806099653244, -82.8814151789993"
- ],
- "entity_id" : [
- "936"
+ "hits" : {
+ "total" : {
+ "value" : 10000,
+ "relation" : "gte"
+ },
+ "max_score" : 0.0,
+ "hits" : []
+ },
+ "aggregations" : {
+ "shapes" : {
+ "meta" : { },
+ "buckets" : {
+ "0DrJu3QB6yyY-xQxv6Ip" : {
+ "doc_count" : 1047,
+ "entitySplit" : {
+ "doc_count_error_upper_bound" : 0,
+ "sum_other_doc_count" : 957,
+ "buckets" : [
+ {
+ "key" : "936",
+ "doc_count" : 9,
+ "entityHits" : {
+ "hits" : {
+ "total" : {
+ "value" : 9,
+ "relation" : "eq"
+ },
+ "max_score" : null,
+ "hits" : [
+ {
+ "_index" : "flight_tracks",
+ "_id" : "N-ng1XQB6yyY-xQxnGSM",
+ "_score" : null,
+ "fields" : {
+ "time_data.@timestamp" : [
+ "2020-09-28T18:01:41.190Z"
+ ],
+ "geo.coords.location" : [
+ "40.62806099653244, -82.8814151789993"
+ ],
+ "entity_id" : [
+ "936"
+ ]
+ },
+ "sort" : [
+ 1601316101190
]
- },
- "sort" : [
- 1601316101190
- ]
- }
- ]
+ }
+ ]
+ }
}
- }
- },
- {
- "key" : "AAL2019",
- "doc_count" : 9,
- "entityHits" : {
- "hits" : {
- "total" : {
- "value" : 9,
- "relation" : "eq"
- },
- "max_score" : null,
- "hits" : [
- {
- "_index" : "flight_tracks",
- "_id" : "iOng1XQB6yyY-xQxnGSM",
- "_score" : null,
- "fields" : {
- "time_data.@timestamp" : [
- "2020-09-28T18:01:41.191Z"
- ],
- "geo.coords.location" : [
- "39.006176185794175, -82.22068064846098"
- ],
- "entity_id" : [
- "AAL2019"
+ },
+ {
+ "key" : "AAL2019",
+ "doc_count" : 9,
+ "entityHits" : {
+ "hits" : {
+ "total" : {
+ "value" : 9,
+ "relation" : "eq"
+ },
+ "max_score" : null,
+ "hits" : [
+ {
+ "_index" : "flight_tracks",
+ "_id" : "iOng1XQB6yyY-xQxnGSM",
+ "_score" : null,
+ "fields" : {
+ "time_data.@timestamp" : [
+ "2020-09-28T18:01:41.191Z"
+ ],
+ "geo.coords.location" : [
+ "39.006176185794175, -82.22068064846098"
+ ],
+ "entity_id" : [
+ "AAL2019"
+ ]
+ },
+ "sort" : [
+ 1601316101191
]
- },
- "sort" : [
- 1601316101191
- ]
- }
- ]
+ }
+ ]
+ }
}
- }
- },
- {
- "key" : "AAL2323",
- "doc_count" : 9,
- "entityHits" : {
- "hits" : {
- "total" : {
- "value" : 9,
- "relation" : "eq"
- },
- "max_score" : null,
- "hits" : [
- {
- "_index" : "flight_tracks",
- "_id" : "n-ng1XQB6yyY-xQxnGSM",
- "_score" : null,
- "fields" : {
- "time_data.@timestamp" : [
- "2020-09-28T18:01:41.191Z"
- ],
- "geo.coords.location" : [
- "41.6677269525826, -84.71324851736426"
- ],
- "entity_id" : [
- "AAL2323"
+ },
+ {
+ "key" : "AAL2323",
+ "doc_count" : 9,
+ "entityHits" : {
+ "hits" : {
+ "total" : {
+ "value" : 9,
+ "relation" : "eq"
+ },
+ "max_score" : null,
+ "hits" : [
+ {
+ "_index" : "flight_tracks",
+ "_id" : "n-ng1XQB6yyY-xQxnGSM",
+ "_score" : null,
+ "fields" : {
+ "time_data.@timestamp" : [
+ "2020-09-28T18:01:41.191Z"
+ ],
+ "geo.coords.location" : [
+ "41.6677269525826, -84.71324851736426"
+ ],
+ "entity_id" : [
+ "AAL2323"
+ ]
+ },
+ "sort" : [
+ 1601316101191
]
- },
- "sort" : [
- 1601316101191
- ]
- }
- ]
+ }
+ ]
+ }
}
- }
- },
- {
- "key" : "ABD5250",
- "doc_count" : 9,
- "entityHits" : {
- "hits" : {
- "total" : {
- "value" : 9,
- "relation" : "eq"
- },
- "max_score" : null,
- "hits" : [
- {
- "_index" : "flight_tracks",
- "_id" : "GOng1XQB6yyY-xQxnGWM",
- "_score" : null,
- "fields" : {
- "time_data.@timestamp" : [
- "2020-09-28T18:01:41.192Z"
- ],
- "geo.coords.location" : [
- "39.07997465226799, 6.073727197945118"
- ],
- "entity_id" : [
- "ABD5250"
+ },
+ {
+ "key" : "ABD5250",
+ "doc_count" : 9,
+ "entityHits" : {
+ "hits" : {
+ "total" : {
+ "value" : 9,
+ "relation" : "eq"
+ },
+ "max_score" : null,
+ "hits" : [
+ {
+ "_index" : "flight_tracks",
+ "_id" : "GOng1XQB6yyY-xQxnGWM",
+ "_score" : null,
+ "fields" : {
+ "time_data.@timestamp" : [
+ "2020-09-28T18:01:41.192Z"
+ ],
+ "geo.coords.location" : [
+ "39.07997465226799, 6.073727197945118"
+ ],
+ "entity_id" : [
+ "ABD5250"
+ ]
+ },
+ "sort" : [
+ 1601316101192
]
- },
- "sort" : [
- 1601316101192
- ]
- }
- ]
+ }
+ ]
+ }
}
}
- }
- ]
+ ]
+ }
}
}
}
diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts
index 429331916ea7d..df2e9df4ba189 100644
--- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts
+++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts
@@ -6,67 +6,101 @@
*/
import _ from 'lodash';
-import sampleJsonResponse from './es_sample_response.json';
-import sampleJsonResponseWithNesting from './es_sample_response_with_nesting.json';
-import { getActiveEntriesAndGenerateAlerts, transformResults } from '../geo_containment';
+import { loggingSystemMock } from 'src/core/server/mocks';
+import { AlertServicesMock, alertsMock } from '../../../../../alerting/server/mocks';
+import sampleAggsJsonResponse from './es_sample_response.json';
+import sampleShapesJsonResponse from './es_sample_response_shapes.json';
+import sampleAggsJsonResponseWithNesting from './es_sample_response_with_nesting.json';
+import {
+ getActiveEntriesAndGenerateAlerts,
+ transformResults,
+ getGeoContainmentExecutor,
+} from '../geo_containment';
import { OTHER_CATEGORY } from '../es_query_builder';
-import { alertsMock } from '../../../../../alerting/server/mocks';
-import { GeoContainmentInstanceContext, GeoContainmentInstanceState } from '../alert_type';
+import {
+ GeoContainmentInstanceContext,
+ GeoContainmentInstanceState,
+ GeoContainmentParams,
+} from '../alert_type';
import { SearchResponse } from 'elasticsearch';
+const alertInstanceFactory = (contextKeys: unknown[], testAlertActionArr: unknown[]) => (
+ instanceId: string
+) => {
+ const alertInstance = alertsMock.createAlertInstanceFactory<
+ GeoContainmentInstanceState,
+ GeoContainmentInstanceContext
+ >();
+ alertInstance.scheduleActions.mockImplementation(
+ (actionGroupId: string, context?: GeoContainmentInstanceContext) => {
+ // Check subset of alert for comparison to expected results
+ // @ts-ignore
+ const contextSubset = _.pickBy(context, (v, k) => contextKeys.includes(k));
+ testAlertActionArr.push({
+ actionGroupId,
+ instanceId,
+ context: contextSubset,
+ });
+ }
+ );
+ return alertInstance;
+};
+
describe('geo_containment', () => {
describe('transformResults', () => {
const dateField = '@timestamp';
const geoField = 'location';
it('should correctly transform expected results', async () => {
const transformedResults = transformResults(
- (sampleJsonResponse as unknown) as SearchResponse,
+ // @ts-ignore
+ (sampleAggsJsonResponse.body as unknown) as SearchResponse,
dateField,
geoField
);
expect(transformedResults).toEqual(
new Map([
[
- '936',
+ '0',
[
{
- dateInShape: '2020-09-28T18:01:41.190Z',
- docId: 'N-ng1XQB6yyY-xQxnGSM',
- location: [-82.8814151789993, 40.62806099653244],
- shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
+ dateInShape: '2021-04-28T16:56:11.923Z',
+ docId: 'ZVBoGXkBsFLYN2Tj1wmV',
+ location: [-73.99018926545978, 40.751759740523994],
+ shapeLocationId: 'kFATGXkBsFLYN2Tj6AAk',
},
- ],
- ],
- [
- 'AAL2019',
- [
{
- dateInShape: '2020-09-28T18:01:41.191Z',
- docId: 'iOng1XQB6yyY-xQxnGSM',
- location: [-82.22068064846098, 39.006176185794175],
- shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
+ dateInShape: '2021-04-28T16:56:01.896Z',
+ docId: 'YlBoGXkBsFLYN2TjsAlp',
+ location: [-73.98968475870788, 40.7506317878142],
+ shapeLocationId: 'other',
},
],
],
[
- 'AAL2323',
+ '1',
[
{
- dateInShape: '2020-09-28T18:01:41.191Z',
- docId: 'n-ng1XQB6yyY-xQxnGSM',
- location: [-84.71324851736426, 41.6677269525826],
- shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
+ dateInShape: '2021-04-28T16:56:11.923Z',
+ docId: 'ZlBoGXkBsFLYN2Tj1wmV',
+ location: [-73.99561604484916, 40.75449890457094],
+ shapeLocationId: 'kFATGXkBsFLYN2Tj6AAk',
+ },
+ {
+ dateInShape: '2021-04-28T16:56:01.896Z',
+ docId: 'Y1BoGXkBsFLYN2TjsAlp',
+ location: [-73.99459345266223, 40.755913141183555],
+ shapeLocationId: 'other',
},
],
],
[
- 'ABD5250',
+ '2',
[
{
- dateInShape: '2020-09-28T18:01:41.192Z',
- docId: 'GOng1XQB6yyY-xQxnGWM',
- location: [6.073727197945118, 39.07997465226799],
- shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
+ dateInShape: '2021-04-28T16:56:11.923Z',
+ docId: 'Z1BoGXkBsFLYN2Tj1wmV',
+ location: [-73.98662586696446, 40.7667087810114],
+ shapeLocationId: 'other',
},
],
],
@@ -78,7 +112,8 @@ describe('geo_containment', () => {
const nestedGeoField = 'geo.coords.location';
it('should correctly transform expected results if fields are nested', async () => {
const transformedResults = transformResults(
- (sampleJsonResponseWithNesting as unknown) as SearchResponse,
+ // @ts-ignore
+ (sampleAggsJsonResponseWithNesting.body as unknown) as SearchResponse,
nestedDateField,
nestedGeoField
);
@@ -181,7 +216,7 @@ describe('geo_containment', () => {
],
]);
- const expectedContext = [
+ const expectedAlertResults = [
{
actionGroupId: 'Tracked entity contained',
context: {
@@ -213,28 +248,9 @@ describe('geo_containment', () => {
instanceId: 'c-789',
},
];
+ const contextKeys = Object.keys(expectedAlertResults[0].context);
const emptyShapesIdsNamesMap = {};
- const alertInstanceFactory = (instanceId: string) => {
- const alertInstance = alertsMock.createAlertInstanceFactory<
- GeoContainmentInstanceState,
- GeoContainmentInstanceContext
- >();
- alertInstance.scheduleActions.mockImplementation(
- (actionGroupId: string, context?: GeoContainmentInstanceContext) => {
- const contextKeys = Object.keys(expectedContext[0].context);
- const contextSubset = _.pickBy(context, (v, k) => contextKeys.includes(k));
- testAlertActionArr.push({
- actionGroupId,
- instanceId,
- context: contextSubset,
- });
- return alertInstance;
- }
- );
- return alertInstance;
- };
-
const currentDateTime = new Date();
it('should use currently active entities if no older entity entries', () => {
@@ -242,13 +258,14 @@ describe('geo_containment', () => {
const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts(
emptyPrevLocationMap,
currLocationMap,
- alertInstanceFactory,
+ alertInstanceFactory(contextKeys, testAlertActionArr),
emptyShapesIdsNamesMap,
currentDateTime
);
expect(allActiveEntriesMap).toEqual(currLocationMap);
- expect(testAlertActionArr).toMatchObject(expectedContext);
+ expect(testAlertActionArr).toMatchObject(expectedAlertResults);
});
+
it('should overwrite older identical entity entries', () => {
const prevLocationMapWithIdenticalEntityEntry = new Map([
[
@@ -266,13 +283,14 @@ describe('geo_containment', () => {
const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts(
prevLocationMapWithIdenticalEntityEntry,
currLocationMap,
- alertInstanceFactory,
+ alertInstanceFactory(contextKeys, testAlertActionArr),
emptyShapesIdsNamesMap,
currentDateTime
);
expect(allActiveEntriesMap).toEqual(currLocationMap);
- expect(testAlertActionArr).toMatchObject(expectedContext);
+ expect(testAlertActionArr).toMatchObject(expectedAlertResults);
});
+
it('should preserve older non-identical entity entries', () => {
const prevLocationMapWithNonIdenticalEntityEntry = new Map([
[
@@ -287,7 +305,7 @@ describe('geo_containment', () => {
],
],
]);
- const expectedContextPlusD = [
+ const expectedAlertResultsPlusD = [
{
actionGroupId: 'Tracked entity contained',
context: {
@@ -298,19 +316,19 @@ describe('geo_containment', () => {
},
instanceId: 'd-999',
},
- ...expectedContext,
+ ...expectedAlertResults,
];
const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts(
prevLocationMapWithNonIdenticalEntityEntry,
currLocationMap,
- alertInstanceFactory,
+ alertInstanceFactory(contextKeys, testAlertActionArr),
emptyShapesIdsNamesMap,
currentDateTime
);
expect(allActiveEntriesMap).not.toEqual(currLocationMap);
expect(allActiveEntriesMap.has('d')).toBeTruthy();
- expect(testAlertActionArr).toMatchObject(expectedContextPlusD);
+ expect(testAlertActionArr).toMatchObject(expectedAlertResultsPlusD);
});
it('should remove "other" entries and schedule the expected number of actions', () => {
@@ -327,12 +345,12 @@ describe('geo_containment', () => {
const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts(
emptyPrevLocationMap,
currLocationMapWithOther,
- alertInstanceFactory,
+ alertInstanceFactory(contextKeys, testAlertActionArr),
emptyShapesIdsNamesMap,
currentDateTime
);
expect(allActiveEntriesMap).toEqual(currLocationMap);
- expect(testAlertActionArr).toMatchObject(expectedContext);
+ expect(testAlertActionArr).toMatchObject(expectedAlertResults);
});
it('should generate multiple alerts per entity if found in multiple shapes in interval', () => {
@@ -360,7 +378,7 @@ describe('geo_containment', () => {
getActiveEntriesAndGenerateAlerts(
emptyPrevLocationMap,
currLocationMapWithThreeMore,
- alertInstanceFactory,
+ alertInstanceFactory(contextKeys, testAlertActionArr),
emptyShapesIdsNamesMap,
currentDateTime
);
@@ -397,7 +415,7 @@ describe('geo_containment', () => {
const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts(
emptyPrevLocationMap,
currLocationMapWithOther,
- alertInstanceFactory,
+ alertInstanceFactory(contextKeys, testAlertActionArr),
emptyShapesIdsNamesMap,
currentDateTime
);
@@ -429,7 +447,7 @@ describe('geo_containment', () => {
const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts(
emptyPrevLocationMap,
currLocationMapWithOther,
- alertInstanceFactory,
+ alertInstanceFactory(contextKeys, testAlertActionArr),
emptyShapesIdsNamesMap,
currentDateTime
);
@@ -445,4 +463,171 @@ describe('geo_containment', () => {
);
});
});
+
+ describe('getGeoContainmentExecutor', () => {
+ // Params needed for all tests
+ const expectedAlertResults = [
+ {
+ actionGroupId: 'Tracked entity contained',
+ context: {
+ containingBoundaryId: 'kFATGXkBsFLYN2Tj6AAk',
+ entityDocumentId: 'ZVBoGXkBsFLYN2Tj1wmV',
+ entityId: '0',
+ entityLocation: 'POINT (-73.99018926545978 40.751759740523994)',
+ },
+ instanceId: '0-kFATGXkBsFLYN2Tj6AAk',
+ },
+ {
+ actionGroupId: 'Tracked entity contained',
+ context: {
+ containingBoundaryId: 'kFATGXkBsFLYN2Tj6AAk',
+ entityDocumentId: 'ZlBoGXkBsFLYN2Tj1wmV',
+ entityId: '1',
+ entityLocation: 'POINT (-73.99561604484916 40.75449890457094)',
+ },
+ instanceId: '1-kFATGXkBsFLYN2Tj6AAk',
+ },
+ ];
+ const testAlertActionArr: unknown[] = [];
+ const mockLogger = loggingSystemMock.createLogger();
+ const previousStartedAt = new Date('2021-04-27T16:56:11.923Z');
+ const startedAt = new Date('2021-04-29T16:56:11.923Z');
+ const geoContainmentParams: GeoContainmentParams = {
+ index: 'testIndex',
+ indexId: 'testIndexId',
+ geoField: 'location',
+ entity: 'testEntity',
+ dateField: '@timestamp',
+ boundaryType: 'testBoundaryType',
+ boundaryIndexTitle: 'testBoundaryIndexTitle',
+ boundaryIndexId: 'testBoundaryIndexId',
+ boundaryGeoField: 'testBoundaryGeoField',
+ };
+ const alertId = 'testAlertId';
+ const geoContainmentState = {
+ shapesFilters: {
+ testShape: 'thisIsAShape',
+ },
+ shapesIdsNamesMap: {},
+ prevLocationMap: {},
+ };
+
+ // Boundary test mocks
+ const boundaryCall = jest.fn();
+ const esAggCall = jest.fn();
+ const contextKeys = Object.keys(expectedAlertResults[0].context);
+ const alertServicesWithSearchMock: AlertServicesMock = {
+ ...alertsMock.createAlertServices(),
+ // @ts-ignore
+ alertInstanceFactory: alertInstanceFactory(contextKeys, testAlertActionArr),
+ scopedClusterClient: {
+ asCurrentUser: {
+ // @ts-ignore
+ search: jest.fn(({ index }: { index: string }) => {
+ if (index === geoContainmentParams.boundaryIndexTitle) {
+ boundaryCall();
+ return sampleShapesJsonResponse;
+ } else {
+ esAggCall();
+ return sampleAggsJsonResponse;
+ }
+ }),
+ },
+ },
+ };
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ testAlertActionArr.length = 0;
+ });
+
+ it('should query for shapes if state does not contain shapes', async () => {
+ const executor = await getGeoContainmentExecutor(mockLogger);
+ const executionResult = await executor({
+ previousStartedAt,
+ startedAt,
+ // @ts-ignore
+ services: alertServicesWithSearchMock,
+ params: geoContainmentParams,
+ alertId,
+ // @ts-ignore
+ state: {},
+ });
+ if (executionResult && executionResult.shapesFilters) {
+ expect(boundaryCall.mock.calls.length).toBe(1);
+ expect(esAggCall.mock.calls.length).toBe(1);
+ }
+ expect(testAlertActionArr).toMatchObject(expectedAlertResults);
+ });
+
+ it('should not query for shapes if state contains shapes', async () => {
+ const executor = await getGeoContainmentExecutor(mockLogger);
+ const executionResult = await executor({
+ previousStartedAt,
+ startedAt,
+ // @ts-ignore
+ services: alertServicesWithSearchMock,
+ params: geoContainmentParams,
+ alertId,
+ state: geoContainmentState,
+ });
+ if (executionResult && executionResult.shapesFilters) {
+ expect(boundaryCall.mock.calls.length).toBe(0);
+ expect(esAggCall.mock.calls.length).toBe(1);
+ }
+ expect(testAlertActionArr).toMatchObject(expectedAlertResults);
+ });
+
+ it('should carry through shapes filters in state to next call unmodified', async () => {
+ const executor = await getGeoContainmentExecutor(mockLogger);
+ const executionResult = await executor({
+ previousStartedAt,
+ startedAt,
+ // @ts-ignore
+ services: alertServicesWithSearchMock,
+ params: geoContainmentParams,
+ alertId,
+ state: geoContainmentState,
+ });
+ if (executionResult && executionResult.shapesFilters) {
+ expect(executionResult.shapesFilters).toEqual(geoContainmentState.shapesFilters);
+ }
+ expect(testAlertActionArr).toMatchObject(expectedAlertResults);
+ });
+
+ it('should return previous locations map', async () => {
+ const expectedPrevLocationMap = {
+ '0': [
+ {
+ dateInShape: '2021-04-28T16:56:11.923Z',
+ docId: 'ZVBoGXkBsFLYN2Tj1wmV',
+ location: [-73.99018926545978, 40.751759740523994],
+ shapeLocationId: 'kFATGXkBsFLYN2Tj6AAk',
+ },
+ ],
+ '1': [
+ {
+ dateInShape: '2021-04-28T16:56:11.923Z',
+ docId: 'ZlBoGXkBsFLYN2Tj1wmV',
+ location: [-73.99561604484916, 40.75449890457094],
+ shapeLocationId: 'kFATGXkBsFLYN2Tj6AAk',
+ },
+ ],
+ };
+ const executor = await getGeoContainmentExecutor(mockLogger);
+ const executionResult = await executor({
+ previousStartedAt,
+ startedAt,
+ // @ts-ignore
+ services: alertServicesWithSearchMock,
+ params: geoContainmentParams,
+ alertId,
+ state: geoContainmentState,
+ });
+ if (executionResult && executionResult.prevLocationMap) {
+ expect(executionResult.prevLocationMap).toEqual(expectedPrevLocationMap);
+ }
+ expect(testAlertActionArr).toMatchObject(expectedAlertResults);
+ });
+ });
});
From 4af6e9c6916c2100f7112073f048e8b16e9fa98c Mon Sep 17 00:00:00 2001
From: Tiago Costa
Date: Mon, 3 May 2021 22:34:53 +0100
Subject: [PATCH 004/106] chore(NA): moving @kbn/analytics into bazel (#98917)
* chore(NA): moving @kbn/analytics into bazel
* chore(NA): fix type check for package migration
* chore(NA): fix type check for package migration
* chore(NA): fix type check for package migration
* chore(NA): separate type generating from server code
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
BUILD.bazel | 1 +
.../monorepo-packages.asciidoc | 1 +
package.json | 2 +-
packages/BUILD.bazel | 1 +
packages/kbn-analytics/BUILD.bazel | 124 ++++++++++++++++++
packages/kbn-analytics/babel.config.js | 21 ---
packages/kbn-analytics/package.json | 9 +-
packages/kbn-analytics/scripts/build.js | 85 ------------
packages/kbn-analytics/tsconfig.browser.json | 18 +++
packages/kbn-analytics/tsconfig.json | 6 +-
packages/kbn-ui-shared-deps/package.json | 1 -
yarn.lock | 2 +-
12 files changed, 152 insertions(+), 119 deletions(-)
create mode 100644 packages/kbn-analytics/BUILD.bazel
delete mode 100644 packages/kbn-analytics/babel.config.js
delete mode 100644 packages/kbn-analytics/scripts/build.js
create mode 100644 packages/kbn-analytics/tsconfig.browser.json
diff --git a/BUILD.bazel b/BUILD.bazel
index 4502daeaacb59..1f6e3030e5d0c 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -3,6 +3,7 @@
exports_files(
[
"tsconfig.base.json",
+ "tsconfig.browser.json",
"tsconfig.json",
"package.json"
],
diff --git a/docs/developer/getting-started/monorepo-packages.asciidoc b/docs/developer/getting-started/monorepo-packages.asciidoc
index 969226df53cb7..02f96f304ec1e 100644
--- a/docs/developer/getting-started/monorepo-packages.asciidoc
+++ b/docs/developer/getting-started/monorepo-packages.asciidoc
@@ -64,6 +64,7 @@ yarn kbn watch-bazel
- @elastic/datemath
- @elastic/eslint-config-kibana
- @elastic/safer-lodash-set
+- @kbn/analytics
- @kbn/apm-config-loader
- @kbn/apm-utils
- @kbn/babel-code-parser
diff --git a/package.json b/package.json
index 04bce960ab1a2..81727c4028610 100644
--- a/package.json
+++ b/package.json
@@ -122,7 +122,7 @@
"@hapi/podium": "^4.1.1",
"@hapi/wreck": "^17.1.0",
"@kbn/ace": "link:packages/kbn-ace",
- "@kbn/analytics": "link:packages/kbn-analytics",
+ "@kbn/analytics": "link:bazel-bin/packages/kbn-analytics/npm_module",
"@kbn/apm-config-loader": "link:bazel-bin/packages/kbn-apm-config-loader/npm_module",
"@kbn/apm-utils": "link:bazel-bin/packages/kbn-apm-utils/npm_module",
"@kbn/config": "link:packages/kbn-config",
diff --git a/packages/BUILD.bazel b/packages/BUILD.bazel
index 39285fb9ea66a..eb93a1055e7b7 100644
--- a/packages/BUILD.bazel
+++ b/packages/BUILD.bazel
@@ -6,6 +6,7 @@ filegroup(
"//packages/elastic-datemath:build",
"//packages/elastic-eslint-config-kibana:build",
"//packages/elastic-safer-lodash-set:build",
+ "//packages/kbn-analytics:build",
"//packages/kbn-apm-config-loader:build",
"//packages/kbn-apm-utils:build",
"//packages/kbn-babel-code-parser:build",
diff --git a/packages/kbn-analytics/BUILD.bazel b/packages/kbn-analytics/BUILD.bazel
new file mode 100644
index 0000000000000..a5506598baeac
--- /dev/null
+++ b/packages/kbn-analytics/BUILD.bazel
@@ -0,0 +1,124 @@
+load("@npm//@bazel/typescript:index.bzl", "ts_config", "ts_project")
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library", "pkg_npm")
+
+PKG_BASE_NAME = "kbn-analytics"
+PKG_REQUIRE_NAME = "@kbn/analytics"
+
+SOURCE_FILES = glob(
+ [
+ "src/**/*.ts",
+ ],
+)
+
+SRCS = SOURCE_FILES
+
+filegroup(
+ name = "srcs",
+ srcs = SRCS,
+)
+
+NPM_MODULE_EXTRA_FILES = [
+ "package.json"
+]
+
+SRC_DEPS = [
+ "@npm//moment-timezone",
+ "@npm//tslib",
+]
+
+TYPES_DEPS = [
+ "@npm//@types/moment-timezone",
+ "@npm//@types/node",
+]
+
+DEPS = SRC_DEPS + TYPES_DEPS
+
+ts_config(
+ name = "tsconfig",
+ src = "tsconfig.json",
+ deps = [
+ "//:tsconfig.base.json",
+ ],
+)
+
+ts_config(
+ name = "tsconfig_browser",
+ src = "tsconfig.browser.json",
+ deps = [
+ "//:tsconfig.base.json",
+ "//:tsconfig.browser.json",
+ ],
+)
+
+ts_project(
+ name = "tsc",
+ args = ['--pretty'],
+ srcs = SRCS,
+ deps = DEPS,
+ declaration = True,
+ declaration_dir = "types",
+ declaration_map = True,
+ incremental = True,
+ out_dir = "node",
+ source_map = True,
+ root_dir = "src",
+ tsconfig = ":tsconfig",
+)
+
+ts_project(
+ name = "tsc_browser",
+ args = ['--pretty'],
+ srcs = SRCS,
+ deps = DEPS,
+ declaration = False,
+ incremental = True,
+ out_dir = "web",
+ source_map = True,
+ root_dir = "src",
+ tsconfig = ":tsconfig_browser",
+)
+
+filegroup(
+ name = "tsc_types",
+ srcs = [":tsc"],
+ output_group = "types",
+)
+
+filegroup(
+ name = "target_files",
+ srcs = [
+ ":tsc",
+ ":tsc_browser",
+ ":tsc_types",
+ ],
+)
+
+pkg_npm(
+ name = "target",
+ deps = [
+ ":target_files",
+ ],
+)
+
+js_library(
+ name = PKG_BASE_NAME,
+ srcs = NPM_MODULE_EXTRA_FILES,
+ deps = [":target"] + DEPS,
+ package_name = PKG_REQUIRE_NAME,
+ visibility = ["//visibility:public"],
+)
+
+pkg_npm(
+ name = "npm_module",
+ deps = [
+ ":%s" % PKG_BASE_NAME,
+ ]
+)
+
+filegroup(
+ name = "build",
+ srcs = [
+ ":npm_module",
+ ],
+ visibility = ["//visibility:public"],
+)
diff --git a/packages/kbn-analytics/babel.config.js b/packages/kbn-analytics/babel.config.js
deleted file mode 100644
index cdbc4feb60176..0000000000000
--- a/packages/kbn-analytics/babel.config.js
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-// We can't use common Kibana presets here because of babel versions incompatibility
-module.exports = {
- plugins: ['@babel/plugin-proposal-class-properties'],
- env: {
- web: {
- presets: ['@kbn/babel-preset/webpack_preset'],
- },
- node: {
- presets: ['@kbn/babel-preset/node_preset'],
- },
- },
- ignore: ['**/*.test.ts', '**/*.test.tsx'],
-};
diff --git a/packages/kbn-analytics/package.json b/packages/kbn-analytics/package.json
index 2195de578081e..726b62e1c61b8 100644
--- a/packages/kbn-analytics/package.json
+++ b/packages/kbn-analytics/package.json
@@ -4,13 +4,8 @@
"version": "1.0.0",
"description": "Kibana Analytics tool",
"main": "target/node/index.js",
- "browser": "target/web/index.js",
"types": "target/types/index.d.ts",
+ "browser": "target/web/index.js",
"author": "Ahmad Bamieh ",
- "license": "SSPL-1.0 OR Elastic License 2.0",
- "scripts": {
- "build": "node scripts/build",
- "kbn:bootstrap": "node scripts/build --source-maps",
- "kbn:watch": "node scripts/build --source-maps --watch"
- }
+ "license": "SSPL-1.0 OR Elastic License 2.0"
}
\ No newline at end of file
diff --git a/packages/kbn-analytics/scripts/build.js b/packages/kbn-analytics/scripts/build.js
deleted file mode 100644
index b9677d6a07a88..0000000000000
--- a/packages/kbn-analytics/scripts/build.js
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-const { resolve } = require('path');
-
-const del = require('del');
-const supportsColor = require('supports-color');
-const { run, withProcRunner } = require('@kbn/dev-utils');
-
-const ROOT_DIR = resolve(__dirname, '..');
-const BUILD_DIR = resolve(ROOT_DIR, 'target');
-
-const padRight = (width, str) =>
- str.length >= width ? str : `${str}${' '.repeat(width - str.length)}`;
-
-run(
- async ({ log, flags }) => {
- await withProcRunner(log, async (proc) => {
- log.info('Deleting old output');
- await del(BUILD_DIR);
-
- const cwd = ROOT_DIR;
- const env = { ...process.env };
- if (supportsColor.stdout) {
- env.FORCE_COLOR = 'true';
- }
-
- log.info(`Starting babel and typescript${flags.watch ? ' in watch mode' : ''}`);
- await Promise.all([
- ...['web', 'node'].map((subTask) =>
- proc.run(padRight(10, `babel:${subTask}`), {
- cmd: 'babel',
- args: [
- 'src',
- '--config-file',
- require.resolve('../babel.config.js'),
- '--out-dir',
- resolve(BUILD_DIR, subTask),
- '--extensions',
- '.ts,.js,.tsx',
- ...(flags.watch ? ['--watch'] : ['--quiet']),
- ...(!flags['source-maps'] || !!process.env.CODE_COVERAGE
- ? []
- : ['--source-maps', 'inline']),
- ],
- wait: true,
- env: {
- ...env,
- BABEL_ENV: subTask,
- },
- cwd,
- })
- ),
-
- proc.run(padRight(10, 'tsc'), {
- cmd: 'tsc',
- args: [
- ...(flags.watch ? ['--watch', '--preserveWatchOutput', 'true'] : []),
- ...(flags['source-maps'] ? ['--declarationMap', 'true'] : []),
- ],
- wait: true,
- env,
- cwd,
- }),
- ]);
-
- log.success('Complete');
- });
- },
- {
- description: 'Simple build tool for @kbn/analytics package',
- flags: {
- boolean: ['watch', 'source-maps'],
- help: `
- --watch Run in watch mode
- --source-maps Include sourcemaps
- `,
- },
- }
-);
diff --git a/packages/kbn-analytics/tsconfig.browser.json b/packages/kbn-analytics/tsconfig.browser.json
new file mode 100644
index 0000000000000..12f70b77008e7
--- /dev/null
+++ b/packages/kbn-analytics/tsconfig.browser.json
@@ -0,0 +1,18 @@
+{
+ "extends": "../../tsconfig.browser.json",
+ "compilerOptions": {
+ "incremental": true,
+ "outDir": "./target/web",
+ "stripInternal": true,
+ "declaration": false,
+ "isolatedModules": true,
+ "sourceMap": true,
+ "sourceRoot": "../../../../../packages/kbn-analytics/src",
+ "types": [
+ "node"
+ ]
+ },
+ "include": [
+ "src/**/*"
+ ]
+}
diff --git a/packages/kbn-analytics/tsconfig.json b/packages/kbn-analytics/tsconfig.json
index 80a2255d71805..165a8b695db57 100644
--- a/packages/kbn-analytics/tsconfig.json
+++ b/packages/kbn-analytics/tsconfig.json
@@ -1,10 +1,10 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
- "incremental": false,
- "outDir": "./target/types",
+ "incremental": true,
+ "declarationDir": "./target/types",
+ "outDir": "./target/node",
"stripInternal": true,
- "emitDeclarationOnly": true,
"declaration": true,
"declarationMap": true,
"isolatedModules": true,
diff --git a/packages/kbn-ui-shared-deps/package.json b/packages/kbn-ui-shared-deps/package.json
index 8b08f64ba0f62..54d983bf1bf44 100644
--- a/packages/kbn-ui-shared-deps/package.json
+++ b/packages/kbn-ui-shared-deps/package.json
@@ -9,7 +9,6 @@
"kbn:watch": "node scripts/build --dev --watch"
},
"dependencies": {
- "@kbn/analytics": "link:../kbn-analytics",
"@kbn/i18n": "link:../kbn-i18n",
"@kbn/monaco": "link:../kbn-monaco"
}
diff --git a/yarn.lock b/yarn.lock
index 39b223b9b00b0..29db5e0d4830a 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2587,7 +2587,7 @@
version "0.0.0"
uid ""
-"@kbn/analytics@link:packages/kbn-analytics":
+"@kbn/analytics@link:bazel-bin/packages/kbn-analytics/npm_module":
version "0.0.0"
uid ""
From b2a20fc9af1d0ba765bb59f8cbfee5566b97c907 Mon Sep 17 00:00:00 2001
From: Nicolas Chaulet
Date: Mon, 3 May 2021 17:42:07 -0400
Subject: [PATCH 005/106] [Fleet] Fix fleet server host port during settings
creation (#99084)
---
x-pack/plugins/fleet/server/services/settings.ts | 12 ++++++------
.../fleet_api_integration/apis/settings/update.ts | 12 ++++++++++++
2 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/x-pack/plugins/fleet/server/services/settings.ts b/x-pack/plugins/fleet/server/services/settings.ts
index 6b1a17fc5b060..e493234d1cff8 100644
--- a/x-pack/plugins/fleet/server/services/settings.ts
+++ b/x-pack/plugins/fleet/server/services/settings.ts
@@ -79,14 +79,14 @@ export async function saveSettings(
soClient: SavedObjectsClientContract,
newData: Partial>
): Promise & Pick> {
+ const data = { ...newData };
+ if (data.fleet_server_hosts) {
+ data.fleet_server_hosts = data.fleet_server_hosts.map(normalizeFleetServerHost);
+ }
+
try {
const settings = await getSettings(soClient);
- const data = { ...newData };
- if (data.fleet_server_hosts) {
- data.fleet_server_hosts = data.fleet_server_hosts.map(normalizeFleetServerHost);
- }
-
const res = await soClient.update(
GLOBAL_SETTINGS_SAVED_OBJECT_TYPE,
settings.id,
@@ -102,7 +102,7 @@ export async function saveSettings(
const defaultSettings = createDefaultSettings();
const res = await soClient.create(GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, {
...defaultSettings,
- ...newData,
+ ...data,
});
return {
diff --git a/x-pack/test/fleet_api_integration/apis/settings/update.ts b/x-pack/test/fleet_api_integration/apis/settings/update.ts
index 2c4992b21d71a..31fcb74627915 100644
--- a/x-pack/test/fleet_api_integration/apis/settings/update.ts
+++ b/x-pack/test/fleet_api_integration/apis/settings/update.ts
@@ -38,6 +38,18 @@ export default function (providerContext: FtrProviderContext) {
after(async () => {
await esArchiver.unload('fleet/empty_fleet_server');
});
+
+ it('should explicitly set port on fleet_server_hosts', async function () {
+ await supertest
+ .put(`/api/fleet/settings`)
+ .set('kbn-xsrf', 'xxxx')
+ .send({ fleet_server_hosts: ['https://test.fr'] })
+ .expect(200);
+
+ const { body: getSettingsRes } = await supertest.get(`/api/fleet/settings`).expect(200);
+ expect(getSettingsRes.item.fleet_server_hosts).to.eql(['https://test.fr:443']);
+ });
+
it("should bump all agent policy's revision", async function () {
const { body: testPolicy1PostRes } = await supertest
.post(`/api/fleet/agent_policies`)
From c7977614ed7ab11516dfbaf733da6ef7f8cceebe Mon Sep 17 00:00:00 2001
From: Jonathan Budzenski
Date: Mon, 3 May 2021 17:04:34 -0500
Subject: [PATCH 006/106] [docs/build] Update build dependencies list (#98545)
Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
docs/developer/getting-started/building-kibana.asciidoc | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/docs/developer/getting-started/building-kibana.asciidoc b/docs/developer/getting-started/building-kibana.asciidoc
index 04771b34bf69f..b015be79abca2 100644
--- a/docs/developer/getting-started/building-kibana.asciidoc
+++ b/docs/developer/getting-started/building-kibana.asciidoc
@@ -18,12 +18,12 @@ yarn build --help
[discrete]
=== Building OS packages
-Packages are built using fpm, dpkg, and rpm. Package building has only been tested on Linux and is not supported on any other platform.
-
+Packages are built using fpm, dpkg, and rpm, and Docker. Package building has only been tested on Linux and is not supported on any other platform.
+Docker installation instructions can be found at https://docs.docker.com/engine/install/[Install Docker Engine].
[source,bash]
----
-apt-get install ruby-dev rpm
+apt-get install ruby ruby-dev rpm dpkg build-essential
gem install fpm -v 1.5.0
yarn build --skip-archives
----
From 7a92ea3e10c88913baa378cb0bce1c61ed12cb43 Mon Sep 17 00:00:00 2001
From: Aaron Caldwell
Date: Mon, 3 May 2021 16:42:01 -0600
Subject: [PATCH 007/106] [Maps][File upload] Add validation check that ensure
index name has been provided (#99118)
---
x-pack/plugins/file_upload/public/validate_index_name.ts | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/x-pack/plugins/file_upload/public/validate_index_name.ts b/x-pack/plugins/file_upload/public/validate_index_name.ts
index cd190188b6a63..c13f09f58ae21 100644
--- a/x-pack/plugins/file_upload/public/validate_index_name.ts
+++ b/x-pack/plugins/file_upload/public/validate_index_name.ts
@@ -23,6 +23,12 @@ export function checkIndexPatternValid(name: string) {
}
export const validateIndexName = async (indexName: string) => {
+ if (!indexName) {
+ return i18n.translate('xpack.fileUpload.indexNameRequired', {
+ defaultMessage: 'Index name required',
+ });
+ }
+
if (!checkIndexPatternValid(indexName)) {
return i18n.translate('xpack.fileUpload.indexNameContainsIllegalCharactersErrorMessage', {
defaultMessage: 'Index name contains illegal characters.',
From 4d3499822d62647536b02be3cf4d79965680c0db Mon Sep 17 00:00:00 2001
From: Liza Katz
Date: Tue, 4 May 2021 01:58:06 +0300
Subject: [PATCH 008/106] [Bug] Fix saved query parsing (#99014)
* Fix saved query parsing
* handle null string
---
.../saved_query/saved_query_service.test.ts | 68 +++++++++++++++++++
.../query/saved_query/saved_query_service.ts | 14 ++--
2 files changed, 77 insertions(+), 5 deletions(-)
diff --git a/src/plugins/data/public/query/saved_query/saved_query_service.test.ts b/src/plugins/data/public/query/saved_query/saved_query_service.test.ts
index 5e566a9ec0135..a74f81b6a046d 100644
--- a/src/plugins/data/public/query/saved_query/saved_query_service.test.ts
+++ b/src/plugins/data/public/query/saved_query/saved_query_service.test.ts
@@ -278,6 +278,74 @@ describe('saved query service', () => {
await getSavedQuery('foo');
expect(mockSavedObjectsClient.get).toHaveBeenCalledWith('query', 'foo');
});
+
+ it('should parse a json query', async () => {
+ mockSavedObjectsClient.get.mockReturnValue({
+ id: 'food',
+ attributes: {
+ title: 'food',
+ description: 'bar',
+ query: {
+ language: 'kuery',
+ query: '{"x": "y"}',
+ },
+ },
+ });
+
+ const response = await getSavedQuery('food');
+ expect(response.attributes.query.query).toEqual({ x: 'y' });
+ });
+
+ it('should handle null string', async () => {
+ mockSavedObjectsClient.get.mockReturnValue({
+ id: 'food',
+ attributes: {
+ title: 'food',
+ description: 'bar',
+ query: {
+ language: 'kuery',
+ query: 'null',
+ },
+ },
+ });
+
+ const response = await getSavedQuery('food');
+ expect(response.attributes.query.query).toEqual('null');
+ });
+
+ it('should handle null quoted string', async () => {
+ mockSavedObjectsClient.get.mockReturnValue({
+ id: 'food',
+ attributes: {
+ title: 'food',
+ description: 'bar',
+ query: {
+ language: 'kuery',
+ query: '"null"',
+ },
+ },
+ });
+
+ const response = await getSavedQuery('food');
+ expect(response.attributes.query.query).toEqual('"null"');
+ });
+
+ it('should not lose quotes', async () => {
+ mockSavedObjectsClient.get.mockReturnValue({
+ id: 'food',
+ attributes: {
+ title: 'food',
+ description: 'bar',
+ query: {
+ language: 'kuery',
+ query: '"Bob"',
+ },
+ },
+ });
+
+ const response = await getSavedQuery('food');
+ expect(response.attributes.query.query).toEqual('"Bob"');
+ });
});
describe('deleteSavedQuery', function () {
diff --git a/src/plugins/data/public/query/saved_query/saved_query_service.ts b/src/plugins/data/public/query/saved_query/saved_query_service.ts
index e29066c8892cf..0f3da8f80a5ec 100644
--- a/src/plugins/data/public/query/saved_query/saved_query_service.ts
+++ b/src/plugins/data/public/query/saved_query/saved_query_service.ts
@@ -6,6 +6,7 @@
* Side Public License, v 1.
*/
+import { isObject } from 'lodash';
import { SavedObjectsClientContract, SavedObjectAttributes } from 'src/core/public';
import { SavedQueryAttributes, SavedQuery, SavedQueryService } from './types';
@@ -119,12 +120,15 @@ export const createSavedQueryService = (
id: string;
attributes: SerializedSavedQueryAttributes;
}) => {
- let queryString;
+ let queryString: string | object = savedQuery.attributes.query.query;
+
try {
- queryString = JSON.parse(savedQuery.attributes.query.query);
- } catch (error) {
- queryString = savedQuery.attributes.query.query;
- }
+ const parsedQueryString: object = JSON.parse(savedQuery.attributes.query.query);
+ if (isObject(parsedQueryString)) {
+ queryString = parsedQueryString;
+ }
+ } catch (e) {} // eslint-disable-line no-empty
+
const savedQueryItems: SavedQueryAttributes = {
title: savedQuery.attributes.title || '',
description: savedQuery.attributes.description || '',
From 582c6c7ae3bada799af169cad0a43180e5e6e242 Mon Sep 17 00:00:00 2001
From: ymao1
Date: Mon, 3 May 2021 19:18:13 -0400
Subject: [PATCH 009/106] [Alerting][Docs] Adding query to identify long
running rules to docs (#98773)
* Adding query to identify long running rules to docs
* Wording suggestsion from PR review
* Adding event.provider to query. Allowing copy to console
* Adding note for system privileges
* Adding runtime field to query
* Removing extra dollar sign
* PR fixes
---
.../alerting-troubleshooting.asciidoc | 164 ++++++++++++++++++
1 file changed, 164 insertions(+)
diff --git a/docs/user/alerting/alerting-troubleshooting.asciidoc b/docs/user/alerting/alerting-troubleshooting.asciidoc
index 6d4a0e9375678..b7fd98d1c674e 100644
--- a/docs/user/alerting/alerting-troubleshooting.asciidoc
+++ b/docs/user/alerting/alerting-troubleshooting.asciidoc
@@ -69,3 +69,167 @@ Configuration options are available to specialize connections to TLS servers,
including ignoring server certificate validation, and providing certificate
authority data to verify servers using custom certificates. For more details,
see <>.
+
+[float]
+[[rules-long-execution-time]]
+=== Identify long-running rules
+
+The following query can help you identify rules that are taking a long time to execute and might impact the overall health of your deployment.
+
+[IMPORTANT]
+==============================================
+By default, only users with a `superuser` role can query the {kib} event log because it is a system index. To enable additional users to execute this query, assign `read` privileges to the `.kibana-event-log*` index.
+==============================================
+
+Query for a list of rule ids, bucketed by their execution times:
+
+[source,console]
+--------------------------------------------------
+GET /.kibana-event-log*/_search
+{
+ "size": 0,
+ "query": {
+ "bool": {
+ "filter": [
+ {
+ "range": {
+ "@timestamp": {
+ "gte": "now-1d", <1>
+ "lte": "now"
+ }
+ }
+ },
+ {
+ "term": {
+ "event.action": {
+ "value": "execute"
+ }
+ }
+ },
+ {
+ "term": {
+ "event.provider": {
+ "value": "alerting" <2>
+ }
+ }
+ }
+ ]
+ }
+ },
+ "runtime_mappings": { <3>
+ "event.duration_in_seconds": {
+ "type": "double",
+ "script": {
+ "source": "emit(doc['event.duration'].value / 1E9)"
+ }
+ }
+ },
+ "aggs": {
+ "ruleIdsByExecutionDuration": {
+ "histogram": {
+ "field": "event.duration_in_seconds",
+ "min_doc_count": 1,
+ "interval": 1 <4>
+ },
+ "aggs": {
+ "ruleId": {
+ "nested": {
+ "path": "kibana.saved_objects"
+ },
+ "aggs": {
+ "ruleId": {
+ "terms": {
+ "field": "kibana.saved_objects.id",
+ "size": 10 <5>
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+--------------------------------------------------
+// TEST
+
+<1> This queries for rules executed in the last day. Update the values of `lte` and `gte` to query over a different time range.
+<2> Use `event.provider: actions` to query for long-running action executions.
+<3> Execution durations are stored as nanoseconds. This adds a runtime field to convert that duration into seconds.
+<4> This interval buckets the event.duration_in_seconds runtime field into 1 second intervals. Update this value to change the granularity of the buckets. If you are unable to use runtime fields, make sure this aggregation targets `event.duration` and use nanoseconds for the interval.
+<5> This retrieves the top 10 rule ids for this duration interval. Update this value to retrieve more rule ids.
+
+This query returns the following:
+
+[source,json]
+--------------------------------------------------
+{
+ "took" : 322,
+ "timed_out" : false,
+ "_shards" : {
+ "total" : 1,
+ "successful" : 1,
+ "skipped" : 0,
+ "failed" : 0
+ },
+ "hits" : {
+ "total" : {
+ "value" : 326,
+ "relation" : "eq"
+ },
+ "max_score" : null,
+ "hits" : [ ]
+ },
+ "aggregations" : {
+ "ruleIdsByExecutionDuration" : {
+ "buckets" : [
+ {
+ "key" : 0.0, <1>
+ "doc_count" : 320,
+ "ruleId" : {
+ "doc_count" : 320,
+ "ruleId" : {
+ "doc_count_error_upper_bound" : 0,
+ "sum_other_doc_count" : 0,
+ "buckets" : [
+ {
+ "key" : "1923ada0-a8f3-11eb-a04b-13d723cdfdc5",
+ "doc_count" : 140
+ },
+ {
+ "key" : "15415ecf-cdb0-4fef-950a-f824bd277fe4",
+ "doc_count" : 130
+ },
+ {
+ "key" : "dceeb5d0-6b41-11eb-802b-85b0c1bc8ba2",
+ "doc_count" : 50
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key" : 30.0, <2>
+ "doc_count" : 6,
+ "ruleId" : {
+ "doc_count" : 6,
+ "ruleId" : {
+ "doc_count_error_upper_bound" : 0,
+ "sum_other_doc_count" : 0,
+ "buckets" : [
+ {
+ "key" : "41893910-6bca-11eb-9e0d-85d233e3ee35",
+ "doc_count" : 6
+ }
+ ]
+ }
+ }
+ }
+ ]
+ }
+ }
+}
+--------------------------------------------------
+<1> Most rule execution durations fall within the first bucket (0 - 1 seconds).
+<2> A single rule with id `41893910-6bca-11eb-9e0d-85d233e3ee35` took between 30 and 31 seconds to execute.
+
+Use the <> to retrieve additional information about rules that take a long time to execute.
\ No newline at end of file
From 992ff93a6e3dce29afb43ee5c505ef76f877a6db Mon Sep 17 00:00:00 2001
From: Constance
Date: Mon, 3 May 2021 16:24:52 -0700
Subject: [PATCH 010/106] [Enterprise Search] Refactor shared
SchemaErrorsAccordion component (#99069)
* Move SchemaErrorsAccordion to its own folder
+ give it its own export
* Move errors-specific copy to its own file
+ Capitalize ID heading
* [UI polish] Accordion header tweaks
- Fix responsive behavior
- Make Review button more accessible (a within a button is not semantically correct)
- Add markup for field type
* [UI polish] CSS tweaks
- use OOTB EUI CSS classes for borders (see https://elastic.github.io/eui/#/layout/accordion#styled-for-forms)
- tweak spacing
- misc fixes
* [Code polish] Misc cleanup
- Simplify / DRY out props types
- Prefer unique IDs/names over map indexes for React keys
- Improve var names
* Simplify conditional "view" column + misc fixes
- Rename prop to match AS's generateSomePath helpers
- Remove need for itemId - AS should be able to generate its own URL route without it
- Fix accessibility for view column - should be totally hidden to screen readers if not present
- Fix semantics of view link - should be an link not a button
* Update WS's use of SchemaErrorsAccordion
---
.../applications/shared/schema/constants.ts | 28 ----
.../schema/errors_accordion/constants.ts | 28 ++++
.../index.test.tsx} | 23 ++--
.../shared/schema/errors_accordion/index.tsx | 113 ++++++++++++++++
.../schema_errors_accordion.scss | 26 ++++
.../applications/shared/schema/index.ts | 1 +
.../schema/schema_errors_accordion.scss | 20 ---
.../shared/schema/schema_errors_accordion.tsx | 121 ------------------
.../schema/schema_change_errors.test.tsx | 2 +-
.../schema/schema_change_errors.tsx | 8 +-
10 files changed, 185 insertions(+), 185 deletions(-)
create mode 100644 x-pack/plugins/enterprise_search/public/applications/shared/schema/errors_accordion/constants.ts
rename x-pack/plugins/enterprise_search/public/applications/shared/schema/{schema_errors_accordion.test.tsx => errors_accordion/index.test.tsx} (55%)
create mode 100644 x-pack/plugins/enterprise_search/public/applications/shared/schema/errors_accordion/index.tsx
create mode 100644 x-pack/plugins/enterprise_search/public/applications/shared/schema/errors_accordion/schema_errors_accordion.scss
delete mode 100644 x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_errors_accordion.scss
delete mode 100644 x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_errors_accordion.tsx
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/constants.ts b/x-pack/plugins/enterprise_search/public/applications/shared/schema/constants.ts
index e85050aa22ef1..134e5e7c3dde3 100644
--- a/x-pack/plugins/enterprise_search/public/applications/shared/schema/constants.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/schema/constants.ts
@@ -14,31 +14,3 @@ export const FIELD_NAME = i18n.translate('xpack.enterpriseSearch.schema.fieldNam
export const FIELD_TYPE = i18n.translate('xpack.enterpriseSearch.schema.fieldTypeLabel', {
defaultMessage: 'Field type',
});
-
-export const ERROR_TABLE_ID_HEADER = i18n.translate(
- 'xpack.enterpriseSearch.schema.errorsTable.heading.id',
- {
- defaultMessage: 'id',
- }
-);
-
-export const ERROR_TABLE_ERROR_HEADER = i18n.translate(
- 'xpack.enterpriseSearch.schema.errorsTable.heading.error',
- {
- defaultMessage: 'Error',
- }
-);
-
-export const ERROR_TABLE_REVIEW_CONTROL = i18n.translate(
- 'xpack.enterpriseSearch.schema.errorsTable.control.review',
- {
- defaultMessage: 'Review',
- }
-);
-
-export const ERROR_TABLE_VIEW_LINK = i18n.translate(
- 'xpack.enterpriseSearch.schema.errorsTable.link.view',
- {
- defaultMessage: 'View',
- }
-);
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/errors_accordion/constants.ts b/x-pack/plugins/enterprise_search/public/applications/shared/schema/errors_accordion/constants.ts
new file mode 100644
index 0000000000000..844f0c8041915
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/schema/errors_accordion/constants.ts
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { i18n } from '@kbn/i18n';
+
+export const ERROR_TABLE_ID_HEADER = i18n.translate(
+ 'xpack.enterpriseSearch.schema.errorsTable.heading.id',
+ { defaultMessage: 'ID' }
+);
+
+export const ERROR_TABLE_ERROR_HEADER = i18n.translate(
+ 'xpack.enterpriseSearch.schema.errorsTable.heading.error',
+ { defaultMessage: 'Error' }
+);
+
+export const ERROR_TABLE_REVIEW_CONTROL = i18n.translate(
+ 'xpack.enterpriseSearch.schema.errorsTable.control.review',
+ { defaultMessage: 'Review' }
+);
+
+export const ERROR_TABLE_VIEW_LINK = i18n.translate(
+ 'xpack.enterpriseSearch.schema.errorsTable.link.view',
+ { defaultMessage: 'View' }
+);
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_errors_accordion.test.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/schema/errors_accordion/index.test.tsx
similarity index 55%
rename from x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_errors_accordion.test.tsx
rename to x-pack/plugins/enterprise_search/public/applications/shared/schema/errors_accordion/index.test.tsx
index a15d39c447126..5413da9d56161 100644
--- a/x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_errors_accordion.test.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/schema/errors_accordion/index.test.tsx
@@ -9,11 +9,13 @@ import React from 'react';
import { shallow } from 'enzyme';
-import { EuiAccordion, EuiTableRow } from '@elastic/eui';
+import { EuiAccordion, EuiTableRow, EuiTableHeaderCell } from '@elastic/eui';
-import { EuiButtonEmptyTo } from '../react_router_helpers';
+import { EuiLinkTo } from '../../react_router_helpers';
-import { SchemaErrorsAccordion } from './schema_errors_accordion';
+import { SchemaType } from '../types';
+
+import { SchemaErrorsAccordion } from './';
describe('SchemaErrorsAccordion', () => {
const props = {
@@ -30,8 +32,7 @@ describe('SchemaErrorsAccordion', () => {
],
},
schema: {
- id: 'string',
- name: 'boolean',
+ id: SchemaType.Text,
},
};
@@ -40,12 +41,16 @@ describe('SchemaErrorsAccordion', () => {
expect(wrapper.find(EuiAccordion)).toHaveLength(1);
expect(wrapper.find(EuiTableRow)).toHaveLength(2);
- expect(wrapper.find(EuiButtonEmptyTo)).toHaveLength(0);
});
- it('renders document buttons', () => {
- const wrapper = shallow();
+ it('conditionally renders a view column', () => {
+ const generateViewPath = jest.fn((id: string) => `/documents/${id}`);
+ const wrapper = shallow(
+
+ );
- expect(wrapper.find(EuiButtonEmptyTo)).toHaveLength(2);
+ expect(wrapper.find(EuiTableHeaderCell)).toHaveLength(3);
+ expect(wrapper.find(EuiLinkTo)).toHaveLength(2);
+ expect(wrapper.find(EuiLinkTo).first().prop('to')).toEqual('/documents/foo');
});
});
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/errors_accordion/index.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/schema/errors_accordion/index.tsx
new file mode 100644
index 0000000000000..35675d246a15e
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/schema/errors_accordion/index.tsx
@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+
+import {
+ EuiAccordion,
+ EuiFlexGroup,
+ EuiFlexItem,
+ EuiTable,
+ EuiTableBody,
+ EuiTableHeader,
+ EuiTableHeaderCell,
+ EuiTableRow,
+ EuiTableRowCell,
+} from '@elastic/eui';
+
+import { EuiLinkTo } from '../../react_router_helpers';
+import { TruncatedContent } from '../../truncate';
+
+import { Schema, FieldCoercionErrors } from '../types';
+
+import {
+ ERROR_TABLE_ID_HEADER,
+ ERROR_TABLE_ERROR_HEADER,
+ ERROR_TABLE_REVIEW_CONTROL,
+ ERROR_TABLE_VIEW_LINK,
+} from './constants';
+
+import './schema_errors_accordion.scss';
+
+interface Props {
+ fieldCoercionErrors: FieldCoercionErrors;
+ schema: Schema;
+ generateViewPath?(externalId: string): string;
+}
+
+export const SchemaErrorsAccordion: React.FC = ({
+ fieldCoercionErrors,
+ schema,
+ generateViewPath,
+}) => (
+ <>
+ {Object.keys(fieldCoercionErrors).map((fieldName) => {
+ const fieldType = schema[fieldName];
+ const errors = fieldCoercionErrors[fieldName];
+
+ const accordionHeader = (
+
+
+
+
+
+
+
+ {fieldType}
+
+
+ {/* Mock an EuiButton without actually creating one - we shouldn't nest a button within a button */}
+
+ {ERROR_TABLE_REVIEW_CONTROL}
+
+
+
+ );
+
+ return (
+
+
+
+ {ERROR_TABLE_ID_HEADER}
+ {ERROR_TABLE_ERROR_HEADER}
+ {generateViewPath && }
+
+
+ {errors.map((error) => {
+ const { external_id: id, error: errorMessage } = error;
+ return (
+
+
+
+
+ {errorMessage}
+ {generateViewPath && (
+
+ {ERROR_TABLE_VIEW_LINK}
+
+ )}
+
+ );
+ })}
+
+
+
+ );
+ })}
+ >
+);
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/errors_accordion/schema_errors_accordion.scss b/x-pack/plugins/enterprise_search/public/applications/shared/schema/errors_accordion/schema_errors_accordion.scss
new file mode 100644
index 0000000000000..3b69019e9d460
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/schema/errors_accordion/schema_errors_accordion.scss
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+.schemaErrorsAccordion {
+ // Fixes the accordion button content not stretching to full-width
+ .euiAccordion__button > :last-child {
+ flex-grow: 1;
+ }
+
+ // Tweak spacing
+ .euiAccordion__childWrapper {
+ position: relative;
+ top: -($euiSizeS);
+ padding-left: $euiSizeL;
+ padding-right: $euiSizeL;
+ }
+
+ // Remove extra border-bottom on last table row
+ .euiTableRow:last-child .euiTableRowCell {
+ border-bottom: none;
+ }
+}
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/index.ts b/x-pack/plugins/enterprise_search/public/applications/shared/schema/index.ts
index 657ddef64b7ca..81fa785a9c553 100644
--- a/x-pack/plugins/enterprise_search/public/applications/shared/schema/index.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/shared/schema/index.ts
@@ -8,3 +8,4 @@
export { SchemaAddFieldModal } from './add_field_modal';
export { SchemaFieldTypeSelect } from './field_type_select';
export { SchemaErrorsCallout } from './errors_callout';
+export { SchemaErrorsAccordion } from './errors_accordion';
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_errors_accordion.scss b/x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_errors_accordion.scss
deleted file mode 100644
index e8e55ad2827c5..0000000000000
--- a/x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_errors_accordion.scss
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-.schemaFieldError {
- border-top: 1px solid $euiColorLightShade;
-
- &:last-child {
- border-bottom: 1px solid $euiColorLightShade;
- }
-
- // Something about the EuiFlexGroup being inside a button collapses the row of items.
- // This wrapper div was injected by EUI and had 'with: auto' on it.
- .euiIEFlexWrapFix {
- width: 100%;
- }
-}
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_errors_accordion.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_errors_accordion.tsx
deleted file mode 100644
index c2b3ab614fe47..0000000000000
--- a/x-pack/plugins/enterprise_search/public/applications/shared/schema/schema_errors_accordion.tsx
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import React from 'react';
-
-import {
- EuiAccordion,
- EuiButton,
- EuiFlexGroup,
- EuiFlexItem,
- EuiTable,
- EuiTableBody,
- EuiTableHeader,
- EuiTableHeaderCell,
- EuiTableRow,
- EuiTableRowCell,
-} from '@elastic/eui';
-
-import { EuiButtonEmptyTo } from '../react_router_helpers';
-
-import { TruncatedContent } from '../truncate';
-
-import './schema_errors_accordion.scss';
-
-import {
- ERROR_TABLE_ID_HEADER,
- ERROR_TABLE_ERROR_HEADER,
- ERROR_TABLE_REVIEW_CONTROL,
- ERROR_TABLE_VIEW_LINK,
-} from './constants';
-import { FieldCoercionErrors } from './types';
-
-interface ISchemaErrorsAccordionProps {
- fieldCoercionErrors: FieldCoercionErrors;
- schema: { [key: string]: string };
- itemId?: string;
- getRoute?(itemId: string, externalId: string): string;
-}
-
-export const SchemaErrorsAccordion: React.FC = ({
- fieldCoercionErrors,
- schema,
- itemId,
- getRoute,
-}) => (
- <>
- {Object.keys(fieldCoercionErrors).map((fieldName, fieldNameIndex) => {
- const errorInfos = fieldCoercionErrors[fieldName];
-
- const accordionHeader = (
-
-
-
-
-
-
-
-
- {schema[fieldName]}
-
-
-
- {/* href is needed here because a button cannot be nested in a button or console will error and EuiAccordion uses a button to wrap this. */}
-
- {ERROR_TABLE_REVIEW_CONTROL}
-
-
-
- );
-
- return (
-
-
-
- {ERROR_TABLE_ID_HEADER}
- {ERROR_TABLE_ERROR_HEADER}
-
-
-
- {errorInfos.map((error, errorIndex) => {
- const showViewButton = getRoute && itemId;
- const documentPath = getRoute && itemId ? getRoute(itemId, error.external_id) : '';
-
- const viewButton = showViewButton && (
-
- {ERROR_TABLE_VIEW_LINK}
-
- );
-
- return (
-
-
-
-
- {error.error}
- {showViewButton ? viewButton : }
-
- );
- })}
-
-
-
- );
- })}
- >
-);
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_change_errors.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_change_errors.test.tsx
index e9276b8ec3878..f600d089c6e06 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_change_errors.test.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_change_errors.test.tsx
@@ -14,7 +14,7 @@ import { useParams } from 'react-router-dom';
import { shallow } from 'enzyme';
-import { SchemaErrorsAccordion } from '../../../../../shared/schema/schema_errors_accordion';
+import { SchemaErrorsAccordion } from '../../../../../shared/schema';
import { SchemaChangeErrors } from './schema_change_errors';
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_change_errors.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_change_errors.tsx
index 7f7b26e380c55..e300823aa3ed3 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_change_errors.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_change_errors.tsx
@@ -10,7 +10,7 @@ import { useParams } from 'react-router-dom';
import { useActions, useValues } from 'kea';
-import { SchemaErrorsAccordion } from '../../../../../shared/schema/schema_errors_accordion';
+import { SchemaErrorsAccordion } from '../../../../../shared/schema';
import { ViewContentHeader } from '../../../../components/shared/view_content_header';
import { SCHEMA_ERRORS_HEADING } from './constants';
@@ -32,11 +32,7 @@ export const SchemaChangeErrors: React.FC = () => {
return (
<>
-
+
>
);
};
From f736df990c0bf24c78ca02bb3c06d6276b7b706a Mon Sep 17 00:00:00 2001
From: Nathan L Smith
Date: Mon, 3 May 2021 20:53:33 -0500
Subject: [PATCH 011/106] Observability alerting flyout and page layout
improvements (#99103)
* Make the flyout `s` instead of `m` size
* Remove flyout tabs
* Make flyout description list `compressed`
* Make spacer before description list `s` size
* Use `EuiPageTemplate` on alerts and cases pages.
---
.../pages/alerts/alerts_flyout/index.tsx | 27 ++---
.../public/pages/alerts/index.tsx | 114 +++++++++---------
.../public/pages/cases/index.tsx | 48 ++++----
3 files changed, 88 insertions(+), 101 deletions(-)
diff --git a/x-pack/plugins/observability/public/pages/alerts/alerts_flyout/index.tsx b/x-pack/plugins/observability/public/pages/alerts/alerts_flyout/index.tsx
index 892274b2fb8b0..09fe464aa8cb5 100644
--- a/x-pack/plugins/observability/public/pages/alerts/alerts_flyout/index.tsx
+++ b/x-pack/plugins/observability/public/pages/alerts/alerts_flyout/index.tsx
@@ -16,17 +16,16 @@ import {
EuiFlyoutHeader,
EuiFlyoutProps,
EuiSpacer,
- EuiTabbedContent,
EuiText,
EuiTitle,
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import moment from 'moment-timezone';
import React from 'react';
+import { TopAlert } from '../';
import { useUiSetting } from '../../../../../../../src/plugins/kibana_react/public';
import { asDuration } from '../../../../common/utils/formatters';
import { usePluginContext } from '../../../hooks/use_plugin_context';
-import { TopAlert } from '../';
import { SeverityBadge } from '../severity_badge';
type AlertsFlyoutProps = { alert: TopAlert } & EuiFlyoutProps;
@@ -83,23 +82,8 @@ export function AlertsFlyout({ onClose, alert }: AlertsFlyoutProps) {
},
];
- const tabs = [
- {
- id: 'overview',
- name: i18n.translate('xpack.observability.alerts.flyoutOverviewTabTitle', {
- defaultMessage: 'Overview',
- }),
- content: (
- <>
-
-
- >
- ),
- },
- ];
-
return (
-
+
{alert['rule.name']}
@@ -108,7 +92,12 @@ export function AlertsFlyout({ onClose, alert }: AlertsFlyoutProps) {
{alert.reason}
-
+
+
{alert.link && (
diff --git a/x-pack/plugins/observability/public/pages/alerts/index.tsx b/x-pack/plugins/observability/public/pages/alerts/index.tsx
index 76e5d62369029..a6d5c6926973e 100644
--- a/x-pack/plugins/observability/public/pages/alerts/index.tsx
+++ b/x-pack/plugins/observability/public/pages/alerts/index.tsx
@@ -11,21 +11,20 @@ import {
EuiFlexGroup,
EuiFlexItem,
EuiLink,
- EuiPage,
- EuiPageHeader,
+ EuiPageTemplate,
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import React from 'react';
import { useHistory } from 'react-router-dom';
import { format, parse } from 'url';
-import type { ObservabilityAPIReturnType } from '../../services/call_observability_api/types';
+import { asDuration, asPercent } from '../../../common/utils/formatters';
import { ExperimentalBadge } from '../../components/shared/experimental_badge';
import { useFetcher } from '../../hooks/use_fetcher';
import { usePluginContext } from '../../hooks/use_plugin_context';
import { RouteParams } from '../../routes';
import { callObservabilityApi } from '../../services/call_observability_api';
+import type { ObservabilityAPIReturnType } from '../../services/call_observability_api/types';
import { getAbsoluteDateRange } from '../../utils/date';
-import { asDuration, asPercent } from '../../../common/utils/formatters';
import { AlertsSearchBar } from './alerts_search_bar';
import { AlertsTable } from './alerts_table';
@@ -107,72 +106,71 @@ export function AlertsPage({ routeParams }: AlertsPageProps) {
);
return (
-
-
{i18n.translate('xpack.observability.alertsTitle', { defaultMessage: 'Alerts' })}{' '}
>
- }
- rightSideItems={[
+ ),
+
+ rightSideItems: [
{i18n.translate('xpack.observability.alerts.manageDetectionRulesButtonLabel', {
defaultMessage: 'Manage detection rules',
})}
,
- ]}
- >
-
-
-
+
+
+
+
+ {i18n.translate('xpack.observability.alertsDisclaimerText', {
+ defaultMessage:
+ 'This page shows an experimental alerting view. The data shown here will probably not be an accurate representation of alerts. A non-experimental list of alerts is available in the Alerts and Actions settings in Stack Management.',
})}
- color="warning"
- iconType="beaker"
- >
-
- {i18n.translate('xpack.observability.alertsDisclaimerText', {
- defaultMessage:
- 'This page shows an experimental alerting view. The data shown here will probably not be an accurate representation of alerts. A non-experimental list of alerts is available in the Alerts and Actions settings in Stack Management.',
+
+
+
+ {i18n.translate('xpack.observability.alertsDisclaimerLinkText', {
+ defaultMessage: 'Alerts and Actions',
})}
-
-
-
- {i18n.translate('xpack.observability.alertsDisclaimerLinkText', {
- defaultMessage: 'Alerts and Actions',
- })}
-
-
-
-
-
- {
- const nextSearchParams = new URLSearchParams(history.location.search);
+
+
+
+
+
+ {
+ const nextSearchParams = new URLSearchParams(history.location.search);
- nextSearchParams.set('rangeFrom', dateRange.from);
- nextSearchParams.set('rangeTo', dateRange.to);
- nextSearchParams.set('kuery', query ?? '');
+ nextSearchParams.set('rangeFrom', dateRange.from);
+ nextSearchParams.set('rangeTo', dateRange.to);
+ nextSearchParams.set('kuery', query ?? '');
- history.push({
- ...history.location,
- search: nextSearchParams.toString(),
- });
- }}
- />
-
-
-
-
-
-
-
+ history.push({
+ ...history.location,
+ search: nextSearchParams.toString(),
+ });
+ }}
+ />
+
+
+
+
+
+
);
}
diff --git a/x-pack/plugins/observability/public/pages/cases/index.tsx b/x-pack/plugins/observability/public/pages/cases/index.tsx
index 4af5de274dd65..dd7f7875b568e 100644
--- a/x-pack/plugins/observability/public/pages/cases/index.tsx
+++ b/x-pack/plugins/observability/public/pages/cases/index.tsx
@@ -5,7 +5,7 @@
* 2.0.
*/
-import { EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiPage, EuiPageHeader } from '@elastic/eui';
+import { EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiPageTemplate } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import React from 'react';
import { ExperimentalBadge } from '../../components/shared/experimental_badge';
@@ -17,33 +17,33 @@ interface CasesProps {
export function CasesPage(props: CasesProps) {
return (
-
-
{i18n.translate('xpack.observability.casesTitle', { defaultMessage: 'Cases' })}{' '}
>
- }
- >
-
-
-
+
+
+
+
+ {i18n.translate('xpack.observability.casesDisclaimerText', {
+ defaultMessage: 'This is the future home of cases.',
})}
- color="warning"
- iconType="beaker"
- >
-
- {i18n.translate('xpack.observability.casesDisclaimerText', {
- defaultMessage: 'This is the future home of cases.',
- })}
-
-
-
-
-
-
+
+
+
+
+
);
}
From 8b8d4d83b6f8808985ee1bcd326fc2d99e47c0e4 Mon Sep 17 00:00:00 2001
From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com>
Date: Tue, 4 May 2021 00:02:38 -0400
Subject: [PATCH 012/106] [SECURITY SOLUTION] Fix unmapped field timeline
(#99130)
* add unmapped include_unmapped
* bringing back unmapped field timeline
* add unit test
---
.../timeline/events/all/index.ts | 2 +-
.../components/event_details/columns.test.tsx | 4 +-
.../components/event_details/columns.tsx | 92 +-
.../timeline/body/renderers/cti/helpers.ts | 11 +-
.../factory/events/all/helpers.test.ts | 1594 +++++++++++++----
.../timeline/factory/events/all/helpers.ts | 7 +
.../timeline/factory/events/all/index.ts | 8 +-
.../details/query.events_details.dsl.test.ts | 5 +-
.../details/query.events_details.dsl.ts | 2 +-
9 files changed, 1348 insertions(+), 377 deletions(-)
diff --git a/x-pack/plugins/security_solution/common/search_strategy/timeline/events/all/index.ts b/x-pack/plugins/security_solution/common/search_strategy/timeline/events/all/index.ts
index ada22437a1530..d747758640fab 100644
--- a/x-pack/plugins/security_solution/common/search_strategy/timeline/events/all/index.ts
+++ b/x-pack/plugins/security_solution/common/search_strategy/timeline/events/all/index.ts
@@ -35,7 +35,7 @@ export interface TimelineEventsAllStrategyResponse extends IEsSearchResponse {
}
export interface TimelineEventsAllRequestOptions extends TimelineRequestOptionsPaginated {
- fields: string[];
+ fields: string[] | Array<{ field: string; include_unmapped: boolean }>;
fieldRequested: string[];
language: 'eql' | 'kuery' | 'lucene';
}
diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/columns.test.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/columns.test.tsx
index 7da514a551037..198277e1fb941 100644
--- a/x-pack/plugins/security_solution/public/common/components/event_details/columns.test.tsx
+++ b/x-pack/plugins/security_solution/public/common/components/event_details/columns.test.tsx
@@ -55,14 +55,14 @@ describe('getColumns', () => {
checkboxColumn = getColumns(defaultProps)[0] as Column;
});
- test('should be disabled when the field does not exist', () => {
+ test('should be enabled when the field does not exist', () => {
const testField = 'nonExistingField';
const wrapper = mount(
{checkboxColumn.render(testField, testData)}
) as ReactWrapper;
expect(
wrapper.find(`[data-test-subj="toggle-field-${testField}"]`).first().prop('disabled')
- ).toBe(true);
+ ).toBe(false);
});
test('should be enabled when the field does exist', () => {
diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/columns.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/columns.tsx
index 3a891222c11a5..204b8c088304b 100644
--- a/x-pack/plugins/security_solution/public/common/components/event_details/columns.tsx
+++ b/x-pack/plugins/security_solution/public/common/components/event_details/columns.tsx
@@ -91,10 +91,6 @@ export const getColumns = ({
width: '30px',
render: (field: string, data: EventFieldsData) => {
const label = data.isObjectArray ? i18n.NESTED_COLUMN(field) : i18n.VIEW_COLUMN(field);
- const fieldFromBrowserField = getFieldFromBrowserField(
- [data.category, 'fields', field],
- browserFields
- );
return (
);
@@ -192,40 +186,56 @@ export const getColumns = ({
name: i18n.VALUE,
sortable: true,
truncateText: false,
- render: (values: string[] | null | undefined, data: EventFieldsData) => (
-
- {values != null &&
- values.map((value, i) => (
-
-
- {data.field === MESSAGE_FIELD_NAME ? (
-
- ) : (
-
- )}
-
-
- ))}
-
- ),
+ render: (values: string[] | null | undefined, data: EventFieldsData) => {
+ const fieldFromBrowserField = getFieldFromBrowserField(
+ [data.category, 'fields', data.field],
+ browserFields
+ );
+ return (
+
+ {values != null &&
+ values.map((value, i) => {
+ if (fieldFromBrowserField == null) {
+ return {value};
+ }
+ return (
+
+
+ {data.field === MESSAGE_FIELD_NAME ? (
+
+ ) : (
+
+ )}
+
+
+ );
+ })}
+
+ );
+ },
},
{
field: 'valuesConcatenated',
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/helpers.ts b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/helpers.ts
index 84dcef327736b..6ee39d9852056 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/helpers.ts
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/helpers.ts
@@ -13,8 +13,15 @@ import { INDICATOR_MATCH_SUBFIELDS } from '../../../../../../../common/cti/const
import { Ecs } from '../../../../../../../common/ecs';
import { ThreatIndicatorEcs } from '../../../../../../../common/ecs/threat';
-const getIndicatorEcs = (data: Ecs): ThreatIndicatorEcs[] =>
- get(data, INDICATOR_DESTINATION_PATH) ?? [];
+const getIndicatorEcs = (data: Ecs): ThreatIndicatorEcs[] => {
+ const threatData = get(data, INDICATOR_DESTINATION_PATH);
+ if (threatData == null) {
+ return [];
+ } else if (!Array.isArray(threatData)) {
+ return [threatData];
+ }
+ return threatData;
+};
export const hasThreatMatchValue = (data: Ecs): boolean =>
getIndicatorEcs(data).some((indicator) =>
diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts
index da19df32ac87a..5f732d2bacdac 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts
@@ -8,329 +8,177 @@
import { eventHit } from '../../../../../../common/utils/mock_event_details';
import { EventHit } from '../../../../../../common/search_strategy';
import { TIMELINE_EVENTS_FIELDS } from './constants';
-import { buildObjectForFieldPath, formatTimelineData } from './helpers';
+import { buildFieldsRequest, buildObjectForFieldPath, formatTimelineData } from './helpers';
-describe('#formatTimelineData', () => {
- it('happy path', async () => {
- const res = await formatTimelineData(
- [
- '@timestamp',
- 'host.name',
- 'destination.ip',
- 'source.ip',
- 'source.geo.location',
- 'threat.indicator.matched.field',
- ],
- TIMELINE_EVENTS_FIELDS,
- eventHit
- );
- expect(res).toEqual({
- cursor: {
- tiebreaker: 'beats-ci-immutable-ubuntu-1804-1605624279743236239',
- value: '1605624488922',
- },
- node: {
- _id: 'tkCt1nUBaEgqnrVSZ8R_',
- _index: 'auditbeat-7.8.0-2020.11.05-000003',
- data: [
- {
- field: '@timestamp',
- value: ['2020-11-17T14:48:08.922Z'],
- },
- {
- field: 'host.name',
- value: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'],
- },
- {
- field: 'threat.indicator.matched.field',
- value: ['matched_field', 'other_matched_field', 'matched_field_2'],
- },
- {
- field: 'source.geo.location',
- value: [`{"lon":118.7778,"lat":32.0617}`],
- },
+describe('search strategy timeline helpers', () => {
+ describe('#formatTimelineData', () => {
+ it('happy path', async () => {
+ const res = await formatTimelineData(
+ [
+ '@timestamp',
+ 'host.name',
+ 'destination.ip',
+ 'source.ip',
+ 'source.geo.location',
+ 'threat.indicator.matched.field',
],
- ecs: {
- '@timestamp': ['2020-11-17T14:48:08.922Z'],
+ TIMELINE_EVENTS_FIELDS,
+ eventHit
+ );
+ expect(res).toEqual({
+ cursor: {
+ tiebreaker: 'beats-ci-immutable-ubuntu-1804-1605624279743236239',
+ value: '1605624488922',
+ },
+ node: {
_id: 'tkCt1nUBaEgqnrVSZ8R_',
_index: 'auditbeat-7.8.0-2020.11.05-000003',
- agent: {
- type: ['auditbeat'],
- },
- event: {
- action: ['process_started'],
- category: ['process'],
- dataset: ['process'],
- kind: ['event'],
- module: ['system'],
- type: ['start'],
- },
- host: {
- id: ['e59991e835905c65ed3e455b33e13bd6'],
- ip: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'],
- name: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'],
- os: {
- family: ['debian'],
+ data: [
+ {
+ field: '@timestamp',
+ value: ['2020-11-17T14:48:08.922Z'],
},
- },
- message: ['Process go (PID: 4313) by user jenkins STARTED'],
- process: {
- args: ['go', 'vet', './...'],
- entity_id: ['Z59cIkAAIw8ZoK0H'],
- executable: [
- '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go',
- ],
- hash: {
- sha1: ['1eac22336a41e0660fb302add9d97daa2bcc7040'],
+ {
+ field: 'host.name',
+ value: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'],
},
- name: ['go'],
- pid: ['4313'],
- ppid: ['3977'],
- working_directory: [
- '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat',
- ],
- },
- timestamp: '2020-11-17T14:48:08.922Z',
- user: {
- name: ['jenkins'],
- },
- threat: {
- indicator: [
- {
- event: {
- dataset: [],
- reference: [],
- },
- matched: {
- atomic: ['matched_atomic'],
- field: ['matched_field', 'other_matched_field'],
- type: [],
- },
- provider: ['yourself'],
+ {
+ field: 'threat.indicator.matched.field',
+ value: ['matched_field', 'other_matched_field', 'matched_field_2'],
+ },
+ {
+ field: 'source.geo.location',
+ value: [`{"lon":118.7778,"lat":32.0617}`],
+ },
+ ],
+ ecs: {
+ '@timestamp': ['2020-11-17T14:48:08.922Z'],
+ _id: 'tkCt1nUBaEgqnrVSZ8R_',
+ _index: 'auditbeat-7.8.0-2020.11.05-000003',
+ agent: {
+ type: ['auditbeat'],
+ },
+ event: {
+ action: ['process_started'],
+ category: ['process'],
+ dataset: ['process'],
+ kind: ['event'],
+ module: ['system'],
+ type: ['start'],
+ },
+ host: {
+ id: ['e59991e835905c65ed3e455b33e13bd6'],
+ ip: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'],
+ name: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'],
+ os: {
+ family: ['debian'],
},
- {
- event: {
- dataset: [],
- reference: [],
+ },
+ message: ['Process go (PID: 4313) by user jenkins STARTED'],
+ process: {
+ args: ['go', 'vet', './...'],
+ entity_id: ['Z59cIkAAIw8ZoK0H'],
+ executable: [
+ '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go',
+ ],
+ hash: {
+ sha1: ['1eac22336a41e0660fb302add9d97daa2bcc7040'],
+ },
+ name: ['go'],
+ pid: ['4313'],
+ ppid: ['3977'],
+ working_directory: [
+ '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat',
+ ],
+ },
+ timestamp: '2020-11-17T14:48:08.922Z',
+ user: {
+ name: ['jenkins'],
+ },
+ threat: {
+ indicator: [
+ {
+ event: {
+ dataset: [],
+ reference: [],
+ },
+ matched: {
+ atomic: ['matched_atomic'],
+ field: ['matched_field', 'other_matched_field'],
+ type: [],
+ },
+ provider: ['yourself'],
},
- matched: {
- atomic: ['matched_atomic_2'],
- field: ['matched_field_2'],
- type: [],
+ {
+ event: {
+ dataset: [],
+ reference: [],
+ },
+ matched: {
+ atomic: ['matched_atomic_2'],
+ field: ['matched_field_2'],
+ type: [],
+ },
+ provider: ['other_you'],
},
- provider: ['other_you'],
- },
- ],
+ ],
+ },
},
},
- },
+ });
});
- });
- it('rule signal results', async () => {
- const response: EventHit = {
- _index: '.siem-signals-patrykkopycinski-default-000007',
- _id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562',
- _score: 0,
- _source: {
- signal: {
- threshold_result: {
- count: 10000,
- value: '2a990c11-f61b-4c8e-b210-da2574e9f9db',
- },
- parent: {
- depth: 0,
- index:
- 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,winlogbeat-*',
- id: '0268af90-d8da-576a-9747-2a191519416a',
- type: 'event',
- },
- depth: 1,
- _meta: {
- version: 14,
- },
- rule: {
- note: null,
- throttle: null,
- references: [],
- severity_mapping: [],
- description: 'asdasd',
- created_at: '2021-01-09T11:25:45.046Z',
- language: 'kuery',
- threshold: {
- field: '',
- value: 200,
- },
- building_block_type: null,
- output_index: '.siem-signals-patrykkopycinski-default',
- type: 'threshold',
- rule_name_override: null,
- enabled: true,
- exceptions_list: [],
- updated_at: '2021-01-09T13:36:39.204Z',
- timestamp_override: null,
- from: 'now-360s',
- id: '696c24e0-526d-11eb-836c-e1620268b945',
- timeline_id: null,
- max_signals: 100,
- severity: 'low',
- risk_score: 21,
- risk_score_mapping: [],
- author: [],
- query: '_id :*',
- index: [
- 'apm-*-transaction*',
- 'auditbeat-*',
- 'endgame-*',
- 'filebeat-*',
- 'logs-*',
- 'packetbeat-*',
- 'winlogbeat-*',
- ],
- filters: [
- {
- $state: {
- store: 'appState',
- },
- meta: {
- negate: false,
- alias: null,
- disabled: false,
- type: 'exists',
- value: 'exists',
- key: '_index',
- },
- exists: {
- field: '_index',
- },
- },
- {
- $state: {
- store: 'appState',
- },
- meta: {
- negate: false,
- alias: 'id_exists',
- disabled: false,
- type: 'exists',
- value: 'exists',
- key: '_id',
- },
- exists: {
- field: '_id',
- },
- },
- ],
- created_by: 'patryk_test_user',
- version: 1,
- saved_id: null,
- tags: [],
- rule_id: '2a990c11-f61b-4c8e-b210-da2574e9f9db',
- license: '',
- immutable: false,
- timeline_title: null,
- meta: {
- from: '1m',
- kibana_siem_app_url: 'http://localhost:5601/app/security',
+ it('rule signal results', async () => {
+ const response: EventHit = {
+ _index: '.siem-signals-patrykkopycinski-default-000007',
+ _id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562',
+ _score: 0,
+ _source: {
+ signal: {
+ threshold_result: {
+ count: 10000,
+ value: '2a990c11-f61b-4c8e-b210-da2574e9f9db',
},
- name: 'Threshold test',
- updated_by: 'patryk_test_user',
- interval: '5m',
- false_positives: [],
- to: 'now',
- threat: [],
- actions: [],
- },
- original_time: '2021-01-09T13:39:32.595Z',
- ancestors: [
- {
+ parent: {
depth: 0,
index:
'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,winlogbeat-*',
id: '0268af90-d8da-576a-9747-2a191519416a',
type: 'event',
},
- ],
- parents: [
- {
- depth: 0,
- index:
- 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,winlogbeat-*',
- id: '0268af90-d8da-576a-9747-2a191519416a',
- type: 'event',
+ depth: 1,
+ _meta: {
+ version: 14,
},
- ],
- status: 'open',
- },
- },
- fields: {
- 'signal.rule.output_index': ['.siem-signals-patrykkopycinski-default'],
- 'signal.rule.from': ['now-360s'],
- 'signal.rule.language': ['kuery'],
- '@timestamp': ['2021-01-09T13:41:40.517Z'],
- 'signal.rule.query': ['_id :*'],
- 'signal.rule.type': ['threshold'],
- 'signal.rule.id': ['696c24e0-526d-11eb-836c-e1620268b945'],
- 'signal.rule.risk_score': [21],
- 'signal.status': ['open'],
- 'event.kind': ['signal'],
- 'signal.original_time': ['2021-01-09T13:39:32.595Z'],
- 'signal.rule.severity': ['low'],
- 'signal.rule.version': ['1'],
- 'signal.rule.index': [
- 'apm-*-transaction*',
- 'auditbeat-*',
- 'endgame-*',
- 'filebeat-*',
- 'logs-*',
- 'packetbeat-*',
- 'winlogbeat-*',
- ],
- 'signal.rule.name': ['Threshold test'],
- 'signal.rule.to': ['now'],
- },
- _type: '',
- sort: ['1610199700517'],
- aggregations: {},
- };
-
- expect(
- await formatTimelineData(
- ['@timestamp', 'host.name', 'destination.ip', 'source.ip'],
- TIMELINE_EVENTS_FIELDS,
- response
- )
- ).toEqual({
- cursor: {
- tiebreaker: null,
- value: '',
- },
- node: {
- _id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562',
- _index: '.siem-signals-patrykkopycinski-default-000007',
- data: [
- {
- field: '@timestamp',
- value: ['2021-01-09T13:41:40.517Z'],
- },
- ],
- ecs: {
- '@timestamp': ['2021-01-09T13:41:40.517Z'],
- timestamp: '2021-01-09T13:41:40.517Z',
- _id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562',
- _index: '.siem-signals-patrykkopycinski-default-000007',
- event: {
- kind: ['signal'],
- },
- signal: {
- original_time: ['2021-01-09T13:39:32.595Z'],
- status: ['open'],
- threshold_result: ['{"count":10000,"value":"2a990c11-f61b-4c8e-b210-da2574e9f9db"}'],
rule: {
- building_block_type: [],
+ note: null,
+ throttle: null,
+ references: [],
+ severity_mapping: [],
+ description: 'asdasd',
+ created_at: '2021-01-09T11:25:45.046Z',
+ language: 'kuery',
+ threshold: {
+ field: '',
+ value: 200,
+ },
+ building_block_type: null,
+ output_index: '.siem-signals-patrykkopycinski-default',
+ type: 'threshold',
+ rule_name_override: null,
+ enabled: true,
exceptions_list: [],
- from: ['now-360s'],
- id: ['696c24e0-526d-11eb-836c-e1620268b945'],
+ updated_at: '2021-01-09T13:36:39.204Z',
+ timestamp_override: null,
+ from: 'now-360s',
+ id: '696c24e0-526d-11eb-836c-e1620268b945',
+ timeline_id: null,
+ max_signals: 100,
+ severity: 'low',
+ risk_score: 21,
+ risk_score_mapping: [],
+ author: [],
+ query: '_id :*',
index: [
'apm-*-transaction*',
'auditbeat-*',
@@ -340,27 +188,8 @@ describe('#formatTimelineData', () => {
'packetbeat-*',
'winlogbeat-*',
],
- language: ['kuery'],
- name: ['Threshold test'],
- output_index: ['.siem-signals-patrykkopycinski-default'],
- risk_score: ['21'],
- query: ['_id :*'],
- severity: ['low'],
- to: ['now'],
- type: ['threshold'],
- version: ['1'],
- timeline_id: [],
- timeline_title: [],
- saved_id: [],
- note: [],
- threshold: [
- JSON.stringify({
- field: '',
- value: 200,
- }),
- ],
filters: [
- JSON.stringify({
+ {
$state: {
store: 'appState',
},
@@ -375,8 +204,8 @@ describe('#formatTimelineData', () => {
exists: {
field: '_index',
},
- }),
- JSON.stringify({
+ },
+ {
$state: {
store: 'appState',
},
@@ -391,16 +220,189 @@ describe('#formatTimelineData', () => {
exists: {
field: '_id',
},
- }),
+ },
],
+ created_by: 'patryk_test_user',
+ version: 1,
+ saved_id: null,
+ tags: [],
+ rule_id: '2a990c11-f61b-4c8e-b210-da2574e9f9db',
+ license: '',
+ immutable: false,
+ timeline_title: null,
+ meta: {
+ from: '1m',
+ kibana_siem_app_url: 'http://localhost:5601/app/security',
+ },
+ name: 'Threshold test',
+ updated_by: 'patryk_test_user',
+ interval: '5m',
+ false_positives: [],
+ to: 'now',
+ threat: [],
+ actions: [],
+ },
+ original_time: '2021-01-09T13:39:32.595Z',
+ ancestors: [
+ {
+ depth: 0,
+ index:
+ 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,winlogbeat-*',
+ id: '0268af90-d8da-576a-9747-2a191519416a',
+ type: 'event',
+ },
+ ],
+ parents: [
+ {
+ depth: 0,
+ index:
+ 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,winlogbeat-*',
+ id: '0268af90-d8da-576a-9747-2a191519416a',
+ type: 'event',
+ },
+ ],
+ status: 'open',
+ },
+ },
+ fields: {
+ 'signal.rule.output_index': ['.siem-signals-patrykkopycinski-default'],
+ 'signal.rule.from': ['now-360s'],
+ 'signal.rule.language': ['kuery'],
+ '@timestamp': ['2021-01-09T13:41:40.517Z'],
+ 'signal.rule.query': ['_id :*'],
+ 'signal.rule.type': ['threshold'],
+ 'signal.rule.id': ['696c24e0-526d-11eb-836c-e1620268b945'],
+ 'signal.rule.risk_score': [21],
+ 'signal.status': ['open'],
+ 'event.kind': ['signal'],
+ 'signal.original_time': ['2021-01-09T13:39:32.595Z'],
+ 'signal.rule.severity': ['low'],
+ 'signal.rule.version': ['1'],
+ 'signal.rule.index': [
+ 'apm-*-transaction*',
+ 'auditbeat-*',
+ 'endgame-*',
+ 'filebeat-*',
+ 'logs-*',
+ 'packetbeat-*',
+ 'winlogbeat-*',
+ ],
+ 'signal.rule.name': ['Threshold test'],
+ 'signal.rule.to': ['now'],
+ },
+ _type: '',
+ sort: ['1610199700517'],
+ aggregations: {},
+ };
+
+ expect(
+ await formatTimelineData(
+ ['@timestamp', 'host.name', 'destination.ip', 'source.ip'],
+ TIMELINE_EVENTS_FIELDS,
+ response
+ )
+ ).toEqual({
+ cursor: {
+ tiebreaker: null,
+ value: '',
+ },
+ node: {
+ _id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562',
+ _index: '.siem-signals-patrykkopycinski-default-000007',
+ data: [
+ {
+ field: '@timestamp',
+ value: ['2021-01-09T13:41:40.517Z'],
+ },
+ ],
+ ecs: {
+ '@timestamp': ['2021-01-09T13:41:40.517Z'],
+ timestamp: '2021-01-09T13:41:40.517Z',
+ _id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562',
+ _index: '.siem-signals-patrykkopycinski-default-000007',
+ event: {
+ kind: ['signal'],
+ },
+ signal: {
+ original_time: ['2021-01-09T13:39:32.595Z'],
+ status: ['open'],
+ threshold_result: ['{"count":10000,"value":"2a990c11-f61b-4c8e-b210-da2574e9f9db"}'],
+ rule: {
+ building_block_type: [],
+ exceptions_list: [],
+ from: ['now-360s'],
+ id: ['696c24e0-526d-11eb-836c-e1620268b945'],
+ index: [
+ 'apm-*-transaction*',
+ 'auditbeat-*',
+ 'endgame-*',
+ 'filebeat-*',
+ 'logs-*',
+ 'packetbeat-*',
+ 'winlogbeat-*',
+ ],
+ language: ['kuery'],
+ name: ['Threshold test'],
+ output_index: ['.siem-signals-patrykkopycinski-default'],
+ risk_score: ['21'],
+ query: ['_id :*'],
+ severity: ['low'],
+ to: ['now'],
+ type: ['threshold'],
+ version: ['1'],
+ timeline_id: [],
+ timeline_title: [],
+ saved_id: [],
+ note: [],
+ threshold: [
+ JSON.stringify({
+ field: '',
+ value: 200,
+ }),
+ ],
+ filters: [
+ JSON.stringify({
+ $state: {
+ store: 'appState',
+ },
+ meta: {
+ negate: false,
+ alias: null,
+ disabled: false,
+ type: 'exists',
+ value: 'exists',
+ key: '_index',
+ },
+ exists: {
+ field: '_index',
+ },
+ }),
+ JSON.stringify({
+ $state: {
+ store: 'appState',
+ },
+ meta: {
+ negate: false,
+ alias: 'id_exists',
+ disabled: false,
+ type: 'exists',
+ value: 'exists',
+ key: '_id',
+ },
+ exists: {
+ field: '_id',
+ },
+ }),
+ ],
+ },
},
},
},
- },
+ });
});
});
- describe('buildObjectForFieldPath', () => {
+ describe('#buildObjectForFieldPath', () => {
it('builds an object from a single non-nested field', () => {
expect(buildObjectForFieldPath('@timestamp', eventHit)).toEqual({
'@timestamp': ['2020-11-17T14:48:08.922Z'],
@@ -568,4 +570,946 @@ describe('#formatTimelineData', () => {
});
});
});
+
+ describe('#buildFieldsRequest', () => {
+ it('happy path', async () => {
+ const res = await buildFieldsRequest([
+ '@timestamp',
+ 'host.name',
+ 'destination.ip',
+ 'source.ip',
+ 'source.geo.location',
+ 'threat.indicator.matched.field',
+ ]);
+ expect(res).toEqual([
+ {
+ field: '@timestamp',
+ include_unmapped: true,
+ },
+ {
+ field: 'host.name',
+ include_unmapped: true,
+ },
+ {
+ field: 'destination.ip',
+ include_unmapped: true,
+ },
+ {
+ field: 'source.ip',
+ include_unmapped: true,
+ },
+ {
+ field: 'source.geo.location',
+ include_unmapped: true,
+ },
+ {
+ field: 'threat.indicator.matched.field',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.status',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.group.id',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.original_time',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.filters',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.from',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.language',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.query',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.name',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.to',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.id',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.index',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.type',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.original_event.kind',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.original_event.module',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.version',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.severity',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.risk_score',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.threshold_result',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.code',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.module',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.action',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.category',
+ include_unmapped: true,
+ },
+ {
+ field: 'user.name',
+ include_unmapped: true,
+ },
+ {
+ field: 'message',
+ include_unmapped: true,
+ },
+ {
+ field: 'system.auth.ssh.signature',
+ include_unmapped: true,
+ },
+ {
+ field: 'system.auth.ssh.method',
+ include_unmapped: true,
+ },
+ {
+ field: 'system.audit.package.arch',
+ include_unmapped: true,
+ },
+ {
+ field: 'system.audit.package.entity_id',
+ include_unmapped: true,
+ },
+ {
+ field: 'system.audit.package.name',
+ include_unmapped: true,
+ },
+ {
+ field: 'system.audit.package.size',
+ include_unmapped: true,
+ },
+ {
+ field: 'system.audit.package.summary',
+ include_unmapped: true,
+ },
+ {
+ field: 'system.audit.package.version',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.created',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.dataset',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.duration',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.end',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.hash',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.id',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.kind',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.original',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.outcome',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.risk_score',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.risk_score_norm',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.severity',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.start',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.timezone',
+ include_unmapped: true,
+ },
+ {
+ field: 'event.type',
+ include_unmapped: true,
+ },
+ {
+ field: 'agent.type',
+ include_unmapped: true,
+ },
+ {
+ field: 'auditd.result',
+ include_unmapped: true,
+ },
+ {
+ field: 'auditd.session',
+ include_unmapped: true,
+ },
+ {
+ field: 'auditd.data.acct',
+ include_unmapped: true,
+ },
+ {
+ field: 'auditd.data.terminal',
+ include_unmapped: true,
+ },
+ {
+ field: 'auditd.data.op',
+ include_unmapped: true,
+ },
+ {
+ field: 'auditd.summary.actor.primary',
+ include_unmapped: true,
+ },
+ {
+ field: 'auditd.summary.actor.secondary',
+ include_unmapped: true,
+ },
+ {
+ field: 'auditd.summary.object.primary',
+ include_unmapped: true,
+ },
+ {
+ field: 'auditd.summary.object.secondary',
+ include_unmapped: true,
+ },
+ {
+ field: 'auditd.summary.object.type',
+ include_unmapped: true,
+ },
+ {
+ field: 'auditd.summary.how',
+ include_unmapped: true,
+ },
+ {
+ field: 'auditd.summary.message_type',
+ include_unmapped: true,
+ },
+ {
+ field: 'auditd.summary.sequence',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.Ext.original.path',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.name',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.target_path',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.extension',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.type',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.device',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.inode',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.uid',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.owner',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.gid',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.group',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.mode',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.size',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.mtime',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.ctime',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.path',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.Ext.code_signature',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.Ext.code_signature.subject_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.Ext.code_signature.trusted',
+ include_unmapped: true,
+ },
+ {
+ field: 'file.hash.sha256',
+ include_unmapped: true,
+ },
+ {
+ field: 'host.os.family',
+ include_unmapped: true,
+ },
+ {
+ field: 'host.id',
+ include_unmapped: true,
+ },
+ {
+ field: 'host.ip',
+ include_unmapped: true,
+ },
+ {
+ field: 'registry.key',
+ include_unmapped: true,
+ },
+ {
+ field: 'registry.path',
+ include_unmapped: true,
+ },
+ {
+ field: 'rule.reference',
+ include_unmapped: true,
+ },
+ {
+ field: 'source.bytes',
+ include_unmapped: true,
+ },
+ {
+ field: 'source.packets',
+ include_unmapped: true,
+ },
+ {
+ field: 'source.port',
+ include_unmapped: true,
+ },
+ {
+ field: 'source.geo.continent_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'source.geo.country_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'source.geo.country_iso_code',
+ include_unmapped: true,
+ },
+ {
+ field: 'source.geo.city_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'source.geo.region_iso_code',
+ include_unmapped: true,
+ },
+ {
+ field: 'source.geo.region_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'destination.bytes',
+ include_unmapped: true,
+ },
+ {
+ field: 'destination.packets',
+ include_unmapped: true,
+ },
+ {
+ field: 'destination.port',
+ include_unmapped: true,
+ },
+ {
+ field: 'destination.geo.continent_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'destination.geo.country_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'destination.geo.country_iso_code',
+ include_unmapped: true,
+ },
+ {
+ field: 'destination.geo.city_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'destination.geo.region_iso_code',
+ include_unmapped: true,
+ },
+ {
+ field: 'destination.geo.region_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'dns.question.name',
+ include_unmapped: true,
+ },
+ {
+ field: 'dns.question.type',
+ include_unmapped: true,
+ },
+ {
+ field: 'dns.resolved_ip',
+ include_unmapped: true,
+ },
+ {
+ field: 'dns.response_code',
+ include_unmapped: true,
+ },
+ {
+ field: 'endgame.exit_code',
+ include_unmapped: true,
+ },
+ {
+ field: 'endgame.file_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'endgame.file_path',
+ include_unmapped: true,
+ },
+ {
+ field: 'endgame.logon_type',
+ include_unmapped: true,
+ },
+ {
+ field: 'endgame.parent_process_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'endgame.pid',
+ include_unmapped: true,
+ },
+ {
+ field: 'endgame.process_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'endgame.subject_domain_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'endgame.subject_logon_id',
+ include_unmapped: true,
+ },
+ {
+ field: 'endgame.subject_user_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'endgame.target_domain_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'endgame.target_logon_id',
+ include_unmapped: true,
+ },
+ {
+ field: 'endgame.target_user_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.saved_id',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.timeline_id',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.timeline_title',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.output_index',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.note',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.threshold',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.exceptions_list',
+ include_unmapped: true,
+ },
+ {
+ field: 'signal.rule.building_block_type',
+ include_unmapped: true,
+ },
+ {
+ field: 'suricata.eve.proto',
+ include_unmapped: true,
+ },
+ {
+ field: 'suricata.eve.flow_id',
+ include_unmapped: true,
+ },
+ {
+ field: 'suricata.eve.alert.signature',
+ include_unmapped: true,
+ },
+ {
+ field: 'suricata.eve.alert.signature_id',
+ include_unmapped: true,
+ },
+ {
+ field: 'network.bytes',
+ include_unmapped: true,
+ },
+ {
+ field: 'network.community_id',
+ include_unmapped: true,
+ },
+ {
+ field: 'network.direction',
+ include_unmapped: true,
+ },
+ {
+ field: 'network.packets',
+ include_unmapped: true,
+ },
+ {
+ field: 'network.protocol',
+ include_unmapped: true,
+ },
+ {
+ field: 'network.transport',
+ include_unmapped: true,
+ },
+ {
+ field: 'http.version',
+ include_unmapped: true,
+ },
+ {
+ field: 'http.request.method',
+ include_unmapped: true,
+ },
+ {
+ field: 'http.request.body.bytes',
+ include_unmapped: true,
+ },
+ {
+ field: 'http.request.body.content',
+ include_unmapped: true,
+ },
+ {
+ field: 'http.request.referrer',
+ include_unmapped: true,
+ },
+ {
+ field: 'http.response.status_code',
+ include_unmapped: true,
+ },
+ {
+ field: 'http.response.body.bytes',
+ include_unmapped: true,
+ },
+ {
+ field: 'http.response.body.content',
+ include_unmapped: true,
+ },
+ {
+ field: 'tls.client_certificate.fingerprint.sha1',
+ include_unmapped: true,
+ },
+ {
+ field: 'tls.fingerprints.ja3.hash',
+ include_unmapped: true,
+ },
+ {
+ field: 'tls.server_certificate.fingerprint.sha1',
+ include_unmapped: true,
+ },
+ {
+ field: 'user.domain',
+ include_unmapped: true,
+ },
+ {
+ field: 'winlog.event_id',
+ include_unmapped: true,
+ },
+ {
+ field: 'process.exit_code',
+ include_unmapped: true,
+ },
+ {
+ field: 'process.hash.md5',
+ include_unmapped: true,
+ },
+ {
+ field: 'process.hash.sha1',
+ include_unmapped: true,
+ },
+ {
+ field: 'process.hash.sha256',
+ include_unmapped: true,
+ },
+ {
+ field: 'process.parent.name',
+ include_unmapped: true,
+ },
+ {
+ field: 'process.parent.pid',
+ include_unmapped: true,
+ },
+ {
+ field: 'process.pid',
+ include_unmapped: true,
+ },
+ {
+ field: 'process.name',
+ include_unmapped: true,
+ },
+ {
+ field: 'process.ppid',
+ include_unmapped: true,
+ },
+ {
+ field: 'process.args',
+ include_unmapped: true,
+ },
+ {
+ field: 'process.entity_id',
+ include_unmapped: true,
+ },
+ {
+ field: 'process.executable',
+ include_unmapped: true,
+ },
+ {
+ field: 'process.title',
+ include_unmapped: true,
+ },
+ {
+ field: 'process.working_directory',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.session_id',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.connection.local_resp',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.connection.local_orig',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.connection.missed_bytes',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.connection.state',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.connection.history',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.notice.suppress_for',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.notice.msg',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.notice.note',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.notice.sub',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.notice.dst',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.notice.dropped',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.notice.peer_descr',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.dns.AA',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.dns.qclass_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.dns.RD',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.dns.qtype_name',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.dns.qtype',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.dns.query',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.dns.trans_id',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.dns.qclass',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.dns.RA',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.dns.TC',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.http.resp_mime_types',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.http.trans_depth',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.http.status_msg',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.http.resp_fuids',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.http.tags',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.session_ids',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.timedout',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.local_orig',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.tx_host',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.source',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.is_orig',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.overflow_bytes',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.sha1',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.duration',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.depth',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.analyzers',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.mime_type',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.rx_host',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.total_bytes',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.fuid',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.seen_bytes',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.missing_bytes',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.files.md5',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.ssl.cipher',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.ssl.established',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.ssl.resumed',
+ include_unmapped: true,
+ },
+ {
+ field: 'zeek.ssl.version',
+ include_unmapped: true,
+ },
+ {
+ field: 'threat.indicator.matched.atomic',
+ include_unmapped: true,
+ },
+ {
+ field: 'threat.indicator.matched.type',
+ include_unmapped: true,
+ },
+ {
+ field: 'threat.indicator.event.dataset',
+ include_unmapped: true,
+ },
+ {
+ field: 'threat.indicator.event.reference',
+ include_unmapped: true,
+ },
+ {
+ field: 'threat.indicator.provider',
+ include_unmapped: true,
+ },
+ ]);
+ });
+
+ it('remove internal attributes starting with _', async () => {
+ const res = await buildFieldsRequest([
+ '@timestamp',
+ '_id',
+ 'host.name',
+ 'destination.ip',
+ 'source.ip',
+ 'source.geo.location',
+ '_type',
+ 'threat.indicator.matched.field',
+ ]);
+ expect(res.some((f) => f.field === '_id')).toEqual(false);
+ expect(res.some((f) => f.field === '_type')).toEqual(false);
+ });
+ });
});
diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts
index 6c20843058ff1..8e0e5e9655193 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts
@@ -19,6 +19,7 @@ import {
getDataFromFieldsHits,
getDataSafety,
} from '../../../../../../common/utils/field_formatters';
+import { TIMELINE_EVENTS_FIELDS } from './constants';
const getTimestamp = (hit: EventHit): string => {
if (hit.fields && hit.fields['@timestamp']) {
@@ -29,6 +30,12 @@ const getTimestamp = (hit: EventHit): string => {
return '';
};
+export const buildFieldsRequest = (fields: string[]) =>
+ uniq([...fields.filter((f) => !f.startsWith('_')), ...TIMELINE_EVENTS_FIELDS]).map((field) => ({
+ field,
+ include_unmapped: true,
+ }));
+
export const formatTimelineData = async (
dataFields: readonly string[],
ecsFields: readonly string[],
diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/index.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/index.ts
index 172c864f7ee4f..a26fbe05f7051 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/index.ts
@@ -5,7 +5,7 @@
* 2.0.
*/
-import { cloneDeep, uniq } from 'lodash/fp';
+import { cloneDeep } from 'lodash/fp';
import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../../../../common/constants';
import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common';
@@ -20,7 +20,7 @@ import { inspectStringifyObject } from '../../../../../utils/build_query';
import { SecuritySolutionTimelineFactory } from '../../types';
import { buildTimelineEventsAllQuery } from './query.events_all.dsl';
import { TIMELINE_EVENTS_FIELDS } from './constants';
-import { formatTimelineData } from './helpers';
+import { buildFieldsRequest, formatTimelineData } from './helpers';
export const timelineEventsAll: SecuritySolutionTimelineFactory = {
buildDsl: (options: TimelineEventsAllRequestOptions) => {
@@ -28,7 +28,7 @@ export const timelineEventsAll: SecuritySolutionTimelineFactory
): Promise => {
const { fieldRequested, ...queryOptions } = cloneDeep(options);
- queryOptions.fields = uniq([...fieldRequested, ...TIMELINE_EVENTS_FIELDS]);
+ queryOptions.fields = buildFieldsRequest(fieldRequested);
const { activePage, querySize } = options.pagination;
const totalCount = response.rawResponse.hits.total || 0;
const hits = response.rawResponse.hits.hits;
diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.test.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.test.ts
index 4545a3a3e136b..3572a2b9c497e 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.test.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.test.ts
@@ -40,7 +40,10 @@ describe('buildTimelineDetailsQuery', () => {
},
],
"fields": Array [
- "*",
+ Object {
+ "field": "*",
+ "include_unmapped": true,
+ },
],
"query": Object {
"terms": Object {
diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.ts
index c624eb14ae969..757b004d23f42 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.ts
@@ -22,7 +22,7 @@ export const buildTimelineDetailsQuery = (
_id: [id],
},
},
- fields: ['*'],
+ fields: [{ field: '*', include_unmapped: true }],
_source: true,
},
size: 1,
From 218abe41fe372706328162d1b6f4e683f25ad892 Mon Sep 17 00:00:00 2001
From: Gloria Hornero
Date: Tue, 4 May 2021 08:00:36 +0200
Subject: [PATCH 013/106] [Security Solution] adds 'Alert details with unmapped
fields' test (#98800)
* adds 'Alert details with unmapped fields' test
* fixes test
* improvements
* fixes
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
.../detection_alerts/alerts_details.spec.ts | 52 +++++++++++++++++++
.../security_solution/cypress/objects/rule.ts | 30 ++++++++---
.../cypress/tasks/alerts_details.ts | 1 +
.../cypress/tasks/api_calls/rules.ts | 2 +-
.../es_archives/unmapped_fields/data.json | 14 +++++
.../es_archives/unmapped_fields/mappings.json | 24 +++++++++
6 files changed, 116 insertions(+), 7 deletions(-)
create mode 100644 x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_details.spec.ts
create mode 100644 x-pack/test/security_solution_cypress/es_archives/unmapped_fields/data.json
create mode 100644 x-pack/test/security_solution_cypress/es_archives/unmapped_fields/mappings.json
diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_details.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_details.spec.ts
new file mode 100644
index 0000000000000..177967377f37f
--- /dev/null
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_details.spec.ts
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { JSON_LINES } from '../../screens/alerts_details';
+
+import {
+ expandFirstAlert,
+ waitForAlertsIndexToBeCreated,
+ waitForAlertsPanelToBeLoaded,
+} from '../../tasks/alerts';
+import { openJsonView, scrollJsonViewToBottom } from '../../tasks/alerts_details';
+import { createCustomRuleActivated } from '../../tasks/api_calls/rules';
+import { cleanKibana } from '../../tasks/common';
+import { esArchiverLoad } from '../../tasks/es_archiver';
+import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login';
+
+import { unmappedRule } from '../../objects/rule';
+
+import { DETECTIONS_URL } from '../../urls/navigation';
+
+describe('Alert details with unmapped fields', () => {
+ before(() => {
+ cleanKibana();
+ esArchiverLoad('unmapped_fields');
+ loginAndWaitForPageWithoutDateRange(DETECTIONS_URL);
+ waitForAlertsPanelToBeLoaded();
+ waitForAlertsIndexToBeCreated();
+ createCustomRuleActivated(unmappedRule);
+ });
+ beforeEach(() => {
+ loginAndWaitForPageWithoutDateRange(DETECTIONS_URL);
+ waitForAlertsPanelToBeLoaded();
+ expandFirstAlert();
+ });
+ it('Displays the unmapped field on the JSON view', () => {
+ const expectedUnmappedField = { line: 2, text: ' "unmapped": "This is the unmapped field"' };
+
+ openJsonView();
+ scrollJsonViewToBottom();
+
+ cy.get(JSON_LINES).then((elements) => {
+ const length = elements.length;
+ cy.wrap(elements)
+ .eq(length - expectedUnmappedField.line)
+ .should('have.text', expectedUnmappedField.text);
+ });
+ });
+});
diff --git a/x-pack/plugins/security_solution/cypress/objects/rule.ts b/x-pack/plugins/security_solution/cypress/objects/rule.ts
index 957046cae003a..12523e39cb597 100644
--- a/x-pack/plugins/security_solution/cypress/objects/rule.ts
+++ b/x-pack/plugins/security_solution/cypress/objects/rule.ts
@@ -170,7 +170,25 @@ export const newRule: CustomRule = {
severity: 'High',
riskScore: '17',
tags: ['test', 'newRule'],
- referenceUrls: ['https://www.google.com/', 'https://elastic.co/'],
+ referenceUrls: ['http://example.com/', 'https://example.com/'],
+ falsePositivesExamples: ['False1', 'False2'],
+ mitre: [mitre1, mitre2],
+ note: '# test markdown',
+ runsEvery,
+ lookBack,
+ timeline,
+ maxSignals: 100,
+};
+
+export const unmappedRule: CustomRule = {
+ customQuery: '*:*',
+ index: ['unmapped*'],
+ name: 'Rule with unmapped fields',
+ description: 'The new rule description.',
+ severity: 'High',
+ riskScore: '17',
+ tags: ['test', 'newRule'],
+ referenceUrls: ['http://example.com/', 'https://example.com/'],
falsePositivesExamples: ['False1', 'False2'],
mitre: [mitre1, mitre2],
note: '# test markdown',
@@ -209,7 +227,7 @@ export const newOverrideRule: OverrideRule = {
severity: 'High',
riskScore: '17',
tags: ['test', 'newRule'],
- referenceUrls: ['https://www.google.com/', 'https://elastic.co/'],
+ referenceUrls: ['http://example.com/', 'https://example.com/'],
falsePositivesExamples: ['False1', 'False2'],
mitre: [mitre1, mitre2],
note: '# test markdown',
@@ -231,7 +249,7 @@ export const newThresholdRule: ThresholdRule = {
severity: 'High',
riskScore: '17',
tags: ['test', 'newRule'],
- referenceUrls: ['https://www.google.com/', 'https://elastic.co/'],
+ referenceUrls: ['http://example.com/', 'https://example.com/'],
falsePositivesExamples: ['False1', 'False2'],
mitre: [mitre1, mitre2],
note: '# test markdown',
@@ -267,7 +285,7 @@ export const eqlRule: CustomRule = {
severity: 'High',
riskScore: '17',
tags: ['test', 'newRule'],
- referenceUrls: ['https://www.google.com/', 'https://elastic.co/'],
+ referenceUrls: ['http://example.com/', 'https://example.com/'],
falsePositivesExamples: ['False1', 'False2'],
mitre: [mitre1, mitre2],
note: '# test markdown',
@@ -288,7 +306,7 @@ export const eqlSequenceRule: CustomRule = {
severity: 'High',
riskScore: '17',
tags: ['test', 'newRule'],
- referenceUrls: ['https://www.google.com/', 'https://elastic.co/'],
+ referenceUrls: ['http://example.com/', 'https://example.com/'],
falsePositivesExamples: ['False1', 'False2'],
mitre: [mitre1, mitre2],
note: '# test markdown',
@@ -305,7 +323,7 @@ export const newThreatIndicatorRule: ThreatIndicatorRule = {
severity: 'Critical',
riskScore: '20',
tags: ['test', 'threat'],
- referenceUrls: ['https://www.google.com/', 'https://elastic.co/'],
+ referenceUrls: ['http://example.com/', 'https://example.com/'],
falsePositivesExamples: ['False1', 'False2'],
mitre: [mitre1, mitre2],
note: '# test markdown',
diff --git a/x-pack/plugins/security_solution/cypress/tasks/alerts_details.ts b/x-pack/plugins/security_solution/cypress/tasks/alerts_details.ts
index 1582f35989e2c..45dcc6ea5de5e 100644
--- a/x-pack/plugins/security_solution/cypress/tasks/alerts_details.ts
+++ b/x-pack/plugins/security_solution/cypress/tasks/alerts_details.ts
@@ -14,4 +14,5 @@ export const openJsonView = () => {
export const scrollJsonViewToBottom = () => {
cy.get(JSON_CONTENT).click({ force: true });
cy.get(JSON_CONTENT).type('{pagedown}{pagedown}{pagedown}');
+ cy.get(JSON_CONTENT).should('be.visible');
};
diff --git a/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts b/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts
index 5a816a71744cb..617a06cc8e79a 100644
--- a/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts
+++ b/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts
@@ -85,7 +85,7 @@ export const createCustomRuleActivated = (
severity: rule.severity.toLocaleLowerCase(),
type: 'query',
from: 'now-17520h',
- index: ['auditbeat-*'],
+ index: rule.index,
query: rule.customQuery,
language: 'kuery',
enabled: true,
diff --git a/x-pack/test/security_solution_cypress/es_archives/unmapped_fields/data.json b/x-pack/test/security_solution_cypress/es_archives/unmapped_fields/data.json
new file mode 100644
index 0000000000000..b1e5d16e44b43
--- /dev/null
+++ b/x-pack/test/security_solution_cypress/es_archives/unmapped_fields/data.json
@@ -0,0 +1,14 @@
+{
+ "type": "doc",
+ "value": {
+ "id": "_eZE7mwBOpWiDweStB_c",
+ "index": "unmapped-7.12.0-2021.03.10-000001",
+ "source": {
+ "@timestamp":"2021-02-22T21:00:49.337Z",
+ "mydestination":{
+ "ip": ["127.0.0.1", "127.0.0.2"]
+ },
+ "unmapped": "This is the unmapped field"
+ }
+ }
+}
diff --git a/x-pack/test/security_solution_cypress/es_archives/unmapped_fields/mappings.json b/x-pack/test/security_solution_cypress/es_archives/unmapped_fields/mappings.json
new file mode 100644
index 0000000000000..7ae04b1949cc0
--- /dev/null
+++ b/x-pack/test/security_solution_cypress/es_archives/unmapped_fields/mappings.json
@@ -0,0 +1,24 @@
+{
+ "type": "index",
+ "value": {
+ "aliases": {
+ },
+ "index": "unmapped-7.12.0-2021.03.10-000001",
+ "mappings": {
+ "dynamic": false,
+ "properties":{
+ "@timestamp":{
+ "type":"date"
+ },
+ "mydestination":{
+ "properties":{
+ "ip":{
+ "index": "false",
+ "type":"ip"
+ }
+ }
+ }
+ }
+ }
+ }
+}
From 7ede7a8eca815db738e53e3b5ee51d9daa633732 Mon Sep 17 00:00:00 2001
From: Jason Rhodes
Date: Tue, 4 May 2021 02:23:54 -0400
Subject: [PATCH 014/106] Fixes assignment v comparison and null type check in
metrics detail page (#95102)
* Fixes assignment v comparison and null type check in metrics detail page
* Updated docs to specify new optional TS generic
* Switches new generic to extending the interface
* Removes previously added core generic type and defaults to unknown
* Reverts unknown change, saves for later
* Reverts unknown/any change for now
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
src/core/public/http/types.ts | 2 +-
x-pack/plugins/infra/public/hooks/use_http_request.tsx | 6 +++---
.../pages/metrics/metric_detail/components/page_error.tsx | 5 +++--
x-pack/plugins/infra/public/types.ts | 8 ++++++++
4 files changed, 15 insertions(+), 6 deletions(-)
diff --git a/src/core/public/http/types.ts b/src/core/public/http/types.ts
index d9612d113632a..73418eafccd71 100644
--- a/src/core/public/http/types.ts
+++ b/src/core/public/http/types.ts
@@ -339,7 +339,7 @@ export interface IHttpFetchError extends Error {
* @deprecated Provided for legacy compatibility. Prefer the `response` property instead.
*/
readonly res?: Response;
- readonly body?: any;
+ readonly body?: any; // TODO: this should be unknown
}
/** @public */
diff --git a/x-pack/plugins/infra/public/hooks/use_http_request.tsx b/x-pack/plugins/infra/public/hooks/use_http_request.tsx
index 7e72865214618..1d5c1ef4abfbb 100644
--- a/x-pack/plugins/infra/public/hooks/use_http_request.tsx
+++ b/x-pack/plugins/infra/public/hooks/use_http_request.tsx
@@ -6,12 +6,12 @@
*/
import React, { useMemo, useState } from 'react';
-import { IHttpFetchError } from 'src/core/public';
import { i18n } from '@kbn/i18n';
import { HttpHandler } from 'src/core/public';
import { ToastInput } from 'src/core/public';
import { useTrackedPromise, CanceledPromiseError } from '../utils/use_tracked_promise';
import { useKibana } from '../../../../../src/plugins/kibana_react/public';
+import { InfraHttpError } from '../types';
export function useHTTPRequest(
pathname: string,
@@ -25,7 +25,7 @@ export function useHTTPRequest(
const fetchService = fetch ? fetch : kibana.services.http?.fetch;
const toast = toastDanger ? toastDanger : kibana.notifications.toasts.danger;
const [response, setResponse] = useState(null);
- const [error, setError] = useState(null);
+ const [error, setError] = useState(null);
const [request, makeRequest] = useTrackedPromise(
{
cancelPreviousOn: 'resolution',
@@ -40,7 +40,7 @@ export function useHTTPRequest(
},
onResolve: (resp) => setResponse(decode(resp)),
onReject: (e: unknown) => {
- const err = e as IHttpFetchError;
+ const err = e as InfraHttpError;
if (e && e instanceof CanceledPromiseError) {
return;
}
diff --git a/x-pack/plugins/infra/public/pages/metrics/metric_detail/components/page_error.tsx b/x-pack/plugins/infra/public/pages/metrics/metric_detail/components/page_error.tsx
index 6451206f8cc1a..a6665e0d244f2 100644
--- a/x-pack/plugins/infra/public/pages/metrics/metric_detail/components/page_error.tsx
+++ b/x-pack/plugins/infra/public/pages/metrics/metric_detail/components/page_error.tsx
@@ -7,13 +7,14 @@
import React from 'react';
import { i18n } from '@kbn/i18n';
-import { IHttpFetchError } from 'src/core/public';
import { InvalidNodeError } from './invalid_node';
import { DocumentTitle } from '../../../../components/document_title';
import { ErrorPageBody } from '../../../error';
+import { InfraHttpError } from '../../../../types';
+
interface Props {
name: string;
- error: IHttpFetchError;
+ error: InfraHttpError;
}
export const PageError = ({ error, name }: Props) => {
diff --git a/x-pack/plugins/infra/public/types.ts b/x-pack/plugins/infra/public/types.ts
index 068abd0e0f20f..f4a4d41081a79 100644
--- a/x-pack/plugins/infra/public/types.ts
+++ b/x-pack/plugins/infra/public/types.ts
@@ -6,6 +6,7 @@
*/
import type { CoreSetup, CoreStart, Plugin as PluginClass } from 'kibana/public';
+import { IHttpFetchError } from 'src/core/public';
import type { DataPublicPluginStart } from '../../../../src/plugins/data/public';
import type { HomePublicPluginSetup } from '../../../../src/plugins/home/public';
import type { EmbeddableSetup } from '../../../../src/plugins/embeddable/public';
@@ -59,3 +60,10 @@ export type InfraClientPluginClass = PluginClass<
InfraClientSetupDeps,
InfraClientStartDeps
>;
+
+export interface InfraHttpError extends IHttpFetchError {
+ readonly body?: {
+ statusCode: number;
+ message?: string;
+ };
+}
From a2c13a76b6015e8b769cdac42dacf8571e6965c8 Mon Sep 17 00:00:00 2001
From: Liza Katz
Date: Tue, 4 May 2021 10:35:47 +0300
Subject: [PATCH 015/106] [Search] use core's sha256, fix cached search abort
bug (#98720)
* use core's sha256
* fix bug with aborting first request
* fix types
Co-authored-by: Anton Dosov
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
.../search_interceptor/search_interceptor.test.ts | 15 +++++++++++----
.../search_interceptor/search_interceptor.ts | 6 +++++-
.../public/search/search_interceptor/utils.ts | 6 ++----
3 files changed, 18 insertions(+), 9 deletions(-)
diff --git a/src/plugins/data/public/search/search_interceptor/search_interceptor.test.ts b/src/plugins/data/public/search/search_interceptor/search_interceptor.test.ts
index 0e81f362a030d..dfbc912453d6e 100644
--- a/src/plugins/data/public/search/search_interceptor/search_interceptor.test.ts
+++ b/src/plugins/data/public/search/search_interceptor/search_interceptor.test.ts
@@ -49,17 +49,24 @@ const complete = jest.fn();
function mockFetchImplementation(responses: any[]) {
let i = 0;
- fetchMock.mockImplementation((r) => {
+ fetchMock.mockImplementation((r, abortSignal) => {
if (!r.request.id) i = 0;
const { time = 0, value = {}, isError = false } = responses[i++];
value.meta = {
size: 10,
};
- return new Promise((resolve, reject) =>
+ return new Promise((resolve, reject) => {
setTimeout(() => {
return (isError ? reject : resolve)(value);
- }, time)
- );
+ }, time);
+
+ if (abortSignal) {
+ if (abortSignal.aborted) reject(new AbortError());
+ abortSignal.addEventListener('abort', () => {
+ reject(new AbortError());
+ });
+ }
+ });
});
}
diff --git a/src/plugins/data/public/search/search_interceptor/search_interceptor.ts b/src/plugins/data/public/search/search_interceptor/search_interceptor.ts
index 5fa0a5c301019..57b156a9b3c00 100644
--- a/src/plugins/data/public/search/search_interceptor/search_interceptor.ts
+++ b/src/plugins/data/public/search/search_interceptor/search_interceptor.ts
@@ -198,7 +198,11 @@ export class SearchInterceptor {
options: IAsyncSearchOptions,
searchAbortController: SearchAbortController
) {
- const search = () => this.runSearch({ id, ...request }, options);
+ const search = () =>
+ this.runSearch(
+ { id, ...request },
+ { ...options, abortSignal: searchAbortController.getSignal() }
+ );
const { sessionId, strategy } = options;
// track if this search's session will be send to background
diff --git a/src/plugins/data/public/search/search_interceptor/utils.ts b/src/plugins/data/public/search/search_interceptor/utils.ts
index 2f2d06d3a5494..02c7059a8fdf5 100644
--- a/src/plugins/data/public/search/search_interceptor/utils.ts
+++ b/src/plugins/data/public/search/search_interceptor/utils.ts
@@ -7,10 +7,8 @@
*/
import stringify from 'json-stable-stringify';
+import { Sha256 } from '../../../../../core/public/utils';
export async function createRequestHash(keys: Record) {
- const msgBuffer = new TextEncoder().encode(stringify(keys));
- const hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer);
- const hashArray = Array.from(new Uint8Array(hashBuffer));
- return hashArray.map((b) => ('00' + b.toString(16)).slice(-2)).join('');
+ return new Sha256().update(stringify(keys), 'utf8').digest('hex');
}
From a67f0112125ef1ea94e8442e2ad25ed34759a19e Mon Sep 17 00:00:00 2001
From: Matthias Wilhelm
Date: Tue, 4 May 2021 10:18:44 +0200
Subject: [PATCH 016/106] [Discover] Fix horizontal scrolling of legacy table
(#98770)
* Add CSS class to fix scrolling of legacy table
* Add functional test
---
.../application/components/discover.tsx | 2 +-
test/functional/apps/discover/_doc_table.ts | 29 +++++++++++++++++++
2 files changed, 30 insertions(+), 1 deletion(-)
diff --git a/src/plugins/discover/public/application/components/discover.tsx b/src/plugins/discover/public/application/components/discover.tsx
index 030d7be8ea7e1..90dfd2ef9dce9 100644
--- a/src/plugins/discover/public/application/components/discover.tsx
+++ b/src/plugins/discover/public/application/components/discover.tsx
@@ -394,7 +394,7 @@ export function Discover({
{
+ const scrollWidth = await dscTable.getAttribute('scrollWidth');
+ const clientWidth = await dscTable.getAttribute('clientWidth');
+ log.debug(`scrollWidth: ${scrollWidth}, clientWidth: ${clientWidth}`);
+ return scrollWidth > clientWidth;
+ };
+ const addColumn = async () => {
+ await PageObjects.discover.clickFieldListItemAdd(fieldNames[fieldCounter++]);
+ };
+
+ await addColumn();
+ const isScrollable = await checkScrollable();
+ expect(isScrollable).to.be(false);
+
+ await retry.waitFor('container to be scrollable', async () => {
+ await addColumn();
+ return await checkScrollable();
+ });
+ // so now we need to check if the horizontal scrollbar is displayed
+ const newClientHeight = await dscTable.getAttribute('clientHeight');
+ expect(Number(clientHeight)).to.be.above(Number(newClientHeight));
+ });
});
});
}
From fda2ad45753e6cb3c590cad7fc90fe8e5505638e Mon Sep 17 00:00:00 2001
From: Anton Dosov
Date: Tue, 4 May 2021 10:46:45 +0200
Subject: [PATCH 017/106] [Search Sessions] Fix search sessions docs link
(#98918)
---
.../public/kibana-plugin-core-public.doclinksstart.links.md | 3 +++
src/core/public/doc_links/doc_links_service.ts | 6 ++++++
src/core/public/public.api.md | 3 +++
.../public/search/sessions_mgmt/components/main.test.tsx | 2 +-
.../public/search/sessions_mgmt/lib/documentation.ts | 3 +--
5 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md
index 1830e8f140e60..180d376ceaf51 100644
--- a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md
+++ b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md
@@ -104,6 +104,9 @@ readonly links: {
readonly painlessWalkthrough: string;
readonly luceneExpressions: string;
};
+ readonly search: {
+ readonly sessions: string;
+ };
readonly indexPatterns: {
readonly introduction: string;
readonly fieldFormattersNumber: string;
diff --git a/src/core/public/doc_links/doc_links_service.ts b/src/core/public/doc_links/doc_links_service.ts
index e69006911e7f4..581d614c9a371 100644
--- a/src/core/public/doc_links/doc_links_service.ts
+++ b/src/core/public/doc_links/doc_links_service.ts
@@ -199,6 +199,9 @@ export class DocLinksService {
percolate: `${ELASTICSEARCH_DOCS}query-dsl-percolate-query.html`,
queryDsl: `${ELASTICSEARCH_DOCS}query-dsl.html`,
},
+ search: {
+ sessions: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/search-sessions.html`,
+ },
date: {
dateMath: `${ELASTICSEARCH_DOCS}common-options.html#date-math`,
dateMathIndexNames: `${ELASTICSEARCH_DOCS}date-math-index-names.html`,
@@ -501,6 +504,9 @@ export interface DocLinksStart {
readonly painlessWalkthrough: string;
readonly luceneExpressions: string;
};
+ readonly search: {
+ readonly sessions: string;
+ };
readonly indexPatterns: {
readonly introduction: string;
readonly fieldFormattersNumber: string;
diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md
index 13660da598ea0..0523c523baf6f 100644
--- a/src/core/public/public.api.md
+++ b/src/core/public/public.api.md
@@ -589,6 +589,9 @@ export interface DocLinksStart {
readonly painlessWalkthrough: string;
readonly luceneExpressions: string;
};
+ readonly search: {
+ readonly sessions: string;
+ };
readonly indexPatterns: {
readonly introduction: string;
readonly fieldFormattersNumber: string;
diff --git a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/main.test.tsx b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/main.test.tsx
index dcc39e9fb385a..13dfe12ccb5e4 100644
--- a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/main.test.tsx
+++ b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/main.test.tsx
@@ -60,7 +60,7 @@ describe('Background Search Session Management Main', () => {
ELASTIC_WEBSITE_URL: `boo/`,
DOC_LINK_VERSION: `#foo`,
links: {
- elasticsearch: { asyncSearch: `mock-url` } as any,
+ search: { sessions: `mock-url` } as any,
} as any,
};
diff --git a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/documentation.ts b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/documentation.ts
index 38db89e88a6e1..545c7f7ec26b2 100644
--- a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/documentation.ts
+++ b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/documentation.ts
@@ -12,8 +12,7 @@ export class AsyncSearchIntroDocumentation {
constructor(docs: DocLinksStart) {
const { links } = docs;
- // TODO: There should be Kibana documentation link about Search Sessions in Kibana
- this.docUrl = links.elasticsearch.asyncSearch;
+ this.docUrl = links.search.sessions;
}
public getElasticsearchDocLink() {
From 8426024f6e802774509d7069024da7103399cadc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ester=20Mart=C3=AD=20Vilaseca?=
Date: Tue, 4 May 2021 10:50:12 +0200
Subject: [PATCH 018/106] [Metrics UI] Don't mount AlertFlyout and
NodeContextPopover when they aren't visible (#99039)
---
.../inventory_view/components/waffle/node.tsx | 46 +++++++++++--------
.../components/waffle/node_context_menu.tsx | 24 +++++-----
2 files changed, 39 insertions(+), 31 deletions(-)
diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/node.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/node.tsx
index d6934c6846b79..4484e1281978c 100644
--- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/node.tsx
+++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/node.tsx
@@ -106,26 +106,32 @@ export class Node extends React.PureComponent {
-
-
+
+ {this.state.isOverlayOpen && (
+
+ )}
+
+ {isAlertFlyoutVisible && (
+
+ )}
>
);
}
diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/node_context_menu.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/node_context_menu.tsx
index 95b78e222a72b..6e33c270d8e7a 100644
--- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/node_context_menu.tsx
+++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/node_context_menu.tsx
@@ -201,17 +201,19 @@ export const NodeContextMenu: React.FC = withTheme
-
+ {flyoutVisible && (
+
+ )}
>
);
}
From d03176e1c37b937cdffffcfccf3c67d54b7fc345 Mon Sep 17 00:00:00 2001
From: Marco Liberati
Date: Tue, 4 May 2021 11:25:16 +0200
Subject: [PATCH 019/106] [Lens] Avoid to have fieldless operations by restored
last valid state (#98588)
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
.../dimension_panel/dimension_editor.tsx | 10 ----------
.../dimension_panel/dimension_panel.test.tsx | 17 ++---------------
.../dimension_panel/field_select.tsx | 4 ++--
3 files changed, 4 insertions(+), 27 deletions(-)
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx
index 2c503a7bd6967..b74e97df4a895 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx
@@ -27,7 +27,6 @@ import {
getOperationDisplay,
insertOrReplaceColumn,
replaceColumn,
- deleteColumn,
updateColumnParam,
resetIncomplete,
FieldBasedIndexPatternColumn,
@@ -422,15 +421,6 @@ export function DimensionEditor(props: DimensionEditorProps) {
: (selectedColumn as FieldBasedIndexPatternColumn)?.sourceField
}
incompleteOperation={incompleteOperation}
- onDeleteColumn={() => {
- setStateWrapper(
- deleteColumn({
- layer: state.layers[layerId],
- columnId,
- indexPattern: currentIndexPattern,
- })
- );
- }}
onChoose={(choice) => {
setStateWrapper(
insertOrReplaceColumn({
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx
index 5e79fde0fa8fa..f80b12aecabde 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx
@@ -1795,7 +1795,7 @@ describe('IndexPatternDimensionEditorPanel', () => {
);
});
- it('should clear the dimension when removing the selection in field combobox', () => {
+ it('should keep the latest valid dimension when removing the selection in field combobox', () => {
wrapper = mount();
act(() => {
@@ -1805,20 +1805,7 @@ describe('IndexPatternDimensionEditorPanel', () => {
.prop('onChange')!([]);
});
- expect(setState).toHaveBeenCalledWith(
- {
- ...state,
- layers: {
- first: {
- indexPatternId: '1',
- columns: {},
- columnOrder: [],
- incompleteColumns: {},
- },
- },
- },
- { shouldRemoveDimension: false, shouldReplaceDimension: false }
- );
+ expect(setState).not.toHaveBeenCalled();
});
it('allows custom format', () => {
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/field_select.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/field_select.tsx
index ffd4ac2498133..b80d90ba78b1d 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/field_select.tsx
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/field_select.tsx
@@ -39,7 +39,7 @@ export interface FieldSelectProps extends EuiComboBoxProps void;
- onDeleteColumn: () => void;
+ onDeleteColumn?: () => void;
existingFields: IndexPatternPrivateState['existingFields'];
fieldIsInvalid: boolean;
markAllFieldsCompatible?: boolean;
@@ -195,7 +195,7 @@ export function FieldSelect({
singleSelection={{ asPlainText: true }}
onChange={(choices) => {
if (choices.length === 0) {
- onDeleteColumn();
+ onDeleteColumn?.();
return;
}
From c815e4cd9b9c3fbb0c8f9ee609e7bc3c1ed64e51 Mon Sep 17 00:00:00 2001
From: Mikhail Shustov
Date: Tue, 4 May 2021 11:53:43 +0200
Subject: [PATCH 020/106] Use PiT for outdated document search (#98797)
* use PiT for OUTDATED_DOCUEMENT_SEARCH step
* update tests
* fix typo
* fix so migrations unit tests
* TEMP: use wait_for when transformin docs
* add a step to refresh target index if transformed outdated docs
* add unit tests
* refresh before searching outdated docs
* add integration test for outdated docs migration
* add a step to refresh target index if transformed outdated docs
* make query required
* address comments
---
.../migrations/core/document_migrator.ts | 2 +-
.../migrations/core/elastic_index.ts | 2 +-
.../migrations/kibana/kibana_migrator.test.ts | 4 +-
.../migrationsv2/actions/index.test.ts | 19 +-
.../migrationsv2/actions/index.ts | 40 ++--
.../integration_tests/actions.test.ts | 177 +++++++++------
.../8.0.0_migrated_with_outdated_docs.zip | Bin 0 -> 48381 bytes
.../integration_tests/outdated_docs.test.ts | 140 ++++++++++++
.../migrations_state_action_machine.test.ts | 204 ++++++++---------
.../saved_objects/migrationsv2/model.test.ts | 206 +++++++++++-------
.../saved_objects/migrationsv2/model.ts | 75 +++++--
.../server/saved_objects/migrationsv2/next.ts | 54 +++--
.../saved_objects/migrationsv2/types.ts | 124 +++++++----
13 files changed, 691 insertions(+), 356 deletions(-)
create mode 100644 src/core/server/saved_objects/migrationsv2/integration_tests/archives/8.0.0_migrated_with_outdated_docs.zip
create mode 100644 src/core/server/saved_objects/migrationsv2/integration_tests/outdated_docs.test.ts
diff --git a/src/core/server/saved_objects/migrations/core/document_migrator.ts b/src/core/server/saved_objects/migrations/core/document_migrator.ts
index 8e538f6e12384..1dd4a8fbf6388 100644
--- a/src/core/server/saved_objects/migrations/core/document_migrator.ts
+++ b/src/core/server/saved_objects/migrations/core/document_migrator.ts
@@ -169,7 +169,7 @@ export class DocumentMigrator implements VersionedTransformer {
}
/**
- * Gets the latest version of each migratable property.
+ * Gets the latest version of each migrate-able property.
*
* @readonly
* @type {SavedObjectsMigrationVersion}
diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.ts b/src/core/server/saved_objects/migrations/core/elastic_index.ts
index 44dd60097f1cd..76fdd5e73d804 100644
--- a/src/core/server/saved_objects/migrations/core/elastic_index.ts
+++ b/src/core/server/saved_objects/migrations/core/elastic_index.ts
@@ -30,7 +30,7 @@ export interface FullIndexInfo {
// When migrating from the outdated index we use a read query which excludes
// saved objects which are no longer used. These saved objects will still be
-// kept in the outdated index for backup purposes, but won't be availble in
+// kept in the outdated index for backup purposes, but won't be available in
// the upgraded index.
export const excludeUnusedTypesQuery: estypes.QueryContainer = {
bool: {
diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts
index c6dfd2c2d1809..37cea5d2de3d2 100644
--- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts
+++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts
@@ -344,13 +344,13 @@ const mockV2MigrationOptions = () => {
options.client.openPointInTime = jest
.fn()
- .mockImplementationOnce(() =>
+ .mockImplementation(() =>
elasticsearchClientMock.createSuccessTransportRequestPromise({ id: 'pit_id' })
);
options.client.closePointInTime = jest
.fn()
- .mockImplementationOnce(() =>
+ .mockImplementation(() =>
elasticsearchClientMock.createSuccessTransportRequestPromise({ succeeded: true })
);
diff --git a/src/core/server/saved_objects/migrationsv2/actions/index.test.ts b/src/core/server/saved_objects/migrationsv2/actions/index.test.ts
index b144905cf01ad..84862839f7ed2 100644
--- a/src/core/server/saved_objects/migrationsv2/actions/index.test.ts
+++ b/src/core/server/saved_objects/migrationsv2/actions/index.test.ts
@@ -18,7 +18,7 @@ describe('actions', () => {
jest.clearAllMocks();
});
- // Create a mock client that rejects all methods with a 503 statuscode
+ // Create a mock client that rejects all methods with a 503 status code
// response.
const retryableError = new EsErrors.ResponseError(
elasticsearchClientMock.createApiResponse({
@@ -92,7 +92,7 @@ describe('actions', () => {
describe('readWithPit', () => {
it('calls catchRetryableEsClientErrors when the promise rejects', async () => {
- const task = Actions.readWithPit(client, 'pitId', Option.none, 10_000);
+ const task = Actions.readWithPit(client, 'pitId', { match_all: {} }, 10_000);
try {
await task();
} catch (e) {
@@ -134,7 +134,7 @@ describe('actions', () => {
'my_target_index',
Option.none,
false,
- Option.none
+ {}
);
try {
await task();
@@ -263,4 +263,17 @@ describe('actions', () => {
expect(catchRetryableEsClientErrors).toHaveBeenCalledWith(retryableError);
});
});
+
+ describe('refreshIndex', () => {
+ it('calls catchRetryableEsClientErrors when the promise rejects', async () => {
+ const task = Actions.refreshIndex(client, 'target_index');
+ try {
+ await task();
+ } catch (e) {
+ /** ignore */
+ }
+
+ expect(catchRetryableEsClientErrors).toHaveBeenCalledWith(retryableError);
+ });
+ });
});
diff --git a/src/core/server/saved_objects/migrationsv2/actions/index.ts b/src/core/server/saved_objects/migrationsv2/actions/index.ts
index 049cdc41b7527..43afa746de140 100644
--- a/src/core/server/saved_objects/migrationsv2/actions/index.ts
+++ b/src/core/server/saved_objects/migrationsv2/actions/index.ts
@@ -452,21 +452,18 @@ export interface ReadWithPit {
/*
* Requests documents from the index using PIT mechanism.
- * Filter unusedTypesToExclude documents out to exclude them from being migrated.
* */
export const readWithPit = (
client: ElasticsearchClient,
pitId: string,
- /* When reading we use a source query to exclude saved objects types which
- * are no longer used. These saved objects will still be kept in the outdated
- * index for backup purposes, but won't be available in the upgraded index.
- */
- unusedTypesQuery: Option.Option,
+ query: estypes.QueryContainer,
batchSize: number,
- searchAfter?: number[]
+ searchAfter?: number[],
+ seqNoPrimaryTerm?: boolean
): TaskEither.TaskEither => () => {
return client
.search({
+ seq_no_primary_term: seqNoPrimaryTerm,
body: {
// Sort fields are required to use searchAfter
sort: {
@@ -479,8 +476,7 @@ export const readWithPit = (
// Improve performance by not calculating the total number of hits
// matching the query.
track_total_hits: false,
- // Exclude saved object types
- query: Option.isSome(unusedTypesQuery) ? unusedTypesQuery.value : undefined,
+ query,
},
})
.then((response) => {
@@ -531,6 +527,7 @@ export const transformDocs = (
transformRawDocs: TransformRawDocs,
outdatedDocuments: SavedObjectsRawDoc[],
index: string,
+ // used for testing purposes only
refresh: estypes.Refresh
): TaskEither.TaskEither<
RetryableEsClientError | IndexNotFound | TargetIndexHadWriteBlock,
@@ -551,6 +548,22 @@ export interface ReindexResponse {
taskId: string;
}
+/**
+ * Wait for Elasticsearch to reindex all the changes.
+ */
+export const refreshIndex = (
+ client: ElasticsearchClient,
+ targetIndex: string
+): TaskEither.TaskEither => () => {
+ return client.indices
+ .refresh({
+ index: targetIndex,
+ })
+ .then(() => {
+ return Either.right({ refreshed: true });
+ })
+ .catch(catchRetryableEsClientErrors);
+};
/**
* Reindex documents from the `sourceIndex` into the `targetIndex`. Returns a
* task ID which can be tracked for progress.
@@ -569,7 +582,7 @@ export const reindex = (
* are no longer used. These saved objects will still be kept in the outdated
* index for backup purposes, but won't be available in the upgraded index.
*/
- unusedTypesQuery: Option.Option
+ unusedTypesQuery: estypes.QueryContainer
): TaskEither.TaskEither => () => {
return client
.reindex({
@@ -584,10 +597,7 @@ export const reindex = (
// Set reindex batch size
size: BATCH_SIZE,
// Exclude saved object types
- query: Option.fold(
- () => undefined,
- (query) => query
- )(unusedTypesQuery),
+ query: unusedTypesQuery,
},
dest: {
index: targetIndex,
@@ -997,6 +1007,8 @@ interface SearchForOutdatedDocumentsOptions {
* Search for outdated saved object documents with the provided query. Will
* return one batch of documents. Searching should be repeated until no more
* outdated documents can be found.
+ *
+ * Used for testing only
*/
export const searchForOutdatedDocuments = (
client: ElasticsearchClient,
diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts
index b31f20950ae77..832d322037465 100644
--- a/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts
+++ b/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts
@@ -408,7 +408,7 @@ describe('migration actions', () => {
'reindex_target',
Option.none,
false,
- Option.none
+ { match_all: {} }
)()) as Either.Right;
const task = waitForReindexTask(client, res.right.taskId, '10s');
await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -440,13 +440,13 @@ describe('migration actions', () => {
'reindex_target_excluded_docs',
Option.none,
false,
- Option.of({
+ {
bool: {
must_not: ['f_agent_event', 'another_unused_type'].map((type) => ({
term: { type },
})),
},
- })
+ }
)()) as Either.Right;
const task = waitForReindexTask(client, res.right.taskId, '10s');
await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -477,7 +477,7 @@ describe('migration actions', () => {
'reindex_target_2',
Option.some(`ctx._source.title = ctx._source.title + '_updated'`),
false,
- Option.none
+ { match_all: {} }
)()) as Either.Right;
const task = waitForReindexTask(client, res.right.taskId, '10s');
await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -510,7 +510,7 @@ describe('migration actions', () => {
'reindex_target_3',
Option.some(`ctx._source.title = ctx._source.title + '_updated'`),
false,
- Option.none
+ { match_all: {} }
)()) as Either.Right;
let task = waitForReindexTask(client, res.right.taskId, '10s');
await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -527,7 +527,7 @@ describe('migration actions', () => {
'reindex_target_3',
Option.none,
false,
- Option.none
+ { match_all: {} }
)()) as Either.Right;
task = waitForReindexTask(client, res.right.taskId, '10s');
await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -577,7 +577,7 @@ describe('migration actions', () => {
'reindex_target_4',
Option.some(`ctx._source.title = ctx._source.title + '_updated'`),
false,
- Option.none
+ { match_all: {} }
)()) as Either.Right;
const task = waitForReindexTask(client, res.right.taskId, '10s');
await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -627,7 +627,7 @@ describe('migration actions', () => {
'reindex_target_5',
Option.none,
false,
- Option.none
+ { match_all: {} }
)()) as Either.Right;
const task = waitForReindexTask(client, reindexTaskId, '10s');
@@ -662,7 +662,7 @@ describe('migration actions', () => {
'reindex_target_6',
Option.none,
false,
- Option.none
+ { match_all: {} }
)()) as Either.Right;
const task = waitForReindexTask(client, reindexTaskId, '10s');
@@ -677,14 +677,9 @@ describe('migration actions', () => {
});
it('resolves left index_not_found_exception if source index does not exist', async () => {
expect.assertions(1);
- const res = (await reindex(
- client,
- 'no_such_index',
- 'reindex_target',
- Option.none,
- false,
- Option.none
- )()) as Either.Right;
+ const res = (await reindex(client, 'no_such_index', 'reindex_target', Option.none, false, {
+ match_all: {},
+ })()) as Either.Right;
const task = waitForReindexTask(client, res.right.taskId, '10s');
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
@@ -704,7 +699,7 @@ describe('migration actions', () => {
'existing_index_with_write_block',
Option.none,
false,
- Option.none
+ { match_all: {} }
)()) as Either.Right;
const task = waitForReindexTask(client, res.right.taskId, '10s');
@@ -726,7 +721,7 @@ describe('migration actions', () => {
'existing_index_with_write_block',
Option.none,
true,
- Option.none
+ { match_all: {} }
)()) as Either.Right;
const task = waitForReindexTask(client, res.right.taskId, '10s');
@@ -748,7 +743,7 @@ describe('migration actions', () => {
'reindex_target',
Option.none,
false,
- Option.none
+ { match_all: {} }
)()) as Either.Right;
const task = waitForReindexTask(client, res.right.taskId, '0s');
@@ -775,7 +770,7 @@ describe('migration actions', () => {
'reindex_target_7',
Option.none,
false,
- Option.none
+ { match_all: {} }
)()) as Either.Right;
await waitForReindexTask(client, res.right.taskId, '10s')();
@@ -840,7 +835,7 @@ describe('migration actions', () => {
const readWithPitTask = readWithPit(
client,
pitResponse.right.pitId,
- Option.none,
+ { match_all: {} },
1000,
undefined
);
@@ -856,7 +851,7 @@ describe('migration actions', () => {
const readWithPitTask = readWithPit(
client,
pitResponse.right.pitId,
- Option.none,
+ { match_all: {} },
3,
undefined
);
@@ -865,14 +860,14 @@ describe('migration actions', () => {
await expect(docsResponse.right.outdatedDocuments.length).toBe(3);
});
- it('it excludes documents not matching the provided "unusedTypesQuery"', async () => {
+ it('it excludes documents not matching the provided "query"', async () => {
const openPitTask = openPit(client, 'existing_index_with_docs');
const pitResponse = (await openPitTask()) as Either.Right;
const readWithPitTask = readWithPit(
client,
pitResponse.right.pitId,
- Option.some({
+ {
bool: {
must_not: [
{
@@ -887,7 +882,7 @@ describe('migration actions', () => {
},
],
},
- }),
+ },
1000,
undefined
);
@@ -904,8 +899,93 @@ describe('migration actions', () => {
`);
});
+ it('only returns documents that match the provided "query"', async () => {
+ const openPitTask = openPit(client, 'existing_index_with_docs');
+ const pitResponse = (await openPitTask()) as Either.Right;
+
+ const readWithPitTask = readWithPit(
+ client,
+ pitResponse.right.pitId,
+ {
+ match: { title: { query: 'doc' } },
+ },
+ 1000,
+ undefined
+ );
+
+ const docsResponse = (await readWithPitTask()) as Either.Right;
+
+ expect(docsResponse.right.outdatedDocuments.map((doc) => doc._source.title).sort())
+ .toMatchInlineSnapshot(`
+ Array [
+ "doc 1",
+ "doc 2",
+ "doc 3",
+ ]
+ `);
+ });
+
+ it('returns docs with _seq_no and _primary_term when specified', async () => {
+ const openPitTask = openPit(client, 'existing_index_with_docs');
+ const pitResponse = (await openPitTask()) as Either.Right;
+
+ const readWithPitTask = readWithPit(
+ client,
+ pitResponse.right.pitId,
+ {
+ match: { title: { query: 'doc' } },
+ },
+ 1000,
+ undefined,
+ true
+ );
+
+ const docsResponse = (await readWithPitTask()) as Either.Right;
+
+ expect(docsResponse.right.outdatedDocuments).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ _seq_no: expect.any(Number),
+ _primary_term: expect.any(Number),
+ }),
+ ])
+ );
+ });
+
+ it('does not return docs with _seq_no and _primary_term if not specified', async () => {
+ const openPitTask = openPit(client, 'existing_index_with_docs');
+ const pitResponse = (await openPitTask()) as Either.Right;
+
+ const readWithPitTask = readWithPit(
+ client,
+ pitResponse.right.pitId,
+ {
+ match: { title: { query: 'doc' } },
+ },
+ 1000,
+ undefined
+ );
+
+ const docsResponse = (await readWithPitTask()) as Either.Right;
+
+ expect(docsResponse.right.outdatedDocuments).toEqual(
+ expect.arrayContaining([
+ expect.not.objectContaining({
+ _seq_no: expect.any(Number),
+ _primary_term: expect.any(Number),
+ }),
+ ])
+ );
+ });
+
it('rejects if PIT does not exist', async () => {
- const readWithPitTask = readWithPit(client, 'no_such_pit', Option.none, 1000, undefined);
+ const readWithPitTask = readWithPit(
+ client,
+ 'no_such_pit',
+ { match_all: {} },
+ 1000,
+ undefined
+ );
await expect(readWithPitTask()).rejects.toThrow('illegal_argument_exception');
});
});
@@ -973,49 +1053,6 @@ describe('migration actions', () => {
});
});
- describe('searchForOutdatedDocuments', () => {
- it('only returns documents that match the outdatedDocumentsQuery', async () => {
- expect.assertions(2);
- const resultsWithQuery = ((await searchForOutdatedDocuments(client, {
- batchSize: 1000,
- targetIndex: 'existing_index_with_docs',
- outdatedDocumentsQuery: {
- match: { title: { query: 'doc' } },
- },
- })()) as Either.Right).right.outdatedDocuments;
- expect(resultsWithQuery.length).toBe(3);
-
- const resultsWithoutQuery = ((await searchForOutdatedDocuments(client, {
- batchSize: 1000,
- targetIndex: 'existing_index_with_docs',
- outdatedDocumentsQuery: undefined,
- })()) as Either.Right).right.outdatedDocuments;
- expect(resultsWithoutQuery.length).toBe(5);
- });
- it('resolves with _id, _source, _seq_no and _primary_term', async () => {
- expect.assertions(1);
- const results = ((await searchForOutdatedDocuments(client, {
- batchSize: 1000,
- targetIndex: 'existing_index_with_docs',
- outdatedDocumentsQuery: {
- match: { title: { query: 'doc' } },
- },
- })()) as Either.Right).right.outdatedDocuments;
- expect(results).toEqual(
- expect.arrayContaining([
- expect.objectContaining({
- _id: expect.any(String),
- _seq_no: expect.any(Number),
- _primary_term: expect.any(Number),
- _source: expect.any(Object),
- }),
- ])
- );
- });
- // I haven't been able to find a way to reproduce a partial search result
- // it.todo('rejects if only partial search results can be obtained');
- });
-
describe('waitForPickupUpdatedMappingsTask', () => {
it('rejects if there are failures', async () => {
const res = (await pickupUpdatedMappings(
diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/archives/8.0.0_migrated_with_outdated_docs.zip b/src/core/server/saved_objects/migrationsv2/integration_tests/archives/8.0.0_migrated_with_outdated_docs.zip
new file mode 100644
index 0000000000000000000000000000000000000000..ca936c6448b43361a310d948309c54b398862bda
GIT binary patch
literal 48381
zcmd43W0Ymxk}jOKZQHhOSEACkZQHg{Y1_8#O51kcde7pD8=I~o6d6qkQJ%3q^F0Q@zI$kH+X=$~KNe;$JV
z52H95n_3&&I63Mv8vl(g=f7C`?`$>nE@QuLze29z@$(}l{LJ6PzH2fzAK0Gz(Wg7ucr
zWpZxMqD(muIz&&R`eUYtk?G4f$H(XqYq;IrDJ$(yC`Cpv-`}s&!ua*hE3uH*wUY2L
za&k81k+6c2O&A#rCu;pv0fAY6Nmwk<5)*c*J(@&ebcDZtxe-~J*OTA61Bgw)*;j}q
zTImJGO4MZ#=&Lc~Q~aBH|F>cQ$bXR}{!t9tAH^X5gJN_U_2~>v{*OwbyHd``%1E`a
z*vZP(jML3X-TR}Ur1YPY03ePzEV5kh?H4r;1CbjAmu*ilI5ZySl&CNWdEmvkFZ+Zh
z>S=@6C&dTk!bA6Yjrb8>k|uO2UhqkDf{FBg5X@KthrEi1j>>s=$qE_J2Nb3GGE)k&
z_kIEH3W6r0M#-cIaFJ{(^RIrx2j~jgFw26x&Ho_Zs*6$s{2$p2{}9qYk`v%={D42pHigObkSh02qmh(UpA|Lv0P}1AJ?RKwMP~GzxSyEOexN
z5MvFcd<4UN1ub$?BiZ3q3Q9Sc`D4dsqwxD@f+6d$GR(0vZx8*lLC_m#8)lgeX6Tot
zX{4t0Dry-b*_jam&uOTpsK+K{MVF>kvvhwZWsPT~DF4jr16>45HfCp;WoDUU-Ul(K
zq?4Sa-C<>6W@2DqWnpD+Ze%bpiDj2rl$MhYo*%LzXy-^KRIf>LYy>Eu*waFXN;f
zvUwG43n%qxd!;PXlB@0`yzK+5uBL+lXCdp+^8@-_jGPAemuLPRI)6EKlw96H+P{Pj
z>wiGU>5nh{MgQRDfBDn@19XI7a6WDk4$;VN!8}OQLH~vzUWLD~v1hFCh7hUqQwlU_
z$_x|NoPzW?X0j5HLN}9lmj1N@nuq|~Q_
zqtxpJFnWwW&no1EzdYhU;m7umkn;d!{`C#@9e052msO-Tz@zt;!#?|u*G
zRPyVn7>zUqU=5ty=%~0zb&yyj!uR9DTfp!R8jNDL)z=jHKXLn)NoZ2YbCdtgB>xTV
zPXA$%zXI=H^iSCQzqbgdD<;Pu4<4tL*@Jyhx=X|)5P+9@&mT=SIuaVP%yWYLV-hHt
z8{niIaHLAVTFm#n05~%;RDr`8!M{5-LBGtNg0mKZurvXVi73RB2>{Au;LjOM2xa%b
zNto6&ynFr+0^Uwi6#V7RdVdJK8kcK8Wx}l0E{Ez5^C{=+SNQN<|dyO?>bY^D+5@a-oq8
zSXma>*!Cv?xgqJy@=fzi2Xk~QQ`M8Q`c(Dx(QHkKfR;5>GSm{%Gb75gt2lZZDL9i^
zX@50x1VWd>Qcc*H<(Zllm<~hED`_R9Y4%#0nHcHmS(sVam|2+jO`_=*q-LqY=(p~H|2u!0)tj#1|B1i9GWd1wDxVPu0Duqpze-e&
z|5GGyQt_6-Hpb|I9~TQHky`<^KMA6|D8$2U6djOiuGU9l$eYn)65zk6d)QyQ~O0STR-{ChZUK0NuPUg
z#HK^ZTBs=h%JPnX>-L#+L3Y;vjmtJ^(9}m~VWxut
zT}acZv0;3`AA;JG5W2m~%2oAat=^D1P5-
zYJZm4LK1&s6?#sF(weGmEJS{c7^I&V!EJpeH024)GaAbQCeO`FmzP1XAy?dG~Q;e9TutFBrSWt
z9MO$eES@dA=NlT7f4b;kpsD1ClZMB0O38p=_#3jr5-)@L7r7T_9wL(DC|2rU)-i2m
z9O^l+;2gEvLrv(<$(5u2N>{whIB;SauN(YEWKs?{7-q?*vai{_G*bY9to3N&iK@it
zi7K8FvY5eCW3Q^2S$u%KSrR(DN
zQBlPJy+9BFct`B>poym^7iU?kB!pyuawv|k?bX$9QG`Y-FE1F^cU;G{J;4xJ1|#_i
zv4Dg?tQ8h-S+*~_8itqGY*2&{8JRFUR_7TNULom0c>M>-jT;-O#4DhH+E>|D;T
zXh=ahvqccPiN{|8f=`R_>#oP`)b-m&io`LK-i9Vv@1gf0yi2yhH^~Nb>&mn1V!dUh
z1Zol;icjPC>vE{B(3y8BMHh>Wh1>$Hdzk;GS;mW}(V>UgsrGZ`I&ys5gW|Zmi#b`j
zVc#@}7(eCv;o~5q25tM|HrckqtD5I}H|O4W<2SGP65A!7y0e|f_rb4N=rQ5f;_6ZL
z0DN!&_c*mE9|P7rcDtfe6ON(y?;%Fu
zm=!lR&Sz>O5YL$6R=si=;l{e3VQ&-51esG-E1wC{rC=kFMwpeNR2O{@HeYx$0Ds2x
zA$;1h9~fn2SO)ZY(@uxMlU+Q%!#Q#B#L#t5cbHnPcv&bQA)tlJIGTMYmZ2Y{RRVMy
zsqeb^9Yvv=hx=a=tf2?vDx3t6mZ?xV2A5=w)R=VVx`kl8f&bU(5$<6)!44e&zyRMr
z3s`?mkNzI8{>AjD$@~2#TlxC?gm%Oh?1T
z=+$(4!^BWpgabIe+sdGb*2760$Ta$Mpo9sNBcV>~~g7n*$4;C^#xB0#2
zG!4c=$@Tk6>vUa9J%(jU4qO_0@8FEykOW!I#oqthgwlN_hX4#P8iFIw3!X9?~ET
z%@3)D6Z}s_kr89+<(1t}vi^kMOdQiEMrtu%l|3$?Z7p;@vD=A^TrT>-G_Sn&(kdQY
z-AfKD+uzhCLyqmBd(ZTHkMGohI>B`Quc@Axbn}=AeuOtmC+N&gs!->y(dqd@kl=$7
z-D=_s**jmPXBr
zYQLFGvo2@0$zv0_ndA~ehd+5w&F(z+yJg&XZv`hTA#=%}Vmh?aZqD9~#x_NStqA<8
zX9|(SADblgf2am<8eP|J;QyFw5MrV7Jc~qQLw7MwJvkTo%Y9v
zwjo7`BSsuc^tJ=ilCh(fS|tV_f`CIl9|i=X$GzK^IUaab>0~=e)}&iOuhODr2|Jv7
zOm@F)q9*4t
zlCZFn_vy_g2TS%lNhM{aUk-#%?Wb>bpjlXWdm3dS3W^JBfZoqua-U7|EY?jX(y=S8
z%W=Qa_))n+?#ZKM=T3r?{6?(LSo)TnvYJPZg_ddDgL6uQyfmGs$xR_f3^h$m0>&$(
z4)to>Qkrt-9+Ok0NV$o*aT(ufu!xV?xyZ>{@H-dcI3bvA4nJ8%cHs@!fI>u=V42np
z#kEr0Eyb&ESj#x_{?$td`_>6;$nOG;FSul)4%mX^A_$sHP$gKktN~Xo_iJ+l!5Fxs`{=S>$af
z@8o)uESP%3JFUe+!by7FCcV)asvKLm0;e0pRYc`S8!1Ly03sm)Xp-HpS(@^CEK3|e
z$0KE;;jmRA$SFnF3KYbfF&oJr-vaAI4gyF(7O+Lxlm1O+07G5LZE*x{qHFlD(o`P&
zSW!D7A5yiV*g4RUr*X~vxN>8H^jMT9DgEqy@p(at$7?vqngrHnbV<7?f#(GKvqXCM
z#MHB5^A&b?aX3lD^(W}Vh*$G3f*Pm?1aInagILrVs6+JOf>o}O^E9J5$+bQZ$E#W+
z81O<<66#jJK|-ntv*4YV-v?D7m;H?o91Fw-1t3gA!wK?bHMjkYJ2JU-0olA~Ld+O4~L-^Yr8D)uI3=GM$)^Lq8`reQul0dZ#cyF*FCVVW9COoWV^Ha#vzo#G~{{
zFPGI#)$P7og(D~#CD&DFfssJUS#&T0F@Sw00=k&3>ws>}i>*$2C+cf~*L8x+EcC^N
z5!!>%H1*1H8i7mm7Nvbnmppbs4Ub6HmB7jpl7xF|ZS2PrC1B@^rWx0=t9lJOHA(1S
zrjdkj!@7zvtlPig>C^A8#fGDwb`Fy9jNS<=WhP433YO}g2~51MFiyIxI-@a*KuYyd
zSW+G{HB8fh%||6ViGQAR;w^mBZ$i)=vvHolNzd4W&jyF_>92YB5bi>lZn*}GsS*47
ziC@747qe-Emj^%%eqh_gmo#Cn#ks%$syb}QJh-rYkxB6L0HaCtWkO(RBMgR4?8>1R
z!M6e6nZkO)oO&|p2kM%q_Upz!2qXJ@4(VMc$y6GQ;-rI-mQyunk%e$c@H3z;?+f$Z
zxujS!V$}p9h{-~SgeMwyM4yVM3c!j4px?DV#WuTBS;wh$A(+
z?yWZuVfjwWbsF=am}k)Am*6#H&`w!N9d(N`(5)mfvZNa@yAGPg2n@$t>CxAd0F^}Q
zrOz<7%DnzC;Px0cD-W<2*5eLC|M0_21q~E-U_gn6nCfL<(-R5_`n9_FndqktE18%N
zsW0{fK^_%8lgZRF$za^rj5zU*VeFYg)^CeEMh^4LZnz#kU==V(`fO(|ZVKoL)eDCl
z99&{9ULqWjy&ikY*0+BYbQ;~ute4@*k23hvb)K@&dQPd*{EL(HLs&`Kjx0V6NzvFF4&}$k%n*=`fE|joJ
zwFv?#FJw{+WF^J9h!;hHc>5ejT7)3M%Y$lR_GJtx*+mO&@z^m&sr5UOCd>|
z0}L31?-GFL^MjfG|}&;1Tq(=K}g!;Rf0jgv#{EML=lw6=r1fjlpCi3*`F3
zL`(B)N20S*Yp&Z+JZ&vLs1JBjT%5tD3kfJ1oKPxl$Y|76Amm#^g8Kn4k=9)(&JNuO
zCkIS(5AqYJz!r@}MlW5d*;3C^1x5^^beBl8n#ER-<0LVF=xv-?7}`m2{YGr3PGoc1
z?y<6koTf2mHN&N*$ENdf$g%zz-ovLx7`jpUM~u9Yrfvhroj$
z4e#Z2H7&relU~NMm0Zi%zpP0`d2L`wTxAtIqj
zmgmZQe1W{;S;T;)++-_at-~Ip(QYv_4O9Jpqr++&MXO<4k+L=7`-|MRYz=cTAY0L}
zV^(&_Gpf2%5gH%Vb^D3u?&I1Lb;C!3aOEo|WR}xZjZezk#a(*1*qZuv!DCbECryqF3?zKu0|@;@LWpdkl0
z$9=@z6)B*Pt|TJALp)a?aKeLS@00-)xIK8x~T;~9Wi%1cnSb#KJ
z1M5#qSkSOUd5p@gCaVDAU1(un2{=g%G$gqB(4Do2%}>f@zEv-BOH`M%He)c3v69Ce
z=B!~;cVm7e67ycGt#oT6xP*RfDJS~?Oa8k}&`KBF!rf
zL904%$Okw4QXm}gO&QJ91Y2;##C(!4gg4HeP>?I6MS8fJfs13zm}x=;iTc0}8Z9b`
z5YY_YzwV&2r0qfrl0aWk=ZK?_6h1l^@);#wsk{q*Q@h>9B3hg3LrwlZln9=oU|jat
zqSYtGDnRQeRL~T;>FJ0M6=BE=rnIx)V=X3^a#VmQ$WjRQO5ryVgr|IXH25n
zjT|(zm&jA6eMOu^II`=n-)jSuVcv1fTarU1>aIl!CjevfM@^1rL{6{)eSyppO2VRz
z4g~Pb|0S<$%@DA;5mYM*P2(YrWf)*~&W8X;*aI&IybrOBo8rVRVEawc)Qu+vwpjX(
z#!%8PiW4Z6K*2g&3)jPSzYUxo-3Efs$AHB~dL#9Uv9?X=X)>=cEK}Ey^Hpb5>+=m=
z%X_buHug<`IPV+g_ija0l*Iw}9;~492#LYo!Vp?BJf*1+Q1a%@yShyebic31t0b9*
zI$DDuV*`Lkeec+-o?Z#8+#4l*Vq0i1)CB=3PM97cOJ?&?dz*d`cFxtmW
zXw86XLN(_|lKl-sd|J4sxAh4z0E1XjglLFvlyd%
z(;w83sMo(grZ;Us)=Ji&+v6I2wv(F%vP_N-dJm93@J$FkdER@+qsMpc7|@1$wI0X{
zYzgQ=I+!$wKqCv%YI*y{(^+-)g9>i7Vt#k3Q>#nr)^+$L5w^{4?!y{lD!1IF*Hpt^
zFQ$;Y$G1^^s6u(*M_sr?3n!Z&I+n%)fV*Q2e3KDIB%P|?C{Ky4W2N{*sMp@efzrG^
zdxYzcRxzaEG5<|6%u!L7!NNxHA5!0>i|#1sQu$`7C}pC991)EIy$_bLFF_1Q3mGu+}od@{t%2m4PWbJs4(BdbHec-cLXZpE2#i#g$-n
zORyo@QIGyOtK~rqO9$4bW2%HQ?g>+x1wdR0Iy;mab}9Z+^jqiZRSaA7XOiKftxp0h>Xk$SC;Y{?C7IwR^5xdQx6PFR!r(M@NS(q@0D%&Mn
zX;|XQj+mKu^$rl6m2Be%gxSkr&!oVDt6vU*$yq-U%tjZM$!;1!~(4Vw60qsy;t2g-$TU+rcZ96*{uI@Q}{*h4w{8y(ln|0%w=e
zQRrT8Bi&bynDpy-wE&dYuQ?cz^{oAl`a3ZYy_RsAi<|H6Q+n4QpW^V6m#&=WT}=~Qtq7wT{e0u`!P
zeDO2TfFcf@<`spOs$DG+LDjgj{S1OpwL#tZozX;CN?}VP3OTO~{YN^|7u@30LM3B^
z76jbmIc-JWiF}{I!@CGWvLE;m_|c0<rWESW-7{!hlW^<04=a!FXGFl%d4tYcvN=i#Ha7s|l
z(hVG)4Mb>foM=uC=x|HE-mm#~P^EWyyI~E4p^5^*A6O(VBUIsxJVT5l5+2ut30a`O
z;-J8`6jd_&I>HJ&v}sI`y(hIs&bUUMRWy24J;B>8VUa>Tv^LBQHuMH=G(wFGPPpT)
zbc7vnU>9y8EIweFcn2;)?icFp5$4%tLmo1b^l>6iyJ5iER~2%_?+o6W7N_JFB7@)%q_(
zDOHs|I0266h&$!Pz{E7t@3@Mdx-IPT?caWbX%4dg9_qNLHa<1d4205Ut^`1_3F
zR09sc=NEtn+cw-zBA7IFpn|(@L}7c>ae&dZ_qaIRps7C%cqNL!5XW7$#`ar2LjvGq
z_<-Di&j%gM(zEL_CGTx|_E?6bGcZLu{I+Ak_z1GYtIm0En_;_xAZ41AVu_cBisSo22U!VgJ&yWD64d_Q}0pilPzV4V+g|`*Ar_3IXrXRpx@w
z>7t)PN~E$3PyMcL
z2E9<~{`uFuoVY%dkqaqdLY|2X|
z|JDnvcZ}aDL6(&bgxsA8gRIad>xL1&4I3`$O16B4djCNpY54(`X36E=!TfE(aS(mB
zjc{XuC%@x;%o=nx?5_7=;E^@=S;Jy2g9%@uKr)&mJUVqP7vdTd<8uG%Y4Fj7yh}8a
zyZl(|(o}wLS_1S=XO!)R5Or+D@)CbtCgfE^oawM(DP3DMJU!6t2CBm?i9YJ<5By;U
zn-U@kIaMM(hydIL^bK6;c@;t?Xp8+F)HpK>X?$8p05=~&!Wcyf-1^%e*fJIRONcsk
zn4jS%v@Yr8M_~JvcEkiHb_+w33)#|`roTH1XD80XXj)`HCWmbiY`aaw$_uii3S5}J
zh7>qncogP7*)g3;3PkIh34C>k!B3=FGz`^Z%s7`>u%3d;5yr9tIZzdXu3Mi)rB=z%
z$>QGi-g=bQfs%U+N#-`s(DfvdJwN)sRZ?av-Y$!@r@*z)J#v6%W`bet}+cK)eq%
zqvb=WpN|^Vq5uJhro|Z&(teM?=w*Zpdhmc+wcrl5ZRbb;ui6_htzHSNx^7~Adm-`X
zO7-3JK=<9)d~|~^?C30#fBNnDZArK4CQMoo8@l3bfk>JP;LK`lg>DjO8iI>+m-&JY
zvl29o`I^AM-GTK7OSM3it!qO}0?9HX?2i!}CiBR&Rwb27?55a|ZRPtT$D3m*3uzdE
zPoMLF?Y@4A?0|bYwB-|EMHLzBDG!QJrh~a%+x)!x8vR+`o3;O9gSmY#L8Puj5YGEq
zz9hM!aRBH?8d#PCN6}7h-APf_3DE*?zkHZ68b(!1hkod6
zNehNafFeiGDYn4Y`>4H>s~0;hw(-@aZY|H=*mZOYyT5ct)uswD$l&a;-b)X8rvrHX
zrU@resge;I#$Lo)Bw%NPwCkh^ZLwALA!|xQ4y6e#Qz7?`|5)7X4$1HxsAbxvn;Dt<
z3?0zIhS_zfa_>Bh-ddlXMl7A*SPFq@GhZ_lRQ%f3$N>jl4}RAZ@53tc%O{BsfADYl
z{paV~OsZqk*+l**&(}4$S{_L>I`RFs6>_`O)&xAGhaNT@oXZvqZ(r8E+A@Y}1U*($
z$Lye_KI}Zfym~Jpdl=BcU&wk?Y3vwG@TA0QUPPECoH@#QRqWdtGXAF-N1y|=C2KK*
zq%U;cD`NJG8jmBHR9m4t#
zWhJ{z`Jvga#lR&o?T)_~tC`c6J8kG+4S19!SV_Q_EL8;;wCC1ru`6+lG}?nO`Vlsz
z;;S~F4S`T7rS^!aNJ9I9o2KM;cGhw8^_c5Z}dI!Bk
zhj<3{DM(y4076XG3^OHu^V0ER#t`ThGg7#F%`UAFfhyv)rV1Z*+!N?Cimlb)Dl}Ax
zMjQx=sXEgc@i2u2>sZAz%-M!ag2o^N8C{%P{?ZbJ;wBilIyogeDju{Vn3bKz6l>W#
z9TvvkqO~1N>q!qnC^kE=@&B{mY^9tFcjefdSC_t|VHbW;oK4pv0S^eF^09Y^)AJ3N
zzR3$of|8<{bX2?0t)JM;h{mqUXcHxe-v%
zPMDfk!r^6BhGEID+P5{y$fp&)S8yDB^Xf+9WE)w`O7wIHTlQY>m?0!b910qUQPvOF
zvlEIc%;mvK>RVN);vQONeGcj+D?HZ%aTe)H6?Y%czcK#Qj^6sBY+hAsj6{N;Zo$D&
zhCdE=KNawv+)!DI|6?qz6;muGpKnnU*q87?G-V2OfN3CTpV`lSFfIwUti?2-h4buD
zYPSbFjUQovI*tJu%~0{DnO?a}y0|W&CxsJ$dSNu|Eqc43wOa;FWZ+Zq_
zwzc9I`big!bjutaJgvUXfZz>mRUmjqS%}qAt=LRheC&Dflz?3TwMNe6)`VhD*@E(*
z{Gj>V{R^kdX8wRMmTg){Ye*4-aCiit8a13$y*A8*^K2TA53yRj_TK7+SX&jQkebzX
zZu9Ra4YCdDN#YBq(xtIsdcU^J^tT%vJyWI&gP6+T*MwP7rmH`(NyES%nA;9!Al0S!
zDzG(2Sfo99(#1*mfyx6L8FuL{wY+F
zmKTONi~Veuaa>}U;jtCh;)f&@R1H8UjCsGj-$ckS3KzWOfV8;o&l~r6q?iexh_>*R
zo#njk2UN3>#|*lo_&J0aadd0Y^90&GFb|?UxshG-S*M_O?zi7|JaUn7?rZw_CD|O1
zlUf|I_w?JwovAjao6xE1Br_HjIm0Myc2xLsq8G2VxJ=m0y>ryLCZte6OyJOI{fY%MmaZFFY7N!as3D-87sz4^_I>;x?g=GW#3=WR6_NtiC+A
zUS^PWYM;KdcMHowtb~Y}HyEAKdY>hf;Vm9ZP2owL{56b)ysZJ(Il0AKQLP|Dd?t=7
z83!vbwo5=v33_pT!^?{A_L~Hkxn}RK&!I^1L05!NE9!=s$kC2hh6OKe+JE0km*~{=
zq_@L)(TP3WDFZ3MHElUXl3_+!>Xk5iBC3Nga8hpLHl2FCbNp0%nspMe(wNi`5_s`e
z(qzkeqLzXv_Se0j0lh~E^QK~gJ7qXNdlDrYh^xB+tbNi$^~f^fnkN)rXlCI2`7L=B
zo7CoMgo^5eV0Z>%NfWTRTO$G4-%g%bThWSMW`}8ADY2k&BW(No#Aen|p&J^I4RbTY
z1w$Oe)-M-Rj~b<*kQiVh9pw@Ud;X0v%zj$tr!G-eF7^g$GVj%plK;UC8rNM!WY?5C
zXfWtDPhIcs^#0bB!LJo%lJJ7UI+$CD2YeB*D9$G&%hud?@JW&twJP~c*CU0G)iD+~
zmF&90$4LOtw;fK3u`#T*T$p3Xt@kB%xI&DYvj=}Mx5)8C3(2X%sijV}D9WxxKX^4s
zpM99MPvn8%nsqR<9&Our-&%m4qoxk6YUgSw;|d)0S_e_s-@9tm=g?;Z+f_?Lu+EBY
zddBA=)*Ize;}*?ysaiSxu$HRgno{Rp_h>X)IOgUt)tel)I=3Y;{Zik}&XLPCHoNDV
z)E1T%=#v#YmgiKL9l&i>1EDL5hRi$pS?=zb#!8S
z&GYp8Rtfs)Tu0(7UVF!W_e@p-JNS-EVdT}!fq=egU)3>n+*j&v&
zmbg<^=;B2cBrGA9Bp$OatUuF3U})D`Mqv|3w^2Ke>;-
z%n6mk5s^ycbbyYIZapt|==q71FrI#3rt=|?ld&R$=iF{EJ)h?2KB=K9Ei`?X)JJ69jfgVos{i}AQe&xIq6^E;jB<~4>}?Oh
zoC7!{ZQ9#phZ|!CtL*dZM$F-bC@D#
z{8scuKB$I#C*bD2=m(KkMm3fWiAnJK#t*0Pjb?)5MS6cvD|^B(jSb<$E%$+7SfDNN
z3UxJIZEcM%q4S~o*d5IM+)kW*el6@7zZy*c+R-`0eQo^3-Ng$-aMFu81!)XNu_jLx
z*CaMd6>j4!m#j4HTSKNbd}gkT2zK20mCE2M7d7i*-U1^5MFopo@Z(laQOC=vY?m4t
zY}qFhlENy;-VX1*ty#m=K#rJ#2)#`(*(oI6vR1V1>&8g^v1;Ow15SXDd+8&ZjLdK2
ztY0uOO<4nHhw8Ro9r(UyO9jKYr=!0Pb?fPyzzcnPN`~4s7tQ{xCSQLgLG!tl+dkMi
z4rF}@C26=vU*6vx(Pbp2^4ne^@Me>=7i%26*5imP!p+B1XB?5sn8fE+idKX@k;a{A
zio>)dEOBUf((VQX(brdGjuJ8&Jd2q>o|4lo;UTccAAt2o6O&G3)9Z4=>B?-&YhFC$
zk>9n?Evzz{DqHD#O-N34c&xYaL${w9V$+X2Rm6i%f+l)Qr5~-zGIlwxsmBLcSo!Ap
z%Oa2aS&@|;M(#>u%dHARuIYvQ6zr|vw{?)PllE4#stm!(O1^AH=mlQonW}=FNRoXT
zIabzzl>wkC#q(mkMxO#ezsghlz`9h`K&7015@t`DeP~Esc4J#QjpbhsLx*Ra2ZMo_
zYwvs1GscL?yKeY`yDa&H_yrq_>cudyc@oxb940F4iAy^9*SQ`v)x~<3nY%_U2fVK(
z@x*HcHHRUEHgKYmyvjYBFwLBn5pl|yb8757v_o_ZNMfhT{XQ0;frhZu?q<S0GK`lnufv6ZV4Nj60PRFj5()6YDvHb2-Vc^)VU#jb
zIRYuWov8?H(rNG)#JYhSj&W><#rGt;oEWSZ`lI#VFjlA>hrn%7hopK_&kQLw5NCk0
z-13!TkTg?+XM;Vya;>fyt5aHf4+^gBJuw$dPZKOtr<5Z%NvHk=ZW7$ZbrLQf<~ZCt
zMJ3T8e*FnFSmn$w^4?L0bjvOutyQ|85@TJ@1JZ0D!s>OirRcEU#WuiJoG!}^uM?!n
zMpTkC)^w>ya^mD|#X6Z#Y^n8#r1NDY*4V*Iqi)6pU+LQQ$Kk_mpcfS|UjZG#P0~Ew(^eJE&Hqzax)jJw`je_qt5jBcMB*qH6g5=7HgGmg=!B%Ws)yoem`=2qS
zDR2M)!5a)Th=!rwzq74NlPH+gj`F$aO%?AyURfwb3)BNoiFG&qfCPs
z`q^JlYc%$8l<7mJshGT!WlCnH&Y50d7~pimLE{J&
z&x?Z;a53~jaPyLbu-b5JHFw1KdoYCe(>`;X?1V{Pm86i~=#JpH0ZhMm*^2YouyJ#U
z#6bgfMFD26MK^(I;gOubQww1AB`Z40B`$ltM+DaIj_XL>bOXH)Y|EV7jSX#`$h#cQ
zvM`VDzf~Si+)baV`>HD(N!Qy*ge=2@ANdN@3}Y#PB9lZWY`A4~(WD3RRC%ZgcArM1
zUVZu=hL9AaTKNVHXV%Xl-hfJ!-+88@HsbpY|7Xhq3ZKC)Q?^Wa0;7TfP)6dqgP`iV
zMn(wK;B_-oUkgFlmfEshMBj%E)8a9^Px>ahOK83BcbPdbL=)Z`cy4BQJcKvPMI6oG
z{0>Ufn++ol8n-8*J5$=0UGQ3PtGg^4CwsE+yqw!kIkF+Q{h8XYPpXoET_cKN4KFb5
zjAq^In-4^()449#Kd(8B$3dGeB5>`Wim~0IPnLEd#B0KKzgOo3$U88>-YL=%x4K*O
zT5Gf|o+zcxw-~PMy$0)&88WNU@j}^SeZA?FocKYQRdy+7)yEuVxy!{xrS<}S89J6;
z&PBOOiJf_0d0@^EesQYuf;!?>&?+=C7vKbGeVxqgwh_Wubu#9;R(tW<4AlhL`e{GU
zay^)*6BCj1T{c<7R3KK&jn*caNGM+-lX`623TpTh?2dYG|;B!gMu<5fS)Op#p%62!4-<+3r)$>yZ2BK?pi8nk+&vs*6hm0q!oO%
zM8C1?q767jKWYKNFQa2L&K;8YUeSy+g>r7W{#!DiGMufBiW=EvP{86=w!$&M1_j*>&W$=?`}gE13_
z*SA9O22OtCliZ_$z#gW7!>>5&d!E2ZA^IsLUE)$F`b)>K`_kJ>d{i~}lZK%6Sf-1AdU2#WN3YI(74w(6g1hQ^`w~*9
zA`T!ffIjI`7sM7_eKmx8j;s4UU$E0|<2crmZSj>oxgb87
zij@7>SnfM@cIR+!=g*tEQry>zq1hk^Nvh4}cRT6e()-vi$AN@X=rqyG;m=}iF5dUaRNVcqj63tHuK!)uizuv}GyYla=WWDvl$
zv@xXx@=v1Agg`dP%sRL0MIGw(mQgBC@b?5ryVc_ZE73*HyvCUKs$E3HzKpS0Z=zr1
z>+#x#eF$Ekv~_oOdw@Oh^-gU*4^D>}{4FV0L=O=gGZYKkUJ3nmXn7GEN%e96y@g-n
zh`7;|sChHjuyA@n=JU?IY#j0K>pswawm$b&xV875^4vLLJ@an*jjK67mqXYdXGUk}T}7P*{p|EcV;{8KmlPYc)nepdc(MQi`PaIHzzQw3QK
zD_0T#va8XiMVh|d2v+w0p`dC}r@8|dK_jetQ
z%|c&uF|ry!l(58-`6B-#$||U1JCN!wSN06QA#a^N>wxFc{_J=Eq@LGQPFdRTb_XV0HnGOL-x5NRVFU6+SdeV
z8aY{jt`u4H!Bn7;Cb)ytLCwukU7DxpS=
z;=3(-l8bq^ZItJ}NT}!S5(*dJA2v%7>!}Bn&@INd_5rjFc*UnQThr{|h`bRFb7{%%
z(AK+KSqr5O9ORvmqXUfrJE+^QEW!vc#`jg`(k!eI!Q+x>=Y*IC(K>T*
zixg%0piY3~MS!P4ls8PYOaMGB=oEpgD&E!bvyF=Et*JCQ~gr
zg60pMXc7!DeslslPJVz%V4ZMVBaE=HgcLTBi5NNLjcti)<1wj-#c@XNVWh)-f0iCW
zPRdmBYt`BkVsP;unJO03N4GmOC#}n#PpXUdVI#r7pcB@;p7EEH5`U!pWUlo_E_9pg
z=sKC@7W2r%m#hC!_F=vLyTX`0h{EQ9imfVKJwK_s-}+M5
zRpr3A+IHqDt+~6!+}v@sD>Ol#t^{N{`*0x*>34YsZo3_c?A(fPw?X^wYT54(*MR}(
zl=WVV9w$3D$$GOlUFVX@5M|-~RQgJ;4bycbQ4zkdqFDDFm`&MO94MI&5r;ey!&jQv
zVN63#Xm($`gZmLnaw#ZujmVYkHk~aNyK7o%6TA)k_j8DamT#!G#SDCnCWjr+=Qya2
zdM`m;4e8i2OEY2fi=+rnT4~|Wp}Q(>rv-WD>%COwW7+7{h3sXOFUuL%jWZYaMa|sKjiu#{
z&r?_5?k-2!-!*a}BsH_!Zx<)oF*@4)eFy0y7B%LP9nDsWp35kPOF5o8kv#UZcSC$s
z9EWdx^FIqJg4ypn*h!pO
z$m2UEK1_8Xdvbk@e^+52(>9Gq_49iV#(FJSOO8X0_24)j%S`ZHN0#g`XIG3?_b<{z
zftgXY#n`r&+Q4En7j-Gtog5s}@KoP>?XS;2Wq%Hj4|kS3E(lc_*NEYBE_}yUKYUMn
zI_ZABOx#4zMLeqMY|pj64JGE9u5`ZTTSi^Am12~`
zU>&d~HVNooRgwz)R2Q$DBEg;3jxSj#{kHd2QCTm~o9=Fq?(S5&q`L*_Mp^^`rCYii
z=@JnHDd|p;R6(U%2?;^?y)PKRl`HtWe|){?J;&oYcb?hV-PzgMndftEs*rw66|hnJ
z*A!U-A|>GOT$}%|DL+rog7`y@xwVnGp|QgW!3uD8{*|WXycqKHVp#5I$lO)rmRB)j
zwG-ztl9JHpa^Wz&BnixU>2#Nrewrf&7&l%pgkO9XH^}r@7$HGGxS&8lIIaeAOh?1O
z@`DEA0{#Ja6-xzlb&OCg%XLuFHwLQVq!>YMKB&Uc8L)Z?0eKAR#6np2v#E@riNj-Kt_}I77m%3B+
zp6~tM3+c^CQ9m)whUxV-zK3yNK=FV4
z(gTzp9cC7Wig3SQVAqA5=i&k(DJ%Z*2k)oT!pbzTf<88x{@@H!EN
z;-l0?+`vHU5#0VWVRph@J+2R*D3Dq|g;NPVV)sBx$9R
z5_PTOS~PW<`q_>#aS0a}Pq84Cl=*1LybT@6v7Hb3*Z9iW!lX
ziBxBNx}9aC_W@*
zY|sBV@w~s{xquwaXIAwc#5aVYyu3E!I2c$;A7*J?H%9C-tRD9J=T@h(i!WGw4Jzm1Hjtjq}y*CJ!WsgO2gf#z0b2K97ZyVyi@l{I>o-=
zu7HGQ!MixZSt7}>+gv-VkqMc0-A<_~s~hgFvennEi0?W~+*Q_m_=yTUz@_~2HUm*}
znE>At70UfNtA#YqSIuToMo`;EDvr2s{3;gXWLW!3G-ek$g~|&R7Tr>(|g
z^hqc%b$-X8^NxT2eV!)64ijD=LTk_5r>@aO6|U{lg;ZMIxXB#pGOU-=4tXNCWA`Fn
z$9_HVZrz7YwKgyOw6?pRmS&B8XUV00L?kg_g4caR&Y_ZZCf;!*e_~a`;+C~%2MrS@
z3lD+T$6F37>s3nja=Tv)d_r7>pkOyt8K6MRhHnBM2UOUD8gszQl
zI)9|F`kXbHwK1$W+JsX0J+?`sOYTes&$moWGs(5SHLEAjDz;A#XoSrag5F!AmCEJB
zof~>ij(TS)z@gwlQ1qOGvFtOc62)S-=PX6r7W8;SgBp=JHq=Y6jpN8?w3DZJ!b``F
zM0MVOx}*4z$<0}QYbJ8X>7kw4GPEznwcOEEov2FZ65B|5-G!IYFXmp0jcOxIWlW`Y
zGD-%P?aB0FT6TTehP~|#tCVKxS!HCqBKe-Q-zC!)HY`-`gXS}Wq@n6fij*SGTZBeL
zo`d%xHUjkgmKnl5U^1yNJLtez)V%e$`4yk2bd1miz6cM{Q^HO|#g;Eo$kdvQuCqm6Mz3YM9qrkI?=_)
z6y+tIGxJ}bFbtQLf8{d<5q&XO1Q23L0Di((g|Rnw1n`uZ+gLML8S6V3JN%+#x!}M#
zrY!ph?+|=RuNm+V1hQsyE9UvPLPI`loJT?9)5*FY&
zyfLVG_Z~oXcfgP9s_LH{l0UhUuA=iBxD%aYZm?}CDbY!-n;G#lfq=Ievyk{fe$
z_?<^7Pyay+^
z5E%_EZI7dJASm#6F62922+7Gh0QRYd9uUwu1ek^cB1FMfo`IPof`Rx0u$;jCg?-#p
zvkg>kkn2`D+wn|;fbjplatv&YjsUCtQ9nhc=y>(ysI)$Sk_asZPlWaeHC-YwJ$+-=
z6;Ub7I66stZo+}eP^Fyxgv@#
z`{~=xZ-UXw-*$MI{)gGAjI0bgd$4Q(>{PLE89{%;PPKZo2mc(^9MWK6OgCU9#sG}Z
zdR4?5qn9Fo^wQ|@_;hY4OBEfAD;FT6DyJ
zvaBFgSqK*B-70w|WbAtw0o-rx@-klKF?RZPmC*_Ha)JB8s)Un)70}=;MDpgz;umHx
zM)Ie#>_Mu4Ht(nRke=GW!1XF#ylc__w`ODckImMPAP{D&_LAxHvDLVfo~Wb14rGd)y_UWq
zl;>w6%wSvx5u*$0C<&($UJOUejZCfqs$V882uUC~BBJ{2rXb}0s}LrSE&=~8*9>1I
z6#;(|9}`xvaTTzSwn7tq+#io=Bf)W;Y$bJ}&Y7-<9A2*PZ%xPcADfN|2e9G>m1vJ`
zcH?uqNY7K|0ImUP8fnjUnA(gIGUHR!|GtPG1Xy2Ot@^cQpX(G~KI+
zSE2WiIuXFpm$C6m!9^f^Go7>ai4R;KctSL+ep!_Wq>iAMLX6dfLgND!E@R=@@$B@C
zyTsGd>0WHl-5*z4RmGuY9)*jaNf
zRnD7&6hT13)p&&7U6KRsc$C77go;vJU8FHdTOoKpcs{c21B1Gj$1?~OaPVc$Z0M*@
z`yLP0M)tf@QCYQW-OCp1%r7$M$!^8f7QCaN=_CTqCxXt$CKnz%71{C*?6wFS892DI
zh_VBjAv&2!g)?~FOu330v1&5xM+~+uk>+;Z7v%NS9IXs&98}q34E5AprJ_t+0u(Xb
zogCcc{M-%0!@kE$7C`}}k82(LoZV9epGo)r?TTgnk5_Cz?#X69!wp#q*z8$?JRAyd
zh{TkEM6U%-#LSaCv~MK#>=KJ
zg$TBZKx)2DPy8xGsVWFrFAPaAfUgCDYQp9V*x=(3U#M^Ksqo{QnGp_AR#+;SN@U_C
z0-VSm>JnERkO5Z^QEzuh?y_%)VyCJ`-Hk
zH#2aRfLB2RZ#d5nk0Vf)8wYWK#i)dbndS&7H6JO{$Cf1D!BjUXB3X-Mu2CO
zVQD!!KZX1v!GdEBYyQh~SbLS*^opJHVPkkyE`NP!{?`=Oa0s>Z_s40fj-CEKFyfO+
z2V5<_5(S!E}1Psxvd6aF#$39z$(x=%CyPy-&2u+0|m{i4m+tLAhhqHV|Kt?N&r_AaN31qCq$j(x$Kp4K0YR_9`E>T!?FLjhb9jb@J0qWB7rwD
z!2MzKOn~evJnY*%D=WCM*Qu}v2?&2N_9A8S_If_0Z@3FeE7KBN#p30qgD2*IAp>}fN8tbRDtl51b-`7(S54OJ
z+D(kq1f6d~j<1KzSNy8^^Q;#eE7YD*-140o8V}&M_YQ~|Lxe0H3~k~Zd{Qe=@QyM)
zR@h90CM)+at6dn}-0YJt3LIZF#C>xKVYzScyU!|xWvBZjYPKcbJa>>tM{Y?<*_o_-
zU%RvYXwF*IKTWf|Tcep~J^Xgzaz&2BGYW80-NVDf)@#d;9_pG+)YlxjeL-fvS(~=%
z+@G^qU7%^A#%1yB%XOpxVlZ`qk5%aN>tyshzh8Q)vj|~
zot6JoV`X>B&tEmCYB4#?SsHouI
z5MLw)EqD?_e4E03?a%n!E5sI)Kj7)g1O8Y-Q+e)`QCaC>xukAC3w@EnZuWw>_a5^)
z7}%f?;bOIi`qo~gNPX&GHWIu%?Le<_C@Xbi6*nO%WDvcI`SQx<=;c=b*M(t$`aJ`(
zy>G=OT!>Z%nVvDQNei&rF|hTl0^AThYxFR7@>tw3F3z!qPJmK~8O6fjF(zYUZ?w0g
zvORRuayikfdVq2flHB-3R*&
zBG#|#L1s1-^|5XM6>1;-p5B)U4IR$jN&+FS=rOv|(k$Lsl=hX^z3&2zEDFeFgDh`u
zhT6VMzK)re7BY%dZ`YDsI+Y-7PGKE`m!l)aEnLkKr5cBAdDGU4*N>x0r4^!w)ryzU
z0m8C46xLCF$#DJyN^v6D*65w!=rtJn{1DAvhHRz9X7k0}XUXw*Ay$hQ**=Xqy<;8O
z+8WA_k$KE$5=OBRIKG%gHYPhODtw=z{+)c6ebdXVF)%mB^+~9=Uq}}>qwI&_vy5INEwvwcwFS7=c$76sXAiRGUXM?DgZPk^tMum6`6~OS6
zWx)*b$~aWJ1X3e@wZb7TpAX@ctidNap&jKMsx`_tYdM)u4nun1c=w~bf*sL799E|1
zMa054q8rc~9gHqBJ%@kgDHFWvS(~ov_`0NlxCv7gGJRFZBemZ!lVAojQ^fSypqe7n
z7a{8$o%}pmYA=abErnvTRJb0uK4jh=Qd{189PIgkkk>~sLxXE5(W(ejvSet$&;^mt
z*ZP`q52zD}XfUExe8H3Tu*KpvcNAgfnZ`|I&3J#~(p#YSOgmpEv4s#H9c6bsz#D7_
zJaqL((an#XRCoM(;O>3S%wq_+=%fRF0#^(ib(wWpbY<=u8e1E4GwU)l8adrLjvz1a
zL&x{@#qS@^_z;nWSj?aYKZ{F5Mg9x0%3MH%eK
z;NVozox4OZdK)1Kc?|?@yI~N=Ar!hmuH6vcIk&1$`X-AU;8wvA{=%&SQ(b>>t2zPK
zR54OrO*f{W-N0C7+9wO4gL~U%H<2SNR*{so_(bO}N2y&fjtLvr+i^;^&I6;U=Te`=
zLXfo{3S#Y3!%g*zq_y~ydQk_wgr(aO>Qd4k+9ih6ubjM5^}27S5}Co7IJv57-fgF5
zKG}6G<&dJ@W~VkrjP^QxTe+}enFk2{24k%-J4i@VO9N;DYZxy2fZtf`^XXkzYY
znk6Zr!9ra0MM!7$Sdy9ndcI%{d_L$@eXj)8K}0d-s+w|IrR;3pNO$f!B;I>TfmJ$K
z-_XX=C`{FYJCH$ljLw$j8s7w+z#C>vunmjA7U7YChb|PKohfx`zIxYfJsT`*r)~
z*HlO;%CY3CT{x=EqP}E4U`Y}=pOruxybzVv_(8ZdVT9naLhsLy%d&oge%dFruqrUA
z*s|O{97yzNG)m>+6w=f!fs!K;HzY?3O^1RIS_0RRtZRlxUu|@&S+sozV(C
zb)VX^+35?O(>pVQXLm{qbsnHZZnV~e76b+$E!gGgBzM}bWIl;XhjIe5c6xa5dDXhf
z+6LOm!G6A_`WrMAqt43xlvk_I$xtYrFvaCt*LV0phF%coy<(GojvU9MV;s5Eo>)8<
z%RMZ~mBO`z&V=@1zgFxchIvUr!)xoT9n*VHs^Pas&^=b9){YQ}T^)6ZWUIcwcE0aB
zOg?y~ax-sE`;h+$a{Oi`_)-TtqicJ;ruIFQ@R}&S2Nm#mMp}?#mTWywDsX8<7?wY)
zq}h)nI67E!Y=yw`-D{-=51f~54H>}Wj%4Q`=9sBzm0a=M#X^xc&39;)C!Y@}Duq#Mq?c3`m36F56KS;R!H0jKebc804HG
zL)dq3>pZ(9DT7E;RYh=cUB>AhAALw&0JB^z7H9sp1Nz$b3KJ(&?KcKB#(1`lO96
zxNL7ji(O6fem>%M`dnYT?)&o~?WvClZ*D1^8L@|HRiC{n6Jh{lnM(
ztDohZ^ZQSpxPi7(Y%-q7;05?rn{?hce5uLYHs-i3n-ZRtk^BwRyTq{yxOR$%xdHNoPKOc&(>fIU$b9?g~P7<1qQGcyOH}JhN0zVZC5d
z%O>KM0f*uROi<+uur#T{Q0}4KYD{+IWCr=62wBLaF7j{=Slh3vDvKeUbi`i>mG{Es
zv-EcO!q&QpMu0*$@={peVj8@YeY=+#^Ggxx(3X$UNQ>26rufh-I-62yJS&9QhetSO
z#E&gdKwf~{UNmEHPD0n|p_^|IS)hAFewTt|u^AP85!PBPmE^V=4PTHmzB(dgIzfyOHh-d}gP?TxrBS!ubX~!c64IYGM*)>|jm>
z{;#3wG4nWe>`4ctZwJwD@Pw~d`^X3fQCv@F2>pn_6Nh-`?WV3=lM9pO#7F=4H#Ky1
zGRp4s;MQniWlk^q$GJ4i>u4hgy(>eXc!S%ptcsGu$G@;UYpX5ks4Bg8*rQNJBj|!`
z+1ChN^I&!%*#nKEaWJJh%t)167m9=nYxg8@`wFbd_qCH98tgJriad>xTs&vjgy54@SZoy~r2OoJUNdN#VHH?p7q}a!*3`d^
z#QXYadg4p&S2UeF%>*M=3i9bW^WhI@OY29|o;7tkJ#=k)e_si2>=x1Uavb6Lux$d=
z0!ZGM46OCOCd%>|7LE87kuV}mFYYJq(rOz_)E@0mxK>yn9-@T%bax+Ngui2bpNH^G
zM|hh%=^&&wQ+I|21p$m}7L1HlJ7WYBY+)}>uhosP+-kN5_HfDjd9XEgj7eRY-v(KT
zj3oc8mC$Hve7jmtY5VJxEQ67nw}S~R(qAZEnnd^^x!vzw6H&L3IXd?M+a^_sa|Cc@
zUI1sn{uiMmqlxvec=4hf)y#KirVISc-aiAySK;0~{A|9Sfe{}CFqk``
zvamewLe5|VK0K9;T{DY4OBuV6?MZ_a(1=C(A{sL;C)-$!8ISpCJ9aKs+x#r>qi9Wt
zao&seo$^T1Jv6ed*43bLcbTVgae17lPZ!4|m>D2;U?0Eh^fY|%#_extK-G=aQb|ro
zYZoTFrM!t2Wv`^Hk$|JA-&Tf#%hFQqr2v8EL?JC{EJO<^6z~3UmsFRR`EXEbN?FN4
z2~ck{&V%F8iHd-ekE3&(SI6sCSVb8?IOlb%;LDYx0frX)8?Os|$@!DlmB$c5SzFW8
z6Kli{f|z}3Fa=Mb2t9!F*qWG_x`VV$l&V>-1a~53JS9~OMV3?CFa&$3B`P&wUEcYF
zb*(cCKN@mkKx6zjP`s7Y#fVh*Mo@IR1V?%GVRcm;g1Z_iDKk$<(J2B
ze94};^wB{5DBmbHP?~xnxw}?iCuc9
z`YPtoUj(_u9Zqz_bjkUdk=ZI~+rA&Vx!eoYr*O0}RJCcHCB81*Fv*V1o*?p%nRF
zb1#5&0Z9Uf?eNxvOWYu4BapQ8dH2+c&z^oT5qec?nWRcp;GyKJcBmpF!mYat&GSgN
zyS7ZB(046t2&utSx@6sYZSk`W4>}|Wwb{ifsRkFz68a<^R03xj4`Dc;uaoD5ho-bz
zH6}qW+f|h05B>jGg5Um1Fw1WmYS4>d>RLDn-8m8g$qcrt+k8`&s
zuvRb|>z|~jZecL!P%Mk
zzeinnNW8?eCEW07MUgzDJ-u!_4_ml{NI`zl)2UW2d$Sfh!4Q@%*(_vOyV&X-2rYk`
z7ng~@Y255Fwk$KrHSdU)!ayo-wzrzs-#TSrqf?Fdto7|Y-0VH-cdG%{9h7zLpNo_b
zs%r{J>@46B3kvXqv30Gj*!s}iKQ7?ExvYEblZjR%h&UJ4z&Bx0S-GG^{6U&zI^WUp
z4j+XQ5(4uwiKUK@52{u_UQb{3vAGjIu*T@oEUY4KPLq?RQin$Mc^y_03?JfJxgB(S
zgbC!M!fFz_(Jy%g#-f>Q#&;jkK5j?MnOCB5A%ts)8+auntVjZ48+8y)@Vx3x(25sb
zUylEPZd!+hz4#Co8q^@`JAnkhBH>}2Ad^r+k{FJ@wyct#p!=b~7
zX!WMt#gsN}96nRd?=h#=pN_89Zj(ds_ONCPymyt0PrTz(_mW1o%@wx<%KHPBo2e$}
z(Q+l>a>tFkVl!E#kLV8}-`R3YLQ4f}NJb(Hc1(aBgmNRp-!q2}EvcM*M(fD_)lWy+
zSvv;P8>^*H(Zx%IzMamnw@X^t7cNC4mTtosVQ*F~Gz(t$Zp##PoL&^tjF4s-bTiqV
zHSa>(=JtZ=u!9GpLj>tdUnvn(MU;dLG3FoP-K!F9u-hRq^SH4(0wRN>r)-1e@Se8}
zhrR6alm7ff3|>Mj*gzBMxB*sovimCa7+A`Na~%UuF>X8*QX*wxzkWx&b;ynqPaYnR>(>>U!mI6l;$+LX9a?O<
zzAs8ka_8vLGYh5i9$~sk;qB-3SrF|wwFJ7;`H=`}LgLKY&zA~Fwp66s)P?3|b;hJ(
zFeX2Qp^T|=3g2??C6*(>N
z!@d^GCZRwNHuX2BFr9!tw8!pTQ7JI1?HW8OaWz|T6%xKy#=+zk4~;D=s2fJ+lKpu-
z-oeG<=dCsQN7de6*ip!r$)yKYm!O
zSFPU?T?k_`VS-c9iov3=^}V?-sp_c3YjGdpYnjh6y#uA>^?T)3-xzVFOcWQZAwYgf
zCtZ+*I06^?Ozey*PYOc_Atto_A>%Qjs0yz{%6QM4adJz0R5z5lQP=k`KlS!-;X23M
zsxcYrBPICKB_jS(Z>$tfN!@C&RlNGl+ZBU^CcB&)qJ!CgGt2U-Y
zNLDW*qbBykNmrcuAd$Qk2dWU=lurEuYeNe}FrRAcsZ&2G%Sc$+_QLd&Tku6pMpX5y
zxeeDeuDjE8H%Q2uQRgPYwwB|-ma*=H8(weP7}N^!absouyyer!@V`u$ELE$GPo?JP((NGo7~@O@k~&5I`WB<>vXcG
z=2Q&t^$wZ~>B)iWpS>4`84@_Dzmi2dftp2^YafuViBzF@uO*Rz80^ujU;$iW{HlgB
zoe4B|$?*!d3j22*sS5U-tmd=BXsQ}=cdO=8A{O~!f_r#}vKc?*ig>*@NUq0zPdp3@
z1yQdGp2`1oa%BlLdI{3|t_MdDsM%~nvNho?0knU
z@8qpswxr>_ze_Np$}Unz%XkwxY`tiDfGVgoR(~tjI__GpwcG3$TK-8WO&V(SlBQ?S
z*A5MY$8>ACMO5t?U^{~S1!EJy0`ltrOdRVWoLoSC!*xYB6
za(W!cv2xqQ2=DpiHA+O%YhTSb$w%mYbYQd|)%#$3QiJUJnWfA7#IjTCRre1Mi_*&G
zgP_)uWJ4iBz@p``OYH2AX4Fi=e5i#Hj6!}T5VSYl;_||A2)S&VRkx}%f3vngbI)teWALoZy
zgE;l-e3rdEMATap#0dS>
zq0disOWdn&>Z|W}y2uZ|AB!+1Tx{>BBBEBh6=Q0tQGB?zzK)zda`V7>V{@6U{@jg=
zIft|)6W|0C|0W0kKCu2F2Ik;tyijl*Xq!@jQfV%H|znT3*aQew-99hD-2kT(J)+1%7PwxS$bHo1GH
z$Mw_BgHOG#Y;VhqhVb#_TAv2K6`n5^9h6GY(zvT4*xp^@AU|1>H7aiuD?pIQ_0pTL
zpiZDj+>N$8cxD@<=P#TiNoHG
zYe_xInzD1p^YhRpah76XfB`uGzdf(zTRf&ZEb(z_AtUlWBd?EVdSv+*AB%o!qzd$f
zv={n{8Zgv4f(HHHbZCte#xuG_0so*4^FYzMT!*3fT>Oc7T&(Jhi
zOXA18Q|dslvlTAzMwx&fe3A#Tsq1%B7jDi;jI;CRmoVI<=6ABU7~dK-41M16;MCdL
zMqM3UR^MUW2cI;v-!^W_FhE`GZWZ??A(+M^m<}~IWay`QjOL$~q0#_NyoKS#XPb7z
zix^%ng-TuXDa9>K5A>V4Yn_rY;We?votgs-T;+L)y%~O`mfn41I2@7)VcYrJ3hac7
zHZ{dcQEcFWmc7_-)^`IqSh@P{o-(_P-zFJ0h@LyX{)kOZhnJ%=M$ZmqLh%LBcX0;^hV3Ot&kPC0y_`|#ox!I0GGV7CH$v1!
zqa@3;K**Jh=}Bl`H!aZ+|0w_02$?w@cUs73t&VyO$iViZ^aI84>1MpPjo_kI;NJG%
zi@|!*gjnCfIDBv*(&-a-pq^O^X063HU-@inTRg(wqrGaPx3kD1SU{Wy=IJ+wXiqec
z1v0+r>ecsAtW%-d1Y&KOLuo`(pUL(`3cbt_6&I*OobJy$rgasPXr<52y5FfQWGqQ3
z_u#x?7v3MnC%=^ZMN_*z&wXc(>ugNmzTXJ<a`My3dof{Lf~(s^EXQ7L1Ec4^RvCT3SxpSE2PUYQa1qDrnAS-3%P-GyZyqvHfRu=VAI;
z&icQ&!_2w>JFMvY4htetp}oQmgYrj*u}ynd0&~vtylwCGMF4wD8t}VvT>{7Xx`hAm
zt*o%Y5C>WLn$7oOT<*iYB-V4fM@T$Y)duLD{-5anD@P8jtN2Y5ALEs3Do*q#vH?+P
z==)3rhm#J{Xe5edVk8JTBC+)G1F%Ysc^$iIXZ&33nfsXq{;#wBfC>2zv;Bn)W*v?A
zX$egYEggU%#2bMt1vgZKZ3F`vJO_g(HiGTZrMCtfFrWbylIYr9LDH`WS?O>PamJ>D
zmEos1H*j%ra6bBVH#%0$Q2A5LMduiBO@QQHk6#Z7!LMD16icWI6pLA>$QP@UVK0T6
z2|gHnJfmWw_qj2|RPRFolQaWuUr)bbMnrMGOQ0lAK24EKND=e2W+(Z$ZuW2M`)OVfeKLxd2-x-@W^=KgTvUroiG1j
z&XBvWsllaYqM#_v%F19W?c%7ea#QBsC3!jQ&YfQWV9vNe`suD_r$2SE0zqI#lAXfA
zKtM16_mzJzXZ$_nmKKrd%*KrKFgnXuGw5yMWtr4F~7Y
zg6?F(YKG#HG!()U;s}v3xd(NVwUodxsQm)*m!x|-W$5L6;qj9+qIKbv(9+ZsujwX7
zXhbU&JsubUs~F>{C*~+2Flj51OmLye$GqPumLEtVSiDXnNWt%CA`S(W{BH29AtaoY&Z_9t_9xbn1F~HVO_3ekK+!4v^-kYs3ZV
zp~C?DPI;&rVJ0>vo?LbcRVF}VpqZ_ZqJ@@)qQ!0d0czqZtqNxq;c8d`_e{!n+;!|HqL7_Ipc
zC_qQt{1KeDodkB)nW^&{aO;~PKY&2#t&t9)
z6TQWHaT!JpdJ*1;OT1ma>27Ip9sVn<&f!HOCJDdOC{
zdF`HI7C|X>#)tL%7u_L)PqqQi(6T{^O7&24{jaMJmtV{vRUUMH3CtER!<4sGXJrAD
zDD?MyE<;QtfACyLc!syROU-s|G3n`jd8z>sPNm?jcuZM{Uc$Q-6YA3|uhXn*q
z=^{c22ycRUH8I%NZi0Ekb@xp4bUV27)rE9FJY1vcf-Zo7vT-xHt#w%PzPL4}l_I~Y
z%N^Jb&@dEx3q?0NJ}&iZVKF4W@ct`MsS_?}6`idLCZ#Hcs0oCB46o5dXnVaKZBb
z&P@Px^S^lx1xAD?^3U(#e{d6ADF5^>zI!puuV;sUN&}V5d4+0BzqU8u2aH
z7BFY9MOt@+%>}}9WMGmauF)yF*!ug3l-Sq=#ewv)tt-D
z!@)G%bRdA?!G3TPoR~$NJw;6~a~uLlJmWAG#ikTh2EW0!DTRY*WRDJDtJw@ff2}l)
zE-k$=&89ph&FY{WERE&>AMc3hA{~Ub8RW3>dKyS+)8a+@>_Bj{T$UkA3$sbD2(GywUH^3?Zx4WctEYq0c2FfGY-wq`H~`&wB=r6mHg9ygM+_+&
z_5@XeBhBpiKynv6I0Pcxm+D~W);2~Q-?Qg{h`s7}CG&r<@Uh1@d11J~V*opmh6FKE
z(;V+HU>;aL*WeR{c=b%Q_>NA5unt`Ip~|g-O5=$hsi%?_k*bA^5bM~yi(;bkaKgx5
zrc7Sgs2Gg`DqF;aK98MMgqCDj4NYGgRrUu;9dc$cc)!?C@AoNGDe#MgX%
ziqH`ujN7r*HglaFe%TAksJvfb$13ubMdY<|M|)Pz8V1(3cFs-aW{pmk3Pcz{v8ltm_!yB@!4B>Rw
zbZ;1l{Sx*zJ2MlBB(sHOqE>NZ!MXqmVZ^knB880#UhNh63rJx{s1Hw4SLTW}5p#)K
zOdI4}dMj?^!M8PPhfY16+)flLjc&Gdy@Mr+=4w2uy!z=A+vx0b*nq+(ai9)y8Cv}W
z_nXq+7_7M~gi(dAJk0*O!MOS9!yzA8>M9DKC@yt1p}ZNR3y81?F*r7?q6hM=rj4)q
z?r-ny=BnD*d*$)yW?!?&)!NSQN_pZ`@X{RLE+|!he+0))ySc)x8Z~~8hocr$wpeXe
zF9w|Gb|?EFwOw1#H0Bg*(6W9Ob^d)V1wI&)7cfKPG|>-3yPP_2?`H0K2`SLo>2(Qv
zaeRa*&vV!f7H*Hs>am-__gzkjly!)77-FuLW712{hk1~=Tx+DvHQ+Xw=u5?BH#2Gw
z#Zr0~34QaCW&lm6GD)*>PuD2XzQG&3Pm^B~!8WxB-fMxwDAZSdRRBe)%lxDl{EpBP
zi?8vNN;L~Yzzc$C?RqqX8I&9
z9lG$pIQSfI&9Knmt;Vw(9p6kD-UulIZB~(#!O`3Tn+yTk6jC-TXxws_DxoUIew-VY
zBb@QYj7H-#k}y^fUAo%C4PGoA>2mbXC?;6gl=#i`*P|f2%v8kF
z;3L1sn|+^L@O1PEtdN)<-DYjfp5MU2HlF_r;o6$1@!n_F5TBdS^U2n-RuoxXnw++_
z9_oAl?f!I~0ZPN{!Bdt893WbNAK?YkejXGo$H5O!F7Wre?1SRJw8KaB0FqRVEsY?c
zJ$jK5R7?UC953mg=W(+MgiSPoz|b=w16#jU+Z6C})?iOXht>c!0owiibCH_ojA;W<
z5%@c&rh+sm7$Nvc+TOsu1w6M*5EW0Bb^mfCFD8{zcUJ8z{mVQ35|i
z{k(1hB>pCYbE4eIWfLe-baJ{5R3*dEp>b
zx#Up*F<1cM=x2nd0{-QA|?%t@&X7!CsI7tqcKegH^+ofiy@{{Ac%
zI8`BX=`-=aRd^h5K%bTSz3Utx_%}Dy)0O~kfR7{
zxrwoZ<9AX)fcW3+gwx`Ideg^JrNw;L`>Z{zAkT`|b^K}P3)kDPRK$7@_2VC5H|iO+
zWdY>8r_~0?{ch$zhy_}Ma5BPE5umOn;D>oeY%bv26Q{)jiIgvG3E+DEls3k11Tg6X
zfS|KW?k3Dx(I-m|5IbFNX%Nf?5`SO>0cqwxFL)Oax1H8mm*soy*PoXLaPH22a^r`4
z0Pa8v-gAP1b9bWfg+@McJLkPP))^xg0UmiyD}36mU2eUde65JA))D{@oxyl)nJ!^m%Y0;IAEjmVEp44CG%8@FbIWLdAXcu~MEHtGLYN
z#tKa4ULA|!3|7viv94&@vuWMamPG_KC?4;}KTIVv(|5=VlDz*m^j|Z4mOJ*e9z9pp
z<8-$6-@s3H4lrE^aQHv}FrBe`x!h%q@FSslGTFajSv`F^0zuD`yPh5~0U$8FoaKLl
z{`kvDIu$VLODJ%b!1c5O)qk(R&y?xE>F|5Y)1R@tvBG7I@FN%cZ-~DZ+F6#?)1&Ub
zBI0+>RsigUXUH$uvmB+TVMPJU?{c$!GSDCC*vsLbCHXv!>#BT3-0zg1m%}~Fka-%n
zLgk9M-}y2xhkKS^@-*&e)hptjuuWbL_AI^PX;?ErkN1_P_dD6*<#5lk6Q0J+(6}P*
zcdo+A;htspJB>RI2%oPs-V?6BKf?lj=2>#P)3Azwd(2f}f8HC}jt}QQxJzKrbC!ke
zG&Z@;)v$H{9DM$A9A`PmPQ#|$zB26bJIbA=vpKawn
zjm&CyMda^|-Iv2X+d_H<_wE&Ozc-X#4)<*9+8JDzE8<=$nQ^vT<}@xB;9C_}IW_<>
z-xYky*|Pl8xZ<8y#62m}|2H%LJ`uqAGpGjUZW_F<274|CbBUY!+4}U;LuU5A8n*69
zwfese`PV6VwmkbZY@W{*VNZ&+F9&U0oK?T4y)F#Bes9;aX)obNm&H15*+0Ai9jCX?d}-;l
zW%I8Jd2!n7!lD22TW8A>PwQb8dc~oiOM6`ieq#53PkSkZT^9UD;_1?9ud{W7r$=0M
zb;%AT=$UBqf&+<}=~Sxk
z4{w3LciCsdzUqXl;hqco{@_Z^KkH%q3^{?%S+t4YvHwx30XT^M@Q4hQe6~>Iv^AYC
yko void 0);
+}
+
+describe('migration v2', () => {
+ let esServer: kbnTestServer.TestElasticsearchUtils;
+ let root: Root;
+
+ beforeAll(async () => {
+ await removeLogFile();
+ });
+
+ afterAll(async () => {
+ if (root) {
+ await root.shutdown();
+ }
+ if (esServer) {
+ await esServer.stop();
+ }
+
+ await new Promise((resolve) => setTimeout(resolve, 10000));
+ });
+
+ it('migrates the documents to the highest version', async () => {
+ const migratedIndex = `.kibana_${pkg.version}_001`;
+ const { startES } = kbnTestServer.createTestServers({
+ adjustTimeout: (t: number) => jest.setTimeout(t),
+ settings: {
+ es: {
+ license: 'basic',
+ // original SO:
+ // {
+ // type: 'foo',
+ // foo: {},
+ // migrationVersion: {
+ // foo: '7.13.0',
+ // },
+ // },
+ // contains migrated index with 8.0 aliases to skip migration, but run outdated doc search
+ dataArchive: Path.join(__dirname, 'archives', '8.0.0_migrated_with_outdated_docs.zip'),
+ },
+ },
+ });
+
+ root = createRoot();
+
+ esServer = await startES();
+ const coreSetup = await root.setup();
+
+ coreSetup.savedObjects.registerType({
+ name: 'foo',
+ hidden: false,
+ mappings: { properties: {} },
+ namespaceType: 'agnostic',
+ migrations: {
+ '7.14.0': (doc) => doc,
+ },
+ });
+
+ const coreStart = await root.start();
+ const esClient = coreStart.elasticsearch.client.asInternalUser;
+
+ const migratedDocs = await fetchDocs(esClient, migratedIndex);
+
+ expect(migratedDocs.length).toBe(1);
+ const [doc] = migratedDocs;
+ expect(doc._source.migrationVersion.foo).toBe('7.14.0');
+ expect(doc._source.coreMigrationVersion).toBe('8.0.0');
+ });
+});
+
+function createRoot() {
+ return kbnTestServer.createRootWithCorePlugins(
+ {
+ migrations: {
+ skip: false,
+ enableV2: true,
+ },
+ logging: {
+ appenders: {
+ file: {
+ type: 'file',
+ fileName: logFilePath,
+ layout: {
+ type: 'json',
+ },
+ },
+ },
+ loggers: [
+ {
+ name: 'root',
+ appenders: ['file'],
+ },
+ ],
+ },
+ },
+ {
+ oss: true,
+ }
+ );
+}
+
+async function fetchDocs(esClient: ElasticsearchClient, index: string) {
+ const { body } = await esClient.search({
+ index,
+ body: {
+ query: {
+ bool: {
+ should: [
+ {
+ term: { type: 'foo' },
+ },
+ ],
+ },
+ },
+ },
+ });
+
+ return body.hits.hits;
+}
diff --git a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts
index bffe590a39432..7a87a645a249c 100644
--- a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts
+++ b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts
@@ -296,38 +296,35 @@ describe('migrationsStateActionMachine', () => {
},
},
"unusedTypesQuery": Object {
- "_tag": "Some",
- "value": Object {
- "bool": Object {
- "must_not": Array [
- Object {
- "term": Object {
- "type": "fleet-agent-events",
- },
+ "bool": Object {
+ "must_not": Array [
+ Object {
+ "term": Object {
+ "type": "fleet-agent-events",
},
- Object {
- "term": Object {
- "type": "tsvb-validation-telemetry",
- },
+ },
+ Object {
+ "term": Object {
+ "type": "tsvb-validation-telemetry",
},
- Object {
- "bool": Object {
- "must": Array [
- Object {
- "match": Object {
- "type": "search-session",
- },
+ },
+ Object {
+ "bool": Object {
+ "must": Array [
+ Object {
+ "match": Object {
+ "type": "search-session",
},
- Object {
- "match": Object {
- "search-session.persisted": false,
- },
+ },
+ Object {
+ "match": Object {
+ "search-session.persisted": false,
},
- ],
- },
+ },
+ ],
},
- ],
- },
+ },
+ ],
},
},
"versionAlias": ".my-so-index_7.11.0",
@@ -396,38 +393,35 @@ describe('migrationsStateActionMachine', () => {
},
},
"unusedTypesQuery": Object {
- "_tag": "Some",
- "value": Object {
- "bool": Object {
- "must_not": Array [
- Object {
- "term": Object {
- "type": "fleet-agent-events",
- },
+ "bool": Object {
+ "must_not": Array [
+ Object {
+ "term": Object {
+ "type": "fleet-agent-events",
},
- Object {
- "term": Object {
- "type": "tsvb-validation-telemetry",
- },
+ },
+ Object {
+ "term": Object {
+ "type": "tsvb-validation-telemetry",
},
- Object {
- "bool": Object {
- "must": Array [
- Object {
- "match": Object {
- "type": "search-session",
- },
+ },
+ Object {
+ "bool": Object {
+ "must": Array [
+ Object {
+ "match": Object {
+ "type": "search-session",
},
- Object {
- "match": Object {
- "search-session.persisted": false,
- },
+ },
+ Object {
+ "match": Object {
+ "search-session.persisted": false,
},
- ],
- },
+ },
+ ],
},
- ],
- },
+ },
+ ],
},
},
"versionAlias": ".my-so-index_7.11.0",
@@ -584,38 +578,35 @@ describe('migrationsStateActionMachine', () => {
},
},
"unusedTypesQuery": Object {
- "_tag": "Some",
- "value": Object {
- "bool": Object {
- "must_not": Array [
- Object {
- "term": Object {
- "type": "fleet-agent-events",
- },
+ "bool": Object {
+ "must_not": Array [
+ Object {
+ "term": Object {
+ "type": "fleet-agent-events",
},
- Object {
- "term": Object {
- "type": "tsvb-validation-telemetry",
- },
+ },
+ Object {
+ "term": Object {
+ "type": "tsvb-validation-telemetry",
},
- Object {
- "bool": Object {
- "must": Array [
- Object {
- "match": Object {
- "type": "search-session",
- },
+ },
+ Object {
+ "bool": Object {
+ "must": Array [
+ Object {
+ "match": Object {
+ "type": "search-session",
},
- Object {
- "match": Object {
- "search-session.persisted": false,
- },
+ },
+ Object {
+ "match": Object {
+ "search-session.persisted": false,
},
- ],
- },
+ },
+ ],
},
- ],
- },
+ },
+ ],
},
},
"versionAlias": ".my-so-index_7.11.0",
@@ -679,38 +670,35 @@ describe('migrationsStateActionMachine', () => {
},
},
"unusedTypesQuery": Object {
- "_tag": "Some",
- "value": Object {
- "bool": Object {
- "must_not": Array [
- Object {
- "term": Object {
- "type": "fleet-agent-events",
- },
+ "bool": Object {
+ "must_not": Array [
+ Object {
+ "term": Object {
+ "type": "fleet-agent-events",
},
- Object {
- "term": Object {
- "type": "tsvb-validation-telemetry",
- },
+ },
+ Object {
+ "term": Object {
+ "type": "tsvb-validation-telemetry",
},
- Object {
- "bool": Object {
- "must": Array [
- Object {
- "match": Object {
- "type": "search-session",
- },
+ },
+ Object {
+ "bool": Object {
+ "must": Array [
+ Object {
+ "match": Object {
+ "type": "search-session",
},
- Object {
- "match": Object {
- "search-session.persisted": false,
- },
+ },
+ Object {
+ "match": Object {
+ "search-session.persisted": false,
},
- ],
- },
+ },
+ ],
},
- ],
- },
+ },
+ ],
},
},
"versionAlias": ".my-so-index_7.11.0",
diff --git a/src/core/server/saved_objects/migrationsv2/model.test.ts b/src/core/server/saved_objects/migrationsv2/model.test.ts
index 57a7a7f2ea24a..213e8b43c0ea0 100644
--- a/src/core/server/saved_objects/migrationsv2/model.test.ts
+++ b/src/core/server/saved_objects/migrationsv2/model.test.ts
@@ -21,9 +21,12 @@ import type {
ReindexSourceToTempRead,
ReindexSourceToTempClosePit,
ReindexSourceToTempIndex,
+ RefreshTarget,
UpdateTargetMappingsState,
UpdateTargetMappingsWaitForTaskState,
- OutdatedDocumentsSearch,
+ OutdatedDocumentsSearchOpenPit,
+ OutdatedDocumentsSearchRead,
+ OutdatedDocumentsSearchClosePit,
OutdatedDocumentsTransform,
MarkVersionIndexReady,
BaseState,
@@ -72,7 +75,7 @@ describe('migrations v2 model', () => {
versionAlias: '.kibana_7.11.0',
versionIndex: '.kibana_7.11.0_001',
tempIndex: '.kibana_7.11.0_reindex_temp',
- unusedTypesQuery: Option.of({
+ unusedTypesQuery: {
bool: {
must_not: [
{
@@ -82,7 +85,7 @@ describe('migrations v2 model', () => {
},
],
},
- }),
+ },
};
describe('exponential retry delays for retryable_es_client_error', () => {
@@ -214,7 +217,7 @@ describe('migrations v2 model', () => {
},
};
- test('INIT -> OUTDATED_DOCUMENTS_SEARCH if .kibana is already pointing to the target index', () => {
+ test('INIT -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT if .kibana is already pointing to the target index', () => {
const res: ResponseType<'INIT'> = Either.right({
'.kibana_7.11.0_001': {
aliases: {
@@ -227,7 +230,7 @@ describe('migrations v2 model', () => {
});
const newState = model(initState, res);
- expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH');
+ expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT');
// This snapshot asserts that we merge the
// migrationMappingPropertyHashes of the existing index, but we leave
// the mappings for the disabled_saved_object_type untouched. There
@@ -888,78 +891,120 @@ describe('migrations v2 model', () => {
sourceIndex: Option.some('.kibana') as Option.Some,
targetIndex: '.kibana_7.11.0_001',
};
- it('CLONE_TEMP_TO_TARGET -> OUTDATED_DOCUMENTS_SEARCH if response is right', () => {
+ it('CLONE_TEMP_TO_TARGET -> REFRESH_TARGET if response is right', () => {
const res: ResponseType<'CLONE_TEMP_TO_TARGET'> = Either.right({
acknowledged: true,
shardsAcknowledged: true,
});
const newState = model(state, res);
- expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH');
- expect(newState.retryCount).toEqual(0);
- expect(newState.retryDelay).toEqual(0);
+ expect(newState.controlState).toBe('REFRESH_TARGET');
+ expect(newState.retryCount).toBe(0);
+ expect(newState.retryDelay).toBe(0);
});
- it('CLONE_TEMP_TO_TARGET -> OUTDATED_DOCUMENTS_SEARCH if response is left index_not_fonud_exception', () => {
+ it('CLONE_TEMP_TO_TARGET -> REFRESH_TARGET if response is left index_not_fonud_exception', () => {
const res: ResponseType<'CLONE_TEMP_TO_TARGET'> = Either.left({
type: 'index_not_found_exception',
index: 'temp_index',
});
const newState = model(state, res);
- expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH');
- expect(newState.retryCount).toEqual(0);
- expect(newState.retryDelay).toEqual(0);
+ expect(newState.controlState).toBe('REFRESH_TARGET');
+ expect(newState.retryCount).toBe(0);
+ expect(newState.retryDelay).toBe(0);
});
});
- describe('OUTDATED_DOCUMENTS_SEARCH', () => {
- const outdatedDocumentsSourchState: OutdatedDocumentsSearch = {
+ describe('OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', () => {
+ const state: OutdatedDocumentsSearchOpenPit = {
...baseState,
- controlState: 'OUTDATED_DOCUMENTS_SEARCH',
+ controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT',
versionIndexReadyActions: Option.none,
sourceIndex: Option.some('.kibana') as Option.Some,
targetIndex: '.kibana_7.11.0_001',
};
- test('OUTDATED_DOCUMENTS_SEARCH -> OUTDATED_DOCUMENTS_TRANSFORM if some outdated documents were found', () => {
- const outdatedDocuments = ([
- Symbol('raw saved object doc'),
- ] as unknown) as SavedObjectsRawDoc[];
- const res: ResponseType<'OUTDATED_DOCUMENTS_SEARCH'> = Either.right({
- outdatedDocuments,
+ it('OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ if action succeeds', () => {
+ const res: ResponseType<'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT'> = Either.right({
+ pitId: 'pit_id',
});
- const newState = model(outdatedDocumentsSourchState, res) as OutdatedDocumentsTransform;
- expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_TRANSFORM');
- expect(newState.outdatedDocuments).toEqual(outdatedDocuments);
- expect(newState.retryCount).toEqual(0);
- expect(newState.retryDelay).toEqual(0);
+ const newState = model(state, res) as OutdatedDocumentsSearchRead;
+ expect(newState.controlState).toBe('OUTDATED_DOCUMENTS_SEARCH_READ');
+ expect(newState.pitId).toBe('pit_id');
+ expect(newState.lastHitSortValue).toBe(undefined);
+ expect(newState.retryCount).toBe(0);
+ expect(newState.retryDelay).toBe(0);
});
- test('OUTDATED_DOCUMENTS_SEARCH -> UPDATE_TARGET_MAPPINGS if none outdated documents were found and some versionIndexReadyActions', () => {
- const aliasActions = ([Symbol('alias action')] as unknown) as AliasAction[];
- const outdatedDocumentsSourchStateWithSomeVersionIndexReadyActions = {
- ...outdatedDocumentsSourchState,
- ...{
- versionIndexReadyActions: Option.some(aliasActions),
- },
- };
- const res: ResponseType<'OUTDATED_DOCUMENTS_SEARCH'> = Either.right({
- outdatedDocuments: [],
+ });
+
+ describe('OUTDATED_DOCUMENTS_SEARCH_READ', () => {
+ const state: OutdatedDocumentsSearchRead = {
+ ...baseState,
+ controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ',
+ versionIndexReadyActions: Option.none,
+ sourceIndex: Option.some('.kibana') as Option.Some,
+ pitId: 'pit_id',
+ targetIndex: '.kibana_7.11.0_001',
+ lastHitSortValue: undefined,
+ hasTransformedDocs: false,
+ };
+
+ it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_TRANSFORM if found documents to transform', () => {
+ const outdatedDocuments = [{ _id: '1', _source: { type: 'vis' } }];
+ const lastHitSortValue = [123456];
+ const res: ResponseType<'OUTDATED_DOCUMENTS_SEARCH_READ'> = Either.right({
+ outdatedDocuments,
+ lastHitSortValue,
});
- const newState = model(
- outdatedDocumentsSourchStateWithSomeVersionIndexReadyActions,
- res
- ) as MarkVersionIndexReady;
- expect(newState.controlState).toEqual('UPDATE_TARGET_MAPPINGS');
- expect(newState.versionIndexReadyActions.value).toEqual(aliasActions);
- expect(newState.retryCount).toEqual(0);
- expect(newState.retryDelay).toEqual(0);
+ const newState = model(state, res) as OutdatedDocumentsTransform;
+ expect(newState.controlState).toBe('OUTDATED_DOCUMENTS_TRANSFORM');
+ expect(newState.outdatedDocuments).toBe(outdatedDocuments);
+ expect(newState.lastHitSortValue).toBe(lastHitSortValue);
});
- test('OUTDATED_DOCUMENTS_SEARCH -> UPDATE_TARGET_MAPPINGS if none outdated documents were found and none versionIndexReadyActions', () => {
- const res: ResponseType<'OUTDATED_DOCUMENTS_SEARCH'> = Either.right({
+
+ it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT if no outdated documents to transform', () => {
+ const res: ResponseType<'OUTDATED_DOCUMENTS_SEARCH_READ'> = Either.right({
outdatedDocuments: [],
+ lastHitSortValue: undefined,
});
- const newState = model(outdatedDocumentsSourchState, res);
- expect(newState.controlState).toEqual('UPDATE_TARGET_MAPPINGS');
- expect(newState.retryCount).toEqual(0);
- expect(newState.retryDelay).toEqual(0);
+ const newState = model(state, res) as OutdatedDocumentsSearchClosePit;
+ expect(newState.controlState).toBe('OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT');
+ expect(newState.pitId).toBe('pit_id');
+ });
+ });
+
+ describe('OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', () => {
+ const state: OutdatedDocumentsSearchClosePit = {
+ ...baseState,
+ controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT',
+ versionIndexReadyActions: Option.none,
+ sourceIndex: Option.some('.kibana') as Option.Some,
+ pitId: 'pit_id',
+ targetIndex: '.kibana_7.11.0_001',
+ hasTransformedDocs: false,
+ };
+
+ it('OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> UPDATE_TARGET_MAPPINGS if action succeeded', () => {
+ const res: ResponseType<'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT'> = Either.right({});
+ const newState = model(state, res) as UpdateTargetMappingsState;
+ expect(newState.controlState).toBe('UPDATE_TARGET_MAPPINGS');
+ // @ts-expect-error pitId shouldn't leak outside
+ expect(newState.pitId).toBe(undefined);
+ });
+ });
+
+ describe('REFRESH_TARGET', () => {
+ const state: RefreshTarget = {
+ ...baseState,
+ controlState: 'REFRESH_TARGET',
+ versionIndexReadyActions: Option.none,
+ sourceIndex: Option.some('.kibana') as Option.Some,
+ targetIndex: '.kibana_7.11.0_001',
+ };
+
+ it('REFRESH_TARGET -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT if action succeeded', () => {
+ const res: ResponseType<'REFRESH_TARGET'> = Either.right({ refreshed: true });
+ const newState = model(state, res) as UpdateTargetMappingsState;
+ expect(newState.controlState).toBe('OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT');
});
});
+
describe('OUTDATED_DOCUMENTS_TRANSFORM', () => {
const outdatedDocuments = ([
Symbol('raw saved object doc'),
@@ -971,17 +1016,21 @@ describe('migrations v2 model', () => {
sourceIndex: Option.some('.kibana') as Option.Some,
targetIndex: '.kibana_7.11.0_001',
outdatedDocuments,
+ pitId: 'pit_id',
+ lastHitSortValue: [3, 4],
+ hasTransformedDocs: false,
};
- test('OUTDATED_DOCUMENTS_TRANSFORM -> OUTDATED_DOCUMENTS_SEARCH if action succeeds', () => {
+ test('OUTDATED_DOCUMENTS_TRANSFORM -> OUTDATED_DOCUMENTS_SEARCH_READ if action succeeds', () => {
const res: ResponseType<'OUTDATED_DOCUMENTS_TRANSFORM'> = Either.right(
'bulk_index_succeeded'
);
const newState = model(outdatedDocumentsTransformState, res);
- expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH');
+ expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH_READ');
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
});
+
describe('UPDATE_TARGET_MAPPINGS', () => {
const updateTargetMappingsState: UpdateTargetMappingsState = {
...baseState,
@@ -1236,38 +1285,35 @@ describe('migrations v2 model', () => {
},
},
"unusedTypesQuery": Object {
- "_tag": "Some",
- "value": Object {
- "bool": Object {
- "must_not": Array [
- Object {
- "term": Object {
- "type": "fleet-agent-events",
- },
+ "bool": Object {
+ "must_not": Array [
+ Object {
+ "term": Object {
+ "type": "fleet-agent-events",
},
- Object {
- "term": Object {
- "type": "tsvb-validation-telemetry",
- },
+ },
+ Object {
+ "term": Object {
+ "type": "tsvb-validation-telemetry",
},
- Object {
- "bool": Object {
- "must": Array [
- Object {
- "match": Object {
- "type": "search-session",
- },
+ },
+ Object {
+ "bool": Object {
+ "must": Array [
+ Object {
+ "match": Object {
+ "type": "search-session",
},
- Object {
- "match": Object {
- "search-session.persisted": false,
- },
+ },
+ Object {
+ "match": Object {
+ "search-session.persisted": false,
},
- ],
- },
+ },
+ ],
},
- ],
- },
+ },
+ ],
},
},
"versionAlias": ".kibana_task_manager_8.1.0",
diff --git a/src/core/server/saved_objects/migrationsv2/model.ts b/src/core/server/saved_objects/migrationsv2/model.ts
index 2097b1de88aab..318eff19d5e24 100644
--- a/src/core/server/saved_objects/migrationsv2/model.ts
+++ b/src/core/server/saved_objects/migrationsv2/model.ts
@@ -189,10 +189,10 @@ export const model = (currentState: State, resW: ResponseType):
) {
return {
...stateP,
- // Skip to 'OUTDATED_DOCUMENTS_SEARCH' so that if a new plugin was
+ // Skip to 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT' so that if a new plugin was
// installed / enabled we can transform any old documents and update
// the mappings for this plugin's types.
- controlState: 'OUTDATED_DOCUMENTS_SEARCH',
+ controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT',
// Source is a none because we didn't do any migration from a source
// index
sourceIndex: Option.none,
@@ -574,51 +574,100 @@ export const model = (currentState: State, resW: ResponseType):
if (Either.isRight(res)) {
return {
...stateP,
- controlState: 'OUTDATED_DOCUMENTS_SEARCH',
+ controlState: 'REFRESH_TARGET',
};
} else {
const left = res.left;
if (isLeftTypeof(left, 'index_not_found_exception')) {
- // index_not_found_exception means another instance alread completed
+ // index_not_found_exception means another instance already completed
// the MARK_VERSION_INDEX_READY step and removed the temp index
- // We still perform the OUTDATED_DOCUMENTS_* and
+ // We still perform the REFRESH_TARGET, OUTDATED_DOCUMENTS_* and
// UPDATE_TARGET_MAPPINGS steps since we might have plugins enabled
// which the other instances don't.
return {
...stateP,
- controlState: 'OUTDATED_DOCUMENTS_SEARCH',
+ controlState: 'REFRESH_TARGET',
};
} else {
throwBadResponse(stateP, left);
}
}
- } else if (stateP.controlState === 'OUTDATED_DOCUMENTS_SEARCH') {
+ } else if (stateP.controlState === 'REFRESH_TARGET') {
+ const res = resW as ExcludeRetryableEsError>;
+ if (Either.isRight(res)) {
+ return {
+ ...stateP,
+ controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT',
+ };
+ } else {
+ throwBadResponse(stateP, res);
+ }
+ } else if (stateP.controlState === 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT') {
+ const res = resW as ExcludeRetryableEsError>;
+ if (Either.isRight(res)) {
+ return {
+ ...stateP,
+ controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ',
+ pitId: res.right.pitId,
+ lastHitSortValue: undefined,
+ hasTransformedDocs: false,
+ };
+ } else {
+ throwBadResponse(stateP, res);
+ }
+ } else if (stateP.controlState === 'OUTDATED_DOCUMENTS_SEARCH_READ') {
const res = resW as ExcludeRetryableEsError>;
if (Either.isRight(res)) {
- // If outdated documents were found, transform them
if (res.right.outdatedDocuments.length > 0) {
return {
...stateP,
controlState: 'OUTDATED_DOCUMENTS_TRANSFORM',
outdatedDocuments: res.right.outdatedDocuments,
+ lastHitSortValue: res.right.lastHitSortValue,
};
} else {
- // If there are no more results we have transformed all outdated
- // documents and can proceed to the next step
return {
...stateP,
- controlState: 'UPDATE_TARGET_MAPPINGS',
+ controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT',
};
}
} else {
throwBadResponse(stateP, res);
}
+ } else if (stateP.controlState === 'OUTDATED_DOCUMENTS_REFRESH') {
+ const res = resW as ExcludeRetryableEsError>;
+ if (Either.isRight(res)) {
+ return {
+ ...stateP,
+ controlState: 'UPDATE_TARGET_MAPPINGS',
+ };
+ } else {
+ throwBadResponse(stateP, res);
+ }
+ } else if (stateP.controlState === 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT') {
+ const res = resW as ExcludeRetryableEsError