From 00a7cf6cb88d51965f7c09c190f20bb5d34184ee Mon Sep 17 00:00:00 2001 From: Tomasz Ciecierski Date: Mon, 14 Nov 2022 17:18:04 +0100 Subject: [PATCH 01/16] [Osquery] Global packs (#143948) --- .../migrations/check_registered_types.test.ts | 4 +- .../osquery/common/schemas/common/utils.ts | 46 ++++++ .../osquery/cypress/e2e/all/alerts.cy.ts | 5 +- .../osquery/cypress/e2e/all/cases.cy.ts | 2 + .../osquery/cypress/e2e/all/packs.cy.ts | 70 ++++++++ .../cypress/e2e/roles/alert_test.cy.ts | 11 +- .../osquery/cypress/tasks/live_query.ts | 14 +- .../osquery/cypress/tasks/saved_queries.ts | 1 + .../osquery/public/packs/form/index.tsx | 155 ++++++++++++++++-- .../packs/form/policy_id_combobox_field.tsx | 11 +- .../packs/form/shards/pack_shards_field.tsx | 117 +++++++++++++ .../form/shards/pack_type_selectable.tsx | 128 +++++++++++++++ .../public/packs/form/shards/shards_form.tsx | 90 ++++++++++ .../form/shards/shards_percentage_field.tsx | 81 +++++++++ .../packs/form/shards/shards_policy_field.tsx | 101 ++++++++++++ .../packs/queries/use_pack_query_form.tsx | 2 + x-pack/plugins/osquery/public/packs/types.ts | 2 + .../public/routes/saved_queries/edit/tabs.tsx | 2 + x-pack/plugins/osquery/public/types.ts | 2 + x-pack/plugins/osquery/server/common/types.ts | 3 + .../lib/saved_query/saved_object_mappings.ts | 8 + .../osquery/server/lib/update_global_packs.ts | 92 +++++++++++ x-pack/plugins/osquery/server/plugin.ts | 20 ++- .../server/routes/pack/create_pack_route.ts | 48 ++++-- .../server/routes/pack/read_pack_route.ts | 2 + .../server/routes/pack/update_pack_route.ts | 79 ++++++--- .../osquery/server/routes/pack/utils.ts | 37 ++++- x-pack/plugins/osquery/server/routes/utils.ts | 19 +++ .../test/osquery_cypress/artifact_manager.ts | 2 +- 29 files changed, 1076 insertions(+), 78 deletions(-) create mode 100644 x-pack/plugins/osquery/public/packs/form/shards/pack_shards_field.tsx create mode 100644 x-pack/plugins/osquery/public/packs/form/shards/pack_type_selectable.tsx create mode 100644 x-pack/plugins/osquery/public/packs/form/shards/shards_form.tsx create mode 100644 x-pack/plugins/osquery/public/packs/form/shards/shards_percentage_field.tsx create mode 100644 x-pack/plugins/osquery/public/packs/form/shards/shards_policy_field.tsx create mode 100644 x-pack/plugins/osquery/server/lib/update_global_packs.ts diff --git a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts index a53049a26106e..eb9d10f63ca17 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts @@ -113,8 +113,8 @@ describe('checking migration metadata changes on all registered SO types', () => "ml-trained-model": "e39dd10b2da827e194ddcaaf3db141ad1daf0201", "monitoring-telemetry": "af508cea8e22edaa909e462069390650fbbf01b7", "osquery-manager-usage-metric": "fbe3cbea25a96e2ca522ca436878e0162c94dcc2", - "osquery-pack": "afb3b46c5e23fc24ad438e9c4317ff37e4e5164a", - "osquery-pack-asset": "32421669c87c49dfabd4d3957f044e5eb7f7fb20", + "osquery-pack": "a2d675c7af4208e54a5b28d23d324d7c599a5491", + "osquery-pack-asset": "de8783298eb33a577bf1fa0caacd42121dcfae91", "osquery-saved-query": "7b213b4b7a3e59350e99c50e8df9948662ed493a", "query": "4640ef356321500a678869f24117b7091a911cb6", "sample-data-telemetry": "8b10336d9efae6f3d5593c4cc89fb4abcdf84e04", diff --git a/x-pack/plugins/osquery/common/schemas/common/utils.ts b/x-pack/plugins/osquery/common/schemas/common/utils.ts index 6900cac5a7d7e..b6dec6a6f920d 100644 --- a/x-pack/plugins/osquery/common/schemas/common/utils.ts +++ b/x-pack/plugins/osquery/common/schemas/common/utils.ts @@ -8,6 +8,7 @@ import { isEmpty, reduce } from 'lodash'; import type { DefaultValues } from 'react-hook-form'; import type { ECSMapping } from '@kbn/osquery-io-ts-types'; +import type { GetAgentPoliciesResponseItem } from '@kbn/fleet-plugin/common'; export type ECSMappingArray = Array<{ key: string; @@ -52,3 +53,48 @@ export const convertECSMappingToArray = ( }, [] as ECSMappingArray ); + +export type ShardsArray = Array<{ + policy: { + key: string; + label: string; + }; + percentage: number; +}>; + +export type Shard = Record; + +export const convertShardsToObject = (shards: ShardsArray): Shard => + reduce( + shards, + (acc, value) => { + if (!isEmpty(value?.policy)) { + acc[value.policy.key] = value.percentage; + } + + return acc; + }, + {} as Shard + ); + +export const convertShardsToArray = ( + shards: DefaultValues, + policiesById?: Record +): ShardsArray => + reduce( + shards, + (acc, value, key) => { + if (value) { + acc.push({ + policy: { + key, + label: policiesById?.[key]?.name ?? '', + }, + percentage: value, + }); + } + + return acc; + }, + [] as ShardsArray + ); diff --git a/x-pack/plugins/osquery/cypress/e2e/all/alerts.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/alerts.cy.ts index 34d04289ad676..ad939fbad5d44 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/alerts.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/alerts.cy.ts @@ -47,7 +47,9 @@ describe('Alert Event Details', () => { ); findAndClickButton('Update pack'); closeModalIfVisible(); - cy.contains(PACK_NAME); + cy.contains(`Successfully updated "${PACK_NAME}" pack`); + cy.getBySel('toastCloseButton').click(); + cy.visit('/app/security/rules'); cy.contains(RULE_NAME); cy.wait(2000); @@ -109,6 +111,7 @@ describe('Alert Event Details', () => { cy.get('.euiButtonEmpty--flushLeft').contains('Cancel').click(); cy.getBySel('add-to-timeline').first().click(); cy.getBySel('globalToastList').contains('Added'); + cy.getBySel('toastCloseButton').click(); cy.getBySel(RESULTS_TABLE).within(() => { cy.getBySel(RESULTS_TABLE_BUTTON).should('not.exist'); }); diff --git a/x-pack/plugins/osquery/cypress/e2e/all/cases.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/cases.cy.ts index ce586e4d49943..4dd970042bb8f 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/cases.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/cases.cy.ts @@ -23,6 +23,7 @@ describe('Add to Cases', () => { runKbnArchiverScript(ArchiverMethod.UNLOAD, 'case_observability'); }); it('should add result a case and not have add to timeline in result', () => { + cy.waitForReact(); cy.react('CustomItemAction', { props: { index: 1 }, }).click(); @@ -54,6 +55,7 @@ describe('Add to Cases', () => { }); it('should add result a case and have add to timeline in result', () => { + cy.waitForReact(); cy.react('CustomItemAction', { props: { index: 1 }, }).click(); diff --git a/x-pack/plugins/osquery/cypress/e2e/all/packs.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/packs.cy.ts index 7dc3d1e646075..12a242cb29917 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/packs.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/packs.cy.ts @@ -166,6 +166,7 @@ describe('ALL - Packs', () => { findAndClickButton('Save and deploy changes'); cy.contains(PACK_NAME); cy.contains(`Successfully created "${PACK_NAME}" pack`); + cy.getBySel('toastCloseButton').click(); }); it('to click the edit button and edit pack', () => { @@ -186,6 +187,7 @@ describe('ALL - Packs', () => { cy.contains('Save and deploy changes'); findAndClickButton('Save and deploy changes'); cy.contains(`Successfully updated "${PACK_NAME}" pack`); + cy.getBySel('toastCloseButton').click(); }); it('should trigger validation when saved query is being chosen', () => { @@ -409,4 +411,72 @@ describe('ALL - Packs', () => { cy.react('EuiTableRow').should('have.length.above', 5); }); }); + + describe('Global packs', () => { + beforeEach(() => { + login(); + navigateTo('/app/osquery/packs'); + }); + + it('add global packs to polciies', () => { + const globalPack = 'globalPack'; + cy.contains('Packs').click(); + findAndClickButton('Add pack'); + findFormFieldByRowsLabelAndType('Name', globalPack); + cy.getBySel('osqueryPackTypeGlobal').click(); + findAndClickButton('Save pack'); + + cy.contains(globalPack); + cy.contains(`Successfully created "${globalPack}" pack`); + cy.getBySel('toastCloseButton').click(); + + cy.visit(FLEET_AGENT_POLICIES); + cy.contains('Create agent policy').click(); + cy.getBySel('createAgentPolicyNameField').type('testGlobal'); + cy.getBySel('createAgentPolicyFlyoutBtn').click(); + cy.contains(/^Agent policy 'testGlobal' created$/).click(); + cy.contains('testGlobal').click(); + cy.contains('Add integration').click(); + cy.contains(integration).click(); + addIntegration('testGlobal'); + cy.contains('Add Elastic Agent later').click(); + cy.contains('osquery_manager-'); + cy.request('/internal/osquery/fleet_wrapper/package_policies').then((response) => { + const item = response.body.items[0]; + + expect(item.inputs[0].config.osquery.value.packs.globalPack).to.deep.equal({ + shard: 100, + queries: {}, + }); + }); + }); + it('add proper shard to policies packs config', () => { + const shardPack = 'shardPack'; + cy.contains('Packs').click(); + findAndClickButton('Add pack'); + findFormFieldByRowsLabelAndType('Name', shardPack); + + cy.contains('Partial deployment (shards)').click(); + cy.getBySel('shards-field-policy').type('Default{downArrow}{enter}'); + cy.get('#shardsPercentage0').type('{backspace}{backspace}5'); + findAndClickButton('Save pack'); + + cy.contains(`Successfully created "${shardPack}" pack`); + cy.getBySel('toastCloseButton').click(); + + cy.request('/internal/osquery/fleet_wrapper/package_policies').then((response) => { + const shardPolicy = response.body.items.find( + (policy: { policy_id: string }) => policy.policy_id === 'fleet-server-policy' + ); + + expect(shardPolicy.inputs[0].config.osquery.value.packs[shardPack]).to.deep.equal({ + shard: 15, + queries: {}, + }); + }); + cy.contains(shardPack).click(); + cy.contains('Edit').click(); + cy.get('#shardsPercentage0').should('have.value', '15'); + }); + }); }); diff --git a/x-pack/plugins/osquery/cypress/e2e/roles/alert_test.cy.ts b/x-pack/plugins/osquery/cypress/e2e/roles/alert_test.cy.ts index 3adffecd77848..1a94f79232ae4 100644 --- a/x-pack/plugins/osquery/cypress/e2e/roles/alert_test.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/roles/alert_test.cy.ts @@ -14,7 +14,6 @@ import { findFormFieldByRowsLabelAndType, submitQuery, } from '../../tasks/live_query'; -import { preparePack } from '../../tasks/packs'; import { closeModalIfVisible } from '../../tasks/integrations'; import { navigateTo } from '../../tasks/navigation'; @@ -36,7 +35,9 @@ describe('Alert_Test', () => { const PACK_NAME = 'testpack'; const RULE_NAME = 'Test-rule'; navigateTo('/app/osquery'); - preparePack(PACK_NAME); + cy.contains('Packs').click(); + cy.getBySel('pagination-button-next').click(); + cy.contains(PACK_NAME).click(); findAndClickButton('Edit'); cy.contains(`Edit ${PACK_NAME}`); findFormFieldByRowsLabelAndType( @@ -45,7 +46,9 @@ describe('Alert_Test', () => { ); findAndClickButton('Update pack'); closeModalIfVisible(); - cy.contains(PACK_NAME); + cy.contains(`Successfully updated "${PACK_NAME}" pack`); + cy.getBySel('toastCloseButton').click(); + cy.visit('/app/security/rules'); cy.contains(RULE_NAME).click(); cy.wait(2000); @@ -73,6 +76,7 @@ describe('Alert_Test', () => { cy.visit('/app/security/alerts'); cy.getBySel('expand-event').first().click(); + cy.wait(500); cy.contains('Get processes').click(); submitQuery(); checkResults(); @@ -86,6 +90,7 @@ describe('Alert_Test', () => { cy.visit('/app/security/alerts'); cy.getBySel('expand-event').first().click(); + cy.wait(500); cy.contains('Get processes').click(); cy.intercept('POST', '/api/osquery/live_queries', (req) => { diff --git a/x-pack/plugins/osquery/cypress/tasks/live_query.ts b/x-pack/plugins/osquery/cypress/tasks/live_query.ts index 3c571d04ff84c..519ebed2ce9a6 100644 --- a/x-pack/plugins/osquery/cypress/tasks/live_query.ts +++ b/x-pack/plugins/osquery/cypress/tasks/live_query.ts @@ -30,8 +30,18 @@ export const submitQuery = () => { cy.contains('Submit').click(); }; -export const checkResults = () => - cy.getBySel('dataGridRowCell', { timeout: 120000 }).should('have.lengthOf.above', 0); +// sometimes the results get stuck in the tests, this is a workaround +export const checkResults = () => { + cy.getBySel('osqueryResultsTable').then(($table) => { + if ($table.find('div .euiDataGridRow').length > 0) { + cy.getBySel('dataGridRowCell', { timeout: 120000 }).should('have.lengthOf.above', 0); + } else { + cy.getBySel('osquery-status-tab').click(); + cy.getBySel('osquery-results-tab').click(); + cy.getBySel('dataGridRowCell', { timeout: 120000 }).should('have.lengthOf.above', 0); + } + }); +}; export const typeInECSFieldInput = (text: string) => cy.getBySel('ECS-field-input').type(text); export const typeInOsqueryFieldInput = (text: string) => diff --git a/x-pack/plugins/osquery/cypress/tasks/saved_queries.ts b/x-pack/plugins/osquery/cypress/tasks/saved_queries.ts index 4de712bd2f55a..c8461f345b866 100644 --- a/x-pack/plugins/osquery/cypress/tasks/saved_queries.ts +++ b/x-pack/plugins/osquery/cypress/tasks/saved_queries.ts @@ -81,6 +81,7 @@ export const getSavedQueriesComplexTest = (savedQueryId: string, savedQueryDescr findFormFieldByRowsLabelAndType('Description (optional)', savedQueryDescription); cy.react('EuiButtonDisplay').contains('Save').click(); cy.contains('Successfully saved'); + cy.getBySel('toastCloseButton').click(); // play saved query cy.contains('Saved queries').click(); diff --git a/x-pack/plugins/osquery/public/packs/form/index.tsx b/x-pack/plugins/osquery/public/packs/form/index.tsx index ac3892fe9748b..dc19739c32adf 100644 --- a/x-pack/plugins/osquery/public/packs/form/index.tsx +++ b/x-pack/plugins/osquery/public/packs/form/index.tsx @@ -5,7 +5,8 @@ * 2.0. */ -import { reduce } from 'lodash'; +import { filter, isEmpty, map, omit, reduce } from 'lodash'; +import type { EuiAccordionProps } from '@elastic/eui'; import { EuiFlexGroup, EuiFlexItem, @@ -14,12 +15,15 @@ import { EuiSpacer, EuiBottomBar, EuiHorizontalRule, + EuiAccordion, } from '@elastic/eui'; -import React, { useCallback, useMemo, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; import deepEqual from 'fast-deep-equal'; import { FormProvider, useForm as useHookForm } from 'react-hook-form'; +import styled from 'styled-components'; +import { PackShardsField } from './shards/pack_shards_field'; import { useRouterNavigate } from '../../common/lib/kibana'; import { PolicyIdComboBoxField } from './policy_id_combobox_field'; import { QueriesField } from './queries_field'; @@ -32,9 +36,17 @@ import type { PackItem } from '../types'; import { NameField } from './name_field'; import { DescriptionField } from './description_field'; import type { PackQueryFormData } from '../queries/use_pack_query_form'; +import { PackTypeSelectable } from './shards/pack_type_selectable'; type PackFormData = Omit & { queries: PackQueryFormData[] }; +const StyledEuiAccordion = styled(EuiAccordion)` + ${({ isDisabled }: { isDisabled?: boolean }) => isDisabled && 'display: none;'} + .euiAccordion__button { + color: ${({ theme }) => theme.eui.euiColorPrimary}; + } +`; + interface PackFormProps { defaultValue?: PackItem; editMode?: boolean; @@ -46,6 +58,13 @@ const PackFormComponent: React.FC = ({ editMode = false, isReadOnly = false, }) => { + const [shardsToggleState, setShardsToggleState] = + useState('closed'); + const handleToggle = useCallback((isOpen) => { + const newState = isOpen ? 'open' : 'closed'; + setShardsToggleState(newState); + }, []); + const [packType, setPackType] = useState('policy'); const [showConfirmationModal, setShowConfirmationModal] = useState(false); const handleHideConfirmationModal = useCallback(() => setShowConfirmationModal(false), []); @@ -60,16 +79,16 @@ const PackFormComponent: React.FC = ({ withRedirect: true, }); - const deserializer = (payload: PackItem) => ({ - ...payload, - policy_ids: payload.policy_ids ?? [], - queries: convertPackQueriesToSO(payload.queries), - }); + const deserializer = (payload: PackItem) => { + const defaultPolicyIds = filter(payload.policy_ids, (policyId) => !payload.shards?.[policyId]); - const serializer = (payload: PackFormData) => ({ - ...payload, - queries: convertSOQueriesToPack(payload.queries), - }); + return { + ...payload, + policy_ids: defaultPolicyIds ?? [], + queries: convertPackQueriesToSO(payload.queries), + shards: omit(payload.shards, '*') ?? {}, + }; + }; const hooksForm = useHookForm({ defaultValues: defaultValue @@ -83,14 +102,68 @@ const PackFormComponent: React.FC = ({ }, }); + useEffect(() => { + if (!isEmpty(defaultValue?.shards)) { + if (defaultValue?.shards?.['*']) { + setPackType('global'); + } else { + setShardsToggleState('open'); + } + } + }, [defaultValue, defaultValue?.shards]); + const { handleSubmit, watch, formState: { isSubmitting }, } = hooksForm; + const { policy_ids: policyIds, shards } = watch(); + + const getShards = useCallback(() => { + if (packType === 'global') { + return { '*': 100 }; + } + + return reduce( + shards, + (acc, shard, key) => { + if (!isEmpty(key)) { + return { ...acc, [key]: shard }; + } + + return acc; + }, + {} + ); + }, [packType, shards]); const onSubmit = useCallback( async (values: PackFormData) => { + const serializer = ({ + shards: _, + policy_ids: payloadPolicyIds, + queries, + ...restPayload + }: PackFormData) => { + const mappedShards = !isEmpty(shards) + ? (filter( + map(shards, (shard, key) => { + if (!isEmpty(key)) { + return key; + } + }) + ) as string[]) + : []; + const policies = [...payloadPolicyIds, ...mappedShards]; + + return { + ...restPayload, + policy_ids: policies ?? [], + queries: convertSOQueriesToPack(queries), + shards: getShards() ?? {}, + }; + }; + try { if (editMode && defaultValue?.id) { await updateAsync({ id: defaultValue?.id, ...serializer(values) }); @@ -100,13 +173,11 @@ const PackFormComponent: React.FC = ({ // eslint-disable-next-line no-empty } catch (e) {} }, - [createAsync, defaultValue?.id, editMode, updateAsync] + [createAsync, defaultValue?.id, editMode, getShards, shards, updateAsync] ); const handleSubmitForm = useMemo(() => handleSubmit(onSubmit), [handleSubmit, onSubmit]); - const { policy_ids: policyIds } = watch(); - const agentCount = useMemo( () => reduce( @@ -138,6 +209,31 @@ const PackFormComponent: React.FC = ({ const euiFieldProps = useMemo(() => ({ isDisabled: isReadOnly }), [isReadOnly]); + const changePackType = useCallback( + (type: 'global' | 'policy' | 'shards') => { + setPackType(type); + }, + [setPackType] + ); + + const options = useMemo( + () => + Object.entries(agentPoliciesById ?? {}).map(([agentPolicyId, agentPolicy]) => ({ + key: agentPolicyId, + label: agentPolicy.name, + })), + [agentPoliciesById] + ); + + const availableOptions = useMemo(() => { + const currentShardsFieldValues = map(shards, (shard, key) => key); + const currentPolicyIdsFieldValues = map(policyIds, (policy) => policy); + + const currentValues = [...currentShardsFieldValues, ...currentPolicyIdsFieldValues]; + + return options.filter(({ key }) => !currentValues.includes(key)); + }, [shards, policyIds, options]); + return ( <> @@ -154,10 +250,35 @@ const PackFormComponent: React.FC = ({ - - - + + + {packType === 'policy' && ( + <> + + + + + + + + + + + + + + + + )} + + + diff --git a/x-pack/plugins/osquery/public/packs/form/policy_id_combobox_field.tsx b/x-pack/plugins/osquery/public/packs/form/policy_id_combobox_field.tsx index 4f17050f9c537..5fd71cc9cd02c 100644 --- a/x-pack/plugins/osquery/public/packs/form/policy_id_combobox_field.tsx +++ b/x-pack/plugins/osquery/public/packs/form/policy_id_combobox_field.tsx @@ -32,10 +32,12 @@ const AgentPolicyDescriptionColumn = styled(EuiFlexItem)` interface PolicyIdComboBoxFieldProps { euiFieldProps?: EuiComboBoxProps; + options: Array>; } const PolicyIdComboBoxFieldComponent: React.FC = ({ euiFieldProps, + options, }) => { const { data: { agentPoliciesById } = {} } = useAgentPolicies(); @@ -48,15 +50,6 @@ const PolicyIdComboBoxFieldComponent: React.FC = ({ rules: {}, }); - const options = useMemo( - () => - Object.entries(agentPoliciesById ?? {}).map(([agentPolicyId, agentPolicy]) => ({ - key: agentPolicyId, - label: agentPolicy.name, - })), - [agentPoliciesById] - ); - const selectedOptions = useMemo(() => { if (agentPoliciesById) { return castArray(value).map((policyId) => ({ diff --git a/x-pack/plugins/osquery/public/packs/form/shards/pack_shards_field.tsx b/x-pack/plugins/osquery/public/packs/form/shards/pack_shards_field.tsx new file mode 100644 index 0000000000000..c78d36f20aa19 --- /dev/null +++ b/x-pack/plugins/osquery/public/packs/form/shards/pack_shards_field.tsx @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useEffect } from 'react'; +import type { InternalFieldErrors } from 'react-hook-form'; +import { useFieldArray, useForm, useFormContext } from 'react-hook-form'; +import type { EuiComboBoxOptionOption } from '@elastic/eui'; +import { EuiSpacer } from '@elastic/eui'; +import deepEqual from 'fast-deep-equal'; +import { isEmpty, last, reject } from 'lodash'; +import { useAgentPolicies } from '../../../agent_policies'; +import type { ShardsArray } from '../../../../common/schemas/common'; +import { convertShardsToArray, convertShardsToObject } from '../../../../common/schemas/common'; +import { ShardsForm } from './shards_form'; + +export const defaultShardData = { + policy: { + label: '', + key: '', + }, + percentage: 100, +}; + +interface PackShardsFieldProps { + options: Array>; +} + +const PackShardsFieldComponent = ({ options }: PackShardsFieldProps) => { + const { + watch: watchRoot, + register: registerRoot, + setValue: setValueRoot, + formState: { errors: errorsRoot }, + } = useFormContext(); + const { data: { agentPoliciesById } = {} } = useAgentPolicies(); + + const rootShards = watchRoot('shards'); + + const { control, watch, getFieldState, formState, resetField, setValue } = useForm<{ + shardsArray: ShardsArray; + }>({ + mode: 'all', + shouldUnregister: true, + defaultValues: { + shardsArray: !isEmpty(convertShardsToArray(rootShards, agentPoliciesById)) + ? [...convertShardsToArray(rootShards, agentPoliciesById), defaultShardData] + : [defaultShardData], + }, + }); + const { fields, remove, append } = useFieldArray({ + control, + name: 'shardsArray', + }); + + const formValue = watch(); + + const shardsArrayState = getFieldState('shardsArray', formState); + + useEffect(() => { + registerRoot('shards', { + validate: () => { + const nonEmptyErrors = reject(shardsArrayState.error, isEmpty) as InternalFieldErrors[]; + + return !nonEmptyErrors.length; + }, + }); + }, [shardsArrayState.error, errorsRoot, registerRoot]); + + useEffect(() => { + const subscription = watch((data, payload) => { + if (data?.shardsArray) { + const lastShardIndex = data?.shardsArray?.length - 1; + if (payload.name?.startsWith(`shardsArray.${lastShardIndex}.`)) { + const lastShard = last(data.shardsArray); + if (lastShard?.policy?.key) { + append(defaultShardData); + } + } + } + }); + + return () => subscription.unsubscribe(); + }, [formValue, append, watch]); + + useEffect(() => { + const parsedShards = convertShardsToObject(formValue.shardsArray); + if (shardsArrayState.isDirty && !deepEqual(parsedShards, rootShards)) { + setValueRoot('shards', parsedShards, { + shouldTouch: true, + }); + } + }, [setValueRoot, formValue, shardsArrayState.isDirty, rootShards, resetField, setValue]); + + return ( + <> + + + {fields.map((item, index, array) => ( +
+ +
+ ))} + + ); +}; + +export const PackShardsField = React.memo(PackShardsFieldComponent); diff --git a/x-pack/plugins/osquery/public/packs/form/shards/pack_type_selectable.tsx b/x-pack/plugins/osquery/public/packs/form/shards/pack_type_selectable.tsx new file mode 100644 index 0000000000000..35da6a666ea22 --- /dev/null +++ b/x-pack/plugins/osquery/public/packs/form/shards/pack_type_selectable.tsx @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiCard, EuiFlexGroup, EuiFlexItem, EuiFormRow, EuiRadio } from '@elastic/eui'; +import React, { useCallback, useMemo } from 'react'; +import { i18n } from '@kbn/i18n'; +import styled from 'styled-components'; +import { noop } from 'lodash'; + +const StyledEuiCard = styled(EuiCard)` + padding: 16px 92px 16px 16px !important; + border: ${(props) => { + if (props.selectable?.isSelected) { + return `1px solid ${props.theme.eui.euiColorPrimary}`; + } + }}; + + .euiTitle { + font-size: 1rem; + } + + .euiText { + margin-top: 0; + margin-left: 25px; + color: ${(props) => props.theme.eui.EuiTextSubduedColor}; + } + + > button[role='switch'] { + display: none; + } +`; + +interface PackTypeSelectableProps { + packType: string; + setPackType: (type: 'global' | 'policy') => void; + resetFormFields?: () => void; +} + +const PackTypeSelectableComponent = ({ + packType, + setPackType, + resetFormFields, +}: PackTypeSelectableProps) => { + const handleChange = useCallback( + (type) => { + setPackType(type); + if (resetFormFields) { + resetFormFields(); + } + }, + [resetFormFields, setPackType] + ); + const policyCardSelectable = useMemo( + () => ({ + onClick: () => handleChange('policy'), + isSelected: packType === 'policy', + }), + [packType, handleChange] + ); + + const globalCardSelectable = useMemo( + () => ({ + onClick: () => handleChange('global'), + isSelected: packType === 'global', + }), + [packType, handleChange] + ); + + return ( + + + + + + } + titleSize="xs" + hasBorder + description={i18n.translate('xpack.osquery.pack.form.policyDescription', { + defaultMessage: 'Schedule pack for specific policy.', + })} + data-test-subj={'osqueryPackTypePolicy'} + selectable={policyCardSelectable} + {...(packType === 'policy' && { color: 'primary' })} + /> + + + + } + titleSize="xs" + description={i18n.translate('xpack.osquery.pack.form.globalDescription', { + defaultMessage: 'Use pack across all policies', + })} + selectable={globalCardSelectable} + data-test-subj={'osqueryPackTypeGlobal'} + {...(packType === 'global' && { color: 'primary' })} + /> + + + + + ); +}; + +export const PackTypeSelectable = React.memo(PackTypeSelectableComponent); diff --git a/x-pack/plugins/osquery/public/packs/form/shards/shards_form.tsx b/x-pack/plugins/osquery/public/packs/form/shards/shards_form.tsx new file mode 100644 index 0000000000000..7ced94c754409 --- /dev/null +++ b/x-pack/plugins/osquery/public/packs/form/shards/shards_form.tsx @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback } from 'react'; +import { i18n } from '@kbn/i18n'; +import type { EuiComboBoxOptionOption } from '@elastic/eui'; +import { EuiButtonIcon, EuiFlexGroup, EuiFlexItem, EuiSpacer } from '@elastic/eui'; +import styled from 'styled-components'; +import type { UseFieldArrayRemove, UseFormReturn } from 'react-hook-form'; +import type { ShardsArray } from '../../../../common/schemas/common/utils'; +import { ShardsPolicyField } from './shards_policy_field'; +import { ShardsPercentageField } from './shards_percentage_field'; + +const StyledButtonWrapper = styled.div` + margin-top: ${(props: { index: number }) => props.index === 0 && '16px'}; +`; + +export type ShardsFormReturn = UseFormReturn<{ shardsArray: ShardsArray }>; + +interface ShardsFormProps { + index: number; + isLastItem: boolean; + control: ShardsFormReturn['control']; + onDelete?: UseFieldArrayRemove; + options: Array>; +} + +const ShardsFormComponent = ({ + onDelete, + index, + isLastItem, + control, + options, +}: ShardsFormProps) => { + const handleDeleteClick = useCallback(() => { + if (onDelete) { + onDelete(index); + } + }, [index, onDelete]); + + return ( + <> + + + + + + + + + + + + + + + + + + + + + + + + + ); +}; + +export const ShardsForm = React.memo(ShardsFormComponent); diff --git a/x-pack/plugins/osquery/public/packs/form/shards/shards_percentage_field.tsx b/x-pack/plugins/osquery/public/packs/form/shards/shards_percentage_field.tsx new file mode 100644 index 0000000000000..61a98219f3c38 --- /dev/null +++ b/x-pack/plugins/osquery/public/packs/form/shards/shards_percentage_field.tsx @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useCallback, useMemo } from 'react'; +import { useController } from 'react-hook-form'; +import { EuiFlexGroup, EuiFlexItem, EuiFormRow, EuiRange } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import type { ShardsFormReturn } from './shards_form'; + +interface ShardsPercentageFieldComponent { + index: number; + control: ShardsFormReturn['control']; + euiFieldProps?: Record; + hideLabel?: boolean; +} + +const ShardsPercentageFieldComponent = ({ + index, + control, + euiFieldProps, + hideLabel, +}: ShardsPercentageFieldComponent) => { + const { + field: { onChange, value }, + fieldState: { error }, + } = useController({ + control, + name: `shardsArray.${index}.percentage`, + defaultValue: 100, + }); + + const handleChange = useCallback( + (e: React.ChangeEvent | React.MouseEvent) => { + const numberValue = (e.target as { valueAsNumber: number }).valueAsNumber + ? (e.target as { valueAsNumber: number }).valueAsNumber + : 0; + onChange(numberValue); + }, + [onChange] + ); + const hasError = useMemo(() => !!error?.message, [error?.message]); + + return ( + + + + + + + + ); +}; + +export const ShardsPercentageField = React.memo(ShardsPercentageFieldComponent); diff --git a/x-pack/plugins/osquery/public/packs/form/shards/shards_policy_field.tsx b/x-pack/plugins/osquery/public/packs/form/shards/shards_policy_field.tsx new file mode 100644 index 0000000000000..bf4bdf0540bdc --- /dev/null +++ b/x-pack/plugins/osquery/public/packs/form/shards/shards_policy_field.tsx @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useCallback, useMemo, useState, useEffect } from 'react'; +import { useController } from 'react-hook-form'; +import type { EuiComboBoxOptionOption } from '@elastic/eui'; +import { EuiComboBox, EuiFormRow } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { useAgentPolicies } from '../../../agent_policies'; +import type { ShardsFormReturn } from './shards_form'; + +interface ShardsPolicyFieldComponent { + index: number; + control: ShardsFormReturn['control']; + euiFieldProps?: Record; + hideLabel?: boolean; + options: Array>; +} + +const ShardsPolicyFieldComponent = ({ + index, + control, + hideLabel, + options, +}: ShardsPolicyFieldComponent) => { + const { data: { agentPoliciesById } = {} } = useAgentPolicies(); + + const policyFieldValidator = useCallback( + (policy: { key: string; label: string }) => + !policy + ? i18n.translate('xpack.osquery.pack.form.shardsPolicyFieldMissingErrorMessage', { + defaultMessage: 'Policy is a required field', + }) + : undefined, + + [] + ); + + const { + field: { onChange, value }, + fieldState: { error }, + } = useController({ + control, + name: `shardsArray.${index}.policy`, + rules: { + validate: policyFieldValidator, + }, + }); + + const hasError = useMemo(() => !!error?.message, [error?.message]); + + const [selectedOptions, setSelected] = useState([]); + const handleChange = useCallback( + (newSelectedOptions: EuiComboBoxOptionOption[]) => { + setSelected(newSelectedOptions); + onChange(newSelectedOptions[0]); + }, + [onChange] + ); + + useEffect(() => { + const foundPolicy = agentPoliciesById?.[value.key]; + if (value && foundPolicy) { + setSelected([{ label: value.label || foundPolicy.name, value: value.key }]); + } + }, [agentPoliciesById, value]); + + const singleSelectionConfig = useMemo(() => ({ asPlainText: true }), []); + + return ( + + + + ); +}; + +export const ShardsPolicyField = React.memo(ShardsPolicyFieldComponent); diff --git a/x-pack/plugins/osquery/public/packs/queries/use_pack_query_form.tsx b/x-pack/plugins/osquery/public/packs/queries/use_pack_query_form.tsx index 53971345dbb7e..d5f222af8f1a0 100644 --- a/x-pack/plugins/osquery/public/packs/queries/use_pack_query_form.tsx +++ b/x-pack/plugins/osquery/public/packs/queries/use_pack_query_form.tsx @@ -11,6 +11,7 @@ import type { Draft } from 'immer'; import { produce } from 'immer'; import { useMemo } from 'react'; import type { ECSMapping } from '@kbn/osquery-io-ts-types'; +import type { Shard } from '../../../common/schemas/common/utils'; export interface UsePackQueryFormProps { uniqueQueryIds: string[]; @@ -26,6 +27,7 @@ export interface PackSOQueryFormData { platform?: string | undefined; version?: string | undefined; ecs_mapping?: ECSMapping; + shards: Shard; } export type PackQuerySOECSMapping = Array<{ field: string; value: string }>; diff --git a/x-pack/plugins/osquery/public/packs/types.ts b/x-pack/plugins/osquery/public/packs/types.ts index 941d4f400042d..b5d5181f98d03 100644 --- a/x-pack/plugins/osquery/public/packs/types.ts +++ b/x-pack/plugins/osquery/public/packs/types.ts @@ -5,6 +5,7 @@ * 2.0. */ import type { SavedObject } from '@kbn/core/public'; +import type { Shard } from '../../common/schemas/common/utils'; import type { PackQueryFormData } from './queries/use_pack_query_form'; export type PackSavedObject = SavedObject<{ @@ -23,4 +24,5 @@ export type PackItem = PackSavedObject['attributes'] & { id: string; policy_ids: string[]; read_only?: boolean; + shards?: Shard; }; diff --git a/x-pack/plugins/osquery/public/routes/saved_queries/edit/tabs.tsx b/x-pack/plugins/osquery/public/routes/saved_queries/edit/tabs.tsx index 81ccb30d65e10..9fe31e920564c 100644 --- a/x-pack/plugins/osquery/public/routes/saved_queries/edit/tabs.tsx +++ b/x-pack/plugins/osquery/public/routes/saved_queries/edit/tabs.tsx @@ -43,6 +43,7 @@ const ResultTabsComponent: React.FC = ({ { id: 'results', name: 'Results', + 'data-test-subj': 'osquery-results-tab', content: ( = ({ { id: 'status', name: 'Status', + 'data-test-subj': 'osquery-status-tab', content: ( ), diff --git a/x-pack/plugins/osquery/public/types.ts b/x-pack/plugins/osquery/public/types.ts index d3771383e14e8..d591776a41ecd 100644 --- a/x-pack/plugins/osquery/public/types.ts +++ b/x-pack/plugins/osquery/public/types.ts @@ -18,6 +18,7 @@ import type { } from '@kbn/triggers-actions-ui-plugin/public'; import type { CasesUiStart, CasesUiSetup } from '@kbn/cases-plugin/public'; import type { TimelinesUIStart } from '@kbn/timelines-plugin/public'; +import type { SpacesPluginStart } from '@kbn/spaces-plugin/public'; import type { getLazyOsqueryResults, getLazyLiveQueryField, @@ -47,6 +48,7 @@ export interface StartPlugins { fleet: FleetStart; lens?: LensPublicStart; security: SecurityPluginStart; + spaces: SpacesPluginStart; triggersActionsUi: TriggersAndActionsUIPublicPluginStart; cases: CasesUiStart; timelines: TimelinesUIStart; diff --git a/x-pack/plugins/osquery/server/common/types.ts b/x-pack/plugins/osquery/server/common/types.ts index 8508eddae63b9..522f1fa250ada 100644 --- a/x-pack/plugins/osquery/server/common/types.ts +++ b/x-pack/plugins/osquery/server/common/types.ts @@ -14,6 +14,8 @@ export interface IQueryPayload { }; } +export type SOShard = Array<{ key: string; value: number }>; + export interface PackSavedObjectAttributes { name: string; description: string | undefined; @@ -33,6 +35,7 @@ export interface PackSavedObjectAttributes { updated_at: string; updated_by: string | undefined; policy_ids?: string[]; + shards: SOShard; } export type PackSavedObject = SavedObject; diff --git a/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts b/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts index c915b15e603f5..f0c04848aa52f 100644 --- a/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts +++ b/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts @@ -123,6 +123,10 @@ export const packSavedObjectMappings: SavedObjectsType['mappings'] = { enabled: { type: 'boolean', }, + shards: { + type: 'object', + enabled: false, + }, version: { type: 'long', }, @@ -195,6 +199,10 @@ export const packAssetSavedObjectMappings: SavedObjectsType['mappings'] = { version: { type: 'long', }, + shards: { + type: 'object', + enabled: false, + }, queries: { dynamic: false, properties: { diff --git a/x-pack/plugins/osquery/server/lib/update_global_packs.ts b/x-pack/plugins/osquery/server/lib/update_global_packs.ts new file mode 100644 index 0000000000000..40e08ad92d9a2 --- /dev/null +++ b/x-pack/plugins/osquery/server/lib/update_global_packs.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + ElasticsearchClient, + SavedObjectsClient, + SavedObjectsFindResponse, +} from '@kbn/core/server'; +import { has, map, mapKeys, set, unset } from 'lodash'; +import type { PackagePolicy } from '@kbn/fleet-plugin/common'; +import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common'; +import produce from 'immer'; +import { convertShardsToObject } from '../routes/utils'; +import { packSavedObjectType } from '../../common/types'; +import type { OsqueryAppContextService } from './osquery_app_context_services'; +import type { PackSavedObjectAttributes } from '../common/types'; +import { convertSOQueriesToPackConfig } from '../routes/pack/utils'; +import type { PackSavedObject } from '../common/types'; + +export const updateGlobalPacksCreateCallback = async ( + packagePolicy: PackagePolicy, + packsClient: SavedObjectsClient, + allPacks: SavedObjectsFindResponse, + osqueryContext: OsqueryAppContextService, + esClient: ElasticsearchClient +) => { + const agentPolicyService = osqueryContext.getAgentPolicyService(); + + const packagePolicyService = osqueryContext.getPackagePolicyService(); + const agentPoliciesResult = await agentPolicyService?.getByIds(packsClient, [ + packagePolicy.policy_id, + ]); + const agentPolicyResultIds = map(agentPoliciesResult, 'id'); + const agentPolicies = agentPoliciesResult + ? mapKeys(await agentPolicyService?.getByIds(packsClient, agentPolicyResultIds), 'id') + : {}; + + const packsContainingShardForPolicy: PackSavedObject[] = []; + allPacks.saved_objects.map((pack) => { + const shards = convertShardsToObject(pack.attributes.shards); + + return map(shards, (shard, shardName) => { + if (shardName === '*') { + packsContainingShardForPolicy.push(pack); + } + }); + }); + + await Promise.all( + map(packsContainingShardForPolicy, (pack) => { + packsClient.update( + packSavedObjectType, + pack.id, + {}, + { + references: [ + ...pack.references, + { + id: packagePolicy.policy_id, + name: agentPolicies[packagePolicy.policy_id]?.name, + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + }, + ], + } + ); + }) + ); + await packagePolicyService?.update( + packsClient, + esClient, + packagePolicy.id, + produce(packagePolicy, (draft) => { + unset(draft, 'id'); + if (!has(draft, 'inputs[0].streams')) { + set(draft, 'inputs[0].streams', []); + } + + map(packsContainingShardForPolicy, (pack) => { + set(draft, `inputs[0].config.osquery.value.packs.${pack.attributes.name}`, { + shard: 100, + queries: convertSOQueriesToPackConfig(pack.attributes.queries), + }); + }); + + return draft; + }) + ); +}; diff --git a/x-pack/plugins/osquery/server/plugin.ts b/x-pack/plugins/osquery/server/plugin.ts index 601e0e29a3a83..660b5518a66d9 100644 --- a/x-pack/plugins/osquery/server/plugin.ts +++ b/x-pack/plugins/osquery/server/plugin.ts @@ -17,6 +17,9 @@ import type { PackagePolicy } from '@kbn/fleet-plugin/common'; import type { DataRequestHandlerContext } from '@kbn/data-plugin/server'; import type { DataViewsService } from '@kbn/data-views-plugin/common'; +import type { PackSavedObjectAttributes } from './common/types'; +import { updateGlobalPacksCreateCallback } from './lib/update_global_packs'; +import { packSavedObjectType } from '../common/types'; import type { CreateLiveQueryRequestBodySchema } from '../common/schemas/routes/live_query'; import { createConfig } from './create_config'; import type { OsqueryPluginSetup, OsqueryPluginStart, SetupPlugins, StartPlugins } from './types'; @@ -118,9 +121,10 @@ export class OsqueryPlugin implements Plugin => { if (packagePolicy.package?.name === OSQUERY_INTEGRATION_NAME) { await this.initialize(core, dataViewsService); + + const allPacks = await client.find({ + type: packSavedObjectType, + }); + + if (allPacks.saved_objects) { + await updateGlobalPacksCreateCallback( + packagePolicy, + client, + allPacks, + this.osqueryAppContextService, + esClient + ); + } } return packagePolicy; diff --git a/x-pack/plugins/osquery/server/routes/pack/create_pack_route.ts b/x-pack/plugins/osquery/server/routes/pack/create_pack_route.ts index 2726355d23dcd..37d61b908f150 100644 --- a/x-pack/plugins/osquery/server/routes/pack/create_pack_route.ts +++ b/x-pack/plugins/osquery/server/routes/pack/create_pack_route.ts @@ -6,7 +6,7 @@ */ import moment from 'moment-timezone'; -import { has, mapKeys, set, unset, find, some } from 'lodash'; +import { has, set, unset, find, some, mapKeys } from 'lodash'; import { schema } from '@kbn/config-schema'; import { produce } from 'immer'; import type { PackagePolicy } from '@kbn/fleet-plugin/common'; @@ -19,8 +19,13 @@ import type { OsqueryAppContext } from '../../lib/osquery_app_context_services'; import { OSQUERY_INTEGRATION_NAME } from '../../../common'; import { PLUGIN_ID } from '../../../common'; import { packSavedObjectType } from '../../../common/types'; -import { convertPackQueriesToSO, convertSOQueriesToPackConfig } from './utils'; -import { getInternalSavedObjectsClient } from '../utils'; +import { + convertSOQueriesToPackConfig, + convertPackQueriesToSO, + findMatchingShards, + getInitialPolicies, +} from './utils'; +import { convertShardsToArray, getInternalSavedObjectsClient } from '../utils'; import type { PackSavedObjectAttributes } from '../../common/types'; export const createPackRoute = (router: IRouter, osqueryContext: OsqueryAppContext) => { @@ -34,6 +39,7 @@ export const createPackRoute = (router: IRouter, osqueryContext: OsqueryAppConte description: schema.maybe(schema.string()), enabled: schema.maybe(schema.boolean()), policy_ids: schema.maybe(schema.arrayOf(schema.string())), + shards: schema.recordOf(schema.string(), schema.number()), queries: schema.recordOf( schema.string(), schema.object({ @@ -75,8 +81,7 @@ export const createPackRoute = (router: IRouter, osqueryContext: OsqueryAppConte const currentUser = await osqueryContext.security.authc.getCurrentUser(request)?.username; // eslint-disable-next-line @typescript-eslint/naming-convention - const { name, description, queries, enabled, policy_ids } = request.body; - + const { name, description, queries, enabled, policy_ids, shards } = request.body; const conflictingEntries = await savedObjectsClient.find({ type: packSavedObjectType, filter: `${packSavedObjectType}.attributes.name: "${name}"`, @@ -98,17 +103,22 @@ export const createPackRoute = (router: IRouter, osqueryContext: OsqueryAppConte } )) ?? { items: [] }; - const agentPolicies = policy_ids - ? mapKeys(await agentPolicyService?.getByIds(internalSavedObjectsClient, policy_ids), 'id') - : {}; + const policiesList = getInitialPolicies(packagePolicies, policy_ids, shards); + + const agentPolicies = await agentPolicyService?.getByIds( + internalSavedObjectsClient, + policiesList + ); + + const policyShards = findMatchingShards(agentPolicies, shards); + + const agentPoliciesIdMap = mapKeys(agentPolicies, 'id'); - const references = policy_ids - ? policy_ids.map((policyId: string) => ({ - id: policyId, - name: agentPolicies[policyId].name, - type: AGENT_POLICY_SAVED_OBJECT_TYPE, - })) - : []; + const references = policiesList.map((id) => ({ + id, + name: agentPoliciesIdMap[id]?.name, + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + })); const packSO = await savedObjectsClient.create( packSavedObjectType, @@ -121,6 +131,7 @@ export const createPackRoute = (router: IRouter, osqueryContext: OsqueryAppConte created_by: currentUser, updated_at: moment().toISOString(), updated_by: currentUser, + shards: convertShardsToArray(shards), }, { references, @@ -128,9 +139,9 @@ export const createPackRoute = (router: IRouter, osqueryContext: OsqueryAppConte } ); - if (enabled && policy_ids?.length) { + if (enabled && policiesList.length) { await Promise.all( - policy_ids.map((agentPolicyId) => { + policiesList.map((agentPolicyId) => { const packagePolicy = find(packagePolicies, ['policy_id', agentPolicyId]); if (packagePolicy) { return packagePolicyService?.update( @@ -144,6 +155,9 @@ export const createPackRoute = (router: IRouter, osqueryContext: OsqueryAppConte } set(draft, `inputs[0].config.osquery.value.packs.${packSO.attributes.name}`, { + shard: policyShards[packagePolicy.policy_id] + ? policyShards[packagePolicy.policy_id] + : 100, queries: convertSOQueriesToPackConfig(queries), }); diff --git a/x-pack/plugins/osquery/server/routes/pack/read_pack_route.ts b/x-pack/plugins/osquery/server/routes/pack/read_pack_route.ts index 35790b2a61658..7dbea0f197247 100644 --- a/x-pack/plugins/osquery/server/routes/pack/read_pack_route.ts +++ b/x-pack/plugins/osquery/server/routes/pack/read_pack_route.ts @@ -14,6 +14,7 @@ import { PLUGIN_ID } from '../../../common'; import { packSavedObjectType } from '../../../common/types'; import { convertSOQueriesToPack } from './utils'; +import { convertShardsToObject } from '../utils'; export const readPackRoute = (router: IRouter) => { router.get( @@ -45,6 +46,7 @@ export const readPackRoute = (router: IRouter) => { ...rest, ...attributes, queries: convertSOQueriesToPack(attributes.queries), + shards: convertShardsToObject(attributes.shards), policy_ids: policyIds, read_only: attributes.version !== undefined && osqueryPackAssetReference, }, diff --git a/x-pack/plugins/osquery/server/routes/pack/update_pack_route.ts b/x-pack/plugins/osquery/server/routes/pack/update_pack_route.ts index 984f59bb51a29..c8d053d8d92da 100644 --- a/x-pack/plugins/osquery/server/routes/pack/update_pack_route.ts +++ b/x-pack/plugins/osquery/server/routes/pack/update_pack_route.ts @@ -6,7 +6,19 @@ */ import moment from 'moment-timezone'; -import { set, unset, has, difference, filter, find, map, mapKeys, uniq, some } from 'lodash'; +import { + set, + unset, + has, + difference, + filter, + find, + map, + mapKeys, + uniq, + some, + isEmpty, +} from 'lodash'; import { schema } from '@kbn/config-schema'; import { produce } from 'immer'; import type { PackagePolicy } from '@kbn/fleet-plugin/common'; @@ -24,8 +36,11 @@ import { convertSOQueriesToPack, convertPackQueriesToSO, convertSOQueriesToPackConfig, + getInitialPolicies, + findMatchingShards, } from './utils'; -import { getInternalSavedObjectsClient } from '../utils'; + +import { convertShardsToArray, getInternalSavedObjectsClient } from '../utils'; import type { PackSavedObjectAttributes } from '../../common/types'; export const updatePackRoute = (router: IRouter, osqueryContext: OsqueryAppContext) => { @@ -45,6 +60,7 @@ export const updatePackRoute = (router: IRouter, osqueryContext: OsqueryAppConte description: schema.maybe(schema.string()), enabled: schema.maybe(schema.boolean()), policy_ids: schema.maybe(schema.arrayOf(schema.string())), + shards: schema.maybe(schema.recordOf(schema.string(), schema.number())), queries: schema.maybe( schema.recordOf( schema.string(), @@ -87,7 +103,7 @@ export const updatePackRoute = (router: IRouter, osqueryContext: OsqueryAppConte const currentUser = await osqueryContext.security.authc.getCurrentUser(request)?.username; // eslint-disable-next-line @typescript-eslint/naming-convention - const { name, description, queries, enabled, policy_ids } = request.body; + const { name, description, queries, enabled, policy_ids, shards = {} } = request.body; const currentPackSO = await savedObjectsClient.get<{ name: string; enabled: boolean }>( packSavedObjectType, @@ -121,32 +137,40 @@ export const updatePackRoute = (router: IRouter, osqueryContext: OsqueryAppConte const currentPackagePolicies = filter(packagePolicies, (packagePolicy) => has(packagePolicy, `inputs[0].config.osquery.value.packs.${currentPackSO.attributes.name}`) ); - const agentPolicies = policy_ids - ? mapKeys(await agentPolicyService?.getByIds(internalSavedObjectsClient, policy_ids), 'id') - : {}; - const agentPolicyIds = Object.keys(agentPolicies); + + const policiesList = getInitialPolicies(packagePolicies, policy_ids, shards); + + const agentPolicies = await agentPolicyService?.getByIds( + internalSavedObjectsClient, + policiesList + ); + + const policyShards = findMatchingShards(agentPolicies, shards); + + const agentPoliciesIdMap = mapKeys(agentPolicies, 'id'); const nonAgentPolicyReferences = filter( currentPackSO.references, (reference) => reference.type !== AGENT_POLICY_SAVED_OBJECT_TYPE ); - const getUpdatedReferences = () => { - if (policy_ids) { - return [ - ...nonAgentPolicyReferences, - ...policy_ids.map((id) => ({ - id, - name: agentPolicies[id].name, - type: AGENT_POLICY_SAVED_OBJECT_TYPE, - })), - ]; + if (!policy_ids && isEmpty(shards)) { + return currentPackSO.references; } - return currentPackSO.references; + return [ + ...nonAgentPolicyReferences, + ...policiesList.map((id) => ({ + id, + name: agentPoliciesIdMap[id]?.name, + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + })), + ]; }; - await savedObjectsClient.update( + const references = getUpdatedReferences(); + + await savedObjectsClient.update( packSavedObjectType, request.params.id, { @@ -156,10 +180,11 @@ export const updatePackRoute = (router: IRouter, osqueryContext: OsqueryAppConte queries: queries && convertPackQueriesToSO(queries), updated_at: moment().toISOString(), updated_by: currentUser, + shards: convertShardsToArray(shards), }, { refresh: 'wait_for', - references: getUpdatedReferences(), + references, } ); @@ -167,7 +192,6 @@ export const updatePackRoute = (router: IRouter, osqueryContext: OsqueryAppConte filter(currentPackSO.references, ['type', AGENT_POLICY_SAVED_OBJECT_TYPE]), 'id' ); - const updatedPackSO = await savedObjectsClient.get<{ name: string; enabled: boolean; @@ -182,7 +206,7 @@ export const updatePackRoute = (router: IRouter, osqueryContext: OsqueryAppConte if (enabled != null && enabled !== currentPackSO.attributes.enabled) { if (enabled) { - const policyIds = policy_ids ? agentPolicyIds : currentAgentPolicyIds; + const policyIds = policy_ids || !isEmpty(shards) ? policiesList : currentAgentPolicyIds; await Promise.all( policyIds.map((agentPolicyId) => { @@ -237,11 +261,12 @@ export const updatePackRoute = (router: IRouter, osqueryContext: OsqueryAppConte ); } } else { - const agentPolicyIdsToRemove = uniq(difference(currentAgentPolicyIds, agentPolicyIds)); + // TODO double check if policiesList shouldnt be changed into policyIds + const agentPolicyIdsToRemove = uniq(difference(currentAgentPolicyIds, policiesList)); const agentPolicyIdsToUpdate = uniq( difference(currentAgentPolicyIds, agentPolicyIdsToRemove) ); - const agentPolicyIdsToAdd = uniq(difference(agentPolicyIds, currentAgentPolicyIds)); + const agentPolicyIdsToAdd = uniq(difference(policiesList, currentAgentPolicyIds)); await Promise.all( agentPolicyIdsToRemove.map((agentPolicyId) => { @@ -287,6 +312,9 @@ export const updatePackRoute = (router: IRouter, osqueryContext: OsqueryAppConte draft, `inputs[0].config.osquery.value.packs.${updatedPackSO.attributes.name}`, { + shard: policyShards[packagePolicy.policy_id] + ? policyShards[packagePolicy.policy_id] + : 100, queries: convertSOQueriesToPackConfig(updatedPackSO.attributes.queries), } ); @@ -317,6 +345,9 @@ export const updatePackRoute = (router: IRouter, osqueryContext: OsqueryAppConte draft, `inputs[0].config.osquery.value.packs.${updatedPackSO.attributes.name}`, { + shard: policyShards[packagePolicy.policy_id] + ? policyShards[packagePolicy.policy_id] + : 100, queries: convertSOQueriesToPackConfig(updatedPackSO.attributes.queries), } ); diff --git a/x-pack/plugins/osquery/server/routes/pack/utils.ts b/x-pack/plugins/osquery/server/routes/pack/utils.ts index 1630f303d4f79..3be425ad1ec28 100644 --- a/x-pack/plugins/osquery/server/routes/pack/utils.ts +++ b/x-pack/plugins/osquery/server/routes/pack/utils.ts @@ -5,7 +5,10 @@ * 2.0. */ -import { isEmpty, pick, reduce, isArray } from 'lodash'; +import { isEmpty, pick, reduce, isArray, filter, uniq, map, mapKeys } from 'lodash'; +import { satisfies } from 'semver'; +import type { AgentPolicy, PackagePolicy } from '@kbn/fleet-plugin/common'; +import type { Shard } from '../../../common/schemas/common/utils'; import { DEFAULT_PLATFORM } from '../../../common/constants'; import { removeMultilines } from '../../../common/utils/build_query/remove_multilines'; import { convertECSMappingToArray, convertECSMappingToObject } from '../utils'; @@ -88,3 +91,35 @@ export const convertSOQueriesToPackConfig = ( // eslint-disable-next-line @typescript-eslint/no-explicit-any {} as Record ); + +export const getInitialPolicies = ( + packagePolicies: PackagePolicy[] | never[], + policyIds: string[] = [], + shards?: Shard +) => { + // we want to find all policies, because this is a global pack + if (shards?.['*']) { + const supportedPackagePolicyIds = filter(packagePolicies, (packagePolicy) => + satisfies(packagePolicy.package?.version ?? '', '>=0.6.0') + ); + + return uniq(map(supportedPackagePolicyIds, 'policy_id')); + } + + return policyIds; +}; + +export const findMatchingShards = (agentPolicies: AgentPolicy[] | undefined, shards?: Shard) => { + const policyShards: Shard = {}; + if (!isEmpty(shards)) { + const agentPoliciesIdMap = mapKeys(agentPolicies, 'id'); + + map(shards, (shard, shardName) => { + if (agentPoliciesIdMap[shardName]) { + policyShards[agentPoliciesIdMap[shardName].id] = shard; + } + }); + } + + return policyShards; +}; diff --git a/x-pack/plugins/osquery/server/routes/utils.ts b/x-pack/plugins/osquery/server/routes/utils.ts index 95447fa998848..5395d1c3968cb 100644 --- a/x-pack/plugins/osquery/server/routes/utils.ts +++ b/x-pack/plugins/osquery/server/routes/utils.ts @@ -8,6 +8,8 @@ import type { CoreSetup } from '@kbn/core/server'; import { SavedObjectsClient } from '@kbn/core/server'; import { reduce } from 'lodash'; +import type { Shard } from '../../common/schemas/common/utils'; +import type { SOShard } from '../common/types'; export const convertECSMappingToArray = (ecsMapping: Record | undefined) => ecsMapping @@ -30,6 +32,23 @@ export const convertECSMappingToObject = ( {} as Record ); +export const convertShardsToArray = (shards: Shard): SOShard => + Object.entries(shards).map((item) => ({ + key: item[0], + value: item[1], + })); + +export const convertShardsToObject = (shards: Array<{ key: string; value: number }>) => + reduce( + shards, + (acc, value) => { + acc[value.key] = value.value; + + return acc; + }, + {} as Record + ); + export const getInternalSavedObjectsClient = async ( getStartServices: CoreSetup['getStartServices'] ) => { diff --git a/x-pack/test/osquery_cypress/artifact_manager.ts b/x-pack/test/osquery_cypress/artifact_manager.ts index 7ee2680e21f83..74097deb442cc 100644 --- a/x-pack/test/osquery_cypress/artifact_manager.ts +++ b/x-pack/test/osquery_cypress/artifact_manager.ts @@ -6,5 +6,5 @@ */ export async function getLatestVersion(): Promise { - return '8.6.0-SNAPSHOT'; + return '8.5.1-SNAPSHOT'; } From 9e28b7cedcd98f2933d5872af5004de099d8a3e2 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Mon, 14 Nov 2022 17:20:57 +0100 Subject: [PATCH 02/16] [Graph] Move codeownership to kibana-visualizations (#145119) Co-authored-by: Julia Rechkunova --- .github/CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fe89a23ee3573..51968181ef567 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -15,8 +15,6 @@ /test/functional/apps/discover/ @elastic/kibana-data-discovery /test/functional/apps/context/ @elastic/kibana-data-discovery /test/api_integration/apis/unified_field_list/ @elastic/kibana-data-discovery -/x-pack/plugins/graph/ @elastic/kibana-data-discovery -/x-pack/test/functional/apps/graph @elastic/kibana-data-discovery /src/plugins/unified_field_list/ @elastic/kibana-data-discovery /src/plugins/unified_histogram/ @elastic/kibana-data-discovery /src/plugins/saved_objects_finder/ @elastic/kibana-data-discovery @@ -49,6 +47,8 @@ /test/functional/apps/visualize/ @elastic/kibana-visualizations /src/plugins/expressions/ @elastic/kibana-visualizations /src/plugins/unified_search/ @elastic/kibana-visualizations +/x-pack/plugins/graph/ @elastic/kibana-visualizations +/x-pack/test/functional/apps/graph @elastic/kibana-visualizations # Application Services /examples/dashboard_embeddable_examples/ @elastic/kibana-app-services From 90f6ffb353be44024e2b66cd86921550045d1de5 Mon Sep 17 00:00:00 2001 From: Kurt Date: Mon, 14 Nov 2022 11:30:37 -0500 Subject: [PATCH 03/16] Adding readonly view for API Keys page (#144923) ## Summary Adding a `readonly` view for users with `read_security` cluster privileges ## Release Note The API Keys screen can be accessed in a Read Only view with the the cluster privilege `read_security` ## Testing Steps Login as `elastic` and create a `role` with the `read_security` cluster privilege Screen Shot 2022-11-09 at 1 03 05 PM Create a test user and assign the newly create role, as well as `viewer` and `kibana_admin` Screen Shot 2022-11-09 at 1 03 48 PM Login as the new test user and navigate to Stack Management > API Keys Verify there aren't any Create buttons and that the ReadOnly `glasses` icon is in the top right Screen Shot 2022-11-09 at 1 04 59 PM Login as `elastic` and create an API key, remember the name of the key Go to Dev Tools and use the following script to grant usage of the API key to the test user, use the following block: ```json POST /_security/api_key/grant { "grant_type": "password", "username" : "elastic", "password" : "changeme", "run_as": "test_user", "api_key" : { "name": "test-api-key" } } ``` Login as the test user and navigate to the API Keys page, notice the granted API key is displayed, but you are unable to `delete` or `create` new keys Screen Shot 2022-11-09 at 1 06 48 PM Co-authored-by: Thomas Watson --- .../api_keys_grid/api_keys_empty_prompt.tsx | 26 +++ .../api_keys_grid/api_keys_grid_page.test.tsx | 111 ++++++++++++- .../api_keys_grid/api_keys_grid_page.tsx | 154 +++++++++++------- .../api_keys/api_keys_management_app.test.tsx | 18 +- .../api_keys/api_keys_management_app.tsx | 8 + .../server/features/security_features.ts | 6 +- 6 files changed, 260 insertions(+), 63 deletions(-) diff --git a/x-pack/plugins/security/public/management/api_keys/api_keys_grid/api_keys_empty_prompt.tsx b/x-pack/plugins/security/public/management/api_keys/api_keys_grid/api_keys_empty_prompt.tsx index f93095a72ed6d..4e851cc4ea556 100644 --- a/x-pack/plugins/security/public/management/api_keys/api_keys_grid/api_keys_empty_prompt.tsx +++ b/x-pack/plugins/security/public/management/api_keys/api_keys_grid/api_keys_empty_prompt.tsx @@ -17,10 +17,12 @@ import { useHtmlId } from '../../../components/use_html_id'; export interface ApiKeysEmptyPromptProps { error?: Error; + readOnly?: boolean; } export const ApiKeysEmptyPrompt: FunctionComponent = ({ error, + readOnly, children, }) => { const accordionId = useHtmlId('apiKeysEmptyPrompt', 'accordion'); @@ -115,6 +117,30 @@ export const ApiKeysEmptyPrompt: FunctionComponent = ({ ); } + if (readOnly) { + return ( + + + + } + body={ +

+ +

+ } + /> + ); + } + return ( { // since we are using EuiErrorBoundary and react will console.error any errors const consoleWarnMock = jest.spyOn(console, 'error').mockImplementation(); - const coreStart = coreMock.createStart(); + let coreStart: ReturnType; const theme$ = themeServiceMock.createTheme$(); const apiClientMock = apiKeysAPIClientMock.create(); const { authc } = securityMock.createSetup(); beforeEach(() => { + coreStart = coreMock.createStart(); apiClientMock.checkPrivileges.mockClear(); apiClientMock.getApiKeys.mockClear(); coreStart.http.get.mockClear(); @@ -50,6 +51,7 @@ describe('APIKeysGridPage', () => { canManage: true, isAdmin: true, }); + apiClientMock.getApiKeys.mockResolvedValue({ apiKeys: [ { @@ -83,19 +85,29 @@ describe('APIKeysGridPage', () => { }) ); }); + it('loads and displays API keys', async () => { const history = createMemoryHistory({ initialEntries: ['/'] }); - const { findByText } = render( + coreStart.application.capabilities = { + ...coreStart.application.capabilities, + api_keys: { + save: true, + }, + }; + + const { findByText, queryByTestId } = render( ); + expect(await queryByTestId('apiKeysCreateTableButton')).toBeNull(); expect(await findByText(/Loading API keys/)).not.toBeInTheDocument(); await findByText(/first-api-key/); await findByText(/second-api-key/); @@ -114,12 +126,20 @@ describe('APIKeysGridPage', () => { isAdmin: true, }); + coreStart.application.capabilities = { + ...coreStart.application.capabilities, + api_keys: { + save: true, + }, + }; + const { findByText } = render( ); @@ -136,12 +156,20 @@ describe('APIKeysGridPage', () => { isAdmin: false, }); + coreStart.application.capabilities = { + ...coreStart.application.capabilities, + api_keys: { + save: true, + }, + }; + const { findByText } = render( ); @@ -160,12 +188,20 @@ describe('APIKeysGridPage', () => { }); const history = createMemoryHistory({ initialEntries: ['/'] }); + coreStart.application.capabilities = { + ...coreStart.application.capabilities, + api_keys: { + save: true, + }, + }; + const { findByText } = render( ); @@ -173,4 +209,75 @@ describe('APIKeysGridPage', () => { expect(await findByText(/Loading API keys/)).not.toBeInTheDocument(); await findByText(/Could not load API keys/); }); + + describe('Read Only View', () => { + beforeEach(() => { + apiClientMock.checkPrivileges.mockResolvedValueOnce({ + areApiKeysEnabled: true, + canManage: false, + isAdmin: false, + }); + }); + + it('should not display prompt `Create Button` when no API keys are shown', async () => { + const history = createMemoryHistory({ initialEntries: ['/'] }); + + apiClientMock.getApiKeys.mockResolvedValue({ + apiKeys: [], + }); + + coreStart.application.capabilities = { + ...coreStart.application.capabilities, + api_keys: { + save: false, + }, + }; + + const { findByText, queryByText } = render( + + + + ); + expect(await findByText(/Loading API keys/)).not.toBeInTheDocument(); + expect(await findByText('You do not have permission to create API keys')).toBeInTheDocument(); + expect(queryByText('Create API key')).toBeNull(); + }); + + it('should not display table `Create Button` nor `Delete` icons column', async () => { + const history = createMemoryHistory({ initialEntries: ['/'] }); + + coreStart.application.capabilities = { + ...coreStart.application.capabilities, + api_keys: { + save: false, + }, + }; + + const { findByText, queryByText, queryAllByText } = await render( + + + + ); + + expect(await findByText(/Loading API keys/)).not.toBeInTheDocument(); + expect( + await findByText('You only have permission to view your own API keys.') + ).toBeInTheDocument(); + expect( + await findByText('View your API keys. An API key sends requests on your behalf.') + ).toBeInTheDocument(); + expect(queryByText('Create API key')).toBeNull(); + expect(queryAllByText('Delete').length).toBe(0); + }); + }); }); diff --git a/x-pack/plugins/security/public/management/api_keys/api_keys_grid/api_keys_grid_page.tsx b/x-pack/plugins/security/public/management/api_keys/api_keys_grid/api_keys_grid_page.tsx index 04ae1e7bae91e..aa94f4f147789 100644 --- a/x-pack/plugins/security/public/management/api_keys/api_keys_grid/api_keys_grid_page.tsx +++ b/x-pack/plugins/security/public/management/api_keys/api_keys_grid/api_keys_grid_page.tsx @@ -48,6 +48,7 @@ interface Props { history: History; notifications: NotificationsStart; apiKeysAPIClient: PublicMethodsOf; + readOnly?: boolean; } interface State { @@ -65,6 +66,10 @@ interface State { const DATE_FORMAT = 'MMMM Do YYYY HH:mm:ss'; export class APIKeysGridPage extends Component { + static defaultProps: Partial = { + readOnly: false, + }; + constructor(props: any) { super(props); this.state = { @@ -147,25 +152,31 @@ export class APIKeysGridPage extends Component { } if (!isLoadingTable && apiKeys && apiKeys.length === 0) { - return ( - - - - - - ); + if (this.props.readOnly) { + return ; + } else { + return ( + + + + + + ); + } } const concatenated = `${this.state.createdApiKey?.id}:${this.state.createdApiKey?.api_key}`; + const description = this.determineDescription(isAdmin, this.props.readOnly ?? false); + return ( <> { defaultMessage="API Keys" /> } - description={ - <> - {isAdmin ? ( - - ) : ( - - )} - + description={description} + rightSideItems={ + this.props.readOnly + ? undefined + : [ + + + , + ] } - rightSideItems={[ - - - , - ]} /> {this.state.createdApiKey && !this.state.isLoadingTable && ( @@ -421,20 +422,13 @@ export class APIKeysGridPage extends Component { : undefined, }; + const callOutTitle = this.determineCallOutTitle(this.props.readOnly ?? false); + return ( <> {!isAdmin ? ( <> - - } - color="primary" - iconType="user" - /> + ) : undefined} @@ -451,7 +445,7 @@ export class APIKeysGridPage extends Component { columns={this.getColumnConfig(invalidateApiKeyPrompt)} search={search} sorting={sorting} - selection={selection} + selection={this.props.readOnly ? undefined : selection} pagination={pagination} loading={isLoadingTable} error={ @@ -580,7 +574,10 @@ export class APIKeysGridPage extends Component { ); }, }, - { + ]); + + if (!this.props.readOnly) { + config.push({ actions: [ { name: i18n.translate('xpack.security.management.apiKeys.table.deleteAction', { @@ -600,8 +597,8 @@ export class APIKeysGridPage extends Component { 'data-test-subj': 'apiKeysTableDeleteAction', }, ], - }, - ]); + }); + } return config; }; @@ -618,7 +615,7 @@ export class APIKeysGridPage extends Component { await this.props.apiKeysAPIClient.checkPrivileges(); this.setState({ isAdmin, canManage, areApiKeysEnabled }); - if (!canManage || !areApiKeysEnabled) { + if ((!canManage && !this.props.readOnly) || !areApiKeysEnabled) { this.setState({ isLoadingApp: false }); } else { this.loadApiKeys(); @@ -654,4 +651,47 @@ export class APIKeysGridPage extends Component { this.setState({ isLoadingApp: false, isLoadingTable: false }); }; + + private determineDescription(isAdmin: boolean, readOnly: boolean) { + if (isAdmin) { + return ( + + ); + } else if (readOnly) { + return ( + + ); + } else { + return ( + + ); + } + } + + private determineCallOutTitle(readOnly: boolean) { + if (readOnly) { + return ( + + ); + } else { + return ( + + ); + } + } } diff --git a/x-pack/plugins/security/public/management/api_keys/api_keys_management_app.test.tsx b/x-pack/plugins/security/public/management/api_keys/api_keys_management_app.test.tsx index c4f3000277af7..bcde6bbb619b7 100644 --- a/x-pack/plugins/security/public/management/api_keys/api_keys_management_app.test.tsx +++ b/x-pack/plugins/security/public/management/api_keys/api_keys_management_app.test.tsx @@ -34,16 +34,27 @@ describe('apiKeysManagementApp', () => { }); it('mount() works for the `grid` page', async () => { - const { getStartServices } = coreMock.createSetup(); + const coreStart = coreMock.createSetup(); const { authc } = securityMock.createSetup(); - const startServices = await getStartServices(); + const startServices = await coreStart.getStartServices(); + + const [{ application }] = startServices; + application.capabilities = { + ...application.capabilities, + api_keys: { + save: true, + }, + }; + const docTitle = startServices[0].chrome.docTitle; const container = document.createElement('div'); const setBreadcrumbs = jest.fn(); + let unmount: Unmount; + await act(async () => { unmount = await apiKeysManagementApp .create({ authc, getStartServices: () => Promise.resolve(startServices) as any }) @@ -84,7 +95,8 @@ describe('apiKeysManagementApp', () => { "anonymousPaths": {}, "externalUrl": {} } - } + }, + "readOnly": false } `); diff --git a/x-pack/plugins/security/public/management/api_keys/api_keys_management_app.tsx b/x-pack/plugins/security/public/management/api_keys/api_keys_management_app.tsx index 8145a31f3f6f4..071e1ed42f231 100644 --- a/x-pack/plugins/security/public/management/api_keys/api_keys_management_app.tsx +++ b/x-pack/plugins/security/public/management/api_keys/api_keys_management_app.tsx @@ -27,6 +27,7 @@ import { } from '../../components/breadcrumb'; import { AuthenticationProvider } from '../../components/use_current_user'; import type { PluginStartDependencies } from '../../plugin'; +import { ReadonlyBadge } from '../badges/readonly_badge'; interface CreateParams { authc: AuthenticationServiceSetup; @@ -67,6 +68,7 @@ export const apiKeysManagementApp = Object.freeze({ history={history} notifications={coreStart.notifications} apiKeysAPIClient={new APIKeysAPIClient(coreStart.http)} + readOnly={!coreStart.application.capabilities.api_keys.save} /> , @@ -102,6 +104,12 @@ export const Providers: FunctionComponent = ({ + {children} diff --git a/x-pack/plugins/security/server/features/security_features.ts b/x-pack/plugins/security/server/features/security_features.ts index 46184a845b66c..c566399548648 100644 --- a/x-pack/plugins/security/server/features/security_features.ts +++ b/x-pack/plugins/security/server/features/security_features.ts @@ -52,10 +52,14 @@ const apiKeysManagementFeature: ElasticsearchFeatureConfig = { privileges: [ { requiredClusterPrivileges: ['manage_api_key'], - ui: [], + ui: ['save'], }, { requiredClusterPrivileges: ['manage_own_api_key'], + ui: ['save'], + }, + { + requiredClusterPrivileges: ['read_security'], ui: [], }, ], From 045b57093cc6ce7425ee265b8c1e2cd0de1bc152 Mon Sep 17 00:00:00 2001 From: John Dorlus Date: Mon, 14 Nov 2022 11:31:27 -0500 Subject: [PATCH 04/16] CCS Tests for Machine Learning (#144869) --- .../apps/ml/anomaly_detection_jobs/index.ts | 32 +++++++++++-------- .../single_metric_job.ts | 22 +++++++++---- x-pack/test/functional/config.ccs.ts | 1 + .../functional/services/ml/security_common.ts | 3 ++ 4 files changed, 38 insertions(+), 20 deletions(-) diff --git a/x-pack/test/functional/apps/ml/anomaly_detection_jobs/index.ts b/x-pack/test/functional/apps/ml/anomaly_detection_jobs/index.ts index e2f6901b75c31..7696e4f97eea8 100644 --- a/x-pack/test/functional/apps/ml/anomaly_detection_jobs/index.ts +++ b/x-pack/test/functional/apps/ml/anomaly_detection_jobs/index.ts @@ -8,7 +8,9 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService, loadTestFile }: FtrProviderContext) { - const esArchiver = getService('esArchiver'); + const config = getService('config'); + const isCcs = config.get('esTestCluster.ccs'); + const esNode = isCcs ? getService('remoteEsArchiver' as 'esArchiver') : getService('esArchiver'); const ml = getService('ml'); describe('machine learning - anomaly detection', function () { @@ -17,6 +19,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { before(async () => { await ml.securityCommon.createMlRoles(); await ml.securityCommon.createMlUsers(); + await ml.securityUI.loginAsMlPowerUser(); }); after(async () => { @@ -26,22 +29,25 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { await ml.securityCommon.cleanMlUsers(); await ml.securityCommon.cleanMlRoles(); - await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote'); - await esArchiver.unload('x-pack/test/functional/es_archives/ml/ecommerce'); - await esArchiver.unload('x-pack/test/functional/es_archives/ml/categorization_small'); - await esArchiver.unload('x-pack/test/functional/es_archives/ml/event_rate_nanos'); + await esNode.unload('x-pack/test/functional/es_archives/ml/farequote'); + await esNode.unload('x-pack/test/functional/es_archives/ml/ecommerce'); + await esNode.unload('x-pack/test/functional/es_archives/ml/categorization_small'); + await esNode.unload('x-pack/test/functional/es_archives/ml/event_rate_nanos'); await ml.testResources.resetKibanaTimeZone(); }); loadTestFile(require.resolve('./single_metric_job')); - loadTestFile(require.resolve('./single_metric_job_without_datafeed_start')); - loadTestFile(require.resolve('./multi_metric_job')); - loadTestFile(require.resolve('./population_job')); - loadTestFile(require.resolve('./saved_search_job')); - loadTestFile(require.resolve('./advanced_job')); - loadTestFile(require.resolve('./categorization_job')); - loadTestFile(require.resolve('./date_nanos_job')); - loadTestFile(require.resolve('./custom_urls')); + + if (!isCcs) { + loadTestFile(require.resolve('./single_metric_job_without_datafeed_start')); + loadTestFile(require.resolve('./multi_metric_job')); + loadTestFile(require.resolve('./population_job')); + loadTestFile(require.resolve('./saved_search_job')); + loadTestFile(require.resolve('./advanced_job')); + loadTestFile(require.resolve('./categorization_job')); + loadTestFile(require.resolve('./date_nanos_job')); + loadTestFile(require.resolve('./custom_urls')); + } }); } diff --git a/x-pack/test/functional/apps/ml/anomaly_detection_jobs/single_metric_job.ts b/x-pack/test/functional/apps/ml/anomaly_detection_jobs/single_metric_job.ts index cb21f8de77bd2..a4b702e7400b6 100644 --- a/x-pack/test/functional/apps/ml/anomaly_detection_jobs/single_metric_job.ts +++ b/x-pack/test/functional/apps/ml/anomaly_detection_jobs/single_metric_job.ts @@ -8,7 +8,10 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService }: FtrProviderContext) { - const esArchiver = getService('esArchiver'); + const config = getService('config'); + const esNode = config.get('esTestCluster.ccs') + ? getService('remoteEsArchiver' as 'esArchiver') + : getService('esArchiver'); const ml = getService('ml'); const browser = getService('browser'); @@ -70,12 +73,17 @@ export default function ({ getService }: FtrProviderContext) { } const calendarId = `wizard-test-calendar_${Date.now()}`; + const remoteName = 'ftr-remote:'; + const indexPatternName = 'ft_farequote'; + const indexPatternString = config.get('esTestCluster.ccs') + ? remoteName + indexPatternName + : indexPatternName; describe('single metric', function () { this.tags(['ml']); before(async () => { - await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); - await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp'); + await esNode.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); + await ml.testResources.createIndexPatternIfNeeded(indexPatternString, '@timestamp'); await ml.testResources.setKibanaTimeZoneToUTC(); await ml.api.createCalendar(calendarId); @@ -84,7 +92,7 @@ export default function ({ getService }: FtrProviderContext) { after(async () => { await ml.api.cleanMlIndices(); - await ml.testResources.deleteIndexPatternByTitle('ft_farequote'); + await ml.testResources.deleteIndexPatternByTitle(indexPatternString); }); it('job creation loads the single metric wizard for the source data', async () => { @@ -96,7 +104,7 @@ export default function ({ getService }: FtrProviderContext) { await ml.jobManagement.navigateToNewJobSourceSelection(); await ml.testExecution.logTestStep('job creation loads the job type selection page'); - await ml.jobSourceSelection.selectSourceForAnomalyDetectionJob('ft_farequote'); + await ml.jobSourceSelection.selectSourceForAnomalyDetectionJob(indexPatternString); await ml.testExecution.logTestStep('job creation loads the single metric job wizard page'); await ml.jobTypeSelection.selectSingleMetricJob(); @@ -204,7 +212,7 @@ export default function ({ getService }: FtrProviderContext) { it('job cloning fails in the single metric wizard if a matching data view does not exist', async () => { await ml.testExecution.logTestStep('delete data view used by job'); - await ml.testResources.deleteIndexPatternByTitle('ft_farequote'); + await ml.testResources.deleteIndexPatternByTitle(indexPatternString); // Refresh page to ensure page has correct cache of data views await browser.refresh(); @@ -217,7 +225,7 @@ export default function ({ getService }: FtrProviderContext) { it('job cloning opens the existing job in the single metric wizard', async () => { await ml.testExecution.logTestStep('recreate data view used by job'); - await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp'); + await ml.testResources.createIndexPatternIfNeeded(indexPatternString, '@timestamp'); // Refresh page to ensure page has correct cache of data views await browser.refresh(); diff --git a/x-pack/test/functional/config.ccs.ts b/x-pack/test/functional/config.ccs.ts index 62f988d8f2f02..59681a8c6d9f3 100644 --- a/x-pack/test/functional/config.ccs.ts +++ b/x-pack/test/functional/config.ccs.ts @@ -20,6 +20,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { require.resolve('./apps/lens/group1'), require.resolve('./apps/remote_clusters/ccs/remote_clusters_index_management_flow'), require.resolve('./apps/rollup_job'), + require.resolve('./apps/ml/anomaly_detection_jobs'), ], junit: { diff --git a/x-pack/test/functional/services/ml/security_common.ts b/x-pack/test/functional/services/ml/security_common.ts index ae8f973228b39..6952183e7bdef 100644 --- a/x-pack/test/functional/services/ml/security_common.ts +++ b/x-pack/test/functional/services/ml/security_common.ts @@ -24,7 +24,9 @@ export enum USER { } export function MachineLearningSecurityCommonProvider({ getService }: FtrProviderContext) { + const config = getService('config'); const security = getService('security'); + const remoteEsRoles: undefined | Record = config.get('security.remoteEsRoles'); const roles = [ { @@ -172,6 +174,7 @@ export function MachineLearningSecurityCommonProvider({ getService }: FtrProvide 'ft_ml_source', 'ft_ml_dest', 'ft_ml_ui_extras', + ...(remoteEsRoles ? Object.keys(remoteEsRoles) : []), ], }, { From 594e43e5ef4954935c7e5d1fad23b7003020faf1 Mon Sep 17 00:00:00 2001 From: Nav <13634519+navarone-feekery@users.noreply.github.com> Date: Mon, 14 Nov 2022 17:38:29 +0100 Subject: [PATCH 05/16] [Enterprise Search] Update crawler2 configurations to v2 (#145096) ## Summary https://github.com/elastic/enterprise-search-team/issues/3208 The index `crawler2_configurations` was updated to v2 here: https://github.com/elastic/ent-search/pull/7041 This PR ensures that Kibana uses the correct index version. --- x-pack/plugins/enterprise_search/server/index.ts | 2 +- .../enterprise_search/server/lib/crawler/fetch_crawlers.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/enterprise_search/server/index.ts b/x-pack/plugins/enterprise_search/server/index.ts index c91789eadac35..b8f65c23ab674 100644 --- a/x-pack/plugins/enterprise_search/server/index.ts +++ b/x-pack/plugins/enterprise_search/server/index.ts @@ -42,6 +42,6 @@ export const CONNECTORS_INDEX = '.elastic-connectors'; export const CURRENT_CONNECTORS_INDEX = '.elastic-connectors-v1'; export const CONNECTORS_JOBS_INDEX = '.elastic-connectors-sync-jobs'; export const CONNECTORS_VERSION = 1; -export const CRAWLERS_INDEX = '.ent-search-actastic-crawler2_configurations'; +export const CRAWLERS_INDEX = '.ent-search-actastic-crawler2_configurations_v2'; export const ANALYTICS_COLLECTIONS_INDEX = '.elastic-analytics-collections'; export const ANALYTICS_VERSION = '1'; diff --git a/x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawlers.ts b/x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawlers.ts index c0d3c7344d1a8..7cab68248734c 100644 --- a/x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawlers.ts +++ b/x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawlers.ts @@ -11,8 +11,8 @@ import { IScopedClusterClient } from '@kbn/core/server'; import { Crawler, CrawlRequest } from '../../../common/types/crawler'; import { fetchAll } from '../fetch_all'; -const CRAWLER_CONFIGURATIONS_INDEX = '.ent-search-actastic-crawler2_configurations'; -const CRAWLER_CRAWL_REQUESTS_INDEX = '.ent-search-actastic-crawler2_crawl_requests'; +const CRAWLER_CONFIGURATIONS_INDEX = '.ent-search-actastic-crawler2_configurations_v2'; +const CRAWLER_CRAWL_REQUESTS_INDEX = '.ent-search-actastic-crawler2_crawl_requests_v2'; export const fetchMostRecentCrawlerRequestByConfigurationId = async ( client: IScopedClusterClient, From 164de3c5b44e32467778250531f8964e9f209adb Mon Sep 17 00:00:00 2001 From: Abdul Wahab Zahid Date: Mon, 14 Nov 2022 17:38:38 +0100 Subject: [PATCH 06/16] [Synthetics UI] Fix filtering and search on management page (#144838) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Fixes https://github.com/elastic/kibana/issues/144802 --- .../migrations/check_registered_types.test.ts | 2 +- .../monitor_add_edit/monitor_add_page.tsx | 13 +- .../monitor_add_edit/monitor_edit_page.tsx | 9 +- .../hooks/use_monitor_ilst.test.tsx | 102 +++++++++++++++ .../monitors_page/hooks/use_monitor_list.ts | 13 +- .../management/list_filters/filter_button.tsx | 2 +- .../management/monitor_list_container.tsx | 18 ++- .../management/monitor_list_table/columns.tsx | 1 + .../monitor_list_table/monitor_list.tsx | 5 +- .../components/monitors_page/monitor_page.tsx | 14 +- .../public/apps/synthetics/routes.tsx | 27 +--- .../synthetics/state/monitor_list/models.ts | 2 +- .../__mocks__/synthetics_store.mock.ts | 6 +- .../testing/get_supported_url_params.test.ts | 121 ------------------ .../utils/testing/get_supported_url_params.ts | 106 --------------- .../lib/saved_objects/synthetics_monitor.ts | 3 + .../synthetics/server/routes/common.ts | 32 ++++- .../routes/monitor_cruds/get_monitor.ts | 35 ++--- 18 files changed, 210 insertions(+), 301 deletions(-) create mode 100644 x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/hooks/use_monitor_ilst.test.tsx delete mode 100644 x-pack/plugins/synthetics/public/apps/synthetics/utils/testing/get_supported_url_params.test.ts delete mode 100644 x-pack/plugins/synthetics/public/apps/synthetics/utils/testing/get_supported_url_params.ts diff --git a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts index eb9d10f63ca17..1f0a043782311 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts @@ -130,7 +130,7 @@ describe('checking migration metadata changes on all registered SO types', () => "siem-ui-timeline-pinned-event": "e2697b38751506c7fce6e8b7207a830483dc4283", "space": "c4a0acce1bd4b9cce85154f2a350624a53111c59", "spaces-usage-stats": "922d3235bbf519e3fb3b260e27248b1df8249b79", - "synthetics-monitor": "30f1cd04016a37095de60554cbf7fff89aaad177", + "synthetics-monitor": "111811218f7e34f40980665a4eb99976f457bb23", "synthetics-privates-locations": "dd00385f4a27ef062c3e57312eeb3799872fa4af", "tag": "39413f4578cc2128c9a0fda97d0acd1c8862c47a", "task": "ef53d0f070bd54957b8fe22fae3b1ff208913f76", diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/monitor_add_page.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/monitor_add_page.tsx index 651d751902777..e1a7dced64928 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/monitor_add_page.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/monitor_add_page.tsx @@ -9,14 +9,17 @@ import React, { useEffect } from 'react'; import { EuiLoadingSpinner } from '@elastic/eui'; import { useDispatch } from 'react-redux'; import { useTrackPageview } from '@kbn/observability-plugin/public'; -import { useKibanaSpace } from './hooks'; + import { getServiceLocations } from '../../state'; +import { ServiceAllowedWrapper } from '../common/wrappers/service_allowed_wrapper'; + +import { useKibanaSpace } from './hooks'; import { MonitorSteps } from './steps'; import { MonitorForm } from './form'; import { ADD_MONITOR_STEPS } from './steps/step_config'; import { useMonitorAddEditBreadcrumbs } from './use_breadcrumbs'; -export const MonitorAddPage = () => { +const MonitorAddPage = () => { useTrackPageview({ app: 'synthetics', path: 'add-monitor' }); const { space, loading, error } = useKibanaSpace(); useTrackPageview({ app: 'synthetics', path: 'add-monitor', delay: 15000 }); @@ -35,3 +38,9 @@ export const MonitorAddPage = () => { ); }; + +export const MonitorAddPageWithServiceAllowed = React.memo(() => ( + + + +)); diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/monitor_edit_page.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/monitor_edit_page.tsx index a1aba64f43f4c..53057a9dbba5c 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/monitor_edit_page.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/monitor_edit_page.tsx @@ -11,6 +11,7 @@ import { useParams } from 'react-router-dom'; import { useDispatch } from 'react-redux'; import { useTrackPageview, useFetcher } from '@kbn/observability-plugin/public'; import { getServiceLocations } from '../../state'; +import { ServiceAllowedWrapper } from '../common/wrappers/service_allowed_wrapper'; import { MonitorSteps } from './steps'; import { MonitorForm } from './form'; import { MonitorDetailsLinkPortal } from './monitor_details_portal'; @@ -18,7 +19,7 @@ import { useMonitorAddEditBreadcrumbs } from './use_breadcrumbs'; import { getMonitorAPI } from '../../state/monitor_management/api'; import { EDIT_MONITOR_STEPS } from './steps/step_config'; -export const MonitorEditPage: React.FC = () => { +const MonitorEditPage: React.FC = () => { useTrackPageview({ app: 'synthetics', path: 'edit-monitor' }); useTrackPageview({ app: 'synthetics', path: 'edit-monitor', delay: 15000 }); const { monitorId } = useParams<{ monitorId: string }>(); @@ -42,3 +43,9 @@ export const MonitorEditPage: React.FC = () => { ); }; + +export const MonitorEditPageWithServiceAllowed = React.memo(() => ( + + + +)); diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/hooks/use_monitor_ilst.test.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/hooks/use_monitor_ilst.test.tsx new file mode 100644 index 0000000000000..19204efe110df --- /dev/null +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/hooks/use_monitor_ilst.test.tsx @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { renderHook } from '@testing-library/react-hooks'; +import * as redux from 'react-redux'; +import { MONITOR_ROUTE } from '../../../../../../common/constants'; +import { mockState } from '../../../utils/testing/__mocks__/synthetics_store.mock'; +import { WrappedHelper } from '../../../utils/testing'; +import { SyntheticsAppState } from '../../../state/root_reducer'; +import { + selectEncryptedSyntheticsSavedMonitors, + fetchMonitorListAction, + MonitorListPageState, +} from '../../../state'; + +import { useMonitorList } from './use_monitor_list'; + +describe('useMonitorList', () => { + let state: SyntheticsAppState; + let initialState: Omit, 'loadPage' | 'reloadPage'>; + let defaultPageState: MonitorListPageState; + const dispatchMockFn = jest.fn(); + + beforeEach(() => { + jest.resetAllMocks(); + jest.spyOn(redux, 'useDispatch').mockReturnValue(dispatchMockFn); + + state = mockState; + initialState = { + loading: false, + loaded: false, + total: state.monitorList.data.total ?? 0, + error: state.monitorList.error, + absoluteTotal: state.monitorList.data.absoluteTotal ?? 0, + pageState: state.monitorList.pageState, + isDataQueried: false, + syntheticsMonitors: selectEncryptedSyntheticsSavedMonitors.resultFunc(state.monitorList), + }; + + defaultPageState = { + ...state.monitorList.pageState, + query: '', + locations: [], + monitorType: [], + tags: [], + }; + }); + + it('returns expected initial state', () => { + const { + result: { current: hookResult }, + } = renderHook(() => useMonitorList(), { wrapper: WrappedHelper }); + + expect(hookResult).toMatchObject(initialState); + }); + + it('dispatches correct action for query url param', async () => { + const query = 'xyz'; + const url = `/monitor/1?query=${query}`; + const WrapperWithState = ({ children }: { children: React.ReactElement }) => { + return ( + + {children} + + ); + }; + + renderHook(() => useMonitorList(), { wrapper: WrapperWithState }); + + expect(dispatchMockFn).toHaveBeenCalledWith( + fetchMonitorListAction.get({ ...defaultPageState, query }) + ); + }); + + it('dispatches correct action for filter url param', async () => { + const tags = ['abc', 'xyz']; + const locations = ['loc1', 'loc1']; + const monitorType = ['browser']; + + const url = `/monitor/1?tags=${JSON.stringify(tags)}&locations=${JSON.stringify( + locations + )}&monitorType=${JSON.stringify(monitorType)}`; + const WrapperWithState = ({ children }: { children: React.ReactElement }) => { + return ( + + {children} + + ); + }; + + renderHook(() => useMonitorList(), { wrapper: WrapperWithState }); + + expect(dispatchMockFn).toHaveBeenCalledWith( + fetchMonitorListAction.get({ ...defaultPageState, tags, locations, monitorType }) + ); + }); +}); diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/hooks/use_monitor_list.ts b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/hooks/use_monitor_list.ts index ba8b43677809a..9b30c5590f950 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/hooks/use_monitor_list.ts +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/hooks/use_monitor_list.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { useEffect, useCallback, useState } from 'react'; +import { useEffect, useCallback, useRef } from 'react'; import { useDispatch, useSelector } from 'react-redux'; import { useLocation } from 'react-router-dom'; import { useGetUrlParams } from '../../../hooks'; @@ -18,7 +18,7 @@ import { export function useMonitorList() { const dispatch = useDispatch(); - const [isDataQueried, setIsDataQueried] = useState(false); + const isDataQueriedRef = useRef(false); const { pageState, loading, loaded, error, data } = useSelector(selectMonitorListState); const syntheticsMonitors = useSelector(selectEncryptedSyntheticsSavedMonitors); @@ -50,14 +50,15 @@ export function useMonitorList() { // Initial loading useEffect(() => { - if (!loading && !isDataQueried) { + if (!loading && !isDataQueriedRef.current) { + isDataQueriedRef.current = true; reloadPage(); } if (loading) { - setIsDataQueried(true); + isDataQueriedRef.current = true; } - }, [reloadPage, isDataQueried, syntheticsMonitors, loading]); + }, [reloadPage, syntheticsMonitors, loading]); return { loading, @@ -68,7 +69,7 @@ export function useMonitorList() { total: data?.total ?? 0, loadPage, reloadPage, - isDataQueried, + isDataQueried: isDataQueriedRef.current, absoluteTotal: data.absoluteTotal ?? 0, }; } diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/management/list_filters/filter_button.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/management/list_filters/filter_button.tsx index 3338a42e1e0ab..463c4bbeba003 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/management/list_filters/filter_button.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/management/list_filters/filter_button.tsx @@ -15,7 +15,7 @@ export const FilterButton = ({ filter }: { filter: FilterItem }) => { const [query, setQuery] = useState(''); - const updateUrlParams = useUrlParams()[1]; + const [, updateUrlParams] = useUrlParams(); const urlParams = useGetUrlParams(); diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/management/monitor_list_container.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/management/monitor_list_container.tsx index 86663b21a5ea9..cd299b4f66708 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/management/monitor_list_container.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/management/monitor_list_container.tsx @@ -5,14 +5,22 @@ * 2.0. */ +import { EuiSpacer } from '@elastic/eui'; import React from 'react'; -import { useMonitorList } from '../hooks/use_monitor_list'; -import { MonitorList } from './monitor_list_table/monitor_list'; +import type { useMonitorList } from '../hooks/use_monitor_list'; import { MonitorAsyncError } from './monitor_errors/monitor_async_error'; import { useInlineErrors } from '../hooks/use_inline_errors'; +import { ListFilters } from './list_filters/list_filters'; +import { MonitorList } from './monitor_list_table/monitor_list'; -export const MonitorListContainer = ({ isEnabled }: { isEnabled?: boolean }) => { +export const MonitorListContainer = ({ + isEnabled, + monitorListProps, +}: { + isEnabled?: boolean; + monitorListProps: ReturnType; +}) => { const { pageState, error, @@ -22,7 +30,7 @@ export const MonitorListContainer = ({ isEnabled }: { isEnabled?: boolean }) => absoluteTotal, loadPage, reloadPage, - } = useMonitorList(); + } = monitorListProps; const { errorSummaries, loading: errorsLoading } = useInlineErrors({ onlyInvalidMonitors: false, @@ -37,6 +45,8 @@ export const MonitorListContainer = ({ isEnabled }: { isEnabled?: boolean }) => return ( <> + + - - {recordRangeLabel}
diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/monitor_page.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/monitor_page.tsx index 3bbb64418f580..6bd9578fa5374 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/monitor_page.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/monitor_page.tsx @@ -12,6 +12,7 @@ import { useTrackPageview } from '@kbn/observability-plugin/public'; import { GETTING_STARTED_ROUTE } from '../../../../../common/constants'; +import { ServiceAllowedWrapper } from '../common/wrappers/service_allowed_wrapper'; import { useLocations } from '../../hooks'; import { Loader } from './management/loader/loader'; @@ -23,18 +24,19 @@ import { useMonitorListBreadcrumbs } from './hooks/use_breadcrumbs'; import { useMonitorList } from './hooks/use_monitor_list'; import * as labels from './management/labels'; -export const MonitorPage: React.FC = () => { +const MonitorPage: React.FC = () => { useTrackPageview({ app: 'synthetics', path: 'monitors' }); useTrackPageview({ app: 'synthetics', path: 'monitors', delay: 15000 }); useMonitorListBreadcrumbs(); + const monitorListProps = useMonitorList(); const { syntheticsMonitors, loading: monitorsLoading, isDataQueried, absoluteTotal, - } = useMonitorList(); + } = monitorListProps; const { error: enablementError, @@ -85,9 +87,15 @@ export const MonitorPage: React.FC = () => { ) : null} - + {showEmptyState && } ); }; + +export const MonitorsPageWithServiceAllowed = React.memo(() => ( + + + +)); diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/routes.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/routes.tsx index fb28f293ae6ae..5f03369c5f382 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/routes.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/routes.tsx @@ -20,8 +20,8 @@ import { getSettingsRouteConfig } from './components/settings/route_config'; import { TestRunDetails } from './components/test_run_details/test_run_details'; import { ErrorDetailsPage } from './components/error_details/error_details_page'; import { StepTitle } from './components/step_details_page/step_title'; -import { MonitorAddPage } from './components/monitor_add_edit/monitor_add_page'; -import { MonitorEditPage } from './components/monitor_add_edit/monitor_edit_page'; +import { MonitorAddPageWithServiceAllowed } from './components/monitor_add_edit/monitor_add_page'; +import { MonitorEditPageWithServiceAllowed } from './components/monitor_add_edit/monitor_edit_page'; import { MonitorDetailsPageTitle } from './components/monitor_details/monitor_details_page_title'; import { MonitorDetailsPage } from './components/monitor_details/monitor_details_page'; import { GettingStartedPage } from './components/getting_started/getting_started_page'; @@ -30,7 +30,6 @@ import { CreateMonitorButton } from './components/monitors_page/create_monitor_b import { OverviewPage } from './components/monitors_page/overview/overview_page'; import { SyntheticsPageTemplateComponent } from './components/common/pages/synthetics_page_template'; import { NotFoundPage } from './components/common/pages/not_found'; -import { ServiceAllowedWrapper } from './components/common/wrappers/service_allowed_wrapper'; import { MonitorTypePortalNode, MonitorDetailsLinkPortalNode, @@ -49,7 +48,7 @@ import { TEST_RUN_DETAILS_ROUTE, } from '../../../common/constants'; import { PLUGIN } from '../../../common/constants/plugin'; -import { MonitorPage } from './components/monitors_page/monitor_page'; +import { MonitorsPageWithServiceAllowed } from './components/monitors_page/monitor_page'; import { apiService } from '../../utils/api_service'; import { RunTestManually } from './components/monitor_details/run_test_manually'; import { MonitorDetailsStatus } from './components/monitor_details/monitor_details_status'; @@ -179,13 +178,7 @@ const getRoutes = ( values: { baseTitle }, }), path: MONITORS_ROUTE, - component: () => ( - <> - - - - - ), + component: MonitorsPageWithServiceAllowed, dataTestSubj: 'syntheticsMonitorManagementPage', pageHeader: { pageTitle: , @@ -218,11 +211,7 @@ const getRoutes = ( values: { baseTitle }, }), path: MONITOR_ADD_ROUTE, - component: () => ( - - - - ), + component: MonitorAddPageWithServiceAllowed, dataTestSubj: 'syntheticsMonitorAddPage', restrictWidth: true, pageHeader: { @@ -256,11 +245,7 @@ const getRoutes = ( values: { baseTitle }, }), path: MONITOR_EDIT_ROUTE, - component: () => ( - - - - ), + component: MonitorEditPageWithServiceAllowed, dataTestSubj: 'syntheticsMonitorEditPage', restrictWidth: true, pageHeader: { diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/state/monitor_list/models.ts b/x-pack/plugins/synthetics/public/apps/synthetics/state/monitor_list/models.ts index f86efc8bc0349..b43de3a1914d0 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/state/monitor_list/models.ts +++ b/x-pack/plugins/synthetics/public/apps/synthetics/state/monitor_list/models.ts @@ -10,7 +10,7 @@ import { FetchMonitorManagementListQueryArgs, } from '../../../../../common/runtime_types'; -export type MonitorListSortField = `${keyof EncryptedSyntheticsSavedMonitor}.keyword`; +export type MonitorListSortField = `${keyof EncryptedSyntheticsSavedMonitor}.keyword` | 'enabled'; export interface MonitorListPageState { query?: string; diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/utils/testing/__mocks__/synthetics_store.mock.ts b/x-pack/plugins/synthetics/public/apps/synthetics/utils/testing/__mocks__/synthetics_store.mock.ts index 2c9953a51a26e..3a62b6df37df3 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/utils/testing/__mocks__/synthetics_store.mock.ts +++ b/x-pack/plugins/synthetics/public/apps/synthetics/utils/testing/__mocks__/synthetics_store.mock.ts @@ -66,10 +66,14 @@ export const mockState: SyntheticsAppState = { }, monitorList: { pageState: { + query: undefined, pageIndex: 0, pageSize: 10, - sortOrder: 'asc', sortField: `${ConfigKey.NAME}.keyword`, + sortOrder: 'asc', + tags: undefined, + monitorType: undefined, + locations: undefined, }, monitorUpsertStatuses: {}, data: { diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/utils/testing/get_supported_url_params.test.ts b/x-pack/plugins/synthetics/public/apps/synthetics/utils/testing/get_supported_url_params.test.ts deleted file mode 100644 index 0b1c82aebe06b..0000000000000 --- a/x-pack/plugins/synthetics/public/apps/synthetics/utils/testing/get_supported_url_params.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import DateMath from '@kbn/datemath'; -import { getSupportedUrlParams } from '../url_params'; -import { CLIENT_DEFAULTS } from '../../../../../common/constants'; - -describe('getSupportedUrlParams', () => { - let dateMathSpy: any; - const MOCK_DATE_VALUE = 20; - - beforeEach(() => { - dateMathSpy = jest.spyOn(DateMath, 'parse'); - dateMathSpy.mockReturnValue(MOCK_DATE_VALUE); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('returns custom values', () => { - const customValues = { - autorefreshInterval: '23', - autorefreshIsPaused: 'false', - dateRangeStart: 'now-15m', - dateRangeEnd: 'now', - monitorListPageIndex: '23', - monitorListPageSize: '50', - monitorListSortDirection: 'desc', - monitorListSortField: 'monitor.status', - search: 'monitor.status: down', - selectedPingStatus: 'up', - }; - - const expected = { - absoluteDateRangeEnd: 20, - absoluteDateRangeStart: 20, - autorefreshInterval: 23, - autorefreshIsPaused: false, - dateRangeEnd: 'now', - dateRangeStart: 'now-15m', - search: 'monitor.status: down', - }; - - const result = getSupportedUrlParams(customValues); - expect(result).toMatchObject(expected); - }); - - it('returns default values', () => { - const { - AUTOREFRESH_INTERVAL, - AUTOREFRESH_IS_PAUSED, - DATE_RANGE_START, - DATE_RANGE_END, - FILTERS, - SEARCH, - STATUS_FILTER, - } = CLIENT_DEFAULTS; - const result = getSupportedUrlParams({}); - expect(result).toEqual({ - absoluteDateRangeStart: MOCK_DATE_VALUE, - absoluteDateRangeEnd: MOCK_DATE_VALUE, - autorefreshInterval: AUTOREFRESH_INTERVAL, - autorefreshIsPaused: AUTOREFRESH_IS_PAUSED, - dateRangeStart: DATE_RANGE_START, - dateRangeEnd: DATE_RANGE_END, - excludedFilters: '', - filters: FILTERS, - focusConnectorField: false, - pagination: undefined, - search: SEARCH, - statusFilter: STATUS_FILTER, - query: '', - locations: [], - monitorType: [], - tags: [], - }); - }); - - it('returns the first item for string arrays', () => { - const result = getSupportedUrlParams({ - dateRangeStart: ['now-18d', 'now-11d', 'now-5m'], - }); - - const expected = { - absoluteDateRangeEnd: 20, - absoluteDateRangeStart: 20, - autorefreshInterval: 60000, - }; - - expect(result).toMatchObject(expected); - }); - - it('provides defaults for undefined values', () => { - const result = getSupportedUrlParams({ - dateRangeStart: undefined, - }); - - const expected = { - absoluteDateRangeStart: 20, - }; - - expect(result).toMatchObject(expected); - }); - - it('provides defaults for empty string array values', () => { - const result = getSupportedUrlParams({ - dateRangeStart: [], - }); - - const expected = { - absoluteDateRangeStart: 20, - }; - - expect(result).toMatchObject(expected); - }); -}); diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/utils/testing/get_supported_url_params.ts b/x-pack/plugins/synthetics/public/apps/synthetics/utils/testing/get_supported_url_params.ts deleted file mode 100644 index c9badbcf7f728..0000000000000 --- a/x-pack/plugins/synthetics/public/apps/synthetics/utils/testing/get_supported_url_params.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { parseIsPaused } from '../url_params/parse_is_paused'; -import { parseUrlInt } from '../url_params/parse_url_int'; -import { CLIENT_DEFAULTS } from '../../../../../common/constants'; -import { parseAbsoluteDate } from '../url_params/parse_absolute_date'; - -export interface SyntheticsUrlParams { - absoluteDateRangeStart: number; - absoluteDateRangeEnd: number; - autorefreshInterval: number; - autorefreshIsPaused: boolean; - dateRangeStart: string; - dateRangeEnd: string; - pagination?: string; - filters: string; - excludedFilters: string; - search: string; - statusFilter: string; - focusConnectorField?: boolean; - query?: string; -} - -const { - ABSOLUTE_DATE_RANGE_START, - ABSOLUTE_DATE_RANGE_END, - AUTOREFRESH_INTERVAL, - AUTOREFRESH_IS_PAUSED, - DATE_RANGE_START, - DATE_RANGE_END, - SEARCH, - FILTERS, - STATUS_FILTER, -} = CLIENT_DEFAULTS; - -/** - * Gets the current URL values for the application. If no item is present - * for the URL, a default value is supplied. - * - * @param params A set of key-value pairs where the value is either - * undefined or a string/string array. If a string array is passed, - * only the first item is chosen. Support for lists in the URL will - * require further development. - */ -export const getSupportedUrlParams = (params: { - [key: string]: string | string[] | undefined | null; -}): SyntheticsUrlParams => { - const filteredParams: { [key: string]: string | undefined } = {}; - Object.keys(params).forEach((key) => { - let value: string | undefined; - if (params[key] === undefined) { - value = undefined; - } else if (Array.isArray(params[key])) { - // @ts-ignore this must be an array, and it's ok if the - // 0th element is undefined - value = params[key][0]; - } else { - // @ts-ignore this will not be an array because the preceding - // block tests for that - value = params[key]; - } - filteredParams[key] = value; - }); - - const { - autorefreshInterval, - autorefreshIsPaused, - dateRangeStart, - dateRangeEnd, - filters, - excludedFilters, - search, - statusFilter, - pagination, - focusConnectorField, - query, - } = filteredParams; - - return { - pagination, - absoluteDateRangeStart: parseAbsoluteDate( - dateRangeStart || DATE_RANGE_START, - ABSOLUTE_DATE_RANGE_START - ), - absoluteDateRangeEnd: parseAbsoluteDate( - dateRangeEnd || DATE_RANGE_END, - ABSOLUTE_DATE_RANGE_END, - { roundUp: true } - ), - autorefreshInterval: parseUrlInt(autorefreshInterval, AUTOREFRESH_INTERVAL), - autorefreshIsPaused: parseIsPaused(autorefreshIsPaused, AUTOREFRESH_IS_PAUSED), - dateRangeStart: dateRangeStart || DATE_RANGE_START, - dateRangeEnd: dateRangeEnd || DATE_RANGE_END, - filters: filters || FILTERS, - excludedFilters: excludedFilters || '', - search: search || SEARCH, - statusFilter: statusFilter || STATUS_FILTER, - focusConnectorField: !!focusConnectorField, - query: query || '', - }; -}; diff --git a/x-pack/plugins/synthetics/server/legacy_uptime/lib/saved_objects/synthetics_monitor.ts b/x-pack/plugins/synthetics/server/legacy_uptime/lib/saved_objects/synthetics_monitor.ts index 4967ebb5af446..ab25be0f3f3d6 100644 --- a/x-pack/plugins/synthetics/server/legacy_uptime/lib/saved_objects/synthetics_monitor.ts +++ b/x-pack/plugins/synthetics/server/legacy_uptime/lib/saved_objects/synthetics_monitor.ts @@ -124,6 +124,9 @@ export const getSyntheticsMonitorSavedObjectType = ( }, }, }, + enabled: { + type: 'boolean', + }, }, }, management: { diff --git a/x-pack/plugins/synthetics/server/routes/common.ts b/x-pack/plugins/synthetics/server/routes/common.ts index 48f528030616d..5c49308469b27 100644 --- a/x-pack/plugins/synthetics/server/routes/common.ts +++ b/x-pack/plugins/synthetics/server/routes/common.ts @@ -59,10 +59,14 @@ export const getMonitors = ( const locationFilter = parseLocationFilter(syntheticsService.locations, locations); - const filters = - getKqlFilter({ field: 'tags', values: tags }) + - getKqlFilter({ field: 'type', values: monitorType }) + - getKqlFilter({ field: 'locations.id', values: locationFilter }); + const filterStr = [ + filter, + getKqlFilter({ field: 'tags', values: tags }), + getKqlFilter({ field: 'type', values: monitorType }), + getKqlFilter({ field: 'locations.id', values: locationFilter }), + ] + .filter((f) => !!f) + .join(' AND '); return savedObjectsClient.find({ type: syntheticsMonitorType, @@ -72,7 +76,7 @@ export const getMonitors = ( sortOrder, searchFields: ['name', 'tags.text', 'locations.id.text', 'urls', 'project_id.text'], search: query ? `${query}*` : undefined, - filter: filters + filter, + filter: filterStr, fields, searchAfter, }); @@ -123,3 +127,21 @@ const parseLocationFilter = (serviceLocations: ServiceLocations, locations?: str export const findLocationItem = (query: string, locations: ServiceLocations) => { return locations.find(({ id, label }) => query === id || label === query); }; + +/** + * Returns whether the query is likely to return a subset of monitor objects. + * Useful where `absoluteTotal` needs to be determined with a separate call + * @param monitorQuery { MonitorsQuery } + */ +export const isMonitorsQueryFiltered = (monitorQuery: MonitorsQuery) => { + const { query, tags, monitorType, locations, status, filter } = monitorQuery; + + return ( + !!query || + !!filter || + !!locations?.length || + !!monitorType?.length || + !!tags?.length || + !!status?.length + ); +}; diff --git a/x-pack/plugins/synthetics/server/routes/monitor_cruds/get_monitor.ts b/x-pack/plugins/synthetics/server/routes/monitor_cruds/get_monitor.ts index c3c2b1829ceea..54b3b0bda0217 100644 --- a/x-pack/plugins/synthetics/server/routes/monitor_cruds/get_monitor.ts +++ b/x-pack/plugins/synthetics/server/routes/monitor_cruds/get_monitor.ts @@ -12,7 +12,7 @@ import { SyntheticsRestApiRouteFactory } from '../../legacy_uptime/routes/types' import { API_URLS, SYNTHETICS_API_URLS } from '../../../common/constants'; import { syntheticsMonitorType } from '../../legacy_uptime/lib/saved_objects/synthetics_monitor'; import { getMonitorNotFoundResponse } from '../synthetics_service/service_errors'; -import { getMonitors, QuerySchema, SEARCH_FIELDS } from '../common'; +import { getMonitors, isMonitorsQueryFiltered, QuerySchema, SEARCH_FIELDS } from '../common'; export const getSyntheticsMonitorRoute: SyntheticsRestApiRouteFactory = (libs: UMServerLibs) => ({ method: 'GET', @@ -53,40 +53,27 @@ export const getAllSyntheticsMonitorRoute: SyntheticsRestApiRouteFactory = () => query: QuerySchema, }, handler: async ({ request, savedObjectsClient, syntheticsMonitorClient }): Promise => { - const { filters, query } = request.query; - const monitorsPromise = getMonitors( + const queryResult = await getMonitors( request.query, syntheticsMonitorClient.syntheticsService, savedObjectsClient ); - if (filters || query) { - const totalMonitorsPromise = savedObjectsClient.find({ - type: syntheticsMonitorType, - perPage: 0, - page: 1, - }); - - const allResolved = await Promise.all([monitorsPromise, totalMonitorsPromise]); - const { saved_objects: monitors, per_page: perPageT, ...rest } = allResolved[0]; - const { total } = allResolved[1]; - - return { - ...rest, - monitors, - perPage: perPageT, - absoluteTotal: total, - syncErrors: syntheticsMonitorClient.syntheticsService.syncErrors, - }; - } + const countResult = isMonitorsQueryFiltered(request.query) + ? await savedObjectsClient.find({ + type: syntheticsMonitorType, + perPage: 0, + page: 1, + }) + : queryResult; - const { saved_objects: monitors, per_page: perPageT, ...rest } = await monitorsPromise; + const { saved_objects: monitors, per_page: perPageT, ...rest } = queryResult; return { ...rest, monitors, perPage: perPageT, - absoluteTotal: rest.total, + absoluteTotal: countResult.total, syncErrors: syntheticsMonitorClient.syntheticsService.syncErrors, }; }, From f1117c89593f1d8ea2fb3c7e1e44a88fb1e829aa Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Mon, 14 Nov 2022 16:55:20 +0000 Subject: [PATCH 07/16] [Security Solution][Alerts] adds support for multi fields in new terms rule (#143943) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - addresses https://github.com/elastic/kibana/issues/142862 - allows up to 3 fields in `New terms` - displays new terms fields in alerts details - For multiple new terms fields(['source.host', 'source.ip']), in terms aggregation uses a runtime field. Which is created by joining values from new terms fields into one single keyword. Field values encoded in base64 and joined with a configured delimiter symbol, which is not part of base64 symbols(a–Z, 0–9, +, /, =) to avoid a situation when delimiter can be part of field value. Include parameter consists of encoded in base64 results from Phase 1. For single field, implementation remains the same to avoid performance penalties - Performance measurements: - [msearch POC for one field](https://github.com/elastic/kibana/pull/131010#issuecomment-1135280478) - [historical POC of multi fields](https://github.com/elastic/kibana/issues/142862#issuecomment-1290471400) - [current implementation](https://github.com/elastic/kibana/issues/142862#issuecomment-1298910940) ## UI ### Alert details #### Before Screenshot 2022-10-26 at 18 29 00 #### After Screenshot 2022-11-09 at 19 05 10 Delete any items that are not applicable to this PR. - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --- .../security_solution/common/constants.ts | 2 + .../new_terms_attributes.ts | 9 +- .../common/field_maps/field_names.ts | 3 +- .../event_details/alert_summary_view.test.tsx | 9 +- .../event_details/get_alert_summary_rows.tsx | 11 +- .../components/alerts_table/translations.ts | 7 + .../rules/step_define_rule/index.tsx | 8 +- .../rules/step_define_rule/schema.tsx | 7 +- .../rules/step_define_rule/utils.ts | 21 ++ .../rule_types/new_terms/README.md | 5 +- .../build_new_terms_aggregation.test.ts.snap | 34 ++ .../build_new_terms_aggregation.test.ts | 13 +- .../new_terms/build_new_terms_aggregation.ts | 22 +- .../new_terms/create_new_terms_alert_type.ts | 48 ++- .../rule_types/new_terms/utils.test.ts | 146 ++++++- .../rule_types/new_terms/utils.ts | 107 ++++++ .../group1/create_new_terms.ts | 17 + .../rule_execution_logic/new_terms.ts | 356 ++++++++++++++++++ .../utils/index.ts | 1 + .../utils/perform_search_query.ts | 44 +++ .../security_solution/new_terms/data.json | 220 +++++++++++ .../security_solution/new_terms/mappings.json | 70 ++++ 22 files changed, 1116 insertions(+), 44 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/utils.ts create mode 100644 x-pack/test/detection_engine_api_integration/utils/perform_search_query.ts create mode 100644 x-pack/test/functional/es_archives/security_solution/new_terms/data.json create mode 100644 x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts index e88580e1a338f..1566723f9c6c5 100644 --- a/x-pack/plugins/security_solution/common/constants.ts +++ b/x-pack/plugins/security_solution/common/constants.ts @@ -469,4 +469,6 @@ export const RISKY_HOSTS_DOC_LINK = export const RISKY_USERS_DOC_LINK = 'https://www.elastic.co/guide/en/security/current/user-risk-score.html'; +export const MAX_NUMBER_OF_NEW_TERMS_FIELDS = 3; + export const BULK_ADD_TO_TIMELINE_LIMIT = 2000; diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts index 15bf73ba150e5..6d9f39011b675 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts @@ -7,14 +7,19 @@ import * as t from 'io-ts'; import { LimitedSizeArray, NonEmptyString } from '@kbn/securitysolution-io-ts-types'; +import { MAX_NUMBER_OF_NEW_TERMS_FIELDS } from '../../../../constants'; // Attributes specific to New Terms rules /** - * New terms rule type currently only supports a single term, but should support more in the future + * New terms rule type supports a limited number of fields. Max number of fields is 3 and defined in common constants as MAX_NUMBER_OF_NEW_TERMS_FIELDS */ export type NewTermsFields = t.TypeOf; -export const NewTermsFields = LimitedSizeArray({ codec: t.string, minSize: 1, maxSize: 1 }); +export const NewTermsFields = LimitedSizeArray({ + codec: t.string, + minSize: 1, + maxSize: MAX_NUMBER_OF_NEW_TERMS_FIELDS, +}); export type HistoryWindowStart = t.TypeOf; export const HistoryWindowStart = NonEmptyString; diff --git a/x-pack/plugins/security_solution/common/field_maps/field_names.ts b/x-pack/plugins/security_solution/common/field_maps/field_names.ts index 6a7b4efff8a7c..53ebfc5c188d1 100644 --- a/x-pack/plugins/security_solution/common/field_maps/field_names.ts +++ b/x-pack/plugins/security_solution/common/field_maps/field_names.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { ALERT_NAMESPACE, ALERT_RULE_NAMESPACE } from '@kbn/rule-data-utils'; +import { ALERT_NAMESPACE, ALERT_RULE_NAMESPACE, ALERT_RULE_PARAMETERS } from '@kbn/rule-data-utils'; export const ALERT_ANCESTORS = `${ALERT_NAMESPACE}.ancestors` as const; export const ALERT_BUILDING_BLOCK_TYPE = `${ALERT_NAMESPACE}.building_block_type` as const; @@ -16,6 +16,7 @@ export const ALERT_ORIGINAL_TIME = `${ALERT_NAMESPACE}.original_time` as const; export const ALERT_THRESHOLD_RESULT = `${ALERT_NAMESPACE}.threshold_result` as const; export const ALERT_THRESHOLD_RESULT_COUNT = `${ALERT_THRESHOLD_RESULT}.count` as const; export const ALERT_NEW_TERMS = `${ALERT_NAMESPACE}.new_terms` as const; +export const ALERT_NEW_TERMS_FIELDS = `${ALERT_RULE_PARAMETERS}.new_terms_fields` as const; export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event` as const; export const ALERT_ORIGINAL_EVENT_ACTION = `${ALERT_ORIGINAL_EVENT}.action` as const; diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.test.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.test.tsx index 662e1983680bd..beda9c0aa4d13 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.test.tsx @@ -703,18 +703,25 @@ describe('AlertSummaryView', () => { values: ['127.0.0.1'], originalValue: ['127.0.0.1'], }, + { + category: 'kibana', + field: 'kibana.alert.rule.parameters.new_terms_fields', + values: ['host.ip'], + originalValue: ['host.ip'], + }, ] as TimelineEventsDetailsItem[]; const renderProps = { ...props, data: enhancedData, }; + const { getByText } = render( ); - ['New Terms'].forEach((fieldId) => { + ['New Terms', '127.0.0.1', 'New Terms fields', 'host.ip'].forEach((fieldId) => { expect(getByText(fieldId)); }); }); diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx index 5caca1fcd7253..96bc3e030f7a2 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx @@ -16,8 +16,13 @@ import { ALERTS_HEADERS_THRESHOLD_TERMS, ALERTS_HEADERS_RULE_DESCRIPTION, ALERTS_HEADERS_NEW_TERMS, + ALERTS_HEADERS_NEW_TERMS_FIELDS, } from '../../../detections/components/alerts_table/translations'; -import { ALERT_NEW_TERMS, ALERT_THRESHOLD_RESULT } from '../../../../common/field_maps/field_names'; +import { + ALERT_NEW_TERMS_FIELDS, + ALERT_NEW_TERMS, + ALERT_THRESHOLD_RESULT, +} from '../../../../common/field_maps/field_names'; import { AGENT_STATUS_FIELD_NAME } from '../../../timelines/components/timeline/body/renderers/constants'; import type { AlertSummaryRow } from './helpers'; import { getEnrichedFieldInfo } from './helpers'; @@ -172,6 +177,10 @@ function getFieldsByRuleType(ruleType?: string): EventSummaryField[] { ]; case 'new_terms': return [ + { + id: ALERT_NEW_TERMS_FIELDS, + label: ALERTS_HEADERS_NEW_TERMS_FIELDS, + }, { id: ALERT_NEW_TERMS, label: ALERTS_HEADERS_NEW_TERMS, diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts index 3f2f3ca8c5a15..21e80a6770c1f 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts @@ -123,6 +123,13 @@ export const ALERTS_HEADERS_NEW_TERMS = i18n.translate( } ); +export const ALERTS_HEADERS_NEW_TERMS_FIELDS = i18n.translate( + 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.newTermsFields', + { + defaultMessage: 'New Terms fields', + } +); + export const ACTION_INVESTIGATE_IN_TIMELINE = i18n.translate( 'xpack.securitySolution.detectionEngine.alerts.actions.investigateInTimelineTitle', { diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index dc926fd4f74e2..9cbe2362acb6d 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -59,6 +59,7 @@ import { useFormData, } from '../../../../shared_imports'; import { schema } from './schema'; +import { getTermsAggregationFields } from './utils'; import * as i18n from './translations'; import { isEqlRule, @@ -297,6 +298,11 @@ const StepDefineRuleComponent: FC = ({ setAggregatableFields(aggregatableFields(fields as BrowserField[])); }, [indexPattern]); + const termsAggregationFields: BrowserField[] = useMemo( + () => getTermsAggregationFields(aggFields), + [aggFields] + ); + const [ threatIndexPatternsLoading, { browserFields: threatBrowserFields, indexPatterns: threatIndexPatterns }, @@ -836,7 +842,7 @@ const StepDefineRuleComponent: FC = ({ path="newTermsFields" component={NewTermsFields} componentProps={{ - browserFields: aggFields, + browserFields: termsAggregationFields, }} /> = { i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.newTermsFieldsMin', { - defaultMessage: 'Number of fields must be 1.', + defaultMessage: 'A minimum of one field is required.', } ) )(...args); @@ -601,11 +602,11 @@ export const schema: FormSchema = { return; } return fieldValidators.maxLengthField({ - length: 1, + length: MAX_NUMBER_OF_NEW_TERMS_FIELDS, message: i18n.translate( 'xpack.securitySolution.detectionEngine.validations.stepDefineRule.newTermsFieldsMax', { - defaultMessage: 'Number of fields must be 1.', + defaultMessage: 'Number of fields must be 3 or less.', } ), })(...args); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/utils.ts b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/utils.ts new file mode 100644 index 0000000000000..d8b63f5801159 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/utils.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { BrowserField } from '../../../../common/containers/source'; + +/** + * Filters out fields, that are not supported in terms aggregation. + * Terms aggregation supports limited number of types: + * Keyword, Numeric, ip, boolean, or binary. + * https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html + */ +export const getTermsAggregationFields = (fields: BrowserField[]): BrowserField[] => { + // binary types is excluded, as binary field has property aggregatable === false + const allowedTypesSet = new Set(['string', 'number', 'ip', 'boolean']); + + return fields.filter((field) => field.aggregatable === true && allowedTypesSet.has(field.type)); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md index 50ec5e7682a28..694fdd53fe2f4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md @@ -2,7 +2,7 @@ The rule accepts 2 new parameters that are unique to the new_terms rule type, in addition to common Security rule parameters such as query, index, and filters, to, from, etc. The new parameters are: -- `new_terms_fields`: an array of field names, currently limited to an array of size 1. In the future we will likely allow multiple field names to be specified here. +- `new_terms_fields`: an array of field names, currently limited to an array of size 3. Example: ['host.ip'] - `history_window_start`: defines the additional time range to search over when determining if a term is "new". If a term is found between the times `history_window_start` and from then it will not be classified as a new term. Example: now-30d @@ -12,6 +12,7 @@ Each page is evaluated in 3 phases. Phase 1: Collect "recent" terms - terms that have appeared in the last rule interval, without regard to whether or not they have appeared in historical data. This is done using a composite aggregation to ensure we can iterate over every term. Phase 2: Check if the page of terms contains any new terms. This uses a regular terms agg with the include parameter - every term is added to the array of include values, so the terms agg is limited to only aggregating on the terms of interest from phase 1. This avoids issues with the terms agg providing approximate results due to getting different terms from different shards. +For multiple new terms fields(['source.host', 'source.ip']), in terms aggregation uses a runtime field. Which is created by joining values from new terms fields into one single keyword value. Fields values encoded in base64 and joined with configured a delimiter symbol, which is not part of base64 symbols(a–Z, 0–9, +, /, =) to avoid a situation when delimiter can be part of field value. Include parameter consists of encoded in base64 results from Phase 1. Phase 3: Any new terms from phase 2 are processed and the first document to contain that term is retrieved. The document becomes the basis of the generated alert. This is done with an aggregation query that is very similar to the agg used in phase 2, except it also includes a top_hits agg. top_hits is moved to a separate, later phase for efficiency - top_hits is slow and most terms will not be new in phase 2. This means we only execute the top_hits agg on the terms that are actually new which is faster. @@ -26,4 +27,4 @@ The new terms rule type reuses the singleSearchAfter function which implements t ## Limitations and future enhancements - Value list exceptions are not supported at the moment. Commit ead04ce removes an experimental method I tried for evaluating value list exceptions. -- In the future we may want to support searching for new sets of terms, e.g. a pair of `host.ip` and `host.id` that has never been seen together before. +- Runtime field supports only 100 emitted values. So for large arrays or combination of values greater than 100, results may not be exhaustive. This applies only to new terms with multiple fields diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/__snapshots__/build_new_terms_aggregation.test.ts.snap b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/__snapshots__/build_new_terms_aggregation.test.ts.snap index 72df8a34cfa18..022f4eba9365c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/__snapshots__/build_new_terms_aggregation.test.ts.snap +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/__snapshots__/build_new_terms_aggregation.test.ts.snap @@ -135,3 +135,37 @@ Object { }, } `; + +exports[`aggregations buildRecentTermsAgg builds a correct composite aggregation with multiple fields 1`] = ` +Object { + "new_terms": Object { + "composite": Object { + "after": undefined, + "size": 10000, + "sources": Array [ + Object { + "host.name": Object { + "terms": Object { + "field": "host.name", + }, + }, + }, + Object { + "host.port": Object { + "terms": Object { + "field": "host.port", + }, + }, + }, + Object { + "host.url": Object { + "terms": Object { + "field": "host.url", + }, + }, + }, + ], + }, + }, +} +`; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.test.ts index 9b853a730ba4c..ec81c06b92837 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.test.ts @@ -16,7 +16,7 @@ describe('aggregations', () => { describe('buildRecentTermsAgg', () => { test('builds a correct composite agg without `after`', () => { const aggregation = buildRecentTermsAgg({ - field: 'host.name', + fields: ['host.name'], after: undefined, }); @@ -25,12 +25,21 @@ describe('aggregations', () => { test('builds a correct composite aggregation with `after`', () => { const aggregation = buildRecentTermsAgg({ - field: 'host.name', + fields: ['host.name'], after: { 'host.name': 'myHost' }, }); expect(aggregation).toMatchSnapshot(); }); + + test('builds a correct composite aggregation with multiple fields', () => { + const aggregation = buildRecentTermsAgg({ + fields: ['host.name', 'host.port', 'host.url'], + after: undefined, + }); + + expect(aggregation).toMatchSnapshot(); + }); }); describe('buildNewTermsAggregation', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts index 41f2f7c6dc0ab..e9bf89554941f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts @@ -31,24 +31,24 @@ const PAGE_SIZE = 10000; * without regard to whether or not they're actually new. */ export const buildRecentTermsAgg = ({ - field, + fields, after, }: { - field: string; + fields: string[]; after: Record | undefined; }) => { + const sources = fields.map((field) => ({ + [field]: { + terms: { + field, + }, + }, + })); + return { new_terms: { composite: { - sources: [ - { - [field]: { - terms: { - field, - }, - }, - }, - ], + sources, size: PAGE_SIZE, after, }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 8fce33897bd0f..bc2746ddf7888 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -5,7 +5,6 @@ * 2.0. */ -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { validateNonExact } from '@kbn/securitysolution-io-ts-utils'; import { NEW_TERMS_RULE_TYPE_ID } from '@kbn/securitysolution-rules'; import { SERVER_APP_ID } from '../../../../../common/constants'; @@ -16,6 +15,7 @@ import type { CreateRuleOptions, SecurityAlertType } from '../types'; import { singleSearchAfter } from '../../signals/single_search_after'; import { getFilter } from '../../signals/get_filter'; import { wrapNewTermsAlerts } from '../factories/utils/wrap_new_terms_alerts'; +import type { EventsAndTerms } from '../factories/utils/wrap_new_terms_alerts'; import type { DocFetchAggResult, RecentTermsAggResult, @@ -26,9 +26,15 @@ import { buildRecentTermsAgg, buildNewTermsAgg, } from './build_new_terms_aggregation'; -import type { SignalSource } from '../../signals/types'; import { validateIndexPatterns } from '../utils'; -import { parseDateString, validateHistoryWindowStart } from './utils'; +import { + parseDateString, + validateHistoryWindowStart, + transformBucketsToValues, + getNewTermsRuntimeMappings, + getAggregationField, + decodeMatchedValues, +} from './utils'; import { addToSearchAfterReturn, createSearchAfterReturnType, @@ -154,7 +160,7 @@ export const createNewTermsAlertType = ( // ones are new. const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ aggregations: buildRecentTermsAgg({ - field: params.newTermsFields[0], + fields: params.newTermsFields, after: afterKey, }), searchAfterSortIds: undefined, @@ -187,10 +193,7 @@ export const createNewTermsAlertType = ( break; } const bucketsForField = searchResultWithAggs.aggregations.new_terms.buckets; - const includeValues = bucketsForField - .map((bucket) => Object.values(bucket.key)[0]) - .filter((value): value is string | number => value != null); - + const includeValues = transformBucketsToValues(params.newTermsFields, bucketsForField); // PHASE 2: Take the page of results from Phase 1 and determine if each term exists in the history window. // The aggregation filters out buckets for terms that exist prior to `tuple.from`, so the buckets in the // response correspond to each new term. @@ -202,10 +205,13 @@ export const createNewTermsAlertType = ( aggregations: buildNewTermsAgg({ newValueWindowStart: tuple.from, timestampField: aggregatableTimestampField, - field: params.newTermsFields[0], + field: getAggregationField(params.newTermsFields), include: includeValues, }), - runtimeMappings, + runtimeMappings: { + ...runtimeMappings, + ...getNewTermsRuntimeMappings(params.newTermsFields), + }, searchAfterSortIds: undefined, index: inputIndex, // For Phase 2, we expand the time range to aggregate over the history window @@ -245,10 +251,13 @@ export const createNewTermsAlertType = ( } = await singleSearchAfter({ aggregations: buildDocFetchAgg({ timestampField: aggregatableTimestampField, - field: params.newTermsFields[0], + field: getAggregationField(params.newTermsFields), include: actualNewTerms, }), - runtimeMappings, + runtimeMappings: { + ...runtimeMappings, + ...getNewTermsRuntimeMappings(params.newTermsFields), + }, searchAfterSortIds: undefined, index: inputIndex, // For phase 3, we go back to aggregating only over the rule interval - excluding the history window @@ -270,13 +279,14 @@ export const createNewTermsAlertType = ( throw new Error('Aggregations were missing on document fetch search result'); } - const eventsAndTerms: Array<{ - event: estypes.SearchHit; - newTerms: Array; - }> = docFetchResultWithAggs.aggregations.new_terms.buckets.map((bucket) => ({ - event: bucket.docs.hits.hits[0], - newTerms: [bucket.key], - })); + const eventsAndTerms: EventsAndTerms[] = + docFetchResultWithAggs.aggregations.new_terms.buckets.map((bucket) => { + const newTerms = decodeMatchedValues(params.newTermsFields, bucket.key); + return { + event: bucket.docs.hits.hits[0], + newTerms, + }; + }); const alertTimestampOverride = isPreview ? startedAt : undefined; const wrappedAlerts = wrapNewTermsAlerts({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts index e1207eccf82b0..2b04b617ba9ba 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts @@ -5,7 +5,15 @@ * 2.0. */ -import { parseDateString, validateHistoryWindowStart } from './utils'; +import { + parseDateString, + validateHistoryWindowStart, + transformBucketsToValues, + getAggregationField, + decodeMatchedValues, + getNewTermsRuntimeMappings, + AGG_FIELD_NAME, +} from './utils'; describe('new terms utils', () => { describe('parseDateString', () => { @@ -64,4 +72,140 @@ describe('new terms utils', () => { ); }); }); + + describe('transformBucketsToValues', () => { + it('should return correct value for a single new terms field', () => { + expect( + transformBucketsToValues( + ['source.host'], + [ + { + key: { + 'source.host': 'host-0', + }, + doc_count: 1, + }, + { + key: { + 'source.host': 'host-1', + }, + doc_count: 3, + }, + ] + ) + ).toEqual(['host-0', 'host-1']); + }); + + it('should filter null values for a single new terms field', () => { + expect( + transformBucketsToValues( + ['source.host'], + [ + { + key: { + 'source.host': 'host-0', + }, + doc_count: 1, + }, + { + key: { + 'source.host': null, + }, + doc_count: 3, + }, + ] + ) + ).toEqual(['host-0']); + }); + + it('should return correct value for multiple new terms fields', () => { + expect( + transformBucketsToValues( + ['source.host', 'source.ip'], + [ + { + key: { + 'source.host': 'host-0', + 'source.ip': '127.0.0.1', + }, + doc_count: 1, + }, + { + key: { + 'source.host': 'host-1', + 'source.ip': '127.0.0.1', + }, + doc_count: 1, + }, + ] + ) + ).toEqual(['aG9zdC0w_MTI3LjAuMC4x', 'aG9zdC0x_MTI3LjAuMC4x']); + }); + + it('should filter null values for multiple new terms fields', () => { + expect( + transformBucketsToValues( + ['source.host', 'source.ip'], + [ + { + key: { + 'source.host': 'host-0', + 'source.ip': '127.0.0.1', + }, + doc_count: 1, + }, + { + key: { + 'source.host': 'host-1', + 'source.ip': null, + }, + doc_count: 1, + }, + ] + ) + ).toEqual(['aG9zdC0w_MTI3LjAuMC4x']); + }); + }); + + describe('getAggregationField', () => { + it('should return correct value for a single new terms field', () => { + expect(getAggregationField(['source.ip'])).toBe('source.ip'); + }); + it('should return correct value for multiple new terms fields', () => { + expect(getAggregationField(['source.host', 'source.ip'])).toBe(AGG_FIELD_NAME); + }); + }); + + describe('decodeMatchedValues', () => { + it('should return correct value for a single new terms field', () => { + expect(decodeMatchedValues(['source.ip'], '127.0.0.1')).toEqual(['127.0.0.1']); + }); + it('should return correct value for multiple new terms fields', () => { + expect(decodeMatchedValues(['source.host', 'source.ip'], 'aG9zdC0w_MTI3LjAuMC4x')).toEqual([ + 'host-0', + '127.0.0.1', + ]); + }); + }); + + describe('getNewTermsRuntimeMappings', () => { + it('should not return runtime field if new terms fields is empty', () => { + expect(getNewTermsRuntimeMappings([])).toBeUndefined(); + }); + it('should not return runtime field if new terms fields has only one field', () => { + expect(getNewTermsRuntimeMappings(['host.name'])).toBeUndefined(); + }); + + it('should return runtime field if new terms fields has more than one field', () => { + const runtimeMappings = getNewTermsRuntimeMappings(['host.name', 'host.ip']); + + expect(runtimeMappings?.[AGG_FIELD_NAME]).toMatchObject({ + type: 'keyword', + script: { + params: { fields: ['host.name', 'host.ip'] }, + source: expect.any(String), + }, + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index 4a87ec9edbbda..cebd63f17e663 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -7,6 +7,10 @@ import dateMath from '@elastic/datemath'; import moment from 'moment'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +export const AGG_FIELD_NAME = 'new_terms_values'; +const DELIMITER = '_'; export const parseDateString = ({ date, @@ -46,3 +50,106 @@ export const validateHistoryWindowStart = ({ ); } }; + +/** + * Takes a list of buckets and creates value from them to be used in 'include' clause of terms aggregation. + * For a single new terms field, value equals to bucket name + * For multiple new terms fields and buckets, value equals to concatenated base64 encoded bucket names + * @returns for buckets('host-0', 'test'), resulted value equals to: 'aG9zdC0w_dGVzdA==' + */ +export const transformBucketsToValues = ( + newTermsFields: string[], + buckets: estypes.AggregationsCompositeBucket[] +): Array => { + // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together + if (newTermsFields.length === 1) { + return buckets + .map((bucket) => Object.values(bucket.key)[0]) + .filter((value): value is string | number => value != null); + } + + return buckets + .map((bucket) => Object.values(bucket.key)) + .filter((values) => !values.some((value) => value == null)) + .map((values) => + values + .map((value) => + Buffer.from(typeof value !== 'string' ? value.toString() : value).toString('base64') + ) + .join(DELIMITER) + ); +}; + +export const getNewTermsRuntimeMappings = ( + newTermsFields: string[] +): undefined | { [AGG_FIELD_NAME]: estypes.MappingRuntimeField } => { + // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together + if (newTermsFields.length <= 1) { + return undefined; + } + + return { + [AGG_FIELD_NAME]: { + type: 'keyword', + script: { + params: { fields: newTermsFields }, + source: ` + def stack = new Stack(); + // ES has limit in 100 values for runtime field, after this query will fail + int emitLimit = 100; + stack.add([0, '']); + + while (stack.length > 0) { + if (emitLimit == 0) { + break; + } + def tuple = stack.pop(); + def index = tuple[0]; + def line = tuple[1]; + if (index === params['fields'].length) { + emit(line); + emitLimit = emitLimit - 1; + } else { + for (field in doc[params['fields'][index]]) { + def delimiter = index === 0 ? '' : '${DELIMITER}'; + def nextLine = line + delimiter + String.valueOf(field).encodeBase64(); + + stack.add([index + 1, nextLine]) + } + } + } + `, + }, + }, + }; +}; + +/** + * For a single new terms field, aggregation field equals to new terms field + * For multiple new terms fields, aggregation field equals to defined AGG_FIELD_NAME, which is runtime field + */ +export const getAggregationField = (newTermsFields: string[]): string => { + // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together + if (newTermsFields.length === 1) { + return newTermsFields[0]; + } + + return AGG_FIELD_NAME; +}; + +const decodeBucketKey = (bucketKey: string): string[] => { + return bucketKey + .split(DELIMITER) + .map((encodedValue) => Buffer.from(encodedValue, 'base64').toString()); +}; + +/** + * decodes matched values(bucket keys) from terms aggregation and returns fields as array + * @returns 'aG9zdC0w_dGVzdA==' bucket key will result in ['host-0', 'test'] + */ +export const decodeMatchedValues = (newTermsFields: string[], bucketKey: string | number) => { + // if newTermsFields has length greater than 1, bucketKey can't be number, so casting is safe here + const values = newTermsFields.length === 1 ? [bucketKey] : decodeBucketKey(bucketKey as string); + + return values; +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts index 095ce3766918d..cd961fce7aed0 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts @@ -40,5 +40,22 @@ export default ({ getService }: FtrProviderContext) => { "params invalid: History window size is smaller than rule interval + additional lookback, 'historyWindowStart' must be earlier than 'from'" ); }); + + it('should not be able to create a new terms rule with fields number greater than 3', async () => { + const rule = { + ...getCreateNewTermsRulesSchemaMock('rule-1'), + history_window_start: 'now-5m', + new_terms_fields: ['field1', 'field2', 'field3', 'field4'], + }; + const response = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(rule); + + expect(response.status).to.equal(400); + expect(response.body.message).to.be( + '[request body]: Array size (4) is out of bounds: min: 1, max: 3' + ); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index 4bfbe92118599..3b1304f12e6c3 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -11,6 +11,10 @@ import { NewTermsRuleCreateProps } from '@kbn/security-solution-plugin/common/de import { orderBy } from 'lodash'; import { getCreateNewTermsRulesSchemaMock } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema/mocks'; import { DetectionAlert } from '@kbn/security-solution-plugin/common/detection_engine/schemas/alerts'; +import { + getNewTermsRuntimeMappings, + AGG_FIELD_NAME, +} from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/new_terms/utils'; import { createRule, deleteAllAlerts, @@ -18,6 +22,7 @@ import { getOpenSignals, getPreviewAlerts, previewRule, + performSearchQuery, } from '../../utils'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { previewRuleWithExceptionEntries } from '../../utils/preview_rule_with_exception_entries'; @@ -50,10 +55,12 @@ export default ({ getService }: FtrProviderContext) => { describe('New terms type rules', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/new_terms'); }); after(async () => { await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); + await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/new_terms'); await deleteSignalsIndex(supertest, log); await deleteAllAlerts(supertest, log); }); @@ -228,6 +235,130 @@ export default ({ getService }: FtrProviderContext) => { ]); }); + it('should generate 3 alerts when 1 document has 3 new values for multiple fields', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + new_terms_fields: ['host.name', 'host.ip'], + from: '2019-02-19T20:42:00.000Z', + history_window_start: '2019-01-19T20:42:00.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(3); + + const newTerms = orderBy( + previewAlerts.map((item) => item._source?.['kibana.alert.new_terms']), + ['0', '1'] + ); + + expect(newTerms).eql([ + ['zeek-newyork-sha-aa8df15', '10.10.0.6'], + ['zeek-newyork-sha-aa8df15', '157.230.208.30'], + ['zeek-newyork-sha-aa8df15', 'fe80::24ce:f7ff:fede:a571'], + ]); + }); + + it('should generate 1 alert for unique combination of existing terms', async () => { + // ensure there are no alerts for single new terms fields, it means values are not new + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['host.name', 'host.ip'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + // shouldn't be terms for 'host.ip' + const hostIpPreview = await previewRule({ + supertest, + rule: { ...rule, new_terms_fields: ['host.ip'] }, + }); + const hostIpPreviewAlerts = await getPreviewAlerts({ + es, + previewId: hostIpPreview.previewId, + }); + expect(hostIpPreviewAlerts.length).eql(0); + + // shouldn't be terms for 'host.name' + const hostNamePreview = await previewRule({ + supertest, + rule: { ...rule, new_terms_fields: ['host.name'] }, + }); + const hostNamePreviewAlerts = await getPreviewAlerts({ + es, + previewId: hostNamePreview.previewId, + }); + expect(hostNamePreviewAlerts.length).eql(0); + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(1); + + expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['host-0', '127.0.0.2']); + }); + + it('should generate 5 alerts, 1 for each new unique combination in 2 fields', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['source.ip', 'tags'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(5); + + const newTerms = orderBy( + previewAlerts.map((item) => item._source?.['kibana.alert.new_terms']), + ['0', '1'] + ); + + expect(newTerms).eql([ + ['192.168.1.1', 'tag-new-1'], + ['192.168.1.1', 'tag-new-3'], + ['192.168.1.2', 'tag-2'], + ['192.168.1.2', 'tag-new-1'], + ['192.168.1.2', 'tag-new-3'], + ]); + }); + + it('should generate 1 alert for unique combination of terms, one of which is a number', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['user.name', 'user.id'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(1); + expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['user-0', '1']); + }); + + it('should generate 1 alert for unique combination of terms, one of which is a boolean', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['user.name', 'user.enabled'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(1); + expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['user-0', 'false']); + }); + it('should generate alerts for every term when history window is small', async () => { const rule: NewTermsRuleCreateProps = { ...getCreateNewTermsRulesSchemaMock('rule-1', true), @@ -251,6 +382,92 @@ export default ({ getService }: FtrProviderContext) => { expect(hostNames[4]).eql(['zeek-sensor-san-francisco']); }); + describe('null values', () => { + it('should not generate alerts with null values for single field', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['possibly_null_field'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(0); + }); + + it('should not generate alerts with null values for multiple fields', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['possibly_null_field', 'host.name'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(0); + }); + }); + + describe('large arrays values', () => { + it('should generate alerts for unique values in large array for single field from a single document', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['large_array_20'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId, size: 100 }); + + expect(previewAlerts.length).eql(20); + }); + + // There is a limit in ES for a number of emitted values in runtime field (100) + // This test ensures rule run doesn't fail if processed fields in runtime script generates 100 values, hard limit for ES + // For this test case: large_array_10 & large_array_5 have 100 unique combination in total + it('should generate alerts for array fields that have 100 unique combination of values in runtime field', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['large_array_10', 'large_array_5'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId, size: 200 }); + + expect(previewAlerts.length).eql(100); + }); + + // There is a limit in ES for a number of emitted values in runtime field (100) + // This test ensures rule run doesn't fail if processed fields in runtime script generates 200 values + // In case of this test case: large_array_10 & large_array_20 have 200 unique combination in total + // Rule run should not fail and should generate alerts + it('should generate alert for array fields that have more than 200 unique combination of values in runtime field', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['large_array_10', 'large_array_20'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId, size: 200 }); + + expect(previewAlerts.length).eql(100); + }); + }); + describe('timestamp override and fallback', () => { before(async () => { await esArchiver.load( @@ -381,5 +598,144 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts[0]?._source?.host?.risk?.calculated_score_norm).to.eql(23); }); }); + + describe('runtime field', () => { + it('should return runtime field created from 2 single values', async () => { + // encoded base64 values of "host-0" and "127.0.0.1" joined with underscore + const expectedEncodedValues = ['aG9zdC0w_MTI3LjAuMC4x']; + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'first_doc' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip']), + }); + + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); + }); + + it('should return runtime field created from 2 single values, including number value', async () => { + // encoded base64 values of "user-0" and 0 joined with underscore + const expectedEncodedValues = ['dXNlci0w_MA==']; + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'first_doc' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['user.name', 'user.id']), + }); + + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); + }); + + it('should return runtime field created from 2 single values, including boolean value', async () => { + // encoded base64 values of "user-0" and true joined with underscore + const expectedEncodedValues = ['dXNlci0w_dHJ1ZQ==']; + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'first_doc' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['user.name', 'user.enabled']), + }); + + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); + }); + + it('should return runtime field created from 3 single values', async () => { + // encoded base64 values of "host-0" and "127.0.0.1" and "user-0" joined with underscore + const expectedEncodedValues = ['aG9zdC0w_MTI3LjAuMC4x_dXNlci0w']; + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'first_doc' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip', 'user.name']), + }); + + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); + }); + + it('should return runtime field created from fields of arrays', async () => { + // encoded base64 values of all combinations of ["192.168.1.1", "192.168.1.2"] + // and ["tag-new-1", "tag-2", "tag-new-3"] joined with underscore + const expectedEncodedValues = [ + 'MTkyLjE2OC4xLjE=_dGFnLTI=', + 'MTkyLjE2OC4xLjE=_dGFnLW5ldy0x', + 'MTkyLjE2OC4xLjE=_dGFnLW5ldy0z', + 'MTkyLjE2OC4xLjI=_dGFnLTI=', + 'MTkyLjE2OC4xLjI=_dGFnLW5ldy0x', + 'MTkyLjE2OC4xLjI=_dGFnLW5ldy0z', + ]; + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'doc_with_source_ip_as_array' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['source.ip', 'tags']), + }); + + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); + }); + + it('should return runtime field without duplicated values', async () => { + // encoded base64 values of "host-0" and ["tag-1", "tag-2", "tag-2", "tag-1", "tag-1"] + // joined with underscore, without duplicates in tags + const expectedEncodedValues = ['aG9zdC0w_dGFnLTE=', 'aG9zdC0w_dGFnLTI=']; + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'doc_with_duplicated_tags' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'tags']), + }); + + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); + }); + + it('should not return runtime field if one of fields is null', async () => { + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'doc_with_null_field' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME, 'possibly_null_field', 'host.name'], + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'possibly_null_field']), + }); + + expect(hits.hits.length).to.be(1); + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.be(undefined); + expect(hits.hits[0].fields?.possibly_null_field).to.be(undefined); + expect(hits.hits[0].fields?.['host.name']).to.eql(['host-0']); + }); + + it('should not return runtime field if one of fields is not defined', async () => { + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'doc_without_large_arrays' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'large_array_5']), + }); + + expect(hits.hits.length).to.be(1); + expect(hits.hits[0].fields).to.be(undefined); + }); + + // There is a limit in ES for a number of emitted values in runtime field (100) + // This test makes sure runtime script doesn't cause query failure and returns first 100 results + it('should return runtime field if number of emitted values greater than 100', async () => { + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'first_doc' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['large_array_20', 'large_array_10']), + }); + + // runtime field should have 100 values, as large_array_20 and large_array_10 + // give in total 200 combinations + expect(hits.hits[0].fields?.[AGG_FIELD_NAME].length).to.be(100); + }); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/utils/index.ts b/x-pack/test/detection_engine_api_integration/utils/index.ts index b686589addc09..7d03141f58f10 100644 --- a/x-pack/test/detection_engine_api_integration/utils/index.ts +++ b/x-pack/test/detection_engine_api_integration/utils/index.ts @@ -81,6 +81,7 @@ export * from './get_web_hook_action'; export * from './index_event_log_execution_events'; export * from './install_prepackaged_rules'; export * from './machine_learning_setup'; +export * from './perform_search_query'; export * from './preview_rule_with_exception_entries'; export * from './preview_rule'; export * from './refresh_index'; diff --git a/x-pack/test/detection_engine_api_integration/utils/perform_search_query.ts b/x-pack/test/detection_engine_api_integration/utils/perform_search_query.ts new file mode 100644 index 0000000000000..6afd1eebb501c --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/utils/perform_search_query.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client } from '@elastic/elasticsearch'; + +import type { + QueryDslQueryContainer, + MappingRuntimeFields, + IndexName, + Field, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +interface PerformSearchQueryArgs { + es: Client; + query: QueryDslQueryContainer; + index: IndexName; + size?: number; + runtimeMappings?: MappingRuntimeFields; + fields?: Field[]; +} + +/** + * run ES search query + */ +export const performSearchQuery = async ({ + es, + query, + index, + size = 10, + runtimeMappings, + fields, +}: PerformSearchQueryArgs) => { + return es.search({ + index, + size, + fields, + query, + runtime_mappings: runtimeMappings, + }); +}; diff --git a/x-pack/test/functional/es_archives/security_solution/new_terms/data.json b/x-pack/test/functional/es_archives/security_solution/new_terms/data.json new file mode 100644 index 0000000000000..6970a37472c3b --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/new_terms/data.json @@ -0,0 +1,220 @@ +{ + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-14T05:00:01.000Z", + "id": "first_doc", + "possibly_null_field": "test-value", + "host": { + "name": "host-0", + "ip": "127.0.0.1" + }, + "user.name": "user-0", + "user.id": 0, + "user.enabled": true, + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2"], + "large_array_10": ["value-of-10-0","value-of-10-1","value-of-10-2","value-of-10-3","value-of-10-4","value-of-10-5","value-of-10-6","value-of-10-7","value-of-10-8","value-of-10-9"], + "large_array_5": ["value-of-5-0","value-of-5-1","value-of-5-2","value-of-5-3","value-of-5-4"], + "large_array_20": ["value-of-20-0","value-of-20-1","value-of-20-2","value-of-20-3","value-of-20-4","value-of-20-5","value-of-20-6","value-of-20-7","value-of-20-8","value-of-20-9","value-of-20-10","value-of-20-11","value-of-20-12","value-of-20-13","value-of-20-14","value-of-20-15","value-of-20-16","value-of-20-17","value-of-20-18","value-of-20-19"] + } + , + "type": "_doc" + } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-14T05:00:01.000Z", + "id": "doc_without_large_arrays", + "possibly_null_field": "test-value", + "host": { + "name": "host-0", + "ip": "127.0.0.1" + }, + "user.name": "user-1", + "user.id": 1, + "user.enabled": false, + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2"] + }, + "type": "_doc" + } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-14T05:00:01.000Z", + "id": "doc_with_duplicated_tags", + "possibly_null_field": "test-value", + "host": { + "name": "host-0", + "ip": "127.0.0.1" + }, + "user.name": "user-1", + "user.id": 1, + "user.enabled": false, + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2", "tag-2", "tag-1", "tag-1"] + }, + "type": "_doc" + } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-14T05:00:02.000Z", + "possibly_null_field": "test-value", + "host": { + "name": "host-1", + "ip": "127.0.0.1" + }, + "user.name": "user-0", + "user.id": 0, + "user.enabled": true, + "source.ip": "192.168.1.1" + }, + "type": "_doc" + } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-14T05:00:02.000Z", + "possibly_null_field": "test-value", + "host": { + "name": "host-1" + }, + "user.name": "user-0", + "user.id": 0, + "user.enabled": true, + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2"] + }, + "type": "_doc" + } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-14T05:00:03.000Z", + "possibly_null_field": "test-value", + "host": { + "name": "host-1", + "ip": "127.0.0.2" + }, + "user.name": "user-0", + "user.id": 0, + "user.enabled": true, + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2"] + }, + "type": "_doc" + } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-14T05:00:04.000Z", + "possibly_null_field": "test-value", + "host": { + "name": "host-0", + "ip": "127.0.0.1" + }, + "user.name": "user-0", + "user.id": 0, + "user.enabled": true, + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2"] + }, + "type": "_doc" + } + } + + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-20T05:00:04.000Z", + "possibly_null_field": "test-value", + "host": { + "name": "host-0", + "ip": "127.0.0.2" + }, + "user.name": "user-0", + "user.id": 0, + "user.enabled": true, + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2"] + }, + "type": "_doc" + } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-20T05:00:04.000Z", + "id": "doc_with_source_ip_as_array", + "possibly_null_field": "test-value", + "host": { + "name": "host-0", + "ip": "127.0.0.2" + }, + "user.name": "user-0", + "user.id": 0, + "user.enabled": true, + "source.ip": ["192.168.1.1", "192.168.1.2"], + "tags": ["tag-new-1", "tag-2", "tag-new-3"] + }, + "type": "_doc" + } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-20T05:00:04.000Z", + "id": "doc_with_null_field", + "possibly_null_field": null, + "host": { + "name": "host-0", + "ip": "127.0.0.2" + }, + "user.name": "user-0", + "user.id": 1, + "user.enabled": false, + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2"], + "large_array_10": ["a-new-value-of-10-0","a-new-value-of-10-1","a-new-value-of-10-2","a-new-value-of-10-3","a-new-value-of-10-4","a-new-value-of-10-5","a-new-value-of-10-6","a-new-value-of-10-7","a-new-value-of-10-8","a-new-value-of-10-9"], + "large_array_5": ["another-new-value-of-10-0","another-new-value-of-10-1","another-new-value-of-10-2","another-new-value-of-10-3","another-new-value-of-10-4","another-new-value-of-10-5","another-new-value-of-10-6","another-new-value-of-10-7","another-new-value-of-10-8","another-new-value-of-10-9"], + "large_array_20": ["a-new-value-of-20-0","a-new-value-of-20-1","a-new-value-of-20-2","a-new-value-of-20-3","a-new-value-of-20-4","a-new-value-of-20-5","a-new-value-of-20-6","a-new-value-of-20-7","a-new-value-of-20-8","a-new-value-of-20-9","a-new-value-of-20-10","a-new-value-of-20-11","a-new-value-of-20-12","a-new-value-of-20-13","a-new-value-of-20-14","a-new-value-of-20-15","a-new-value-of-20-16","a-new-value-of-20-17","a-new-value-of-20-18","a-new-value-of-20-19"] + }, + "type": "_doc" + } + } \ No newline at end of file diff --git a/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json b/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json new file mode 100644 index 0000000000000..2f156ddedf580 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json @@ -0,0 +1,70 @@ +{ + "type": "index", + "value": { + "index": "new_terms", + "mappings": { + "properties": { + "id": { + "type": "keyword" + }, + "@timestamp": { + "type": "date" + }, + "host": { + "properties": { + "name": { + "type": "keyword" + }, + "ip": { + "type": "ip" + } + } + }, + "user": { + "properties": { + "name": { + "type": "keyword" + }, + "id": { + "type": "integer" + }, + "enabled": { + "type": "boolean" + } + } + }, + "source": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "tags": { + "type": "keyword" + }, + "blob": { + "type": "binary" + }, + "possibly_null_field": { + "type": "keyword" + }, + "large_array_10": { + "type": "keyword" + }, + "large_array_20": { + "type": "keyword" + }, + "large_array_5": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } + } \ No newline at end of file From 261231f8c8697223c744efa5e023b2d996316b72 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Mon, 14 Nov 2022 18:06:23 +0100 Subject: [PATCH 08/16] Replace default warning by an info message (#145086) Customers upgrading to 8.6 should not have warning messages for a new feature that they have not enabled. This PR changes the default behaviour, logging an `info` message (instead of a `warn`) when the default connector configuration is missing for the `'notifications'` plugin. --- .../connectors_email_service_provider.test.ts | 7 ++++--- .../connectors_email_service_provider.ts | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/notifications/server/services/connectors_email_service_provider.test.ts b/x-pack/plugins/notifications/server/services/connectors_email_service_provider.test.ts index 6c36f94db1a7c..7db7502640054 100644 --- a/x-pack/plugins/notifications/server/services/connectors_email_service_provider.test.ts +++ b/x-pack/plugins/notifications/server/services/connectors_email_service_provider.test.ts @@ -79,15 +79,16 @@ describe('ConnectorsEmailServiceProvider', () => { expect(serviceProvider['setupSuccessful']).toEqual(false); }); - it('should log a warning if no default email connector has been defined', () => { + it('should log an info message if no default email connector has been defined', () => { const serviceProvider = new EmailServiceProvider(missingConnectorConfig, logger); serviceProvider.setup({ actions: actionsSetup, licensing: licensingMock.createSetup(), }); - expect(logger.warn).toHaveBeenCalledTimes(1); - expect(logger.warn).toHaveBeenCalledWith( + expect(logger.warn).not.toHaveBeenCalled(); + expect(logger.info).toHaveBeenCalledTimes(1); + expect(logger.info).toHaveBeenCalledWith( `Email Service Error: Email connector not specified.` ); // eslint-disable-next-line dot-notation diff --git a/x-pack/plugins/notifications/server/services/connectors_email_service_provider.ts b/x-pack/plugins/notifications/server/services/connectors_email_service_provider.ts index f034116eb701c..b3364f31d3689 100755 --- a/x-pack/plugins/notifications/server/services/connectors_email_service_provider.ts +++ b/x-pack/plugins/notifications/server/services/connectors_email_service_provider.ts @@ -41,16 +41,18 @@ export class EmailServiceProvider const { actions, licensing } = plugins; if (!actions || !licensing) { - return this._registerServiceError(`Error: 'actions' and 'licensing' plugins are required.`); + return this._registerInitializationError( + `Error: 'actions' and 'licensing' plugins are required.` + ); } const emailConnector = this.config.connectors?.default?.email; if (!emailConnector) { - return this._registerServiceError('Error: Email connector not specified.'); + return this._registerInitializationError('Error: Email connector not specified.', 'info'); } if (!actions.isPreconfiguredConnector(emailConnector)) { - return this._registerServiceError( + return this._registerInitializationError( `Error: Unexisting email connector '${emailConnector}' specified.` ); } @@ -75,7 +77,7 @@ export class EmailServiceProvider this.logger ); } catch (err) { - this._registerServiceError(err); + this._registerInitializationError(err); } } @@ -90,9 +92,13 @@ export class EmailServiceProvider }; } - private _registerServiceError(error: string) { + private _registerInitializationError(error: string, level: 'info' | 'warn' = 'warn') { const message = `Email Service ${error}`; this.setupError = message; - this.logger.warn(message); + if (level === 'info') { + this.logger.info(message); + } else { + this.logger.warn(message); + } } } From e7730f0ab17f7031d945fa0219a8cfda21d5672d Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 14 Nov 2022 17:10:41 +0000 Subject: [PATCH 09/16] skip flaky suite (#145135) --- x-pack/test/api_integration/apis/security/privileges.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/security/privileges.ts b/x-pack/test/api_integration/apis/security/privileges.ts index d11ad982a0f45..81c38e5e1ceba 100644 --- a/x-pack/test/api_integration/apis/security/privileges.ts +++ b/x-pack/test/api_integration/apis/security/privileges.ts @@ -84,7 +84,8 @@ export default function ({ getService }: FtrProviderContext) { }; describe('Privileges', () => { - describe('GET /api/security/privileges', () => { + // FLAKY: https://github.com/elastic/kibana/issues/145135 + describe.skip('GET /api/security/privileges', () => { it('should return a privilege map with all known privileges, without actions', async () => { // If you're adding a privilege to the following, that's great! // If you're removing a privilege, this breaks backwards compatibility From 46522d76da06d74ab2da4e2e2a8baf5b15762af6 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Mon, 14 Nov 2022 18:11:46 +0100 Subject: [PATCH 10/16] Remove non-essential ES-client stats from /api/stats (#145120) [PR #141434](https://github.com/elastic/kibana/pull/141434) exposes a bunch of metrics related to the Elasticsearch Client in the `/api/stats` endpoint. While all these stats are interesting, some of them might be less relevant than others right now. Let's start by exposing only those stats that are more critical from a monitoring standpoint. image --- .../src/status/lib/load_status.test.ts | 8 -- .../src/get_agents_sockets_stats.test.ts | 94 +------------------ .../src/get_agents_sockets_stats.ts | 35 +------ .../src/metrics_service.mock.ts | 8 -- .../core-metrics-server/src/metrics.ts | 18 ---- .../ops_stats_collector.test.ts.snap | 8 -- 6 files changed, 2 insertions(+), 169 deletions(-) diff --git a/packages/core/apps/core-apps-browser-internal/src/status/lib/load_status.test.ts b/packages/core/apps/core-apps-browser-internal/src/status/lib/load_status.test.ts index dd750a56fbf2d..54fb3ed083105 100644 --- a/packages/core/apps/core-apps-browser-internal/src/status/lib/load_status.test.ts +++ b/packages/core/apps/core-apps-browser-internal/src/status/lib/load_status.test.ts @@ -62,17 +62,9 @@ const mockedResponse: StatusResponse = { }, }, elasticsearch_client: { - protocol: 'https', - connectedNodes: 3, - nodesWithActiveSockets: 3, - nodesWithIdleSockets: 1, totalActiveSockets: 25, totalIdleSockets: 2, totalQueuedRequests: 0, - mostActiveNodeSockets: 15, - averageActiveSocketsPerNode: 8, - mostIdleNodeSockets: 2, - averageIdleSocketsPerNode: 0.5, }, process: { pid: 1, diff --git a/packages/core/metrics/core-metrics-collectors-server-internal/src/get_agents_sockets_stats.test.ts b/packages/core/metrics/core-metrics-collectors-server-internal/src/get_agents_sockets_stats.test.ts index 513bf2caa8545..d72a33dd7a8a1 100644 --- a/packages/core/metrics/core-metrics-collectors-server-internal/src/get_agents_sockets_stats.test.ts +++ b/packages/core/metrics/core-metrics-collectors-server-internal/src/get_agents_sockets_stats.test.ts @@ -32,7 +32,7 @@ describe('getAgentsSocketsStats()', () => { }, }); - const agent2 = getHttpAgentMock({ + const agent2 = getHttpsAgentMock({ sockets: { node1: [mockSocket, mockSocket, mockSocket], node4: [mockSocket], @@ -47,101 +47,9 @@ describe('getAgentsSocketsStats()', () => { const stats = getAgentsSocketsStats(new Set([agent1, agent2])); expect(stats).toEqual({ - averageActiveSocketsPerNode: 2.6666666666666665, - averageIdleSocketsPerNode: 4.5, - connectedNodes: 4, - mostActiveNodeSockets: 6, - mostIdleNodeSockets: 8, - nodesWithActiveSockets: 3, - nodesWithIdleSockets: 2, - protocol: 'http', totalActiveSockets: 8, totalIdleSockets: 9, totalQueuedRequests: 6, }); }); - - it('takes into account Agent types to determine the `protocol`', () => { - const httpAgent = getHttpAgentMock({ - sockets: { node1: [mockSocket] }, - freeSockets: {}, - requests: {}, - }); - - const httpsAgent = getHttpsAgentMock({ - sockets: { node1: [mockSocket] }, - freeSockets: {}, - requests: {}, - }); - - const noAgents = new Set(); - const httpAgents = new Set([httpAgent, httpAgent]); - const httpsAgents = new Set([httpsAgent, httpsAgent]); - const mixedAgents = new Set([httpAgent, httpsAgent]); - - expect(getAgentsSocketsStats(noAgents).protocol).toEqual('none'); - expect(getAgentsSocketsStats(httpAgents).protocol).toEqual('http'); - expect(getAgentsSocketsStats(httpsAgents).protocol).toEqual('https'); - expect(getAgentsSocketsStats(mixedAgents).protocol).toEqual('mixed'); - }); - - it('does not take into account those Agents that have not had any connection to any node', () => { - const pristineAgentProps = { - sockets: {}, - freeSockets: {}, - requests: {}, - }; - const agent1 = getHttpAgentMock(pristineAgentProps); - const agent2 = getHttpAgentMock(pristineAgentProps); - const agent3 = getHttpAgentMock(pristineAgentProps); - - const stats = getAgentsSocketsStats(new Set([agent1, agent2, agent3])); - - expect(stats).toEqual({ - averageActiveSocketsPerNode: 0, - averageIdleSocketsPerNode: 0, - connectedNodes: 0, - mostActiveNodeSockets: 0, - mostIdleNodeSockets: 0, - nodesWithActiveSockets: 0, - nodesWithIdleSockets: 0, - protocol: 'none', - totalActiveSockets: 0, - totalIdleSockets: 0, - totalQueuedRequests: 0, - }); - }); - - it('takes into account those Agents that have hold mappings to one or more nodes, but that do not currently have any pending requests, active connections or idle connections', () => { - const emptyAgentProps = { - sockets: { - node1: [], - }, - freeSockets: { - node2: [], - }, - requests: { - node3: [], - }, - }; - - const agent1 = getHttpAgentMock(emptyAgentProps); - const agent2 = getHttpAgentMock(emptyAgentProps); - - const stats = getAgentsSocketsStats(new Set([agent1, agent2])); - - expect(stats).toEqual({ - averageActiveSocketsPerNode: 0, - averageIdleSocketsPerNode: 0, - connectedNodes: 3, - mostActiveNodeSockets: 0, - mostIdleNodeSockets: 0, - nodesWithActiveSockets: 0, - nodesWithIdleSockets: 0, - protocol: 'http', - totalActiveSockets: 0, - totalIdleSockets: 0, - totalQueuedRequests: 0, - }); - }); }); diff --git a/packages/core/metrics/core-metrics-collectors-server-internal/src/get_agents_sockets_stats.ts b/packages/core/metrics/core-metrics-collectors-server-internal/src/get_agents_sockets_stats.ts index e28c92a56a8a4..e513528c07697 100644 --- a/packages/core/metrics/core-metrics-collectors-server-internal/src/get_agents_sockets_stats.ts +++ b/packages/core/metrics/core-metrics-collectors-server-internal/src/get_agents_sockets_stats.ts @@ -7,23 +7,13 @@ */ import { NetworkAgent } from '@kbn/core-elasticsearch-client-server-internal'; -import { Agent as HttpsAgent } from 'https'; -import { mean } from 'lodash'; -import type { - ElasticsearchClientProtocol, - ElasticsearchClientsMetrics, -} from '@kbn/core-metrics-server'; +import type { ElasticsearchClientsMetrics } from '@kbn/core-metrics-server'; export const getAgentsSocketsStats = (agents: Set): ElasticsearchClientsMetrics => { const nodes = new Set(); let totalActiveSockets = 0; let totalIdleSockets = 0; let totalQueuedRequests = 0; - let http: boolean = false; - let https: boolean = false; - - const nodesWithActiveSockets: Record = {}; - const nodesWithIdleSockets: Record = {}; agents.forEach((agent) => { const agentRequests = Object.entries(agent.requests) ?? []; @@ -31,9 +21,6 @@ export const getAgentsSocketsStats = (agents: Set): ElasticsearchC const agentFreeSockets = Object.entries(agent.freeSockets) ?? []; if (agentRequests.length || agentSockets.length || agentFreeSockets.length) { - if (agent instanceof HttpsAgent) https = true; - else http = true; - agentRequests.forEach(([node, queue]) => { nodes.add(node); totalQueuedRequests += queue?.length ?? 0; @@ -43,39 +30,19 @@ export const getAgentsSocketsStats = (agents: Set): ElasticsearchC nodes.add(node); const activeSockets = sockets?.length ?? 0; totalActiveSockets += activeSockets; - nodesWithActiveSockets[node] = (nodesWithActiveSockets[node] ?? 0) + activeSockets; }); agentFreeSockets.forEach(([node, freeSockets]) => { nodes.add(node); const idleSockets = freeSockets?.length ?? 0; totalIdleSockets += idleSockets; - nodesWithIdleSockets[node] = (nodesWithIdleSockets[node] ?? 0) + idleSockets; }); } }); - const activeSocketCounters = Object.values(nodesWithActiveSockets); - const idleSocketCounters = Object.values(nodesWithIdleSockets); - const protocol: ElasticsearchClientProtocol = http - ? https - ? 'mixed' - : 'http' - : https - ? 'https' - : 'none'; - return { - protocol, - connectedNodes: nodes.size, - nodesWithActiveSockets: activeSocketCounters.filter(Boolean).length, - nodesWithIdleSockets: idleSocketCounters.filter(Boolean).length, totalActiveSockets, totalIdleSockets, totalQueuedRequests, - mostActiveNodeSockets: activeSocketCounters.length ? Math.max(...activeSocketCounters) : 0, - averageActiveSocketsPerNode: activeSocketCounters.length ? mean(activeSocketCounters) : 0, - mostIdleNodeSockets: idleSocketCounters.length ? Math.max(...idleSocketCounters) : 0, - averageIdleSocketsPerNode: idleSocketCounters.length ? mean(idleSocketCounters) : 0, }; }; diff --git a/packages/core/metrics/core-metrics-server-mocks/src/metrics_service.mock.ts b/packages/core/metrics/core-metrics-server-mocks/src/metrics_service.mock.ts index 44601caeaa85c..cd49d704d0afc 100644 --- a/packages/core/metrics/core-metrics-server-mocks/src/metrics_service.mock.ts +++ b/packages/core/metrics/core-metrics-server-mocks/src/metrics_service.mock.ts @@ -24,17 +24,9 @@ import type { } from '@kbn/core-metrics-server'; export const sampleEsClientMetrics: ElasticsearchClientsMetrics = { - protocol: 'https', - connectedNodes: 3, - nodesWithActiveSockets: 3, - nodesWithIdleSockets: 1, totalActiveSockets: 25, totalIdleSockets: 2, totalQueuedRequests: 0, - mostActiveNodeSockets: 15, - averageActiveSocketsPerNode: 8, - mostIdleNodeSockets: 2, - averageIdleSocketsPerNode: 0.5, }; const createInternalSetupContractMock = () => { diff --git a/packages/core/metrics/core-metrics-server/src/metrics.ts b/packages/core/metrics/core-metrics-server/src/metrics.ts index 958f6b75f55e4..c20b2eb78d5d3 100644 --- a/packages/core/metrics/core-metrics-server/src/metrics.ts +++ b/packages/core/metrics/core-metrics-server/src/metrics.ts @@ -52,30 +52,12 @@ export type ElasticsearchClientProtocol = 'none' | 'http' | 'https' | 'mixed'; * @public */ export interface ElasticsearchClientsMetrics { - /** The protocol (or protocols) that these Agents are using */ - protocol: ElasticsearchClientProtocol; - /** Number of ES nodes that ES-js client is connecting to */ - connectedNodes: number; - /** Number of nodes with active connections */ - nodesWithActiveSockets: number; - /** Number of nodes with available connections (alive but idle). - * Note that a node can have both active and idle connections at the same time - */ - nodesWithIdleSockets: number; /** Total number of active sockets (all nodes, all connections) */ totalActiveSockets: number; /** Total number of available sockets (alive but idle, all nodes, all connections) */ totalIdleSockets: number; /** Total number of queued requests (all nodes, all connections) */ totalQueuedRequests: number; - /** Number of active connections of the node with most active connections */ - mostActiveNodeSockets: number; - /** Average of active sockets per node (all connections) */ - averageActiveSocketsPerNode: number; - /** Number of idle connections of the node with most idle connections */ - mostIdleNodeSockets: number; - /** Average of available (idle) sockets per node (all connections) */ - averageIdleSocketsPerNode: number; } /** diff --git a/src/plugins/kibana_usage_collection/server/collectors/ops_stats/__snapshots__/ops_stats_collector.test.ts.snap b/src/plugins/kibana_usage_collection/server/collectors/ops_stats/__snapshots__/ops_stats_collector.test.ts.snap index d77d43293480b..9005e4ba24bf7 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ops_stats/__snapshots__/ops_stats_collector.test.ts.snap +++ b/src/plugins/kibana_usage_collection/server/collectors/ops_stats/__snapshots__/ops_stats_collector.test.ts.snap @@ -4,14 +4,6 @@ exports[`telemetry_ops_stats should return something when there is a metric 1`] Object { "concurrent_connections": 1, "elasticsearch_client": Object { - "averageActiveSocketsPerNode": 8, - "averageIdleSocketsPerNode": 0.5, - "connectedNodes": 3, - "mostActiveNodeSockets": 15, - "mostIdleNodeSockets": 2, - "nodesWithActiveSockets": 3, - "nodesWithIdleSockets": 1, - "protocol": "https", "totalActiveSockets": 25, "totalIdleSockets": 2, "totalQueuedRequests": 0, From 773f8deaeb4b47af06736601d2a39290986a4bef Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 14 Nov 2022 17:12:32 +0000 Subject: [PATCH 11/16] skip flaky suite (#145136) --- x-pack/test/api_integration/apis/security/privileges.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/security/privileges.ts b/x-pack/test/api_integration/apis/security/privileges.ts index 81c38e5e1ceba..242b70d3f48d9 100644 --- a/x-pack/test/api_integration/apis/security/privileges.ts +++ b/x-pack/test/api_integration/apis/security/privileges.ts @@ -194,7 +194,8 @@ export default function ({ getService }: FtrProviderContext) { }); // In this non-Basic case, results should be exactly the same as not supplying the respectLicenseLevel flag - describe('GET /api/security/privileges?respectLicenseLevel=false', () => { + // FLAKY: https://github.com/elastic/kibana/issues/145136 + describe.skip('GET /api/security/privileges?respectLicenseLevel=false', () => { it('should return a privilege map with all known privileges, without actions', async () => { // If you're adding a privilege to the following, that's great! // If you're removing a privilege, this breaks backwards compatibility From 00b5e88ef3bf6ca9d593ea768649f5f75f3bfeea Mon Sep 17 00:00:00 2001 From: jennypavlova Date: Mon, 14 Nov 2022 18:33:29 +0100 Subject: [PATCH 12/16] [infrastructure UI] Metrics should use inventory metric formatter (#145085) Closes #144637 ## Summary This PR fixes the formatting issue on the host view page. To keep it consistent with the inventory page the formatters used there are extended and used on the Host View. I kept the 'N/A' fallback for missing values similar to the APM tables as I think it looks better in the table ( the inventory formatter will return an empty string in that case but it is also used for the tooltips and there maybe it makes sense to leave it empty) ### Testing Open the Host View and check the formatting of the values of the table: image --- .../plugins/infra/common/formatters/bytes.ts | 2 +- .../host/metrics/snapshot/memory_total.ts | 2 +- .../hosts/components/hosts_table_columns.tsx | 26 +++++++++++-------- .../lib/create_inventory_metric_formatter.ts | 12 +++++++++ 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/infra/common/formatters/bytes.ts b/x-pack/plugins/infra/common/formatters/bytes.ts index 64cd075efce6a..797168c300f64 100644 --- a/x-pack/plugins/infra/common/formatters/bytes.ts +++ b/x-pack/plugins/infra/common/formatters/bytes.ts @@ -43,7 +43,7 @@ export const createBytesFormatter = (format: InfraWaffleMapDataFormat) => (bytes const labels = LABELS[format]; const base = BASES[format]; const value = format === InfraWaffleMapDataFormat.bitsDecimal ? bytes * 8 : bytes; - // Use an exponetial equation to get the power to determine which label to use. If the power + // Use an exponential equation to get the power to determine which label to use. If the power // is greater then the max label then use the max label. const power = Math.min(Math.floor(Math.log(Math.abs(value)) / Math.log(base)), labels.length - 1); if (power < 0) { diff --git a/x-pack/plugins/infra/common/inventory_models/host/metrics/snapshot/memory_total.ts b/x-pack/plugins/infra/common/inventory_models/host/metrics/snapshot/memory_total.ts index c3d0a8063b93f..e1990555ad550 100644 --- a/x-pack/plugins/infra/common/inventory_models/host/metrics/snapshot/memory_total.ts +++ b/x-pack/plugins/infra/common/inventory_models/host/metrics/snapshot/memory_total.ts @@ -19,7 +19,7 @@ export const memoryTotal: MetricsUIAggregation = { memoryTotal: 'memory_total', }, script: { - source: 'params.memoryTotal / 1000000', // Convert to MB + source: 'params.memoryTotal', lang: 'painless', }, gap_policy: 'skip', diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table_columns.tsx b/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table_columns.tsx index 1bf4aab9ce054..03f0f243445ce 100644 --- a/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table_columns.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table_columns.tsx @@ -9,9 +9,8 @@ import { EuiBasicTableColumn } from '@elastic/eui'; import React from 'react'; import { i18n } from '@kbn/i18n'; import { EuiText } from '@elastic/eui'; -import type { SnapshotNodeMetric } from '../../../../../common/http_api'; -import { scaleUpPercentage } from '../../../../components/infrastructure_node_metrics_tables/shared/hooks'; -import { NumberCell } from '../../../../components/infrastructure_node_metrics_tables/shared/components'; +import type { SnapshotMetricInput, SnapshotNodeMetric } from '../../../../../common/http_api'; +import { createInventoryMetricFormatter } from '../../inventory_view/lib/create_inventory_metric_formatter'; interface HostNodeRow extends HostMetics { os?: string | null; @@ -28,6 +27,9 @@ export interface HostMetics { memoryTotal: SnapshotNodeMetric; } +const formatMetric = (type: SnapshotMetricInput['type'], value: number | undefined | null) => + value || value === 0 ? createInventoryMetricFormatter({ type })(value) : 'N/A'; + export const HostsTableColumns: Array> = [ { name: i18n.translate('xpack.infra.hostsTable.nameColumnHeader', { @@ -50,9 +52,11 @@ export const HostsTableColumns: Array> = [ name: i18n.translate('xpack.infra.hostsTable.numberOfCpusColumnHeader', { defaultMessage: '# of CPUs', }), - field: 'cpuCores.value', + field: 'cpuCores', sortable: true, - render: (value: number) => , + render: (cpuCores: SnapshotNodeMetric) => ( + <>{formatMetric('cpuCores', cpuCores?.value ?? cpuCores?.max)} + ), }, { name: i18n.translate('xpack.infra.hostsTable.diskLatencyColumnHeader', { @@ -60,7 +64,7 @@ export const HostsTableColumns: Array> = [ }), field: 'diskLatency.avg', sortable: true, - render: (avg: number) => , + render: (avg: number) => <>{formatMetric('diskLatency', avg)}, }, { name: i18n.translate('xpack.infra.hostsTable.averageTxColumnHeader', { @@ -68,7 +72,7 @@ export const HostsTableColumns: Array> = [ }), field: 'tx.avg', sortable: true, - render: (avg: number) => , + render: (avg: number) => <>{formatMetric('tx', avg)}, }, { name: i18n.translate('xpack.infra.hostsTable.averageRxColumnHeader', { @@ -76,7 +80,7 @@ export const HostsTableColumns: Array> = [ }), field: 'rx.avg', sortable: true, - render: (avg: number) => , + render: (avg: number) => <>{formatMetric('rx', avg)}, }, { name: i18n.translate('xpack.infra.hostsTable.averageMemoryTotalColumnHeader', { @@ -84,7 +88,7 @@ export const HostsTableColumns: Array> = [ }), field: 'memoryTotal.avg', sortable: true, - render: (avg: number) => , + render: (avg: number) => <>{formatMetric('memoryTotal', avg)}, }, { name: i18n.translate('xpack.infra.hostsTable.servicesOnHostColumnHeader', { @@ -92,7 +96,7 @@ export const HostsTableColumns: Array> = [ }), field: 'servicesOnHost', sortable: true, - render: (servicesOnHost: number) => , + render: (servicesOnHost: number) => <>{formatMetric('cpuCores', servicesOnHost)}, }, { name: i18n.translate('xpack.infra.hostsTable.averageMemoryUsageColumnHeader', { @@ -100,6 +104,6 @@ export const HostsTableColumns: Array> = [ }), field: 'memory.avg', sortable: true, - render: (avg: number) => , + render: (avg: number) => <>{formatMetric('memory', avg)}, }, ]; diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/lib/create_inventory_metric_formatter.ts b/x-pack/plugins/infra/public/pages/metrics/inventory_view/lib/create_inventory_metric_formatter.ts index ca09762442d20..40aa9cd693e7f 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/lib/create_inventory_metric_formatter.ts +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/lib/create_inventory_metric_formatter.ts @@ -30,10 +30,22 @@ const METRIC_FORMATTERS: MetricFormatters = { formatter: InfraFormatterType.percent, template: '{{value}}', }, + ['cpuCores']: { + formatter: InfraFormatterType.number, + template: '{{value}}', + }, ['memory']: { formatter: InfraFormatterType.percent, template: '{{value}}', }, + ['memoryTotal']: { + formatter: InfraFormatterType.bytes, + template: '{{value}}', + }, + ['diskLatency']: { + formatter: InfraFormatterType.number, + template: '{{value}} ms', + }, ['rx']: { formatter: InfraFormatterType.bits, template: '{{value}}/s' }, ['tx']: { formatter: InfraFormatterType.bits, template: '{{value}}/s' }, ['logRate']: { From 34d8a68d102c4988aa8f3e15744901634d9e11e8 Mon Sep 17 00:00:00 2001 From: Jordan <51442161+JordanSh@users.noreply.github.com> Date: Mon, 14 Nov 2022 19:36:38 +0200 Subject: [PATCH 13/16] [Cloud Posture] Dashboard Redesign trend graph (#144814) --- .../common/constants.ts | 1 - .../api/use_cis_kubernetes_integration.tsx | 4 +- .../public/components/chart_panel.tsx | 2 +- .../public/components/cis_benchmark_icon.tsx | 2 +- .../public/components/csp_counter_card.tsx | 51 +++-- .../compliance_charts/cases_table.tsx | 29 --- .../cloud_posture_score_chart.tsx | 207 +++++++++--------- .../compliance_charts/risks_table.tsx | 42 ++-- .../compliance_dashboard.tsx | 24 +- .../dashboard_sections/benchmarks_section.tsx | 113 ---------- .../cloud_benchmarks_section.tsx | 30 +-- .../cloud_summary_section.tsx | 41 ++-- .../cluster_details_box.tsx | 33 ++- .../dashboard_sections/summary_section.tsx | 90 -------- .../cloud_security_posture/public/plugin.tsx | 8 +- .../translations/translations/fr-FR.json | 5 - .../translations/translations/ja-JP.json | 5 - .../translations/translations/zh-CN.json | 5 - 18 files changed, 217 insertions(+), 475 deletions(-) delete mode 100644 x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_charts/cases_table.tsx delete mode 100644 x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/benchmarks_section.tsx delete mode 100644 x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/summary_section.tsx diff --git a/x-pack/plugins/cloud_security_posture/common/constants.ts b/x-pack/plugins/cloud_security_posture/common/constants.ts index 11498efbb7fc6..052f3cb743d37 100644 --- a/x-pack/plugins/cloud_security_posture/common/constants.ts +++ b/x-pack/plugins/cloud_security_posture/common/constants.ts @@ -40,7 +40,6 @@ export const INTERNAL_FEATURE_FLAGS = { showManageRulesMock: false, showFindingFlyoutEvidence: false, showFindingsGroupBy: true, - showNewDashboard: false, } as const; export const CSP_RULE_SAVED_OBJECT_TYPE = 'csp_rule'; diff --git a/x-pack/plugins/cloud_security_posture/public/common/api/use_cis_kubernetes_integration.tsx b/x-pack/plugins/cloud_security_posture/public/common/api/use_cis_kubernetes_integration.tsx index 26b885ad44344..0ee08d915c0d1 100644 --- a/x-pack/plugins/cloud_security_posture/public/common/api/use_cis_kubernetes_integration.tsx +++ b/x-pack/plugins/cloud_security_posture/public/common/api/use_cis_kubernetes_integration.tsx @@ -21,8 +21,6 @@ export const useCisKubernetesIntegration = () => { const { http } = useKibana().services; return useQuery(['integrations'], () => - http.get(epmRouteService.getInfoPath(CLOUD_SECURITY_POSTURE_PACKAGE_NAME), { - query: { experimental: true }, - }) + http.get(epmRouteService.getInfoPath(CLOUD_SECURITY_POSTURE_PACKAGE_NAME)) ); }; diff --git a/x-pack/plugins/cloud_security_posture/public/components/chart_panel.tsx b/x-pack/plugins/cloud_security_posture/public/components/chart_panel.tsx index 409460be59287..01291b287293a 100644 --- a/x-pack/plugins/cloud_security_posture/public/components/chart_panel.tsx +++ b/x-pack/plugins/cloud_security_posture/public/components/chart_panel.tsx @@ -60,7 +60,7 @@ export const ChartPanel: React.FC = ({ return ( - + {title && ( diff --git a/x-pack/plugins/cloud_security_posture/public/components/cis_benchmark_icon.tsx b/x-pack/plugins/cloud_security_posture/public/components/cis_benchmark_icon.tsx index a5cc0d34466e2..f72dff1206cce 100644 --- a/x-pack/plugins/cloud_security_posture/public/components/cis_benchmark_icon.tsx +++ b/x-pack/plugins/cloud_security_posture/public/components/cis_benchmark_icon.tsx @@ -30,6 +30,6 @@ const getBenchmarkIdIconType = (props: Props): string => { export const CISBenchmarkIcon = (props: Props) => ( - + ); diff --git a/x-pack/plugins/cloud_security_posture/public/components/csp_counter_card.tsx b/x-pack/plugins/cloud_security_posture/public/components/csp_counter_card.tsx index dbe54990be505..4b953d7030251 100644 --- a/x-pack/plugins/cloud_security_posture/public/components/csp_counter_card.tsx +++ b/x-pack/plugins/cloud_security_posture/public/components/csp_counter_card.tsx @@ -5,32 +5,31 @@ * 2.0. */ -import React from 'react'; +import React, { MouseEventHandler } from 'react'; import { css } from '@emotion/react'; -import { EuiCard, EuiIcon, EuiText, EuiTitle, useEuiTheme } from '@elastic/eui'; -import type { EuiTextProps, EuiCardProps } from '@elastic/eui'; +import { EuiIcon, EuiPanel, EuiStat, useEuiTheme } from '@elastic/eui'; +import type { EuiStatProps } from '@elastic/eui'; -export type CspCounterCardProps = Pick & { - descriptionColor?: EuiTextProps['color']; -}; +export interface CspCounterCardProps { + id: string; + onClick?: MouseEventHandler; + title: EuiStatProps['title']; + titleColor?: EuiStatProps['titleColor']; + description: EuiStatProps['description']; +} export const CspCounterCard = (counter: CspCounterCardProps) => { const { euiTheme } = useEuiTheme(); return ( - -
{counter.title}
- - } + { `} data-test-subj={counter.id} > - - -

{counter.description}

- - + {counter.onClick && ( )} - + ); }; diff --git a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_charts/cases_table.tsx b/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_charts/cases_table.tsx deleted file mode 100644 index f978f19bc9a29..0000000000000 --- a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_charts/cases_table.tsx +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React from 'react'; -import { EuiFlexGroup, EuiFlexItem, EuiText } from '@elastic/eui'; -import { EuiIcon } from '@elastic/eui'; -import { FormattedMessage } from '@kbn/i18n-react'; - -export const CasesTable = () => { - return ( - - - - - - - - - - - ); -}; diff --git a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_charts/cloud_posture_score_chart.tsx b/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_charts/cloud_posture_score_chart.tsx index 98a3b140aa233..20ebd4b893f20 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_charts/cloud_posture_score_chart.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_charts/cloud_posture_score_chart.tsx @@ -10,100 +10,51 @@ import { AreaSeries, Axis, Chart, - ElementClickListener, niceTimeFormatByDay, - Partition, - PartitionElementEvent, - PartitionLayout, Settings, timeFormatter, } from '@elastic/charts'; -import { EuiFlexGroup, EuiText, EuiHorizontalRule, EuiFlexItem } from '@elastic/eui'; +import { + useEuiTheme, + EuiFlexGroup, + EuiFlexItem, + EuiLink, + EuiText, + EuiTitle, + type EuiLinkButtonProps, + type EuiTextProps, + EuiToolTip, + EuiToolTipProps, +} from '@elastic/eui'; import { FormattedDate, FormattedTime } from '@kbn/i18n-react'; import moment from 'moment'; -import { statusColors } from '../../../common/constants'; -import type { PostureTrend, Stats } from '../../../../common/types'; -import { CompactFormattedNumber } from '../../../components/compact_formatted_number'; +import { i18n } from '@kbn/i18n'; import { RULE_FAILED, RULE_PASSED } from '../../../../common/constants'; +import { CompactFormattedNumber } from '../../../components/compact_formatted_number'; +import type { Evaluation, PostureTrend, Stats } from '../../../../common/types'; import { useKibana } from '../../../common/hooks/use_kibana'; interface CloudPostureScoreChartProps { + compact?: boolean; trend: PostureTrend[]; data: Stats; id: string; - partitionOnElementClick: (elements: PartitionElementEvent[]) => void; + onEvalCounterClick: (evaluation: Evaluation) => void; } const getPostureScorePercentage = (postureScore: number): string => `${Math.round(postureScore)}%`; -const ScoreChart = ({ - data: { totalPassed, totalFailed }, - id, - partitionOnElementClick, -}: Omit) => { - const data = [ - { label: RULE_PASSED, value: totalPassed }, - { label: RULE_FAILED, value: totalFailed }, - ]; - const { - services: { charts }, - } = useKibana(); - - return ( - - - d.value} - layout={PartitionLayout.sunburst} - layers={[ - { - groupByRollup: (d: { label: string }) => d.label, - shape: { - fillColor: (d, index) => - d.dataName === RULE_PASSED ? statusColors.success : statusColors.danger, - }, - }, - ]} - /> - - ); -}; - const PercentageInfo = ({ + compact, postureScore, - totalPassed, - totalFindings, -}: CloudPostureScoreChartProps['data']) => { +}: CloudPostureScoreChartProps['data'] & { compact?: CloudPostureScoreChartProps['compact'] }) => { + const { euiTheme } = useEuiTheme(); const percentage = getPostureScorePercentage(postureScore); return ( - - {percentage} - - - {'/'} - - {' Findings passed'} - - + +

{percentage}

+ ); }; @@ -149,38 +100,94 @@ const ComplianceTrendChart = ({ trend }: { trend: PostureTrend[] }) => { tickFormat={timeFormatter(niceTimeFormatByDay(2))} ticks={4} /> - getPostureScorePercentage(rawScore)} - /> + ); }; +const CounterLink = ({ + text, + count, + color, + onClick, + tooltipContent, +}: { + count: number; + text: string; + color: EuiTextProps['color']; + onClick: EuiLinkButtonProps['onClick']; + tooltipContent: EuiToolTipProps['content']; +}) => { + const { euiTheme } = useEuiTheme(); + + return ( + + + + +   + + {text} + + + ); +}; + export const CloudPostureScoreChart = ({ data, trend, - id, - partitionOnElementClick, -}: CloudPostureScoreChartProps) => ( - - - - - - - - - - - - - - - - -); + onEvalCounterClick, + compact, +}: CloudPostureScoreChartProps) => { + const { euiTheme } = useEuiTheme(); + + return ( + + + + + + + + + onEvalCounterClick(RULE_PASSED)} + tooltipContent={i18n.translate( + 'xpack.csp.cloudPostureScoreChart.counterLink.passedFindingsTooltip', + { defaultMessage: 'Passed findings' } + )} + /> +  {`-`}  + onEvalCounterClick(RULE_FAILED)} + tooltipContent={i18n.translate( + 'xpack.csp.cloudPostureScoreChart.counterLink.failedFindingsTooltip', + { defaultMessage: 'Failed findings' } + )} + /> + + + + + + + + + ); +}; diff --git a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_charts/risks_table.tsx b/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_charts/risks_table.tsx index b911688dc0cf1..a1850a793ab3e 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_charts/risks_table.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_charts/risks_table.tsx @@ -25,6 +25,7 @@ export interface RisksTableProps { maxItems: number; onCellClick: (name: string) => void; onViewAllClick: () => void; + compact?: boolean; } export const getTopRisks = ( @@ -42,26 +43,31 @@ export const RisksTable = ({ maxItems, onCellClick, onViewAllClick, + compact, }: RisksTableProps) => { const columns: Array> = useMemo( () => [ { field: 'name', truncateText: true, - name: i18n.translate('xpack.csp.dashboard.risksTable.cisSectionColumnLabel', { - defaultMessage: 'CIS Section', - }), + name: compact + ? '' + : i18n.translate('xpack.csp.dashboard.risksTable.cisSectionColumnLabel', { + defaultMessage: 'CIS Section', + }), render: (name: GroupedFindingsEvaluation['name']) => ( - onCellClick(name)} className="eui-textTruncate"> + onCellClick(name)} className="eui-textTruncate" color="text"> {name} ), }, { field: 'totalFailed', - name: i18n.translate('xpack.csp.dashboard.risksTable.findingsColumnLabel', { - defaultMessage: 'Findings', - }), + name: compact + ? '' + : i18n.translate('xpack.csp.dashboard.risksTable.findingsColumnLabel', { + defaultMessage: 'Findings', + }), render: ( totalFailed: GroupedFindingsEvaluation['totalFailed'], resource: GroupedFindingsEvaluation @@ -78,13 +84,13 @@ export const RisksTable = ({ ), }, ], - [onCellClick] + [compact, onCellClick] ); const items = useMemo(() => getTopRisks(resourcesTypes, maxItems), [resourcesTypes, maxItems]); return ( - + rowHeader="name" @@ -93,16 +99,14 @@ export const RisksTable = ({ /> - - - - - - - +
+ + + +
); diff --git a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_dashboard.tsx b/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_dashboard.tsx index c72bc286f7d0d..33fa8756c631d 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_dashboard.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_dashboard.tsx @@ -9,13 +9,10 @@ import React from 'react'; import { EuiSpacer, EuiPageHeader } from '@elastic/eui'; import { css } from '@emotion/react'; import { i18n } from '@kbn/i18n'; -import { INTERNAL_FEATURE_FLAGS } from '../../../common/constants'; import { CloudSummarySection } from './dashboard_sections/cloud_summary_section'; import { CloudPosturePageTitle } from '../../components/cloud_posture_page_title'; import { CloudPosturePage } from '../../components/cloud_posture_page'; import { DASHBOARD_CONTAINER } from './test_subjects'; -import { SummarySection } from './dashboard_sections/summary_section'; -import { BenchmarksSection } from './dashboard_sections/benchmarks_section'; import { useComplianceDashboardDataApi } from '../../common/api'; import { useCspSetupStatusApi } from '../../common/api/use_setup_status_api'; import { NoFindingsStates } from '../../components/no_findings_states'; @@ -51,21 +48,12 @@ export const ComplianceDashboard = () => { margin-right: auto; `} > - {INTERNAL_FEATURE_FLAGS.showNewDashboard ? ( - <> - - - - - - ) : ( - <> - - - - - - )} + <> + + + + + ); diff --git a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/benchmarks_section.tsx b/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/benchmarks_section.tsx deleted file mode 100644 index cbdbbba49d4fd..0000000000000 --- a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/benchmarks_section.tsx +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React from 'react'; -import { EuiFlexItem, EuiPanel, EuiSpacer, EuiFlexGroup, useEuiTheme } from '@elastic/eui'; -import { PartitionElementEvent } from '@elastic/charts'; -import { i18n } from '@kbn/i18n'; -import { CloudPostureScoreChart } from '../compliance_charts/cloud_posture_score_chart'; -import { ChartPanel } from '../../../components/chart_panel'; -import type { ComplianceDashboardData, Evaluation } from '../../../../common/types'; -import { RisksTable } from '../compliance_charts/risks_table'; -import { RULE_FAILED } from '../../../../common/constants'; -import { useNavigateFindings } from '../../../common/hooks/use_navigate_findings'; -import { ClusterDetailsBox } from './cluster_details_box'; - -const cardHeight = 300; - -export const BenchmarksSection = ({ - complianceData, -}: { - complianceData: ComplianceDashboardData; -}) => { - const { euiTheme } = useEuiTheme(); - const navToFindings = useNavigateFindings(); - - const handleElementClick = (clusterId: string, elements: PartitionElementEvent[]) => { - const [element] = elements; - const [layerValue] = element; - const evaluation = layerValue[0].groupByRollup as Evaluation; - - navToFindings({ cluster_id: clusterId, 'result.evaluation': evaluation }); - }; - - const handleCellClick = (clusterId: string, ruleSection: string) => { - navToFindings({ - cluster_id: clusterId, - 'rule.section': ruleSection, - 'result.evaluation': RULE_FAILED, - }); - }; - - const handleViewAllClick = (clusterId: string) => { - navToFindings({ cluster_id: clusterId, 'result.evaluation': RULE_FAILED }); - }; - - return ( - <> - {complianceData.clusters.map((cluster) => ( - - - - - - - - - - handleElementClick(cluster.meta.clusterId, elements) - } - /> - - - - - - handleCellClick(cluster.meta.clusterId, resourceTypeName) - } - onViewAllClick={() => handleViewAllClick(cluster.meta.clusterId)} - /> - - - - - - - ))} - - ); -}; diff --git a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/cloud_benchmarks_section.tsx b/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/cloud_benchmarks_section.tsx index 11c836080a23c..26bce94cd49a3 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/cloud_benchmarks_section.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/cloud_benchmarks_section.tsx @@ -7,9 +7,7 @@ import React from 'react'; import { EuiFlexItem, EuiFlexGroup, useEuiTheme, EuiTitle } from '@elastic/eui'; -import { PartitionElementEvent } from '@elastic/charts'; import { FormattedMessage } from '@kbn/i18n-react'; -import { ChartPanel } from '../../../components/chart_panel'; import { CloudPostureScoreChart } from '../compliance_charts/cloud_posture_score_chart'; import type { ComplianceDashboardData, Evaluation } from '../../../../common/types'; import { RisksTable } from '../compliance_charts/risks_table'; @@ -26,11 +24,7 @@ export const CloudBenchmarksSection = ({ const { euiTheme } = useEuiTheme(); const navToFindings = useNavigateFindings(); - const handleElementClick = (clusterId: string, elements: PartitionElementEvent[]) => { - const [element] = elements; - const [layerValue] = element; - const evaluation = layerValue[0].groupByRollup as Evaluation; - + const handleEvalCounterClick = (clusterId: string, evaluation: Evaluation) => { navToFindings({ cluster_id: clusterId, 'result.evaluation': evaluation }); }; @@ -54,7 +48,7 @@ export const CloudBenchmarksSection = ({ style={{ borderBottom: euiTheme.border.thick, borderBottomColor: euiTheme.colors.text, - marginBottom: euiTheme.size.l, + marginBottom: euiTheme.size.m, paddingBottom: euiTheme.size.s, }} > @@ -105,20 +99,28 @@ export const CloudBenchmarksSection = ({ - +
- handleElementClick(cluster.meta.clusterId, elements) + onEvalCounterClick={(evaluation) => + handleEvalCounterClick(cluster.meta.clusterId, evaluation) } /> - +
- +
@@ -126,7 +128,7 @@ export const CloudBenchmarksSection = ({ } onViewAllClick={() => handleViewAllClick(cluster.meta.clusterId)} /> - +
))} diff --git a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/cloud_summary_section.tsx b/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/cloud_summary_section.tsx index 02cf6124280b9..005d174079f02 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/cloud_summary_section.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/cloud_summary_section.tsx @@ -7,7 +7,6 @@ import React, { useMemo } from 'react'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import { PartitionElementEvent } from '@elastic/charts'; import { i18n } from '@kbn/i18n'; import { FlexItemGrowSize } from '@elastic/eui/src/components/flex/flex_item'; import { DASHBOARD_COUNTER_CARDS } from '../test_subjects'; @@ -23,13 +22,6 @@ import { } from '../../../common/hooks/use_navigate_findings'; import { RULE_FAILED } from '../../../../common/constants'; -const defaultHeight = 360; - -// TODO: limit this to desktop media queries only -const summarySectionWrapperStyle = { - height: defaultHeight, -}; - export const dashboardColumnsGrow: Record = { first: 3, second: 8, @@ -44,11 +36,7 @@ export const CloudSummarySection = ({ const navToFindings = useNavigateFindings(); const navToFindingsByResource = useNavigateFindingsByResource(); - const handleElementClick = (elements: PartitionElementEvent[]) => { - const [element] = elements; - const [layerValue] = element; - const evaluation = layerValue[0].groupByRollup as Evaluation; - + const handleEvalCounterClick = (evaluation: Evaluation) => { navToFindings({ 'result.evaluation': evaluation }); }; @@ -67,33 +55,31 @@ export const CloudSummarySection = ({ () => [ { id: DASHBOARD_COUNTER_CARDS.CLUSTERS_EVALUATED, - title: i18n.translate( + description: i18n.translate( 'xpack.csp.dashboard.summarySection.counterCard.clustersEvaluatedDescription', { defaultMessage: 'Clusters Evaluated' } ), - description: , + title: , }, { id: DASHBOARD_COUNTER_CARDS.RESOURCES_EVALUATED, - title: i18n.translate( + description: i18n.translate( 'xpack.csp.dashboard.summarySection.counterCard.resourcesEvaluatedDescription', { defaultMessage: 'Resources Evaluated' } ), - description: ( - - ), + title: , onClick: () => { navToFindingsByResource(); }, }, { id: DASHBOARD_COUNTER_CARDS.FAILING_FINDINGS, - title: i18n.translate( + description: i18n.translate( 'xpack.csp.dashboard.summarySection.counterCard.failingFindingsDescription', { defaultMessage: 'Failing Findings' } ), - description: , - descriptionColor: complianceData.stats.totalFailed > 0 ? 'danger' : 'text', + title: , + titleColor: complianceData.stats.totalFailed > 0 ? 'danger' : 'text', onClick: () => { navToFindings({ 'result.evaluation': RULE_FAILED }); }, @@ -109,7 +95,7 @@ export const CloudSummarySection = ({ ); return ( - + {counters.map((counter) => ( @@ -129,15 +115,16 @@ export const CloudSummarySection = ({ id="cloud_posture_score_chart" data={complianceData.stats} trend={complianceData.trend} - partitionOnElementClick={handleElementClick} + onEvalCounterClick={handleEvalCounterClick} /> { + const { euiTheme } = useEuiTheme(); const navToFindings = useNavigateFindings(); const shortId = cluster.meta.clusterId.slice(0, 6); @@ -40,7 +41,7 @@ export const ClusterDetailsBox = ({ cluster }: { cluster: Cluster }) => { }; return ( - + { } > handleClusterTitleClick(cluster.meta.clusterId)} color="text"> - -

+ +
{ shortId, }} /> -
- +

+ - - - - + + {INTERNAL_FEATURE_FLAGS.showManageRulesMock && ( diff --git a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/summary_section.tsx b/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/summary_section.tsx deleted file mode 100644 index e05a844777d22..0000000000000 --- a/x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/summary_section.tsx +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React from 'react'; -import { EuiFlexGrid, EuiFlexItem } from '@elastic/eui'; -import { PartitionElementEvent } from '@elastic/charts'; -import { i18n } from '@kbn/i18n'; -import { ChartPanel } from '../../../components/chart_panel'; -import { CloudPostureScoreChart } from '../compliance_charts/cloud_posture_score_chart'; -import type { ComplianceDashboardData, Evaluation } from '../../../../common/types'; -import { RisksTable } from '../compliance_charts/risks_table'; -import { CasesTable } from '../compliance_charts/cases_table'; -import { useNavigateFindings } from '../../../common/hooks/use_navigate_findings'; -import { RULE_FAILED } from '../../../../common/constants'; - -const defaultHeight = 360; - -// TODO: limit this to desktop media queries only -const summarySectionWrapperStyle = { - height: defaultHeight, -}; - -export const SummarySection = ({ complianceData }: { complianceData: ComplianceDashboardData }) => { - const navToFindings = useNavigateFindings(); - - const handleElementClick = (elements: PartitionElementEvent[]) => { - const [element] = elements; - const [layerValue] = element; - const evaluation = layerValue[0].groupByRollup as Evaluation; - - navToFindings({ 'result.evaluation': evaluation }); - }; - - const handleCellClick = (ruleSection: string) => { - navToFindings({ - 'rule.section': ruleSection, - 'result.evaluation': RULE_FAILED, - }); - }; - - const handleViewAllClick = () => { - navToFindings({ 'result.evaluation': RULE_FAILED }); - }; - - return ( - - - - - - - - - - - - - - - - - - ); -}; diff --git a/x-pack/plugins/cloud_security_posture/public/plugin.tsx b/x-pack/plugins/cloud_security_posture/public/plugin.tsx index cf281884f9d91..3644bd19d49b7 100755 --- a/x-pack/plugins/cloud_security_posture/public/plugin.tsx +++ b/x-pack/plugins/cloud_security_posture/public/plugin.tsx @@ -78,9 +78,11 @@ export class CspPlugin ( - - - +
+ + + +
), diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index c8cbb70072d23..6fb0a0ab1c247 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -9853,16 +9853,11 @@ "xpack.csp.cspEvaluationBadge.failLabel": "Échec", "xpack.csp.cspEvaluationBadge.passLabel": "Réussite", "xpack.csp.cspSettings.rules": "Règles de sécurité du CSP - ", - "xpack.csp.dashboard.benchmarkSection.complianceScorePanelTitle": "Score de conformité", - "xpack.csp.dashboard.benchmarkSection.failedFindingsPanelTitle": "Échec des résultats", - "xpack.csp.dashboard.casesTable.placeholderTitle": "Bientôt disponible", "xpack.csp.dashboard.cspPageTemplate.pageTitle": "Niveau du cloud", "xpack.csp.dashboard.risksTable.cisSectionColumnLabel": "Section CIS", "xpack.csp.dashboard.risksTable.findingsColumnLabel": "Résultats", "xpack.csp.dashboard.risksTable.viewAllButtonTitle": "Afficher tous les échecs des résultats", "xpack.csp.dashboard.summarySection.cloudPostureScorePanelTitle": "Score du niveau du cloud", - "xpack.csp.dashboard.summarySection.failedFindingsPanelTitle": "Échec des résultats", - "xpack.csp.dashboard.summarySection.openCasesPanelTitle": "Cas ouverts", "xpack.csp.expandColumnDescriptionLabel": "Développer", "xpack.csp.expandColumnNameLabel": "Développer", "xpack.csp.findings.distributionBar.totalFailedLabel": "Échec des résultats", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 4d7f13bdd9aaa..e6e439aa4196a 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -9840,16 +9840,11 @@ "xpack.csp.cspEvaluationBadge.failLabel": "失敗", "xpack.csp.cspEvaluationBadge.passLabel": "合格", "xpack.csp.cspSettings.rules": "CSPセキュリティルール - ", - "xpack.csp.dashboard.benchmarkSection.complianceScorePanelTitle": "コンプライアンススコア", - "xpack.csp.dashboard.benchmarkSection.failedFindingsPanelTitle": "失敗した調査結果", - "xpack.csp.dashboard.casesTable.placeholderTitle": "まもなくリリース", "xpack.csp.dashboard.cspPageTemplate.pageTitle": "クラウド態勢", "xpack.csp.dashboard.risksTable.cisSectionColumnLabel": "CISセクション", "xpack.csp.dashboard.risksTable.findingsColumnLabel": "調査結果", "xpack.csp.dashboard.risksTable.viewAllButtonTitle": "すべてのフィールド調査結果を表示", "xpack.csp.dashboard.summarySection.cloudPostureScorePanelTitle": "クラウド態勢スコア", - "xpack.csp.dashboard.summarySection.failedFindingsPanelTitle": "失敗した調査結果", - "xpack.csp.dashboard.summarySection.openCasesPanelTitle": "ケースを開く", "xpack.csp.expandColumnDescriptionLabel": "拡張", "xpack.csp.expandColumnNameLabel": "拡張", "xpack.csp.findings.distributionBar.totalFailedLabel": "失敗した調査結果", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 0d1d73517f78b..539b91c88a933 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -9858,16 +9858,11 @@ "xpack.csp.cspEvaluationBadge.failLabel": "失败", "xpack.csp.cspEvaluationBadge.passLabel": "通过", "xpack.csp.cspSettings.rules": "CSP 安全规则 - ", - "xpack.csp.dashboard.benchmarkSection.complianceScorePanelTitle": "合规性分数", - "xpack.csp.dashboard.benchmarkSection.failedFindingsPanelTitle": "失败的结果", - "xpack.csp.dashboard.casesTable.placeholderTitle": "即将推出", "xpack.csp.dashboard.cspPageTemplate.pageTitle": "云态势", "xpack.csp.dashboard.risksTable.cisSectionColumnLabel": "CIS 部分", "xpack.csp.dashboard.risksTable.findingsColumnLabel": "结果", "xpack.csp.dashboard.risksTable.viewAllButtonTitle": "查看所有失败的结果", "xpack.csp.dashboard.summarySection.cloudPostureScorePanelTitle": "云态势分数", - "xpack.csp.dashboard.summarySection.failedFindingsPanelTitle": "失败的结果", - "xpack.csp.dashboard.summarySection.openCasesPanelTitle": "未结案例", "xpack.csp.expandColumnDescriptionLabel": "展开", "xpack.csp.expandColumnNameLabel": "展开", "xpack.csp.findings.distributionBar.totalFailedLabel": "失败的结果", From 19c4e198132ef8385ba23688967f2ded2a3f2888 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 14 Nov 2022 17:40:57 +0000 Subject: [PATCH 14/16] skip flaky suite (#145134) --- x-pack/test/api_integration/apis/security/privileges_basic.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/security/privileges_basic.ts b/x-pack/test/api_integration/apis/security/privileges_basic.ts index ba4fefd9ae691..2e1b01a6bc715 100644 --- a/x-pack/test/api_integration/apis/security/privileges_basic.ts +++ b/x-pack/test/api_integration/apis/security/privileges_basic.ts @@ -13,7 +13,8 @@ export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); describe('Privileges', () => { - describe('GET /api/security/privileges', () => { + // FLAKY: https://github.com/elastic/kibana/issues/145134 + describe.skip('GET /api/security/privileges', () => { it('should return a privilege map with all known privileges, without actions', async () => { // If you're adding a privilege to the following, that's great! // If you're removing a privilege, this breaks backwards compatibility From 90f38a6388e148fd029c6707c0b6ce4860b86a77 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 14 Nov 2022 19:29:03 +0100 Subject: [PATCH 15/16] [Synthetics] Update duration chart legend labels (#145116) --- .../configurations/lens_attributes.test.ts | 4 +-- .../configurations/lens_attributes.ts | 27 ++++++-------- .../monitor_summary/duration_trend.tsx | 35 ++++++++++++++++--- 3 files changed, 44 insertions(+), 22 deletions(-) diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.test.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.test.ts index aaef957422172..bce05a998fafe 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.test.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.test.ts @@ -108,7 +108,7 @@ describe('Lens Attribute', () => { to: 'now', }, dataView: mockDataView, - name: 'ux-series-1', + name: 'Page load time', breakdown: 'percentile', reportDefinitions: {}, selectedMetricField: 'transaction.duration.us', @@ -139,7 +139,7 @@ describe('Lens Attribute', () => { query: 'transaction.type: page-load and processor.event: transaction', }, isBucketed: false, - label: `${rank} percentile of page load time`, + label: 'Page load time', operationType: 'percentile', params: { percentile: Number(rank.slice(0, 2)), diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.ts index e029e8479fde5..c93b6ef2cadd0 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.ts @@ -331,7 +331,6 @@ export class LensAttributes { columnType, columnFilter, operationType, - shortLabel, }: { sourceField: string; columnType?: string; @@ -339,7 +338,6 @@ export class LensAttributes { operationType?: SupportedOperations | 'last_value'; label?: string; seriesConfig: SeriesConfig; - shortLabel?: boolean; }) { if (columnType === 'operation' || operationType) { if ( @@ -352,7 +350,6 @@ export class LensAttributes { label, seriesConfig, columnFilter, - shortLabel, }); } if (operationType === 'last_value') { @@ -365,7 +362,7 @@ export class LensAttributes { }); } if (operationType?.includes('th')) { - return this.getPercentileNumberColumn(sourceField, operationType, seriesConfig!); + return this.getPercentileNumberColumn(sourceField, operationType, seriesConfig!, label); } } return this.getNumberRangeColumn(sourceField, seriesConfig!, label); @@ -402,14 +399,12 @@ export class LensAttributes { seriesConfig, operationType, columnFilter, - shortLabel, }: { sourceField: string; operationType: SupportedOperations; label?: string; seriesConfig: SeriesConfig; columnFilter?: ColumnFilter; - shortLabel?: boolean; }): | MinIndexPatternColumn | MaxIndexPatternColumn @@ -469,14 +464,17 @@ export class LensAttributes { getPercentileNumberColumn( sourceField: string, percentileValue: string, - seriesConfig: SeriesConfig + seriesConfig: SeriesConfig, + label?: string ): PercentileIndexPatternColumn { return { ...buildNumberColumn(sourceField), - label: i18n.translate('xpack.observability.expView.columns.label', { - defaultMessage: '{percentileValue} percentile of {sourceField}', - values: { sourceField: seriesConfig.labels[sourceField]?.toLowerCase(), percentileValue }, - }), + label: + label ?? + i18n.translate('xpack.observability.expView.columns.label', { + defaultMessage: '{percentileValue} percentile of {sourceField}', + values: { sourceField: seriesConfig.labels[sourceField]?.toLowerCase(), percentileValue }, + }), operationType: 'percentile', params: getPercentileParam(percentileValue), customLabel: true, @@ -552,7 +550,6 @@ export class LensAttributes { colIndex, layerId, metricOption, - shortLabel, }: { sourceField: string; metricOption?: MetricOption; @@ -561,7 +558,6 @@ export class LensAttributes { layerId: string; layerConfig: LayerConfig; colIndex?: number; - shortLabel?: boolean; }) { const { breakdown, seriesConfig } = layerConfig; const fieldMetaInfo = this.getFieldMeta(sourceField, layerConfig, metricOption); @@ -614,7 +610,8 @@ export class LensAttributes { ...this.getPercentileNumberColumn( fieldName, operationType || PERCENTILE_RANKS[0], - seriesConfig! + seriesConfig!, + label || columnLabel ), filter: colIndex !== undefined ? columnFilters?.[colIndex] : undefined, }; @@ -628,7 +625,6 @@ export class LensAttributes { operationType, label: label || columnLabel, seriesConfig: layerConfig.seriesConfig, - shortLabel, }); } if (operationType === 'unique_count' || fieldType === 'string') { @@ -745,7 +741,6 @@ export class LensAttributes { return this.getColumnBasedOnType({ layerConfig, layerId, - shortLabel: true, label: item.label, sourceField: REPORT_METRIC_FIELD, metricOption: item, diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_details/monitor_summary/duration_trend.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_details/monitor_summary/duration_trend.tsx index e104eeb87abdd..000706140d267 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_details/monitor_summary/duration_trend.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_details/monitor_summary/duration_trend.tsx @@ -7,6 +7,7 @@ import React from 'react'; import { useKibana } from '@kbn/kibana-react-plugin/public'; +import { i18n } from '@kbn/i18n'; import { ClientPluginsStart } from '../../../../../plugin'; import { useMonitorQueryId } from '../hooks/use_monitor_query_id'; import { useSelectedLocation } from '../hooks/use_selected_location'; @@ -24,8 +25,6 @@ export const MonitorDurationTrend = (props: MonitorDurationTrendProps) => { const monitorId = useMonitorQueryId(); const selectedLocation = useSelectedLocation(); - const metricsToShow = ['min', 'max', 'median', '25th', '75th']; - if (!selectedLocation) { return null; } @@ -34,10 +33,10 @@ export const MonitorDurationTrend = (props: MonitorDurationTrendProps) => { ({ + attributes={Object.keys(metricsToShow).map((metric) => ({ dataType: 'synthetics', time: props, - name: metric + ' Series', + name: metricsToShow[metric], selectedMetricField: 'monitor.duration.us', reportDefinitions: { 'monitor.id': [monitorId], @@ -49,3 +48,31 @@ export const MonitorDurationTrend = (props: MonitorDurationTrendProps) => { /> ); }; + +const MIN_LABEL = i18n.translate('xpack.synthetics.durationTrend.min', { + defaultMessage: 'Min', +}); + +const MAX_LABEL = i18n.translate('xpack.synthetics.durationTrend.max', { + defaultMessage: 'Max', +}); + +const MEDIAN_LABEL = i18n.translate('xpack.synthetics.durationTrend.median', { + defaultMessage: 'Median', +}); + +const PERCENTILE_25_LABEL = i18n.translate('xpack.synthetics.durationTrend.percentile25', { + defaultMessage: '25th', +}); + +const PERCENTILE_75_LABEL = i18n.translate('xpack.synthetics.durationTrend.percentile75', { + defaultMessage: '75th', +}); + +const metricsToShow: Record = { + max: MAX_LABEL, + '75th': PERCENTILE_75_LABEL, + median: MEDIAN_LABEL, + '25th': PERCENTILE_25_LABEL, + min: MIN_LABEL, +}; From bebcd354d36f59464f1899ce80c47af673c024a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Mon, 14 Nov 2022 19:42:24 +0100 Subject: [PATCH 16/16] [Uptime] Fix monitor alert label (#145112) ## Summary Closes #143895 Updates the label for the status check labels to reflect what the alert does. Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../__snapshots__/down_number_select.test.tsx.snap | 4 ++-- .../legacy_uptime/components/overview/alerts/translations.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/synthetics/public/legacy_uptime/components/overview/alerts/monitor_expressions/__snapshots__/down_number_select.test.tsx.snap b/x-pack/plugins/synthetics/public/legacy_uptime/components/overview/alerts/monitor_expressions/__snapshots__/down_number_select.test.tsx.snap index eb417a875de9e..527247ec9f072 100644 --- a/x-pack/plugins/synthetics/public/legacy_uptime/components/overview/alerts/monitor_expressions/__snapshots__/down_number_select.test.tsx.snap +++ b/x-pack/plugins/synthetics/public/legacy_uptime/components/overview/alerts/monitor_expressions/__snapshots__/down_number_select.test.tsx.snap @@ -16,7 +16,7 @@ exports[`DownNoExpressionSelect component should renders against props 1`] = ` - matching monitors are down > + matching monitors are down >= } data-test-subj="xpack.synthetics.alerts.monitorStatus.numTimesExpression" - description="matching monitors are down >" + description="matching monitors are down >=" id="ping-count" value="5 times" /> diff --git a/x-pack/plugins/synthetics/public/legacy_uptime/components/overview/alerts/translations.ts b/x-pack/plugins/synthetics/public/legacy_uptime/components/overview/alerts/translations.ts index 0580528b6b38c..de9da94460338 100644 --- a/x-pack/plugins/synthetics/public/legacy_uptime/components/overview/alerts/translations.ts +++ b/x-pack/plugins/synthetics/public/legacy_uptime/components/overview/alerts/translations.ts @@ -117,14 +117,14 @@ export const ENTER_NUMBER_OF_DOWN_COUNTS = i18n.translate( export const MATCHING_MONITORS_DOWN = i18n.translate( 'xpack.synthetics.alerts.monitorStatus.numTimesExpression.matchingMonitors.description', { - defaultMessage: 'matching monitors are down >', + defaultMessage: 'matching monitors are down >=', } ); export const ANY_MONITOR_DOWN = i18n.translate( 'xpack.synthetics.alerts.monitorStatus.numTimesExpression.anyMonitors.description', { - defaultMessage: 'any monitor is down >', + defaultMessage: 'any monitor is down >=', } );