From 17ed210a421eebc1296bf2797a8dadc97916d8e4 Mon Sep 17 00:00:00 2001 From: Paulo Henrique Date: Mon, 10 Apr 2023 10:28:39 -0300 Subject: [PATCH 001/117] [Cloud Posture] Add Coming soon options for KSPM (#154513) --- .../common/constants.ts | 2 + .../public/assets/icons/cis_aks_logo.svg | 5 +++ .../public/assets/icons/cis_gke_logo.svg | 21 ++++++++++ .../public/common/constants.ts | 41 +++++++++++++++++-- .../policy_template_form.test.tsx | 10 ++--- 5 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 x-pack/plugins/cloud_security_posture/public/assets/icons/cis_aks_logo.svg create mode 100644 x-pack/plugins/cloud_security_posture/public/assets/icons/cis_gke_logo.svg diff --git a/x-pack/plugins/cloud_security_posture/common/constants.ts b/x-pack/plugins/cloud_security_posture/common/constants.ts index 37453e1364f8d..fd99edd8705e7 100644 --- a/x-pack/plugins/cloud_security_posture/common/constants.ts +++ b/x-pack/plugins/cloud_security_posture/common/constants.ts @@ -63,6 +63,8 @@ export const CSP_RULE_TEMPLATE_SAVED_OBJECT_TYPE = 'csp-rule-template'; export const CLOUDBEAT_VANILLA = 'cloudbeat/cis_k8s'; export const CLOUDBEAT_EKS = 'cloudbeat/cis_eks'; +export const CLOUDBEAT_AKS = 'cloudbeat/cis_aks'; +export const CLOUDBEAT_GKE = 'cloudbeat/cis_gke'; export const CLOUDBEAT_AWS = 'cloudbeat/cis_aws'; export const CLOUDBEAT_GCP = 'cloudbeat/cis_gcp'; export const CLOUDBEAT_AZURE = 'cloudbeat/cis_azure'; diff --git a/x-pack/plugins/cloud_security_posture/public/assets/icons/cis_aks_logo.svg b/x-pack/plugins/cloud_security_posture/public/assets/icons/cis_aks_logo.svg new file mode 100644 index 0000000000000..f81d0ae7d0370 --- /dev/null +++ b/x-pack/plugins/cloud_security_posture/public/assets/icons/cis_aks_logo.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/x-pack/plugins/cloud_security_posture/public/assets/icons/cis_gke_logo.svg b/x-pack/plugins/cloud_security_posture/public/assets/icons/cis_gke_logo.svg new file mode 100644 index 0000000000000..6dd28d61ebf0d --- /dev/null +++ b/x-pack/plugins/cloud_security_posture/public/assets/icons/cis_gke_logo.svg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/x-pack/plugins/cloud_security_posture/public/common/constants.ts b/x-pack/plugins/cloud_security_posture/public/common/constants.ts index 54e0b7467069a..185cd19f200cf 100644 --- a/x-pack/plugins/cloud_security_posture/public/common/constants.ts +++ b/x-pack/plugins/cloud_security_posture/public/common/constants.ts @@ -20,9 +20,13 @@ import { VULN_MGMT_POLICY_TEMPLATE, CLOUDBEAT_VULN_MGMT_GCP, CLOUDBEAT_VULN_MGMT_AZURE, + CLOUDBEAT_AKS, + CLOUDBEAT_GKE, } from '../../common/constants'; import eksLogo from '../assets/icons/cis_eks_logo.svg'; +import aksLogo from '../assets/icons/cis_aks_logo.svg'; +import gkeLogo from '../assets/icons/cis_gke_logo.svg'; export const statusColors = { passed: euiThemeVars.euiColorVis0, @@ -48,7 +52,7 @@ export interface CloudPostureIntegrationProps { name: string; shortName: string; options: Array<{ - type: PostureInput; + type: PostureInput | typeof CLOUDBEAT_AKS | typeof CLOUDBEAT_GKE; name: string; benchmark: string; disabled?: boolean; @@ -119,7 +123,7 @@ export const cloudPostureIntegrations: CloudPostureIntegrations = { { type: CLOUDBEAT_VANILLA, name: i18n.translate('xpack.csp.kspmIntegration.vanillaOption.nameTitle', { - defaultMessage: 'Self-Managed/Vanilla Kubernetes', + defaultMessage: 'Self-Managed', }), benchmark: i18n.translate('xpack.csp.kspmIntegration.vanillaOption.benchmarkTitle', { defaultMessage: 'CIS Kubernetes', @@ -129,12 +133,43 @@ export const cloudPostureIntegrations: CloudPostureIntegrations = { { type: CLOUDBEAT_EKS, name: i18n.translate('xpack.csp.kspmIntegration.eksOption.nameTitle', { - defaultMessage: 'EKS (Elastic Kubernetes Service)', + defaultMessage: 'EKS', }), benchmark: i18n.translate('xpack.csp.kspmIntegration.eksOption.benchmarkTitle', { defaultMessage: 'CIS EKS', }), icon: eksLogo, + tooltip: i18n.translate('xpack.csp.kspmIntegration.eksOption.tooltipContent', { + defaultMessage: 'Elastic Kubernetes Service', + }), + }, + { + type: CLOUDBEAT_AKS, + name: i18n.translate('xpack.csp.kspmIntegration.aksOption.nameTitle', { + defaultMessage: 'AKS', + }), + benchmark: i18n.translate('xpack.csp.kspmIntegration.aksOption.benchmarkTitle', { + defaultMessage: 'CIS AKS', + }), + disabled: true, + icon: aksLogo, + tooltip: i18n.translate('xpack.csp.kspmIntegration.aksOption.tooltipContent', { + defaultMessage: 'Azure Kubernetes Service - Coming soon', + }), + }, + { + type: CLOUDBEAT_GKE, + name: i18n.translate('xpack.csp.kspmIntegration.gkeOption.nameTitle', { + defaultMessage: 'GKE', + }), + benchmark: i18n.translate('xpack.csp.kspmIntegration.gkeOption.benchmarkTitle', { + defaultMessage: 'CIS GKE', + }), + disabled: true, + icon: gkeLogo, + tooltip: i18n.translate('xpack.csp.kspmIntegration.gkeOption.tooltipContent', { + defaultMessage: 'Google Kubernetes Engine - Coming soon', + }), }, ], }, diff --git a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/policy_template_form.test.tsx b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/policy_template_form.test.tsx index 7f3b7dc51319e..3427c25e49c19 100644 --- a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/policy_template_form.test.tsx +++ b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/policy_template_form.test.tsx @@ -101,8 +101,8 @@ describe('', () => { it('renders KSPM input selector', () => { const { getByLabelText } = render(); - const option1 = getByLabelText('Self-Managed/Vanilla Kubernetes'); - const option2 = getByLabelText('EKS (Elastic Kubernetes Service)'); + const option1 = getByLabelText('Self-Managed'); + const option2 = getByLabelText('EKS'); expect(option1).toBeInTheDocument(); expect(option2).toBeInTheDocument(); @@ -116,7 +116,7 @@ describe('', () => { const eksPolicy = getMockPolicyEKS(); const { getByLabelText } = render(); - const option = getByLabelText('EKS (Elastic Kubernetes Service)'); + const option = getByLabelText('EKS'); userEvent.click(option); // Listen to the 2nd triggered by the test. @@ -148,8 +148,8 @@ describe('', () => { ); - const option1 = getByLabelText('Self-Managed/Vanilla Kubernetes'); - const option2 = getByLabelText('EKS (Elastic Kubernetes Service)'); + const option1 = getByLabelText('Self-Managed'); + const option2 = getByLabelText('EKS'); expect(option1).toBeInTheDocument(); expect(option2).toBeInTheDocument(); From 1f8dc16a0eac616aae8e79077b03ed7b9b8c3be3 Mon Sep 17 00:00:00 2001 From: Saarika Bhasi <55930906+saarikabhasi@users.noreply.github.com> Date: Mon, 10 Apr 2023 10:45:01 -0400 Subject: [PATCH 002/117] [Enterprise Search] [Search application] Adding filter by field type to Schema Page (#154466) ## Summary Adding Filter by field type component to Schema page and display callout when showing field conflicts [thread](https://elastic.slack.com/archives/C02U50QNEAG/p1680627458702419?thread_ts=1680288487.778129&cid=C02U50QNEAG) ### Screen Recording https://user-images.githubusercontent.com/55930906/230146041-4978335d-ad6e-4594-8e01-4e688a580b5a.mov #### [Edit] Updated Popover clear all button with eraser Popover with clear all button with eraser --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../components/engine/engine_schema.tsx | 187 +++++++++++++++--- .../plugins/enterprise_search/tsconfig.json | 1 + 2 files changed, 166 insertions(+), 22 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_schema.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_schema.tsx index 76bae7ec6ee1f..b44d644b89336 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_schema.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_schema.tsx @@ -13,19 +13,31 @@ import { EuiBadge, EuiBasicTable, EuiBasicTableColumn, + EuiButton, EuiButtonEmpty, EuiCallOut, + EuiFilterButton, EuiFlexGroup, + EuiFlexItem, EuiIcon, EuiLink, EuiPanel, + EuiPopover, + EuiPopoverFooter, + EuiPopoverTitle, + EuiSelectable, EuiSwitch, EuiText, } from '@elastic/eui'; + +import { ES_FIELD_TYPES } from '@kbn/field-types'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; +import { FieldIcon } from '@kbn/react-field'; + import { SchemaField } from '../../../../../common/types/engines'; + import { docLinks } from '../../../shared/doc_links'; import { generateEncodedPath } from '../../../shared/encode_path_params'; import { KibanaLogic } from '../../../shared/kibana'; @@ -36,7 +48,6 @@ import { EnterpriseSearchEnginesPageTemplate } from '../layout/engines_page_temp import { EngineIndicesLogic } from './engine_indices_logic'; import { EngineViewLogic } from './engine_view_logic'; -import { FieldIcon } from './field_icon'; const SchemaFieldDetails: React.FC<{ schemaField: SchemaField }> = ({ schemaField }) => { const { navigateToUrl } = useValues(KibanaLogic); @@ -147,20 +158,49 @@ export const EngineSchema: React.FC = () => { const [onlyShowConflicts, setOnlyShowConflicts] = useState(false); const { isLoadingEngineSchema, schemaFields } = useValues(EngineViewLogic); const { fetchEngineSchema } = useActions(EngineViewLogic); + + const [isFilterByPopoverOpen, setIsFilterByPopoverOpen] = useState(false); const [itemIdToExpandedRowMap, setItemIdToExpandedRowMap] = useState>( {} ); + // get all the elasticsearch Field Types + const esFieldTypes = Object.values(ES_FIELD_TYPES).map((fieldDataTypeName) => ({ + checked: undefined, + label: fieldDataTypeName.toString(), + })); + const [selectedEsFieldTypes, setSelectedEsFieldTypes] = useState(esFieldTypes); const toggleOnlyShowConflicts = useCallback(() => { setOnlyShowConflicts(!onlyShowConflicts); setItemIdToExpandedRowMap({}); }, [onlyShowConflicts]); - const filteredSchemaFields = useMemo(() => { + // update filteredDataTypes as filter by field types are selected + const filteredDataTypes = useMemo(() => { + const selectedDataTypes = selectedEsFieldTypes + .filter((option) => option.checked === 'on') + .map((option) => option.label); + return selectedDataTypes; + }, [selectedEsFieldTypes]); + + // return schema fields may be with conflicts + const schemaFieldsMaybeWithConflicts = useMemo(() => { if (onlyShowConflicts) return schemaFields.filter((field) => field.type === 'conflict'); return schemaFields; }, [onlyShowConflicts, schemaFields]); + const filteredSchemaFields = useMemo(() => { + if (filteredDataTypes.length > 0) + return schemaFieldsMaybeWithConflicts.filter((field) => + field.indices.some((i) => filteredDataTypes.includes(i.type)) + ); + return schemaFieldsMaybeWithConflicts; + }, [onlyShowConflicts, schemaFields, filteredDataTypes]); + + const totalConflictsHiddenByTypeFilters = onlyShowConflicts + ? schemaFieldsMaybeWithConflicts.length - filteredSchemaFields.length + : 0; + useEffect(() => { fetchEngineSchema({ engineName }); }, [engineName]); @@ -287,6 +327,21 @@ export const EngineSchema: React.FC = () => { width: '10%', }, ]; + const filterButton = ( + setIsFilterByPopoverOpen(!isFilterByPopoverOpen)} + numFilters={selectedEsFieldTypes.length} + hasActiveFilters={filteredDataTypes.length > 0} + numActiveFilters={filteredDataTypes.length} + isSelected={isFilterByPopoverOpen} + > + {i18n.translate('xpack.enterpriseSearch.content.engine.schema.filters', { + defaultMessage: 'Field types', + })} + + ); return ( { }} engineName={engineName} > - - - + + + + + + {i18n.translate('xpack.enterpriseSearch.content.engine.schema.filters.label', { + defaultMessage: 'Filter By', + })} + + + setIsFilterByPopoverOpen(false)} + panelPaddingSize="none" + anchorPosition="downCenter" + > + setSelectedEsFieldTypes(options)} + > + {(list, search) => ( +
+ {search} + {list} +
+ )} + + + + setSelectedEsFieldTypes(esFieldTypes)} + > + {i18n.translate( + 'xpack.enterpriseSearch.content.engine.schema.filters.clearAll', + { + defaultMessage: 'Clear all ', + } + )} + + + + + + + + + + {totalConflictsHiddenByTypeFilters > 0 && ( + + } + color="danger" + iconType="iInCircle" + > +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.engine.schema.filters.conflict.callout.subTitle', + { + defaultMessage: + 'In order to see all field conflicts you must clear your field filters', + } + )} +

+ setSelectedEsFieldTypes(esFieldTypes)}> + {i18n.translate( + 'xpack.enterpriseSearch.content.engine.schema.filters.conflict.callout.clearFilters', + { + defaultMessage: 'Clear filters ', + } + )} + + + )} - - + ); }; diff --git a/x-pack/plugins/enterprise_search/tsconfig.json b/x-pack/plugins/enterprise_search/tsconfig.json index ec465b9d78027..8d83fd25fdf84 100644 --- a/x-pack/plugins/enterprise_search/tsconfig.json +++ b/x-pack/plugins/enterprise_search/tsconfig.json @@ -56,5 +56,6 @@ "@kbn/datemath", "@kbn/expressions-plugin", "@kbn/react-field", + "@kbn/field-types", ] } From 5c759a1290ead0b9ca4372369b2999717e3d34e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cau=C3=AA=20Marcondes?= <55978943+cauemarcondes@users.noreply.github.com> Date: Mon, 10 Apr 2023 11:50:11 -0400 Subject: [PATCH 003/117] [Profiling] renaming CPU incl and CPU excl (#154560) Screenshot 2023-04-06 at 9 28 13 AM Screenshot 2023-04-06 at 9 28 23 AM Screenshot 2023-04-06 at 9 38 57 AM --- .../flamegraph/flamegraph_tooltip.tsx | 23 +++++-- .../components/flamegraph/tooltip_row.tsx | 4 +- ...get_impact_rows.ts => get_impact_rows.tsx} | 12 ++-- .../key_value_list.tsx | 6 +- .../shared/cpu_label_with_hint/index.tsx | 66 +++++++++++++++++++ .../components/topn_functions/index.tsx | 37 ++++------- .../translations/translations/fr-FR.json | 8 --- .../translations/translations/ja-JP.json | 8 --- .../translations/translations/zh-CN.json | 8 --- 9 files changed, 106 insertions(+), 66 deletions(-) rename x-pack/plugins/profiling/public/components/frame_information_window/{get_impact_rows.ts => get_impact_rows.tsx} (93%) create mode 100644 x-pack/plugins/profiling/public/components/shared/cpu_label_with_hint/index.tsx diff --git a/x-pack/plugins/profiling/public/components/flamegraph/flamegraph_tooltip.tsx b/x-pack/plugins/profiling/public/components/flamegraph/flamegraph_tooltip.tsx index e8ae3633ef8c4..2d42da4a5b621 100644 --- a/x-pack/plugins/profiling/public/components/flamegraph/flamegraph_tooltip.tsx +++ b/x-pack/plugins/profiling/public/components/flamegraph/flamegraph_tooltip.tsx @@ -22,6 +22,7 @@ import { calculateImpactEstimates } from '../../utils/calculate_impact_estimates import { asCost } from '../../utils/formatters/as_cost'; import { asPercentage } from '../../utils/formatters/as_percentage'; import { asWeight } from '../../utils/formatters/as_weight'; +import { CPULabelWithHint } from '../shared/cpu_label_with_hint'; import { TooltipRow } from './tooltip_row'; interface Props { @@ -86,9 +87,14 @@ export function FlameGraphTooltip({ {isRoot === false && ( <> + } value={impactEstimates.percentage} comparison={comparisonImpactEstimates?.percentage} formatValue={asPercentage} @@ -96,9 +102,14 @@ export function FlameGraphTooltip({ formatDifferenceAsPercentage /> + } value={impactEstimates.percentageNoChildren} comparison={comparisonImpactEstimates?.percentageNoChildren} showDifference diff --git a/x-pack/plugins/profiling/public/components/flamegraph/tooltip_row.tsx b/x-pack/plugins/profiling/public/components/flamegraph/tooltip_row.tsx index 3c7f0a413c388..5f3511169ecdc 100644 --- a/x-pack/plugins/profiling/public/components/flamegraph/tooltip_row.tsx +++ b/x-pack/plugins/profiling/public/components/flamegraph/tooltip_row.tsx @@ -19,7 +19,7 @@ export function TooltipRow({ formatValue, }: { value: number; - label: string; + label: string | React.ReactElement; comparison?: number; formatDifferenceAsPercentage: boolean; showDifference: boolean; @@ -58,7 +58,7 @@ export function TooltipRow({ - {label}: + {label}: diff --git a/x-pack/plugins/profiling/public/components/frame_information_window/get_impact_rows.ts b/x-pack/plugins/profiling/public/components/frame_information_window/get_impact_rows.tsx similarity index 93% rename from x-pack/plugins/profiling/public/components/frame_information_window/get_impact_rows.ts rename to x-pack/plugins/profiling/public/components/frame_information_window/get_impact_rows.tsx index 80fe7d1d2bb74..1a0b259fc359d 100644 --- a/x-pack/plugins/profiling/public/components/frame_information_window/get_impact_rows.ts +++ b/x-pack/plugins/profiling/public/components/frame_information_window/get_impact_rows.tsx @@ -6,12 +6,14 @@ */ import { i18n } from '@kbn/i18n'; +import React from 'react'; import { calculateImpactEstimates } from '../../utils/calculate_impact_estimates'; import { asCost } from '../../utils/formatters/as_cost'; import { asDuration } from '../../utils/formatters/as_duration'; import { asNumber } from '../../utils/formatters/as_number'; import { asPercentage } from '../../utils/formatters/as_percentage'; import { asWeight } from '../../utils/formatters/as_weight'; +import { CPULabelWithHint } from '../shared/cpu_label_with_hint'; export function getImpactRows({ countInclusive, @@ -48,17 +50,11 @@ export function getImpactRows({ const impactRows = [ { - label: i18n.translate( - 'xpack.profiling.flameGraphInformationWindow.percentageCpuTimeInclusiveLabel', - { defaultMessage: '% of CPU time' } - ), + label: , value: asPercentage(percentage), }, { - label: i18n.translate( - 'xpack.profiling.flameGraphInformationWindow.percentageCpuTimeExclusiveLabel', - { defaultMessage: '% of CPU time (excl. children)' } - ), + label: , value: asPercentage(percentageNoChildren), }, { diff --git a/x-pack/plugins/profiling/public/components/frame_information_window/key_value_list.tsx b/x-pack/plugins/profiling/public/components/frame_information_window/key_value_list.tsx index cbdf00dae7146..752be18349be1 100644 --- a/x-pack/plugins/profiling/public/components/frame_information_window/key_value_list.tsx +++ b/x-pack/plugins/profiling/public/components/frame_information_window/key_value_list.tsx @@ -9,7 +9,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiHorizontalRule } from '@elastic/eui'; import React from 'react'; interface Props { - rows: Array<{ label: string; value: React.ReactNode }>; + rows: Array<{ label: string | React.ReactNode; value: React.ReactNode }>; } export function KeyValueList({ rows }: Props) { @@ -19,7 +19,9 @@ export function KeyValueList({ rows }: Props) { <> - {row.label}: + + {row.label}: + {row.value} diff --git a/x-pack/plugins/profiling/public/components/shared/cpu_label_with_hint/index.tsx b/x-pack/plugins/profiling/public/components/shared/cpu_label_with_hint/index.tsx new file mode 100644 index 0000000000000..f0a339b65974f --- /dev/null +++ b/x-pack/plugins/profiling/public/components/shared/cpu_label_with_hint/index.tsx @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiIcon, + EuiIconProps, + EuiText, + EuiTextProps, + EuiToolTip, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +type CPUType = 'self' | 'total'; + +interface Props { + type: CPUType; + labelSize?: EuiTextProps['size']; + labelStyle?: EuiTextProps['style']; + iconSize?: EuiIconProps['size']; +} + +const CPULabelHintMap: Record = { + self: { + label: i18n.translate('xpack.profiling.cpu.self.label', { + defaultMessage: 'Self CPU', + }), + hint: i18n.translate('xpack.profiling.cpu.self.hint', { + defaultMessage: + 'Indicates how much CPU time was spent by the code in the function body, excluding the work done by functions that were called by it', + }), + }, + total: { + label: i18n.translate('xpack.profiling.cpu.total.label', { + defaultMessage: 'Total CPU', + }), + hint: i18n.translate('xpack.profiling.cpu.total.hint', { + defaultMessage: + 'Indicates how much CPU time was spent by the function and any functions called by it', + }), + }, +}; + +export function CPULabelWithHint({ iconSize, labelSize, labelStyle, type }: Props) { + const { label, hint } = CPULabelHintMap[type]; + + return ( + + + + {label} + + + + + + + + + ); +} diff --git a/x-pack/plugins/profiling/public/components/topn_functions/index.tsx b/x-pack/plugins/profiling/public/components/topn_functions/index.tsx index 020bcf4fdff0a..dc1796f07ebab 100644 --- a/x-pack/plugins/profiling/public/components/topn_functions/index.tsx +++ b/x-pack/plugins/profiling/public/components/topn_functions/index.tsx @@ -26,6 +26,7 @@ import { calculateImpactEstimates } from '../../utils/calculate_impact_estimates import { asCost } from '../../utils/formatters/as_cost'; import { asWeight } from '../../utils/formatters/as_weight'; import { FrameInformationTooltip } from '../frame_information_window/frame_information_tooltip'; +import { CPULabelWithHint } from '../shared/cpu_label_with_hint'; import { StackFrameSummary } from '../stack_frame_summary'; import { GetLabel } from './get_label'; @@ -250,18 +251,12 @@ export function TopNFunctionsTable({ { field: TopNFunctionSortField.ExclusiveCPU, name: ( - - - {i18n.translate('xpack.profiling.functionsView.cpuColumnLabel1Exclusive', { - defaultMessage: 'CPU excl.', - })} - - - {i18n.translate('xpack.profiling.functionsView.cpuColumnLabel2Exclusive', { - defaultMessage: 'subfunctions', - })} - - + ), render: (_, { exclusiveCPU, diff }) => { return ; @@ -271,18 +266,12 @@ export function TopNFunctionsTable({ { field: TopNFunctionSortField.InclusiveCPU, name: ( - - - {i18n.translate('xpack.profiling.functionsView.cpuColumnLabel1Inclusive', { - defaultMessage: 'CPU incl.', - })} - - - {i18n.translate('xpack.profiling.functionsView.cpuColumnLabel2Inclusive', { - defaultMessage: 'subfunctions', - })} - - + ), render: (_, { inclusiveCPU, diff }) => { return ; diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 751d0f97a6981..43d43e26d7837 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -26359,8 +26359,6 @@ "xpack.profiling.flameGraphInformationWindow.executableLabel": "Exécutable", "xpack.profiling.flameGraphInformationWindow.frameTypeLabel": "Type de cadre", "xpack.profiling.flameGraphInformationWindow.functionLabel": "Fonction", - "xpack.profiling.flameGraphInformationWindow.percentageCpuTimeExclusiveLabel": "% de temps processeur (enfants excl.)", - "xpack.profiling.flameGraphInformationWindow.percentageCpuTimeInclusiveLabel": "% de temps processeur", "xpack.profiling.flameGraphInformationWindow.samplesExclusiveLabel": "Échantillons (enfants excl.)", "xpack.profiling.flameGraphInformationWindow.samplesInclusiveLabel": "Échantillons", "xpack.profiling.flameGraphInformationWindow.sourceFileLabel": "Fichier source", @@ -26378,13 +26376,7 @@ "xpack.profiling.flameGraphsView.differentialFlameGraphComparisonModeTitle": "Format", "xpack.profiling.flameGraphsView.differentialFlameGraphTabLabel": "Flame-graph différentiel", "xpack.profiling.flameGraphsView.flameGraphTabLabel": "Flame-graph", - "xpack.profiling.flameGraphTooltip.exclusiveCpuLabel": "CPU", - "xpack.profiling.flameGraphTooltip.inclusiveCpuLabel": "CPU incl. sous-fonctions", "xpack.profiling.flameGraphTooltip.samplesLabel": "Échantillons", - "xpack.profiling.functionsView.cpuColumnLabel1Exclusive": "CPU excl.", - "xpack.profiling.functionsView.cpuColumnLabel1Inclusive": "CPU incl.", - "xpack.profiling.functionsView.cpuColumnLabel2Exclusive": "sous-fonctions", - "xpack.profiling.functionsView.cpuColumnLabel2Inclusive": "sous-fonctions", "xpack.profiling.functionsView.diffColumnLabel": "Diff", "xpack.profiling.functionsView.differentialFunctionsTabLabel": "Fonctions TopN différentielles", "xpack.profiling.functionsView.functionColumnLabel": "Fonction", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index b395853501d98..679150ed7a407 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -26340,8 +26340,6 @@ "xpack.profiling.flameGraphInformationWindow.executableLabel": "実行ファイル", "xpack.profiling.flameGraphInformationWindow.frameTypeLabel": "フレームタイプ", "xpack.profiling.flameGraphInformationWindow.functionLabel": "関数", - "xpack.profiling.flameGraphInformationWindow.percentageCpuTimeExclusiveLabel": "CPU時間の割合（子を除く）", - "xpack.profiling.flameGraphInformationWindow.percentageCpuTimeInclusiveLabel": "CPU時間の割合", "xpack.profiling.flameGraphInformationWindow.samplesExclusiveLabel": "サンプル（子を除く）", "xpack.profiling.flameGraphInformationWindow.samplesInclusiveLabel": "サンプル", "xpack.profiling.flameGraphInformationWindow.sourceFileLabel": "ソースファイル", @@ -26359,13 +26357,7 @@ "xpack.profiling.flameGraphsView.differentialFlameGraphComparisonModeTitle": "フォーマット", "xpack.profiling.flameGraphsView.differentialFlameGraphTabLabel": "差分flamegraph", "xpack.profiling.flameGraphsView.flameGraphTabLabel": "Flamegraph", - "xpack.profiling.flameGraphTooltip.exclusiveCpuLabel": "CPU", - "xpack.profiling.flameGraphTooltip.inclusiveCpuLabel": "CPU（サブ関数を含む）", "xpack.profiling.flameGraphTooltip.samplesLabel": "サンプル", - "xpack.profiling.functionsView.cpuColumnLabel1Exclusive": "CPU（", - "xpack.profiling.functionsView.cpuColumnLabel1Inclusive": "CPU（", - "xpack.profiling.functionsView.cpuColumnLabel2Exclusive": "サブ関数を含む）", - "xpack.profiling.functionsView.cpuColumnLabel2Inclusive": "サブ関数を含む）", "xpack.profiling.functionsView.diffColumnLabel": "差分", "xpack.profiling.functionsView.differentialFunctionsTabLabel": "差分上位N関数", "xpack.profiling.functionsView.functionColumnLabel": "関数", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 4d688bf73539c..f9c07c04ed3d9 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -26356,8 +26356,6 @@ "xpack.profiling.flameGraphInformationWindow.executableLabel": "可执行", "xpack.profiling.flameGraphInformationWindow.frameTypeLabel": "帧类型", "xpack.profiling.flameGraphInformationWindow.functionLabel": "函数", - "xpack.profiling.flameGraphInformationWindow.percentageCpuTimeExclusiveLabel": "CPU 时间百分比（不包括子项）", - "xpack.profiling.flameGraphInformationWindow.percentageCpuTimeInclusiveLabel": "CPU 时间百分比", "xpack.profiling.flameGraphInformationWindow.samplesExclusiveLabel": "样例（不包括子项）", "xpack.profiling.flameGraphInformationWindow.samplesInclusiveLabel": "样例", "xpack.profiling.flameGraphInformationWindow.sourceFileLabel": "源文件", @@ -26375,13 +26373,7 @@ "xpack.profiling.flameGraphsView.differentialFlameGraphComparisonModeTitle": "格式", "xpack.profiling.flameGraphsView.differentialFlameGraphTabLabel": "差异火焰图", "xpack.profiling.flameGraphsView.flameGraphTabLabel": "火焰图", - "xpack.profiling.flameGraphTooltip.exclusiveCpuLabel": "CPU", - "xpack.profiling.flameGraphTooltip.inclusiveCpuLabel": "CPU 非独占时间（子函数）", "xpack.profiling.flameGraphTooltip.samplesLabel": "样例", - "xpack.profiling.functionsView.cpuColumnLabel1Exclusive": "CPU 独占时间", - "xpack.profiling.functionsView.cpuColumnLabel1Inclusive": "CPU 非独占时间", - "xpack.profiling.functionsView.cpuColumnLabel2Exclusive": "子函数", - "xpack.profiling.functionsView.cpuColumnLabel2Inclusive": "子函数", "xpack.profiling.functionsView.diffColumnLabel": "差异", "xpack.profiling.functionsView.differentialFunctionsTabLabel": "差异 TopN 函数", "xpack.profiling.functionsView.functionColumnLabel": "函数", From c98fa8214ed9a8166dca4e9d4bac65043aeb3c2a Mon Sep 17 00:00:00 2001 From: Rodney Norris Date: Mon, 10 Apr 2023 11:00:27 -0500 Subject: [PATCH 004/117] Enterprise Search: remove search applications feature flag (#154609) ## Summary Remove the feature flagging of the search applications (engines) section for nav and routes. ![image](https://user-images.githubusercontent.com/1972968/230502703-694c480d-b997-4d9d-bea6-1f5ba763d6be.png) *Note we should do some more work on the side nav separate from this PR. --- .../common/__mocks__/initial_app_data.ts | 2 - .../enterprise_search/common/constants.ts | 1 - .../enterprise_search/common/types/index.ts | 2 - .../__mocks__/kea_logic/kibana_logic.mock.ts | 2 - .../components/engines/engines_router.tsx | 14 - .../public/applications/index.tsx | 1 - .../applications/shared/layout/nav.test.tsx | 383 ++++-------------- .../public/applications/shared/layout/nav.tsx | 196 +++------ .../server/lib/check_access.test.ts | 12 - .../server/lib/check_access.ts | 2 - .../lib/enterprise_search_config_api.test.ts | 8 - .../lib/enterprise_search_config_api.ts | 4 - 12 files changed, 150 insertions(+), 477 deletions(-) diff --git a/x-pack/plugins/enterprise_search/common/__mocks__/initial_app_data.ts b/x-pack/plugins/enterprise_search/common/__mocks__/initial_app_data.ts index 837cc9da1d87c..f3f1af4e87263 100644 --- a/x-pack/plugins/enterprise_search/common/__mocks__/initial_app_data.ts +++ b/x-pack/plugins/enterprise_search/common/__mocks__/initial_app_data.ts @@ -29,14 +29,12 @@ export const DEFAULT_INITIAL_APP_DATA = { }, access: { hasAppSearchAccess: true, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: true, }, features: { hasConnectors: true, hasDefaultIngestPipeline: true, hasNativeConnectors: true, - hasSearchApplications: false, hasWebCrawler: true, }, appSearch: { diff --git a/x-pack/plugins/enterprise_search/common/constants.ts b/x-pack/plugins/enterprise_search/common/constants.ts index 14cb73fb6027b..d2d9d8a5a3302 100644 --- a/x-pack/plugins/enterprise_search/common/constants.ts +++ b/x-pack/plugins/enterprise_search/common/constants.ts @@ -167,6 +167,5 @@ export const DEFAULT_PRODUCT_FEATURES: ProductFeatures = { hasConnectors: true, hasDefaultIngestPipeline: true, hasNativeConnectors: true, - hasSearchApplications: false, hasWebCrawler: true, }; diff --git a/x-pack/plugins/enterprise_search/common/types/index.ts b/x-pack/plugins/enterprise_search/common/types/index.ts index c85aba9b88863..2b10d55023b8f 100644 --- a/x-pack/plugins/enterprise_search/common/types/index.ts +++ b/x-pack/plugins/enterprise_search/common/types/index.ts @@ -33,7 +33,6 @@ export interface ConfiguredLimits { export interface ProductAccess { hasAppSearchAccess: boolean; - hasSearchEnginesAccess: boolean; hasWorkplaceSearchAccess: boolean; } @@ -41,7 +40,6 @@ export interface ProductFeatures { hasConnectors: boolean; hasDefaultIngestPipeline: boolean; hasNativeConnectors: boolean; - hasSearchApplications: boolean; hasWebCrawler: boolean; } diff --git a/x-pack/plugins/enterprise_search/public/applications/__mocks__/kea_logic/kibana_logic.mock.ts b/x-pack/plugins/enterprise_search/public/applications/__mocks__/kea_logic/kibana_logic.mock.ts index 93681c5b7b299..5cfd5e7029459 100644 --- a/x-pack/plugins/enterprise_search/public/applications/__mocks__/kea_logic/kibana_logic.mock.ts +++ b/x-pack/plugins/enterprise_search/public/applications/__mocks__/kea_logic/kibana_logic.mock.ts @@ -44,12 +44,10 @@ export const mockKibanaValues = { navigateToUrl: jest.fn(), productAccess: { hasAppSearchAccess: true, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: true, }, productFeatures: { hasNativeConnectors: true, - hasSearchApplications: false, hasWebCrawler: true, }, uiSettings: uiSettingsServiceMock.createStartContract(), diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/engines_router.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/engines_router.tsx index de01ca4500bd8..662e1d7c21417 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/engines_router.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/engines_router.tsx @@ -8,11 +8,8 @@ import React from 'react'; import { Switch } from 'react-router-dom'; -import { useValues } from 'kea'; - import { Route } from '@kbn/shared-ux-router'; -import { KibanaLogic } from '../../../shared/kibana'; import { ENGINES_PATH, ENGINE_PATH } from '../../routes'; import { EngineRouter } from '../engine/engine_router'; @@ -21,17 +18,6 @@ import { NotFound } from '../not_found'; import { EnginesList } from './engines_list'; export const EnginesRouter: React.FC = () => { - const { productAccess } = useValues(KibanaLogic); - const enginesSectionEnabled = productAccess.hasSearchEnginesAccess; - if (!enginesSectionEnabled) { - return ( - - - - - - ); - } return ( diff --git a/x-pack/plugins/enterprise_search/public/applications/index.tsx b/x-pack/plugins/enterprise_search/public/applications/index.tsx index 93da5e0d20111..40c3d8bdaf105 100644 --- a/x-pack/plugins/enterprise_search/public/applications/index.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/index.tsx @@ -46,7 +46,6 @@ export const renderApp = ( const noProductAccess: ProductAccess = { hasAppSearchAccess: false, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: false, }; const productAccess = data.access || noProductAccess; diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.test.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.test.tsx index 8745c9d2758e0..79588ba3361ea 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.test.tsx @@ -24,7 +24,6 @@ import { const DEFAULT_PRODUCT_ACCESS: ProductAccess = { hasAppSearchAccess: true, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: true, }; @@ -38,196 +37,6 @@ describe('useEnterpriseSearchContentNav', () => { const fullProductAccess: ProductAccess = DEFAULT_PRODUCT_ACCESS; setMockValues({ productAccess: fullProductAccess, productFeatures: DEFAULT_PRODUCT_FEATURES }); - expect(useEnterpriseSearchNav()).toEqual([ - { - href: '/app/enterprise_search/overview', - id: 'es_overview', - name: 'Overview', - }, - { - id: 'content', - items: [ - { - href: '/app/enterprise_search/content/search_indices', - id: 'search_indices', - name: 'Indices', - }, - { - href: '/app/enterprise_search/content/settings', - id: 'settings', - items: undefined, - name: 'Settings', - }, - ], - name: 'Content', - }, - { - id: 'enterpriseSearchAnalytics', - items: [ - { - href: '/app/enterprise_search/analytics', - id: 'analytics_collections', - name: 'Collections', - }, - ], - name: 'Behavioral Analytics', - }, - { - id: 'search', - items: [ - { - href: '/app/enterprise_search/elasticsearch', - id: 'elasticsearch', - name: 'Elasticsearch', - }, - { - href: '/app/enterprise_search/search_experiences', - id: 'searchExperiences', - name: 'Search Experiences', - }, - { - href: '/app/enterprise_search/app_search', - id: 'app_search', - name: 'App Search', - }, - { - href: '/app/enterprise_search/workplace_search', - id: 'workplace_search', - name: 'Workplace Search', - }, - ], - name: 'Search', - }, - ]); - }); - - it('excludes legacy products when the user has no access to them', () => { - const noProductAccess: ProductAccess = { - ...DEFAULT_PRODUCT_ACCESS, - hasAppSearchAccess: false, - hasWorkplaceSearchAccess: false, - }; - - setMockValues({ productAccess: noProductAccess, productFeatures: DEFAULT_PRODUCT_FEATURES }); - mockKibanaValues.uiSettings.get.mockReturnValue(false); - - const esNav = useEnterpriseSearchNav(); - const searchNav = esNav?.find((item) => item.id === 'search'); - expect(searchNav).not.toBeUndefined(); - expect(searchNav).toEqual({ - id: 'search', - items: [ - { - href: '/app/enterprise_search/elasticsearch', - id: 'elasticsearch', - name: 'Elasticsearch', - }, - { - href: '/app/enterprise_search/search_experiences', - id: 'searchExperiences', - name: 'Search Experiences', - }, - ], - name: 'Search', - }); - }); - - it('excludes App Search when the user has no access to it', () => { - const workplaceSearchProductAccess: ProductAccess = { - ...DEFAULT_PRODUCT_ACCESS, - hasAppSearchAccess: false, - hasWorkplaceSearchAccess: true, - }; - - setMockValues({ - productAccess: workplaceSearchProductAccess, - productFeatures: DEFAULT_PRODUCT_FEATURES, - }); - - const esNav = useEnterpriseSearchNav(); - const searchNav = esNav?.find((item) => item.id === 'search'); - expect(searchNav).not.toBeUndefined(); - expect(searchNav).toEqual({ - id: 'search', - items: [ - { - href: '/app/enterprise_search/elasticsearch', - id: 'elasticsearch', - name: 'Elasticsearch', - }, - { - href: '/app/enterprise_search/search_experiences', - id: 'searchExperiences', - name: 'Search Experiences', - }, - { - href: '/app/enterprise_search/workplace_search', - id: 'workplace_search', - name: 'Workplace Search', - }, - ], - name: 'Search', - }); - }); - - it('excludes Workplace Search when the user has no access to it', () => { - const appSearchProductAccess: ProductAccess = { - ...DEFAULT_PRODUCT_ACCESS, - hasWorkplaceSearchAccess: false, - }; - - setMockValues({ - productAccess: appSearchProductAccess, - productFeatures: DEFAULT_PRODUCT_FEATURES, - }); - - const esNav = useEnterpriseSearchNav(); - const searchNav = esNav?.find((item) => item.id === 'search'); - expect(searchNav).not.toBeUndefined(); - expect(searchNav).toEqual({ - id: 'search', - items: [ - { - href: '/app/enterprise_search/elasticsearch', - id: 'elasticsearch', - name: 'Elasticsearch', - }, - { - href: '/app/enterprise_search/search_experiences', - id: 'searchExperiences', - name: 'Search Experiences', - }, - { - href: '/app/enterprise_search/app_search', - id: 'app_search', - name: 'App Search', - }, - ], - name: 'Search', - }); - }); - - it('excludes engines when feature flag is off', () => { - const fullProductAccess: ProductAccess = DEFAULT_PRODUCT_ACCESS; - setMockValues({ productAccess: fullProductAccess, productFeatures: DEFAULT_PRODUCT_FEATURES }); - - const esNav = useEnterpriseSearchNav(); - expect(esNav?.find((item) => item.id === 'enginesSearch')).toBeUndefined(); - }); -}); - -describe('useEnterpriseSearchContentNav Engines feature flag', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('returns an array of top-level Enterprise Search nav items', () => { - const fullProductAccess: ProductAccess = { - ...DEFAULT_PRODUCT_ACCESS, - hasSearchEnginesAccess: true, - }; - setMockValues({ productAccess: fullProductAccess, productFeatures: DEFAULT_PRODUCT_FEATURES }); - expect(useEnterpriseSearchNav()).toEqual([ { href: '/app/enterprise_search/overview', @@ -302,25 +111,32 @@ describe('useEnterpriseSearchContentNav Engines feature flag', () => { ]); }); - it('excludes standalone experiences when the user has no access to them', () => { - const fullProductAccess: ProductAccess = { + it('excludes legacy products when the user has no access to them', () => { + const noProductAccess: ProductAccess = { ...DEFAULT_PRODUCT_ACCESS, hasAppSearchAccess: false, - hasSearchEnginesAccess: true, hasWorkplaceSearchAccess: false, }; - setMockValues({ productAccess: fullProductAccess, productFeatures: DEFAULT_PRODUCT_FEATURES }); + + setMockValues({ productAccess: noProductAccess, productFeatures: DEFAULT_PRODUCT_FEATURES }); + mockKibanaValues.uiSettings.get.mockReturnValue(false); const esNav = useEnterpriseSearchNav(); - expect(esNav?.find((item) => item.id === 'standaloneExperiences')).toBeUndefined(); + const standAloneNav = esNav?.find((item) => item.id === 'standaloneExperiences'); + expect(standAloneNav).toBeUndefined(); }); + it('excludes App Search when the user has no access to it', () => { - const fullProductAccess: ProductAccess = { + const workplaceSearchProductAccess: ProductAccess = { ...DEFAULT_PRODUCT_ACCESS, hasAppSearchAccess: false, - hasSearchEnginesAccess: true, + hasWorkplaceSearchAccess: true, }; - setMockValues({ productAccess: fullProductAccess, productFeatures: DEFAULT_PRODUCT_FEATURES }); + + setMockValues({ + productAccess: workplaceSearchProductAccess, + productFeatures: DEFAULT_PRODUCT_FEATURES, + }); const esNav = useEnterpriseSearchNav(); const standAloneNav = esNav?.find((item) => item.id === 'standaloneExperiences'); @@ -337,14 +153,17 @@ describe('useEnterpriseSearchContentNav Engines feature flag', () => { name: 'Standalone Experiences', }); }); + it('excludes Workplace Search when the user has no access to it', () => { - const fullProductAccess: ProductAccess = { + const appSearchProductAccess: ProductAccess = { ...DEFAULT_PRODUCT_ACCESS, - hasAppSearchAccess: true, - hasSearchEnginesAccess: true, hasWorkplaceSearchAccess: false, }; - setMockValues({ productAccess: fullProductAccess, productFeatures: DEFAULT_PRODUCT_FEATURES }); + + setMockValues({ + productAccess: appSearchProductAccess, + productFeatures: DEFAULT_PRODUCT_FEATURES, + }); const esNav = useEnterpriseSearchNav(); const standAloneNav = esNav?.find((item) => item.id === 'standaloneExperiences'); @@ -367,11 +186,10 @@ describe('useEnterpriseSearchEngineNav', () => { beforeEach(() => { jest.clearAllMocks(); mockKibanaValues.uiSettings.get.mockReturnValue(true); - const fullProductAccess: ProductAccess = { - ...DEFAULT_PRODUCT_ACCESS, - hasSearchEnginesAccess: true, - }; - setMockValues({ productAccess: fullProductAccess, productFeatures: DEFAULT_PRODUCT_FEATURES }); + setMockValues({ + productAccess: DEFAULT_PRODUCT_ACCESS, + productFeatures: DEFAULT_PRODUCT_FEATURES, + }); }); it('returns an array of top-level Enterprise Search nav items', () => { @@ -555,6 +373,27 @@ describe('useEnterpriseSearchAnalyticsNav', () => { ], name: 'Content', }, + { + id: 'enginesSearch', + name: 'Search', + items: [ + { + href: '/app/enterprise_search/elasticsearch', + id: 'elasticsearch', + name: 'Elasticsearch', + }, + { + id: 'enterpriseSearchEngines', + name: 'Engines', + href: '/app/enterprise_search/content/engines', + }, + { + id: 'searchExperiences', + name: 'Search Experiences', + href: '/app/enterprise_search/search_experiences', + }, + ], + }, { id: 'enterpriseSearchAnalytics', items: [ @@ -567,18 +406,8 @@ describe('useEnterpriseSearchAnalyticsNav', () => { name: 'Behavioral Analytics', }, { - id: 'search', + id: 'standaloneExperiences', items: [ - { - href: '/app/enterprise_search/elasticsearch', - id: 'elasticsearch', - name: 'Elasticsearch', - }, - { - href: '/app/enterprise_search/search_experiences', - id: 'searchExperiences', - name: 'Search Experiences', - }, { href: '/app/enterprise_search/app_search', id: 'app_search', @@ -590,7 +419,7 @@ describe('useEnterpriseSearchAnalyticsNav', () => { name: 'Workplace Search', }, ], - name: 'Search', + name: 'Standalone Experiences', }, ]; @@ -615,83 +444,41 @@ describe('useEnterpriseSearchAnalyticsNav', () => { integration: '/integration-path', overview: '/overview-path', }); - expect(navItems).toEqual([ - { - href: '/app/enterprise_search/overview', - id: 'es_overview', - name: 'Overview', - }, - { - id: 'content', - items: [ - { - href: '/app/enterprise_search/content/search_indices', - id: 'search_indices', - name: 'Indices', - }, - ], - name: 'Content', - }, - { - id: 'enterpriseSearchAnalytics', - items: [ - { - href: '/app/enterprise_search/analytics', - id: 'analytics_collections', - items: [ - { - id: 'analytics_collections', - items: [ - { - href: '/app/enterprise_search/analytics/overview-path', - id: 'enterpriseSearchEngineOverview', - name: 'Overview', - }, - { - href: '/app/enterprise_search/analytics/explorer-path', - id: 'enterpriseSearchEngineIndices', - name: 'Explorer', - }, - { - href: '/app/enterprise_search/analytics/integration-path', - id: 'enterpriseSearchEngineSchema', - name: 'Integration', - }, - ], - name: 'my-test-collection', - }, - ], - name: 'Collections', - }, - ], - name: 'Behavioral Analytics', - }, - { - id: 'search', - items: [ - { - href: '/app/enterprise_search/elasticsearch', - id: 'elasticsearch', - name: 'Elasticsearch', - }, - { - href: '/app/enterprise_search/search_experiences', - id: 'searchExperiences', - name: 'Search Experiences', - }, - { - href: '/app/enterprise_search/app_search', - id: 'app_search', - name: 'App Search', - }, - { - href: '/app/enterprise_search/workplace_search', - id: 'workplace_search', - name: 'Workplace Search', - }, - ], - name: 'Search', - }, - ]); + const analyticsNav = navItems?.find((item) => item.id === 'enterpriseSearchAnalytics'); + expect(analyticsNav).not.toBeUndefined(); + expect(analyticsNav).toEqual({ + id: 'enterpriseSearchAnalytics', + items: [ + { + href: '/app/enterprise_search/analytics', + id: 'analytics_collections', + items: [ + { + id: 'analytics_collections', + items: [ + { + href: '/app/enterprise_search/analytics/overview-path', + id: 'enterpriseSearchEngineOverview', + name: 'Overview', + }, + { + href: '/app/enterprise_search/analytics/explorer-path', + id: 'enterpriseSearchEngineIndices', + name: 'Explorer', + }, + { + href: '/app/enterprise_search/analytics/integration-path', + id: 'enterpriseSearchEngineSchema', + name: 'Integration', + }, + ], + name: 'my-test-collection', + }, + ], + name: 'Collections', + }, + ], + name: 'Behavioral Analytics', + }); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.tsx index 8a4601c60792f..51b22389b87f8 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.tsx @@ -32,8 +32,6 @@ import { generateNavLink } from './nav_link_helpers'; export const useEnterpriseSearchNav = () => { const { productAccess, productFeatures } = useValues(KibanaLogic); - const enginesSectionEnabled = productAccess.hasSearchEnginesAccess; - const navItems: Array> = [ { id: 'es_overview', @@ -80,34 +78,26 @@ export const useEnterpriseSearchNav = () => { }), }, { - id: 'enterpriseSearchAnalytics', + id: 'enginesSearch', items: [ { - id: 'analytics_collections', - name: i18n.translate('xpack.enterpriseSearch.nav.analyticsCollectionsTitle', { - defaultMessage: 'Collections', + id: 'elasticsearch', + name: i18n.translate('xpack.enterpriseSearch.nav.elasticsearchTitle', { + defaultMessage: 'Elasticsearch', }), ...generateNavLink({ shouldNotCreateHref: true, - to: ANALYTICS_PLUGIN.URL, + to: ELASTICSEARCH_PLUGIN.URL, }), }, - ], - name: i18n.translate('xpack.enterpriseSearch.nav.analyticsTitle', { - defaultMessage: 'Behavioral Analytics', - }), - }, - { - id: 'search', - items: [ { - id: 'elasticsearch', - name: i18n.translate('xpack.enterpriseSearch.nav.elasticsearchTitle', { - defaultMessage: 'Elasticsearch', + id: 'enterpriseSearchEngines', + name: i18n.translate('xpack.enterpriseSearch.nav.enginesTitle', { + defaultMessage: 'Engines', }), ...generateNavLink({ shouldNotCreateHref: true, - to: ELASTICSEARCH_PLUGIN.URL, + to: ENTERPRISE_SEARCH_CONTENT_PLUGIN.URL + ENGINES_PATH, }), }, { @@ -120,126 +110,70 @@ export const useEnterpriseSearchNav = () => { to: SEARCH_EXPERIENCES_PLUGIN.URL, }), }, - ...(productAccess.hasAppSearchAccess - ? [ - { - id: 'app_search', - name: i18n.translate('xpack.enterpriseSearch.nav.appSearchTitle', { - defaultMessage: 'App Search', - }), - ...generateNavLink({ - shouldNotCreateHref: true, - to: APP_SEARCH_PLUGIN.URL, - }), - }, - ] - : []), - ...(productAccess.hasWorkplaceSearchAccess - ? [ - { - id: 'workplace_search', - name: i18n.translate('xpack.enterpriseSearch.nav.workplaceSearchTitle', { - defaultMessage: 'Workplace Search', - }), - ...generateNavLink({ - shouldNotCreateHref: true, - to: WORKPLACE_SEARCH_PLUGIN.URL, - }), - }, - ] - : []), ], name: i18n.translate('xpack.enterpriseSearch.nav.searchTitle', { defaultMessage: 'Search', }), }, - ]; - - if (enginesSectionEnabled) { - return [ - navItems[0], // Overview - navItems[1], // Content - { - id: 'enginesSearch', // TODO: just search? or wait for that - items: [ - { - id: 'elasticsearch', - name: i18n.translate('xpack.enterpriseSearch.nav.elasticsearchTitle', { - defaultMessage: 'Elasticsearch', - }), - ...generateNavLink({ - shouldNotCreateHref: true, - to: ELASTICSEARCH_PLUGIN.URL, - }), - }, - { - id: 'enterpriseSearchEngines', - name: i18n.translate('xpack.enterpriseSearch.nav.enginesTitle', { - defaultMessage: 'Engines', - }), - ...generateNavLink({ - shouldNotCreateHref: true, - to: ENTERPRISE_SEARCH_CONTENT_PLUGIN.URL + ENGINES_PATH, - }), - }, + { + id: 'enterpriseSearchAnalytics', + items: [ + { + id: 'analytics_collections', + name: i18n.translate('xpack.enterpriseSearch.nav.analyticsCollectionsTitle', { + defaultMessage: 'Collections', + }), + ...generateNavLink({ + shouldNotCreateHref: true, + to: ANALYTICS_PLUGIN.URL, + }), + }, + ], + name: i18n.translate('xpack.enterpriseSearch.nav.analyticsTitle', { + defaultMessage: 'Behavioral Analytics', + }), + }, + ...(productAccess.hasAppSearchAccess || productAccess.hasWorkplaceSearchAccess + ? [ { - id: 'searchExperiences', - name: i18n.translate('xpack.enterpriseSearch.nav.searchExperiencesTitle', { - defaultMessage: 'Search Experiences', - }), - ...generateNavLink({ - shouldNotCreateHref: true, - to: SEARCH_EXPERIENCES_PLUGIN.URL, + id: 'standaloneExperiences', + items: [ + ...(productAccess.hasAppSearchAccess + ? [ + { + id: 'app_search', + name: i18n.translate('xpack.enterpriseSearch.nav.appSearchTitle', { + defaultMessage: 'App Search', + }), + ...generateNavLink({ + shouldNotCreateHref: true, + to: APP_SEARCH_PLUGIN.URL, + }), + }, + ] + : []), + ...(productAccess.hasWorkplaceSearchAccess + ? [ + { + id: 'workplace_search', + name: i18n.translate('xpack.enterpriseSearch.nav.workplaceSearchTitle', { + defaultMessage: 'Workplace Search', + }), + ...generateNavLink({ + shouldNotCreateHref: true, + to: WORKPLACE_SEARCH_PLUGIN.URL, + }), + }, + ] + : []), + ], + name: i18n.translate('xpack.enterpriseSearch.nav.standaloneExperiencesTitle', { + defaultMessage: 'Standalone Experiences', }), }, - ], - name: i18n.translate('xpack.enterpriseSearch.nav.searchTitle', { - defaultMessage: 'Search', - }), - }, - navItems[2], // Behavioural Analytics - ...(productAccess.hasAppSearchAccess || productAccess.hasWorkplaceSearchAccess - ? [ - { - id: 'standaloneExperiences', - items: [ - ...(productAccess.hasAppSearchAccess - ? [ - { - id: 'app_search', - name: i18n.translate('xpack.enterpriseSearch.nav.appSearchTitle', { - defaultMessage: 'App Search', - }), - ...generateNavLink({ - shouldNotCreateHref: true, - to: APP_SEARCH_PLUGIN.URL, - }), - }, - ] - : []), - ...(productAccess.hasWorkplaceSearchAccess - ? [ - { - id: 'workplace_search', - name: i18n.translate('xpack.enterpriseSearch.nav.workplaceSearchTitle', { - defaultMessage: 'Workplace Search', - }), - ...generateNavLink({ - shouldNotCreateHref: true, - to: WORKPLACE_SEARCH_PLUGIN.URL, - }), - }, - ] - : []), - ], - name: i18n.translate('xpack.enterpriseSearch.nav.standaloneExperiencesTitle', { - defaultMessage: 'Standalone Experiences', - }), - }, - ] - : []), - ]; - } + ] + : []), + ]; return navItems; }; diff --git a/x-pack/plugins/enterprise_search/server/lib/check_access.test.ts b/x-pack/plugins/enterprise_search/server/lib/check_access.test.ts index 046b152d5e103..e19cdc7153215 100644 --- a/x-pack/plugins/enterprise_search/server/lib/check_access.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/check_access.test.ts @@ -62,7 +62,6 @@ describe('checkAccess', () => { }; expect(await checkAccess({ ...mockDependencies, security })).toEqual({ hasAppSearchAccess: false, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: false, }); }); @@ -75,7 +74,6 @@ describe('checkAccess', () => { }; expect(await checkAccess({ ...mockDependencies, request })).toEqual({ hasAppSearchAccess: false, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: false, }); }); @@ -86,7 +84,6 @@ describe('checkAccess', () => { mockSpaces.spacesService.getActiveSpace.mockResolvedValueOnce(disabledSpace); expect(await checkAccess({ ...mockDependencies })).toEqual({ hasAppSearchAccess: false, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: false, }); }); @@ -100,7 +97,6 @@ describe('checkAccess', () => { ); expect(await checkAccess({ ...mockDependencies })).toEqual({ hasAppSearchAccess: false, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: false, }); }); @@ -123,7 +119,6 @@ describe('checkAccess', () => { it('should not throw', async () => { await expect(checkAccess({ ...mockDependencies, spaces: undefined })).resolves.toEqual({ hasAppSearchAccess: false, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: false, }); }); @@ -151,7 +146,6 @@ describe('checkAccess', () => { }; expect(await checkAccess({ ...mockDependencies, security })).toEqual({ hasAppSearchAccess: true, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: true, }); }); @@ -167,7 +161,6 @@ describe('checkAccess', () => { }; expect(await checkAccess({ ...mockDependencies, security })).toEqual({ hasAppSearchAccess: false, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: false, }); }); @@ -189,7 +182,6 @@ describe('checkAccess', () => { const config = { host: undefined }; expect(await checkAccess({ ...mockDependencies, config })).toEqual({ hasAppSearchAccess: false, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: false, }); }); @@ -200,13 +192,11 @@ describe('checkAccess', () => { (callEnterpriseSearchConfigAPI as jest.Mock).mockImplementationOnce(() => ({ access: { hasAppSearchAccess: false, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: true, }, })); expect(await checkAccess(mockDependencies)).toEqual({ hasAppSearchAccess: false, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: true, }); }); @@ -215,7 +205,6 @@ describe('checkAccess', () => { (callEnterpriseSearchConfigAPI as jest.Mock).mockImplementationOnce(() => ({})); expect(await checkAccess(mockDependencies)).toEqual({ hasAppSearchAccess: false, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: false, }); }); @@ -227,7 +216,6 @@ describe('checkAccess', () => { })); expect(await checkAccess(mockDependencies)).toEqual({ hasAppSearchAccess: false, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: false, }); }); diff --git a/x-pack/plugins/enterprise_search/server/lib/check_access.ts b/x-pack/plugins/enterprise_search/server/lib/check_access.ts index ad051bc3d7468..f362066b5def4 100644 --- a/x-pack/plugins/enterprise_search/server/lib/check_access.ts +++ b/x-pack/plugins/enterprise_search/server/lib/check_access.ts @@ -25,12 +25,10 @@ interface CheckAccess { const ALLOW_ALL_PLUGINS: ProductAccess = { hasAppSearchAccess: true, - hasSearchEnginesAccess: false, // still false unless Feature Flag explicitly enabled on backend hasWorkplaceSearchAccess: true, }; const DENY_ALL_PLUGINS: ProductAccess = { hasAppSearchAccess: false, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: false, }; diff --git a/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.test.ts b/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.test.ts index c8a6713216798..6243955ca63ad 100644 --- a/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.test.ts @@ -72,7 +72,6 @@ describe('callEnterpriseSearchConfigAPI', () => { name: 'someuser', access: { app_search: true, - search_engines: true, workplace_search: false, }, app_search: { @@ -126,12 +125,10 @@ describe('callEnterpriseSearchConfigAPI', () => { kibanaVersion: '1.0.0', access: { hasAppSearchAccess: true, - hasSearchEnginesAccess: true, hasWorkplaceSearchAccess: false, }, features: { hasNativeConnectors: true, - hasSearchApplications: true, hasWebCrawler: true, }, publicUrl: 'http://some.vanity.url', @@ -145,12 +142,10 @@ describe('callEnterpriseSearchConfigAPI', () => { kibanaVersion: '1.0.0', access: { hasAppSearchAccess: false, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: false, }, features: { hasNativeConnectors: true, - hasSearchApplications: false, hasWebCrawler: true, }, publicUrl: undefined, @@ -210,7 +205,6 @@ describe('callEnterpriseSearchConfigAPI', () => { hasConnectors: false, hasDefaultIngestPipeline: false, hasNativeConnectors: false, - hasSearchApplications: false, hasWebCrawler: false, host: '', }; @@ -218,14 +212,12 @@ describe('callEnterpriseSearchConfigAPI', () => { expect(await callEnterpriseSearchConfigAPI({ ...mockDependencies, config })).toEqual({ access: { hasAppSearchAccess: false, - hasSearchEnginesAccess: false, hasWorkplaceSearchAccess: false, }, features: { hasConnectors: false, hasDefaultIngestPipeline: false, hasNativeConnectors: false, - hasSearchApplications: false, hasWebCrawler: false, }, kibanaVersion: '1.0.0', diff --git a/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.ts b/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.ts index e7652402da17d..e0758feef9d94 100644 --- a/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.ts +++ b/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.ts @@ -48,14 +48,12 @@ export const callEnterpriseSearchConfigAPI = async ({ return { access: { hasAppSearchAccess: false, - hasSearchEnginesAccess: false, // TODO: update to read ES feature flag, or just refactor out hasWorkplaceSearchAccess: false, }, features: { hasConnectors: config.hasConnectors, hasDefaultIngestPipeline: config.hasDefaultIngestPipeline, hasNativeConnectors: config.hasNativeConnectors, - hasSearchApplications: false, // TODO: update to read ES feature flag, or just refactor out hasWebCrawler: config.hasWebCrawler, }, kibanaVersion: kibanaPackageJson.version, @@ -103,14 +101,12 @@ export const callEnterpriseSearchConfigAPI = async ({ kibanaVersion: kibanaPackageJson.version, access: { hasAppSearchAccess: !!data?.current_user?.access?.app_search, - hasSearchEnginesAccess: !!data?.current_user?.access?.search_engines, hasWorkplaceSearchAccess: !!data?.current_user?.access?.workplace_search, }, features: { hasConnectors: config.hasConnectors, hasDefaultIngestPipeline: config.hasDefaultIngestPipeline, hasNativeConnectors: config.hasNativeConnectors, - hasSearchApplications: !!data?.current_user?.access?.search_engines, hasWebCrawler: config.hasWebCrawler, }, publicUrl: stripTrailingSlash(data?.settings?.external_url), From b6a113ccfa7e8caa384452dc3900ac14fb73c2f5 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Mon, 10 Apr 2023 12:20:58 -0400 Subject: [PATCH 005/117] Cases delete files case deletion (#153979) This PR extends the case deletion functionality to delete any files that are attached to the case when a case is deleted. To avoid attempting to delete too many files at once I chunk case ids into 50 at a time such that we're at most only deleting 5000 (50 case * 100 files per case) at once. That way we don't exceed the 10k find api limit. ## Testing Run the python script from here: https://github.com/elastic/cases-files-generator to generate some files for a case Deleting the case should remove all files --- .../server/file_service/file_action_types.ts | 2 +- x-pack/plugins/cases/common/api/helpers.ts | 5 + .../plugins/cases/common/constants/index.ts | 1 + x-pack/plugins/cases/common/files/index.ts | 2 +- .../server/client/attachments/bulk_delete.ts | 34 +- .../cases/server/client/cases/delete.test.ts | 79 +++++ .../cases/server/client/cases/delete.ts | 51 ++- .../cases/server/client/files/index.test.ts | 77 +++++ .../cases/server/client/files/index.ts | 39 +++ .../routes/api/comments/find_comments.ts | 4 +- .../common/lib/api/attachments.ts | 25 ++ .../tests/common/cases/delete_cases.ts | 295 +++++++++++++++++- 12 files changed, 568 insertions(+), 46 deletions(-) create mode 100644 x-pack/plugins/cases/server/client/cases/delete.test.ts create mode 100644 x-pack/plugins/cases/server/client/files/index.test.ts create mode 100644 x-pack/plugins/cases/server/client/files/index.ts diff --git a/src/plugins/files/server/file_service/file_action_types.ts b/src/plugins/files/server/file_service/file_action_types.ts index c1264ed2b2452..4247f567802ed 100644 --- a/src/plugins/files/server/file_service/file_action_types.ts +++ b/src/plugins/files/server/file_service/file_action_types.ts @@ -105,5 +105,5 @@ export interface FindFileArgs extends Pagination { /** * File metadata values. These values are governed by the consumer. */ - meta?: Record; + meta?: Record; } diff --git a/x-pack/plugins/cases/common/api/helpers.ts b/x-pack/plugins/cases/common/api/helpers.ts index 95180c9695950..b02d46e85fb18 100644 --- a/x-pack/plugins/cases/common/api/helpers.ts +++ b/x-pack/plugins/cases/common/api/helpers.ts @@ -21,6 +21,7 @@ import { INTERNAL_CONNECTORS_URL, INTERNAL_CASE_USERS_URL, INTERNAL_DELETE_FILE_ATTACHMENTS_URL, + CASE_FIND_ATTACHMENTS_URL, } from '../constants'; export const getCaseDetailsUrl = (id: string): string => { @@ -39,6 +40,10 @@ export const getCaseCommentDetailsUrl = (caseId: string, commentId: string): str return CASE_COMMENT_DETAILS_URL.replace('{case_id}', caseId).replace('{comment_id}', commentId); }; +export const getCaseFindAttachmentsUrl = (caseId: string): string => { + return CASE_FIND_ATTACHMENTS_URL.replace('{case_id}', caseId); +}; + export const getCaseCommentDeleteUrl = (caseId: string, commentId: string): string => { return CASE_COMMENT_DELETE_URL.replace('{case_id}', caseId).replace('{comment_id}', commentId); }; diff --git a/x-pack/plugins/cases/common/constants/index.ts b/x-pack/plugins/cases/common/constants/index.ts index b0bf34eef6c29..83bd0cb660093 100644 --- a/x-pack/plugins/cases/common/constants/index.ts +++ b/x-pack/plugins/cases/common/constants/index.ts @@ -47,6 +47,7 @@ export const CASE_CONFIGURE_DETAILS_URL = `${CASES_URL}/configure/{configuration export const CASE_CONFIGURE_CONNECTORS_URL = `${CASE_CONFIGURE_URL}/connectors` as const; export const CASE_COMMENTS_URL = `${CASE_DETAILS_URL}/comments` as const; +export const CASE_FIND_ATTACHMENTS_URL = `${CASE_COMMENTS_URL}/_find` as const; export const CASE_COMMENT_DETAILS_URL = `${CASE_DETAILS_URL}/comments/{comment_id}` as const; export const CASE_COMMENT_DELETE_URL = `${CASE_DETAILS_URL}/comments/{comment_id}` as const; export const CASE_PUSH_URL = `${CASE_DETAILS_URL}/connector/{connector_id}/_push` as const; diff --git a/x-pack/plugins/cases/common/files/index.ts b/x-pack/plugins/cases/common/files/index.ts index 6ab2268eeb0ce..2e72430a24d88 100644 --- a/x-pack/plugins/cases/common/files/index.ts +++ b/x-pack/plugins/cases/common/files/index.ts @@ -18,7 +18,7 @@ export const CaseFileMetadataForDeletionRt = rt.type({ caseIds: rt.array(rt.string), }); -export type CaseFileMetadata = rt.TypeOf; +export type CaseFileMetadataForDeletion = rt.TypeOf; const FILE_KIND_DELIMITER = 'FilesCases'; diff --git a/x-pack/plugins/cases/server/client/attachments/bulk_delete.ts b/x-pack/plugins/cases/server/client/attachments/bulk_delete.ts index c604c13b0970f..75cdf6f42e05f 100644 --- a/x-pack/plugins/cases/server/client/attachments/bulk_delete.ts +++ b/x-pack/plugins/cases/server/client/attachments/bulk_delete.ts @@ -12,18 +12,18 @@ import { identity } from 'fp-ts/lib/function'; import pMap from 'p-map'; import { partition } from 'lodash'; -import type { File } from '@kbn/files-plugin/common'; +import type { File, FileJSON } from '@kbn/files-plugin/common'; import type { FileServiceStart } from '@kbn/files-plugin/server'; import { FileNotFoundError } from '@kbn/files-plugin/server/file_service/errors'; import { BulkDeleteFileAttachmentsRequestRt, excess, throwErrors } from '../../../common/api'; import { MAX_CONCURRENT_SEARCHES } from '../../../common/constants'; import type { CasesClientArgs } from '../types'; import { createCaseError } from '../../common/error'; -import type { OwnerEntity } from '../../authorization'; import { Operations } from '../../authorization'; import type { BulkDeleteFileArgs } from './types'; -import { constructOwnerFromFileKind, CaseFileMetadataForDeletionRt } from '../../../common/files'; +import { CaseFileMetadataForDeletionRt } from '../../../common/files'; import type { CasesClient } from '../client'; +import { createFileEntities, deleteFiles } from '../files'; export const bulkDeleteFileAttachments = async ( { caseId, fileIds }: BulkDeleteFileArgs, @@ -67,9 +67,7 @@ export const bulkDeleteFileAttachments = async ( }); await Promise.all([ - pMap(request.ids, async (fileId: string) => fileService.delete({ id: fileId }), { - concurrency: MAX_CONCURRENT_SEARCHES, - }), + deleteFiles(request.ids, fileService), attachmentService.bulkDelete({ attachmentIds: fileAttachments.map((so) => so.id), refresh: false, @@ -117,7 +115,7 @@ const getFiles = async ( caseId: BulkDeleteFileArgs['caseId'], fileIds: BulkDeleteFileArgs['fileIds'], fileService: FileServiceStart -) => { +): Promise => { // it's possible that we're trying to delete a file when an attachment wasn't created (for example if the create // attachment request failed) const files = await pMap(fileIds, async (fileId: string) => fileService.getById({ id: fileId }), { @@ -143,25 +141,5 @@ const getFiles = async ( throw Boom.badRequest('Failed to find files to delete'); } - return validFiles; -}; - -const createFileEntities = (files: File[]) => { - const fileEntities: OwnerEntity[] = []; - - // It's possible that the owner array could have invalid information in it so we'll use the file kind for determining if the user - // has the correct authorization for deleting these files - for (const fileInfo of files) { - const ownerFromFileKind = constructOwnerFromFileKind(fileInfo.data.fileKind); - - if (ownerFromFileKind == null) { - throw Boom.badRequest( - `File id ${fileInfo.id} has invalid file kind ${fileInfo.data.fileKind}` - ); - } - - fileEntities.push({ id: fileInfo.id, owner: ownerFromFileKind }); - } - - return fileEntities; + return validFiles.map((fileInfo) => fileInfo.data); }; diff --git a/x-pack/plugins/cases/server/client/cases/delete.test.ts b/x-pack/plugins/cases/server/client/cases/delete.test.ts new file mode 100644 index 0000000000000..043b08733dd3f --- /dev/null +++ b/x-pack/plugins/cases/server/client/cases/delete.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { MAX_FILES_PER_CASE } from '../../../common/constants'; +import type { FindFileArgs } from '@kbn/files-plugin/server'; +import { createFileServiceMock } from '@kbn/files-plugin/server/mocks'; +import type { FileJSON } from '@kbn/shared-ux-file-types'; +import type { CaseFileMetadataForDeletion } from '../../../common/files'; +import { constructFileKindIdByOwner } from '../../../common/files'; +import { getFileEntities } from './delete'; + +const getCaseIds = (numIds: number) => { + return Array.from(Array(numIds).keys()).map((key) => key.toString()); +}; +describe('delete', () => { + describe('getFileEntities', () => { + const numCaseIds = 1000; + const caseIds = getCaseIds(numCaseIds); + const mockFileService = createFileServiceMock(); + mockFileService.find.mockImplementation(async (args: FindFileArgs) => { + const caseMeta = args.meta as unknown as CaseFileMetadataForDeletion; + const numFilesToGen = caseMeta.caseIds.length * MAX_FILES_PER_CASE; + const files = Array.from(Array(numFilesToGen).keys()).map(() => createMockFileJSON()); + + return { + files, + total: files.length, + }; + }); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('only provides 50 case ids in a single call to the find api', async () => { + await getFileEntities(caseIds, mockFileService); + + for (const call of mockFileService.find.mock.calls) { + const callMeta = call[0].meta as unknown as CaseFileMetadataForDeletion; + expect(callMeta.caseIds.length).toEqual(50); + } + }); + + it('calls the find function the number of case ids divided by the chunk size', async () => { + await getFileEntities(caseIds, mockFileService); + + const chunkSize = 50; + + expect(mockFileService.find).toHaveBeenCalledTimes(numCaseIds / chunkSize); + }); + + it('returns the number of entities equal to the case ids times the max files per case limit', async () => { + const expectedEntities = Array.from(Array(numCaseIds * MAX_FILES_PER_CASE).keys()).map( + () => ({ + id: '123', + owner: 'securitySolution', + }) + ); + + const entities = await getFileEntities(caseIds, mockFileService); + + expect(entities.length).toEqual(numCaseIds * MAX_FILES_PER_CASE); + expect(entities).toEqual(expectedEntities); + }); + }); +}); + +const createMockFileJSON = (): FileJSON => { + return { + id: '123', + fileKind: constructFileKindIdByOwner('securitySolution'), + meta: { + owner: ['securitySolution'], + }, + } as unknown as FileJSON; +}; diff --git a/x-pack/plugins/cases/server/client/cases/delete.ts b/x-pack/plugins/cases/server/client/cases/delete.ts index d9e0b6383af1b..a81e0db4dcf66 100644 --- a/x-pack/plugins/cases/server/client/cases/delete.ts +++ b/x-pack/plugins/cases/server/client/cases/delete.ts @@ -6,27 +6,32 @@ */ import { Boom } from '@hapi/boom'; +import pMap from 'p-map'; +import { chunk } from 'lodash'; import type { SavedObjectsBulkDeleteObject } from '@kbn/core/server'; +import type { FileServiceStart } from '@kbn/files-plugin/server'; import { CASE_COMMENT_SAVED_OBJECT, CASE_SAVED_OBJECT, CASE_USER_ACTION_SAVED_OBJECT, + MAX_FILES_PER_CASE, + MAX_DOCS_PER_PAGE, } from '../../../common/constants'; import type { CasesClientArgs } from '..'; import { createCaseError } from '../../common/error'; import type { OwnerEntity } from '../../authorization'; import { Operations } from '../../authorization'; +import { createFileEntities, deleteFiles } from '../files'; /** * Deletes the specified cases and their attachments. - * - * @ignore */ export async function deleteCases(ids: string[], clientArgs: CasesClientArgs): Promise { const { services: { caseService, attachmentService, userActionService }, logger, authorization, + fileService, } = clientArgs; try { const cases = await caseService.getCases({ caseIds: ids }); @@ -44,9 +49,11 @@ export async function deleteCases(ids: string[], clientArgs: CasesClientArgs): P entities.set(theCase.id, { id: theCase.id, owner: theCase.attributes.owner }); } + const fileEntities = await getFileEntities(ids, fileService); + await authorization.ensureAuthorized({ operation: Operations.deleteCase, - entities: Array.from(entities.values()), + entities: [...Array.from(entities.values()), ...fileEntities], }); const attachmentIds = await attachmentService.getter.getAttachmentIdsForCases({ @@ -61,10 +68,14 @@ export async function deleteCases(ids: string[], clientArgs: CasesClientArgs): P ...userActionIds.map((id) => ({ id, type: CASE_USER_ACTION_SAVED_OBJECT })), ]; - await caseService.bulkDeleteCaseEntities({ - entities: bulkDeleteEntities, - options: { refresh: 'wait_for' }, - }); + const fileIds = fileEntities.map((entity) => entity.id); + await Promise.all([ + deleteFiles(fileIds, fileService), + caseService.bulkDeleteCaseEntities({ + entities: bulkDeleteEntities, + options: { refresh: 'wait_for' }, + }), + ]); await userActionService.creator.bulkAuditLogCaseDeletion( cases.saved_objects.map((caseInfo) => caseInfo.id) @@ -77,3 +88,29 @@ export async function deleteCases(ids: string[], clientArgs: CasesClientArgs): P }); } } + +export const getFileEntities = async ( + caseIds: string[], + fileService: FileServiceStart +): Promise => { + // using 50 just to be safe, each case can have 100 files = 50 * 100 = 5000 which is half the max number of docs that + // the client can request + const chunkSize = MAX_FILES_PER_CASE / 2; + const chunkedIds = chunk(caseIds, chunkSize); + + const entityResults = await pMap(chunkedIds, async (ids: string[]) => { + const findRes = await fileService.find({ + perPage: MAX_DOCS_PER_PAGE, + meta: { + caseIds: ids, + }, + }); + + const fileEntities = createFileEntities(findRes.files); + return fileEntities; + }); + + const entities = entityResults.flatMap((res) => res); + + return entities; +}; diff --git a/x-pack/plugins/cases/server/client/files/index.test.ts b/x-pack/plugins/cases/server/client/files/index.test.ts new file mode 100644 index 0000000000000..e1727f5885d4a --- /dev/null +++ b/x-pack/plugins/cases/server/client/files/index.test.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createFileServiceMock } from '@kbn/files-plugin/server/mocks'; +import { OBSERVABILITY_OWNER, SECURITY_SOLUTION_OWNER } from '../../../common'; +import { constructFileKindIdByOwner } from '../../../common/files'; +import { createFileEntities, deleteFiles } from '.'; + +describe('server files', () => { + describe('createFileEntities', () => { + it('returns an empty array when passed no files', () => { + expect(createFileEntities([])).toEqual([]); + }); + + it('throws an error when the file kind is not valid', () => { + expect.assertions(1); + + expect(() => + createFileEntities([{ fileKind: 'abc', id: '1' }]) + ).toThrowErrorMatchingInlineSnapshot(`"File id 1 has invalid file kind abc"`); + }); + + it('throws an error when one of the file entities does not have a valid file kind', () => { + expect.assertions(1); + + expect(() => + createFileEntities([ + { fileKind: constructFileKindIdByOwner(SECURITY_SOLUTION_OWNER), id: '1' }, + { fileKind: 'abc', id: '2' }, + ]) + ).toThrowErrorMatchingInlineSnapshot(`"File id 2 has invalid file kind abc"`); + }); + + it('returns an array of entities when the file kind is valid', () => { + expect.assertions(1); + + expect( + createFileEntities([ + { fileKind: constructFileKindIdByOwner(SECURITY_SOLUTION_OWNER), id: '1' }, + { fileKind: constructFileKindIdByOwner(OBSERVABILITY_OWNER), id: '2' }, + ]) + ).toEqual([ + { id: '1', owner: 'securitySolution' }, + { id: '2', owner: 'observability' }, + ]); + }); + }); + + describe('deleteFiles', () => { + it('calls delete twice with the ids passed in', async () => { + const fileServiceMock = createFileServiceMock(); + + expect.assertions(2); + await deleteFiles(['1', '2'], fileServiceMock); + + expect(fileServiceMock.delete).toBeCalledTimes(2); + expect(fileServiceMock.delete.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + Object { + "id": "1", + }, + ], + Array [ + Object { + "id": "2", + }, + ], + ] + `); + }); + }); +}); diff --git a/x-pack/plugins/cases/server/client/files/index.ts b/x-pack/plugins/cases/server/client/files/index.ts new file mode 100644 index 0000000000000..31e0bcf8fb2dd --- /dev/null +++ b/x-pack/plugins/cases/server/client/files/index.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Boom from '@hapi/boom'; +import type { FileJSON } from '@kbn/files-plugin/common'; +import type { FileServiceStart } from '@kbn/files-plugin/server'; +import pMap from 'p-map'; +import { constructOwnerFromFileKind } from '../../../common/files'; +import { MAX_CONCURRENT_SEARCHES } from '../../../common/constants'; +import type { OwnerEntity } from '../../authorization'; + +type FileEntityInfo = Pick; + +export const createFileEntities = (files: FileEntityInfo[]): OwnerEntity[] => { + const fileEntities: OwnerEntity[] = []; + + // It's possible that the owner array could have invalid information in it so we'll use the file kind for determining if the user + // has the correct authorization for deleting these files + for (const fileInfo of files) { + const ownerFromFileKind = constructOwnerFromFileKind(fileInfo.fileKind); + + if (ownerFromFileKind == null) { + throw Boom.badRequest(`File id ${fileInfo.id} has invalid file kind ${fileInfo.fileKind}`); + } + + fileEntities.push({ id: fileInfo.id, owner: ownerFromFileKind }); + } + + return fileEntities; +}; + +export const deleteFiles = async (fileIds: string[], fileService: FileServiceStart) => + pMap(fileIds, async (fileId: string) => fileService.delete({ id: fileId }), { + concurrency: MAX_CONCURRENT_SEARCHES, + }); diff --git a/x-pack/plugins/cases/server/routes/api/comments/find_comments.ts b/x-pack/plugins/cases/server/routes/api/comments/find_comments.ts index 23a7ecb601534..ede44d0784f89 100644 --- a/x-pack/plugins/cases/server/routes/api/comments/find_comments.ts +++ b/x-pack/plugins/cases/server/routes/api/comments/find_comments.ts @@ -13,13 +13,13 @@ import { fold } from 'fp-ts/lib/Either'; import { identity } from 'fp-ts/lib/function'; import { FindQueryParamsRt, throwErrors, excess } from '../../../../common/api'; -import { CASE_COMMENTS_URL } from '../../../../common/constants'; +import { CASE_FIND_ATTACHMENTS_URL } from '../../../../common/constants'; import { createCasesRoute } from '../create_cases_route'; import { createCaseError } from '../../../common/error'; export const findCommentsRoute = createCasesRoute({ method: 'get', - path: `${CASE_COMMENTS_URL}/_find`, + path: CASE_FIND_ATTACHMENTS_URL, params: { params: schema.object({ case_id: schema.string(), diff --git a/x-pack/test/cases_api_integration/common/lib/api/attachments.ts b/x-pack/test/cases_api_integration/common/lib/api/attachments.ts index 4f035a100e69f..c49a375ed0b94 100644 --- a/x-pack/test/cases_api_integration/common/lib/api/attachments.ts +++ b/x-pack/test/cases_api_integration/common/lib/api/attachments.ts @@ -15,7 +15,9 @@ import { CommentPatchRequest, CommentRequest, CommentResponse, + CommentsResponse, CommentType, + getCaseFindAttachmentsUrl, getCasesDeleteFileAttachmentsUrl, } from '@kbn/cases-plugin/common/api'; import { User } from '../authentication/types'; @@ -280,3 +282,26 @@ export const bulkDeleteFileAttachments = async ({ .auth(auth.user.username, auth.user.password) .expect(expectedHttpCode); }; + +export const findAttachments = async ({ + supertest, + caseId, + query = {}, + expectedHttpCode = 200, + auth = { user: superUser, space: null }, +}: { + supertest: SuperTest.SuperTest; + caseId: string; + query?: Record; + expectedHttpCode?: number; + auth?: { user: User; space: string | null }; +}): Promise => { + const { body } = await supertest + .get(`${getSpaceUrlPrefix(auth.space)}${getCaseFindAttachmentsUrl(caseId)}`) + .set('kbn-xsrf', 'true') + .query(query) + .auth(auth.user.username, auth.user.password) + .expect(expectedHttpCode); + + return body; +}; diff --git a/x-pack/test/cases_api_integration/security_and_spaces/tests/common/cases/delete_cases.ts b/x-pack/test/cases_api_integration/security_and_spaces/tests/common/cases/delete_cases.ts index 957f304b8e829..981a368019e8f 100644 --- a/x-pack/test/cases_api_integration/security_and_spaces/tests/common/cases/delete_cases.ts +++ b/x-pack/test/cases_api_integration/security_and_spaces/tests/common/cases/delete_cases.ts @@ -6,13 +6,16 @@ */ import expect from '@kbn/expect'; +import type SuperTest from 'supertest'; +import { MAX_DOCS_PER_PAGE } from '@kbn/cases-plugin/common/constants'; import { FtrProviderContext } from '../../../../common/ftr_provider_context'; -import { getPostCaseRequest, postCommentUserReq } from '../../../../common/lib/mock'; import { - deleteCasesByESQuery, - deleteCasesUserActions, - deleteComments, + getFilesAttachmentReq, + getPostCaseRequest, + postCommentUserReq, +} from '../../../../common/lib/mock'; +import { createCase, deleteCases, createComment, @@ -20,6 +23,12 @@ import { getCase, superUserSpace1Auth, getCaseUserActions, + deleteAllCaseItems, + createAndUploadFile, + deleteAllFiles, + listFiles, + findAttachments, + bulkCreateAttachments, } from '../../../../common/lib/api'; import { secOnly, @@ -31,6 +40,17 @@ import { obsOnly, superUser, } from '../../../../common/lib/authentication/users'; +import { + secAllUser, + users as api_int_users, +} from '../../../../../api_integration/apis/cases/common/users'; +import { roles as api_int_roles } from '../../../../../api_integration/apis/cases/common/roles'; +import { createUsersAndRoles, deleteUsersAndRoles } from '../../../../common/lib/authentication'; +import { + OBSERVABILITY_FILE_KIND, + SECURITY_SOLUTION_FILE_KIND, +} from '../../../../common/lib/constants'; +import { User } from '../../../../common/lib/authentication/types'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { @@ -40,9 +60,7 @@ export default ({ getService }: FtrProviderContext): void => { describe('delete_cases', () => { afterEach(async () => { - await deleteCasesByESQuery(es); - await deleteComments(es); - await deleteCasesUserActions(es); + await deleteAllCaseItems(es); }); it('should delete a case', async () => { @@ -106,7 +124,204 @@ export default ({ getService }: FtrProviderContext): void => { await deleteCases({ supertest, caseIDs: ['fake-id'], expectedHttpCode: 404 }); }); + describe('files', () => { + afterEach(async () => { + await deleteAllFiles({ + supertest, + }); + }); + + it('should delete all files associated with a case', async () => { + const { caseInfo: postedCase } = await createCaseWithFiles({ + supertest: supertestWithoutAuth, + fileKind: SECURITY_SOLUTION_FILE_KIND, + owner: 'securitySolution', + }); + + await deleteCases({ supertest: supertestWithoutAuth, caseIDs: [postedCase.id] }); + + const [filesAfterDelete, attachmentsAfterDelete] = await Promise.all([ + listFiles({ + supertest: supertestWithoutAuth, + params: { + kind: SECURITY_SOLUTION_FILE_KIND, + }, + }), + findAttachments({ + supertest: supertestWithoutAuth, + caseId: postedCase.id, + query: { + perPage: MAX_DOCS_PER_PAGE, + }, + }), + ]); + + expect(filesAfterDelete.total).to.be(0); + expect(attachmentsAfterDelete.comments.length).to.be(0); + }); + + it('should delete all files associated with multiple cases', async () => { + const [{ caseInfo: postedCase1 }, { caseInfo: postedCase2 }] = await Promise.all([ + createCaseWithFiles({ + supertest: supertestWithoutAuth, + fileKind: SECURITY_SOLUTION_FILE_KIND, + owner: 'securitySolution', + }), + createCaseWithFiles({ + supertest: supertestWithoutAuth, + fileKind: SECURITY_SOLUTION_FILE_KIND, + owner: 'securitySolution', + }), + ]); + + await deleteCases({ + supertest: supertestWithoutAuth, + caseIDs: [postedCase1.id, postedCase2.id], + }); + + const [filesAfterDelete, attachmentsAfterDelete, attachmentsAfterDelete2] = + await Promise.all([ + listFiles({ + supertest: supertestWithoutAuth, + params: { + kind: SECURITY_SOLUTION_FILE_KIND, + }, + }), + findAttachments({ + supertest: supertestWithoutAuth, + caseId: postedCase1.id, + query: { + perPage: MAX_DOCS_PER_PAGE, + }, + }), + findAttachments({ + supertest: supertestWithoutAuth, + caseId: postedCase2.id, + query: { + perPage: MAX_DOCS_PER_PAGE, + }, + }), + ]); + + expect(filesAfterDelete.total).to.be(0); + expect(attachmentsAfterDelete.comments.length).to.be(0); + expect(attachmentsAfterDelete2.comments.length).to.be(0); + }); + }); + describe('rbac', () => { + describe('files', () => { + // we need api_int_users and roles because they have authorization for the actual plugins (not the fixtures). This + // is needed because the fixture plugins are not registered as file kinds + before(async () => { + await createUsersAndRoles(getService, api_int_users, api_int_roles); + }); + + after(async () => { + await deleteUsersAndRoles(getService, api_int_users, api_int_roles); + }); + + it('should delete a case when the user has access to delete the case and files', async () => { + const { caseInfo: postedCase } = await createCaseWithFiles({ + supertest: supertestWithoutAuth, + fileKind: SECURITY_SOLUTION_FILE_KIND, + owner: 'securitySolution', + auth: { user: secAllUser, space: 'space1' }, + }); + + await deleteCases({ + supertest: supertestWithoutAuth, + caseIDs: [postedCase.id], + auth: { user: secAllUser, space: 'space1' }, + }); + + const [filesAfterDelete, attachmentsAfterDelete] = await Promise.all([ + listFiles({ + supertest: supertestWithoutAuth, + params: { + kind: SECURITY_SOLUTION_FILE_KIND, + }, + auth: { user: secAllUser, space: 'space1' }, + }), + findAttachments({ + supertest: supertestWithoutAuth, + caseId: postedCase.id, + query: { + perPage: MAX_DOCS_PER_PAGE, + }, + auth: { user: secAllUser, space: 'space1' }, + }), + ]); + + expect(filesAfterDelete.total).to.be(0); + expect(attachmentsAfterDelete.comments.length).to.be(0); + }); + + it('should not delete a case when the user does not have access to the file kind of the files', async () => { + const postedCase = await createCase( + supertestWithoutAuth, + getPostCaseRequest({ owner: 'securitySolution' }), + 200, + { user: secAllUser, space: 'space1' } + ); + const { create: createdFile } = await createAndUploadFile({ + supertest: supertestWithoutAuth, + createFileParams: { + name: 'testfile', + // use observability for the file kind which the security user should not have access to + kind: OBSERVABILITY_FILE_KIND, + mimeType: 'text/plain', + meta: { + caseIds: [postedCase.id], + owner: [postedCase.owner], + }, + }, + data: 'abc', + auth: { user: superUser, space: 'space1' }, + }); + + await bulkCreateAttachments({ + supertest: supertestWithoutAuth, + caseId: postedCase.id, + params: [ + getFilesAttachmentReq({ + externalReferenceId: createdFile.file.id, + owner: 'securitySolution', + }), + ], + auth: { user: secAllUser, space: 'space1' }, + }); + + await deleteCases({ + supertest: supertestWithoutAuth, + caseIDs: [postedCase.id], + auth: { user: secAllUser, space: 'space1' }, + expectedHttpCode: 403, + }); + + const [filesAfterDelete, attachmentsAfterDelete] = await Promise.all([ + listFiles({ + supertest: supertestWithoutAuth, + params: { + kind: OBSERVABILITY_FILE_KIND, + }, + auth: { user: superUser, space: 'space1' }, + }), + findAttachments({ + supertest: supertestWithoutAuth, + caseId: postedCase.id, + query: { + perPage: MAX_DOCS_PER_PAGE, + }, + auth: { user: secAllUser, space: 'space1' }, + }), + ]); + + expect(filesAfterDelete.total).to.be(1); + expect(attachmentsAfterDelete.comments.length).to.be(1); + }); + }); + it('User: security solution only - should delete a case', async () => { const postedCase = await createCase( supertestWithoutAuth, @@ -255,3 +470,69 @@ export default ({ getService }: FtrProviderContext): void => { }); }); }; + +const createCaseWithFiles = async ({ + supertest, + fileKind, + owner, + auth = { user: superUser, space: null }, +}: { + supertest: SuperTest.SuperTest; + fileKind: string; + owner: string; + auth?: { user: User; space: string | null }; +}) => { + const postedCase = await createCase(supertest, getPostCaseRequest({ owner }), 200, auth); + + const files = await Promise.all([ + createAndUploadFile({ + supertest, + createFileParams: { + name: 'testfile', + kind: fileKind, + mimeType: 'text/plain', + meta: { + caseIds: [postedCase.id], + owner: [postedCase.owner], + }, + }, + data: 'abc', + auth, + }), + createAndUploadFile({ + supertest, + createFileParams: { + name: 'testfile', + kind: fileKind, + mimeType: 'text/plain', + meta: { + caseIds: [postedCase.id], + owner: [postedCase.owner], + }, + }, + data: 'abc', + auth, + }), + ]); + + const caseWithAttachments = await bulkCreateAttachments({ + supertest, + caseId: postedCase.id, + params: [ + getFilesAttachmentReq({ + externalReferenceId: files[0].create.file.id, + owner, + }), + getFilesAttachmentReq({ + externalReferenceId: files[1].create.file.id, + owner, + }), + ], + auth, + }); + + return { + caseInfo: caseWithAttachments, + attachments: files, + }; +}; From 04f1be1eddddfb0f37f8514a33833ad42108bb46 Mon Sep 17 00:00:00 2001 From: Zacqary Adam Xeper Date: Mon, 10 Apr 2023 11:29:37 -0500 Subject: [PATCH 006/117] [RAM] Fix error message flash and throttle value reset (#154497) ## Summary Fixes #152943 - Error message will no longer flash when switching notify when to `On custom action intervals` - Switching away and back to `On custom action intervals` will preserve the throttle value as well as the throttle unit --- .../sections/action_connector_form/action_type_form.tsx | 6 ++++-- .../public/application/sections/rule_form/rule_errors.ts | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_type_form.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_type_form.tsx index 5079f2b2cff73..b5df2e5d6ac6b 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_type_form.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_type_form.tsx @@ -258,8 +258,10 @@ export const ActionTypeForm = ({ )} onThrottleChange={useCallback( (throttle: number | null, throttleUnit: string) => { - setActionThrottle(throttle); - setActionThrottleUnit(throttleUnit); + if (throttle) { + setActionThrottle(throttle); + setActionThrottleUnit(throttleUnit); + } setActionFrequencyProperty( 'throttle', throttle ? `${throttle}${throttleUnit}` : null, diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_errors.ts b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_errors.ts index ec66f810b3097..02e2e7c084009 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_errors.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_errors.ts @@ -60,7 +60,8 @@ export function validateBaseProperties( } const invalidThrottleActions = ruleObject.actions.filter((a) => { - const throttleDuration = a.frequency?.throttle ? parseDuration(a.frequency.throttle) : 0; + if (!a.frequency?.throttle) return false; + const throttleDuration = parseDuration(a.frequency.throttle); const intervalDuration = ruleObject.schedule.interval && ruleObject.schedule.interval.length > 1 ? parseDuration(ruleObject.schedule.interval) From a0d754c5953458e5fa4c17ec26afccb0cbc5757a Mon Sep 17 00:00:00 2001 From: Zacqary Adam Xeper Date: Mon, 10 Apr 2023 11:29:49 -0500 Subject: [PATCH 007/117] [RAM] Fix Delete Schedule button padding issue (#154503) ## Summary Closes #152398 Fixes the bug with the `Delete schedule` button losing its padding when a date value goes invalid --- .../rules_list/components/rule_snooze/scheduler.tsx | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/rules_list/components/rule_snooze/scheduler.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/rules_list/components/rule_snooze/scheduler.tsx index 7ef4aefecc28c..c4e6160955e28 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/rules_list/components/rule_snooze/scheduler.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/rules_list/components/rule_snooze/scheduler.tsx @@ -362,7 +362,16 @@ const RuleSnoozeSchedulerPanel: React.FunctionComponent = ({ {(initialSchedule || showDelete) && ( <> {!inPopover && } - + {!inPopover && } From 394174eedf756d5676773f22a72828b786bf6b56 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 10 Apr 2023 12:38:45 -0400 Subject: [PATCH 008/117] [Fleet] Disable ILM policy in serverless (#154460) --- config/serverless.yml | 1 + x-pack/plugins/fleet/common/types/index.ts | 3 + x-pack/plugins/fleet/server/config.ts | 8 ++ .../template/default_settings.ts | 14 ++- .../epm/packages/_install_package.test.ts | 118 +++++++++++++++++- .../services/epm/packages/_install_package.ts | 30 +++-- 6 files changed, 156 insertions(+), 18 deletions(-) diff --git a/config/serverless.yml b/config/serverless.yml index 6eabe0049fbfd..80e4ab584f9f6 100644 --- a/config/serverless.yml +++ b/config/serverless.yml @@ -1 +1,2 @@ xpack.fleet.enableExperimental: ['fleetServerStandalone'] +xpack.fleet.internal.ILMPoliciesDisabled: true diff --git a/x-pack/plugins/fleet/common/types/index.ts b/x-pack/plugins/fleet/common/types/index.ts index d3a7f9ee95bd1..0daea71cd8271 100644 --- a/x-pack/plugins/fleet/common/types/index.ts +++ b/x-pack/plugins/fleet/common/types/index.ts @@ -44,6 +44,9 @@ export interface FleetConfigType { disableRegistryVersionCheck?: boolean; bundledPackageLocation?: string; }; + internal?: { + disableILMPolicies: boolean; + }; } // Calling Object.entries(PackagesGroupedByStatus) gave `status: string` diff --git a/x-pack/plugins/fleet/server/config.ts b/x-pack/plugins/fleet/server/config.ts index d5312bf9bc65c..267d9873aaa2d 100644 --- a/x-pack/plugins/fleet/server/config.ts +++ b/x-pack/plugins/fleet/server/config.ts @@ -158,6 +158,14 @@ export const config: PluginConfigDescriptor = { } }, }), + + internal: schema.maybe( + schema.object({ + disableILMPolicies: schema.boolean({ + defaultValue: false, + }), + }) + ), }), }; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts index 658dde2ed760a..fe90954966f24 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts @@ -61,12 +61,18 @@ export function buildDefaultSettings({ : defaultFields ).map((field) => field.name); + const isILMPolicyDisabled = appContextService.getConfig()?.internal?.disableILMPolicies ?? false; + return { index: { - // ILM Policy must be added here, for now point to the default global ILM policy name - lifecycle: { - name: ilmPolicy ? ilmPolicy : type, - }, + ...(isILMPolicyDisabled + ? {} + : { + // ILM Policy must be added here, for now point to the default global ILM policy name + lifecycle: { + name: ilmPolicy ? ilmPolicy : type, + }, + }), // What should be our default for the compression? codec: 'best_compression', // All the default fields which should be queried have to be added here. diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts index 23cdd399344e7..7deadc45df8cd 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts @@ -8,11 +8,13 @@ import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/server'; import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; import { loggerMock } from '@kbn/logging-mocks'; - import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; import { appContextService } from '../../app_context'; import { createAppContextStartContractMock } from '../../../mocks'; +import { saveArchiveEntries } from '../archive/storage'; +import { installILMPolicy } from '../elasticsearch/ilm/install'; +import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install'; jest.mock('../elasticsearch/template/template'); jest.mock('../kibana/assets/install'); @@ -20,6 +22,10 @@ jest.mock('../kibana/index_pattern/install'); jest.mock('./install'); jest.mock('./get'); +jest.mock('../archive/storage'); +jest.mock('../elasticsearch/ilm/install'); +jest.mock('../elasticsearch/datastream_ilm/install'); + import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; import { installKibanaAssetsAndReferences } from '../kibana/assets/install'; @@ -47,8 +53,22 @@ describe('_installPackage', () => { beforeEach(async () => { soClient = savedObjectsClientMock.create(); + + soClient.update.mockImplementation(async (type, id, attributes) => { + return { id, attributes } as any; + }); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; appContextService.start(createAppContextStartContractMock()); + jest.mocked(installILMPolicy).mockReset(); + jest.mocked(installIlmForDataStream).mockReset(); + jest.mocked(installIlmForDataStream).mockResolvedValue({ + esReferences: [], + installedIlms: [], + }); + jest.mocked(saveArchiveEntries).mockResolvedValue({ + saved_objects: [], + }); }); afterEach(async () => { appContextService.stop(); @@ -96,4 +116,100 @@ describe('_installPackage', () => { await expect(installationPromise).rejects.toThrow('mocked'); await expect(installationPromise).rejects.toThrow('should be caught'); }); + + it('do not install ILM policies if disabled in config', async () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + }, + }) + ); + // force errors from this function + mockedInstallKibanaAssetsAndReferences.mockResolvedValue([]); + // pick any function between when those are called and when await Promise.all is defined later + // and force it to take long enough for the errors to occur + // @ts-expect-error about call signature + mockedUpdateCurrentWriteIndices.mockImplementation(async () => await sleep(1000)); + mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({ + installedTemplates: [], + esReferences: [], + }); + await _installPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(installILMPolicy).not.toBeCalled(); + expect(installIlmForDataStream).not.toBeCalled(); + // if we have a .catch this will fail nicely (test pass) + // otherwise the test will fail with either of the mocked errors + // await expect(installationPromise).rejects.toThrow('mocked'); + // await expect(installationPromise).rejects.toThrow('should be caught'); + }); + + it('install ILM policies if not disabled in config', async () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: false, + }, + }) + ); + // force errors from this function + mockedInstallKibanaAssetsAndReferences.mockResolvedValue([]); + // pick any function between when those are called and when await Promise.all is defined later + // and force it to take long enough for the errors to occur + // @ts-expect-error about call signature + mockedUpdateCurrentWriteIndices.mockImplementation(async () => await sleep(1000)); + mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({ + installedTemplates: [], + esReferences: [], + }); + await _installPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(installILMPolicy).toBeCalled(); + expect(installIlmForDataStream).toBeCalled(); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 7da483d0794a2..68c981a308f82 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -153,20 +153,24 @@ export async function _installPackage({ // currently only the base package has an ILM policy // at some point ILM policies can be installed/modified // per data stream and we should then save them - esReferences = await withPackageSpan('Install ILM policies', () => - installILMPolicy(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences) - ); + const isILMPoliciesDisabled = + appContextService.getConfig()?.internal?.disableILMPolicies ?? false; + if (!isILMPoliciesDisabled) { + esReferences = await withPackageSpan('Install ILM policies', () => + installILMPolicy(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences) + ); - ({ esReferences } = await withPackageSpan('Install Data Stream ILM policies', () => - installIlmForDataStream( - packageInfo, - paths, - esClient, - savedObjectsClient, - logger, - esReferences - ) - )); + ({ esReferences } = await withPackageSpan('Install Data Stream ILM policies', () => + installIlmForDataStream( + packageInfo, + paths, + esClient, + savedObjectsClient, + logger, + esReferences + ) + )); + } // installs ml models esReferences = await withPackageSpan('Install ML models', () => From 77498a9b69a929127bffe0da4bf836862b53e8ba Mon Sep 17 00:00:00 2001 From: Karl Godard Date: Mon, 10 Apr 2023 10:10:23 -0700 Subject: [PATCH 009/117] Fix for k8s dashboard getting letterboxed when there is no data. (#154094) ## Summary Fixes an display issue with the k8s dashboard. Currently this page doesn't render an empty state like other pages do, and so I've removed it from the list of pages with "empty state" in order to avoid the whole page having a restricted width of ~960px. Before: ![image](https://user-images.githubusercontent.com/16198204/228934077-add3ecdf-2845-4714-98f2-e791133505e8.png) After: ![image](https://user-images.githubusercontent.com/16198204/228934155-f785c0b9-2fbc-4a04-9522-4eecdd5d2a6c.png) --- .../common/utils/empty_view/use_show_pages_with_empty_view.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/common/utils/empty_view/use_show_pages_with_empty_view.tsx b/x-pack/plugins/security_solution/public/common/utils/empty_view/use_show_pages_with_empty_view.tsx index 7f9688ef3c04f..e9f54f8a9284f 100644 --- a/x-pack/plugins/security_solution/public/common/utils/empty_view/use_show_pages_with_empty_view.tsx +++ b/x-pack/plugins/security_solution/public/common/utils/empty_view/use_show_pages_with_empty_view.tsx @@ -18,7 +18,6 @@ const isPageNameWithEmptyView = (currentName: string) => { SecurityPageName.timelines, SecurityPageName.overview, SecurityPageName.users, - SecurityPageName.kubernetes, ]; return pageNamesWithEmptyView.includes(currentName); }; From a0f8d910b7715e6a0c9124e829ffb27681d5c8a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cau=C3=AA=20Marcondes?= <55978943+cauemarcondes@users.noreply.github.com> Date: Mon, 10 Apr 2023 13:11:04 -0400 Subject: [PATCH 010/117] [Profling] Adding symbols callout on frame information window (#154478) This PR adds a callout on the Frame information window for each symbol status and Storybook support. `Symbolized`: Screenshot 2023-04-05 at 2 46 36 PM `Native language`: Screenshot 2023-04-05 at 2 46 24 PM `Interpreted language`: Screenshot 2023-04-05 at 2 46 16 PM --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- src/dev/storybook/aliases.ts | 1 + .../profiling/.storybook/jest_setup.js | 11 +++ x-pack/plugins/profiling/.storybook/main.js | 8 ++ .../plugins/profiling/.storybook/preview.js | 10 ++ .../profiling/common/profiling.test.ts | 43 +++++++++ x-pack/plugins/profiling/common/profiling.ts | 55 ++++++++--- .../mock_profiling_dependencies_storybook.tsx | 93 +++++++++++++++++++ .../frame_information_tooltip.tsx | 2 +- .../frame_information_window/index.tsx | 13 +++ .../missing_symbols_callout.stories.tsx | 42 +++++++++ .../missing_symbols_callout.tsx | 90 ++++++++++++++++++ .../profiling_app_page_template/index.tsx | 2 +- x-pack/plugins/profiling/tsconfig.json | 3 + 13 files changed, 360 insertions(+), 13 deletions(-) create mode 100644 x-pack/plugins/profiling/.storybook/jest_setup.js create mode 100644 x-pack/plugins/profiling/.storybook/main.js create mode 100644 x-pack/plugins/profiling/.storybook/preview.js create mode 100644 x-pack/plugins/profiling/public/components/contexts/profiling_dependencies/mock_profiling_dependencies_storybook.tsx create mode 100644 x-pack/plugins/profiling/public/components/frame_information_window/missing_symbols_callout.stories.tsx create mode 100644 x-pack/plugins/profiling/public/components/frame_information_window/missing_symbols_callout.tsx diff --git a/src/dev/storybook/aliases.ts b/src/dev/storybook/aliases.ts index 76a0c23992d7a..3ad6852936aed 100644 --- a/src/dev/storybook/aliases.ts +++ b/src/dev/storybook/aliases.ts @@ -50,4 +50,5 @@ export const storybookAliases = { triggers_actions_ui: 'x-pack/plugins/triggers_actions_ui/.storybook', ui_actions_enhanced: 'src/plugins/ui_actions_enhanced/.storybook', unified_search: 'src/plugins/unified_search/.storybook', + profiling: 'x-pack/plugins/profiling/.storybook', }; diff --git a/x-pack/plugins/profiling/.storybook/jest_setup.js b/x-pack/plugins/profiling/.storybook/jest_setup.js new file mode 100644 index 0000000000000..32071b8aa3f62 --- /dev/null +++ b/x-pack/plugins/profiling/.storybook/jest_setup.js @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { setGlobalConfig } from '@storybook/testing-react'; +import * as globalStorybookConfig from './preview'; + +setGlobalConfig(globalStorybookConfig); diff --git a/x-pack/plugins/profiling/.storybook/main.js b/x-pack/plugins/profiling/.storybook/main.js new file mode 100644 index 0000000000000..86b48c32f103e --- /dev/null +++ b/x-pack/plugins/profiling/.storybook/main.js @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = require('@kbn/storybook').defaultConfig; diff --git a/x-pack/plugins/profiling/.storybook/preview.js b/x-pack/plugins/profiling/.storybook/preview.js new file mode 100644 index 0000000000000..3200746243d47 --- /dev/null +++ b/x-pack/plugins/profiling/.storybook/preview.js @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiThemeProviderDecorator } from '@kbn/kibana-react-plugin/common'; + +export const decorators = [EuiThemeProviderDecorator]; diff --git a/x-pack/plugins/profiling/common/profiling.test.ts b/x-pack/plugins/profiling/common/profiling.test.ts index df014bada46bb..5115055ad3c94 100644 --- a/x-pack/plugins/profiling/common/profiling.test.ts +++ b/x-pack/plugins/profiling/common/profiling.test.ts @@ -8,11 +8,14 @@ import { createStackFrameID, createStackFrameMetadata, + FrameSymbolStatus, FrameType, getAddressFromStackFrameID, getCalleeFunction, getCalleeSource, getFileIDFromStackFrameID, + getFrameSymbolStatus, + getLanguageType, } from './profiling'; describe('Stack frame operations', () => { @@ -88,3 +91,43 @@ describe('Stack frame metadata operations', () => { expect(getCalleeSource(metadata)).toEqual('runtime/malloc.go'); }); }); + +describe('getFrameSymbolStatus', () => { + it('returns partially symbolized when metadata has executable name but no source name and source line', () => { + expect(getFrameSymbolStatus({ sourceFilename: '', sourceLine: 0, exeFileName: 'foo' })).toEqual( + FrameSymbolStatus.PARTIALLY_SYMBOLYZED + ); + }); + it('returns not symbolized when metadata has no source name and source line and executable name', () => { + expect(getFrameSymbolStatus({ sourceFilename: '', sourceLine: 0 })).toEqual( + FrameSymbolStatus.NOT_SYMBOLIZED + ); + }); + + it('returns symbolized when metadata has source name and source line', () => { + expect(getFrameSymbolStatus({ sourceFilename: 'foo', sourceLine: 10 })).toEqual( + FrameSymbolStatus.SYMBOLIZED + ); + }); +}); + +describe('getLanguageType', () => { + [FrameType.Native, FrameType.Kernel].map((type) => + it(`returns native for ${type}`, () => { + expect(getLanguageType({ frameType: type })).toEqual('NATIVE'); + }) + ); + [ + FrameType.JVM, + FrameType.JavaScript, + FrameType.PHP, + FrameType.PHPJIT, + FrameType.Perl, + FrameType.Python, + FrameType.Ruby, + ].map((type) => + it(`returns interpreted for ${type}`, () => { + expect(getLanguageType({ frameType: type })).toEqual('INTERPRETED'); + }) + ); +}); diff --git a/x-pack/plugins/profiling/common/profiling.ts b/x-pack/plugins/profiling/common/profiling.ts index 32f4e920306c4..8cb9558fcfc64 100644 --- a/x-pack/plugins/profiling/common/profiling.ts +++ b/x-pack/plugins/profiling/common/profiling.ts @@ -234,23 +234,56 @@ export function getCalleeFunction(frame: StackFrameMetadata): string { // When there is no function name, only use the executable name return frame.FunctionName ? exeDisplayName + ': ' + frame.FunctionName : exeDisplayName; } +export enum FrameSymbolStatus { + PARTIALLY_SYMBOLYZED = 'PARTIALLY_SYMBOLYZED', + NOT_SYMBOLIZED = 'NOT_SYMBOLIZED', + SYMBOLIZED = 'SYMBOLIZED', +} +export function getFrameSymbolStatus({ + sourceFilename, + sourceLine, + exeFileName, +}: { + sourceFilename: string; + sourceLine: number; + exeFileName?: string; +}) { + if (sourceFilename === '' && sourceLine === 0) { + if (exeFileName) { + return FrameSymbolStatus.PARTIALLY_SYMBOLYZED; + } + + return FrameSymbolStatus.NOT_SYMBOLIZED; + } + + return FrameSymbolStatus.SYMBOLIZED; +} + +const nativeLanguages = [FrameType.Native, FrameType.Kernel]; +export function getLanguageType({ frameType }: { frameType: FrameType }) { + return nativeLanguages.includes(frameType) ? 'NATIVE' : 'INTERPRETED'; +} export function getCalleeSource(frame: StackFrameMetadata): string { - if (frame.SourceFilename === '' && frame.SourceLine === 0) { - if (frame.ExeFileName) { + const frameSymbolStatus = getFrameSymbolStatus({ + sourceFilename: frame.SourceFilename, + sourceLine: frame.SourceLine, + exeFileName: frame.ExeFileName, + }); + + switch (frameSymbolStatus) { + case FrameSymbolStatus.NOT_SYMBOLIZED: { + // If we don't have the executable filename, display + return ''; + } + case FrameSymbolStatus.PARTIALLY_SYMBOLYZED: { // If no source line or filename available, display the executable offset return frame.ExeFileName + '+0x' + frame.AddressOrLine.toString(16); } - - // If we don't have the executable filename, display - return ''; - } - - if (frame.SourceFilename !== '' && frame.SourceLine === 0) { - return frame.SourceFilename; + case FrameSymbolStatus.SYMBOLIZED: { + return frame.SourceFilename + (frame.SourceLine !== 0 ? `#${frame.SourceLine}` : ''); + } } - - return frame.SourceFilename + (frame.SourceLine !== 0 ? `#${frame.SourceLine}` : ''); } export function groupStackFrameMetadataByStackTrace( diff --git a/x-pack/plugins/profiling/public/components/contexts/profiling_dependencies/mock_profiling_dependencies_storybook.tsx b/x-pack/plugins/profiling/public/components/contexts/profiling_dependencies/mock_profiling_dependencies_storybook.tsx new file mode 100644 index 0000000000000..67a28c6839979 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/contexts/profiling_dependencies/mock_profiling_dependencies_storybook.tsx @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { CoreStart } from '@kbn/core/public'; +import { EuiThemeProvider } from '@kbn/kibana-react-plugin/common'; +import { createKibanaReactContext } from '@kbn/kibana-react-plugin/public'; +import { MlLocatorDefinition } from '@kbn/ml-plugin/public'; +import { UrlService } from '@kbn/share-plugin/common/url_service'; +import React, { ReactNode } from 'react'; +import { Observable } from 'rxjs'; +import { ProfilingDependenciesContextProvider } from './profiling_dependencies_context'; + +const urlService = new UrlService({ + navigate: async () => {}, + getUrl: async ({ app, path }, { absolute }) => { + return `${absolute ? 'http://localhost:8888' : ''}/app/${app}${path}`; + }, + shortUrls: () => ({ get: () => {} } as any), +}); +const locator = urlService.locators.create(new MlLocatorDefinition()); + +const mockPlugin = { + ml: { + locator, + }, + data: { + query: { + timefilter: { timefilter: { setTime: () => {}, getTime: () => ({}) } }, + }, + }, +}; + +const mockCore = { + application: { + currentAppId$: new Observable(), + getUrlForApp: (appId: string) => '', + navigateToUrl: (url: string) => {}, + }, + chrome: { + docTitle: { change: () => {} }, + setBreadcrumbs: () => {}, + setHelpExtension: () => {}, + setBadge: () => {}, + }, + docLinks: { + DOC_LINK_VERSION: 'current', + ELASTIC_WEBSITE_URL: 'https://www.elastic.co/', + links: { observability: { guide: '' } }, + }, + http: { + basePath: { + prepend: (path: string) => `/basepath${path}`, + get: () => '/basepath', + }, + }, + i18n: { + Context: ({ children }: { children: ReactNode }) => children, + }, + notifications: { + toasts: { + addWarning: () => {}, + addDanger: () => {}, + add: () => {}, + }, + }, +}; + +const mockProfilingDependenciesContext = { + core: mockCore, + plugins: mockPlugin, +} as any; + +export function MockProfilingDependenciesStorybook({ children }: { children?: ReactNode }) { + const KibanaReactContext = createKibanaReactContext( + mockProfilingDependenciesContext.core as unknown as Partial + ); + + return ( + + + + {children} + + + + ); +} diff --git a/x-pack/plugins/profiling/public/components/frame_information_window/frame_information_tooltip.tsx b/x-pack/plugins/profiling/public/components/frame_information_window/frame_information_tooltip.tsx index 587e660702c75..2ff6a36636d61 100644 --- a/x-pack/plugins/profiling/public/components/frame_information_window/frame_information_tooltip.tsx +++ b/x-pack/plugins/profiling/public/components/frame_information_window/frame_information_tooltip.tsx @@ -14,7 +14,7 @@ interface Props extends FrameInformationWindowProps { export function FrameInformationTooltip({ onClose, ...props }: Props) { return ( - + diff --git a/x-pack/plugins/profiling/public/components/frame_information_window/index.tsx b/x-pack/plugins/profiling/public/components/frame_information_window/index.tsx index 72913b8387707..520136c1f40ac 100644 --- a/x-pack/plugins/profiling/public/components/frame_information_window/index.tsx +++ b/x-pack/plugins/profiling/public/components/frame_information_window/index.tsx @@ -7,10 +7,12 @@ import { EuiFlexGroup, EuiFlexItem, EuiText, EuiTitle } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import React from 'react'; +import { FrameSymbolStatus, getFrameSymbolStatus } from '../../../common/profiling'; import { FrameInformationPanel } from './frame_information_panel'; import { getImpactRows } from './get_impact_rows'; import { getInformationRows } from './get_information_rows'; import { KeyValueList } from './key_value_list'; +import { MissingSymbolsCallout } from './missing_symbols_callout'; export interface Props { frame?: { @@ -41,6 +43,12 @@ export function FrameInformationWindow({ frame, totalSamples, totalSeconds }: Pr ); } + const symbolStatus = getFrameSymbolStatus({ + sourceFilename: frame.sourceFileName, + sourceLine: frame.sourceLine, + exeFileName: frame.exeFileName, + }); + const { fileID, frameType, @@ -76,6 +84,11 @@ export function FrameInformationWindow({ frame, totalSamples, totalSeconds }: Pr + {symbolStatus !== FrameSymbolStatus.SYMBOLIZED && ( + + + + )} diff --git a/x-pack/plugins/profiling/public/components/frame_information_window/missing_symbols_callout.stories.tsx b/x-pack/plugins/profiling/public/components/frame_information_window/missing_symbols_callout.stories.tsx new file mode 100644 index 0000000000000..ba353d48c6995 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/frame_information_window/missing_symbols_callout.stories.tsx @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { Meta } from '@storybook/react'; +import React from 'react'; +import { FrameType } from '../../../common/profiling'; +import { MockProfilingDependenciesStorybook } from '../contexts/profiling_dependencies/mock_profiling_dependencies_storybook'; +import { MissingSymbolsCallout } from './missing_symbols_callout'; + +const stories: Meta<{}> = { + title: 'shared/Frame information window/Missing symbols', + component: MissingSymbolsCallout, + decorators: [ + (StoryComponent, { globals }) => { + return ( + + + + ); + }, + ], +}; + +export default stories; + +export function Examples() { + return ( + + + + + + + + + ); +} diff --git a/x-pack/plugins/profiling/public/components/frame_information_window/missing_symbols_callout.tsx b/x-pack/plugins/profiling/public/components/frame_information_window/missing_symbols_callout.tsx new file mode 100644 index 0000000000000..72fbe41c46993 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/frame_information_window/missing_symbols_callout.tsx @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiButton, EuiCallOut, EuiLink } from '@elastic/eui'; +import React from 'react'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { i18n } from '@kbn/i18n'; +import { FrameType, getLanguageType } from '../../../common/profiling'; +import { PROFILING_FEEDBACK_LINK } from '../profiling_app_page_template'; +import { useProfilingDependencies } from '../contexts/profiling_dependencies/use_profiling_dependencies'; + +interface Props { + frameType: FrameType; +} + +export function MissingSymbolsCallout({ frameType }: Props) { + const languageType = getLanguageType({ frameType }); + const { docLinks } = useProfilingDependencies().start.core; + + if (languageType === 'NATIVE') { + return ( + +

+ + {i18n.translate( + 'xpack.profiling.frameInformationWindow.missingSymbols.native.readMore', + { defaultMessage: 'Read more' } + )} + + ), + }} + /> +

+ + {i18n.translate( + 'xpack.profiling.frameInformationWindow.missingSymbols.native.downloadBinary', + { defaultMessage: 'Download elastic-profiling binary' } + )} + + + ); + } + + return ( + +

+ {i18n.translate('xpack.profiling.frameInformationWindow.missingSymbols.interpreted', { + defaultMessage: + 'Symbols are not available because of an error in the unwinder for this language or an unknown error with the interpreter.', + })} +

+ + {i18n.translate( + 'xpack.profiling.frameInformationWindow.missingSymbols.interpreted.reportProblem', + { defaultMessage: 'Report a problem' } + )} + + + ); +} diff --git a/x-pack/plugins/profiling/public/components/profiling_app_page_template/index.tsx b/x-pack/plugins/profiling/public/components/profiling_app_page_template/index.tsx index 38a6f379643ed..4114082efe9e5 100644 --- a/x-pack/plugins/profiling/public/components/profiling_app_page_template/index.tsx +++ b/x-pack/plugins/profiling/public/components/profiling_app_page_template/index.tsx @@ -19,7 +19,7 @@ import { NoDataPageProps } from '@kbn/shared-ux-page-no-data-types'; import { useProfilingDependencies } from '../contexts/profiling_dependencies/use_profiling_dependencies'; import { PrimaryProfilingSearchBar } from './primary_profiling_search_bar'; -const PROFILING_FEEDBACK_LINK = 'https://ela.st/profiling-feedback'; +export const PROFILING_FEEDBACK_LINK = 'https://ela.st/profiling-feedback'; export function ProfilingAppPageTemplate({ children, diff --git a/x-pack/plugins/profiling/tsconfig.json b/x-pack/plugins/profiling/tsconfig.json index 6364139982415..310a8c0b24bef 100644 --- a/x-pack/plugins/profiling/tsconfig.json +++ b/x-pack/plugins/profiling/tsconfig.json @@ -41,6 +41,9 @@ "@kbn/spaces-plugin", "@kbn/cloud-plugin", "@kbn/shared-ux-prompt-not-found", + "@kbn/i18n-react", + "@kbn/ml-plugin", + "@kbn/share-plugin", // add references to other TypeScript projects the plugin depends on // requiredPlugins from ./kibana.json From 9b84061d3b41cd218d4c15c6fb04c6f36d5e700a Mon Sep 17 00:00:00 2001 From: Rickyanto Ang Date: Mon, 10 Apr 2023 10:40:12 -0700 Subject: [PATCH 011/117] [Cloud Posture] Center Framework Sources icons centered (#154604) ## Summary This ticket is part of Quick Wins ticket This PR fixes the issue with Center Framework Sources icons not being centered on findings and rule flyout Screen Shot 2023-04-06 at 1 42 13 PM --- .../pages/configurations/findings_flyout/findings_flyout.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/cloud_security_posture/public/pages/configurations/findings_flyout/findings_flyout.tsx b/x-pack/plugins/cloud_security_posture/public/pages/configurations/findings_flyout/findings_flyout.tsx index 9a611c1059ceb..b1b58a107a5e4 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/configurations/findings_flyout/findings_flyout.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/configurations/findings_flyout/findings_flyout.tsx @@ -93,7 +93,7 @@ export const CisKubernetesIcons = ({ benchmarkId: BenchmarkId; benchmarkName: BenchmarkName; }) => ( - + From a6642d0efe331fd51fa718f4578793d25202e667 Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 10 Apr 2023 12:49:34 -0500 Subject: [PATCH 012/117] Upgrade chromedriver to 112.0.0 (#154652) --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index b4a7a8ac9632c..e277bc93b5d36 100644 --- a/package.json +++ b/package.json @@ -1313,7 +1313,7 @@ "backport": "^8.9.7", "callsites": "^3.1.0", "chance": "1.0.18", - "chromedriver": "^110.0.0", + "chromedriver": "^112.0.0", "clean-webpack-plugin": "^3.0.0", "compression-webpack-plugin": "^4.0.0", "copy-webpack-plugin": "^6.0.2", diff --git a/yarn.lock b/yarn.lock index 4fe380b77943b..29b3711e877d2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11866,10 +11866,10 @@ chrome-trace-event@^1.0.2: dependencies: tslib "^1.9.0" -chromedriver@^110.0.0: - version "110.0.0" - resolved "https://registry.yarnpkg.com/chromedriver/-/chromedriver-110.0.0.tgz#d00a1a2976592d933faa8e9839e97692922834a4" - integrity sha512-Le6q8xrA/3fAt+g8qiN0YjsYxINIhQMC6wj9X3W5L77uN4NspEzklDrqYNwBcEVn7PcAEJ73nLlS7mTyZRspHA== +chromedriver@^112.0.0: + version "112.0.0" + resolved "https://registry.yarnpkg.com/chromedriver/-/chromedriver-112.0.0.tgz#34c86f6cfc8f5e208100088dbb9563439243f8b2" + integrity sha512-fEw1tI05dmK1KK8MGh99LAppP7zCOPEXUxxbYX5wpIBCCmKasyrwZhk/qsdnxJYKd/h0TfiHvGEj7ReDQXW1AA== dependencies: "@testim/chrome-version" "^1.1.3" axios "^1.2.1" From da47a1b7a53d8a1c6a97493fd2828d0e5fd16452 Mon Sep 17 00:00:00 2001 From: Rickyanto Ang Date: Mon, 10 Apr 2023 12:02:01 -0700 Subject: [PATCH 013/117] [Cloud Security] Added new Benchmark Error message component (#154636) ## Summary This ticket is part of Quick Wins ticket. This PR updates the Error message that user receive when the app can't retrieve Benchmark data due to issue with the API Screen Shot 2023-04-09 at 12 39 31 AM --- .../pages/benchmarks/benchmarks.test.tsx | 3 +- .../pages/benchmarks/benchmarks_table.tsx | 43 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/cloud_security_posture/public/pages/benchmarks/benchmarks.test.tsx b/x-pack/plugins/cloud_security_posture/public/pages/benchmarks/benchmarks.test.tsx index 0d9a4eff3f958..2a123f6cfa201 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/benchmarks/benchmarks.test.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/benchmarks/benchmarks.test.tsx @@ -17,6 +17,7 @@ import { useCspBenchmarkIntegrations } from './use_csp_benchmark_integrations'; import { useCspSetupStatusApi } from '../../common/api/use_setup_status_api'; import { useSubscriptionStatus } from '../../common/hooks/use_subscription_status'; import { useCspIntegrationLink } from '../../common/navigation/use_csp_integration_link'; +import { ERROR_STATE_TEST_SUBJECT } from './benchmarks_table'; jest.mock('./use_csp_benchmark_integrations'); jest.mock('../../common/api/use_setup_status_api'); @@ -80,7 +81,7 @@ describe('', () => { const error = new Error('message'); renderBenchmarks(createReactQueryResponse({ status: 'error', error })); - expect(screen.getByText(error.message)).toBeInTheDocument(); + expect(screen.getByTestId(ERROR_STATE_TEST_SUBJECT)).toBeInTheDocument(); }); it('renders the benchmarks table', () => { diff --git a/x-pack/plugins/cloud_security_posture/public/pages/benchmarks/benchmarks_table.tsx b/x-pack/plugins/cloud_security_posture/public/pages/benchmarks/benchmarks_table.tsx index 35515ea1cca52..e37a9d6fcbe9c 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/benchmarks/benchmarks_table.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/benchmarks/benchmarks_table.tsx @@ -14,18 +14,24 @@ import { EuiLink, EuiToolTip, EuiAvatar, + EuiEmptyPrompt, } from '@elastic/eui'; import React from 'react'; import { generatePath } from 'react-router-dom'; import { pagePathGetters } from '@kbn/fleet-plugin/public'; import { i18n } from '@kbn/i18n'; import type { PackagePolicy } from '@kbn/fleet-plugin/common'; +import { FormattedMessage } from '@kbn/i18n-react'; import { TimestampTableCell } from '../../components/timestamp_table_cell'; import type { Benchmark } from '../../../common/types'; import { useKibana } from '../../common/hooks/use_kibana'; import { benchmarksNavigation } from '../../common/navigation/constants'; import * as TEST_SUBJ from './test_subjects'; import { getEnabledCspIntegrationDetails } from '../../common/utils/get_enabled_csp_integration_details'; +import { isCommonError } from '../../components/cloud_posture_page'; +import { FullSizeCenteredPage } from '../../components/full_size_centered_page'; + +export const ERROR_STATE_TEST_SUBJECT = 'benchmark_page_error'; interface BenchmarksTableProps extends Pick, 'loading' | 'error' | 'noItemsMessage' | 'sorting'>, @@ -67,6 +73,39 @@ const IntegrationButtonLink = ({ ); }; +const ErrorMessageComponent = (error: { error: unknown }) => ( + + + + + } + body={ + isCommonError(error) ? ( +

+ +

+ ) : undefined + } + /> + +); + const BENCHMARKS_TABLE_COLUMNS: Array> = [ { field: 'package_policy.name', @@ -193,6 +232,10 @@ export const BenchmarksTable = ({ setQuery({ page: { ...page, index: page.index + 1 }, sort }); }; + if (error) { + return ; + } + return ( Date: Mon, 10 Apr 2023 15:11:12 -0400 Subject: [PATCH 014/117] [Fleet] Loosen validation for experimental data stream features (#154661) ## Summary Closes https://github.com/elastic/kibana/issues/154585 Allow `PUT`ing partial request bodies for experimental features, which ensures we don't break package policies from prior versions of Kibana where not all experimental features were available. Added an FTR test to catch the error, which is an API schema validation one. Verified the test fails before the schema changes are made. ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../plugins/fleet/common/types/models/epm.ts | 4 ++-- .../server/services/epm/packages/update.ts | 2 +- ...experimental_datastream_features_helper.ts | 9 ++++++-- .../server/types/models/package_policy.ts | 8 +++---- .../apis/epm/data_stream.ts | 23 ++++++++++++++++++- 5 files changed, 36 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 96977bc351b12..41974a5fa04d5 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -471,7 +471,7 @@ export type ExperimentalIndexingFeature = export interface ExperimentalDataStreamFeature { data_stream: string; - features: Record; + features: Partial>; } export interface Installation extends SavedObjectAttributes { @@ -493,7 +493,7 @@ export interface Installation extends SavedObjectAttributes { // TypeScript doesn't like using the `ExperimentalDataStreamFeature` type defined above here experimental_data_stream_features?: Array<{ data_stream: string; - features: Record; + features: Partial>; }>; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/update.ts b/x-pack/plugins/fleet/server/services/epm/packages/update.ts index 3f3633708b94c..3072dfed86636 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/update.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/update.ts @@ -56,7 +56,7 @@ export async function updateDatastreamExperimentalFeatures( pkgName: string, dataStreamFeatureMapping: Array<{ data_stream: string; - features: Record; + features: Partial>; }> ) { auditLoggingService.writeCustomSoAuditLog({ diff --git a/x-pack/plugins/fleet/server/services/experimental_datastream_features_helper.ts b/x-pack/plugins/fleet/server/services/experimental_datastream_features_helper.ts index 029215e282383..a33f936a613f4 100644 --- a/x-pack/plugins/fleet/server/services/experimental_datastream_features_helper.ts +++ b/x-pack/plugins/fleet/server/services/experimental_datastream_features_helper.ts @@ -46,7 +46,11 @@ export const applyDocOnlyValueToMapping = ( 'scaled_float', 'unsigned_long', ]; - if (isDocValueOnlyNumericChanged && numericTypes.includes(mapping.type ?? '')) { + if ( + isDocValueOnlyNumericChanged && + numericTypes.includes(mapping.type ?? '') && + featureMap.features.doc_value_only_numeric !== undefined + ) { updateMapping(mapping, featureMap.features.doc_value_only_numeric); } @@ -54,7 +58,8 @@ export const applyDocOnlyValueToMapping = ( if ( isDocValueOnlyOtherChanged && name !== '@timestamp' && - otherSupportedTypes.includes(mapping.type ?? '') + otherSupportedTypes.includes(mapping.type ?? '') && + featureMap.features.doc_value_only_other !== undefined ) { updateMapping(mapping, featureMap.features.doc_value_only_other); } diff --git a/x-pack/plugins/fleet/server/types/models/package_policy.ts b/x-pack/plugins/fleet/server/types/models/package_policy.ts index e1851693d3367..32199c2e8be48 100644 --- a/x-pack/plugins/fleet/server/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/server/types/models/package_policy.ts @@ -83,10 +83,10 @@ const ExperimentalDataStreamFeatures = schema.arrayOf( schema.object({ data_stream: schema.string(), features: schema.object({ - synthetic_source: schema.boolean(), - tsdb: schema.boolean(), - doc_value_only_numeric: schema.boolean(), - doc_value_only_other: schema.boolean(), + synthetic_source: schema.maybe(schema.boolean({ defaultValue: false })), + tsdb: schema.maybe(schema.boolean({ defaultValue: false })), + doc_value_only_numeric: schema.maybe(schema.boolean({ defaultValue: false })), + doc_value_only_other: schema.maybe(schema.boolean({ defaultValue: false })), }), }) ); diff --git a/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts b/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts index 2e5197f4543c8..80a5a2f5738fc 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts @@ -228,7 +228,7 @@ export default function (providerContext: FtrProviderContext) { return resLogsDatastream.body.data_streams[0].indices.length; } - it('should rollover datstream after enabling a expiremental datastream feature that need a rollover', async () => { + it('should rollover datstream after enabling a experimental datastream feature that need a rollover', async () => { expect(await getLogsDefaultBackingIndicesLength()).to.be(1); await supertest @@ -256,6 +256,27 @@ export default function (providerContext: FtrProviderContext) { // Datastream should have been rolled over expect(await getLogsDefaultBackingIndicesLength()).to.be(2); }); + + it('should allow updating a package policy with only a partial set of experimental datastream features', async () => { + await supertest + .put(`/api/fleet/package_policies/${packagePolicyId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + ...packagePolicyData, + package: { + ...packagePolicyData.package, + experimental_data_stream_features: [ + { + data_stream: logsTemplateName, + features: { + synthetic_source: true, + }, + }, + ], + }, + }) + .expect(200); + }); }); }); } From 813c2a3c64d3eb3309c6f6a3de6d1d3b53072598 Mon Sep 17 00:00:00 2001 From: Philippe Oberti Date: Mon, 10 Apr 2023 14:18:38 -0500 Subject: [PATCH 015/117] [Security Solution] skip failing related_integrations e2e test (#154662) --- .../cypress/e2e/detection_rules/related_integrations.cy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/cypress/e2e/detection_rules/related_integrations.cy.ts b/x-pack/plugins/security_solution/cypress/e2e/detection_rules/related_integrations.cy.ts index 663317f334751..5b316b4136708 100644 --- a/x-pack/plugins/security_solution/cypress/e2e/detection_rules/related_integrations.cy.ts +++ b/x-pack/plugins/security_solution/cypress/e2e/detection_rules/related_integrations.cy.ts @@ -111,7 +111,7 @@ describe('Related integrations', () => { }); }); - context( + context.skip( 'installed integrations: Amazon CloudFront, AWS CloudTrail, System, enabled integrations: Amazon CloudFront, Aws Cloudfront, System', () => { const rule = { From ee424cb3add25e755798dae6ddab71bb6e8e041d Mon Sep 17 00:00:00 2001 From: Sean Story Date: Mon, 10 Apr 2023 16:12:40 -0500 Subject: [PATCH 016/117] Update field mappings for ELSER pipelines (#154291) ## Summary When creating an ML inference pipeline that uses the new ELSER model, we need to be able to specify which fields to apply it to, and where the resulting values should go. These fields can then determine what mapping changes need to be made to the underlying index (setting `rank_features` fields). This PR adds `field_mappings` as a param to our kibana endpoint for creating these ML Inference pipelines, and based on that parameter, attempts to update mappings. A sample request might be: ``` curl -XPOST -u elastic:changeme "http://localhost:5601/mob/internal/enterprise_search/indices/search-test/ml_inference/pipeline_processors" \ -H 'kbn-xsrf: kibana' \ -H 'Content-Type: application/json' \ -d '{ "field_mappings": { "input": "output" }, "pipeline_definition": { "description" : "text", "processors": [] }, "pipeline_name": "test" }' | jq ``` And sample output might be: ``` { "created": "ml-inference-test", "mapping_updated": true } ``` OR ``` { "statusCode": 409, "error": "Conflict", "message": "One or more target fields for this pipeline already exist with a type that is incompatible with the specified model. Ensure that each target field is unique and not already in use.", "attributes": { "error_code": "mapping_update_failed" } } ``` ### Checklist - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --- .../common/types/error_codes.ts | 3 +- .../common/types/pipelines.ts | 13 ++- .../create_ml_inference_pipeline.ts | 25 +++-- .../update_ml_inference_mappings.test.ts | 90 +++++++++++++++++ .../update_ml_inference_mappings.ts | 99 +++++++++++++++++++ .../routes/enterprise_search/indices.test.ts | 35 ++++--- .../routes/enterprise_search/indices.ts | 32 ++++-- .../server/utils/identify_exceptions.ts | 3 + .../plugins/enterprise_search/tsconfig.json | 1 + 9 files changed, 268 insertions(+), 33 deletions(-) create mode 100644 x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.test.ts create mode 100644 x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.ts diff --git a/x-pack/plugins/enterprise_search/common/types/error_codes.ts b/x-pack/plugins/enterprise_search/common/types/error_codes.ts index f250aa2dd9dd3..0ed9a2e1fad99 100644 --- a/x-pack/plugins/enterprise_search/common/types/error_codes.ts +++ b/x-pack/plugins/enterprise_search/common/types/error_codes.ts @@ -11,8 +11,10 @@ export enum ErrorCode { CONNECTOR_DOCUMENT_ALREADY_EXISTS = 'connector_document_already_exists', CRAWLER_ALREADY_EXISTS = 'crawler_already_exists', DOCUMENT_NOT_FOUND = 'document_not_found', + ENGINE_NOT_FOUND = 'engine_not_found', INDEX_ALREADY_EXISTS = 'index_already_exists', INDEX_NOT_FOUND = 'index_not_found', + MAPPING_UPDATE_FAILED = 'mapping_update_failed', PARAMETER_CONFLICT = 'parameter_conflict', PIPELINE_ALREADY_EXISTS = 'pipeline_already_exists', PIPELINE_IS_IN_USE = 'pipeline_is_in_use', @@ -20,5 +22,4 @@ export enum ErrorCode { RESOURCE_NOT_FOUND = 'resource_not_found', UNAUTHORIZED = 'unauthorized', UNCAUGHT_EXCEPTION = 'uncaught_exception', - ENGINE_NOT_FOUND = 'engine_not_found', } diff --git a/x-pack/plugins/enterprise_search/common/types/pipelines.ts b/x-pack/plugins/enterprise_search/common/types/pipelines.ts index a44c8e532e4ed..3e652d97c6f4f 100644 --- a/x-pack/plugins/enterprise_search/common/types/pipelines.ts +++ b/x-pack/plugins/enterprise_search/common/types/pipelines.ts @@ -43,9 +43,16 @@ export interface MlInferenceError { timestamp: string | undefined; // Date string } -export interface CreateMlInferencePipelineResponse { - addedToParentPipeline?: boolean; - created?: boolean; +export interface PreparePipelineAndIndexForMlInferenceResult { + added_to_parent_pipeline?: boolean; + created_pipeline?: boolean; + pipeline_id: string; + + mapping_updated: boolean; +} + +export interface CreatePipelineResult { + created: boolean; id: string; } diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline.ts index 6a2354cde1ac1..eb8cdecb64a02 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline.ts @@ -8,38 +8,43 @@ import { IngestGetPipelineResponse, IngestPipeline } from '@elastic/elasticsearch/lib/api/types'; import { ElasticsearchClient } from '@kbn/core/server'; -import { formatPipelineName } from '../../../../../../common/ml_inference_pipeline'; +import { FieldMapping, formatPipelineName } from '../../../../../../common/ml_inference_pipeline'; import { ErrorCode } from '../../../../../../common/types/error_codes'; import type { - CreateMlInferencePipelineResponse, + PreparePipelineAndIndexForMlInferenceResult, InferencePipelineInferenceConfig, + CreatePipelineResult, } from '../../../../../../common/types/pipelines'; import { addSubPipelineToIndexSpecificMlPipeline } from '../../../../../utils/create_ml_inference_pipeline'; import { getPrefixedInferencePipelineProcessorName } from '../../../../../utils/ml_inference_pipeline_utils'; import { formatMlPipelineBody } from '../../../../pipelines/create_pipeline_definitions'; +import { updateMlInferenceMappings } from '../update_ml_inference_mappings'; /** * Creates a Machine Learning Inference pipeline with the given settings, if it doesn't exist yet, * then references it in the "parent" ML Inference pipeline that is associated with the index. + * Finally, updates the index's mappings to accommodate the specified outputs of the inference model (if able) * @param indexName name of the index this pipeline corresponds to. * @param pipelineName pipeline name set by the user. * @param pipelineDefinition * @param modelId model ID selected by the user. * @param sourceField The document field that model will read. * @param destinationField The document field that the model will write to. + * @param fieldMappings The array of objects representing the source field (text) names and target fields (ML output) names * @param inferenceConfig The configuration for the model. * @param esClient the Elasticsearch Client to use when retrieving pipeline and model details. */ -export const createAndReferenceMlInferencePipeline = async ( +export const preparePipelineAndIndexForMlInference = async ( indexName: string, pipelineName: string, pipelineDefinition: IngestPipeline | undefined, modelId: string | undefined, sourceField: string | undefined, destinationField: string | null | undefined, + fieldMappings: FieldMapping[] | undefined, inferenceConfig: InferencePipelineInferenceConfig | undefined, esClient: ElasticsearchClient -): Promise => { +): Promise => { const createPipelineResult = await createMlInferencePipeline( pipelineName, pipelineDefinition, @@ -56,9 +61,15 @@ export const createAndReferenceMlInferencePipeline = async ( esClient ); + const mappingResponse = fieldMappings + ? (await updateMlInferenceMappings(indexName, fieldMappings, esClient)).acknowledged + : false; + return { - ...createPipelineResult, - addedToParentPipeline: addSubPipelineResult.addedToParentPipeline, + added_to_parent_pipeline: addSubPipelineResult.addedToParentPipeline, + created_pipeline: createPipelineResult.created, + mapping_updated: mappingResponse, + pipeline_id: createPipelineResult.id, }; }; @@ -80,7 +91,7 @@ export const createMlInferencePipeline = async ( destinationField: string | null | undefined, inferenceConfig: InferencePipelineInferenceConfig | undefined, esClient: ElasticsearchClient -): Promise => { +): Promise => { const inferencePipelineGeneratedName = getPrefixedInferencePipelineProcessorName(pipelineName); // Check that a pipeline with the same name doesn't already exist diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.test.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.test.ts new file mode 100644 index 0000000000000..27425274608e1 --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.test.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; + +import { ErrorCode } from '../../../../../common/types/error_codes'; + +import { updateMlInferenceMappings } from './update_ml_inference_mappings'; + +describe('updateMlInferenceMappings', () => { + const indexName = 'my-index'; + + const mockClient = elasticsearchServiceMock.createScopedClusterClient(); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + const expectedMapping = { + ml: { + properties: { + inference: { + properties: { + input_one_expanded: { + properties: { + predicted_value: { + type: 'rank_features', + }, + model_id: { + type: 'keyword', + }, + }, + }, + input_two_expanded: { + properties: { + predicted_value: { + type: 'rank_features', + }, + model_id: { + type: 'keyword', + }, + }, + }, + }, + }, + }, + }, + }; + + const fieldMappings = [ + { + sourceField: 'input_one', + targetField: 'ml.inference.input_one_expanded', + }, + { + sourceField: 'input_two', + targetField: 'ml.inference.input_two_expanded', + }, + ]; + + it('should update mappings for default output', async () => { + await updateMlInferenceMappings(indexName, fieldMappings, mockClient.asCurrentUser); + expect(mockClient.asCurrentUser.indices.putMapping).toHaveBeenLastCalledWith({ + index: indexName, + properties: expectedMapping, + }); + }); + + it('should raise an error if the update fails', async () => { + mockClient.asCurrentUser.indices.putMapping.mockImplementation(() => + Promise.reject({ + meta: { + body: { + error: { + type: 'illegal_argument_exception', + }, + }, + statusCode: 400, + }, + }) + ); + await expect( + updateMlInferenceMappings(indexName, fieldMappings, mockClient.asCurrentUser) + ).rejects.toThrowError(ErrorCode.MAPPING_UPDATE_FAILED); + }); +}); diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.ts new file mode 100644 index 0000000000000..0108187efd120 --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; + +import { FieldMapping } from '../../../../../common/ml_inference_pipeline'; + +import { ErrorCode } from '../../../../../common/types/error_codes'; + +import { isIllegalArgumentException } from '../../../../utils/identify_exceptions'; + +/** + * Creates Elasticsearch field mappings so that the outputs of ML Inference pipelines are not indexed as `float` fields + * @param indexName - the index whose mapping will be updated + * @param fieldMappings - the array of objects representing the source field (text) names and target fields (ML output) names + * @param esClient + */ +export const updateMlInferenceMappings = async ( + indexName: string, + fieldMappings: FieldMapping[], + esClient: ElasticsearchClient +) => { + const sourceFields = fieldMappings.map(({ sourceField }) => sourceField); + + const nonDefaultTargetFields = fieldMappings + .filter( + (fieldMapping) => + fieldMapping.targetField !== `ml.inference.${fieldMapping.sourceField}_expanded` + ) + .map((fieldMapping) => fieldMapping.targetField); + + // Today, we only update mappings for text_expansion fields. + const mapping = generateTextExpansionMappingProperties(sourceFields, nonDefaultTargetFields); + try { + return await esClient.indices.putMapping({ + index: indexName, + properties: mapping, + }); + } catch (e) { + if (isIllegalArgumentException(e)) { + throw new Error(ErrorCode.MAPPING_UPDATE_FAILED); + } else { + throw e; + } + } +}; + +const generateTextExpansionMappingProperties = (sourceFields: string[], targetFields: string[]) => { + return { + ml: { + properties: { + inference: { + properties: { + ...formDefaultElserMappingProps(sourceFields), + }, + }, + }, + }, + ...targetFields.reduce( + (previous, targetField) => ({ + ...previous, + [targetField]: { + properties: { + model_id: { + type: 'keyword', + }, + predicted_value: { + type: 'rank_features', + }, + }, + }, + }), + {} + ), + }; +}; + +const formDefaultElserMappingProps = (sourceFields: string[]) => { + return sourceFields.reduce( + (previous, sourceField) => ({ + ...previous, + [`${sourceField}_expanded`]: { + properties: { + model_id: { + type: 'keyword', + }, + predicted_value: { + type: 'rank_features', + }, + }, + }, + }), + {} + ); +}; diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts index be3c1be297df8..92977df6021d6 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts @@ -26,7 +26,7 @@ jest.mock( jest.mock( '../../lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline', () => ({ - createAndReferenceMlInferencePipeline: jest.fn(), + preparePipelineAndIndexForMlInference: jest.fn(), }) ); jest.mock( @@ -61,7 +61,7 @@ import { indexOrAliasExists } from '../../lib/indices/exists_index'; import { getMlInferenceErrors } from '../../lib/indices/pipelines/ml_inference/get_ml_inference_errors'; import { fetchMlInferencePipelineHistory } from '../../lib/indices/pipelines/ml_inference/get_ml_inference_pipeline_history'; import { attachMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/attach_ml_pipeline'; -import { createAndReferenceMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline'; +import { preparePipelineAndIndexForMlInference } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline'; import { deleteMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline'; import { detachMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline'; import { fetchMlInferencePipelineProcessors } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/get_ml_inference_pipeline_processors'; @@ -306,11 +306,12 @@ describe('Enterprise Search Managed Indices', () => { }); it('creates an ML inference pipeline from model and source_field', async () => { - (createAndReferenceMlInferencePipeline as jest.Mock).mockImplementationOnce(() => { + (preparePipelineAndIndexForMlInference as jest.Mock).mockImplementationOnce(() => { return Promise.resolve({ - id: 'ml-inference-my-pipeline-name', - created: true, - addedToParentPipeline: true, + added_to_parent_pipeline: true, + created_pipeline: true, + mapping_updated: false, + pipeline_id: 'ml-inference-my-pipeline-name', }); }); @@ -319,7 +320,7 @@ describe('Enterprise Search Managed Indices', () => { body: mockRequestBody, }); - expect(createAndReferenceMlInferencePipeline).toHaveBeenCalledWith( + expect(preparePipelineAndIndexForMlInference).toHaveBeenCalledWith( 'my-index-name', mockRequestBody.pipeline_name, undefined, @@ -327,37 +328,41 @@ describe('Enterprise Search Managed Indices', () => { mockRequestBody.source_field, mockRequestBody.destination_field, undefined, + undefined, mockClient.asCurrentUser ); expect(mockRouter.response.ok).toHaveBeenCalledWith({ body: { created: 'ml-inference-my-pipeline-name', + mapping_updated: false, }, headers: { 'content-type': 'application/json' }, }); }); it('creates an ML inference pipeline from pipeline definition', async () => { - (createAndReferenceMlInferencePipeline as jest.Mock).mockImplementationOnce(() => { + (preparePipelineAndIndexForMlInference as jest.Mock).mockImplementationOnce(() => { return Promise.resolve({ - id: 'ml-inference-my-pipeline-name', - created: true, - addedToParentPipeline: true, + added_to_parent_pipeline: true, + created_pipeline: true, + mapping_updated: true, + pipeline_id: 'ml-inference-my-pipeline-name', }); }); await mockRouter.callRoute({ params: { indexName: 'my-index-name' }, body: { - pipeline_name: 'my-pipeline-name', + field_mappings: [], pipeline_definition: { processors: [], }, + pipeline_name: 'my-pipeline-name', }, }); - expect(createAndReferenceMlInferencePipeline).toHaveBeenCalledWith( + expect(preparePipelineAndIndexForMlInference).toHaveBeenCalledWith( 'my-index-name', mockRequestBody.pipeline_name, { @@ -366,6 +371,7 @@ describe('Enterprise Search Managed Indices', () => { undefined, undefined, undefined, + [], undefined, mockClient.asCurrentUser ); @@ -373,13 +379,14 @@ describe('Enterprise Search Managed Indices', () => { expect(mockRouter.response.ok).toHaveBeenCalledWith({ body: { created: 'ml-inference-my-pipeline-name', + mapping_updated: true, }, headers: { 'content-type': 'application/json' }, }); }); it('responds with 409 CONFLICT if the pipeline already exists', async () => { - (createAndReferenceMlInferencePipeline as jest.Mock).mockImplementationOnce(() => { + (preparePipelineAndIndexForMlInference as jest.Mock).mockImplementationOnce(() => { return Promise.reject(new Error(ErrorCode.PIPELINE_ALREADY_EXISTS)); }); diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts index 75cb055b34aac..cad33e310a832 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts @@ -33,7 +33,7 @@ import { generateApiKey } from '../../lib/indices/generate_api_key'; import { getMlInferenceErrors } from '../../lib/indices/pipelines/ml_inference/get_ml_inference_errors'; import { fetchMlInferencePipelineHistory } from '../../lib/indices/pipelines/ml_inference/get_ml_inference_pipeline_history'; import { attachMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/attach_ml_pipeline'; -import { createAndReferenceMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline'; +import { preparePipelineAndIndexForMlInference } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline'; import { deleteMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline'; import { detachMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline'; import { fetchMlInferencePipelineProcessors } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/get_ml_inference_pipeline_processors'; @@ -388,6 +388,11 @@ export function registerIndexRoutes({ }), body: schema.object({ destination_field: schema.maybe(schema.nullable(schema.string())), + field_mappings: schema.maybe( + schema.arrayOf( + schema.object({ sourceField: schema.string(), targetField: schema.string() }) + ) + ), inference_config: schema.maybe( schema.object({ zero_shot_classification: schema.maybe( @@ -421,30 +426,31 @@ export function registerIndexRoutes({ source_field: sourceField, destination_field: destinationField, inference_config: inferenceConfig, + field_mappings: fieldMappings, } = request.body; // additional validations - if (pipelineDefinition && (sourceField || destinationField || modelId)) { + if ((pipelineDefinition || fieldMappings) && (sourceField || destinationField || modelId)) { return createError({ errorCode: ErrorCode.PARAMETER_CONFLICT, message: i18n.translate( 'xpack.enterpriseSearch.server.routes.createMlInferencePipeline.ParameterConflictError', { defaultMessage: - 'pipeline_definition should only be provided if source_field and destination_field and model_id are not provided', + 'pipeline_definition and field_mappings should only be provided if source_field and destination_field and model_id are not provided', } ), response, statusCode: 400, }); - } else if (!(pipelineDefinition || (sourceField && modelId))) { + } else if (!((pipelineDefinition && fieldMappings) || (sourceField && modelId))) { return createError({ errorCode: ErrorCode.PARAMETER_CONFLICT, message: i18n.translate( 'xpack.enterpriseSearch.server.routes.createMlInferencePipeline.ParameterMissingError', { defaultMessage: - 'either pipeline_definition or source_field AND model_id must be provided', + 'either pipeline_definition AND fieldMappings or source_field AND model_id must be provided', } ), response, @@ -454,19 +460,21 @@ export function registerIndexRoutes({ try { // Create the sub-pipeline for inference - const createPipelineResult = await createAndReferenceMlInferencePipeline( + const createPipelineResult = await preparePipelineAndIndexForMlInference( indexName, pipelineName, pipelineDefinition, modelId, sourceField, destinationField, + fieldMappings, inferenceConfig, client.asCurrentUser ); return response.ok({ body: { - created: createPipelineResult?.id, + created: createPipelineResult.pipeline_id, + mapping_updated: createPipelineResult.mapping_updated, }, headers: { 'content-type': 'application/json' }, }); @@ -484,7 +492,15 @@ export function registerIndexRoutes({ statusCode: 409, }); } - + if (fieldMappings && (error as Error).message === ErrorCode.MAPPING_UPDATE_FAILED) { + return createError({ + errorCode: (error as Error).message as ErrorCode, + message: + 'One or more target fields for this pipeline already exist with a type that is incompatible with the specified model. Ensure that each target field is unique and not already in use.', + response, + statusCode: 409, + }); + } throw error; } }) diff --git a/x-pack/plugins/enterprise_search/server/utils/identify_exceptions.ts b/x-pack/plugins/enterprise_search/server/utils/identify_exceptions.ts index 5874659c690f8..04b8a089fa009 100644 --- a/x-pack/plugins/enterprise_search/server/utils/identify_exceptions.ts +++ b/x-pack/plugins/enterprise_search/server/utils/identify_exceptions.ts @@ -36,3 +36,6 @@ export const isPipelineIsInUseException = (error: Error) => export const isNotFoundException = (error: ElasticsearchResponseError) => error.meta?.statusCode === 404; + +export const isIllegalArgumentException = (error: ElasticsearchResponseError) => + error.meta?.body?.error?.type === 'illegal_argument_exception'; diff --git a/x-pack/plugins/enterprise_search/tsconfig.json b/x-pack/plugins/enterprise_search/tsconfig.json index 8d83fd25fdf84..af660c1169d5a 100644 --- a/x-pack/plugins/enterprise_search/tsconfig.json +++ b/x-pack/plugins/enterprise_search/tsconfig.json @@ -57,5 +57,6 @@ "@kbn/expressions-plugin", "@kbn/react-field", "@kbn/field-types", + "@kbn/core-elasticsearch-server-mocks", ] } From 8770a1ceed41959fa1792b360c2295add288a3e4 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Mon, 10 Apr 2023 14:52:43 -0700 Subject: [PATCH 017/117] Update search session saved object mappings (#154401) ## Summary Part of https://github.com/elastic/kibana/pull/153070. Cleans up the saved object mappings for search session saved objects. Removes the non-searchable/sortable properties from the `mappings` property and introduces a `schema` property for validation. --- .../group2/check_registered_types.test.ts | 2 +- .../search/saved_objects/search_session.ts | 56 +++++++++---------- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts index d876484aa16af..53176bd7b9860 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts @@ -126,7 +126,7 @@ describe('checking migration metadata changes on all registered SO types', () => "rules-settings": "9854495c3b54b16a6625fb250c35e5504da72266", "sample-data-telemetry": "c38daf1a49ed24f2a4fb091e6e1e833fccf19935", "search": "ed3a9b1681b57d69560909d51933fdf17576ea68", - "search-session": "58a44d14ec991739166b2ec28d718001ab0f4b28", + "search-session": "fae0dfc63274d6a3b90ca583802c48cab8760637", "search-telemetry": "1bbaf2db531b97fa04399440fa52d46e86d54dd8", "security-rule": "1ff82dfb2298c3caf6888fc3ef15c6bf7a628877", "security-solution-signals-migration": "c2db409c1857d330beb3d6fd188fa186f920302c", diff --git a/src/plugins/data/server/search/saved_objects/search_session.ts b/src/plugins/data/server/search/saved_objects/search_session.ts index ef0f960c84e8c..598229075df42 100644 --- a/src/plugins/data/server/search/saved_objects/search_session.ts +++ b/src/plugins/data/server/search/saved_objects/search_session.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { schema } from '@kbn/config-schema'; import { SavedObjectsType } from '@kbn/core/server'; import { SEARCH_SESSION_TYPE } from '../../../common'; import { searchSessionSavedObjectMigrations } from './search_session_migration'; @@ -15,37 +16,14 @@ export const searchSessionSavedObjectType: SavedObjectsType = { namespaceType: 'single', hidden: true, mappings: { + dynamic: false, properties: { sessionId: { type: 'keyword', }, - name: { - type: 'keyword', - }, created: { type: 'date', }, - expires: { - type: 'date', - }, - appId: { - type: 'keyword', - }, - locatorId: { - type: 'keyword', - }, - initialState: { - dynamic: false, - properties: {}, - }, - restoreState: { - dynamic: false, - properties: {}, - }, - idMapping: { - dynamic: false, - properties: {}, - }, realmType: { type: 'keyword', }, @@ -55,14 +33,32 @@ export const searchSessionSavedObjectType: SavedObjectsType = { username: { type: 'keyword', }, - version: { - type: 'keyword', - }, - isCanceled: { - type: 'boolean', - }, }, }, + schemas: { + '8.8.0': schema.object({ + sessionId: schema.string(), + name: schema.maybe(schema.string()), + created: schema.string(), + expires: schema.string(), + appId: schema.maybe(schema.string()), + locatorId: schema.maybe(schema.string()), + initialState: schema.maybe(schema.object({}, { unknowns: 'allow' })), + restoreState: schema.maybe(schema.object({}, { unknowns: 'allow' })), + idMapping: schema.mapOf( + schema.string(), + schema.object({ + id: schema.string(), + strategy: schema.string(), + }) + ), + realmType: schema.maybe(schema.string()), + realmName: schema.maybe(schema.string()), + username: schema.maybe(schema.string()), + version: schema.string(), + isCanceled: schema.maybe(schema.boolean()), + }), + }, migrations: searchSessionSavedObjectMigrations, excludeOnUpgrade: async () => { return { From 31b50a3906270b7e52a0b74fe236a22cc615e7bc Mon Sep 17 00:00:00 2001 From: Cee Chen <549407+cee-chen@users.noreply.github.com> Date: Mon, 10 Apr 2023 15:06:26 -0700 Subject: [PATCH 018/117] Upgrade EUI to v77.0.0 (#154379) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > ⚠️ Synthetic failures are not related to any EUI changes and are likely already failing on main. Please ignore the failing CI status when reviewing. ## [`77.0.0`](https://github.com/elastic/eui/tree/v77.0.0) **Bug fixes** - Fixed named `EuiBadge` colors to reflect custom theme overrides ([#6659](https://github.com/elastic/eui/pull/6659)) - Fixed user-defined SCSS variables failing to override variables defined in Amsterdam typography overrides. ([#6665](https://github.com/elastic/eui/pull/6665)) - Fixed bold `EuiCode` tokens to actually be bold ([#6666](https://github.com/elastic/eui/pull/6666)) **Breaking changes** - Success- and accent-colored `EuiBadge`s and `EuiButton`s have had their fill colors tinted slightly on light mode to be more readable ([#6659](https://github.com/elastic/eui/pull/6659)) --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Jon --- package.json | 2 +- .../__snapshots__/server_status.test.tsx.snap | 6 +++--- src/dev/license_checker/config.ts | 2 +- .../customize_panel_action.tsx | 2 -- .../__snapshots__/index.test.tsx.snap | 2 +- .../pages/endpoint_hosts/view/index.test.tsx | 19 ------------------- .../__snapshots__/index.test.tsx.snap | 4 ++-- .../__snapshots__/tag_label.test.tsx.snap | 2 +- .../__snapshots__/field.test.tsx.snap | 2 +- .../__snapshots__/tlp_badge.test.tsx.snap | 4 ++-- .../services/ml/data_visualizer_table.ts | 1 + yarn.lock | 8 ++++---- 12 files changed, 17 insertions(+), 37 deletions(-) diff --git a/package.json b/package.json index e277bc93b5d36..7e4a88d72b98e 100644 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "@elastic/datemath": "5.0.3", "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@8.6.0-canary.3", "@elastic/ems-client": "8.4.0", - "@elastic/eui": "76.4.0", + "@elastic/eui": "77.0.0", "@elastic/filesaver": "1.1.2", "@elastic/node-crypto": "1.2.1", "@elastic/numeral": "^2.5.1", diff --git a/packages/core/apps/core-apps-browser-internal/src/status/components/__snapshots__/server_status.test.tsx.snap b/packages/core/apps/core-apps-browser-internal/src/status/components/__snapshots__/server_status.test.tsx.snap index 60d1bad95e3e1..fe17c420479eb 100644 --- a/packages/core/apps/core-apps-browser-internal/src/status/components/__snapshots__/server_status.test.tsx.snap +++ b/packages/core/apps/core-apps-browser-internal/src/status/components/__snapshots__/server_status.test.tsx.snap @@ -5,7 +5,7 @@ exports[`ServerStatus renders correctly for green state 2`] = ` aria-label="Green" class="euiBadge emotion-euiBadge" data-test-subj="serverStatusTitleBadge" - style="background-color: rgb(109, 204, 177); color: rgb(0, 0, 0);" + style="background-color: rgb(77, 210, 202); color: rgb(0, 0, 0);" title="Green" > { size: 's', 'data-test-subj': 'customizePanel', - // @ts-ignore - TODO: Remove this once https://github.com/elastic/eui/pull/6645 lands in Kibana - focusTrapProps: { scrollLock: true }, } ); overlayTracker?.openOverlay(handle); diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/overview/__snapshots__/index.test.tsx.snap b/x-pack/plugins/security_solution/public/common/components/event_details/overview/__snapshots__/index.test.tsx.snap index 89d876e5efa88..b6e9ea7a2ea0f 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/overview/__snapshots__/index.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/common/components/event_details/overview/__snapshots__/index.test.tsx.snap @@ -91,7 +91,7 @@ exports[`Event Details Overview Cards renders rows and spacers correctly 1`] = `