diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index abc6436e7ee0a..6437742d31c15 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -212,13 +212,6 @@ steps: automatic: false soft_fail: true - - command: .buildkite/scripts/steps/code_generation/security_solution_codegen.sh - label: 'Security Solution OpenAPI codegen' - agents: - queue: n2-2-spot - timeout_in_minutes: 60 - parallelism: 1 - - command: .buildkite/scripts/steps/functional/osquery_cypress_burn.sh label: 'Osquery Cypress Tests, burning changed specs' agents: diff --git a/.buildkite/pipelines/quality-gates/pipeline.tests-production-canary.yaml b/.buildkite/pipelines/quality-gates/pipeline.tests-production-canary.yaml deleted file mode 100644 index 8b30d4e141b08..0000000000000 --- a/.buildkite/pipelines/quality-gates/pipeline.tests-production-canary.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# These pipeline steps constitute the quality gate for your service within the production-canary environment. -# Incorporate any necessary additional logic to validate the service's integrity. -# A failure in this pipeline build will prevent further progression to the subsequent stage. - -steps: - - label: ":pipeline::kibana::seedling: Trigger SLO check" - trigger: "serverless-quality-gates" # https://buildkite.com/elastic/serverless-quality-gates - build: - message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production-canary.yaml)" - env: - TARGET_ENV: production-canary - CHECK_SLO: true - CHECK_SLO_TAG: kibana - soft_fail: true - - - label: ":pipeline::rocket::seedling: Trigger control-plane e2e tests" - trigger: "ess-k8s-production-e2e-tests" # https://buildkite.com/elastic/ess-k8s-production-e2e-tests - build: - env: - REGION_ID: aws-us-east-1 - NAME_PREFIX: ci_test_kibana-promotion_ - message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production-canary.yaml)" - - - label: ":cookie: 24h bake time before continuing promotion" - command: "sleep 86400" diff --git a/.buildkite/pipelines/quality-gates/pipeline.tests-production-noncanary.yaml b/.buildkite/pipelines/quality-gates/pipeline.tests-production-noncanary.yaml deleted file mode 100644 index 13c974a344f98..0000000000000 --- a/.buildkite/pipelines/quality-gates/pipeline.tests-production-noncanary.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# These pipeline steps constitute the quality gate for your service within the production-noncanary environment. -# Incorporate any necessary additional logic to validate the service's integrity. -# A failure in this pipeline build will prevent further progression to the subsequent stage. - -steps: - - label: ":pipeline::kibana::seedling: Trigger SLO check" - trigger: "serverless-quality-gates" # https://buildkite.com/elastic/serverless-quality-gates - build: - message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production-noncanary.yaml)" - env: - TARGET_ENV: production-noncanary - CHECK_SLO: true - CHECK_SLO_TAG: kibana - soft_fail: true - - - label: ":pipeline::rocket::seedling: Trigger control-plane e2e tests" - trigger: "ess-k8s-production-e2e-tests" # https://buildkite.com/elastic/ess-k8s-production-e2e-tests - build: - env: - REGION_ID: aws-us-east-1 - NAME_PREFIX: ci_test_kibana-promotion_ - message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production-noncanary.yaml)" diff --git a/.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml b/.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml index 4b0bb30d3084c..fd2fbac8a7b30 100644 --- a/.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml +++ b/.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml @@ -2,10 +2,6 @@ # Incorporate any necessary additional logic to validate the service's integrity. # A failure in this pipeline build will prevent further progression to the subsequent stage. -# DEPRECATION NOTICE: -# PRODUCTION WILL SOON BE SPLIT INTO "CANARY" AND "NONCANARY" AND THIS FILE WILL BE DELETED. -# ENSURE ANY CHANGE MADE TO THIS FILE IS REFLECTED IN THOSE FILES AS WELL. - steps: - label: ":pipeline::kibana::seedling: Trigger SLO check" trigger: "serverless-quality-gates" # https://buildkite.com/elastic/serverless-quality-gates @@ -18,9 +14,14 @@ steps: soft_fail: true - label: ":pipeline::rocket::seedling: Trigger control-plane e2e tests" + if: build.env("ENVIRONMENT") == "production-canary" trigger: "ess-k8s-production-e2e-tests" # https://buildkite.com/elastic/ess-k8s-production-e2e-tests build: env: REGION_ID: aws-us-east-1 NAME_PREFIX: ci_test_kibana-promotion_ message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production.yaml)" + + - label: ":cookie: 24h bake time before continuing promotion" + if: build.env("ENVIRONMENT") == "production-canary" + command: "sleep 86400" diff --git a/.buildkite/scripts/steps/checks.sh b/.buildkite/scripts/steps/checks.sh index 12077902c1c13..c2758eb52c738 100755 --- a/.buildkite/scripts/steps/checks.sh +++ b/.buildkite/scripts/steps/checks.sh @@ -22,3 +22,4 @@ export DISABLE_BOOTSTRAP_VALIDATION=false .buildkite/scripts/steps/checks/test_hardening.sh .buildkite/scripts/steps/checks/ftr_configs.sh .buildkite/scripts/steps/checks/saved_objects_compat_changes.sh +.buildkite/scripts/steps/code_generation/security_solution_codegen.sh diff --git a/catalog-info.yaml b/catalog-info.yaml index 00637fb1a039b..e7aa120004486 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -56,6 +56,8 @@ spec: teams: kibana-operations: access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: BUILD_AND_READ security-engineering-productivity: access_level: BUILD_AND_READ fleet: diff --git a/docs/api/saved-objects/bulk_create.asciidoc b/docs/api/saved-objects/bulk_create.asciidoc index 73feb14acf483..1e56735fa52a2 100644 --- a/docs/api/saved-objects/bulk_create.asciidoc +++ b/docs/api/saved-objects/bulk_create.asciidoc @@ -38,7 +38,7 @@ For the most up-to-date API details, refer to the ==== Request body `type`:: - (Required, string) Valid options include `visualization`, `dashboard`, `search`, `index-pattern`, `config`. + (Required, string) Valid options include `visualization`, `dashboard`, `search`, `index-pattern`. `id`:: (Optional, string) Specifies an ID instead of using a randomly generated ID. diff --git a/docs/api/saved-objects/create.asciidoc b/docs/api/saved-objects/create.asciidoc index 7e26a329fb54d..154a58bb72025 100644 --- a/docs/api/saved-objects/create.asciidoc +++ b/docs/api/saved-objects/create.asciidoc @@ -32,7 +32,7 @@ For the most up-to-date API details, refer to the (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. ``:: - (Required, string) Valid options include `visualization`, `dashboard`, `search`, `index-pattern`, `config`. + (Required, string) Valid options include `visualization`, `dashboard`, `search`, `index-pattern`. ``:: (Optional, string) Specifies an ID instead of using a randomly generated ID. diff --git a/docs/management/connectors/images/bedrock-connector.png b/docs/management/connectors/images/bedrock-connector.png index 22a537183171d..cfdb19f3fc6c2 100644 Binary files a/docs/management/connectors/images/bedrock-connector.png and b/docs/management/connectors/images/bedrock-connector.png differ diff --git a/docs/management/connectors/images/bedrock-params.png b/docs/management/connectors/images/bedrock-params.png index f6857e6d0ffee..23dcfe7e12091 100644 Binary files a/docs/management/connectors/images/bedrock-params.png and b/docs/management/connectors/images/bedrock-params.png differ diff --git a/packages/kbn-es/src/serverless_resources/operator_users.yml b/packages/kbn-es/src/serverless_resources/operator_users.yml index 7e8e7163e6a84..769d52c25f65a 100644 --- a/packages/kbn-es/src/serverless_resources/operator_users.yml +++ b/packages/kbn-es/src/serverless_resources/operator_users.yml @@ -1,19 +1,5 @@ operator: - - usernames: - [ - 'elastic_serverless', - 'system_indices_superuser', - 't1_analyst', - 't2_analyst', - 't3_analyst', - 'threat_intelligence_analyst', - 'rule_author', - 'soc_manager', - 'detections_admin', - 'platform_engineer', - 'endpoint_operations_analyst', - 'endpoint_policy_manager', - ] + - usernames: ['elastic_serverless', 'system_indices_superuser'] realm_type: 'file' auth_type: 'realm' - usernames: ['elastic/kibana'] diff --git a/packages/kbn-es/src/serverless_resources/roles.yml b/packages/kbn-es/src/serverless_resources/roles.yml index c631f596a8cac..5777f282ff7a4 100644 --- a/packages/kbn-es/src/serverless_resources/roles.yml +++ b/packages/kbn-es/src/serverless_resources/roles.yml @@ -117,6 +117,7 @@ t1_analyst: - metrics-endpoint.metadata_current_* - ".fleet-agents*" - ".fleet-actions*" + - "risk-score.risk-score-*" privileges: - read applications: @@ -157,6 +158,7 @@ t2_analyst: - metrics-endpoint.metadata_current_* - .fleet-agents* - .fleet-actions* + - "risk-score.risk-score-*" privileges: - read applications: @@ -204,6 +206,7 @@ t3_analyst: - metrics-endpoint.metadata_current_* - .fleet-agents* - .fleet-actions* + - "risk-score.risk-score-*" privileges: - read applications: @@ -256,6 +259,7 @@ threat_intelligence_analyst: - metrics-endpoint.metadata_current_* - .fleet-agents* - .fleet-actions* + - "risk-score.risk-score-*" privileges: - read applications: @@ -307,6 +311,7 @@ rule_author: - metrics-endpoint.metadata_current_* - .fleet-agents* - .fleet-actions* + - "risk-score.risk-score-*" privileges: - read applications: @@ -363,6 +368,7 @@ soc_manager: - metrics-endpoint.metadata_current_* - .fleet-agents* - .fleet-actions* + - risk-score.risk-score-* privileges: - read applications: @@ -391,7 +397,7 @@ soc_manager: resources: "*" detections_admin: - cluster: + cluster: ["manage_index_templates", "manage_transform"] indices: - names: - apm-*-transaction* @@ -418,6 +424,10 @@ detections_admin: - .fleet-actions* privileges: - read + - names: + - risk-score.risk-score-* + privileges: + - all applications: - application: "kibana-.kibana" privileges: @@ -450,6 +460,7 @@ platform_engineer: - .siem-signals-* - .preview.alerts-security* - .internal.preview.alerts-security* + - risk-score.risk-score-* privileges: - all applications: @@ -482,6 +493,7 @@ endpoint_operations_analyst: - metrics-endpoint.metadata_current_* - .fleet-agents* - .fleet-actions* + - risk-score.risk-score-* privileges: - read - names: @@ -537,6 +549,7 @@ endpoint_policy_manager: - metrics-endpoint.metadata_current_* - .fleet-agents* - .fleet-actions* + - risk-score.risk-score-* privileges: - read - names: diff --git a/packages/kbn-search-connectors/types/native_connectors.ts b/packages/kbn-search-connectors/types/native_connectors.ts index 69a2d64184465..399ce74892de5 100644 --- a/packages/kbn-search-connectors/types/native_connectors.ts +++ b/packages/kbn-search-connectors/types/native_connectors.ts @@ -743,7 +743,7 @@ export const NATIVE_CONNECTOR_DEFINITIONS: Record = ({ inline, children, ...rest }) => { +export const SectionLoading: React.FunctionComponent = ({ + inline, + 'data-test-subj': dataTestSubj, + children, + ...rest +}) => { if (inline) { return ( - + @@ -38,7 +49,7 @@ export const SectionLoading: React.FunctionComponent = ({ inline, childre } body={{children}} - data-test-subj="sectionLoading" + data-test-subj={dataTestSubj ?? 'sectionLoading'} /> ); }; diff --git a/versions.json b/versions.json index 624287e414444..b6ac52d0272ce 100644 --- a/versions.json +++ b/versions.json @@ -14,7 +14,7 @@ "previousMinor": true }, { - "version": "8.10.4", + "version": "8.10.5", "branch": "8.10", "currentMajor": true, "previousMinor": true diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/action_type_selector_modal.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/action_type_selector_modal.tsx index 84a72e0cd2b5f..02db4bf391c14 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/action_type_selector_modal.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector_inline/action_type_selector_modal.tsx @@ -18,6 +18,7 @@ import { } from '@elastic/eui'; import { ActionType } from '@kbn/actions-plugin/common'; import { ActionTypeRegistryContract } from '@kbn/triggers-actions-ui-plugin/public'; +import { css } from '@emotion/css'; import * as i18n from '../translations'; interface Props { @@ -26,6 +27,12 @@ interface Props { onClose: () => void; onSelect: (actionType: ActionType) => void; } +const itemClassName = css` + .euiKeyPadMenuItem__label { + white-space: nowrap; + overflow: hidden; + } +`; export const ActionTypeSelectorModal = ({ actionTypes, @@ -46,6 +53,7 @@ export const ActionTypeSelectorModal = ({ return ( ; type APMTransactionErrorRateIndicator = t.OutputOf; type APMTransactionDurationIndicator = t.OutputOf; type MetricCustomIndicator = t.OutputOf; +type TimesliceMetricIndicator = t.OutputOf; +type TimesliceMetricBasicMetricWithField = t.OutputOf; +type TimesliceMetricDocCountMetric = t.OutputOf; +type TimesclieMetricPercentileMetric = t.OutputOf; type HistogramIndicator = t.OutputOf; type KQLCustomIndicator = t.OutputOf; @@ -327,6 +335,10 @@ export type { IndicatorType, Indicator, MetricCustomIndicator, + TimesliceMetricIndicator, + TimesliceMetricBasicMetricWithField, + TimesclieMetricPercentileMetric, + TimesliceMetricDocCountMetric, HistogramIndicator, KQLCustomIndicator, TimeWindow, diff --git a/x-pack/packages/kbn-slo-schema/src/schema/indicators.ts b/x-pack/packages/kbn-slo-schema/src/schema/indicators.ts index 07b9c69f4fb97..f8d795275acc6 100644 --- a/x-pack/packages/kbn-slo-schema/src/schema/indicators.ts +++ b/x-pack/packages/kbn-slo-schema/src/schema/indicators.ts @@ -59,6 +59,83 @@ const kqlCustomIndicatorSchema = t.type({ ]), }); +const timesliceMetricComparatorMapping = { + GT: '>', + GTE: '>=', + LT: '<', + LTE: '<=', +}; + +const timesliceMetricComparator = t.keyof(timesliceMetricComparatorMapping); + +const timesliceMetricBasicMetricWithField = t.intersection([ + t.type({ + name: t.string, + aggregation: t.keyof({ + avg: true, + max: true, + min: true, + sum: true, + cardinality: true, + last_value: true, + std_deviation: true, + }), + field: t.string, + }), + t.partial({ + filter: t.string, + }), +]); + +const timesliceMetricDocCountMetric = t.intersection([ + t.type({ + name: t.string, + aggregation: t.literal('doc_count'), + }), + t.partial({ + filter: t.string, + }), +]); + +const timesliceMetricPercentileMetric = t.intersection([ + t.type({ + name: t.string, + aggregation: t.literal('percentile'), + field: t.string, + percentile: t.number, + }), + t.partial({ + filter: t.string, + }), +]); + +const timesliceMetricMetricDef = t.union([ + timesliceMetricBasicMetricWithField, + timesliceMetricDocCountMetric, + timesliceMetricPercentileMetric, +]); + +const timesliceMetricDef = t.type({ + metrics: t.array(timesliceMetricMetricDef), + equation: t.string, + threshold: t.number, + comparator: timesliceMetricComparator, +}); +const timesliceMetricIndicatorTypeSchema = t.literal('sli.metric.timeslice'); +const timesliceMetricIndicatorSchema = t.type({ + type: timesliceMetricIndicatorTypeSchema, + params: t.intersection([ + t.type({ + index: t.string, + metric: timesliceMetricDef, + timestampField: t.string, + }), + t.partial({ + filter: t.string, + }), + ]), +}); + const metricCustomValidAggregations = t.keyof({ sum: true, }); @@ -149,6 +226,7 @@ const indicatorTypesSchema = t.union([ apmTransactionErrorRateIndicatorTypeSchema, kqlCustomIndicatorTypeSchema, metricCustomIndicatorTypeSchema, + timesliceMetricIndicatorTypeSchema, histogramIndicatorTypeSchema, ]); @@ -176,6 +254,7 @@ const indicatorSchema = t.union([ apmTransactionErrorRateIndicatorSchema, kqlCustomIndicatorSchema, metricCustomIndicatorSchema, + timesliceMetricIndicatorSchema, histogramIndicatorSchema, ]); @@ -186,8 +265,15 @@ export { apmTransactionErrorRateIndicatorTypeSchema, kqlCustomIndicatorSchema, kqlCustomIndicatorTypeSchema, - metricCustomIndicatorTypeSchema, metricCustomIndicatorSchema, + metricCustomIndicatorTypeSchema, + timesliceMetricComparatorMapping, + timesliceMetricIndicatorSchema, + timesliceMetricIndicatorTypeSchema, + timesliceMetricMetricDef, + timesliceMetricBasicMetricWithField, + timesliceMetricDocCountMetric, + timesliceMetricPercentileMetric, histogramIndicatorTypeSchema, histogramIndicatorSchema, indicatorSchema, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts index 2ebfec9a7f257..ba195f0de0e15 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts @@ -27,6 +27,7 @@ import type { } from './types'; const EMPTY_INDEX_NAMES: string[] = []; +export const INTERNAL_API_VERSION = '1'; export const getIndexNames = ({ ilmExplain, @@ -282,7 +283,7 @@ export const getSizeInBytes = ({ }: { indexName: string; stats: Record | null; -}): number => (stats && stats[indexName]?.primaries?.store?.size_in_bytes) ?? 0; +}): number => (stats && stats[indexName]?.primaries?.store?.total_data_set_size_in_bytes) ?? 0; export const getTotalDocsCount = ({ indexNames, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_auditbeat_pattern_rollup.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_auditbeat_pattern_rollup.ts index 7c18523e44aa3..2f83f899dc0d2 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_auditbeat_pattern_rollup.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_auditbeat_pattern_rollup.ts @@ -53,6 +53,7 @@ export const auditbeatNoResults: PatternRollup = { primaries: { store: { size_in_bytes: 18791790, + total_data_set_size_in_bytes: 18791790, reserved_in_bytes: 0, }, }, @@ -70,6 +71,7 @@ export const auditbeatNoResults: PatternRollup = { primaries: { store: { size_in_bytes: 247, + total_data_set_size_in_bytes: 247, reserved_in_bytes: 0, }, }, @@ -87,6 +89,7 @@ export const auditbeatNoResults: PatternRollup = { primaries: { store: { size_in_bytes: 28409, + total_data_set_size_in_bytes: 28409, reserved_in_bytes: 0, }, }, @@ -182,6 +185,7 @@ export const auditbeatWithAllResults: PatternRollup = { primaries: { store: { size_in_bytes: 18791790, + total_data_set_size_in_bytes: 18791790, reserved_in_bytes: 0, }, }, @@ -199,6 +203,7 @@ export const auditbeatWithAllResults: PatternRollup = { primaries: { store: { size_in_bytes: 247, + total_data_set_size_in_bytes: 247, reserved_in_bytes: 0, }, }, @@ -216,6 +221,7 @@ export const auditbeatWithAllResults: PatternRollup = { primaries: { store: { size_in_bytes: 28409, + total_data_set_size_in_bytes: 28409, reserved_in_bytes: 0, }, }, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_packetbeat_pattern_rollup.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_packetbeat_pattern_rollup.ts index b04c8bb87600a..369803a44a3dd 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_packetbeat_pattern_rollup.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_packetbeat_pattern_rollup.ts @@ -51,6 +51,7 @@ export const packetbeatNoResults: PatternRollup = { primaries: { store: { size_in_bytes: 512194751, + total_data_set_size_in_bytes: 512194751, reserved_in_bytes: 0, }, }, @@ -68,6 +69,7 @@ export const packetbeatNoResults: PatternRollup = { primaries: { store: { size_in_bytes: 584326147, + total_data_set_size_in_bytes: 584326147, reserved_in_bytes: 0, }, }, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_ilm_explain/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_ilm_explain/index.tsx index ae643745bd805..4e95549338874 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_ilm_explain/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_ilm_explain/index.tsx @@ -9,6 +9,7 @@ import type { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch import { useEffect, useState } from 'react'; import { useDataQualityContext } from '../data_quality_panel/data_quality_context'; +import { INTERNAL_API_VERSION } from '../helpers'; import * as i18n from '../translations'; const ILM_EXPLAIN_ENDPOINT = '/internal/ecs_data_quality_dashboard/ilm_explain'; @@ -43,6 +44,7 @@ export const useIlmExplain = (pattern: string): UseIlmExplain => { { method: 'GET', signal: abortController.signal, + version: INTERNAL_API_VERSION, } ); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_mappings/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_mappings/helpers.ts index 8aab64729df2f..809f543c0c0ae 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_mappings/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_mappings/helpers.ts @@ -9,6 +9,7 @@ import type { HttpHandler } from '@kbn/core-http-browser'; import type { IndicesGetMappingIndexMappingRecord } from '@elastic/elasticsearch/lib/api/types'; import * as i18n from '../translations'; +import { INTERNAL_API_VERSION } from '../helpers'; export const MAPPINGS_API_ROUTE = '/internal/ecs_data_quality_dashboard/mappings'; @@ -29,6 +30,7 @@ export async function fetchMappings({ { method: 'GET', signal: abortController.signal, + version: INTERNAL_API_VERSION, } ); } catch (e) { diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.tsx index 6875dad3d4dfc..fce940de15f75 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.tsx @@ -11,6 +11,7 @@ import { HttpFetchQuery } from '@kbn/core/public'; import { useDataQualityContext } from '../data_quality_panel/data_quality_context'; import * as i18n from '../translations'; +import { INTERNAL_API_VERSION } from '../helpers'; const STATS_ENDPOINT = '/internal/ecs_data_quality_dashboard/stats'; @@ -53,6 +54,7 @@ export const useStats = ({ const response = await httpFetch>( `${STATS_ENDPOINT}/${encodedIndexName}`, { + version: INTERNAL_API_VERSION, method: 'GET', signal: abortController.signal, query, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/helpers.test.ts index 7b0a77d9af564..cad285a4bc976 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/helpers.test.ts @@ -15,6 +15,7 @@ import { } from './helpers'; import { mockUnallowedValuesResponse } from '../mock/unallowed_values/mock_unallowed_values'; import { UnallowedValueRequestItem, UnallowedValueSearchResult } from '../types'; +import { INTERNAL_API_VERSION } from '../helpers'; describe('helpers', () => { let originalFetch: typeof global['fetch']; @@ -406,6 +407,7 @@ describe('helpers', () => { headers: { 'Content-Type': 'application/json' }, method: 'POST', signal: abortController.signal, + version: INTERNAL_API_VERSION, } ); }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/helpers.ts index 5a331e7e1b8da..a193456d4afa9 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/helpers.ts @@ -6,6 +6,7 @@ */ import type { HttpHandler } from '@kbn/core-http-browser'; +import { INTERNAL_API_VERSION } from '../helpers'; import * as i18n from '../translations'; import type { Bucket, @@ -81,6 +82,7 @@ export async function fetchUnallowedValues({ headers: { 'Content-Type': 'application/json' }, method: 'POST', signal: abortController.signal, + version: INTERNAL_API_VERSION, }); } catch (e) { throw new Error( diff --git a/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_page.tsx b/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_page.tsx index 2411c9096be70..2c45fd37a6858 100644 --- a/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_page.tsx +++ b/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_page.tsx @@ -359,7 +359,6 @@ export const DataDriftPage: FC = ({ initialSettings }) => { label={comparisonIndexPatternLabel} randomSampler={randomSamplerProd} reload={forceRefresh} - brushSelectionUpdateHandler={brushSelectionUpdate} documentCountStats={documentStatsProd.documentCountStats} documentCountStatsSplit={documentStatsProd.documentCountStatsCompare} isBrushCleared={isBrushCleared} diff --git a/x-pack/plugins/data_visualizer/public/application/data_drift/document_count_with_dual_brush.tsx b/x-pack/plugins/data_visualizer/public/application/data_drift/document_count_with_dual_brush.tsx index 0d83879c37486..210d364ce7aa6 100644 --- a/x-pack/plugins/data_visualizer/public/application/data_drift/document_count_with_dual_brush.tsx +++ b/x-pack/plugins/data_visualizer/public/application/data_drift/document_count_with_dual_brush.tsx @@ -32,7 +32,7 @@ export interface DocumentCountContentProps | 'interval' | 'chartPointsSplitLabel' > { - brushSelectionUpdateHandler: BrushSelectionUpdateHandler; + brushSelectionUpdateHandler?: BrushSelectionUpdateHandler; documentCountStats?: DocumentCountStats; documentCountStatsSplit?: DocumentCountStats; documentCountStatsSplitLabel?: string; diff --git a/x-pack/plugins/data_visualizer/public/application/data_drift/use_data_drift_result.ts b/x-pack/plugins/data_visualizer/public/application/data_drift/use_data_drift_result.ts index 8b22e1d94db33..4588595ffcc4f 100644 --- a/x-pack/plugins/data_visualizer/public/application/data_drift/use_data_drift_result.ts +++ b/x-pack/plugins/data_visualizer/public/application/data_drift/use_data_drift_result.ts @@ -29,6 +29,7 @@ import { isDefined } from '@kbn/ml-is-defined'; import { computeChi2PValue, type Histogram } from '@kbn/ml-chi2test'; import { mapAndFlattenFilters } from '@kbn/data-plugin/public'; +import type { AggregationsRangeBucketKeys } from '@elastic/elasticsearch/lib/api/types'; import { createMergedEsQuery } from '../index_data_visualizer/utils/saved_search_utils'; import { useDataVisualizerKibana } from '../kibana_context'; @@ -378,6 +379,7 @@ const fetchComparisonDriftedData = async ({ fields, baselineResponseAggs, baseRequest, + baselineRequest, randomSamplerWrapper, signal, }: { @@ -387,10 +389,19 @@ const fetchComparisonDriftedData = async ({ randomSamplerWrapper: RandomSamplerWrapper; signal: AbortSignal; baselineResponseAggs: object; + baselineRequest: EsRequestParams; }) => { const driftedRequest = { ...baseRequest }; + const driftedRequestAggs: Record = {}; + // Since aggregation is not able to split the values into distinct 5% intervals, + // this breaks our assumption of uniform distributed fractions in the`ks_test`. + // So, to fix this in the general case, we need to run an additional ranges agg to get the doc count for the ranges + // that we get from the percentiles aggregation + // and use it in the bucket_count_ks_test + const rangesRequestAggs: Record = {}; + for (const { field, type } of fields) { if ( isPopulatedObject(baselineResponseAggs, [`${field}_percentiles`]) && @@ -410,19 +421,16 @@ const fetchComparisonDriftedData = async ({ ranges.push({ from: percentiles[idx - 1], to: val }); } }); - // add range and bucket_count_ks_test to the request - driftedRequestAggs[`${field}_ranges`] = { + const rangeAggs = { range: { field, ranges, }, }; - driftedRequestAggs[`${field}_ks_test`] = { - bucket_count_ks_test: { - buckets_path: `${field}_ranges>_count`, - alternative: ['two_sided'], - }, - }; + // add range and bucket_count_ks_test to the request + rangesRequestAggs[`${field}_ranges`] = rangeAggs; + driftedRequestAggs[`${field}_ranges`] = rangeAggs; + // add stats aggregation to the request driftedRequestAggs[`${field}_stats`] = { stats: { @@ -441,6 +449,48 @@ const fetchComparisonDriftedData = async ({ } } + // Compute fractions based on results of ranges + const rangesResp = await dataSearch( + { + ...baselineRequest, + body: { ...baselineRequest.body, aggs: randomSamplerWrapper.wrap(rangesRequestAggs) }, + }, + signal + ); + + const fieldsWithNoOverlap = new Set(); + for (const { field } of fields) { + if (rangesResp.aggregations[`${field}_ranges`]) { + const buckets = rangesResp.aggregations[`${field}_ranges`] + .buckets as AggregationsRangeBucketKeys[]; + + if (buckets) { + const totalSumOfAllBuckets = buckets.reduce((acc, bucket) => acc + bucket.doc_count, 0); + + const fractions = buckets.map((bucket) => ({ + ...bucket, + fraction: bucket.doc_count / totalSumOfAllBuckets, + })); + + if (totalSumOfAllBuckets > 0) { + driftedRequestAggs[`${field}_ks_test`] = { + bucket_count_ks_test: { + buckets_path: `${field}_ranges>_count`, + alternative: ['two_sided'], + ...(totalSumOfAllBuckets > 0 + ? { fractions: fractions.map((bucket) => Number(bucket.fraction.toFixed(3))) } + : {}), + }, + }; + } else { + // If all doc_counts are 0, that means there's no overlap whatsoever + // in which case we don't need to make the ks test agg, because it defaults to astronomically small value + fieldsWithNoOverlap.add(field); + } + } + } + } + const driftedResp = await dataSearch( { ...driftedRequest, @@ -448,6 +498,17 @@ const fetchComparisonDriftedData = async ({ }, signal ); + + fieldsWithNoOverlap.forEach((field) => { + if (driftedResp.aggregations) { + driftedResp.aggregations[`${field}_ks_test`] = { + // Setting -Infinity to represent astronomically small number + // which would be represented as < 0.000001 in table + two_sided: -Infinity, + }; + } + }); + return driftedResp; }; @@ -678,7 +739,7 @@ export const useFetchDataComparisonResult = ( setResult({ data: undefined, status: FETCH_STATUS.LOADING, error: undefined }); - // Place holder for when there might be difference data views in the future + // Placeholder for when there might be difference data views in the future const referenceIndex = initialSettings ? initialSettings.reference : currentDataView?.getIndexPattern(); @@ -802,6 +863,7 @@ export const useFetchDataComparisonResult = ( fetchComparisonDriftedData({ dataSearch, baseRequest: driftedRequest, + baselineRequest, baselineResponseAggs, fields: chunkedFields, randomSamplerWrapper: prodRandomSamplerWrapper, diff --git a/x-pack/plugins/ecs_data_quality_dashboard/common/constants.ts b/x-pack/plugins/ecs_data_quality_dashboard/common/constants.ts index 51455c071b519..52c734797f726 100755 --- a/x-pack/plugins/ecs_data_quality_dashboard/common/constants.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/common/constants.ts @@ -13,3 +13,4 @@ export const GET_INDEX_STATS = `${BASE_PATH}/stats/{pattern}`; export const GET_INDEX_MAPPINGS = `${BASE_PATH}/mappings/{pattern}`; export const GET_UNALLOWED_FIELD_VALUES = `${BASE_PATH}/unallowed_field_values`; export const GET_ILM_EXPLAIN = `${BASE_PATH}/ilm_explain/{pattern}`; +export const INTERNAL_API_VERSION = '1'; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/server.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/server.ts index 7ac44e1beedf1..913c226517ce3 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/server.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/server.ts @@ -5,35 +5,73 @@ * 2.0. */ import { httpServiceMock } from '@kbn/core/server/mocks'; -import type { RequestHandler, RouteConfig, KibanaRequest } from '@kbn/core/server'; +import type { IRouter, RouteMethod, RequestHandler, KibanaRequest } from '@kbn/core/server'; import type { RequestHandlerContext } from '@kbn/core-http-request-handler-context-server'; +import type { RouterMock } from '@kbn/core-http-router-server-mocks'; +import type { AddVersionOpts, VersionedRouteConfig } from '@kbn/core-http-server'; import { requestMock } from './request'; import { responseMock as responseFactoryMock } from './response'; import { requestContextMock } from './request_context'; import { responseAdapter } from './test_adapters'; +import { INTERNAL_API_VERSION } from '../../common/constants'; interface Route { - config: RouteConfig; + config: AddVersionOpts; handler: RequestHandler; } -const getRoute = (routerMock: MockServer['router']): Route => { - const routeCalls = [ - ...routerMock.get.mock.calls, - ...routerMock.post.mock.calls, - ...routerMock.put.mock.calls, - ...routerMock.patch.mock.calls, - ...routerMock.delete.mock.calls, +interface RegisteredVersionedRoute { + routeConfig: VersionedRouteConfig; + versionConfig: AddVersionOpts; + routeHandler: RequestHandler; +} + +type RouterMethod = Extract; + +export const getRegisteredVersionedRouteMock = ( + routerMock: RouterMock, + method: RouterMethod, + path: string, + version: string +): RegisteredVersionedRoute => { + const route = routerMock.versioned.getRoute(method, path); + const routeVersion = route.versions[version]; + + if (!routeVersion) { + throw new Error(`Handler for [${method}][${path}] with version [${version}] no found!`); + } + + return { + routeConfig: route.config, + versionConfig: routeVersion.config, + routeHandler: routeVersion.handler, + }; +}; + +const getRoute = (routerMock: MockServer['router'], request: KibanaRequest): Route => { + const versionedRouteCalls = [ + ...routerMock.versioned.get.mock.calls, + ...routerMock.versioned.post.mock.calls, + ...routerMock.versioned.put.mock.calls, + ...routerMock.versioned.patch.mock.calls, + ...routerMock.versioned.delete.mock.calls, ]; - const [route] = routeCalls; - if (!route) { + const [versionedRoute] = versionedRouteCalls; + + if (!versionedRoute) { throw new Error('No route registered!'); } - const [config, handler] = route; - return { config, handler }; + const { routeHandler, versionConfig } = getRegisteredVersionedRouteMock( + routerMock, + request.route.method, + request.route.path, + INTERNAL_API_VERSION + ); + + return { config: versionConfig, handler: routeHandler }; }; const buildResultMock = () => ({ ok: jest.fn((x) => x), badRequest: jest.fn((x) => x) }); @@ -53,17 +91,19 @@ class MockServer { public async inject(request: KibanaRequest, context: RequestHandlerContext = this.contextMock) { const validatedRequest = this.validateRequest(request); + const [rejection] = this.resultMock.badRequest.mock.calls; if (rejection) { throw new Error(`Request was rejected with message: '${rejection}'`); } - await this.getRoute().handler(context, validatedRequest, this.responseMock); + await this.getRoute(validatedRequest).handler(context, validatedRequest, this.responseMock); + return responseAdapter(this.responseMock); } - private getRoute(): Route { - return getRoute(this.router); + private getRoute(request: KibanaRequest): Route { + return getRoute(this.router, request); } // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -72,7 +112,8 @@ class MockServer { } private validateRequest(request: KibanaRequest): KibanaRequest { - const validations = this.getRoute().config.validate; + const config = this.getRoute(request).config; + const validations = config.validate && config.validate?.request; if (!validations) { return request; } diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/plugin.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/plugin.ts index d93766d2f3a7e..0c1cf336dc10d 100755 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/plugin.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/plugin.ts @@ -29,10 +29,10 @@ export class EcsDataQualityDashboardPlugin const router = core.http.createRouter(); // this would be deleted when plugin is removed // Register server side APIs - getIndexMappingsRoute(router); - getIndexStatsRoute(router); - getUnallowedFieldValuesRoute(router); - getILMExplainRoute(router); + getIndexMappingsRoute(router, this.logger); + getIndexStatsRoute(router, this.logger); + getUnallowedFieldValuesRoute(router, this.logger); + getILMExplainRoute(router, this.logger); return {}; } diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.test.ts index 737ddf781f1a9..329defab80c2b 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.test.ts @@ -12,6 +12,7 @@ import { serverMock } from '../__mocks__/server'; import { requestMock } from '../__mocks__/request'; import { requestContextMock } from '../__mocks__/request_context'; import { getILMExplainRoute } from './get_ilm_explain'; +import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; jest.mock('../lib', () => ({ fetchILMExplain: jest.fn(), @@ -20,6 +21,8 @@ jest.mock('../lib', () => ({ describe('getILMExplainRoute route', () => { let server: ReturnType; let { context } = requestContextMock.createTools(); + let logger: MockedLogger; + const req = requestMock.create({ method: 'get', path: GET_ILM_EXPLAIN, @@ -32,9 +35,10 @@ describe('getILMExplainRoute route', () => { jest.clearAllMocks(); server = serverMock.create(); + logger = loggerMock.create(); ({ context } = requestContextMock.createTools()); - getILMExplainRoute(server.router); + getILMExplainRoute(server.router, logger); }); test('Returns index ilm information', async () => { @@ -91,11 +95,13 @@ describe('getILMExplainRoute route', () => { describe('request validation', () => { let server: ReturnType; + let logger: MockedLogger; beforeEach(() => { server = serverMock.create(); + logger = loggerMock.create(); - getILMExplainRoute(server.router); + getILMExplainRoute(server.router, logger); }); test('disallows invalid pattern', () => { diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.ts index dab234eecaae7..c30271c62e313 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.ts @@ -5,41 +5,51 @@ * 2.0. */ -import { IRouter } from '@kbn/core/server'; +import { IRouter, Logger } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; -import { GET_ILM_EXPLAIN } from '../../common/constants'; +import { GET_ILM_EXPLAIN, INTERNAL_API_VERSION } from '../../common/constants'; import { fetchILMExplain } from '../lib'; import { buildResponse } from '../lib/build_response'; import { buildRouteValidation } from '../schemas/common'; import { GetILMExplainParams } from '../schemas/get_ilm_explain'; -export const getILMExplainRoute = (router: IRouter) => { - router.get( - { +export const getILMExplainRoute = (router: IRouter, logger: Logger) => { + router.versioned + .get({ path: GET_ILM_EXPLAIN, - validate: { params: buildRouteValidation(GetILMExplainParams) }, - }, - async (context, request, response) => { - const resp = buildResponse(response); + access: 'internal', + }) + .addVersion( + { + version: INTERNAL_API_VERSION, + validate: { + request: { + params: buildRouteValidation(GetILMExplainParams), + }, + }, + }, + async (context, request, response) => { + const resp = buildResponse(response); - try { - const { client } = (await context.core).elasticsearch; - const decodedIndexName = decodeURIComponent(request.params.pattern); + try { + const { client } = (await context.core).elasticsearch; + const decodedIndexName = decodeURIComponent(request.params.pattern); - const ilmExplain = await fetchILMExplain(client, decodedIndexName); + const ilmExplain = await fetchILMExplain(client, decodedIndexName); - return response.ok({ - body: ilmExplain.indices, - }); - } catch (err) { - const error = transformError(err); + return response.ok({ + body: ilmExplain.indices, + }); + } catch (err) { + const error = transformError(err); - return resp.error({ - body: error.message, - statusCode: error.statusCode, - }); + logger.error(error.message); + return resp.error({ + body: error.message, + statusCode: error.statusCode, + }); + } } - } - ); + ); }; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.test.ts index 6c76c84396299..34ce98d4f9378 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.test.ts @@ -12,6 +12,7 @@ import { serverMock } from '../__mocks__/server'; import { requestMock } from '../__mocks__/request'; import { requestContextMock } from '../__mocks__/request_context'; import { getIndexMappingsRoute } from './get_index_mappings'; +import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; jest.mock('../lib', () => ({ fetchMappings: jest.fn(), @@ -20,6 +21,8 @@ jest.mock('../lib', () => ({ describe('getIndexMappingsRoute route', () => { let server: ReturnType; let { context } = requestContextMock.createTools(); + let logger: MockedLogger; + const req = requestMock.create({ method: 'get', path: GET_INDEX_MAPPINGS, @@ -32,9 +35,11 @@ describe('getIndexMappingsRoute route', () => { jest.clearAllMocks(); server = serverMock.create(); + logger = loggerMock.create(); + ({ context } = requestContextMock.createTools()); - getIndexMappingsRoute(server.router); + getIndexMappingsRoute(server.router, logger); }); test('Returns index stats', async () => { @@ -58,11 +63,11 @@ describe('getIndexMappingsRoute route', () => { describe('request validation', () => { let server: ReturnType; - + let logger: MockedLogger; beforeEach(() => { server = serverMock.create(); - - getIndexMappingsRoute(server.router); + logger = loggerMock.create(); + getIndexMappingsRoute(server.router, logger); }); test('disallows invalid pattern', () => { diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.ts index 8b81daebeec8d..c7ab5e1d4a790 100755 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.ts @@ -5,41 +5,47 @@ * 2.0. */ -import { IRouter } from '@kbn/core/server'; +import { IRouter, Logger } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { fetchMappings } from '../lib'; import { buildResponse } from '../lib/build_response'; -import { GET_INDEX_MAPPINGS } from '../../common/constants'; +import { GET_INDEX_MAPPINGS, INTERNAL_API_VERSION } from '../../common/constants'; import { GetIndexMappingsParams } from '../schemas/get_index_mappings'; import { buildRouteValidation } from '../schemas/common'; -export const getIndexMappingsRoute = (router: IRouter) => { - router.get( - { +export const getIndexMappingsRoute = (router: IRouter, logger: Logger) => { + router.versioned + .get({ path: GET_INDEX_MAPPINGS, - validate: { params: buildRouteValidation(GetIndexMappingsParams) }, - }, - async (context, request, response) => { - const resp = buildResponse(response); + access: 'internal', + }) + .addVersion( + { + version: INTERNAL_API_VERSION, + validate: { request: { params: buildRouteValidation(GetIndexMappingsParams) } }, + }, + async (context, request, response) => { + const resp = buildResponse(response); - try { - const { client } = (await context.core).elasticsearch; - const decodedIndexName = decodeURIComponent(request.params.pattern); + try { + const { client } = (await context.core).elasticsearch; + const decodedIndexName = decodeURIComponent(request.params.pattern); - const mappings = await fetchMappings(client, decodedIndexName); + const mappings = await fetchMappings(client, decodedIndexName); - return response.ok({ - body: mappings, - }); - } catch (err) { - const error = transformError(err); + return response.ok({ + body: mappings, + }); + } catch (err) { + const error = transformError(err); + logger.error(error.message); - return resp.error({ - body: error.message, - statusCode: error.statusCode, - }); + return resp.error({ + body: error.message, + statusCode: error.statusCode, + }); + } } - } - ); + ); }; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts index aa7fc306b47e8..e000809797a01 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts @@ -12,6 +12,7 @@ import { serverMock } from '../__mocks__/server'; import { requestMock } from '../__mocks__/request'; import { requestContextMock } from '../__mocks__/request_context'; import { getIndexStatsRoute } from './get_index_stats'; +import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; jest.mock('../lib', () => ({ fetchStats: jest.fn(), @@ -21,6 +22,8 @@ jest.mock('../lib', () => ({ describe('getIndexStatsRoute route', () => { let server: ReturnType; let { context } = requestContextMock.createTools(); + let logger: MockedLogger; + const req = requestMock.create({ method: 'get', path: GET_INDEX_STATS, @@ -38,9 +41,11 @@ describe('getIndexStatsRoute route', () => { jest.clearAllMocks(); server = serverMock.create(); + logger = loggerMock.create(); + ({ context } = requestContextMock.createTools()); - getIndexStatsRoute(server.router); + getIndexStatsRoute(server.router, logger); }); test('Returns index stats', async () => { @@ -127,11 +132,13 @@ describe('getIndexStatsRoute route', () => { describe('request validation', () => { let server: ReturnType; + let logger: MockedLogger; beforeEach(() => { server = serverMock.create(); + logger = loggerMock.create(); - getIndexStatsRoute(server.router); + getIndexStatsRoute(server.router, logger); }); test('disallows invalid pattern', () => { diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts index 839d21931a064..f98fa03c27523 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts @@ -5,87 +5,96 @@ * 2.0. */ import { i18n } from '@kbn/i18n'; -import { IRouter } from '@kbn/core/server'; +import { IRouter, Logger } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types'; import { fetchStats, fetchAvailableIndices } from '../lib'; import { buildResponse } from '../lib/build_response'; -import { GET_INDEX_STATS } from '../../common/constants'; +import { GET_INDEX_STATS, INTERNAL_API_VERSION } from '../../common/constants'; import { buildRouteValidation } from '../schemas/common'; import { GetIndexStatsParams, GetIndexStatsQuery } from '../schemas/get_index_stats'; -export const getIndexStatsRoute = (router: IRouter) => { - router.get( - { +export const getIndexStatsRoute = (router: IRouter, logger: Logger) => { + router.versioned + .get({ path: GET_INDEX_STATS, - validate: { - params: buildRouteValidation(GetIndexStatsParams), - query: buildRouteValidation(GetIndexStatsQuery), + access: 'internal', + }) + .addVersion( + { + version: INTERNAL_API_VERSION, + validate: { + request: { + params: buildRouteValidation(GetIndexStatsParams), + query: buildRouteValidation(GetIndexStatsQuery), + }, + }, }, - }, - async (context, request, response) => { - const resp = buildResponse(response); + async (context, request, response) => { + const resp = buildResponse(response); - try { - const { client } = (await context.core).elasticsearch; - const esClient = client.asCurrentUser; + try { + const { client } = (await context.core).elasticsearch; + const esClient = client.asCurrentUser; - const decodedIndexName = decodeURIComponent(request.params.pattern); + const decodedIndexName = decodeURIComponent(request.params.pattern); - const stats = await fetchStats(client, decodedIndexName); - const { isILMAvailable, startDate, endDate } = request.query; + const stats = await fetchStats(client, decodedIndexName); + const { isILMAvailable, startDate, endDate } = request.query; - if (isILMAvailable === true) { - return response.ok({ - body: stats.indices, - }); - } + if (isILMAvailable === true) { + return response.ok({ + body: stats.indices, + }); + } - /** - * If ILM is not available, we need to fetch the available indices with the given date range. - * `fetchAvailableIndices` returns indices that have data in the given date range. - */ - if (startDate && endDate) { - const decodedStartDate = decodeURIComponent(startDate); - const decodedEndDate = decodeURIComponent(endDate); + /** + * If ILM is not available, we need to fetch the available indices with the given date range. + * `fetchAvailableIndices` returns indices that have data in the given date range. + */ + if (startDate && endDate) { + const decodedStartDate = decodeURIComponent(startDate); + const decodedEndDate = decodeURIComponent(endDate); - const indices = await fetchAvailableIndices(esClient, { - indexPattern: decodedIndexName, - startDate: decodedStartDate, - endDate: decodedEndDate, - }); - const availableIndices = indices?.aggregations?.index?.buckets?.reduce( - (acc: Record, { key }: { key: string }) => { - if (stats.indices?.[key]) { - acc[key] = stats.indices?.[key]; - } - return acc; - }, - {} - ); + const indices = await fetchAvailableIndices(esClient, { + indexPattern: decodedIndexName, + startDate: decodedStartDate, + endDate: decodedEndDate, + }); + const availableIndices = indices?.aggregations?.index?.buckets?.reduce( + (acc: Record, { key }: { key: string }) => { + if (stats.indices?.[key]) { + acc[key] = stats.indices?.[key]; + } + return acc; + }, + {} + ); + + return response.ok({ + body: availableIndices, + }); + } else { + return resp.error({ + body: i18n.translate( + 'xpack.ecsDataQualityDashboard.getIndexStats.dateRangeRequiredErrorMessage', + { + defaultMessage: 'startDate and endDate are required', + } + ), + statusCode: 400, + }); + } + } catch (err) { + const error = transformError(err); + logger.error(error.message); - return response.ok({ - body: availableIndices, - }); - } else { return resp.error({ - body: i18n.translate( - 'xpack.ecsDataQualityDashboard.getIndexStats.dateRangeRequiredErrorMessage', - { - defaultMessage: 'startDate and endDate are required', - } - ), - statusCode: 400, + body: error.message, + statusCode: error.statusCode, }); } - } catch (err) { - const error = transformError(err); - return resp.error({ - body: error.message, - statusCode: error.statusCode, - }); } - } - ); + ); }; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.test.ts index bb59356111699..fe18893acaec1 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.test.ts @@ -12,6 +12,7 @@ import { serverMock } from '../__mocks__/server'; import { requestMock } from '../__mocks__/request'; import { requestContextMock } from '../__mocks__/request_context'; import { getUnallowedFieldValuesRoute } from './get_unallowed_field_values'; +import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; jest.mock('../lib', () => ({ getUnallowedFieldValues: jest.fn(), @@ -20,6 +21,8 @@ jest.mock('../lib', () => ({ describe('getUnallowedFieldValuesRoute route', () => { let server: ReturnType; let { context } = requestContextMock.createTools(); + let logger: MockedLogger; + const req = requestMock.create({ method: 'post', path: GET_UNALLOWED_FIELD_VALUES, @@ -37,8 +40,9 @@ describe('getUnallowedFieldValuesRoute route', () => { server = serverMock.create(); ({ context } = requestContextMock.createTools()); + logger = loggerMock.create(); - getUnallowedFieldValuesRoute(server.router); + getUnallowedFieldValuesRoute(server.router, logger); }); test('Returns unallowedValues', async () => { @@ -107,11 +111,13 @@ describe('getUnallowedFieldValuesRoute route', () => { describe('request validation', () => { let server: ReturnType; + let logger: MockedLogger; beforeEach(() => { server = serverMock.create(); + logger = loggerMock.create(); - getUnallowedFieldValuesRoute(server.router); + getUnallowedFieldValuesRoute(server.router, logger); }); test('disallows invalid pattern', () => { diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.ts index be504a812e16b..db8887c2dfa66 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.ts @@ -5,40 +5,46 @@ * 2.0. */ -import { IRouter } from '@kbn/core/server'; +import { IRouter, Logger } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import { getUnallowedFieldValues } from '../lib'; import { buildResponse } from '../lib/build_response'; -import { GET_UNALLOWED_FIELD_VALUES } from '../../common/constants'; +import { GET_UNALLOWED_FIELD_VALUES, INTERNAL_API_VERSION } from '../../common/constants'; import { buildRouteValidation } from '../schemas/common'; import { GetUnallowedFieldValuesBody } from '../schemas/get_unallowed_field_values'; -export const getUnallowedFieldValuesRoute = (router: IRouter) => { - router.post( - { +export const getUnallowedFieldValuesRoute = (router: IRouter, logger: Logger) => { + router.versioned + .post({ path: GET_UNALLOWED_FIELD_VALUES, - validate: { body: buildRouteValidation(GetUnallowedFieldValuesBody) }, - }, - async (context, request, response) => { - const resp = buildResponse(response); - const esClient = (await context.core).elasticsearch.client.asCurrentUser; + access: 'internal', + }) + .addVersion( + { + version: INTERNAL_API_VERSION, + validate: { request: { body: buildRouteValidation(GetUnallowedFieldValuesBody) } }, + }, + async (context, request, response) => { + const resp = buildResponse(response); + const esClient = (await context.core).elasticsearch.client.asCurrentUser; - try { - const items = request.body; + try { + const items = request.body; - const { responses } = await getUnallowedFieldValues(esClient, items); - return response.ok({ - body: responses, - }); - } catch (err) { - const error = transformError(err); + const { responses } = await getUnallowedFieldValues(esClient, items); + return response.ok({ + body: responses, + }); + } catch (err) { + const error = transformError(err); + logger.error(error.message); - return resp.error({ - body: error.message, - statusCode: error.statusCode, - }); + return resp.error({ + body: error.message, + statusCode: error.statusCode, + }); + } } - } - ); + ); }; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/common.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/common.ts index 00e97a9326c5e..57dc45d4071f7 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/common.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/common.ts @@ -7,7 +7,7 @@ import { fold } from 'fp-ts/lib/Either'; import { pipe } from 'fp-ts/lib/pipeable'; -import type * as rt from 'io-ts'; +import * as rt from 'io-ts'; import { exactCheck, formatErrors } from '@kbn/securitysolution-io-ts-utils'; import type { RouteValidationFunction, diff --git a/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json b/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json index b5c1ad152b232..c0603ef91df6b 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json +++ b/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json @@ -20,6 +20,8 @@ "@kbn/securitysolution-io-ts-utils", "@kbn/securitysolution-io-ts-types", "@kbn/i18n", + "@kbn/core-http-router-server-mocks", + "@kbn/logging-mocks", ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts index a519398f158be..f208fe1d5739f 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts @@ -23,7 +23,7 @@ export const getLlmType = (connectorId: string, connectors: ActionResult[]): str // See: https://github.com/langchain-ai/langchainjs/blob/fb699647a310c620140842776f4a7432c53e02fa/langchain/src/agents/openai/index.ts#L185 return 'openai'; } - // TODO: Add support for AWS Bedrock Connector once merged + // TODO: Add support for Amazon Bedrock Connector once merged // Note: Doesn't appear to be a difference between Azure and OpenAI LLM types, so TBD for functions agent on Azure // See: https://github.com/langchain-ai/langchainjs/blob/fb699647a310c620140842776f4a7432c53e02fa/langchain/src/llms/openai.ts#L539 diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/native_connector_configuration/research_configuration.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/native_connector_configuration/research_configuration.tsx index b76d89396384e..8f32044356a68 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/native_connector_configuration/research_configuration.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/native_connector_configuration/research_configuration.tsx @@ -44,17 +44,19 @@ export const ResearchConfiguration: React.FC = ({ )} - - - {i18n.translate( - 'xpack.enterpriseSearch.content.indices.configurationConnector.researchConfiguration.serviceDocumentationLinkLabel', - { - defaultMessage: '{name} documentation', - values: { name }, - } - )} - - + {externalDocsUrl && ( + + + {i18n.translate( + 'xpack.enterpriseSearch.content.indices.configurationConnector.researchConfiguration.serviceDocumentationLinkLabel', + { + defaultMessage: '{name} documentation', + values: { name }, + } + )} + + + )} ); diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_list_grid/controls.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_list_grid/controls.tsx index cb5c61ff57a14..c50d396c143f8 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_list_grid/controls.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_list_grid/controls.tsx @@ -6,6 +6,7 @@ */ import React, { type ReactNode } from 'react'; +import styled from 'styled-components'; import { EuiFlexGroup, EuiSpacer, EuiTitle } from '@elastic/eui'; @@ -14,6 +15,10 @@ interface ControlsColumnProps { title: string | undefined; } +const FlexGroupWithMaxHeight = styled(EuiFlexGroup)` + max-height: calc(100vh - 120px); +`; + export const ControlsColumn = ({ controls, title }: ControlsColumnProps) => { let titleContent; if (title) { @@ -27,9 +32,9 @@ export const ControlsColumn = ({ controls, title }: ControlsColumnProps) => { ); } return ( - + {titleContent} {controls} - + ); }; diff --git a/x-pack/plugins/observability/docs/openapi/slo/bundled.json b/x-pack/plugins/observability/docs/openapi/slo/bundled.json index 3ba6ab7762e93..b4f52b032a9fc 100644 --- a/x-pack/plugins/observability/docs/openapi/slo/bundled.json +++ b/x-pack/plugins/observability/docs/openapi/slo/bundled.json @@ -1240,6 +1240,210 @@ } } }, + "timeslice_metric_basic_metric_with_field": { + "title": "Timeslice Metric Basic Metric with Field", + "required": [ + "name", + "aggregation", + "field" + ], + "type": "object", + "properties": { + "name": { + "description": "The name of the metric. Only valid options are A-Z", + "type": "string", + "example": "A", + "pattern": "^[A-Z]$" + }, + "aggregation": { + "description": "The aggregation type of the metric.", + "type": "string", + "example": "sum", + "enum": [ + "sum", + "avg", + "min", + "max", + "std_deviation", + "last_value", + "cardinality" + ] + }, + "field": { + "description": "The field of the metric.", + "type": "string", + "example": "processor.processed" + }, + "filter": { + "description": "The filter to apply to the metric.", + "type": "string", + "example": "processor.outcome: \"success\"" + } + } + }, + "timeslice_metric_percentile_metric": { + "title": "Timeslice Metric Percentile Metric", + "required": [ + "name", + "aggregation", + "field", + "percentile" + ], + "type": "object", + "properties": { + "name": { + "description": "The name of the metric. Only valid options are A-Z", + "type": "string", + "example": "A", + "pattern": "^[A-Z]$" + }, + "aggregation": { + "description": "The aggregation type of the metric. Only valid option is \"percentile\"", + "type": "string", + "example": "percentile", + "enum": [ + "percentile" + ] + }, + "field": { + "description": "The field of the metric.", + "type": "string", + "example": "processor.processed" + }, + "percentile": { + "description": "The percentile value.", + "type": "number", + "example": 95 + }, + "filter": { + "description": "The filter to apply to the metric.", + "type": "string", + "example": "processor.outcome: \"success\"" + } + } + }, + "timeslice_metric_doc_count_metric": { + "title": "Timeslice Metric Doc Count Metric", + "required": [ + "name", + "aggregation" + ], + "type": "object", + "properties": { + "name": { + "description": "The name of the metric. Only valid options are A-Z", + "type": "string", + "example": "A", + "pattern": "^[A-Z]$" + }, + "aggregation": { + "description": "The aggregation type of the metric. Only valid option is \"doc_count\"", + "type": "string", + "example": "doc_count", + "enum": [ + "doc_count" + ] + }, + "filter": { + "description": "The filter to apply to the metric.", + "type": "string", + "example": "processor.outcome: \"success\"" + } + } + }, + "indicator_properties_timeslice_metric": { + "title": "Timeslice metric", + "required": [ + "type", + "params" + ], + "description": "Defines properties for a timeslice metric indicator type", + "type": "object", + "properties": { + "params": { + "description": "An object containing the indicator parameters.", + "type": "object", + "nullable": false, + "required": [ + "index", + "timestampField", + "metric" + ], + "properties": { + "index": { + "description": "The index or index pattern to use", + "type": "string", + "example": "my-service-*" + }, + "filter": { + "description": "the KQL query to filter the documents with.", + "type": "string", + "example": "field.environment : \"production\" and service.name : \"my-service\"" + }, + "timestampField": { + "description": "The timestamp field used in the source indice.\n", + "type": "string", + "example": "timestamp" + }, + "metric": { + "description": "An object defining the metrics, equation, and threshold to determine if it's a good slice or not\n", + "type": "object", + "required": [ + "metrics", + "equation", + "comparator", + "threshold" + ], + "properties": { + "metrics": { + "description": "List of metrics with their name, aggregation type, and field.", + "type": "array", + "items": { + "anyOf": [ + { + "$ref": "#/components/schemas/timeslice_metric_basic_metric_with_field" + }, + { + "$ref": "#/components/schemas/timeslice_metric_percentile_metric" + }, + { + "$ref": "#/components/schemas/timeslice_metric_doc_count_metric" + } + ] + } + }, + "equation": { + "description": "The equation to calculate the metric.", + "type": "string", + "example": "A" + }, + "comparator": { + "description": "The comparator to use to compare the equation to the threshold.", + "type": "string", + "example": "GT", + "enum": [ + "GT", + "GTE", + "LT", + "LTE" + ] + }, + "threshold": { + "description": "The threshold used to determine if the metric is a good slice or not.", + "type": "number", + "example": 100 + } + } + } + } + }, + "type": { + "description": "The type of indicator.", + "type": "string", + "example": "sli.metric.timeslice" + } + } + }, "time_window": { "title": "Time window", "required": [ @@ -1427,7 +1631,8 @@ "sli.kql.custom": "#/components/schemas/indicator_properties_custom_kql", "sli.apm.transactionDuration": "#/components/schemas/indicator_properties_apm_latency", "sli.metric.custom": "#/components/schemas/indicator_properties_custom_metric", - "sli.histogram.custom": "#/components/schemas/indicator_properties_histogram" + "sli.histogram.custom": "#/components/schemas/indicator_properties_histogram", + "sli.metric.timeslice": "#/components/schemas/indicator_properties_timeslice_metric" } }, "oneOf": [ @@ -1445,6 +1650,9 @@ }, { "$ref": "#/components/schemas/indicator_properties_histogram" + }, + { + "$ref": "#/components/schemas/indicator_properties_timeslice_metric" } ] }, @@ -1661,6 +1869,9 @@ }, { "$ref": "#/components/schemas/indicator_properties_histogram" + }, + { + "$ref": "#/components/schemas/indicator_properties_timeslice_metric" } ] }, @@ -1755,6 +1966,9 @@ }, { "$ref": "#/components/schemas/indicator_properties_histogram" + }, + { + "$ref": "#/components/schemas/indicator_properties_timeslice_metric" } ] }, diff --git a/x-pack/plugins/observability/docs/openapi/slo/bundled.yaml b/x-pack/plugins/observability/docs/openapi/slo/bundled.yaml index c50403e5096f8..8efdbd9dfe2c2 100644 --- a/x-pack/plugins/observability/docs/openapi/slo/bundled.yaml +++ b/x-pack/plugins/observability/docs/openapi/slo/bundled.yaml @@ -837,6 +837,162 @@ components: description: The type of indicator. type: string example: sli.histogram.custom + timeslice_metric_basic_metric_with_field: + title: Timeslice Metric Basic Metric with Field + required: + - name + - aggregation + - field + type: object + properties: + name: + description: The name of the metric. Only valid options are A-Z + type: string + example: A + pattern: ^[A-Z]$ + aggregation: + description: The aggregation type of the metric. + type: string + example: sum + enum: + - sum + - avg + - min + - max + - std_deviation + - last_value + - cardinality + field: + description: The field of the metric. + type: string + example: processor.processed + filter: + description: The filter to apply to the metric. + type: string + example: 'processor.outcome: "success"' + timeslice_metric_percentile_metric: + title: Timeslice Metric Percentile Metric + required: + - name + - aggregation + - field + - percentile + type: object + properties: + name: + description: The name of the metric. Only valid options are A-Z + type: string + example: A + pattern: ^[A-Z]$ + aggregation: + description: The aggregation type of the metric. Only valid option is "percentile" + type: string + example: percentile + enum: + - percentile + field: + description: The field of the metric. + type: string + example: processor.processed + percentile: + description: The percentile value. + type: number + example: 95 + filter: + description: The filter to apply to the metric. + type: string + example: 'processor.outcome: "success"' + timeslice_metric_doc_count_metric: + title: Timeslice Metric Doc Count Metric + required: + - name + - aggregation + type: object + properties: + name: + description: The name of the metric. Only valid options are A-Z + type: string + example: A + pattern: ^[A-Z]$ + aggregation: + description: The aggregation type of the metric. Only valid option is "doc_count" + type: string + example: doc_count + enum: + - doc_count + filter: + description: The filter to apply to the metric. + type: string + example: 'processor.outcome: "success"' + indicator_properties_timeslice_metric: + title: Timeslice metric + required: + - type + - params + description: Defines properties for a timeslice metric indicator type + type: object + properties: + params: + description: An object containing the indicator parameters. + type: object + nullable: false + required: + - index + - timestampField + - metric + properties: + index: + description: The index or index pattern to use + type: string + example: my-service-* + filter: + description: the KQL query to filter the documents with. + type: string + example: 'field.environment : "production" and service.name : "my-service"' + timestampField: + description: | + The timestamp field used in the source indice. + type: string + example: timestamp + metric: + description: | + An object defining the metrics, equation, and threshold to determine if it's a good slice or not + type: object + required: + - metrics + - equation + - comparator + - threshold + properties: + metrics: + description: List of metrics with their name, aggregation type, and field. + type: array + items: + anyOf: + - $ref: '#/components/schemas/timeslice_metric_basic_metric_with_field' + - $ref: '#/components/schemas/timeslice_metric_percentile_metric' + - $ref: '#/components/schemas/timeslice_metric_doc_count_metric' + equation: + description: The equation to calculate the metric. + type: string + example: A + comparator: + description: The comparator to use to compare the equation to the threshold. + type: string + example: GT + enum: + - GT + - GTE + - LT + - LTE + threshold: + description: The threshold used to determine if the metric is a good slice or not. + type: number + example: 100 + type: + description: The type of indicator. + type: string + example: sli.metric.timeslice time_window: title: Time window required: @@ -988,12 +1144,14 @@ components: sli.apm.transactionDuration: '#/components/schemas/indicator_properties_apm_latency' sli.metric.custom: '#/components/schemas/indicator_properties_custom_metric' sli.histogram.custom: '#/components/schemas/indicator_properties_histogram' + sli.metric.timeslice: '#/components/schemas/indicator_properties_timeslice_metric' oneOf: - $ref: '#/components/schemas/indicator_properties_custom_kql' - $ref: '#/components/schemas/indicator_properties_apm_availability' - $ref: '#/components/schemas/indicator_properties_apm_latency' - $ref: '#/components/schemas/indicator_properties_custom_metric' - $ref: '#/components/schemas/indicator_properties_histogram' + - $ref: '#/components/schemas/indicator_properties_timeslice_metric' timeWindow: $ref: '#/components/schemas/time_window' budgetingMethod: @@ -1150,6 +1308,7 @@ components: - $ref: '#/components/schemas/indicator_properties_apm_latency' - $ref: '#/components/schemas/indicator_properties_custom_metric' - $ref: '#/components/schemas/indicator_properties_histogram' + - $ref: '#/components/schemas/indicator_properties_timeslice_metric' timeWindow: $ref: '#/components/schemas/time_window' budgetingMethod: @@ -1212,6 +1371,7 @@ components: - $ref: '#/components/schemas/indicator_properties_apm_latency' - $ref: '#/components/schemas/indicator_properties_custom_metric' - $ref: '#/components/schemas/indicator_properties_histogram' + - $ref: '#/components/schemas/indicator_properties_timeslice_metric' timeWindow: $ref: '#/components/schemas/time_window' budgetingMethod: diff --git a/x-pack/plugins/observability/docs/openapi/slo/components/schemas/create_slo_request.yaml b/x-pack/plugins/observability/docs/openapi/slo/components/schemas/create_slo_request.yaml index f14a1a134abd8..c3a848fe52133 100644 --- a/x-pack/plugins/observability/docs/openapi/slo/components/schemas/create_slo_request.yaml +++ b/x-pack/plugins/observability/docs/openapi/slo/components/schemas/create_slo_request.yaml @@ -27,6 +27,7 @@ properties: - $ref: "indicator_properties_apm_latency.yaml" - $ref: "indicator_properties_custom_metric.yaml" - $ref: 'indicator_properties_histogram.yaml' + - $ref: 'indicator_properties_timeslice_metric.yaml' timeWindow: $ref: "time_window.yaml" budgetingMethod: diff --git a/x-pack/plugins/observability/docs/openapi/slo/components/schemas/indicator_properties_timeslice_metric.yaml b/x-pack/plugins/observability/docs/openapi/slo/components/schemas/indicator_properties_timeslice_metric.yaml new file mode 100644 index 0000000000000..712420f059fdd --- /dev/null +++ b/x-pack/plugins/observability/docs/openapi/slo/components/schemas/indicator_properties_timeslice_metric.yaml @@ -0,0 +1,64 @@ +title: Timeslice metric +required: + - type + - params +description: Defines properties for a timeslice metric indicator type +type: object +properties: + params: + description: An object containing the indicator parameters. + type: object + nullable: false + required: + - index + - timestampField + - metric + properties: + index: + description: The index or index pattern to use + type: string + example: my-service-* + filter: + description: the KQL query to filter the documents with. + type: string + example: 'field.environment : "production" and service.name : "my-service"' + timestampField: + description: > + The timestamp field used in the source indice. + type: string + example: timestamp + metric: + description: > + An object defining the metrics, equation, and threshold to determine if it's a good slice or not + type: object + required: + - metrics + - equation + - comparator + - threshold + properties: + metrics: + description: List of metrics with their name, aggregation type, and field. + type: array + items: + anyOf: + - $ref: './timeslice_metric_basic_metric_with_field.yaml' + - $ref: './timeslice_metric_percentile_metric.yaml' + - $ref: './timeslice_metric_doc_count_metric.yaml' + equation: + description: The equation to calculate the metric. + type: string + example: A + comparator: + description: The comparator to use to compare the equation to the threshold. + type: string + example: GT + enum: [GT, GTE, LT, LTE] + threshold: + description: The threshold used to determine if the metric is a good slice or not. + type: number + example: 100 + type: + description: The type of indicator. + type: string + example: sli.metric.timeslice diff --git a/x-pack/plugins/observability/docs/openapi/slo/components/schemas/slo_response.yaml b/x-pack/plugins/observability/docs/openapi/slo/components/schemas/slo_response.yaml index da81009bc20b3..bd58e88c7b641 100644 --- a/x-pack/plugins/observability/docs/openapi/slo/components/schemas/slo_response.yaml +++ b/x-pack/plugins/observability/docs/openapi/slo/components/schemas/slo_response.yaml @@ -37,14 +37,16 @@ properties: sli.apm.transactionErrorRate: './indicator_properties_apm_availability.yaml' sli.kql.custom: './indicator_properties_custom_kql.yaml' sli.apm.transactionDuration: './indicator_properties_apm_latency.yaml' - sli.metric.custom: 'indicator_properties_custom_metric.yaml' - sli.histogram.custom: 'indicator_properties_histogram.yaml' + sli.metric.custom: './indicator_properties_custom_metric.yaml' + sli.histogram.custom: './indicator_properties_histogram.yaml' + sli.metric.timeslice: './indicator_properties_timeslice_metric.yaml' oneOf: - $ref: "indicator_properties_custom_kql.yaml" - $ref: "indicator_properties_apm_availability.yaml" - $ref: "indicator_properties_apm_latency.yaml" - $ref: "indicator_properties_custom_metric.yaml" - $ref: "indicator_properties_histogram.yaml" + - $ref: "indicator_properties_timeslice_metric.yaml" timeWindow: $ref: "time_window.yaml" budgetingMethod: diff --git a/x-pack/plugins/observability/docs/openapi/slo/components/schemas/timeslice_metric_basic_metric_with_field.yaml b/x-pack/plugins/observability/docs/openapi/slo/components/schemas/timeslice_metric_basic_metric_with_field.yaml new file mode 100644 index 0000000000000..570f4b4dda905 --- /dev/null +++ b/x-pack/plugins/observability/docs/openapi/slo/components/schemas/timeslice_metric_basic_metric_with_field.yaml @@ -0,0 +1,25 @@ +title: Timeslice Metric Basic Metric with Field +required: + - name + - aggregation + - field +type: object +properties: + name: + description: The name of the metric. Only valid options are A-Z + type: string + example: A + pattern: "^[A-Z]$" + aggregation: + description: The aggregation type of the metric. + type: string + example: sum + enum: [sum, avg, min, max, std_deviation, last_value, cardinality] + field: + description: The field of the metric. + type: string + example: processor.processed + filter: + description: The filter to apply to the metric. + type: string + example: 'processor.outcome: "success"' diff --git a/x-pack/plugins/observability/docs/openapi/slo/components/schemas/timeslice_metric_doc_count_metric.yaml b/x-pack/plugins/observability/docs/openapi/slo/components/schemas/timeslice_metric_doc_count_metric.yaml new file mode 100644 index 0000000000000..76417fd111975 --- /dev/null +++ b/x-pack/plugins/observability/docs/openapi/slo/components/schemas/timeslice_metric_doc_count_metric.yaml @@ -0,0 +1,21 @@ +title: Timeslice Metric Doc Count Metric +required: + - name + - aggregation +type: object +properties: + name: + description: The name of the metric. Only valid options are A-Z + type: string + example: A + pattern: "^[A-Z]$" + aggregation: + description: The aggregation type of the metric. Only valid option is "doc_count" + type: string + example: doc_count + enum: [doc_count] + filter: + description: The filter to apply to the metric. + type: string + example: 'processor.outcome: "success"' + diff --git a/x-pack/plugins/observability/docs/openapi/slo/components/schemas/timeslice_metric_percentile_metric.yaml b/x-pack/plugins/observability/docs/openapi/slo/components/schemas/timeslice_metric_percentile_metric.yaml new file mode 100644 index 0000000000000..c55b7e1c5abb8 --- /dev/null +++ b/x-pack/plugins/observability/docs/openapi/slo/components/schemas/timeslice_metric_percentile_metric.yaml @@ -0,0 +1,30 @@ +title: Timeslice Metric Percentile Metric +required: + - name + - aggregation + - field + - percentile +type: object +properties: + name: + description: The name of the metric. Only valid options are A-Z + type: string + example: A + pattern: "^[A-Z]$" + aggregation: + description: The aggregation type of the metric. Only valid option is "percentile" + type: string + example: percentile + enum: [percentile] + field: + description: The field of the metric. + type: string + example: processor.processed + percentile: + description: The percentile value. + type: number + example: 95 + filter: + description: The filter to apply to the metric. + type: string + example: 'processor.outcome: "success"' diff --git a/x-pack/plugins/observability/docs/openapi/slo/components/schemas/update_slo_request.yaml b/x-pack/plugins/observability/docs/openapi/slo/components/schemas/update_slo_request.yaml index ddeb2e39159a3..8d2c61c7b2249 100644 --- a/x-pack/plugins/observability/docs/openapi/slo/components/schemas/update_slo_request.yaml +++ b/x-pack/plugins/observability/docs/openapi/slo/components/schemas/update_slo_request.yaml @@ -17,6 +17,7 @@ properties: - $ref: "indicator_properties_apm_latency.yaml" - $ref: "indicator_properties_custom_metric.yaml" - $ref: "indicator_properties_histogram.yaml" + - $ref: "indicator_properties_timeslice_metric.yaml" timeWindow: $ref: "time_window.yaml" budgetingMethod: diff --git a/x-pack/plugins/observability/public/pages/overview/components/sections/metrics/metric_with_sparkline.tsx b/x-pack/plugins/observability/public/pages/overview/components/sections/metrics/metric_with_sparkline.tsx index 56e518e835c43..bbd3b9acd224c 100644 --- a/x-pack/plugins/observability/public/pages/overview/components/sections/metrics/metric_with_sparkline.tsx +++ b/x-pack/plugins/observability/public/pages/overview/components/sections/metrics/metric_with_sparkline.tsx @@ -8,6 +8,7 @@ import { Chart, Settings, AreaSeries, TooltipType, Tooltip } from '@elastic/charts'; import { EuiFlexItem, EuiFlexGroup, EuiIcon, EuiTextColor } from '@elastic/eui'; import React, { useContext } from 'react'; +import { FormattedMessage } from '@kbn/i18n-react'; import { EUI_CHARTS_THEME_DARK, EUI_CHARTS_THEME_LIGHT, @@ -39,7 +40,11 @@ export function MetricWithSparkline({ id, formatter, value, timeseries, color }: return ( -  N/A + Β  + ); } diff --git a/x-pack/plugins/observability/public/pages/slo_details/components/overview/overview.tsx b/x-pack/plugins/observability/public/pages/slo_details/components/overview/overview.tsx index f3a6abb829984..03d76b8dc2a7d 100644 --- a/x-pack/plugins/observability/public/pages/slo_details/components/overview/overview.tsx +++ b/x-pack/plugins/observability/public/pages/slo_details/components/overview/overview.tsx @@ -94,16 +94,26 @@ export function Overview({ slo }: Props) { ) : ( {BUDGETING_METHOD_TIMESLICES} ( - {i18n.translate( - 'xpack.observability.slo.sloDetails.overview.timeslicesBudgetingMethodDetails', - { - defaultMessage: '{duration} slices, {target} target', - values: { - duration: toDurationLabel(slo.objective.timesliceWindow!), - target: numeral(slo.objective.timesliceTarget!).format(percentFormat), - }, - } - )} + {slo.indicator.type === 'sli.metric.timeslice' + ? i18n.translate( + 'xpack.observability.slo.sloDetails.overview.timeslicesBudgetingMethodDetailsForTimesliceMetric', + { + defaultMessage: '{duration} slices', + values: { + duration: toDurationLabel(slo.objective.timesliceWindow!), + }, + } + ) + : i18n.translate( + 'xpack.observability.slo.sloDetails.overview.timeslicesBudgetingMethodDetails', + { + defaultMessage: '{duration} slices, {target} target', + values: { + duration: toDurationLabel(slo.objective.timesliceWindow!), + target: numeral(slo.objective.timesliceTarget!).format(percentFormat), + }, + } + )} ) ) diff --git a/x-pack/plugins/observability/public/pages/slo_edit/components/common/data_preview_chart.tsx b/x-pack/plugins/observability/public/pages/slo_edit/components/common/data_preview_chart.tsx index 8f6fe11cae333..7dc2e00f60829 100644 --- a/x-pack/plugins/observability/public/pages/slo_edit/components/common/data_preview_chart.tsx +++ b/x-pack/plugins/observability/public/pages/slo_edit/components/common/data_preview_chart.tsx @@ -5,7 +5,18 @@ * 2.0. */ -import { AreaSeries, Axis, Chart, Position, ScaleType, Settings, Tooltip } from '@elastic/charts'; +import { + AnnotationDomainType, + AreaSeries, + Axis, + Chart, + LineAnnotation, + Position, + RectAnnotation, + ScaleType, + Settings, + Tooltip, +} from '@elastic/charts'; import { EuiFlexGroup, EuiFlexItem, @@ -22,12 +33,27 @@ import moment from 'moment'; import React from 'react'; import { useFormContext } from 'react-hook-form'; import { FormattedMessage } from '@kbn/i18n-react'; +import { min, max } from 'lodash'; import { useKibana } from '../../../../utils/kibana_react'; import { useDebouncedGetPreviewData } from '../../hooks/use_preview'; import { useSectionFormValidation } from '../../hooks/use_section_form_validation'; import { CreateSLOForm } from '../../types'; -export function DataPreviewChart() { +interface DataPreviewChartProps { + formatPattern?: string; + threshold?: number; + thresholdDirection?: 'above' | 'below'; + thresholdColor?: string; + thresholdMessage?: string; +} + +export function DataPreviewChart({ + formatPattern, + threshold, + thresholdDirection, + thresholdColor, + thresholdMessage, +}: DataPreviewChartProps) { const { watch, getFieldState, formState, getValues } = useFormContext(); const { charts, uiSettings } = useKibana().services; const { isIndicatorSectionValid } = useSectionFormValidation({ @@ -47,7 +73,22 @@ export function DataPreviewChart() { const theme = charts.theme.useChartsTheme(); const baseTheme = charts.theme.useChartsBaseTheme(); const dateFormat = uiSettings.get('dateFormat'); - const percentFormat = uiSettings.get('format:percent:defaultPattern'); + const numberFormat = + formatPattern != null + ? formatPattern + : (uiSettings.get('format:percent:defaultPattern') as string); + + const values = (previewData || []).map((row) => row.sliValue); + const maxValue = max(values); + const minValue = min(values); + const domain = { + fit: true, + min: + threshold != null && minValue != null && threshold < minValue ? threshold : minValue || NaN, + max: + threshold != null && maxValue != null && threshold > maxValue ? threshold : maxValue || NaN, + }; + const title = ( <> @@ -85,6 +126,39 @@ export function DataPreviewChart() { ); } + const annotation = threshold != null && ( + <> + + + + ); + return ( {title} @@ -127,6 +201,8 @@ export function DataPreviewChart() { locale={i18n.getLocale()} /> + {annotation} + numeral(d).format(percentFormat)} - domain={{ - fit: true, - min: NaN, - max: NaN, - }} + tickFormat={(d) => numeral(d).format(numberFormat)} + domain={domain} /> { const defaultEquation = createEquationFromMetric(previousNames); diff --git a/x-pack/plugins/observability/public/pages/slo_edit/components/slo_edit_form_indicator_section.tsx b/x-pack/plugins/observability/public/pages/slo_edit/components/slo_edit_form_indicator_section.tsx index 1be3a5b10e537..3c87168792303 100644 --- a/x-pack/plugins/observability/public/pages/slo_edit/components/slo_edit_form_indicator_section.tsx +++ b/x-pack/plugins/observability/public/pages/slo_edit/components/slo_edit_form_indicator_section.tsx @@ -18,6 +18,7 @@ import { CustomKqlIndicatorTypeForm } from './custom_kql/custom_kql_indicator_ty import { CustomMetricIndicatorTypeForm } from './custom_metric/custom_metric_type_form'; import { HistogramIndicatorTypeForm } from './histogram/histogram_indicator_type_form'; import { maxWidth } from './slo_edit_form'; +import { TimesliceMetricIndicatorTypeForm } from './timeslice_metric/timeslice_metric_indicator'; interface SloEditFormIndicatorSectionProps { isEditMode: boolean; @@ -39,6 +40,8 @@ export function SloEditFormIndicatorSection({ isEditMode }: SloEditFormIndicator return ; case 'sli.histogram.custom': return ; + case 'sli.metric.timeslice': + return ; default: return null; } diff --git a/x-pack/plugins/observability/public/pages/slo_edit/components/slo_edit_form_objective_section.tsx b/x-pack/plugins/observability/public/pages/slo_edit/components/slo_edit_form_objective_section.tsx index 951e879f43cf1..ad1f84183a138 100644 --- a/x-pack/plugins/observability/public/pages/slo_edit/components/slo_edit_form_objective_section.tsx +++ b/x-pack/plugins/observability/public/pages/slo_edit/components/slo_edit_form_objective_section.tsx @@ -6,6 +6,7 @@ */ import { + EuiCallOut, EuiFieldNumber, EuiFlexGrid, EuiFlexItem, @@ -20,6 +21,7 @@ import { i18n } from '@kbn/i18n'; import { TimeWindow } from '@kbn/slo-schema'; import React, { useEffect, useState } from 'react'; import { Controller, useFormContext } from 'react-hook-form'; +import { FormattedMessage } from '@kbn/i18n-react'; import { BUDGETING_METHOD_OPTIONS, CALENDARALIGNED_TIMEWINDOW_OPTIONS, @@ -42,6 +44,7 @@ export function SloEditFormObjectiveSection() { const timeWindowTypeSelect = useGeneratedHtmlId({ prefix: 'timeWindowTypeSelect' }); const timeWindowSelect = useGeneratedHtmlId({ prefix: 'timeWindowSelect' }); const timeWindowType = watch('timeWindow.type'); + const indicator = watch('indicator.type'); const [timeWindowTypeState, setTimeWindowTypeState] = useState( defaultValues?.timeWindow?.type @@ -169,6 +172,19 @@ export function SloEditFormObjectiveSection() { + {indicator === 'sli.metric.timeslice' && ( + + +

+ +

+ + + + )} @@ -198,6 +214,7 @@ export function SloEditFormObjectiveSection() { render={({ field: { ref, ...field } }) => ( (); + const { control, getFieldState, watch } = useFormContext(); + const indicator = watch('indicator.type'); return ( <> @@ -47,6 +48,7 @@ export function SloEditFormObjectiveSectionTimeslices() { String.fromCharCode(c)); +const INVALID_EQUATION_REGEX = /[^A-Z|+|\-|\s|\d+|\.|\(|\)|\/|\*|>|<|=|\?|\:|&|\!|\|]+/; + +const validateEquation = (value: string) => { + const result = value.match(INVALID_EQUATION_REGEX); + return result === null; +}; + +function createEquationFromMetric(names: string[]) { + return names.join(' + '); +} + +const equationLabel = i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.timesliceMetric.equationLabel', + { defaultMessage: 'Equation' } +); + +const equationTooltip = ( + +); + +const thresholdLabel = i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.timesliceMetric.thresholdLabel', + { + defaultMessage: 'Threshold', + } +); + +const thresholdTooltip = ( + +); + +export function MetricIndicator({ indexFields, isLoadingIndex }: MetricIndicatorProps) { + const { control, watch, setValue, register, getFieldState } = useFormContext(); + + const { fields, append, remove } = useFieldArray({ + control, + name: `indicator.params.metric.metrics`, + }); + const equation = watch(`indicator.params.metric.equation`); + const indexPattern = watch('indicator.params.index'); + + const disableAdd = fields?.length === MAX_VARIABLES || !indexPattern; + const disableDelete = fields?.length === 1 || !indexPattern; + + const setDefaultEquationIfUnchanged = (previousNames: string[], nextNames: string[]) => { + const defaultEquation = createEquationFromMetric(previousNames); + if (defaultEquation === equation) { + setValue(`indicator.params.metric.equation`, createEquationFromMetric(nextNames)); + } + }; + + const handleDeleteMetric = (index: number) => () => { + const currentVars = fields.map((m) => m.name) ?? ['A']; + const deletedVar = currentVars[index]; + setDefaultEquationIfUnchanged(currentVars, xor(currentVars, [deletedVar])); + remove(index); + }; + + const handleAddMetric = () => { + const currentVars = fields.map((m) => m.name) ?? ['A']; + const name = first(xor(VAR_NAMES, currentVars))!; + setDefaultEquationIfUnchanged(currentVars, [...currentVars, name]); + append({ ...NEW_TIMESLICE_METRIC, name }); + }; + + return ( + <> + + {fields?.map((metric, index) => ( + + + + + + + + + + + ))} + + + + + + + + + + + + + + ( + + {equationLabel} {equationTooltip} + + } + isInvalid={fieldState.invalid} + error={[ + i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.timesliceMetric.equation.invalidCharacters', + { + defaultMessage: + 'The equation field only supports the following characters: A-Z, +, -, /, *, (, ), ?, !, &, :, |, >, <, =', + } + ), + ]} + > + field.onChange(event.target.value)} + /> + + )} + /> + + + ( + + field.onChange(event.target.value)} + /> + + )} + /> + + + + {thresholdLabel} {thresholdTooltip} + + } + > + ( + field.onChange(Number(event.target.value))} + /> + )} + /> + + + + + + +

+ {i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.timesliceMetric.equationHelpText', + { + defaultMessage: + 'Supports basic math equations, valid charaters are: A-Z, +, -, /, *, (, ), ?, !, &, :, |, >, <, =', + } + )} +

+ + + + + ); +} diff --git a/x-pack/plugins/observability/public/pages/slo_edit/components/timeslice_metric/metric_input.tsx b/x-pack/plugins/observability/public/pages/slo_edit/components/timeslice_metric/metric_input.tsx new file mode 100644 index 0000000000000..0db4c1d585782 --- /dev/null +++ b/x-pack/plugins/observability/public/pages/slo_edit/components/timeslice_metric/metric_input.tsx @@ -0,0 +1,263 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { + EuiComboBox, + EuiComboBoxOptionOption, + EuiFieldNumber, + EuiFlexItem, + EuiFormRow, + EuiIconTip, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { Controller, useFormContext } from 'react-hook-form'; +import { createOptionsFromFields } from '../../helpers/create_options'; +import { QueryBuilder } from '../common/query_builder'; +import { CreateSLOForm } from '../../types'; +import { AGGREGATION_OPTIONS, aggValueToLabel } from '../../helpers/aggregation_options'; +import { Field } from '../../../../hooks/slo/use_fetch_index_pattern_fields'; + +const fieldLabel = i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.timesliceMetric.fieldLabel', + { defaultMessage: 'Field' } +); + +const aggregationLabel = i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.timesliceMetric.aggregationLabel', + { defaultMessage: 'Aggregation' } +); + +const filterLabel = i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.timesliceMetric.filterLabel', + { defaultMessage: 'Filter' } +); + +const fieldTooltip = ( + +); + +const NUMERIC_FIELD_TYPES = ['number', 'histogram']; +const CARDINALITY_FIELD_TYPES = ['number', 'string']; + +interface MetricInputProps { + metricIndex: number; + indexPattern: string; + isLoadingIndex: boolean; + indexFields: Field[]; +} + +export function MetricInput({ + metricIndex: index, + indexPattern, + isLoadingIndex, + indexFields, +}: MetricInputProps) { + const { control, watch } = useFormContext(); + const metric = watch(`indicator.params.metric.metrics.${index}`); + const metricFields = indexFields.filter((field) => + metric.aggregation === 'cardinality' + ? CARDINALITY_FIELD_TYPES.includes(field.type) + : NUMERIC_FIELD_TYPES.includes(field.type) + ); + return ( + <> + + ( + + {aggregationLabel} {metric.name} + + } + isInvalid={fieldState.invalid} + > + { + if (selected.length) { + return field.onChange(selected[0].value); + } + field.onChange(''); + }} + selectedOptions={ + !!indexPattern && + !!field.value && + AGGREGATION_OPTIONS.some((agg) => agg.value === agg.value) + ? [ + { + value: field.value, + label: aggValueToLabel(field.value), + }, + ] + : [] + } + options={AGGREGATION_OPTIONS} + /> + + )} + /> + + {metric.aggregation === 'percentile' && ( + + ( + + {i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.timesliceMetric.percentileLabel', + { defaultMessage: 'Percentile' } + )}{' '} + {metric.name} + + } + > + onChange(Number(event.target.value))} + /> + + )} + /> + + )} + {metric.aggregation !== 'doc_count' && ( + + ( + + {fieldLabel} {metric.name} {fieldTooltip} + + } + > + { + if (selected.length) { + return field.onChange(selected[0].value); + } + field.onChange(''); + }} + selectedOptions={ + !!indexPattern && + !!field.value && + metricFields.some((metricField) => metricField.name === field.value) + ? [ + { + value: field.value, + label: field.value, + }, + ] + : [] + } + options={createOptionsFromFields(metricFields)} + /> + + )} + /> + + )} + + + } + /> + + + ); +} diff --git a/x-pack/plugins/observability/public/pages/slo_edit/components/timeslice_metric/timeslice_metric_indicator.tsx b/x-pack/plugins/observability/public/pages/slo_edit/components/timeslice_metric/timeslice_metric_indicator.tsx new file mode 100644 index 0000000000000..5d455a601e3d7 --- /dev/null +++ b/x-pack/plugins/observability/public/pages/slo_edit/components/timeslice_metric/timeslice_metric_indicator.tsx @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EuiFlexGroup, + EuiFlexItem, + EuiHorizontalRule, + EuiIconTip, + EuiSpacer, + EuiTitle, + useEuiTheme, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import React from 'react'; +import { useFormContext } from 'react-hook-form'; +import { useFetchIndexPatternFields } from '../../../../hooks/slo/use_fetch_index_pattern_fields'; +import { CreateSLOForm } from '../../types'; +import { DataPreviewChart } from '../common/data_preview_chart'; +import { IndexFieldSelector } from '../common/index_field_selector'; +import { QueryBuilder } from '../common/query_builder'; +import { IndexSelection } from '../custom_common/index_selection'; +import { MetricIndicator } from './metric_indicator'; +import { useKibana } from '../../../../utils/kibana_react'; +import { COMPARATOR_MAPPING } from '../../constants'; + +export { NEW_TIMESLICE_METRIC } from './metric_indicator'; + +export function TimesliceMetricIndicatorTypeForm() { + const { watch } = useFormContext(); + const index = watch('indicator.params.index'); + const { isLoading: isIndexFieldsLoading, data: indexFields = [] } = + useFetchIndexPatternFields(index); + const timestampFields = indexFields.filter((field) => field.type === 'date'); + const partitionByFields = indexFields.filter((field) => field.aggregatable); + const { uiSettings } = useKibana().services; + const threshold = watch('indicator.params.metric.threshold'); + const comparator = watch('indicator.params.metric.comparator'); + const { euiTheme } = useEuiTheme(); + + return ( + <> + +

+ +

+ + + + + + + + + + + + + + + } + /> + + + + + + + + +

+ +

+ + + + + + + + + + + {i18n.translate('xpack.observability.slo.sloEdit.groupBy.label', { + defaultMessage: 'Partition by', + })}{' '} + + + } + placeholder={i18n.translate('xpack.observability.slo.sloEdit.groupBy.placeholder', { + defaultMessage: 'Select an optional field to partition by', + })} + isLoading={!!index && isIndexFieldsLoading} + isDisabled={!index} + /> + + + + + ); +} diff --git a/x-pack/plugins/observability/public/pages/slo_edit/constants.ts b/x-pack/plugins/observability/public/pages/slo_edit/constants.ts index 4c38525784a10..9c5284855c772 100644 --- a/x-pack/plugins/observability/public/pages/slo_edit/constants.ts +++ b/x-pack/plugins/observability/public/pages/slo_edit/constants.ts @@ -15,6 +15,7 @@ import { IndicatorType, KQLCustomIndicator, MetricCustomIndicator, + TimesliceMetricIndicator, TimeWindow, } from '@kbn/slo-schema'; import { @@ -25,6 +26,7 @@ import { INDICATOR_CUSTOM_KQL, INDICATOR_CUSTOM_METRIC, INDICATOR_HISTOGRAM, + INDICATOR_TIMESLICE_METRIC, } from '../../utils/slo/labels'; import { CreateSLOForm } from './types'; @@ -40,6 +42,10 @@ export const SLI_OPTIONS: Array<{ value: 'sli.metric.custom', text: INDICATOR_CUSTOM_METRIC, }, + { + value: 'sli.metric.timeslice', + text: INDICATOR_TIMESLICE_METRIC, + }, { value: 'sli.histogram.custom', text: INDICATOR_HISTOGRAM, @@ -125,6 +131,21 @@ export const CUSTOM_METRIC_DEFAULT_VALUES: MetricCustomIndicator = { }, }; +export const TIMESLICE_METRIC_DEFAULT_VALUES: TimesliceMetricIndicator = { + type: 'sli.metric.timeslice' as const, + params: { + index: '', + filter: '', + metric: { + metrics: [{ name: 'A', aggregation: 'avg' as const, field: '' }], + equation: 'A', + comparator: 'GT', + threshold: 0, + }, + timestampField: '', + }, +}; + export const HISTOGRAM_DEFAULT_VALUES: HistogramIndicator = { type: 'sli.histogram.custom' as const, params: { @@ -198,3 +219,57 @@ export const SLO_EDIT_FORM_DEFAULT_VALUES_CUSTOM_METRIC: CreateSLOForm = { }, groupBy: ALL_VALUE, }; + +export const COMPARATOR_GT = i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.timesliceMetric.gtLabel', + { + defaultMessage: 'Greater than', + } +); + +export const COMPARATOR_GTE = i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.timesliceMetric.gteLabel', + { + defaultMessage: 'Greater than or equal to', + } +); + +export const COMPARATOR_LT = i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.timesliceMetric.ltLabel', + { + defaultMessage: 'Less than', + } +); + +export const COMPARATOR_LTE = i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.timesliceMetric.lteLabel', + { + defaultMessage: 'Less than or equal to', + } +); + +export const COMPARATOR_MAPPING = { + GT: COMPARATOR_GT, + GTE: COMPARATOR_GTE, + LT: COMPARATOR_LT, + LTE: COMPARATOR_LTE, +}; + +export const COMPARATOR_OPTIONS = [ + { + text: COMPARATOR_GT, + value: 'GT' as const, + }, + { + text: COMPARATOR_GTE, + value: 'GTE' as const, + }, + { + text: COMPARATOR_LT, + value: 'LT' as const, + }, + { + text: COMPARATOR_LTE, + value: 'LTE' as const, + }, +]; diff --git a/x-pack/plugins/observability/public/pages/slo_edit/helpers/aggregation_options.ts b/x-pack/plugins/observability/public/pages/slo_edit/helpers/aggregation_options.ts new file mode 100644 index 0000000000000..4a3a5fb9cf28a --- /dev/null +++ b/x-pack/plugins/observability/public/pages/slo_edit/helpers/aggregation_options.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +export const AGGREGATION_OPTIONS = [ + { + value: 'avg', + label: i18n.translate('xpack.observability.slo.sloEdit.timesliceMetric.aggregation.average', { + defaultMessage: 'Average', + }), + }, + { + value: 'max', + label: i18n.translate('xpack.observability.slo.sloEdit.timesliceMetric.aggregation.max', { + defaultMessage: 'Max', + }), + }, + { + value: 'min', + label: i18n.translate('xpack.observability.slo.sloEdit.timesliceMetric.aggregation.min', { + defaultMessage: 'Min', + }), + }, + { + value: 'sum', + label: i18n.translate('xpack.observability.slo.sloEdit.timesliceMetric.aggregation.sum', { + defaultMessage: 'Sum', + }), + }, + { + value: 'cardinality', + label: i18n.translate( + 'xpack.observability.slo.sloEdit.timesliceMetric.aggregation.cardinality', + { + defaultMessage: 'Cardinality', + } + ), + }, + { + value: 'last_value', + label: i18n.translate( + 'xpack.observability.slo.sloEdit.timesliceMetric.aggregation.last_value', + { + defaultMessage: 'Last value', + } + ), + }, + { + value: 'std_deviation', + label: i18n.translate( + 'xpack.observability.slo.sloEdit.timesliceMetric.aggregation.std_deviation', + { + defaultMessage: 'Std. Deviation', + } + ), + }, + { + value: 'doc_count', + label: i18n.translate('xpack.observability.slo.sloEdit.timesliceMetric.aggregation.doc_count', { + defaultMessage: 'Doc count', + }), + }, + { + value: 'percentile', + label: i18n.translate( + 'xpack.observability.slo.sloEdit.timesliceMetric.aggregation.percentile', + { + defaultMessage: 'Percentile', + } + ), + }, +]; + +export function aggValueToLabel(value: string) { + const aggregation = AGGREGATION_OPTIONS.find((agg) => agg.value === value); + if (aggregation) { + return aggregation.label; + } + return value; +} diff --git a/x-pack/plugins/observability/public/pages/slo_edit/helpers/process_slo_form_values.ts b/x-pack/plugins/observability/public/pages/slo_edit/helpers/process_slo_form_values.ts index c6de2126adacf..f523cc1ce1ce1 100644 --- a/x-pack/plugins/observability/public/pages/slo_edit/helpers/process_slo_form_values.ts +++ b/x-pack/plugins/observability/public/pages/slo_edit/helpers/process_slo_form_values.ts @@ -15,6 +15,7 @@ import { CUSTOM_KQL_DEFAULT_VALUES, CUSTOM_METRIC_DEFAULT_VALUES, HISTOGRAM_DEFAULT_VALUES, + TIMESLICE_METRIC_DEFAULT_VALUES, } from '../constants'; import { CreateSLOForm } from '../types'; @@ -132,6 +133,11 @@ function transformPartialIndicatorState( type: 'sli.metric.custom' as const, params: Object.assign({}, CUSTOM_METRIC_DEFAULT_VALUES.params, indicator.params ?? {}), }; + case 'sli.metric.timeslice': + return { + type: 'sli.metric.timeslice' as const, + params: Object.assign({}, TIMESLICE_METRIC_DEFAULT_VALUES.params, indicator.params ?? {}), + }; default: assertNever(indicatorType); } diff --git a/x-pack/plugins/observability/public/pages/slo_edit/hooks/use_section_form_validation.ts b/x-pack/plugins/observability/public/pages/slo_edit/hooks/use_section_form_validation.ts index e0c2652bfc46d..6fede4552d6f8 100644 --- a/x-pack/plugins/observability/public/pages/slo_edit/hooks/use_section_form_validation.ts +++ b/x-pack/plugins/observability/public/pages/slo_edit/hooks/use_section_form_validation.ts @@ -5,7 +5,12 @@ * 2.0. */ -import { MetricCustomIndicator } from '@kbn/slo-schema'; +import { + MetricCustomIndicator, + timesliceMetricBasicMetricWithField, + TimesliceMetricIndicator, + timesliceMetricPercentileMetric, +} from '@kbn/slo-schema'; import { FormState, UseFormGetFieldState, UseFormGetValues, UseFormWatch } from 'react-hook-form'; import { isObject } from 'lodash'; import { CreateSLOForm } from '../types'; @@ -54,6 +59,39 @@ export function useSectionFormValidation({ getFieldState, getValues, formState, isGoodParamsValid() && isTotalParamsValid(); break; + case 'sli.metric.timeslice': + const isMetricParamsValid = () => { + const data = getValues( + 'indicator.params.metric' + ) as TimesliceMetricIndicator['params']['metric']; + const isEquationValid = !getFieldState('indicator.params.metric.equation').invalid; + const areMetricsValid = + isObject(data) && + (data.metrics ?? []).every((metric) => { + if (timesliceMetricBasicMetricWithField.is(metric)) { + return Boolean(metric.field); + } + if (timesliceMetricPercentileMetric.is(metric)) { + return Boolean(metric.field) && Boolean(metric.percentile); + } + return true; + }); + return isEquationValid && areMetricsValid; + }; + + isIndicatorSectionValid = + ( + [ + 'indicator.params.index', + 'indicator.params.filter', + 'indicator.params.timestampField', + ] as const + ).every((field) => !getFieldState(field).invalid) && + (['indicator.params.index', 'indicator.params.timestampField'] as const).every( + (field) => !!getValues(field) + ) && + isMetricParamsValid(); + break; case 'sli.histogram.custom': const isRangeValid = (type: 'good' | 'total') => { const aggregation = getValues(`indicator.params.${type}.aggregation`); diff --git a/x-pack/plugins/observability/public/pages/slo_edit/hooks/use_unregister_fields.ts b/x-pack/plugins/observability/public/pages/slo_edit/hooks/use_unregister_fields.ts index c15b5cb7fbbfc..d461c841940a4 100644 --- a/x-pack/plugins/observability/public/pages/slo_edit/hooks/use_unregister_fields.ts +++ b/x-pack/plugins/observability/public/pages/slo_edit/hooks/use_unregister_fields.ts @@ -14,10 +14,12 @@ import { useFetchApmIndex } from '../../../hooks/slo/use_fetch_apm_indices'; import { APM_AVAILABILITY_DEFAULT_VALUES, APM_LATENCY_DEFAULT_VALUES, + BUDGETING_METHOD_OPTIONS, CUSTOM_KQL_DEFAULT_VALUES, CUSTOM_METRIC_DEFAULT_VALUES, HISTOGRAM_DEFAULT_VALUES, SLO_EDIT_FORM_DEFAULT_VALUES, + TIMESLICE_METRIC_DEFAULT_VALUES, } from '../constants'; import { CreateSLOForm } from '../types'; @@ -49,6 +51,22 @@ export function useUnregisterFields({ isEditMode }: { isEditMode: boolean }) { } ); break; + case 'sli.metric.timeslice': + reset( + Object.assign({}, SLO_EDIT_FORM_DEFAULT_VALUES, { + budgetingMethod: BUDGETING_METHOD_OPTIONS[1].value, + objective: { + target: 99, + timesliceTarget: 95, + timesliceWindow: 1, + }, + indicator: TIMESLICE_METRIC_DEFAULT_VALUES, + }), + { + keepDefaultValues: true, + } + ); + break; case 'sli.kql.custom': reset( Object.assign({}, SLO_EDIT_FORM_DEFAULT_VALUES, { diff --git a/x-pack/plugins/observability/public/utils/slo/labels.ts b/x-pack/plugins/observability/public/utils/slo/labels.ts index 40c58e624bb2b..43ed455f5a9e3 100644 --- a/x-pack/plugins/observability/public/utils/slo/labels.ts +++ b/x-pack/plugins/observability/public/utils/slo/labels.ts @@ -21,6 +21,13 @@ export const INDICATOR_CUSTOM_METRIC = i18n.translate( } ); +export const INDICATOR_TIMESLICE_METRIC = i18n.translate( + 'xpack.observability.slo.indicators.timesliceMetric', + { + defaultMessage: 'Timeslice Metric', + } +); + export const INDICATOR_HISTOGRAM = i18n.translate('xpack.observability.slo.indicators.histogram', { defaultMessage: 'Histogram Metric', }); @@ -54,6 +61,9 @@ export function toIndicatorTypeLabel( case 'sli.histogram.custom': return INDICATOR_HISTOGRAM; + case 'sli.metric.timeslice': + return INDICATOR_TIMESLICE_METRIC; + default: assertNever(indicatorType as never); } diff --git a/x-pack/plugins/observability/server/routes/slo/route.ts b/x-pack/plugins/observability/server/routes/slo/route.ts index 2dfdcb2308ee3..ade3f1714ddfb 100644 --- a/x-pack/plugins/observability/server/routes/slo/route.ts +++ b/x-pack/plugins/observability/server/routes/slo/route.ts @@ -49,6 +49,7 @@ import { KQLCustomTransformGenerator, MetricCustomTransformGenerator, TransformGenerator, + TimesliceMetricTransformGenerator, } from '../../services/slo/transform_generators'; import type { ObservabilityRequestHandlerContext } from '../../types'; import { createObservabilityServerRoute } from '../create_observability_server_route'; @@ -59,6 +60,7 @@ const transformGenerators: Record = { 'sli.kql.custom': new KQLCustomTransformGenerator(), 'sli.metric.custom': new MetricCustomTransformGenerator(), 'sli.histogram.custom': new HistogramTransformGenerator(), + 'sli.metric.timeslice': new TimesliceMetricTransformGenerator(), }; const assertPlatinumLicense = async (context: ObservabilityRequestHandlerContext) => { diff --git a/x-pack/plugins/observability/server/services/slo/aggregations/__snapshots__/get_timeslice_metric_indicator_aggregation.test.ts.snap b/x-pack/plugins/observability/server/services/slo/aggregations/__snapshots__/get_timeslice_metric_indicator_aggregation.test.ts.snap new file mode 100644 index 0000000000000..8288087ce3327 --- /dev/null +++ b/x-pack/plugins/observability/server/services/slo/aggregations/__snapshots__/get_timeslice_metric_indicator_aggregation.test.ts.snap @@ -0,0 +1,213 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`GetTimesliceMetricIndicatorAggregation should generate an aggregation for basic metrics 1`] = ` +Object { + "_A": Object { + "aggs": Object { + "metric": Object { + "avg": Object { + "field": "test.field", + }, + }, + }, + "filter": Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "test.category": "test", + }, + }, + ], + }, + }, + }, + "_B": Object { + "aggs": Object { + "metric": Object { + "max": Object { + "field": "test.field", + }, + }, + }, + "filter": Object { + "match_all": Object {}, + }, + }, + "_C": Object { + "aggs": Object { + "metric": Object { + "min": Object { + "field": "test.field", + }, + }, + }, + "filter": Object { + "match_all": Object {}, + }, + }, + "_D": Object { + "aggs": Object { + "metric": Object { + "sum": Object { + "field": "test.field", + }, + }, + }, + "filter": Object { + "match_all": Object {}, + }, + }, + "_E": Object { + "aggs": Object { + "metric": Object { + "cardinality": Object { + "field": "test.field", + }, + }, + }, + "filter": Object { + "match_all": Object {}, + }, + }, + "_metric": Object { + "bucket_script": Object { + "buckets_path": Object { + "A": "_A>metric", + "B": "_B>metric", + "C": "_C>metric", + "D": "_D>metric", + "E": "_E>metric", + }, + "script": Object { + "lang": "painless", + "source": "(params.A + params.B + params.C + params.D + params.E) / params.A", + }, + }, + }, +} +`; + +exports[`GetTimesliceMetricIndicatorAggregation should generate an aggregation for doc_count 1`] = ` +Object { + "_A": Object { + "filter": Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "test.category": "test", + }, + }, + ], + }, + }, + }, + "_metric": Object { + "bucket_script": Object { + "buckets_path": Object { + "A": "_A>_count", + }, + "script": Object { + "lang": "painless", + "source": "params.A", + }, + }, + }, +} +`; + +exports[`GetTimesliceMetricIndicatorAggregation should generate an aggregation for last_value 1`] = ` +Object { + "_A": Object { + "aggs": Object { + "metric": Object { + "top_metrics": Object { + "metrics": Object { + "field": "test.field", + }, + "sort": Object { + "@timestamp": "desc", + }, + }, + }, + }, + "filter": Object { + "match_all": Object {}, + }, + }, + "_metric": Object { + "bucket_script": Object { + "buckets_path": Object { + "A": "_A>metric[test.field]", + }, + "script": Object { + "lang": "painless", + "source": "params.A", + }, + }, + }, +} +`; + +exports[`GetTimesliceMetricIndicatorAggregation should generate an aggregation for percentile 1`] = ` +Object { + "_A": Object { + "aggs": Object { + "metric": Object { + "percentiles": Object { + "field": "test.field", + "keyed": true, + "percents": Array [ + 97, + ], + }, + }, + }, + "filter": Object { + "match_all": Object {}, + }, + }, + "_metric": Object { + "bucket_script": Object { + "buckets_path": Object { + "A": "_A>metric[97]", + }, + "script": Object { + "lang": "painless", + "source": "params.A", + }, + }, + }, +} +`; + +exports[`GetTimesliceMetricIndicatorAggregation should generate an aggregation for std_deviation 1`] = ` +Object { + "_A": Object { + "aggs": Object { + "metric": Object { + "extended_stats": Object { + "field": "test.field", + }, + }, + }, + "filter": Object { + "match_all": Object {}, + }, + }, + "_metric": Object { + "bucket_script": Object { + "buckets_path": Object { + "A": "_A>metric[std_deviation]", + }, + "script": Object { + "lang": "painless", + "source": "params.A", + }, + }, + }, +} +`; diff --git a/x-pack/plugins/observability/server/services/slo/aggregations/get_custom_metric_indicator_aggregation.ts b/x-pack/plugins/observability/server/services/slo/aggregations/get_custom_metric_indicator_aggregation.ts index ceb51bdfef199..73bbb91b1041f 100644 --- a/x-pack/plugins/observability/server/services/slo/aggregations/get_custom_metric_indicator_aggregation.ts +++ b/x-pack/plugins/observability/server/services/slo/aggregations/get_custom_metric_indicator_aggregation.ts @@ -37,7 +37,7 @@ export class GetCustomMetricIndicatorAggregation { private convertEquationToPainless(bucketsPath: Record, equation: string) { const workingEquation = equation || Object.keys(bucketsPath).join(' + '); return Object.keys(bucketsPath).reduce((acc, key) => { - return acc.replace(key, `params.${key}`); + return acc.replaceAll(key, `params.${key}`); }, workingEquation); } diff --git a/x-pack/plugins/observability/server/services/slo/aggregations/get_timeslice_metric_indicator_aggregation.test.ts b/x-pack/plugins/observability/server/services/slo/aggregations/get_timeslice_metric_indicator_aggregation.test.ts new file mode 100644 index 0000000000000..4f73f186fd343 --- /dev/null +++ b/x-pack/plugins/observability/server/services/slo/aggregations/get_timeslice_metric_indicator_aggregation.test.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createTimesliceMetricIndicator } from '../fixtures/slo'; +import { GetTimesliceMetricIndicatorAggregation } from './get_timeslice_metric_indicator_aggregation'; + +describe('GetTimesliceMetricIndicatorAggregation', () => { + it('should generate an aggregation for basic metrics', () => { + const indicator = createTimesliceMetricIndicator( + [ + { + name: 'A', + aggregation: 'avg' as const, + field: 'test.field', + filter: 'test.category: test', + }, + { + name: 'B', + aggregation: 'max' as const, + field: 'test.field', + }, + { + name: 'C', + aggregation: 'min' as const, + field: 'test.field', + }, + { + name: 'D', + aggregation: 'sum' as const, + field: 'test.field', + }, + { + name: 'E', + aggregation: 'cardinality' as const, + field: 'test.field', + }, + ], + '(A + B + C + D + E) / A' + ); + const getIndicatorAggregation = new GetTimesliceMetricIndicatorAggregation(indicator); + expect(getIndicatorAggregation.execute('_metric')).toMatchSnapshot(); + }); + + it('should generate an aggregation for doc_count', () => { + const indicator = createTimesliceMetricIndicator( + [ + { + name: 'A', + aggregation: 'doc_count' as const, + filter: 'test.category: test', + }, + ], + 'A' + ); + const getIndicatorAggregation = new GetTimesliceMetricIndicatorAggregation(indicator); + expect(getIndicatorAggregation.execute('_metric')).toMatchSnapshot(); + }); + + it('should generate an aggregation for std_deviation', () => { + const indicator = createTimesliceMetricIndicator( + [ + { + name: 'A', + aggregation: 'std_deviation' as const, + field: 'test.field', + }, + ], + 'A' + ); + const getIndicatorAggregation = new GetTimesliceMetricIndicatorAggregation(indicator); + expect(getIndicatorAggregation.execute('_metric')).toMatchSnapshot(); + }); + + it('should generate an aggregation for percentile', () => { + const indicator = createTimesliceMetricIndicator( + [ + { + name: 'A', + aggregation: 'percentile' as const, + field: 'test.field', + percentile: 97, + }, + ], + 'A' + ); + const getIndicatorAggregation = new GetTimesliceMetricIndicatorAggregation(indicator); + expect(getIndicatorAggregation.execute('_metric')).toMatchSnapshot(); + }); + + it('should generate an aggregation for last_value', () => { + const indicator = createTimesliceMetricIndicator( + [ + { + name: 'A', + aggregation: 'last_value' as const, + field: 'test.field', + }, + ], + 'A' + ); + const getIndicatorAggregation = new GetTimesliceMetricIndicatorAggregation(indicator); + expect(getIndicatorAggregation.execute('_metric')).toMatchSnapshot(); + }); +}); diff --git a/x-pack/plugins/observability/server/services/slo/aggregations/get_timeslice_metric_indicator_aggregation.ts b/x-pack/plugins/observability/server/services/slo/aggregations/get_timeslice_metric_indicator_aggregation.ts new file mode 100644 index 0000000000000..e715038e324f0 --- /dev/null +++ b/x-pack/plugins/observability/server/services/slo/aggregations/get_timeslice_metric_indicator_aggregation.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TimesliceMetricIndicator, timesliceMetricMetricDef } from '@kbn/slo-schema'; +import * as t from 'io-ts'; +import { assertNever } from '@kbn/std'; + +import { getElastichsearchQueryOrThrow } from '../transform_generators'; + +type TimesliceMetricDef = TimesliceMetricIndicator['params']['metric']; +type TimesliceMetricMetricDef = t.TypeOf; + +export class GetTimesliceMetricIndicatorAggregation { + constructor(private indicator: TimesliceMetricIndicator) {} + + private buildAggregation(metric: TimesliceMetricMetricDef) { + const { aggregation } = metric; + switch (aggregation) { + case 'doc_count': + return {}; + case 'std_deviation': + return { + extended_stats: { field: metric.field }, + }; + case 'percentile': + if (metric.percentile == null) { + throw new Error('You must provide a percentile value for percentile aggregations.'); + } + return { + percentiles: { + field: metric.field, + percents: [metric.percentile], + keyed: true, + }, + }; + case 'last_value': + return { + top_metrics: { + metrics: { field: metric.field }, + sort: { [this.indicator.params.timestampField]: 'desc' }, + }, + }; + case 'avg': + case 'max': + case 'min': + case 'sum': + case 'cardinality': + if (metric.field == null) { + throw new Error('You must provide a field for basic metric aggregations.'); + } + return { + [aggregation]: { field: metric.field }, + }; + default: + assertNever(aggregation); + } + } + + private buildBucketPath(prefix: string, metric: TimesliceMetricMetricDef) { + const { aggregation } = metric; + switch (aggregation) { + case 'doc_count': + return `${prefix}>_count`; + case 'std_deviation': + return `${prefix}>metric[std_deviation]`; + case 'percentile': + return `${prefix}>metric[${metric.percentile}]`; + case 'last_value': + return `${prefix}>metric[${metric.field}]`; + case 'avg': + case 'max': + case 'min': + case 'sum': + case 'cardinality': + return `${prefix}>metric`; + default: + assertNever(aggregation); + } + } + + private buildMetricAggregations(metricDef: TimesliceMetricDef) { + return metricDef.metrics.reduce((acc, metric) => { + const filter = metric.filter + ? getElastichsearchQueryOrThrow(metric.filter) + : { match_all: {} }; + const aggs = { metric: this.buildAggregation(metric) }; + return { + ...acc, + [`_${metric.name}`]: { + filter, + ...(metric.aggregation !== 'doc_count' ? { aggs } : {}), + }, + }; + }, {}); + } + + private convertEquationToPainless(bucketsPath: Record, equation: string) { + const workingEquation = equation || Object.keys(bucketsPath).join(' + '); + return Object.keys(bucketsPath).reduce((acc, key) => { + return acc.replaceAll(key, `params.${key}`); + }, workingEquation); + } + + private buildMetricEquation(definition: TimesliceMetricDef) { + const bucketsPath = definition.metrics.reduce( + (acc, metric) => ({ ...acc, [metric.name]: this.buildBucketPath(`_${metric.name}`, metric) }), + {} + ); + return { + bucket_script: { + buckets_path: bucketsPath, + script: { + source: this.convertEquationToPainless(bucketsPath, definition.equation), + lang: 'painless', + }, + }, + }; + } + + public execute(aggregationKey: string) { + return { + ...this.buildMetricAggregations(this.indicator.params.metric), + [aggregationKey]: this.buildMetricEquation(this.indicator.params.metric), + }; + } +} diff --git a/x-pack/plugins/observability/server/services/slo/aggregations/index.ts b/x-pack/plugins/observability/server/services/slo/aggregations/index.ts index b814152a4fcd5..6df05b4b2eac5 100644 --- a/x-pack/plugins/observability/server/services/slo/aggregations/index.ts +++ b/x-pack/plugins/observability/server/services/slo/aggregations/index.ts @@ -7,3 +7,4 @@ export * from './get_histogram_indicator_aggregation'; export * from './get_custom_metric_indicator_aggregation'; +export * from './get_timeslice_metric_indicator_aggregation'; diff --git a/x-pack/plugins/observability/server/services/slo/fixtures/slo.ts b/x-pack/plugins/observability/server/services/slo/fixtures/slo.ts index 494c54cd65741..e423a0441f9d5 100644 --- a/x-pack/plugins/observability/server/services/slo/fixtures/slo.ts +++ b/x-pack/plugins/observability/server/services/slo/fixtures/slo.ts @@ -6,7 +6,13 @@ */ import { SavedObject } from '@kbn/core-saved-objects-server'; -import { ALL_VALUE, CreateSLOParams, HistogramIndicator, sloSchema } from '@kbn/slo-schema'; +import { + ALL_VALUE, + CreateSLOParams, + HistogramIndicator, + sloSchema, + TimesliceMetricIndicator, +} from '@kbn/slo-schema'; import { cloneDeep } from 'lodash'; import { v1 as uuidv1 } from 'uuid'; import { @@ -90,6 +96,25 @@ export const createMetricCustomIndicator = ( }, }); +export const createTimesliceMetricIndicator = ( + metrics: TimesliceMetricIndicator['params']['metric']['metrics'] = [], + equation: TimesliceMetricIndicator['params']['metric']['equation'] = '', + queryFilter = '' +): TimesliceMetricIndicator => ({ + type: 'sli.metric.timeslice', + params: { + index: 'test-*', + timestampField: '@timestamp', + filter: queryFilter, + metric: { + metrics, + equation, + threshold: 100, + comparator: 'GTE', + }, + }, +}); + export const createHistogramIndicator = ( params: Partial = {} ): HistogramIndicator => ({ diff --git a/x-pack/plugins/observability/server/services/slo/get_preview_data.ts b/x-pack/plugins/observability/server/services/slo/get_preview_data.ts index 98f07e1f8ed5e..2fd80c09d3875 100644 --- a/x-pack/plugins/observability/server/services/slo/get_preview_data.ts +++ b/x-pack/plugins/observability/server/services/slo/get_preview_data.ts @@ -15,6 +15,7 @@ import { HistogramIndicator, KQLCustomIndicator, MetricCustomIndicator, + TimesliceMetricIndicator, } from '@kbn/slo-schema'; import { assertNever } from '@kbn/std'; import { APMTransactionDurationIndicator } from '../../domain/models'; @@ -23,6 +24,7 @@ import { InvalidQueryError } from '../../errors'; import { GetCustomMetricIndicatorAggregation, GetHistogramIndicatorAggregation, + GetTimesliceMetricIndicatorAggregation, } from './aggregations'; export class GetPreviewData { @@ -55,6 +57,7 @@ export class GetPreviewData { const result = await this.esClient.search({ index: indicator.params.index, + size: 0, query: { bool: { filter: [ @@ -130,6 +133,7 @@ export class GetPreviewData { const result = await this.esClient.search({ index: indicator.params.index, + size: 0, query: { bool: { filter: [ @@ -186,6 +190,7 @@ export class GetPreviewData { const timestampField = indicator.params.timestampField; const options = { index: indicator.params.index, + size: 0, query: { bool: { filter: [{ range: { [timestampField]: { gte: 'now-60m' } } }, filterQuery], @@ -228,6 +233,7 @@ export class GetPreviewData { const getCustomMetricIndicatorAggregation = new GetCustomMetricIndicatorAggregation(indicator); const result = await this.esClient.search({ index: indicator.params.index, + size: 0, query: { bool: { filter: [{ range: { [timestampField]: { gte: 'now-60m' } } }, filterQuery], @@ -261,6 +267,42 @@ export class GetPreviewData { })); } + private async getTimesliceMetricPreviewData( + indicator: TimesliceMetricIndicator + ): Promise { + const timestampField = indicator.params.timestampField; + const filterQuery = getElastichsearchQueryOrThrow(indicator.params.filter); + const getCustomMetricIndicatorAggregation = new GetTimesliceMetricIndicatorAggregation( + indicator + ); + const result = await this.esClient.search({ + index: indicator.params.index, + size: 0, + query: { + bool: { + filter: [{ range: { [timestampField]: { gte: 'now-60m' } } }, filterQuery], + }, + }, + aggs: { + perMinute: { + date_histogram: { + field: timestampField, + fixed_interval: '1m', + }, + aggs: { + ...getCustomMetricIndicatorAggregation.execute('metric'), + }, + }, + }, + }); + + // @ts-ignore buckets is not improperly typed + return result.aggregations?.perMinute.buckets.map((bucket) => ({ + date: bucket.key_as_string, + sliValue: !!bucket.metric ? bucket.metric.value : null, + })); + } + private async getCustomKQLPreviewData( indicator: KQLCustomIndicator ): Promise { @@ -270,6 +312,7 @@ export class GetPreviewData { const timestampField = indicator.params.timestampField; const result = await this.esClient.search({ index: indicator.params.index, + size: 0, query: { bool: { filter: [{ range: { [timestampField]: { gte: 'now-60m' } } }, filterQuery], @@ -313,6 +356,8 @@ export class GetPreviewData { return this.getHistogramPreviewData(params.indicator); case 'sli.metric.custom': return this.getCustomMetricPreviewData(params.indicator); + case 'sli.metric.timeslice': + return this.getTimesliceMetricPreviewData(params.indicator); default: assertNever(type); } diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/timeslice_metric.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/timeslice_metric.test.ts.snap new file mode 100644 index 0000000000000..e2698ba3e1793 --- /dev/null +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/timeslice_metric.test.ts.snap @@ -0,0 +1,697 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Timeslice Metric Transform Generator filters the source using the kql query 1`] = ` +Object { + "bool": Object { + "filter": Array [ + Object { + "range": Object { + "@timestamp": Object { + "gte": "now-7d/d", + }, + }, + }, + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match_phrase": Object { + "test.category": "test", + }, + }, + ], + }, + }, + ], + }, +} +`; + +exports[`Timeslice Metric Transform Generator returns the expected transform params for timeslices slo 1`] = ` +Object { + "_meta": Object { + "managed": true, + "managed_by": "observability", + "version": 2, + }, + "description": "Rolled-up SLI data for SLO: irrelevant", + "dest": Object { + "index": ".slo-observability.sli-v2", + "pipeline": ".slo-observability.sli.pipeline", + }, + "frequency": "1m", + "pivot": Object { + "aggregations": Object { + "_A": Object { + "aggs": Object { + "metric": Object { + "avg": Object { + "field": "test.field", + }, + }, + }, + "filter": Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match_phrase": Object { + "test.category": "test", + }, + }, + ], + }, + }, + }, + "_B": Object { + "filter": Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match_phrase": Object { + "test.category": "test", + }, + }, + ], + }, + }, + }, + "_C": Object { + "aggs": Object { + "metric": Object { + "top_metrics": Object { + "metrics": Object { + "field": "test.field", + }, + "sort": Object { + "@timestamp": "desc", + }, + }, + }, + }, + "filter": Object { + "match_all": Object {}, + }, + }, + "_D": Object { + "aggs": Object { + "metric": Object { + "extended_stats": Object { + "field": "test.field", + }, + }, + }, + "filter": Object { + "match_all": Object {}, + }, + }, + "_E": Object { + "aggs": Object { + "metric": Object { + "percentiles": Object { + "field": "test.field", + "keyed": true, + "percents": Array [ + 97, + ], + }, + }, + }, + "filter": Object { + "match_all": Object {}, + }, + }, + "_metric": Object { + "bucket_script": Object { + "buckets_path": Object { + "A": "_A>metric", + "B": "_B>_count", + "C": "_C>metric[test.field]", + "D": "_D>metric[std_deviation]", + "E": "_E>metric[97]", + }, + "script": Object { + "lang": "painless", + "source": "(params.A + params.B + params.C + params.D + params.E) / params.B", + }, + }, + }, + "slo.denominator": Object { + "bucket_script": Object { + "buckets_path": Object {}, + "script": "1", + }, + }, + "slo.isGoodSlice": Object { + "bucket_script": Object { + "buckets_path": Object { + "goodEvents": "slo.numerator>value", + }, + "script": "params.goodEvents == 1 ? 1 : 0", + }, + }, + "slo.numerator": Object { + "bucket_script": Object { + "buckets_path": Object { + "value": "_metric>value", + }, + "script": Object { + "params": Object { + "threshold": 100, + }, + "source": "params.value >= params.threshold ? 1 : 0", + }, + }, + }, + }, + "group_by": Object { + "@timestamp": Object { + "date_histogram": Object { + "field": "@timestamp", + "fixed_interval": "2m", + }, + }, + "slo.budgetingMethod": Object { + "terms": Object { + "field": "slo.budgetingMethod", + }, + }, + "slo.description": Object { + "terms": Object { + "field": "slo.description", + }, + }, + "slo.groupBy": Object { + "terms": Object { + "field": "slo.groupBy", + }, + }, + "slo.id": Object { + "terms": Object { + "field": "slo.id", + }, + }, + "slo.indicator.type": Object { + "terms": Object { + "field": "slo.indicator.type", + }, + }, + "slo.instanceId": Object { + "terms": Object { + "field": "slo.instanceId", + }, + }, + "slo.name": Object { + "terms": Object { + "field": "slo.name", + }, + }, + "slo.objective.sliceDurationInSeconds": Object { + "terms": Object { + "field": "slo.objective.sliceDurationInSeconds", + }, + }, + "slo.objective.target": Object { + "terms": Object { + "field": "slo.objective.target", + }, + }, + "slo.revision": Object { + "terms": Object { + "field": "slo.revision", + }, + }, + "slo.tags": Object { + "terms": Object { + "field": "slo.tags", + }, + }, + "slo.timeWindow.duration": Object { + "terms": Object { + "field": "slo.timeWindow.duration", + }, + }, + "slo.timeWindow.type": Object { + "terms": Object { + "field": "slo.timeWindow.type", + }, + }, + }, + }, + "settings": Object { + "deduce_mappings": false, + "unattended": true, + }, + "source": Object { + "index": "test-*", + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "range": Object { + "@timestamp": Object { + "gte": "now-7d/d", + }, + }, + }, + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match_phrase": Object { + "test.category": "test", + }, + }, + ], + }, + }, + ], + }, + }, + "runtime_mappings": Object { + "slo.budgetingMethod": Object { + "script": Object { + "source": "emit('timeslices')", + }, + "type": "keyword", + }, + "slo.description": Object { + "script": Object { + "source": "emit('irrelevant')", + }, + "type": "keyword", + }, + "slo.groupBy": Object { + "script": Object { + "source": "emit('*')", + }, + "type": "keyword", + }, + "slo.id": Object { + "script": Object { + "source": Any, + }, + "type": "keyword", + }, + "slo.indicator.type": Object { + "script": Object { + "source": "emit('sli.metric.timeslice')", + }, + "type": "keyword", + }, + "slo.instanceId": Object { + "script": Object { + "source": "emit('*')", + }, + "type": "keyword", + }, + "slo.name": Object { + "script": Object { + "source": "emit('irrelevant')", + }, + "type": "keyword", + }, + "slo.objective.sliceDurationInSeconds": Object { + "script": Object { + "source": "emit(120)", + }, + "type": "long", + }, + "slo.objective.target": Object { + "script": Object { + "source": "emit(0.98)", + }, + "type": "double", + }, + "slo.revision": Object { + "script": Object { + "source": "emit(1)", + }, + "type": "long", + }, + "slo.tags": Object { + "script": Object { + "source": "emit('critical,k8s')", + }, + "type": "keyword", + }, + "slo.timeWindow.duration": Object { + "script": Object { + "source": "emit('7d')", + }, + "type": "keyword", + }, + "slo.timeWindow.type": Object { + "script": Object { + "source": "emit('rolling')", + }, + "type": "keyword", + }, + }, + }, + "sync": Object { + "time": Object { + "delay": "1m", + "field": "@timestamp", + }, + }, + "transform_id": Any, +} +`; + +exports[`Timeslice Metric Transform Generator returns the expected transform params with every specified indicator params 1`] = ` +Object { + "_meta": Object { + "managed": true, + "managed_by": "observability", + "version": 2, + }, + "description": "Rolled-up SLI data for SLO: irrelevant", + "dest": Object { + "index": ".slo-observability.sli-v2", + "pipeline": ".slo-observability.sli.pipeline", + }, + "frequency": "1m", + "pivot": Object { + "aggregations": Object { + "_A": Object { + "aggs": Object { + "metric": Object { + "avg": Object { + "field": "test.field", + }, + }, + }, + "filter": Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match_phrase": Object { + "test.category": "test", + }, + }, + ], + }, + }, + }, + "_B": Object { + "filter": Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match_phrase": Object { + "test.category": "test", + }, + }, + ], + }, + }, + }, + "_C": Object { + "aggs": Object { + "metric": Object { + "top_metrics": Object { + "metrics": Object { + "field": "test.field", + }, + "sort": Object { + "@timestamp": "desc", + }, + }, + }, + }, + "filter": Object { + "match_all": Object {}, + }, + }, + "_D": Object { + "aggs": Object { + "metric": Object { + "extended_stats": Object { + "field": "test.field", + }, + }, + }, + "filter": Object { + "match_all": Object {}, + }, + }, + "_E": Object { + "aggs": Object { + "metric": Object { + "percentiles": Object { + "field": "test.field", + "keyed": true, + "percents": Array [ + 97, + ], + }, + }, + }, + "filter": Object { + "match_all": Object {}, + }, + }, + "_metric": Object { + "bucket_script": Object { + "buckets_path": Object { + "A": "_A>metric", + "B": "_B>_count", + "C": "_C>metric[test.field]", + "D": "_D>metric[std_deviation]", + "E": "_E>metric[97]", + }, + "script": Object { + "lang": "painless", + "source": "(params.A + params.B + params.C + params.D + params.E) / params.B", + }, + }, + }, + "slo.denominator": Object { + "bucket_script": Object { + "buckets_path": Object {}, + "script": "1", + }, + }, + "slo.isGoodSlice": Object { + "bucket_script": Object { + "buckets_path": Object { + "goodEvents": "slo.numerator>value", + }, + "script": "params.goodEvents == 1 ? 1 : 0", + }, + }, + "slo.numerator": Object { + "bucket_script": Object { + "buckets_path": Object { + "value": "_metric>value", + }, + "script": Object { + "params": Object { + "threshold": 100, + }, + "source": "params.value >= params.threshold ? 1 : 0", + }, + }, + }, + }, + "group_by": Object { + "@timestamp": Object { + "date_histogram": Object { + "field": "@timestamp", + "fixed_interval": "2m", + }, + }, + "slo.budgetingMethod": Object { + "terms": Object { + "field": "slo.budgetingMethod", + }, + }, + "slo.description": Object { + "terms": Object { + "field": "slo.description", + }, + }, + "slo.groupBy": Object { + "terms": Object { + "field": "slo.groupBy", + }, + }, + "slo.id": Object { + "terms": Object { + "field": "slo.id", + }, + }, + "slo.indicator.type": Object { + "terms": Object { + "field": "slo.indicator.type", + }, + }, + "slo.instanceId": Object { + "terms": Object { + "field": "slo.instanceId", + }, + }, + "slo.name": Object { + "terms": Object { + "field": "slo.name", + }, + }, + "slo.objective.sliceDurationInSeconds": Object { + "terms": Object { + "field": "slo.objective.sliceDurationInSeconds", + }, + }, + "slo.objective.target": Object { + "terms": Object { + "field": "slo.objective.target", + }, + }, + "slo.revision": Object { + "terms": Object { + "field": "slo.revision", + }, + }, + "slo.tags": Object { + "terms": Object { + "field": "slo.tags", + }, + }, + "slo.timeWindow.duration": Object { + "terms": Object { + "field": "slo.timeWindow.duration", + }, + }, + "slo.timeWindow.type": Object { + "terms": Object { + "field": "slo.timeWindow.type", + }, + }, + }, + }, + "settings": Object { + "deduce_mappings": false, + "unattended": true, + }, + "source": Object { + "index": "test-*", + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "range": Object { + "@timestamp": Object { + "gte": "now-7d/d", + }, + }, + }, + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match_phrase": Object { + "test.category": "test", + }, + }, + ], + }, + }, + ], + }, + }, + "runtime_mappings": Object { + "slo.budgetingMethod": Object { + "script": Object { + "source": "emit('timeslices')", + }, + "type": "keyword", + }, + "slo.description": Object { + "script": Object { + "source": "emit('irrelevant')", + }, + "type": "keyword", + }, + "slo.groupBy": Object { + "script": Object { + "source": "emit('*')", + }, + "type": "keyword", + }, + "slo.id": Object { + "script": Object { + "source": Any, + }, + "type": "keyword", + }, + "slo.indicator.type": Object { + "script": Object { + "source": "emit('sli.metric.timeslice')", + }, + "type": "keyword", + }, + "slo.instanceId": Object { + "script": Object { + "source": "emit('*')", + }, + "type": "keyword", + }, + "slo.name": Object { + "script": Object { + "source": "emit('irrelevant')", + }, + "type": "keyword", + }, + "slo.objective.sliceDurationInSeconds": Object { + "script": Object { + "source": "emit(120)", + }, + "type": "long", + }, + "slo.objective.target": Object { + "script": Object { + "source": "emit(0.98)", + }, + "type": "double", + }, + "slo.revision": Object { + "script": Object { + "source": "emit(1)", + }, + "type": "long", + }, + "slo.tags": Object { + "script": Object { + "source": "emit('critical,k8s')", + }, + "type": "keyword", + }, + "slo.timeWindow.duration": Object { + "script": Object { + "source": "emit('7d')", + }, + "type": "keyword", + }, + "slo.timeWindow.type": Object { + "script": Object { + "source": "emit('rolling')", + }, + "type": "keyword", + }, + }, + }, + "sync": Object { + "time": Object { + "delay": "1m", + "field": "@timestamp", + }, + }, + "transform_id": Any, +} +`; diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/index.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/index.ts index 2b9337546d796..8bfaf865f340c 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/index.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/index.ts @@ -11,4 +11,5 @@ export * from './apm_transaction_duration'; export * from './kql_custom'; export * from './metric_custom'; export * from './histogram'; +export * from './timeslice_metric'; export * from './common'; diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/timeslice_metric.test.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/timeslice_metric.test.ts new file mode 100644 index 0000000000000..aa21f7e0ceb0e --- /dev/null +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/timeslice_metric.test.ts @@ -0,0 +1,178 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + createTimesliceMetricIndicator, + createSLOWithTimeslicesBudgetingMethod, + createSLO, +} from '../fixtures/slo'; +import { TimesliceMetricTransformGenerator } from './timeslice_metric'; + +const generator = new TimesliceMetricTransformGenerator(); +const everythingIndicator = createTimesliceMetricIndicator( + [ + { name: 'A', aggregation: 'avg', field: 'test.field', filter: 'test.category: "test"' }, + { name: 'B', aggregation: 'doc_count', filter: 'test.category: "test"' }, + { name: 'C', aggregation: 'last_value', field: 'test.field' }, + { name: 'D', aggregation: 'std_deviation', field: 'test.field' }, + { name: 'E', aggregation: 'percentile', field: 'test.field', percentile: 97 }, + ], + '(A + B + C + D + E) / B', + 'test.category: "test"' +); + +describe('Timeslice Metric Transform Generator', () => { + describe('validation', () => { + it('throws when the budgeting method is occurrences', () => { + const anSLO = createSLO({ + indicator: createTimesliceMetricIndicator( + [{ name: 'A', aggregation: 'avg', field: 'test.field' }], + '(A / 200) + A' + ), + }); + expect(() => generator.getTransformParams(anSLO)).toThrow( + 'The sli.metric.timeslice indicator MUST have a timeslice budgeting method.' + ); + }); + it('throws when the metric equation is invalid', () => { + const anSLO = createSLOWithTimeslicesBudgetingMethod({ + indicator: createTimesliceMetricIndicator( + [{ name: 'A', aggregation: 'avg', field: 'test.field' }], + '(a / 200) + A' + ), + }); + expect(() => generator.getTransformParams(anSLO)).toThrow(/Invalid equation/); + }); + it('throws when the metric filter is invalid', () => { + const anSLO = createSLOWithTimeslicesBudgetingMethod({ + indicator: createTimesliceMetricIndicator( + [{ name: 'A', aggregation: 'avg', field: 'test.field', filter: 'test:' }], + '(A / 200) + A' + ), + }); + expect(() => generator.getTransformParams(anSLO)).toThrow(/Invalid KQL: test:/); + }); + it('throws when the query_filter is invalid', () => { + const anSLO = createSLOWithTimeslicesBudgetingMethod({ + indicator: createTimesliceMetricIndicator( + [{ name: 'A', aggregation: 'avg', field: 'test.field', filter: 'test.category: "test"' }], + '(A / 200) + A', + 'test:' + ), + }); + expect(() => generator.getTransformParams(anSLO)).toThrow(/Invalid KQL/); + }); + }); + + it('returns the expected transform params with every specified indicator params', async () => { + const anSLO = createSLOWithTimeslicesBudgetingMethod({ + indicator: everythingIndicator, + }); + const transform = generator.getTransformParams(anSLO); + + expect(transform).toMatchSnapshot({ + transform_id: expect.any(String), + source: { runtime_mappings: { 'slo.id': { script: { source: expect.any(String) } } } }, + }); + expect(transform.transform_id).toEqual(`slo-${anSLO.id}-${anSLO.revision}`); + expect(transform.source.runtime_mappings!['slo.id']).toMatchObject({ + script: { source: `emit('${anSLO.id}')` }, + }); + expect(transform.source.runtime_mappings!['slo.revision']).toMatchObject({ + script: { source: `emit(${anSLO.revision})` }, + }); + }); + + it('returns the expected transform params for timeslices slo', async () => { + const anSLO = createSLOWithTimeslicesBudgetingMethod({ + indicator: everythingIndicator, + }); + const transform = generator.getTransformParams(anSLO); + + expect(transform).toMatchSnapshot({ + transform_id: expect.any(String), + source: { runtime_mappings: { 'slo.id': { script: { source: expect.any(String) } } } }, + }); + }); + + it('filters the source using the kql query', async () => { + const anSLO = createSLOWithTimeslicesBudgetingMethod({ + indicator: everythingIndicator, + }); + const transform = generator.getTransformParams(anSLO); + + expect(transform.source.query).toMatchSnapshot(); + }); + + it('uses the provided index', async () => { + const anSLO = createSLOWithTimeslicesBudgetingMethod({ + indicator: { + ...everythingIndicator, + params: { ...everythingIndicator.params, index: 'my-own-index*' }, + }, + }); + const transform = generator.getTransformParams(anSLO); + + expect(transform.source.index).toBe('my-own-index*'); + }); + + it('uses the provided timestampField', async () => { + const anSLO = createSLOWithTimeslicesBudgetingMethod({ + indicator: { + ...everythingIndicator, + params: { ...everythingIndicator.params, timestampField: 'my-date-field' }, + }, + }); + const transform = generator.getTransformParams(anSLO); + + expect(transform.sync?.time?.field).toBe('my-date-field'); + // @ts-ignore + expect(transform.pivot?.group_by['@timestamp'].date_histogram.field).toBe('my-date-field'); + }); + + it('aggregates using the _metric equation', async () => { + const anSLO = createSLOWithTimeslicesBudgetingMethod({ + indicator: everythingIndicator, + }); + const transform = generator.getTransformParams(anSLO); + + expect(transform.pivot!.aggregations!._metric).toEqual({ + bucket_script: { + buckets_path: { + A: '_A>metric', + B: '_B>_count', + C: '_C>metric[test.field]', + D: '_D>metric[std_deviation]', + E: '_E>metric[97]', + }, + script: { + lang: 'painless', + source: '(params.A + params.B + params.C + params.D + params.E) / params.B', + }, + }, + }); + expect(transform.pivot!.aggregations!['slo.numerator']).toEqual({ + bucket_script: { + buckets_path: { + value: '_metric>value', + }, + script: { + params: { + threshold: 100, + }, + source: 'params.value >= params.threshold ? 1 : 0', + }, + }, + }); + expect(transform.pivot!.aggregations!['slo.denominator']).toEqual({ + bucket_script: { + buckets_path: {}, + script: '1', + }, + }); + }); +}); diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/timeslice_metric.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/timeslice_metric.ts new file mode 100644 index 0000000000000..944b7d0622aad --- /dev/null +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/timeslice_metric.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types'; +import { + timesliceMetricComparatorMapping, + TimesliceMetricIndicator, + timesliceMetricIndicatorSchema, + timeslicesBudgetingMethodSchema, +} from '@kbn/slo-schema'; + +import { InvalidTransformError } from '../../../errors'; +import { getSLOTransformTemplate } from '../../../assets/transform_templates/slo_transform_template'; +import { getElastichsearchQueryOrThrow, parseIndex, TransformGenerator } from '.'; +import { + SLO_DESTINATION_INDEX_NAME, + SLO_INGEST_PIPELINE_NAME, + getSLOTransformId, +} from '../../../assets/constants'; +import { SLO } from '../../../domain/models'; +import { GetTimesliceMetricIndicatorAggregation } from '../aggregations'; + +const INVALID_EQUATION_REGEX = /[^A-Z|+|\-|\s|\d+|\.|\(|\)|\/|\*|>|<|=|\?|\:|&|\!|\|]+/g; + +export class TimesliceMetricTransformGenerator extends TransformGenerator { + public getTransformParams(slo: SLO): TransformPutTransformRequest { + if (!timesliceMetricIndicatorSchema.is(slo.indicator)) { + throw new InvalidTransformError(`Cannot handle SLO of indicator type: ${slo.indicator.type}`); + } + + return getSLOTransformTemplate( + this.buildTransformId(slo), + this.buildDescription(slo), + this.buildSource(slo, slo.indicator), + this.buildDestination(), + this.buildCommonGroupBy(slo, slo.indicator.params.timestampField), + this.buildAggregations(slo, slo.indicator), + this.buildSettings(slo, slo.indicator.params.timestampField) + ); + } + + private buildTransformId(slo: SLO): string { + return getSLOTransformId(slo.id, slo.revision); + } + + private buildSource(slo: SLO, indicator: TimesliceMetricIndicator) { + return { + index: parseIndex(indicator.params.index), + runtime_mappings: this.buildCommonRuntimeMappings(slo), + query: { + bool: { + filter: [ + { + range: { + [indicator.params.timestampField]: { + gte: `now-${slo.timeWindow.duration.format()}/d`, + }, + }, + }, + getElastichsearchQueryOrThrow(indicator.params.filter), + ], + }, + }, + }; + } + + private buildDestination() { + return { + pipeline: SLO_INGEST_PIPELINE_NAME, + index: SLO_DESTINATION_INDEX_NAME, + }; + } + + private buildAggregations(slo: SLO, indicator: TimesliceMetricIndicator) { + if (indicator.params.metric.equation.match(INVALID_EQUATION_REGEX)) { + throw new Error(`Invalid equation: ${indicator.params.metric.equation}`); + } + + if (!timeslicesBudgetingMethodSchema.is(slo.budgetingMethod)) { + throw new Error('The sli.metric.timeslice indicator MUST have a timeslice budgeting method.'); + } + + const getIndicatorAggregation = new GetTimesliceMetricIndicatorAggregation(indicator); + const comparator = timesliceMetricComparatorMapping[indicator.params.metric.comparator]; + return { + ...getIndicatorAggregation.execute('_metric'), + 'slo.numerator': { + bucket_script: { + buckets_path: { value: '_metric>value' }, + script: { + source: `params.value ${comparator} params.threshold ? 1 : 0`, + params: { threshold: indicator.params.metric.threshold }, + }, + }, + }, + 'slo.denominator': { + bucket_script: { + buckets_path: {}, + script: '1', + }, + }, + 'slo.isGoodSlice': { + bucket_script: { + buckets_path: { + goodEvents: 'slo.numerator>value', + }, + script: `params.goodEvents == 1 ? 1 : 0`, + }, + }, + }; + } +} diff --git a/x-pack/plugins/remote_clusters/public/application/sections/remote_cluster_edit/remote_cluster_edit.js b/x-pack/plugins/remote_clusters/public/application/sections/remote_cluster_edit/remote_cluster_edit.js index 02ae974380683..04daad5dee8fc 100644 --- a/x-pack/plugins/remote_clusters/public/application/sections/remote_cluster_edit/remote_cluster_edit.js +++ b/x-pack/plugins/remote_clusters/public/application/sections/remote_cluster_edit/remote_cluster_edit.js @@ -96,55 +96,51 @@ export class RemoteClusterEdit extends Component { if (isLoading) { return ( - - - - - + + + ); } if (!cluster) { return ( - - - - - } - body={ -

- -

- } - actions={ - - - - } - /> - + + + + } + body={ +

+ +

+ } + actions={ + + + + } + /> ); } @@ -152,36 +148,34 @@ export class RemoteClusterEdit extends Component { if (isConfiguredByNode) { return ( - - - - - } - body={ -

- + + + } + body={ +

+ -

- } - actions={ - - - - } - /> - + /> +

+ } + actions={ + + + + } + /> ); } diff --git a/x-pack/plugins/remote_clusters/public/application/sections/remote_cluster_list/remote_cluster_list.js b/x-pack/plugins/remote_clusters/public/application/sections/remote_cluster_list/remote_cluster_list.js index 06948e25a0583..21ed52d08fa83 100644 --- a/x-pack/plugins/remote_clusters/public/application/sections/remote_cluster_list/remote_cluster_list.js +++ b/x-pack/plugins/remote_clusters/public/application/sections/remote_cluster_list/remote_cluster_list.js @@ -93,28 +93,26 @@ export class RemoteClusterList extends Component { renderNoPermission() { return ( - - - - - } - body={ -

- -

- } - /> - + + + + } + body={ +

+ +

+ } + /> ); } @@ -124,103 +122,91 @@ export class RemoteClusterList extends Component { const { statusCode, error: errorString } = error.body; return ( - - - - - } - body={ -

- {statusCode} {errorString} -

- } - /> - + + + + } + body={ +

+ {statusCode} {errorString} +

+ } + /> ); } renderEmpty() { return ( - - - - - } - body={ -

- + + + } + body={ +

+ -

- } - actions={ - - - - } - footer={ - <> - - - - - {' '} - + /> +

+ } + actions={ + + + + } + footer={ + <> + + - - - } - /> - + + {' '} + + + + + } + /> ); } renderLoading() { return ( - - - - - + + + ); } diff --git a/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/installation_and_upgrade.md b/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/installation_and_upgrade.md index b8ae0c85c82e6..cca0f27be0708 100644 --- a/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/installation_and_upgrade.md +++ b/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/installation_and_upgrade.md @@ -45,6 +45,83 @@ Status: `in progress`. The current test plan matches `Milestone 2` of the [Rule - Kibana should not crash with Out Of Memory exception during package installation. - For test purposes, it should be possible to use detection rules package versions lower than the latest. +### Functional requirements + +- User should be able to install prebuilt rules with and without previewing what exactly they would install (rule properties). +- User should be able to upgrade prebuilt rules with and without previewing what updates they would apply (rule properties of target rule versions). +- If user chooses to preview a prebuilt rule to be installed/upgraded, we currently show this preview in a flyout. +- In the prebuilt rule preview a tab that doesn't have any sections should not be displayed and a section that doesn't have any properties also should not be displayed. + +Examples of rule properties we show in the prebuilt rule preview flyout: + +```Gherkin +Examples: +| rule_type | property | tab | section | +β”‚ All rule types β”‚ Author β”‚ Overview β”‚ About β”‚ +β”‚ All rule types β”‚ Building block β”‚ Overview β”‚ About β”‚ +β”‚ All rule types β”‚ Severity β”‚ Overview β”‚ About β”‚ +β”‚ All rule types β”‚ Severity override β”‚ Overview β”‚ About β”‚ +β”‚ All rule types β”‚ Risk score β”‚ Overview β”‚ About β”‚ +β”‚ All rule types β”‚ Risk score override β”‚ Overview β”‚ About β”‚ +β”‚ All rule types β”‚ Reference URLs β”‚ Overview β”‚ About β”‚ +β”‚ All rule types β”‚ False positive examples β”‚ Overview β”‚ About β”‚ +β”‚ All rule types β”‚ Custom highlighted fields β”‚ Overview β”‚ About β”‚ +β”‚ All rule types β”‚ License β”‚ Overview β”‚ About β”‚ +β”‚ All rule types β”‚ Rule name override β”‚ Overview β”‚ About β”‚ +β”‚ All rule types β”‚ MITRE ATT&CKβ„’ β”‚ Overview β”‚ About β”‚ +β”‚ All rule types β”‚ Timestamp override β”‚ Overview β”‚ About β”‚ +β”‚ All rule types β”‚ Tags β”‚ Overview β”‚ About β”‚ +β”‚ All rule types β”‚ Type β”‚ Overview β”‚ Definition β”‚ +β”‚ All rule types β”‚ Related integrations β”‚ Overview β”‚ Definition β”‚ +β”‚ All rule types β”‚ Required fields β”‚ Overview β”‚ Definition β”‚ +β”‚ All rule types β”‚ Timeline template β”‚ Overview β”‚ Definition β”‚ +β”‚ All rule types β”‚ Runs every β”‚ Overview β”‚ Schedule β”‚ +β”‚ All rule types β”‚ Additional look-back time β”‚ Overview β”‚ Schedule β”‚ +β”‚ All rule types β”‚ Setup guide β”‚ Overview β”‚ Setup guide β”‚ +β”‚ All rule types β”‚ Investigation guide β”‚ Investigation guide β”‚ Investigation guide β”‚ +β”‚ Custom Query β”‚ Index patterns β”‚ Overview β”‚ Definition β”‚ +β”‚ Custom Query β”‚ Data view ID β”‚ Overview β”‚ Definition β”‚ +β”‚ Custom Query β”‚ Data view index pattern β”‚ Overview β”‚ Definition β”‚ +β”‚ Custom Query β”‚ Custom query β”‚ Overview β”‚ Definition β”‚ +β”‚ Custom Query β”‚ Filters β”‚ Overview β”‚ Definition β”‚ +β”‚ Custom Query β”‚ Saved query name β”‚ Overview β”‚ Definition β”‚ +β”‚ Custom Query β”‚ Saved query filters β”‚ Overview β”‚ Definition β”‚ +β”‚ Custom Query β”‚ Saved query β”‚ Overview β”‚ Definition β”‚ +β”‚ Custom Query β”‚ Suppress alerts by β”‚ Overview β”‚ Definition β”‚ +β”‚ Custom Query β”‚ Suppress alerts for β”‚ Overview β”‚ Definition β”‚ +β”‚ Custom Query β”‚ If a suppression field is missing β”‚ Overview β”‚ Definition β”‚ +β”‚ Machine Learning β”‚ Anomaly score threshold β”‚ Overview β”‚ Definition β”‚ +β”‚ Machine Learning β”‚ Machine Learning job β”‚ Overview β”‚ Definition β”‚ +β”‚ Threshold β”‚ Threshold β”‚ Overview β”‚ Definition β”‚ +β”‚ Threshold β”‚ Index patterns β”‚ Overview β”‚ Definition β”‚ +β”‚ Threshold β”‚ Data view ID β”‚ Overview β”‚ Definition β”‚ +β”‚ Threshold β”‚ Data view index pattern β”‚ Overview β”‚ Definition β”‚ +β”‚ Threshold β”‚ Custom query β”‚ Overview β”‚ Definition β”‚ +β”‚ Threshold β”‚ Filters β”‚ Overview β”‚ Definition β”‚ +β”‚ Event Correlation β”‚ EQL query β”‚ Overview β”‚ Definition β”‚ +β”‚ Event Correlation β”‚ Filters β”‚ Overview β”‚ Definition β”‚ +β”‚ Event Correlation β”‚ Index patterns β”‚ Overview β”‚ Definition β”‚ +β”‚ Event Correlation β”‚ Data view ID β”‚ Overview β”‚ Definition β”‚ +β”‚ Event Correlation β”‚ Data view index pattern β”‚ Overview β”‚ Definition β”‚ +β”‚ Indicator Match β”‚ Indicator index patterns β”‚ Overview β”‚ Definition β”‚ +β”‚ Indicator Match β”‚ Indicator mapping β”‚ Overview β”‚ Definition β”‚ +β”‚ Indicator Match β”‚ Indicator filters β”‚ Overview β”‚ Definition β”‚ +β”‚ Indicator Match β”‚ Indicator index query β”‚ Overview β”‚ Definition β”‚ +β”‚ Indicator Match β”‚ Index patterns β”‚ Overview β”‚ Definition β”‚ +β”‚ Indicator Match β”‚ Data view ID β”‚ Overview β”‚ Definition β”‚ +β”‚ Indicator Match β”‚ Data view index pattern β”‚ Overview β”‚ Definition β”‚ +β”‚ Indicator Match β”‚ Custom query β”‚ Overview β”‚ Definition β”‚ +β”‚ Indicator Match β”‚ Filters β”‚ Overview β”‚ Definition β”‚ +β”‚ New Terms β”‚ Fields β”‚ Overview β”‚ Definition β”‚ +β”‚ New Terms β”‚ History Window Size β”‚ Overview β”‚ Definition β”‚ +β”‚ New Terms β”‚ Index patterns β”‚ Overview β”‚ Definition β”‚ +β”‚ New Terms β”‚ Data view ID β”‚ Overview β”‚ Definition β”‚ +β”‚ New Terms β”‚ Data view index pattern β”‚ Overview β”‚ Definition β”‚ +β”‚ New Terms β”‚ Custom query β”‚ Overview β”‚ Definition β”‚ +β”‚ New Terms β”‚ Filters β”‚ Overview β”‚ Definition β”‚ +β”‚ ESQL β”‚ ESQL query β”‚ Overview β”‚ Definition β”‚ +``` + ## Scenarios ### Package installation @@ -366,7 +443,7 @@ Given no prebuilt rules are installed in Kibana And there are X prebuilt rules available to install When user opens the Add Rules page Then prebuilt rules available for installation should be displayed in the table -When user installs one individual rule +When user installs one individual rule without previewing it Then success message should be displayed after installation And the installed rule should be removed from the table When user navigates back to the Rule Management page @@ -429,6 +506,64 @@ Then user should see a message indicating that all available rules have been ins And user should see a CTA that leads to the Rule Management page ``` +#### **Scenario: User can preview a rule before installing** + +**Automation**: 1 e2e test + +```Gherkin +Given no prebuilt rules are installed in Kibana +And there are 2 rules available to install +When user opens the Add Rules page +Then all rules available for installation should be displayed in the table +When user opens the rule preview for the 1st rule +Then the preview should open +When user closes the preview +Then it should disappear +When user opens the rule preview for the 2nd rule +Then the preview should open +When user installs the rule using a CTA in the rule preview +Then the 2nd rule should be installed +And a success message should be displayed after installation +And the 2nd rule should be removed from the Add Rules table +When user navigates back to the Rule Management page +Then user should see a CTA to install prebuilt rules +And user should see the number of rules available to install as 1 +``` + +#### **Scenario: User can see correct rule information in preview before installing** + +**Automation**: 1 e2e test + +```Gherkin +Given no prebuilt rules are installed in Kibana +And there are X prebuilt rules of all types available to install +When user opens the Add Rules page +Then all X rules available for installation should be displayed in the table +When user opens the rule preview for the 1st rule +Then the preview should open +And all properties of the 1st rule should be displayed in the correct tab and section of the preview (see examples of rule properties above) +When user selects the 2nd rule in the table +Then the preview should be updated +And all properties of the 2nd rule should be displayed in the correct tab and section of the preview (see examples of rule properties above) +And user should be able to repeat this for all X rules +``` + +#### **Scenario: Tabs and sections without content should be hidden in preview before installing** + +**Automation**: 1 e2e test + +```Gherkin +Given no prebuilt rules are installed in Kibana +And there is at least 1 rule available to install +And this rule has neither Setup guide nor Investigation guide +When user opens the Add Rules page +Then all rules available for installation should be displayed in the table +When user opens the rule preview for this rule +Then the preview should open +And the Setup Guide section should NOT be displayed in the Overview tab +And the Investigation Guide tab should NOT be displayed +``` + ### Rule installation workflow: filtering, sorting, pagination TODO: add scenarios https://github.com/elastic/kibana/issues/166215 @@ -467,7 +602,7 @@ And for Y of the installed rules there are new versions available And user is on the Rule Management page When user opens the Rule Updates table Then Y rules available for upgrade should be displayed in the table -When user upgrades one individual rule +When user upgrades one individual rule without previewing it Then success message should be displayed after upgrade And the upgraded rule should be removed from the table And user should see the number of rules available to upgrade decreased by 1 @@ -513,6 +648,65 @@ And user should NOT see a number of rules available to upgrade And user should NOT see the Rule Updates table ``` +#### **Scenario: User can preview a rule before upgrading** + +**Automation**: 1 e2e test + +```Gherkin +Given 2 prebuilt rules are installed in Kibana +And for these 2 installed rules there are new versions available +And user is on the Rule Management page +When user opens the Rule Updates table +Then all rules available for upgrade should be displayed in the table +When user opens the rule preview for the 1st rule +Then the preview should open +When user closes the preview +Then it should disappear +When user opens the rule preview for the 2nd rule +Then the preview should open +When user upgrades the rule using a CTA in the rule preview +Then the 2nd rule should be upgraded to the latest version +And a success message should be displayed after upgrade +And the 2nd rule should be removed from the Rule Updates table +And user should see the number of rules available to upgrade as 1 +``` + +#### **Scenario: User can see correct rule information in preview before upgrading** + +**Automation**: 1 e2e test + +```Gherkin +Given X prebuilt rules of all types are installed in Kibana +And for all of the installed rules there are new versions available +And user is on the Rule Management page +When user opens the Rule Updates table +Then all X rules available for upgrade should be displayed in the table +When user opens the rule preview for the 1st rule +Then the preview should open +And all properties of the new version of the 1st rule should be displayed in the correct tab and section of the preview (see examples of rule properties above) +When user selects the 2nd rule in the table +Then the preview should be updated +And all properties of the new version of the 2nd rule should be displayed in the correct tab and section of the preview (see examples of rule properties above) +And user should be able to repeat this for all X rules +``` + +#### **Scenario: Tabs and sections without content should be hidden in preview before upgrading** + +**Automation**: 1 e2e test + +```Gherkin +Given at least 1 prebuilt rule is installed in Kibana +And for this rule there is a new version available +And the updated version of a rule has neither Setup guide nor Investigation guide +And user is on the Rule Management page +When user opens the Rule Updates table +Then all rules available for upgrade should be displayed in the table +When user opens the rule preview for a rule without Setup guide and Investigation guide +Then the preview should open +And the Setup Guide section should NOT be displayed in the Overview tab +And the Investigation Guide tab should NOT be displayed +``` + ### Rule upgrade workflow: filtering, sorting, pagination TODO: add scenarios https://github.com/elastic/kibana/issues/166215 diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx index d91caae855d22..b87de5bc64874 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx @@ -11,7 +11,7 @@ import { EuiAvatar, EuiMarkdownFormat, EuiText, tint } from '@elastic/eui'; import React from 'react'; import { AssistantAvatar } from '@kbn/elastic-assistant'; -import { css } from '@emotion/react/dist/emotion-react.cjs'; +import { css } from '@emotion/react'; import { euiThemeVars } from '@kbn/ui-theme'; import { CommentActions } from '../comment_actions'; import * as i18n from './translations'; diff --git a/x-pack/plugins/security_solution/public/explore/containers/risk_score/kpi/index.tsx b/x-pack/plugins/security_solution/public/explore/containers/risk_score/kpi/index.tsx index 36e0a5a44190f..2d8712556ae24 100644 --- a/x-pack/plugins/security_solution/public/explore/containers/risk_score/kpi/index.tsx +++ b/x-pack/plugins/security_solution/public/explore/containers/risk_score/kpi/index.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import { useEffect, useMemo } from 'react'; +import { useCallback, useEffect, useMemo } from 'react'; import { getHostRiskIndex, @@ -20,12 +20,12 @@ import { isIndexNotFoundError } from '../../../../common/utils/exceptions'; import type { ESQuery } from '../../../../../common/typed_json'; import type { SeverityCount } from '../../../components/risk_score/severity/types'; import { useSpaceId } from '../../../../common/hooks/use_space_id'; -import { useMlCapabilities } from '../../../../common/components/ml/hooks/use_ml_capabilities'; import { useSearchStrategy } from '../../../../common/containers/use_search_strategy'; import type { InspectResponse } from '../../../../types'; import type { inputsModel } from '../../../../common/store'; import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; import { useIsNewRiskScoreModuleInstalled } from '../../../../entity_analytics/api/hooks/use_risk_engine_status'; +import { useRiskScoreFeatureStatus } from '../feature_status'; interface RiskScoreKpi { error: unknown; @@ -52,7 +52,6 @@ export const useRiskScoreKpi = ({ }: UseRiskScoreKpiProps): RiskScoreKpi => { const { addError } = useAppToasts(); const spaceId = useSpaceId(); - const featureEnabled = useMlCapabilities().isPlatinumOrTrialLicense; const isNewRiskScoreModuleInstalled = useIsNewRiskScoreModuleInstalled(); const defaultIndex = spaceId ? riskEntity === RiskScoreEntity.host @@ -60,6 +59,14 @@ export const useRiskScoreKpi = ({ : getUserRiskIndex(spaceId, true, isNewRiskScoreModuleInstalled) : undefined; + const { + isDeprecated, + isEnabled, + isAuthorized, + isLoading: isDeprecatedLoading, + refetch: refetchDeprecated, + } = useRiskScoreFeatureStatus(riskEntity, defaultIndex); + const { loading, result, search, refetch, inspect, error } = useSearchStrategy({ factoryQueryType: RiskQueries.kpiRiskScore, @@ -73,18 +80,42 @@ export const useRiskScoreKpi = ({ const isModuleDisabled = !!error && isIndexNotFoundError(error); useEffect(() => { - if (!skip && defaultIndex && featureEnabled) { + if ( + !skip && + defaultIndex && + !isDeprecatedLoading && + isAuthorized && + isEnabled && + !isDeprecated + ) { search({ filterQuery, defaultIndex: [defaultIndex], entity: riskEntity, }); } - }, [defaultIndex, search, filterQuery, skip, riskEntity, featureEnabled]); + }, [ + isEnabled, + isDeprecated, + isAuthorized, + isDeprecatedLoading, + skip, + defaultIndex, + search, + filterQuery, + riskEntity, + ]); + + const refetchAll = useCallback(() => { + if (defaultIndex) { + refetchDeprecated(defaultIndex); + refetch(); + } + }, [defaultIndex, refetch, refetchDeprecated]); // since query does not take timerange arg, we need to manually refetch when time range updates useEffect(() => { - refetch(); + refetchAll(); // eslint-disable-next-line react-hooks/exhaustive-deps }, [timerange?.to, timerange?.from]); diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/session_preview_container.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/session_preview_container.tsx index 101c067ad661d..10250e74c383c 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/session_preview_container.tsx +++ b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/session_preview_container.tsx @@ -10,7 +10,7 @@ import { TimelineTabs } from '@kbn/securitysolution-data-table'; import { useDispatch } from 'react-redux'; import { EuiLink, useEuiTheme } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; -import { css } from '@emotion/css/dist/emotion-css.cjs'; +import { css } from '@emotion/css'; import { useLicense } from '../../../../common/hooks/use_license'; import { SessionPreview } from './session_preview'; import { useSessionPreview } from '../hooks/use_session_preview'; diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/integration_tests/policy_list.test.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/integration_tests/policy_list.test.tsx index ee520a98632bc..abb68f1d79814 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/integration_tests/policy_list.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/integration_tests/policy_list.test.tsx @@ -25,7 +25,8 @@ jest.mock('../../../../../common/components/user_privileges'); const getPackagePolicies = sendGetEndpointSpecificPackagePolicies as jest.Mock; const useUserPrivilegesMock = useUserPrivileges as jest.Mock; -describe('When on the policy list page', () => { +// Failing: See https://github.com/elastic/kibana/issues/169133 +describe.skip('When on the policy list page', () => { let render: () => ReturnType; let renderResult: ReturnType; let history: AppContextTestRender['history']; diff --git a/x-pack/plugins/stack_connectors/common/bedrock/constants.ts b/x-pack/plugins/stack_connectors/common/bedrock/constants.ts index 361309a84d0a9..8071091194049 100644 --- a/x-pack/plugins/stack_connectors/common/bedrock/constants.ts +++ b/x-pack/plugins/stack_connectors/common/bedrock/constants.ts @@ -10,7 +10,7 @@ import { i18n } from '@kbn/i18n'; export const BEDROCK_TITLE = i18n.translate( 'xpack.stackConnectors.components.bedrock.connectorTypeTitle', { - defaultMessage: 'AWS Bedrock', + defaultMessage: 'Amazon Bedrock', } ); export const BEDROCK_CONNECTOR_ID = '.bedrock'; diff --git a/x-pack/plugins/stack_connectors/public/connector_types/bedrock/bedrock.test.tsx b/x-pack/plugins/stack_connectors/public/connector_types/bedrock/bedrock.test.tsx index 502ec1755bda7..a37762487a822 100644 --- a/x-pack/plugins/stack_connectors/public/connector_types/bedrock/bedrock.test.tsx +++ b/x-pack/plugins/stack_connectors/public/connector_types/bedrock/bedrock.test.tsx @@ -26,8 +26,8 @@ beforeAll(() => { describe('actionTypeRegistry.get() works', () => { test('connector type static data is as expected', () => { expect(actionTypeModel.id).toEqual(ACTION_TYPE_ID); - expect(actionTypeModel.selectMessage).toBe('Send a request to AWS Bedrock.'); - expect(actionTypeModel.actionTypeTitle).toBe('AWS Bedrock'); + expect(actionTypeModel.selectMessage).toBe('Send a request to Amazon Bedrock.'); + expect(actionTypeModel.actionTypeTitle).toBe('Amazon Bedrock'); }); }); diff --git a/x-pack/plugins/stack_connectors/public/connector_types/bedrock/bedrock.tsx b/x-pack/plugins/stack_connectors/public/connector_types/bedrock/bedrock.tsx index e6d674354511a..361caed6882c2 100644 --- a/x-pack/plugins/stack_connectors/public/connector_types/bedrock/bedrock.tsx +++ b/x-pack/plugins/stack_connectors/public/connector_types/bedrock/bedrock.tsx @@ -21,7 +21,7 @@ export function getConnectorType(): BedrockConnector { id: BEDROCK_CONNECTOR_ID, iconClass: lazy(() => import('./logo')), selectMessage: i18n.translate('xpack.stackConnectors.components.bedrock.selectMessageText', { - defaultMessage: 'Send a request to AWS Bedrock.', + defaultMessage: 'Send a request to Amazon Bedrock.', }), actionTypeTitle: BEDROCK_TITLE, validateParams: async ( diff --git a/x-pack/plugins/stack_connectors/public/connector_types/bedrock/constants.tsx b/x-pack/plugins/stack_connectors/public/connector_types/bedrock/constants.tsx index 7ee3e35cecf15..88fc42e004bf8 100644 --- a/x-pack/plugins/stack_connectors/public/connector_types/bedrock/constants.tsx +++ b/x-pack/plugins/stack_connectors/public/connector_types/bedrock/constants.tsx @@ -33,7 +33,7 @@ export const bedrockConfig: ConfigFieldSchema[] = [ defaultValue: DEFAULT_BEDROCK_URL, helpText: ( { configurationUtilities = actionsConfigMock.create(); connectorType = getConnectorType(); }); - test('exposes the connector as `AWS Bedrock` with id `.bedrock`', () => { + test('exposes the connector as `Amazon Bedrock` with id `.bedrock`', () => { expect(connectorType.id).toEqual('.bedrock'); - expect(connectorType.name).toEqual('AWS Bedrock'); + expect(connectorType.name).toEqual('Amazon Bedrock'); }); describe('config validation', () => { test('config validation passes when only required fields are provided', () => { @@ -55,7 +55,7 @@ describe('Bedrock Connector', () => { expect(() => { configValidator(config, { configurationUtilities }); }).toThrowErrorMatchingInlineSnapshot( - '"Error configuring AWS Bedrock action: Error: URL Error: Invalid URL: example.com/do-something"' + '"Error configuring Amazon Bedrock action: Error: URL Error: Invalid URL: example.com/do-something"' ); }); @@ -75,7 +75,7 @@ describe('Bedrock Connector', () => { expect(() => { configValidator(config, { configurationUtilities: configUtils }); }).toThrowErrorMatchingInlineSnapshot( - `"Error configuring AWS Bedrock action: Error: error validating url: target url is not present in allowedHosts"` + `"Error configuring Amazon Bedrock action: Error: error validating url: target url is not present in allowedHosts"` ); }); }); diff --git a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts index 11050628203a9..02b2bff9a93ae 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts @@ -43,7 +43,7 @@ export const configValidator = (configObject: Config, validatorServices: Validat } catch (err) { throw new Error( i18n.translate('xpack.stackConnectors.bedrock.configurationErrorApiProvider', { - defaultMessage: 'Error configuring AWS Bedrock action: {err}', + defaultMessage: 'Error configuring Amazon Bedrock action: {err}', values: { err, }, diff --git a/x-pack/plugins/stack_connectors/server/plugin.test.ts b/x-pack/plugins/stack_connectors/server/plugin.test.ts index c728ee92cea40..c5c16a29647fd 100644 --- a/x-pack/plugins/stack_connectors/server/plugin.test.ts +++ b/x-pack/plugins/stack_connectors/server/plugin.test.ts @@ -164,7 +164,7 @@ describe('Stack Connectors Plugin', () => { 4, expect.objectContaining({ id: '.bedrock', - name: 'AWS Bedrock', + name: 'Amazon Bedrock', }) ); expect(actionsSetup.registerSubActionConnectorType).toHaveBeenNthCalledWith( diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts index 8d4412cf37a19..4983d19d36b69 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts @@ -160,7 +160,7 @@ export default function bedrockTest({ getService }: FtrProviderContext) { statusCode: 400, error: 'Bad Request', message: - 'error validating action type config: Error configuring AWS Bedrock action: Error: error validating url: target url "http://bedrock.mynonexistent.com" is not added to the Kibana config xpack.actions.allowedHosts', + 'error validating action type config: Error configuring Amazon Bedrock action: Error: error validating url: target url "http://bedrock.mynonexistent.com" is not added to the Kibana config xpack.actions.allowedHosts', }); }); }); @@ -280,7 +280,7 @@ export default function bedrockTest({ getService }: FtrProviderContext) { status: 'error', retry: true, message: 'an error occurred while running the action', - service_message: `Sub action "invalidAction" is not registered. Connector id: ${bedrockActionId}. Connector name: AWS Bedrock. Connector type: .bedrock`, + service_message: `Sub action "invalidAction" is not registered. Connector id: ${bedrockActionId}. Connector name: Amazon Bedrock. Connector type: .bedrock`, }); }); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/entity_analytics.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/entity_analytics.cy.ts index 3ec2943223a81..3cc088f53d301 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/entity_analytics.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/entity_analytics.cy.ts @@ -69,7 +69,7 @@ describe('Entity Analytics Dashboard', { tags: ['@ess', '@serverless'] }, () => deleteRiskEngineConfiguration(); }); - describe('legcay risk score', () => { + describe('legacy risk score', () => { describe('Without data', () => { beforeEach(() => { login(); @@ -135,8 +135,7 @@ describe('Entity Analytics Dashboard', { tags: ['@ess', '@serverless'] }, () => }); }); - // FLAKY: https://github.com/elastic/kibana/issues/168490 - describe.skip('With host risk data', () => { + describe('With host risk data', () => { before(() => { cy.task('esArchiverLoad', { archiveName: 'risk_hosts' }); }); @@ -163,7 +162,8 @@ describe('Entity Analytics Dashboard', { tags: ['@ess', '@serverless'] }, () => cy.get(HOSTS_TABLE_ALERT_CELL).should('have.length', 5); }); - it('filters by risk level', () => { + // FLAKY: https://github.com/elastic/kibana/issues/168490 + it.skip('filters by risk level', () => { openRiskTableFilterAndSelectTheLowOption(); cy.get(HOSTS_DONUT_CHART).should('include.text', '1Total');