From bab64949bb78946e59fdefe377b880353ff47f79 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Fri, 4 Mar 2022 13:26:15 -0500 Subject: [PATCH 01/39] wip --- .../authorization/alerting_authorization.ts | 1 + .../server/rules_client/rules_client.ts | 86 ++++ .../server/es/cluster_client_adapter.ts | 367 ++++++++++-------- .../event_log/server/event_log_client.ts | 33 ++ x-pack/plugins/event_log/server/types.ts | 11 +- .../feature_privilege_builder/alerting.ts | 2 +- 6 files changed, 340 insertions(+), 160 deletions(-) diff --git a/x-pack/plugins/alerting/server/authorization/alerting_authorization.ts b/x-pack/plugins/alerting/server/authorization/alerting_authorization.ts index aafeb5003eaab..a079594997521 100644 --- a/x-pack/plugins/alerting/server/authorization/alerting_authorization.ts +++ b/x-pack/plugins/alerting/server/authorization/alerting_authorization.ts @@ -30,6 +30,7 @@ export enum ReadOperations { Get = 'get', GetRuleState = 'getRuleState', GetAlertSummary = 'getAlertSummary', + GetExecutionLog = 'getExecutionLog', Find = 'find', } diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index 6d3ffc822a626..4b3ae09c1457f 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -222,6 +222,22 @@ export interface GetAlertSummaryParams { numberOfExecutions?: number; } +export interface GetExecutionLogByIdParams { + id: string; + dateStart?: string; + // dateEnd? +} + +export interface ExecutionLog { + count: number; + values: Array<{ + timestamp: number; + duration: number; + status: 'failed' | 'succeeded'; + message: string; + }>; +} + // NOTE: Changing this prefix will require a migration to update the prefix in all existing `rule` saved objects const extractedSavedObjectParamReferenceNamePrefix = 'param:'; @@ -615,6 +631,76 @@ export class RulesClient { }); } + public async getExecutionLogForRule({ + id, + dateStart, + }: // dateEnd? + GetExecutionLogByIdParams): Promise { + this.logger.debug(`getExecutionLogForRule(): getting execution log for rule ${id}`); + const rule = (await this.get({ id, includeLegacyId: true })) as SanitizedRuleWithLegacyId; + + // Make sure user has access to this rule + await this.authorization.ensureAuthorized({ + ruleTypeId: rule.alertTypeId, + consumer: rule.consumer, + operation: ReadOperations.GetAlertSummary, + entity: AlertingAuthorizationEntity.Rule, + }); + + // default duration of instance summary is 60 * rule interval + const dateNow = new Date(); + const durationMillis = parseDuration(rule.schedule.interval) * (numberOfExecutions ?? 60); + const defaultDateStart = new Date(dateNow.valueOf() - durationMillis); + const parsedDateStart = parseDate(dateStart, 'dateStart', defaultDateStart); + + const eventLogClient = await this.getEventLogClient(); + + let events: IEvent[]; + let executionEvents: IEvent[]; + + try { + const results = await eventLogClient.aggregateEventsBySavedObjectIds( + 'alert', + [id], + {}, + rule.legacyId !== null ? [rule.legacyId] : undefined + ); + } catch (err) { + this.logger.debug( + `rulesClient.getExecutionLogForRule(): error searching event log for rule ${id}: ${err.message}` + ); + } + + // desired output + // { + // count: number; + // values: [ + // { + // // this is shown in the screenshot + // executionTime: timestamp, + // duration: millis, + // status: failed/succeeded, + // message: log message - execution timeout is separate event log entry + + // // other stuff that might be useful that we could probably get + // number of active alerts + // number of new alerts + // number of recovered alerts + // number of triggered actions + // any errors in the actions + + // es search duration + // total search duration + + // } + // ] + // } + return { + count: 0, + values: [], + }; + } + public async find({ options: { fields, ...options } = {}, excludeFromPublicApi = false, diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index bb958c3ce2b54..067e89a45677a 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -14,7 +14,7 @@ import util from 'util'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; import { IEvent, IValidatedEvent, SAVED_OBJECT_REL_PRIMARY } from '../types'; -import { FindOptionsType } from '../event_log_client'; +import { AggregateOptionsType, FindOptionsType } from '../event_log_client'; import { ParsedIndexAlias } from './init'; export const EVENT_BUFFER_TIME = 1000; // milliseconds @@ -47,7 +47,7 @@ interface QueryOptionsEventsBySavedObjectFilter { namespace: string | undefined; type: string; ids: string[]; - findOptions: FindOptionsType; + findOptions: FindOptionsType | AggregateOptionsType; legacyIds?: string[]; } @@ -329,73 +329,187 @@ export class ClusterClientAdapter { - const { index, namespace, type, ids, findOptions, legacyIds } = queryOptions; + const { index, type, ids, findOptions } = queryOptions; // eslint-disable-next-line @typescript-eslint/naming-convention - const { page, per_page: perPage, start, end, sort_field, sort_order, filter } = findOptions; + const { page, per_page: perPage, sort_field, sort_order } = findOptions; - const defaultNamespaceQuery = { - bool: { - must_not: { - exists: { - field: 'kibana.saved_objects.namespace', - }, - }, - }, - }; - const namedNamespaceQuery = { - term: { - 'kibana.saved_objects.namespace': { - value: namespace, - }, - }, + const esClient = await this.elasticsearchClientPromise; + + const query = getQueryBody(this.logger, queryOptions); + + const body: estypes.SearchRequest['body'] = { + size: perPage, + from: (page - 1) * perPage, + sort: [{ [sort_field]: { order: sort_order } }], + query, }; - const namespaceQuery = namespace === undefined ? defaultNamespaceQuery : namedNamespaceQuery; + + try { + const { + hits: { hits, total }, + } = await esClient.search({ + index, + track_total_hits: true, + body, + }); + return { + page, + per_page: perPage, + total: isNumber(total) ? total : total!.value, + data: hits.map((hit) => hit._source), + }; + } catch (err) { + throw new Error( + `querying for Event Log by for type "${type}" and ids "${ids}" failed with: ${err.message}` + ); + } + } + + public async aggregateEventsBySavedObjects( + queryOptions: QueryOptionsEventsBySavedObjectFilter + ): Promise<{ + aggregations: Record | undefined; + }> { + const { index, type, ids, findOptions } = queryOptions; + const { + page, + per_page: perPage, + // eslint-disable-next-line @typescript-eslint/naming-convention + sort_field, + // eslint-disable-next-line @typescript-eslint/naming-convention + sort_order, + aggs, + } = findOptions as AggregateOptionsType; const esClient = await this.elasticsearchClientPromise; - let dslFilterQuery: estypes.QueryDslBoolQuery['filter']; + + const query = getQueryBody(this.logger, queryOptions); + + const body: estypes.SearchRequest['body'] = { + size: 0, + query, + aggs, + }; + try { - dslFilterQuery = filter ? toElasticsearchQuery(fromKueryExpression(filter)) : []; + const { aggregations } = await esClient.search({ + index, + body, + }); + return { + aggregations, + }; } catch (err) { - this.debug(`Invalid kuery syntax for the filter (${filter}) error:`, { + throw new Error( + `querying for Event Log by for type "${type}" and ids "${ids}" failed with: ${err.message}` + ); + } + } +} + +function getNamespaceQuery(namespace?: string) { + const defaultNamespaceQuery = { + bool: { + must_not: { + exists: { + field: 'kibana.saved_objects.namespace', + }, + }, + }, + }; + const namedNamespaceQuery = { + term: { + 'kibana.saved_objects.namespace': { + value: namespace, + }, + }, + }; + return namespace === undefined ? defaultNamespaceQuery : namedNamespaceQuery; +} + +function getQueryBody(logger: Logger, queryOptions: QueryOptionsEventsBySavedObjectFilter) { + const { namespace, type, ids, findOptions, legacyIds } = queryOptions; + const { start, end, filter } = findOptions; + + const namespaceQuery = getNamespaceQuery(namespace); + let dslFilterQuery: estypes.QueryDslBoolQuery['filter']; + try { + dslFilterQuery = filter ? toElasticsearchQuery(fromKueryExpression(filter)) : []; + } catch (err) { + logger.debug( + `esContext: Invalid kuery syntax for the filter (${filter}) error: ${JSON.stringify({ message: err.message, statusCode: err.statusCode, - }); - throw err; - } - const savedObjectsQueryMust: estypes.QueryDslQueryContainer[] = [ - { - term: { - 'kibana.saved_objects.rel': { - value: SAVED_OBJECT_REL_PRIMARY, - }, + })}` + ); + throw err; + } + + const savedObjectsQueryMust: estypes.QueryDslQueryContainer[] = [ + { + term: { + 'kibana.saved_objects.rel': { + value: SAVED_OBJECT_REL_PRIMARY, }, }, - { - term: { - 'kibana.saved_objects.type': { - value: type, + }, + { + term: { + 'kibana.saved_objects.type': { + value: type, + }, + }, + }, + // @ts-expect-error undefined is not assignable as QueryDslTermQuery value + namespaceQuery, + ]; + + const musts: estypes.QueryDslQueryContainer[] = [ + { + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: reject(savedObjectsQueryMust, isUndefined), }, }, }, - // @ts-expect-error undefined is not assignable as QueryDslTermQuery value - namespaceQuery, - ]; - - const musts: estypes.QueryDslQueryContainer[] = [ - { - nested: { - path: 'kibana.saved_objects', - query: { - bool: { - must: reject(savedObjectsQueryMust, isUndefined), + }, + ]; + + const shouldQuery = []; + shouldQuery.push({ + bool: { + must: [ + { + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: [ + { + terms: { + // default maximum of 65,536 terms, configurable by index.max_terms_count + 'kibana.saved_objects.id': ids, + }, + }, + ], + }, }, }, }, - }, - ]; - - const shouldQuery = []; + { + range: { + 'kibana.version': { + gte: LEGACY_ID_CUTOFF_VERSION, + }, + }, + }, + ], + }, + }); + if (legacyIds && legacyIds.length > 0) { shouldQuery.push({ bool: { must: [ @@ -408,7 +522,7 @@ export class ClusterClientAdapter 0) { - shouldQuery.push({ - bool: { - must: [ - { - nested: { - path: 'kibana.saved_objects', - query: { - bool: { - must: [ - { - terms: { - // default maximum of 65,536 terms, configurable by index.max_terms_count - 'kibana.saved_objects.id': legacyIds, - }, - }, - ], - }, - }, - }, - }, - { - bool: { - should: [ - { - range: { - 'kibana.version': { - lt: LEGACY_ID_CUTOFF_VERSION, - }, + bool: { + should: [ + { + range: { + 'kibana.version': { + lt: LEGACY_ID_CUTOFF_VERSION, }, }, - { - bool: { - must_not: { - exists: { - field: 'kibana.version', - }, + }, + { + bool: { + must_not: { + exists: { + field: 'kibana.version', }, }, }, - ], - }, + }, + ], }, - ], - }, - }); - } - - musts.push({ - bool: { - should: shouldQuery, + }, + ], }, }); + } - if (start) { - musts.push({ - range: { - '@timestamp': { - gte: start, - }, - }, - }); - } - if (end) { - musts.push({ - range: { - '@timestamp': { - lte: end, - }, - }, - }); - } + musts.push({ + bool: { + should: shouldQuery, + }, + }); - const body: estypes.SearchRequest['body'] = { - size: perPage, - from: (page - 1) * perPage, - sort: [{ [sort_field]: { order: sort_order } }], - query: { - bool: { - filter: dslFilterQuery, - must: reject(musts, isUndefined), + if (start) { + musts.push({ + range: { + '@timestamp': { + gte: start, }, }, - }; - - try { - const { - hits: { hits, total }, - } = await esClient.search({ - index, - track_total_hits: true, - body, - }); - return { - page, - per_page: perPage, - total: isNumber(total) ? total : total!.value, - data: hits.map((hit) => hit._source), - }; - } catch (err) { - throw new Error( - `querying for Event Log by for type "${type}" and ids "${ids}" failed with: ${err.message}` - ); - } + }); } - - private debug(message: string, object?: unknown) { - const objectString = object == null ? '' : JSON.stringify(object); - this.logger.debug(`esContext: ${message} ${objectString}`); + if (end) { + musts.push({ + range: { + '@timestamp': { + lte: end, + }, + }, + }); } + + return { + bool: { + filter: dslFilterQuery, + must: reject(musts, isUndefined), + }, + }; } diff --git a/x-pack/plugins/event_log/server/event_log_client.ts b/x-pack/plugins/event_log/server/event_log_client.ts index 39b78296e3875..1ce6f1f6316ef 100644 --- a/x-pack/plugins/event_log/server/event_log_client.ts +++ b/x-pack/plugins/event_log/server/event_log_client.ts @@ -8,6 +8,7 @@ import { Observable } from 'rxjs'; import { schema, TypeOf } from '@kbn/config-schema'; import { IClusterClient, KibanaRequest } from 'src/core/server'; +import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { SpacesServiceStart } from '../../spaces/server'; import { EsContext } from './es'; @@ -59,6 +60,14 @@ export type FindOptionsType = Pick< > & Partial>; +export type AggregateOptionsType = Pick< + TypeOf, + 'start' | 'end' | 'sort_field' | 'sort_order' | 'filter' +> & + Partial> & { + aggs: Record; + }; + interface EventLogServiceCtorParams { esContext: EsContext; savedObjectGetter: SavedObjectBulkGetterResult; @@ -103,4 +112,28 @@ export class EventLogClient implements IEventLogClient { legacyIds, }); } + + async aggregateEventsBySavedObjectIds( + type: string, + ids: string[], + options?: Partial, + legacyIds?: string[] + ) { + // const aggregateOptions = findOptionsSchema.validate(options ?? {}); + + const space = await this.spacesService?.getActiveSpace(this.request); + const namespace = space && this.spacesService?.spaceIdToNamespace(space.id); + + // verify the user has the required permissions to view this saved objects + await this.savedObjectGetter(type, ids); + + return await this.esContext.esAdapter.aggregateEventsBySavedObjects({ + index: this.esContext.esNames.indexPattern, + namespace, + type, + ids, + findOptions: options as AggregateOptionsType, + legacyIds, + }); + } } diff --git a/x-pack/plugins/event_log/server/types.ts b/x-pack/plugins/event_log/server/types.ts index 54305803b090a..2b6b4584af06b 100644 --- a/x-pack/plugins/event_log/server/types.ts +++ b/x-pack/plugins/event_log/server/types.ts @@ -8,10 +8,11 @@ import { schema, TypeOf } from '@kbn/config-schema'; import type { IRouter, KibanaRequest, RequestHandlerContext } from 'src/core/server'; +import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; export type { IEvent, IValidatedEvent } from '../generated/schemas'; export { EventSchema, ECS_VERSION } from '../generated/schemas'; import { IEvent } from '../generated/schemas'; -import { FindOptionsType } from './event_log_client'; +import { AggregateOptionsType, FindOptionsType } from './event_log_client'; import { QueryEventsBySavedObjectResult } from './es/cluster_client_adapter'; export type { QueryEventsBySavedObjectResult } from './es/cluster_client_adapter'; import { SavedObjectProvider } from './saved_object_provider_registry'; @@ -49,6 +50,14 @@ export interface IEventLogClient { options?: Partial, legacyIds?: string[] ): Promise; + aggregateEventsBySavedObjectIds( + type: string, + ids: string[], + options?: Partial, + legacyIds?: string[] + ): Promise<{ + aggregations: Record | undefined; + }>; } export interface IEventLogger { diff --git a/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.ts b/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.ts index f536959a910cd..4d2cc97f75d89 100644 --- a/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.ts +++ b/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.ts @@ -16,7 +16,7 @@ enum AlertingEntity { } const readOperations: Record = { - rule: ['get', 'getRuleState', 'getAlertSummary', 'find'], + rule: ['get', 'getRuleState', 'getAlertSummary', 'getExecutionLog', 'find'], alert: ['get', 'find'], }; From 037d7d404bcb7ce8b66a8ea0c62a45e22e435062 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Fri, 4 Mar 2022 15:07:00 -0500 Subject: [PATCH 02/39] wip --- .../routes/get_rule_execution_log.test.ts | 117 ++++++++++++++++++ .../server/routes/get_rule_execution_log.ts | 82 ++++++++++++ .../plugins/alerting/server/routes/index.ts | 2 + 3 files changed, 201 insertions(+) create mode 100644 x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts create mode 100644 x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts diff --git a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts new file mode 100644 index 0000000000000..4193e769dc513 --- /dev/null +++ b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getRuleAlertSummaryRoute } from './get_rule_alert_summary'; +import { httpServiceMock } from 'src/core/server/mocks'; +import { licenseStateMock } from '../lib/license_state.mock'; +import { mockHandlerArguments } from './_mock_handler_arguments'; +import { SavedObjectsErrorHelpers } from 'src/core/server'; +import { rulesClientMock } from '../rules_client.mock'; +import { AlertSummary } from '../types'; + +const rulesClient = rulesClientMock.create(); +jest.mock('../lib/license_api_access.ts', () => ({ + verifyApiAccess: jest.fn(), +})); + +beforeEach(() => { + jest.resetAllMocks(); +}); + +describe('getRuleAlertSummaryRoute', () => { + const dateString = new Date().toISOString(); + const mockedAlertSummary: AlertSummary = { + id: '', + name: '', + tags: [], + ruleTypeId: '', + consumer: '', + muteAll: false, + throttle: null, + enabled: false, + statusStartDate: dateString, + statusEndDate: dateString, + status: 'OK', + errorMessages: [], + alerts: {}, + executionDuration: { + average: 1, + valuesWithTimestamp: { + '17 Nov 2021 @ 19:19:17': 3, + '18 Nov 2021 @ 19:19:17': 5, + '19 Nov 2021 @ 19:19:17': 5, + }, + }, + }; + + it('gets rule alert summary', async () => { + const licenseState = licenseStateMock.create(); + const router = httpServiceMock.createRouter(); + + getRuleAlertSummaryRoute(router, licenseState); + + const [config, handler] = router.get.mock.calls[0]; + + expect(config.path).toMatchInlineSnapshot(`"/internal/alerting/rule/{id}/_alert_summary"`); + + rulesClient.getAlertSummary.mockResolvedValueOnce(mockedAlertSummary); + + const [context, req, res] = mockHandlerArguments( + { rulesClient }, + { + params: { + id: '1', + }, + query: {}, + }, + ['ok'] + ); + + await handler(context, req, res); + + expect(rulesClient.getAlertSummary).toHaveBeenCalledTimes(1); + expect(rulesClient.getAlertSummary.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + Object { + "dateStart": undefined, + "id": "1", + "numberOfExecutions": undefined, + }, + ] + `); + + expect(res.ok).toHaveBeenCalled(); + }); + + it('returns NOT-FOUND when rule is not found', async () => { + const licenseState = licenseStateMock.create(); + const router = httpServiceMock.createRouter(); + + getRuleAlertSummaryRoute(router, licenseState); + + const [, handler] = router.get.mock.calls[0]; + + rulesClient.getAlertSummary = jest + .fn() + .mockRejectedValueOnce(SavedObjectsErrorHelpers.createGenericNotFoundError('alert', '1')); + + const [context, req, res] = mockHandlerArguments( + { rulesClient }, + { + params: { + id: '1', + }, + query: {}, + }, + ['notFound'] + ); + + expect(handler(context, req, res)).rejects.toMatchInlineSnapshot( + `[Error: Saved object [alert/1] not found]` + ); + }); +}); diff --git a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts new file mode 100644 index 0000000000000..ef8458f97231b --- /dev/null +++ b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IRouter } from 'kibana/server'; +import { schema } from '@kbn/config-schema'; +import { ILicenseState } from '../lib'; +import { GetAlertSummaryParams } from '../rules_client'; +import { RewriteRequestCase, RewriteResponseCase, verifyAccessAndContext } from './lib'; +import { + AlertingRequestHandlerContext, + INTERNAL_BASE_ALERTING_API_PATH, + AlertSummary, +} from '../types'; + +const paramSchema = schema.object({ + id: schema.string(), +}); + +const querySchema = schema.object({ + date_start: schema.maybe(schema.string()), + date_end: schema.maybe(schema.string()), +}); + +const rewriteReq: RewriteRequestCase = ({ + date_start: dateStart, + number_of_executions: numberOfExecutions, + ...rest +}) => ({ + ...rest, + numberOfExecutions, + dateStart, +}); + +const rewriteBodyRes: RewriteResponseCase = ({ + ruleTypeId, + muteAll, + statusStartDate, + statusEndDate, + errorMessages, + lastRun, + executionDuration: { valuesWithTimestamp, ...executionDuration }, + ...rest +}) => ({ + ...rest, + rule_type_id: ruleTypeId, + mute_all: muteAll, + status_start_date: statusStartDate, + status_end_date: statusEndDate, + error_messages: errorMessages, + last_run: lastRun, + execution_duration: { + ...executionDuration, + values_with_timestamp: valuesWithTimestamp, + }, +}); + +export const getRuleExecutionLogRoute = ( + router: IRouter, + licenseState: ILicenseState +) => { + router.get( + { + path: `${INTERNAL_BASE_ALERTING_API_PATH}/rule/{id}/_execution_log`, + validate: { + params: paramSchema, + query: querySchema, + }, + }, + router.handleLegacyErrors( + verifyAccessAndContext(licenseState, async function (context, req, res) { + const rulesClient = context.alerting.getRulesClient(); + const { id } = req.params; + const execLog = await rulesClient.getExecutionLogForRule(rewriteReq({ id, ...req.query })); + return res.ok({ body: rewriteBodyRes(execLog) }); + }) + ) + ); +}; diff --git a/x-pack/plugins/alerting/server/routes/index.ts b/x-pack/plugins/alerting/server/routes/index.ts index f43190ec6d1c2..18fcf0539f128 100644 --- a/x-pack/plugins/alerting/server/routes/index.ts +++ b/x-pack/plugins/alerting/server/routes/index.ts @@ -20,6 +20,7 @@ import { disableRuleRoute } from './disable_rule'; import { enableRuleRoute } from './enable_rule'; import { findRulesRoute, findInternalRulesRoute } from './find_rules'; import { getRuleAlertSummaryRoute } from './get_rule_alert_summary'; +import { getRuleExecutionLogRoute } from './get_rule_execution_log'; import { getRuleStateRoute } from './get_rule_state'; import { healthRoute } from './health'; import { resolveRuleRoute } from './resolve_rule'; @@ -53,6 +54,7 @@ export function defineRoutes(opts: RouteOptions) { findRulesRoute(router, licenseState, usageCounter); findInternalRulesRoute(router, licenseState, usageCounter); getRuleAlertSummaryRoute(router, licenseState); + getRuleExecutionLogRoute(router, licenseState); getRuleStateRoute(router, licenseState); healthRoute(router, licenseState, encryptedSavedObjects); ruleTypesRoute(router, licenseState); From 939340e252f36796f272f663e7e75f275772cf84 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Mon, 7 Mar 2022 11:17:54 -0500 Subject: [PATCH 03/39] Reverting changes not related to event log aggregation --- .../authorization/alerting_authorization.ts | 1 - .../routes/get_rule_execution_log.test.ts | 117 ------------------ .../plugins/alerting/server/routes/index.ts | 2 - .../server/rules_client/rules_client.ts | 86 ------------- .../feature_privilege_builder/alerting.ts | 2 +- 5 files changed, 1 insertion(+), 207 deletions(-) delete mode 100644 x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts diff --git a/x-pack/plugins/alerting/server/authorization/alerting_authorization.ts b/x-pack/plugins/alerting/server/authorization/alerting_authorization.ts index a079594997521..aafeb5003eaab 100644 --- a/x-pack/plugins/alerting/server/authorization/alerting_authorization.ts +++ b/x-pack/plugins/alerting/server/authorization/alerting_authorization.ts @@ -30,7 +30,6 @@ export enum ReadOperations { Get = 'get', GetRuleState = 'getRuleState', GetAlertSummary = 'getAlertSummary', - GetExecutionLog = 'getExecutionLog', Find = 'find', } diff --git a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts deleted file mode 100644 index 4193e769dc513..0000000000000 --- a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getRuleAlertSummaryRoute } from './get_rule_alert_summary'; -import { httpServiceMock } from 'src/core/server/mocks'; -import { licenseStateMock } from '../lib/license_state.mock'; -import { mockHandlerArguments } from './_mock_handler_arguments'; -import { SavedObjectsErrorHelpers } from 'src/core/server'; -import { rulesClientMock } from '../rules_client.mock'; -import { AlertSummary } from '../types'; - -const rulesClient = rulesClientMock.create(); -jest.mock('../lib/license_api_access.ts', () => ({ - verifyApiAccess: jest.fn(), -})); - -beforeEach(() => { - jest.resetAllMocks(); -}); - -describe('getRuleAlertSummaryRoute', () => { - const dateString = new Date().toISOString(); - const mockedAlertSummary: AlertSummary = { - id: '', - name: '', - tags: [], - ruleTypeId: '', - consumer: '', - muteAll: false, - throttle: null, - enabled: false, - statusStartDate: dateString, - statusEndDate: dateString, - status: 'OK', - errorMessages: [], - alerts: {}, - executionDuration: { - average: 1, - valuesWithTimestamp: { - '17 Nov 2021 @ 19:19:17': 3, - '18 Nov 2021 @ 19:19:17': 5, - '19 Nov 2021 @ 19:19:17': 5, - }, - }, - }; - - it('gets rule alert summary', async () => { - const licenseState = licenseStateMock.create(); - const router = httpServiceMock.createRouter(); - - getRuleAlertSummaryRoute(router, licenseState); - - const [config, handler] = router.get.mock.calls[0]; - - expect(config.path).toMatchInlineSnapshot(`"/internal/alerting/rule/{id}/_alert_summary"`); - - rulesClient.getAlertSummary.mockResolvedValueOnce(mockedAlertSummary); - - const [context, req, res] = mockHandlerArguments( - { rulesClient }, - { - params: { - id: '1', - }, - query: {}, - }, - ['ok'] - ); - - await handler(context, req, res); - - expect(rulesClient.getAlertSummary).toHaveBeenCalledTimes(1); - expect(rulesClient.getAlertSummary.mock.calls[0]).toMatchInlineSnapshot(` - Array [ - Object { - "dateStart": undefined, - "id": "1", - "numberOfExecutions": undefined, - }, - ] - `); - - expect(res.ok).toHaveBeenCalled(); - }); - - it('returns NOT-FOUND when rule is not found', async () => { - const licenseState = licenseStateMock.create(); - const router = httpServiceMock.createRouter(); - - getRuleAlertSummaryRoute(router, licenseState); - - const [, handler] = router.get.mock.calls[0]; - - rulesClient.getAlertSummary = jest - .fn() - .mockRejectedValueOnce(SavedObjectsErrorHelpers.createGenericNotFoundError('alert', '1')); - - const [context, req, res] = mockHandlerArguments( - { rulesClient }, - { - params: { - id: '1', - }, - query: {}, - }, - ['notFound'] - ); - - expect(handler(context, req, res)).rejects.toMatchInlineSnapshot( - `[Error: Saved object [alert/1] not found]` - ); - }); -}); diff --git a/x-pack/plugins/alerting/server/routes/index.ts b/x-pack/plugins/alerting/server/routes/index.ts index 18fcf0539f128..f43190ec6d1c2 100644 --- a/x-pack/plugins/alerting/server/routes/index.ts +++ b/x-pack/plugins/alerting/server/routes/index.ts @@ -20,7 +20,6 @@ import { disableRuleRoute } from './disable_rule'; import { enableRuleRoute } from './enable_rule'; import { findRulesRoute, findInternalRulesRoute } from './find_rules'; import { getRuleAlertSummaryRoute } from './get_rule_alert_summary'; -import { getRuleExecutionLogRoute } from './get_rule_execution_log'; import { getRuleStateRoute } from './get_rule_state'; import { healthRoute } from './health'; import { resolveRuleRoute } from './resolve_rule'; @@ -54,7 +53,6 @@ export function defineRoutes(opts: RouteOptions) { findRulesRoute(router, licenseState, usageCounter); findInternalRulesRoute(router, licenseState, usageCounter); getRuleAlertSummaryRoute(router, licenseState); - getRuleExecutionLogRoute(router, licenseState); getRuleStateRoute(router, licenseState); healthRoute(router, licenseState, encryptedSavedObjects); ruleTypesRoute(router, licenseState); diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index 8ba6b6d0e58da..86f0d3becdce7 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -229,22 +229,6 @@ export interface GetAlertSummaryParams { numberOfExecutions?: number; } -export interface GetExecutionLogByIdParams { - id: string; - dateStart?: string; - // dateEnd? -} - -export interface ExecutionLog { - count: number; - values: Array<{ - timestamp: number; - duration: number; - status: 'failed' | 'succeeded'; - message: string; - }>; -} - // NOTE: Changing this prefix will require a migration to update the prefix in all existing `rule` saved objects const extractedSavedObjectParamReferenceNamePrefix = 'param:'; @@ -647,76 +631,6 @@ export class RulesClient { }); } - public async getExecutionLogForRule({ - id, - dateStart, - }: // dateEnd? - GetExecutionLogByIdParams): Promise { - this.logger.debug(`getExecutionLogForRule(): getting execution log for rule ${id}`); - const rule = (await this.get({ id, includeLegacyId: true })) as SanitizedRuleWithLegacyId; - - // Make sure user has access to this rule - await this.authorization.ensureAuthorized({ - ruleTypeId: rule.alertTypeId, - consumer: rule.consumer, - operation: ReadOperations.GetAlertSummary, - entity: AlertingAuthorizationEntity.Rule, - }); - - // default duration of instance summary is 60 * rule interval - const dateNow = new Date(); - const durationMillis = parseDuration(rule.schedule.interval) * (numberOfExecutions ?? 60); - const defaultDateStart = new Date(dateNow.valueOf() - durationMillis); - const parsedDateStart = parseDate(dateStart, 'dateStart', defaultDateStart); - - const eventLogClient = await this.getEventLogClient(); - - let events: IEvent[]; - let executionEvents: IEvent[]; - - try { - const results = await eventLogClient.aggregateEventsBySavedObjectIds( - 'alert', - [id], - {}, - rule.legacyId !== null ? [rule.legacyId] : undefined - ); - } catch (err) { - this.logger.debug( - `rulesClient.getExecutionLogForRule(): error searching event log for rule ${id}: ${err.message}` - ); - } - - // desired output - // { - // count: number; - // values: [ - // { - // // this is shown in the screenshot - // executionTime: timestamp, - // duration: millis, - // status: failed/succeeded, - // message: log message - execution timeout is separate event log entry - - // // other stuff that might be useful that we could probably get - // number of active alerts - // number of new alerts - // number of recovered alerts - // number of triggered actions - // any errors in the actions - - // es search duration - // total search duration - - // } - // ] - // } - return { - count: 0, - values: [], - }; - } - public async find({ options: { fields, ...options } = {}, excludeFromPublicApi = false, diff --git a/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.ts b/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.ts index 4d2cc97f75d89..f536959a910cd 100644 --- a/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.ts +++ b/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.ts @@ -16,7 +16,7 @@ enum AlertingEntity { } const readOperations: Record = { - rule: ['get', 'getRuleState', 'getAlertSummary', 'getExecutionLog', 'find'], + rule: ['get', 'getRuleState', 'getAlertSummary', 'find'], alert: ['get', 'find'], }; From 40a93a4b3c02de15d3fb9448f830dffa385a54d0 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Mon, 7 Mar 2022 11:19:27 -0500 Subject: [PATCH 04/39] Reverting changes not related to event log aggregation --- .../server/routes/get_rule_execution_log.ts | 82 ------------------- 1 file changed, 82 deletions(-) delete mode 100644 x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts diff --git a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts deleted file mode 100644 index ef8458f97231b..0000000000000 --- a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IRouter } from 'kibana/server'; -import { schema } from '@kbn/config-schema'; -import { ILicenseState } from '../lib'; -import { GetAlertSummaryParams } from '../rules_client'; -import { RewriteRequestCase, RewriteResponseCase, verifyAccessAndContext } from './lib'; -import { - AlertingRequestHandlerContext, - INTERNAL_BASE_ALERTING_API_PATH, - AlertSummary, -} from '../types'; - -const paramSchema = schema.object({ - id: schema.string(), -}); - -const querySchema = schema.object({ - date_start: schema.maybe(schema.string()), - date_end: schema.maybe(schema.string()), -}); - -const rewriteReq: RewriteRequestCase = ({ - date_start: dateStart, - number_of_executions: numberOfExecutions, - ...rest -}) => ({ - ...rest, - numberOfExecutions, - dateStart, -}); - -const rewriteBodyRes: RewriteResponseCase = ({ - ruleTypeId, - muteAll, - statusStartDate, - statusEndDate, - errorMessages, - lastRun, - executionDuration: { valuesWithTimestamp, ...executionDuration }, - ...rest -}) => ({ - ...rest, - rule_type_id: ruleTypeId, - mute_all: muteAll, - status_start_date: statusStartDate, - status_end_date: statusEndDate, - error_messages: errorMessages, - last_run: lastRun, - execution_duration: { - ...executionDuration, - values_with_timestamp: valuesWithTimestamp, - }, -}); - -export const getRuleExecutionLogRoute = ( - router: IRouter, - licenseState: ILicenseState -) => { - router.get( - { - path: `${INTERNAL_BASE_ALERTING_API_PATH}/rule/{id}/_execution_log`, - validate: { - params: paramSchema, - query: querySchema, - }, - }, - router.handleLegacyErrors( - verifyAccessAndContext(licenseState, async function (context, req, res) { - const rulesClient = context.alerting.getRulesClient(); - const { id } = req.params; - const execLog = await rulesClient.getExecutionLogForRule(rewriteReq({ id, ...req.query })); - return res.ok({ body: rewriteBodyRes(execLog) }); - }) - ) - ); -}; From c40c902c5bec8e3b553e6b80f963098cd045a950 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Mon, 7 Mar 2022 12:02:14 -0500 Subject: [PATCH 05/39] Updating event log client find to take array of sort options --- .../server/rules_client/rules_client.ts | 4 +- .../tests/get_alert_summary.test.ts | 14 ++++- .../server/es/cluster_client_adapter.test.ts | 6 +-- .../server/es/cluster_client_adapter.ts | 17 ++---- .../event_log/server/event_log_client.test.ts | 16 ++++-- .../event_log/server/event_log_client.ts | 54 +++++++++---------- .../plugins/event_log/server/routes/find.ts | 4 +- .../event_log/server/routes/find_by_ids.ts | 4 +- .../event_log/event_log_reader.ts | 3 +- 9 files changed, 64 insertions(+), 58 deletions(-) diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index 86f0d3becdce7..1512959384ac9 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -594,7 +594,7 @@ export class RulesClient { page: 1, per_page: 10000, start: parsedDateStart.toISOString(), - sort_order: 'desc', + sort: [{ sort_field: '@timestamp', sort_order: 'desc' }], end: dateNow.toISOString(), }, rule.legacyId !== null ? [rule.legacyId] : undefined @@ -606,7 +606,7 @@ export class RulesClient { page: 1, per_page: numberOfExecutions ?? 60, filter: 'event.provider: alerting AND event.action:execute', - sort_order: 'desc', + sort: [{ sort_field: '@timestamp', sort_order: 'desc' }], end: dateNow.toISOString(), }, rule.legacyId !== null ? [rule.legacyId] : undefined diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts index 4e6f627dcd4a6..fcf90bc350362 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts @@ -223,7 +223,12 @@ describe('getAlertSummary()', () => { "end": "2019-02-12T21:01:22.479Z", "page": 1, "per_page": 10000, - "sort_order": "desc", + "sort": Array [ + Object { + "sort_field": "@timestamp", + "sort_order": "desc", + }, + ], "start": "2019-02-12T21:00:22.479Z", }, undefined, @@ -260,7 +265,12 @@ describe('getAlertSummary()', () => { "end": "2019-02-12T21:01:22.479Z", "page": 1, "per_page": 10000, - "sort_order": "desc", + "sort": Array [ + Object { + "sort_field": "@timestamp", + "sort_order": "desc", + }, + ], "start": "2019-02-12T21:00:22.479Z", }, Array [ diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts index 22898ac54db5a..56abb736a7dd5 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts @@ -11,7 +11,7 @@ import { IClusterClientAdapter, EVENT_BUFFER_LENGTH, } from './cluster_client_adapter'; -import { findOptionsSchema } from '../event_log_client'; +import { queryOptionsSchema } from '../event_log_client'; import { delay } from '../lib/delay'; import { times } from 'lodash'; import type * as estypes from '@elastic/elasticsearch/lib/api/types'; @@ -567,7 +567,7 @@ describe('createIndex', () => { }); describe('queryEventsBySavedObject', () => { - const DEFAULT_OPTIONS = findOptionsSchema.validate({}); + const DEFAULT_OPTIONS = queryOptionsSchema.validate({}); test('should call cluster with proper arguments with non-default namespace', async () => { clusterClient.search.mockResponse({ @@ -916,7 +916,7 @@ describe('queryEventsBySavedObject', () => { namespace: 'namespace', type: 'saved-object-type', ids: ['saved-object-id'], - findOptions: { ...DEFAULT_OPTIONS, sort_field: 'event.end', sort_order: 'desc' }, + findOptions: { ...DEFAULT_OPTIONS, sort: [{ sort_field: 'event.end', sort_order: 'desc' }] }, }); const [query] = clusterClient.search.mock.calls[0]; diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index 067e89a45677a..96b29a7728abb 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -47,7 +47,7 @@ interface QueryOptionsEventsBySavedObjectFilter { namespace: string | undefined; type: string; ids: string[]; - findOptions: FindOptionsType | AggregateOptionsType; + findOptions: FindOptionsType; legacyIds?: string[]; } @@ -330,8 +330,7 @@ export class ClusterClientAdapter { const { index, type, ids, findOptions } = queryOptions; - // eslint-disable-next-line @typescript-eslint/naming-convention - const { page, per_page: perPage, sort_field, sort_order } = findOptions; + const { page, per_page: perPage, sort } = findOptions; const esClient = await this.elasticsearchClientPromise; @@ -340,7 +339,7 @@ export class ClusterClientAdapter ({ [s.sort_field]: { order: s.sort_order } })), query, }; @@ -371,15 +370,7 @@ export class ClusterClientAdapter | undefined; }> { const { index, type, ids, findOptions } = queryOptions; - const { - page, - per_page: perPage, - // eslint-disable-next-line @typescript-eslint/naming-convention - sort_field, - // eslint-disable-next-line @typescript-eslint/naming-convention - sort_order, - aggs, - } = findOptions as AggregateOptionsType; + const { page, per_page: perPage, sort, aggs } = findOptions as AggregateOptionsType; const esClient = await this.elasticsearchClientPromise; diff --git a/x-pack/plugins/event_log/server/event_log_client.test.ts b/x-pack/plugins/event_log/server/event_log_client.test.ts index 0acb53e93b81a..690cd8380957b 100644 --- a/x-pack/plugins/event_log/server/event_log_client.test.ts +++ b/x-pack/plugins/event_log/server/event_log_client.test.ts @@ -127,8 +127,12 @@ describe('EventLogStart', () => { findOptions: { page: 1, per_page: 10, - sort_field: '@timestamp', - sort_order: 'asc', + sort: [ + { + sort_field: '@timestamp', + sort_order: 'asc', + }, + ], }, legacyIds: ['legacy-id'], }); @@ -214,8 +218,12 @@ describe('EventLogStart', () => { findOptions: { page: 1, per_page: 10, - sort_field: '@timestamp', - sort_order: 'asc', + sort: [ + { + sort_field: '@timestamp', + sort_order: 'asc', + }, + ], start, end, }, diff --git a/x-pack/plugins/event_log/server/event_log_client.ts b/x-pack/plugins/event_log/server/event_log_client.ts index 1ce6f1f6316ef..ca7abc01e47c4 100644 --- a/x-pack/plugins/event_log/server/event_log_client.ts +++ b/x-pack/plugins/event_log/server/event_log_client.ts @@ -28,43 +28,41 @@ const optionalDateFieldSchema = schema.maybe( }) ); -export const findOptionsSchema = schema.object({ +const sortFieldSchema = schema.object({ + sort_field: schema.oneOf([ + schema.literal('@timestamp'), + schema.literal('event.start'), + schema.literal('event.end'), + schema.literal('event.provider'), + schema.literal('event.duration'), + schema.literal('event.action'), + schema.literal('message'), + ]), + sort_order: schema.oneOf([schema.literal('asc'), schema.literal('desc')]), +}); + +const sortFieldsSchema = schema.arrayOf(sortFieldSchema, { + defaultValue: [{ sort_field: '@timestamp', sort_order: 'asc' }], +}); + +export const queryOptionsSchema = schema.object({ per_page: schema.number({ defaultValue: 10, min: 0 }), page: schema.number({ defaultValue: 1, min: 1 }), start: optionalDateFieldSchema, end: optionalDateFieldSchema, - sort_field: schema.oneOf( - [ - schema.literal('@timestamp'), - schema.literal('event.start'), - schema.literal('event.end'), - schema.literal('event.provider'), - schema.literal('event.duration'), - schema.literal('event.action'), - schema.literal('message'), - ], - { - defaultValue: '@timestamp', - } - ), - sort_order: schema.oneOf([schema.literal('asc'), schema.literal('desc')], { - defaultValue: 'asc', - }), + sort: sortFieldsSchema, filter: schema.maybe(schema.string()), }); // page & perPage are required, other fields are optional // using schema.maybe allows us to set undefined, but not to make the field optional export type FindOptionsType = Pick< - TypeOf, - 'page' | 'per_page' | 'sort_field' | 'sort_order' | 'filter' + TypeOf, + 'page' | 'per_page' | 'sort' | 'filter' > & - Partial>; + Partial>; -export type AggregateOptionsType = Pick< - TypeOf, - 'start' | 'end' | 'sort_field' | 'sort_order' | 'filter' -> & - Partial> & { +export type AggregateOptionsType = Pick, 'sort' | 'filter'> & + Partial> & { aggs: Record; }; @@ -95,7 +93,7 @@ export class EventLogClient implements IEventLogClient { options?: Partial, legacyIds?: string[] ): Promise { - const findOptions = findOptionsSchema.validate(options ?? {}); + const findOptions = queryOptionsSchema.validate(options ?? {}); const space = await this.spacesService?.getActiveSpace(this.request); const namespace = space && this.spacesService?.spaceIdToNamespace(space.id); @@ -119,7 +117,7 @@ export class EventLogClient implements IEventLogClient { options?: Partial, legacyIds?: string[] ) { - // const aggregateOptions = findOptionsSchema.validate(options ?? {}); + // const aggregateOptions = queryOptionsSchema.validate(options ?? {}); const space = await this.spacesService?.getActiveSpace(this.request); const namespace = space && this.spacesService?.spaceIdToNamespace(space.id); diff --git a/x-pack/plugins/event_log/server/routes/find.ts b/x-pack/plugins/event_log/server/routes/find.ts index cbbd2eedd2dbc..fdb699b70e26c 100644 --- a/x-pack/plugins/event_log/server/routes/find.ts +++ b/x-pack/plugins/event_log/server/routes/find.ts @@ -14,7 +14,7 @@ import type { } from 'src/core/server'; import type { EventLogRouter, EventLogRequestHandlerContext } from '../types'; import { BASE_EVENT_LOG_API_PATH } from '../../common'; -import { findOptionsSchema, FindOptionsType } from '../event_log_client'; +import { queryOptionsSchema, FindOptionsType } from '../event_log_client'; const paramSchema = schema.object({ type: schema.string(), @@ -27,7 +27,7 @@ export const findRoute = (router: EventLogRouter, systemLogger: Logger) => { path: `${BASE_EVENT_LOG_API_PATH}/{type}/{id}/_find`, validate: { params: paramSchema, - query: findOptionsSchema, + query: queryOptionsSchema, }, }, router.handleLegacyErrors(async function ( diff --git a/x-pack/plugins/event_log/server/routes/find_by_ids.ts b/x-pack/plugins/event_log/server/routes/find_by_ids.ts index 378b9516631ad..324dbc7f568ba 100644 --- a/x-pack/plugins/event_log/server/routes/find_by_ids.ts +++ b/x-pack/plugins/event_log/server/routes/find_by_ids.ts @@ -15,7 +15,7 @@ import type { import type { EventLogRouter, EventLogRequestHandlerContext } from '../types'; import { BASE_EVENT_LOG_API_PATH } from '../../common'; -import { findOptionsSchema, FindOptionsType } from '../event_log_client'; +import { queryOptionsSchema, FindOptionsType } from '../event_log_client'; const paramSchema = schema.object({ type: schema.string(), @@ -32,7 +32,7 @@ export const findByIdsRoute = (router: EventLogRouter, systemLogger: Logger) => path: `${BASE_EVENT_LOG_API_PATH}/{type}/_find`, validate: { params: paramSchema, - query: findOptionsSchema, + query: queryOptionsSchema, body: bodySchema, }, }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log/event_log_reader.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log/event_log_reader.ts index a3c2421edc85a..3a59c26c769ee 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log/event_log_reader.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log/event_log_reader.ts @@ -55,8 +55,7 @@ export const createEventLogReader = (eventLog: IEventLogClient): IEventLogReader return eventLog.findEventsBySavedObjectIds(soType, soIds, { page: 1, per_page: count, - sort_field: '@timestamp', - sort_order: 'desc', + sort: [{ sort_field: '@timestamp', sort_order: 'desc' }], filter: kqlFilter, }); }); From 1b1369523e89a8e799c2e054e7e34a4b86043232 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Mon, 7 Mar 2022 15:47:32 -0500 Subject: [PATCH 06/39] Updating tests and adding basic aggregation function --- .../server/es/cluster_client_adapter.mock.ts | 1 + .../server/es/cluster_client_adapter.test.ts | 1118 +++++++++-------- .../server/es/cluster_client_adapter.ts | 53 +- .../event_log/server/event_log_client.mock.ts | 1 + .../event_log/server/event_log_client.test.ts | 250 ++-- .../event_log/server/event_log_client.ts | 34 +- x-pack/plugins/event_log/server/types.ts | 2 +- 7 files changed, 733 insertions(+), 726 deletions(-) diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts index 667512ea13f65..53a1b9501b432 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts @@ -24,6 +24,7 @@ const createClusterClientMock = () => { getExistingIndexAliases: jest.fn(), setIndexAliasToHidden: jest.fn(), queryEventsBySavedObjects: jest.fn(), + aggregateEventsBySavedObjects: jest.fn(), shutdown: jest.fn(), }; return mock; diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts index 56abb736a7dd5..93580ceca9911 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts @@ -10,10 +10,12 @@ import { ClusterClientAdapter, IClusterClientAdapter, EVENT_BUFFER_LENGTH, + getQueryBody, + FindEventsOptionsBySavedObjectFilter, } from './cluster_client_adapter'; import { queryOptionsSchema } from '../event_log_client'; import { delay } from '../lib/delay'; -import { times } from 'lodash'; +import { pick, times } from 'lodash'; import type * as estypes from '@elastic/elasticsearch/lib/api/types'; type MockedLogger = ReturnType; @@ -569,11 +571,11 @@ describe('createIndex', () => { describe('queryEventsBySavedObject', () => { const DEFAULT_OPTIONS = queryOptionsSchema.validate({}); - test('should call cluster with proper arguments with non-default namespace', async () => { + test('should call cluster with correct options', async () => { clusterClient.search.mockResponse({ hits: { - hits: [], - total: { relation: 'eq', value: 0 }, + hits: [{ _index: 'index-name-00001', _id: '1', _source: { foo: 'bar' } }], + total: { relation: 'eq', value: 1 }, }, took: 0, timed_out: false, @@ -584,85 +586,197 @@ describe('queryEventsBySavedObject', () => { skipped: 0, }, }); - await clusterClientAdapter.queryEventsBySavedObjects({ + const options = { index: 'index-name', namespace: 'namespace', type: 'saved-object-type', ids: ['saved-object-id'], - findOptions: DEFAULT_OPTIONS, - }); + findOptions: { + ...DEFAULT_OPTIONS, + page: 3, + per_page: 6, + sort: [ + { sort_field: '@timestamp', sort_order: 'asc' }, + { sort_field: 'event.end', sort_order: 'desc' }, + ], + }, + }; + const result = await clusterClientAdapter.queryEventsBySavedObjects(options); const [query] = clusterClient.search.mock.calls[0]; - expect(query).toMatchInlineSnapshot( - { - body: { - from: 0, - query: { - bool: { - filter: [], - must: [ - { - nested: { - path: 'kibana.saved_objects', - query: { + expect(query).toEqual({ + index: 'index-name', + track_total_hits: true, + body: { + size: 6, + from: 12, + query: getQueryBody(logger, options, pick(options.findOptions, ['start', 'end', 'filter'])), + sort: [{ '@timestamp': { order: 'asc' } }, { 'event.end': { order: 'desc' } }], + }, + }); + expect(result).toEqual({ + page: 3, + per_page: 6, + total: 1, + data: [{ foo: 'bar' }], + }); + }); +}); + +describe('getQueryBody', () => { + const options = { + index: 'index-name', + namespace: undefined, + type: 'saved-object-type', + ids: ['saved-object-id'], + }; + test('should correctly build query with namespace filter when namespace is undefined', () => { + expect(getQueryBody(logger, options as FindEventsOptionsBySavedObjectFilter, {})).toEqual({ + bool: { + must: [ + { + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: [ + { + term: { + 'kibana.saved_objects.rel': { + value: 'primary', + }, + }, + }, + { + term: { + 'kibana.saved_objects.type': { + value: 'saved-object-type', + }, + }, + }, + { bool: { - must: [ - { - term: { - 'kibana.saved_objects.rel': { - value: 'primary', - }, - }, + must_not: { + exists: { + field: 'kibana.saved_objects.namespace', }, - { - term: { - 'kibana.saved_objects.type': { - value: 'saved-object-type', - }, + }, + }, + }, + ], + }, + }, + }, + }, + { + bool: { + should: [ + { + bool: { + must: [ + { + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: [ + { + terms: { + 'kibana.saved_objects.id': ['saved-object-id'], + }, + }, + ], }, }, - { - term: { - 'kibana.saved_objects.namespace': { - value: 'namespace', - }, - }, + }, + }, + { + range: { + 'kibana.version': { + gte: '8.0.0', }, - ], + }, }, - }, + ], }, }, + ], + }, + }, + ], + }, + }); + }); + + test('should correctly build query with namespace filter when namespace is specified', () => { + expect( + getQueryBody( + logger, + { ...options, namespace: 'namespace' } as FindEventsOptionsBySavedObjectFilter, + {} + ) + ).toEqual({ + bool: { + must: [ + { + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: [ + { + term: { + 'kibana.saved_objects.rel': { + value: 'primary', + }, + }, + }, + { + term: { + 'kibana.saved_objects.type': { + value: 'saved-object-type', + }, + }, + }, + { + term: { + 'kibana.saved_objects.namespace': { + value: 'namespace', + }, + }, + }, + ], + }, + }, + }, + }, + { + bool: { + should: [ { bool: { - should: [ + must: [ { - bool: { - must: [ - { - nested: { - path: 'kibana.saved_objects', - query: { - bool: { - must: [ - { - terms: { - 'kibana.saved_objects.id': ['saved-object-id'], - }, - }, - ], + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: [ + { + terms: { + 'kibana.saved_objects.id': ['saved-object-id'], }, }, - }, - }, - { - range: { - 'kibana.version': { - gte: '8.0.0', - }, - }, + ], }, - ], + }, + }, + }, + { + range: { + 'kibana.version': { + gte: '8.0.0', + }, }, }, ], @@ -671,86 +785,234 @@ describe('queryEventsBySavedObject', () => { ], }, }, - size: 10, - sort: [ - { - '@timestamp': { - order: 'asc', + ], + }, + }); + }); + + test('should correctly build query when filter is specified', () => { + expect( + getQueryBody(logger, options as FindEventsOptionsBySavedObjectFilter, { + filter: 'event.provider: alerting AND event.action:execute', + }) + ).toEqual({ + bool: { + filter: { + bool: { + filter: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + 'event.provider': 'alerting', + }, + }, + ], + }, }, - }, - ], + { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + 'event.action': 'execute', + }, + }, + ], + }, + }, + ], + }, }, - index: 'index-name', - track_total_hits: true, - }, - ` - Object { - "body": Object { - "from": 0, - "query": Object { - "bool": Object { - "filter": Array [], - "must": Array [ - Object { - "nested": Object { - "path": "kibana.saved_objects", - "query": Object { - "bool": Object { - "must": Array [ - Object { - "term": Object { - "kibana.saved_objects.rel": Object { - "value": "primary", - }, - }, + must: [ + { + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: [ + { + term: { + 'kibana.saved_objects.rel': { + value: 'primary', + }, + }, + }, + { + term: { + 'kibana.saved_objects.type': { + value: 'saved-object-type', + }, + }, + }, + { + bool: { + must_not: { + exists: { + field: 'kibana.saved_objects.namespace', }, - Object { - "term": Object { - "kibana.saved_objects.type": Object { - "value": "saved-object-type", - }, + }, + }, + }, + ], + }, + }, + }, + }, + { + bool: { + should: [ + { + bool: { + must: [ + { + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: [ + { + terms: { + 'kibana.saved_objects.id': ['saved-object-id'], + }, + }, + ], }, }, - Object { - "term": Object { - "kibana.saved_objects.namespace": Object { - "value": "namespace", - }, - }, + }, + }, + { + range: { + 'kibana.version': { + gte: '8.0.0', }, - ], + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }); + }); + + test('should correctly build query when legacyIds are specified', () => { + expect( + getQueryBody( + logger, + { ...options, legacyIds: ['legacy-id-1'] } as FindEventsOptionsBySavedObjectFilter, + {} + ) + ).toEqual({ + bool: { + must: [ + { + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: [ + { + term: { + 'kibana.saved_objects.rel': { + value: 'primary', + }, + }, + }, + { + term: { + 'kibana.saved_objects.type': { + value: 'saved-object-type', + }, }, }, + { + bool: { + must_not: { + exists: { + field: 'kibana.saved_objects.namespace', + }, + }, + }, + }, + ], + }, + }, + }, + }, + { + bool: { + should: [ + { + bool: { + must: [ + { + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: [ + { + terms: { + 'kibana.saved_objects.id': ['saved-object-id'], + }, + }, + ], + }, + }, + }, + }, + { + range: { + 'kibana.version': { + gte: '8.0.0', + }, + }, + }, + ], }, }, - Object { - "bool": Object { - "should": Array [ - Object { - "bool": Object { - "must": Array [ - Object { - "nested": Object { - "path": "kibana.saved_objects", - "query": Object { - "bool": Object { - "must": Array [ - Object { - "terms": Object { - "kibana.saved_objects.id": Array [ - "saved-object-id", - ], - }, - }, - ], + { + bool: { + must: [ + { + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: [ + { + terms: { + 'kibana.saved_objects.id': ['legacy-id-1'], }, }, + ], + }, + }, + }, + }, + { + bool: { + should: [ + { + range: { + 'kibana.version': { + lt: '8.0.0', + }, }, }, - Object { - "range": Object { - "kibana.version": Object { - "gte": "8.0.0", + { + bool: { + must_not: { + exists: { + field: 'kibana.version', + }, }, }, }, @@ -763,471 +1025,283 @@ describe('queryEventsBySavedObject', () => { ], }, }, - "size": 10, - "sort": Array [ - Object { - "@timestamp": Object { - "order": "asc", - }, - }, - ], - }, - "index": "index-name", - "track_total_hits": true, - } - ` - ); - }); - - test('should call cluster with proper arguments with default namespace', async () => { - clusterClient.search.mockResponse({ - hits: { - hits: [], - total: { relation: 'eq', value: 0 }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - successful: 0, - total: 0, - skipped: 0, + ], }, }); - await clusterClientAdapter.queryEventsBySavedObjects({ - index: 'index-name', - namespace: undefined, - type: 'saved-object-type', - ids: ['saved-object-id'], - findOptions: DEFAULT_OPTIONS, - }); + }); - const [query] = clusterClient.search.mock.calls[0]; - expect(query).toMatchObject({ - body: { - from: 0, - query: { - bool: { - filter: [], - must: [ - { - nested: { - path: 'kibana.saved_objects', - query: { - bool: { - must: [ - { - term: { - 'kibana.saved_objects.rel': { - value: 'primary', - }, - }, - }, - { - term: { - 'kibana.saved_objects.type': { - value: 'saved-object-type', - }, - }, + test('should correctly build query when start is specified', () => { + expect( + getQueryBody(logger, options as FindEventsOptionsBySavedObjectFilter, { + start: '2020-07-08T00:52:28.350Z', + }) + ).toEqual({ + bool: { + must: [ + { + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: [ + { + term: { + 'kibana.saved_objects.rel': { + value: 'primary', }, - { - bool: { - must_not: { - exists: { - field: 'kibana.saved_objects.namespace', - }, - }, + }, + }, + { + term: { + 'kibana.saved_objects.type': { + value: 'saved-object-type', + }, + }, + }, + { + bool: { + must_not: { + exists: { + field: 'kibana.saved_objects.namespace', }, }, - ], + }, }, - }, + ], }, }, - { - bool: { - should: [ - { - bool: { - must: [ - { - nested: { - path: 'kibana.saved_objects', - query: { - bool: { - must: [ - { - terms: { - 'kibana.saved_objects.id': ['saved-object-id'], - }, - }, - ], + }, + }, + { + bool: { + should: [ + { + bool: { + must: [ + { + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: [ + { + terms: { + 'kibana.saved_objects.id': ['saved-object-id'], + }, }, - }, + ], }, }, - { - range: { - 'kibana.version': { - gte: '8.0.0', - }, - }, + }, + }, + { + range: { + 'kibana.version': { + gte: '8.0.0', }, - ], + }, }, - }, - ], + ], + }, }, - }, - ], + ], + }, }, - }, - size: 10, - sort: [ { - '@timestamp': { - order: 'asc', + range: { + '@timestamp': { + gte: '2020-07-08T00:52:28.350Z', + }, }, }, ], }, - index: 'index-name', - track_total_hits: true, - }); - }); - - test('should call cluster with sort', async () => { - clusterClient.search.mockResponse({ - hits: { - hits: [], - total: { relation: 'eq', value: 0 }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - successful: 0, - total: 0, - skipped: 0, - }, - }); - await clusterClientAdapter.queryEventsBySavedObjects({ - index: 'index-name', - namespace: 'namespace', - type: 'saved-object-type', - ids: ['saved-object-id'], - findOptions: { ...DEFAULT_OPTIONS, sort: [{ sort_field: 'event.end', sort_order: 'desc' }] }, - }); - - const [query] = clusterClient.search.mock.calls[0]; - expect(query).toMatchObject({ - index: 'index-name', - body: { - sort: [{ 'event.end': { order: 'desc' } }], - }, }); }); - test('supports open ended date', async () => { - clusterClient.search.mockResponse({ - hits: { - hits: [], - total: { relation: 'eq', value: 0 }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - successful: 0, - total: 0, - skipped: 0, - }, - }); - - const start = '2020-07-08T00:52:28.350Z'; - - await clusterClientAdapter.queryEventsBySavedObjects({ - index: 'index-name', - namespace: 'namespace', - type: 'saved-object-type', - ids: ['saved-object-id'], - findOptions: { ...DEFAULT_OPTIONS, start }, - }); - - const [query] = clusterClient.search.mock.calls[0]; - expect(query).toMatchObject({ - body: { - from: 0, - query: { - bool: { - filter: [], - must: [ - { - nested: { - path: 'kibana.saved_objects', - query: { - bool: { - must: [ - { - term: { - 'kibana.saved_objects.rel': { - value: 'primary', - }, - }, + test('should correctly build query when end is specified', () => { + expect( + getQueryBody(logger, options as FindEventsOptionsBySavedObjectFilter, { + end: '2020-07-10T00:52:28.350Z', + }) + ).toEqual({ + bool: { + must: [ + { + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: [ + { + term: { + 'kibana.saved_objects.rel': { + value: 'primary', }, - { - term: { - 'kibana.saved_objects.type': { - value: 'saved-object-type', - }, - }, + }, + }, + { + term: { + 'kibana.saved_objects.type': { + value: 'saved-object-type', }, - { - term: { - 'kibana.saved_objects.namespace': { - value: 'namespace', - }, + }, + }, + { + bool: { + must_not: { + exists: { + field: 'kibana.saved_objects.namespace', }, }, - ], + }, }, - }, + ], }, }, - { - bool: { - should: [ - { - bool: { - must: [ - { - nested: { - path: 'kibana.saved_objects', - query: { - bool: { - must: [ - { - terms: { - 'kibana.saved_objects.id': ['saved-object-id'], - }, - }, - ], + }, + }, + { + bool: { + should: [ + { + bool: { + must: [ + { + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: [ + { + terms: { + 'kibana.saved_objects.id': ['saved-object-id'], + }, }, - }, + ], }, }, - { - range: { - 'kibana.version': { - gte: '8.0.0', - }, - }, + }, + }, + { + range: { + 'kibana.version': { + gte: '8.0.0', }, - ], + }, }, - }, - ], - }, - }, - { - range: { - '@timestamp': { - gte: '2020-07-08T00:52:28.350Z', + ], }, }, - }, - ], + ], + }, }, - }, - size: 10, - sort: [ { - '@timestamp': { - order: 'asc', + range: { + '@timestamp': { + lte: '2020-07-10T00:52:28.350Z', + }, }, }, ], }, - index: 'index-name', - track_total_hits: true, }); }); - test('supports optional date range', async () => { - clusterClient.search.mockResponse({ - hits: { - hits: [], - total: { relation: 'eq', value: 0 }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - successful: 0, - total: 0, - skipped: 0, - }, - }); - - const start = '2020-07-08T00:52:28.350Z'; - const end = '2020-07-08T00:00:00.000Z'; - - await clusterClientAdapter.queryEventsBySavedObjects({ - index: 'index-name', - namespace: 'namespace', - type: 'saved-object-type', - ids: ['saved-object-id'], - findOptions: { ...DEFAULT_OPTIONS, start, end }, - legacyIds: ['legacy-id'], - }); - - const [query] = clusterClient.search.mock.calls[0]; - expect(query).toMatchObject({ - body: { - from: 0, - query: { - bool: { - filter: [], - must: [ - { - nested: { - path: 'kibana.saved_objects', - query: { - bool: { - must: [ - { - term: { - 'kibana.saved_objects.rel': { - value: 'primary', - }, - }, - }, - { - term: { - 'kibana.saved_objects.type': { - value: 'saved-object-type', - }, - }, - }, - { - term: { - 'kibana.saved_objects.namespace': { - value: 'namespace', - }, - }, + test('should correctly build query when start and end are specified', () => { + expect( + getQueryBody(logger, options as FindEventsOptionsBySavedObjectFilter, { + start: '2020-07-08T00:52:28.350Z', + end: '2020-07-10T00:52:28.350Z', + }) + ).toEqual({ + bool: { + must: [ + { + nested: { + path: 'kibana.saved_objects', + query: { + bool: { + must: [ + { + term: { + 'kibana.saved_objects.rel': { + value: 'primary', }, - ], + }, }, - }, - }, - }, - { - bool: { - should: [ { - bool: { - must: [ - { - nested: { - path: 'kibana.saved_objects', - query: { - bool: { - must: [ - { - terms: { - 'kibana.saved_objects.id': ['saved-object-id'], - }, - }, - ], - }, - }, - }, - }, - { - range: { - 'kibana.version': { - gte: '8.0.0', - }, - }, - }, - ], + term: { + 'kibana.saved_objects.type': { + value: 'saved-object-type', + }, }, }, { bool: { - must: [ - { - nested: { - path: 'kibana.saved_objects', - query: { - bool: { - must: [ - { - terms: { - 'kibana.saved_objects.id': ['legacy-id'], - }, - }, - ], - }, - }, - }, + must_not: { + exists: { + field: 'kibana.saved_objects.namespace', }, - { + }, + }, + }, + ], + }, + }, + }, + }, + { + bool: { + should: [ + { + bool: { + must: [ + { + nested: { + path: 'kibana.saved_objects', + query: { bool: { - should: [ - { - range: { - 'kibana.version': { - lt: '8.0.0', - }, - }, - }, + must: [ { - bool: { - must_not: { - exists: { - field: 'kibana.version', - }, - }, + terms: { + 'kibana.saved_objects.id': ['saved-object-id'], }, }, ], }, }, - ], + }, }, - }, - ], - }, - }, - { - range: { - '@timestamp': { - gte: '2020-07-08T00:52:28.350Z', - }, - }, - }, - { - range: { - '@timestamp': { - lte: '2020-07-08T00:00:00.000Z', + { + range: { + 'kibana.version': { + gte: '8.0.0', + }, + }, + }, + ], }, }, + ], + }, + }, + { + range: { + '@timestamp': { + gte: '2020-07-08T00:52:28.350Z', }, - ], + }, }, - }, - size: 10, - sort: [ { - '@timestamp': { - order: 'asc', + range: { + '@timestamp': { + lte: '2020-07-10T00:52:28.350Z', + }, }, }, ], }, - index: 'index-name', - track_total_hits: true, }); }); }); diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index 96b29a7728abb..d7fe22d6b4ca3 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -14,7 +14,7 @@ import util from 'util'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; import { IEvent, IValidatedEvent, SAVED_OBJECT_REL_PRIMARY } from '../types'; -import { AggregateOptionsType, FindOptionsType } from '../event_log_client'; +import { AggregateOptionsType, FindOptionsType, QueryOptionsType } from '../event_log_client'; import { ParsedIndexAlias } from './init'; export const EVENT_BUFFER_TIME = 1000; // milliseconds @@ -47,10 +47,17 @@ interface QueryOptionsEventsBySavedObjectFilter { namespace: string | undefined; type: string; ids: string[]; - findOptions: FindOptionsType; legacyIds?: string[]; } +export type FindEventsOptionsBySavedObjectFilter = QueryOptionsEventsBySavedObjectFilter & { + findOptions: FindOptionsType; +}; + +export type AggregateEventsOptionsBySavedObjectFilter = QueryOptionsEventsBySavedObjectFilter & { + aggregateOptions: AggregateOptionsType; +}; + // eslint-disable-next-line @typescript-eslint/no-explicit-any type AliasAny = any; @@ -327,20 +334,26 @@ export class ClusterClientAdapter { const { index, type, ids, findOptions } = queryOptions; const { page, per_page: perPage, sort } = findOptions; const esClient = await this.elasticsearchClientPromise; - const query = getQueryBody(this.logger, queryOptions); + const query = getQueryBody( + this.logger, + queryOptions, + pick(queryOptions.findOptions, ['start', 'end', 'filter']) + ); const body: estypes.SearchRequest['body'] = { size: perPage, from: (page - 1) * perPage, - sort: (sort ?? []).map((s) => ({ [s.sort_field]: { order: s.sort_order } })), query, + ...(sort + ? { sort: sort.map((s) => ({ [s.sort_field]: { order: s.sort_order } })) as estypes.Sort } + : {}), }; try { @@ -365,16 +378,20 @@ export class ClusterClientAdapter | undefined; + aggregateResults: Record | undefined; }> { - const { index, type, ids, findOptions } = queryOptions; - const { page, per_page: perPage, sort, aggs } = findOptions as AggregateOptionsType; + const { index, type, ids, aggregateOptions } = queryOptions; + const { aggs } = aggregateOptions; const esClient = await this.elasticsearchClientPromise; - const query = getQueryBody(this.logger, queryOptions); + const query = getQueryBody( + this.logger, + queryOptions, + pick(queryOptions.aggregateOptions, ['start', 'end', 'filter']) + ); const body: estypes.SearchRequest['body'] = { size: 0, @@ -388,7 +405,7 @@ export class ClusterClientAdapter { const mock: jest.Mocked = { findEventsBySavedObjectIds: jest.fn(), + aggregateEventsBySavedObjectIds: jest.fn(), }; return mock; }; diff --git a/x-pack/plugins/event_log/server/event_log_client.test.ts b/x-pack/plugins/event_log/server/event_log_client.test.ts index 690cd8380957b..2d53a2c1897e4 100644 --- a/x-pack/plugins/event_log/server/event_log_client.test.ts +++ b/x-pack/plugins/event_log/server/event_log_client.test.ts @@ -7,101 +7,82 @@ import { KibanaRequest } from 'src/core/server'; import { EventLogClient } from './event_log_client'; +import { EsContext } from './es'; import { contextMock } from './es/context.mock'; import { merge } from 'lodash'; import moment from 'moment'; +import { IClusterClientAdapter } from './es/cluster_client_adapter'; + +const expectedSavedObject = { + id: 'saved-object-id', + type: 'saved-object-type', + attributes: {}, + references: [], +}; + +const expectedEvents = [ + fakeEvent({ + kibana: { + saved_objects: [ + { + id: 'saved-object-id', + type: 'saved-object-type', + }, + { + type: 'action', + id: '1', + }, + ], + }, + }), + fakeEvent({ + kibana: { + saved_objects: [ + { + id: 'saved-object-id', + type: 'saved-object-type', + }, + { + type: 'action', + id: '2', + }, + ], + }, + }), +]; describe('EventLogStart', () => { + const savedObjectGetter = jest.fn(); + let esContext: jest.Mocked & { + esAdapter: jest.Mocked; + }; + let eventLogClient: EventLogClient; + beforeEach(() => { + esContext = contextMock.create(); + eventLogClient = new EventLogClient({ + esContext, + savedObjectGetter, + request: FakeRequest(), + }); + }); + afterEach(() => { + jest.resetAllMocks(); + }); + describe('findEventsBySavedObjectIds', () => { test('verifies that the user can access the specified saved object', async () => { - const esContext = contextMock.create(); - const savedObjectGetter = jest.fn(); - - const eventLogClient = new EventLogClient({ - esContext, - savedObjectGetter, - request: FakeRequest(), - }); - - savedObjectGetter.mockResolvedValueOnce({ - id: 'saved-object-id', - type: 'saved-object-type', - attributes: {}, - references: [], - }); - + savedObjectGetter.mockResolvedValueOnce(expectedSavedObject); await eventLogClient.findEventsBySavedObjectIds('saved-object-type', ['saved-object-id']); - expect(savedObjectGetter).toHaveBeenCalledWith('saved-object-type', ['saved-object-id']); }); - test('throws when the user doesnt have permission to access the specified saved object', async () => { - const esContext = contextMock.create(); - - const savedObjectGetter = jest.fn(); - - const eventLogClient = new EventLogClient({ - esContext, - savedObjectGetter, - request: FakeRequest(), - }); - savedObjectGetter.mockRejectedValue(new Error('Fail')); - - expect( + await expect( eventLogClient.findEventsBySavedObjectIds('saved-object-type', ['saved-object-id']) ).rejects.toMatchInlineSnapshot(`[Error: Fail]`); }); - test('fetches all event that reference the saved object', async () => { - const esContext = contextMock.create(); - - const savedObjectGetter = jest.fn(); - - const eventLogClient = new EventLogClient({ - esContext, - savedObjectGetter, - request: FakeRequest(), - }); - - savedObjectGetter.mockResolvedValueOnce({ - id: 'saved-object-id', - type: 'saved-object-type', - attributes: {}, - references: [], - }); - - const expectedEvents = [ - fakeEvent({ - kibana: { - saved_objects: [ - { - id: 'saved-object-id', - type: 'saved-object-type', - }, - { - type: 'action', - id: '1', - }, - ], - }, - }), - fakeEvent({ - kibana: { - saved_objects: [ - { - id: 'saved-object-id', - type: 'saved-object-type', - }, - { - type: 'action', - id: '2', - }, - ], - }, - }), - ]; - + savedObjectGetter.mockResolvedValueOnce(expectedSavedObject); const result = { page: 0, per_page: 10, @@ -109,7 +90,6 @@ describe('EventLogStart', () => { data: expectedEvents, }; esContext.esAdapter.queryEventsBySavedObjects.mockResolvedValue(result); - expect( await eventLogClient.findEventsBySavedObjectIds( 'saved-object-type', @@ -118,7 +98,6 @@ describe('EventLogStart', () => { ['legacy-id'] ) ).toEqual(result); - expect(esContext.esAdapter.queryEventsBySavedObjects).toHaveBeenCalledWith({ index: esContext.esNames.indexPattern, namespace: undefined, @@ -137,56 +116,8 @@ describe('EventLogStart', () => { legacyIds: ['legacy-id'], }); }); - test('fetches all events in time frame that reference the saved object', async () => { - const esContext = contextMock.create(); - - const savedObjectGetter = jest.fn(); - - const eventLogClient = new EventLogClient({ - esContext, - savedObjectGetter, - request: FakeRequest(), - }); - - savedObjectGetter.mockResolvedValueOnce({ - id: 'saved-object-id', - type: 'saved-object-type', - attributes: {}, - references: [], - }); - - const expectedEvents = [ - fakeEvent({ - kibana: { - saved_objects: [ - { - id: 'saved-object-id', - type: 'saved-object-type', - }, - { - type: 'action', - id: '1', - }, - ], - }, - }), - fakeEvent({ - kibana: { - saved_objects: [ - { - id: 'saved-object-id', - type: 'saved-object-type', - }, - { - type: 'action', - id: '2', - }, - ], - }, - }), - ]; - + savedObjectGetter.mockResolvedValueOnce(expectedSavedObject); const result = { page: 0, per_page: 10, @@ -194,10 +125,8 @@ describe('EventLogStart', () => { data: expectedEvents, }; esContext.esAdapter.queryEventsBySavedObjects.mockResolvedValue(result); - const start = moment().subtract(1, 'days').toISOString(); const end = moment().add(1, 'days').toISOString(); - expect( await eventLogClient.findEventsBySavedObjectIds( 'saved-object-type', @@ -209,7 +138,6 @@ describe('EventLogStart', () => { ['legacy-id'] ) ).toEqual(result); - expect(esContext.esAdapter.queryEventsBySavedObjects).toHaveBeenCalledWith({ index: esContext.esNames.indexPattern, namespace: undefined, @@ -230,64 +158,28 @@ describe('EventLogStart', () => { legacyIds: ['legacy-id'], }); }); - test('validates that the start date is valid', async () => { - const esContext = contextMock.create(); - - const savedObjectGetter = jest.fn(); - - const eventLogClient = new EventLogClient({ - esContext, - savedObjectGetter, - request: FakeRequest(), - }); - - savedObjectGetter.mockResolvedValueOnce({ - id: 'saved-object-id', - type: 'saved-object-type', - attributes: {}, - references: [], - }); - + savedObjectGetter.mockResolvedValueOnce(expectedSavedObject); esContext.esAdapter.queryEventsBySavedObjects.mockResolvedValue({ page: 0, per_page: 0, total: 0, data: [], }); - expect( eventLogClient.findEventsBySavedObjectIds('saved-object-type', ['saved-object-id'], { start: 'not a date string', }) ).rejects.toMatchInlineSnapshot(`[Error: [start]: Invalid Date]`); }); - test('validates that the end date is valid', async () => { - const esContext = contextMock.create(); - - const savedObjectGetter = jest.fn(); - - const eventLogClient = new EventLogClient({ - esContext, - savedObjectGetter, - request: FakeRequest(), - }); - - savedObjectGetter.mockResolvedValueOnce({ - id: 'saved-object-id', - type: 'saved-object-type', - attributes: {}, - references: [], - }); - + savedObjectGetter.mockResolvedValueOnce(expectedSavedObject); esContext.esAdapter.queryEventsBySavedObjects.mockResolvedValue({ page: 0, per_page: 0, total: 0, data: [], }); - expect( eventLogClient.findEventsBySavedObjectIds('saved-object-type', ['saved-object-id'], { end: 'not a date string', @@ -295,6 +187,22 @@ describe('EventLogStart', () => { ).rejects.toMatchInlineSnapshot(`[Error: [end]: Invalid Date]`); }); }); + + describe('aggregateEventsBySavedObjectIds', () => { + test('verifies that the user can access the specified saved object', async () => { + savedObjectGetter.mockResolvedValueOnce(expectedSavedObject); + await eventLogClient.aggregateEventsBySavedObjectIds('saved-object-type', [ + 'saved-object-id', + ]); + expect(savedObjectGetter).toHaveBeenCalledWith('saved-object-type', ['saved-object-id']); + }); + test('throws when the user doesnt have permission to access the specified saved object', async () => { + savedObjectGetter.mockRejectedValue(new Error('Fail')); + await expect( + eventLogClient.aggregateEventsBySavedObjectIds('saved-object-type', ['saved-object-id']) + ).rejects.toMatchInlineSnapshot(`[Error: Fail]`); + }); + }); }); function fakeEvent(overrides = {}) { diff --git a/x-pack/plugins/event_log/server/event_log_client.ts b/x-pack/plugins/event_log/server/event_log_client.ts index ca7abc01e47c4..ceaed92e103d1 100644 --- a/x-pack/plugins/event_log/server/event_log_client.ts +++ b/x-pack/plugins/event_log/server/event_log_client.ts @@ -53,6 +53,9 @@ export const queryOptionsSchema = schema.object({ sort: sortFieldsSchema, filter: schema.maybe(schema.string()), }); + +export type QueryOptionsType = Pick, 'start' | 'end' | 'filter'>; + // page & perPage are required, other fields are optional // using schema.maybe allows us to set undefined, but not to make the field optional export type FindOptionsType = Pick< @@ -61,7 +64,7 @@ export type FindOptionsType = Pick< > & Partial>; -export type AggregateOptionsType = Pick, 'sort' | 'filter'> & +export type AggregateOptionsType = Pick, 'filter'> & Partial> & { aggs: Record; }; @@ -87,7 +90,7 @@ export class EventLogClient implements IEventLogClient { this.request = request; } - async findEventsBySavedObjectIds( + public async findEventsBySavedObjectIds( type: string, ids: string[], options?: Partial, @@ -95,15 +98,12 @@ export class EventLogClient implements IEventLogClient { ): Promise { const findOptions = queryOptionsSchema.validate(options ?? {}); - const space = await this.spacesService?.getActiveSpace(this.request); - const namespace = space && this.spacesService?.spaceIdToNamespace(space.id); - - // verify the user has the required permissions to view this saved objects + // verify the user has the required permissions to view this saved object await this.savedObjectGetter(type, ids); return await this.esContext.esAdapter.queryEventsBySavedObjects({ index: this.esContext.esNames.indexPattern, - namespace, + namespace: await this.getNamespace(), type, ids, findOptions, @@ -111,27 +111,29 @@ export class EventLogClient implements IEventLogClient { }); } - async aggregateEventsBySavedObjectIds( + public async aggregateEventsBySavedObjectIds( type: string, ids: string[], - options?: Partial, + options?: AggregateOptionsType, legacyIds?: string[] ) { - // const aggregateOptions = queryOptionsSchema.validate(options ?? {}); + const aggregateOptions = queryOptionsSchema.validate(options ?? {}); - const space = await this.spacesService?.getActiveSpace(this.request); - const namespace = space && this.spacesService?.spaceIdToNamespace(space.id); - - // verify the user has the required permissions to view this saved objects + // verify the user has the required permissions to view this saved object await this.savedObjectGetter(type, ids); return await this.esContext.esAdapter.aggregateEventsBySavedObjects({ index: this.esContext.esNames.indexPattern, - namespace, + namespace: await this.getNamespace(), type, ids, - findOptions: options as AggregateOptionsType, + aggregateOptions: aggregateOptions as AggregateOptionsType, legacyIds, }); } + + private async getNamespace() { + const space = await this.spacesService?.getActiveSpace(this.request); + return space && this.spacesService?.spaceIdToNamespace(space.id); + } } diff --git a/x-pack/plugins/event_log/server/types.ts b/x-pack/plugins/event_log/server/types.ts index 2b6b4584af06b..df4dec55b069a 100644 --- a/x-pack/plugins/event_log/server/types.ts +++ b/x-pack/plugins/event_log/server/types.ts @@ -56,7 +56,7 @@ export interface IEventLogClient { options?: Partial, legacyIds?: string[] ): Promise<{ - aggregations: Record | undefined; + aggregateResults: Record | undefined; }>; } From 9d6e606927ffd438e1a4cbd21316512b765bab02 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Mon, 7 Mar 2022 15:57:52 -0500 Subject: [PATCH 07/39] Adding tests --- .../server/es/cluster_client_adapter.test.ts | 101 +++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts index 93580ceca9911..435f1680f9fd6 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts @@ -12,8 +12,9 @@ import { EVENT_BUFFER_LENGTH, getQueryBody, FindEventsOptionsBySavedObjectFilter, + AggregateEventsOptionsBySavedObjectFilter, } from './cluster_client_adapter'; -import { queryOptionsSchema } from '../event_log_client'; +import { AggregateOptionsType, queryOptionsSchema } from '../event_log_client'; import { delay } from '../lib/delay'; import { pick, times } from 'lodash'; import type * as estypes from '@elastic/elasticsearch/lib/api/types'; @@ -623,6 +624,104 @@ describe('queryEventsBySavedObject', () => { }); }); +describe('aggregateEventsBySavedObject', () => { + const DEFAULT_OPTIONS = { + ...queryOptionsSchema.validate({}), + aggs: { + genericAgg: { + term: { + field: 'event.action', + size: 10, + }, + }, + }, + }; + + test('should call cluster with correct options', async () => { + clusterClient.search.mockResponse({ + aggregations: { + genericAgg: { + buckets: [ + { + key: 'execute', + doc_count: 10, + }, + { + key: 'execute-start', + doc_count: 10, + }, + { + key: 'new-instance', + doc_count: 2, + }, + ], + }, + }, + hits: { + hits: [], + total: { relation: 'eq', value: 0 }, + }, + took: 0, + timed_out: false, + _shards: { + failed: 0, + successful: 0, + total: 0, + skipped: 0, + }, + }); + const options: AggregateEventsOptionsBySavedObjectFilter = { + index: 'index-name', + namespace: 'namespace', + type: 'saved-object-type', + ids: ['saved-object-id'], + aggregateOptions: DEFAULT_OPTIONS as AggregateOptionsType, + }; + const result = await clusterClientAdapter.aggregateEventsBySavedObjects(options); + + const [query] = clusterClient.search.mock.calls[0]; + expect(query).toEqual({ + index: 'index-name', + body: { + size: 0, + query: getQueryBody( + logger, + options, + pick(options.aggregateOptions, ['start', 'end', 'filter']) + ), + aggs: { + genericAgg: { + term: { + field: 'event.action', + size: 10, + }, + }, + }, + }, + }); + expect(result).toEqual({ + aggregateResults: { + genericAgg: { + buckets: [ + { + key: 'execute', + doc_count: 10, + }, + { + key: 'execute-start', + doc_count: 10, + }, + { + key: 'new-instance', + doc_count: 2, + }, + ], + }, + }, + }); + }); +}); + describe('getQueryBody', () => { const options = { index: 'index-name', From 63e8ebff4ca729dde010533b6b45cb666cc8e89b Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Mon, 7 Mar 2022 19:05:07 -0500 Subject: [PATCH 08/39] Fixing functional test --- .../test_suites/event_log/public_api_integration.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/test/plugin_api_integration/test_suites/event_log/public_api_integration.ts b/x-pack/test/plugin_api_integration/test_suites/event_log/public_api_integration.ts index 978123860499c..903b1a606dd7e 100644 --- a/x-pack/test/plugin_api_integration/test_suites/event_log/public_api_integration.ts +++ b/x-pack/test/plugin_api_integration/test_suites/event_log/public_api_integration.ts @@ -97,7 +97,9 @@ export default function ({ getService }: FtrProviderContext) { await retry.try(async () => { const { body: { data: foundEvents }, - } = await findEvents(namespace, id, { sort_field: 'event.end', sort_order: 'desc' }); + } = await findEvents(namespace, id, { + sort: [{ sort_field: 'event.end', sort_order: 'desc' }], + }); expect(foundEvents.length).to.be(expectedEvents.length); assertEventsFromApiMatchCreatedEvents(foundEvents, expectedEvents.reverse()); From c22c460718840bb210387da816d9d1f2f1544d19 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Mon, 7 Mar 2022 20:58:06 -0500 Subject: [PATCH 09/39] Fixing functional test --- .../test_suites/event_log/public_api_integration.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/test/plugin_api_integration/test_suites/event_log/public_api_integration.ts b/x-pack/test/plugin_api_integration/test_suites/event_log/public_api_integration.ts index 903b1a606dd7e..5d809cfbe60c0 100644 --- a/x-pack/test/plugin_api_integration/test_suites/event_log/public_api_integration.ts +++ b/x-pack/test/plugin_api_integration/test_suites/event_log/public_api_integration.ts @@ -243,7 +243,9 @@ export default function ({ getService }: FtrProviderContext) { isEmpty(query) ? '' : `?${Object.entries(query) - .map(([key, val]) => `${key}=${val}`) + .map(([key, val]) => + typeof val === 'object' ? `${key}=${JSON.stringify(val)}` : `${key}=${val}` + ) .join('&')}` }`; await delay(1000); // wait for buffer to be written From 6c87182b9dc8bf32dfb4176d30c9a2877b4672e4 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Tue, 8 Mar 2022 09:55:14 -0500 Subject: [PATCH 10/39] Revert "Reverting changes not related to event log aggregation" This reverts commit 939340e252f36796f272f663e7e75f275772cf84. --- .../authorization/alerting_authorization.ts | 1 + .../routes/get_rule_execution_log.test.ts | 117 ++++++++++++++++++ .../plugins/alerting/server/routes/index.ts | 2 + .../server/rules_client/rules_client.ts | 86 +++++++++++++ .../feature_privilege_builder/alerting.ts | 2 +- 5 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts diff --git a/x-pack/plugins/alerting/server/authorization/alerting_authorization.ts b/x-pack/plugins/alerting/server/authorization/alerting_authorization.ts index aafeb5003eaab..a079594997521 100644 --- a/x-pack/plugins/alerting/server/authorization/alerting_authorization.ts +++ b/x-pack/plugins/alerting/server/authorization/alerting_authorization.ts @@ -30,6 +30,7 @@ export enum ReadOperations { Get = 'get', GetRuleState = 'getRuleState', GetAlertSummary = 'getAlertSummary', + GetExecutionLog = 'getExecutionLog', Find = 'find', } diff --git a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts new file mode 100644 index 0000000000000..4193e769dc513 --- /dev/null +++ b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getRuleAlertSummaryRoute } from './get_rule_alert_summary'; +import { httpServiceMock } from 'src/core/server/mocks'; +import { licenseStateMock } from '../lib/license_state.mock'; +import { mockHandlerArguments } from './_mock_handler_arguments'; +import { SavedObjectsErrorHelpers } from 'src/core/server'; +import { rulesClientMock } from '../rules_client.mock'; +import { AlertSummary } from '../types'; + +const rulesClient = rulesClientMock.create(); +jest.mock('../lib/license_api_access.ts', () => ({ + verifyApiAccess: jest.fn(), +})); + +beforeEach(() => { + jest.resetAllMocks(); +}); + +describe('getRuleAlertSummaryRoute', () => { + const dateString = new Date().toISOString(); + const mockedAlertSummary: AlertSummary = { + id: '', + name: '', + tags: [], + ruleTypeId: '', + consumer: '', + muteAll: false, + throttle: null, + enabled: false, + statusStartDate: dateString, + statusEndDate: dateString, + status: 'OK', + errorMessages: [], + alerts: {}, + executionDuration: { + average: 1, + valuesWithTimestamp: { + '17 Nov 2021 @ 19:19:17': 3, + '18 Nov 2021 @ 19:19:17': 5, + '19 Nov 2021 @ 19:19:17': 5, + }, + }, + }; + + it('gets rule alert summary', async () => { + const licenseState = licenseStateMock.create(); + const router = httpServiceMock.createRouter(); + + getRuleAlertSummaryRoute(router, licenseState); + + const [config, handler] = router.get.mock.calls[0]; + + expect(config.path).toMatchInlineSnapshot(`"/internal/alerting/rule/{id}/_alert_summary"`); + + rulesClient.getAlertSummary.mockResolvedValueOnce(mockedAlertSummary); + + const [context, req, res] = mockHandlerArguments( + { rulesClient }, + { + params: { + id: '1', + }, + query: {}, + }, + ['ok'] + ); + + await handler(context, req, res); + + expect(rulesClient.getAlertSummary).toHaveBeenCalledTimes(1); + expect(rulesClient.getAlertSummary.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + Object { + "dateStart": undefined, + "id": "1", + "numberOfExecutions": undefined, + }, + ] + `); + + expect(res.ok).toHaveBeenCalled(); + }); + + it('returns NOT-FOUND when rule is not found', async () => { + const licenseState = licenseStateMock.create(); + const router = httpServiceMock.createRouter(); + + getRuleAlertSummaryRoute(router, licenseState); + + const [, handler] = router.get.mock.calls[0]; + + rulesClient.getAlertSummary = jest + .fn() + .mockRejectedValueOnce(SavedObjectsErrorHelpers.createGenericNotFoundError('alert', '1')); + + const [context, req, res] = mockHandlerArguments( + { rulesClient }, + { + params: { + id: '1', + }, + query: {}, + }, + ['notFound'] + ); + + expect(handler(context, req, res)).rejects.toMatchInlineSnapshot( + `[Error: Saved object [alert/1] not found]` + ); + }); +}); diff --git a/x-pack/plugins/alerting/server/routes/index.ts b/x-pack/plugins/alerting/server/routes/index.ts index f43190ec6d1c2..18fcf0539f128 100644 --- a/x-pack/plugins/alerting/server/routes/index.ts +++ b/x-pack/plugins/alerting/server/routes/index.ts @@ -20,6 +20,7 @@ import { disableRuleRoute } from './disable_rule'; import { enableRuleRoute } from './enable_rule'; import { findRulesRoute, findInternalRulesRoute } from './find_rules'; import { getRuleAlertSummaryRoute } from './get_rule_alert_summary'; +import { getRuleExecutionLogRoute } from './get_rule_execution_log'; import { getRuleStateRoute } from './get_rule_state'; import { healthRoute } from './health'; import { resolveRuleRoute } from './resolve_rule'; @@ -53,6 +54,7 @@ export function defineRoutes(opts: RouteOptions) { findRulesRoute(router, licenseState, usageCounter); findInternalRulesRoute(router, licenseState, usageCounter); getRuleAlertSummaryRoute(router, licenseState); + getRuleExecutionLogRoute(router, licenseState); getRuleStateRoute(router, licenseState); healthRoute(router, licenseState, encryptedSavedObjects); ruleTypesRoute(router, licenseState); diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index 1512959384ac9..bc0a7c82119e0 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -229,6 +229,22 @@ export interface GetAlertSummaryParams { numberOfExecutions?: number; } +export interface GetExecutionLogByIdParams { + id: string; + dateStart?: string; + // dateEnd? +} + +export interface ExecutionLog { + count: number; + values: Array<{ + timestamp: number; + duration: number; + status: 'failed' | 'succeeded'; + message: string; + }>; +} + // NOTE: Changing this prefix will require a migration to update the prefix in all existing `rule` saved objects const extractedSavedObjectParamReferenceNamePrefix = 'param:'; @@ -631,6 +647,76 @@ export class RulesClient { }); } + public async getExecutionLogForRule({ + id, + dateStart, + }: // dateEnd? + GetExecutionLogByIdParams): Promise { + this.logger.debug(`getExecutionLogForRule(): getting execution log for rule ${id}`); + const rule = (await this.get({ id, includeLegacyId: true })) as SanitizedRuleWithLegacyId; + + // Make sure user has access to this rule + await this.authorization.ensureAuthorized({ + ruleTypeId: rule.alertTypeId, + consumer: rule.consumer, + operation: ReadOperations.GetAlertSummary, + entity: AlertingAuthorizationEntity.Rule, + }); + + // default duration of instance summary is 60 * rule interval + const dateNow = new Date(); + const durationMillis = parseDuration(rule.schedule.interval) * (numberOfExecutions ?? 60); + const defaultDateStart = new Date(dateNow.valueOf() - durationMillis); + const parsedDateStart = parseDate(dateStart, 'dateStart', defaultDateStart); + + const eventLogClient = await this.getEventLogClient(); + + let events: IEvent[]; + let executionEvents: IEvent[]; + + try { + const results = await eventLogClient.aggregateEventsBySavedObjectIds( + 'alert', + [id], + {}, + rule.legacyId !== null ? [rule.legacyId] : undefined + ); + } catch (err) { + this.logger.debug( + `rulesClient.getExecutionLogForRule(): error searching event log for rule ${id}: ${err.message}` + ); + } + + // desired output + // { + // count: number; + // values: [ + // { + // // this is shown in the screenshot + // executionTime: timestamp, + // duration: millis, + // status: failed/succeeded, + // message: log message - execution timeout is separate event log entry + + // // other stuff that might be useful that we could probably get + // number of active alerts + // number of new alerts + // number of recovered alerts + // number of triggered actions + // any errors in the actions + + // es search duration + // total search duration + + // } + // ] + // } + return { + count: 0, + values: [], + }; + } + public async find({ options: { fields, ...options } = {}, excludeFromPublicApi = false, diff --git a/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.ts b/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.ts index f536959a910cd..4d2cc97f75d89 100644 --- a/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.ts +++ b/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.ts @@ -16,7 +16,7 @@ enum AlertingEntity { } const readOperations: Record = { - rule: ['get', 'getRuleState', 'getAlertSummary', 'find'], + rule: ['get', 'getRuleState', 'getAlertSummary', 'getExecutionLog', 'find'], alert: ['get', 'find'], }; From 2f8a87c254ebbacb17ddd5ea1690d0dd20098926 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Tue, 8 Mar 2022 10:30:59 -0500 Subject: [PATCH 11/39] Revert "Reverting changes not related to event log aggregation" This reverts commit 40a93a4b3c02de15d3fb9448f830dffa385a54d0. --- .../server/routes/get_rule_execution_log.ts | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts diff --git a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts new file mode 100644 index 0000000000000..ef8458f97231b --- /dev/null +++ b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IRouter } from 'kibana/server'; +import { schema } from '@kbn/config-schema'; +import { ILicenseState } from '../lib'; +import { GetAlertSummaryParams } from '../rules_client'; +import { RewriteRequestCase, RewriteResponseCase, verifyAccessAndContext } from './lib'; +import { + AlertingRequestHandlerContext, + INTERNAL_BASE_ALERTING_API_PATH, + AlertSummary, +} from '../types'; + +const paramSchema = schema.object({ + id: schema.string(), +}); + +const querySchema = schema.object({ + date_start: schema.maybe(schema.string()), + date_end: schema.maybe(schema.string()), +}); + +const rewriteReq: RewriteRequestCase = ({ + date_start: dateStart, + number_of_executions: numberOfExecutions, + ...rest +}) => ({ + ...rest, + numberOfExecutions, + dateStart, +}); + +const rewriteBodyRes: RewriteResponseCase = ({ + ruleTypeId, + muteAll, + statusStartDate, + statusEndDate, + errorMessages, + lastRun, + executionDuration: { valuesWithTimestamp, ...executionDuration }, + ...rest +}) => ({ + ...rest, + rule_type_id: ruleTypeId, + mute_all: muteAll, + status_start_date: statusStartDate, + status_end_date: statusEndDate, + error_messages: errorMessages, + last_run: lastRun, + execution_duration: { + ...executionDuration, + values_with_timestamp: valuesWithTimestamp, + }, +}); + +export const getRuleExecutionLogRoute = ( + router: IRouter, + licenseState: ILicenseState +) => { + router.get( + { + path: `${INTERNAL_BASE_ALERTING_API_PATH}/rule/{id}/_execution_log`, + validate: { + params: paramSchema, + query: querySchema, + }, + }, + router.handleLegacyErrors( + verifyAccessAndContext(licenseState, async function (context, req, res) { + const rulesClient = context.alerting.getRulesClient(); + const { id } = req.params; + const execLog = await rulesClient.getExecutionLogForRule(rewriteReq({ id, ...req.query })); + return res.ok({ body: rewriteBodyRes(execLog) }); + }) + ) + ); +}; From a03ce1df4bc03b6a60b706bd7e6d309550b914c0 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Tue, 8 Mar 2022 15:04:46 -0500 Subject: [PATCH 12/39] Getting aggregation and parsing aggregation results --- .../lib/get_execution_log_aggregation.test.ts | 549 ++++++++++++++++++ .../lib/get_execution_log_aggregation.ts | 261 +++++++++ 2 files changed, 810 insertions(+) create mode 100644 x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts create mode 100644 x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts new file mode 100644 index 0000000000000..1585edf192bca --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts @@ -0,0 +1,549 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + getExecutionLogAggregation, + formatExecutionLogResult, +} from './get_execution_log_aggregation'; +import { AggregateEventsBySavedObjectResult } from '../../../event_log/server'; + +describe('getExecutionLogAggregation', () => { + test('should throw error when given bad sort field', () => { + expect(() => { + getExecutionLogAggregation({ + numExecutions: 5, + page: 1, + perPage: 10, + sortField: 'notsortable', + sortOrder: 'asc', + }); + }).toThrowErrorMatchingInlineSnapshot( + `"Invalid sort field \\"notsortable\\" - must be one of [timestamp,duration]"` + ); + }); + + test('should throw error when given bad page field', () => { + expect(() => { + getExecutionLogAggregation({ + numExecutions: 5, + page: 0, + perPage: 10, + sortField: 'timestamp', + sortOrder: 'asc', + }); + }).toThrowErrorMatchingInlineSnapshot(`"Invalid page field \\"0\\" - must be greater than 0"`); + }); + + test('should correctly generate aggregation', () => { + expect( + getExecutionLogAggregation({ + numExecutions: 5, + page: 2, + perPage: 10, + sortField: 'timestamp', + sortOrder: 'asc', + }) + ).toEqual({ + executionUuidCardinality: { cardinality: { field: 'kibana.alert.rule.execution.uuid' } }, + executionUuid: { + terms: { field: 'kibana.alert.rule.execution.uuid', size: 5 }, + aggs: { + executionUuidSorted: { + bucket_sort: { + sort: [{ 'ruleExecution>executeStartTime': { order: 'asc' } }], + from: 10, + size: 10, + }, + }, + alertCounts: { + filters: { + filters: { + newAlerts: { match: { 'event.action': 'new-instance' } }, + activeAlerts: { match: { 'event.action': 'active-instance' } }, + recoveredAlerts: { match: { 'event.action': 'recovered-instance' } }, + }, + }, + }, + actionExecution: { + filter: { + bool: { + must: [ + { match: { 'event.action': 'execute' } }, + { match: { 'event.provider': 'actions' } }, + ], + }, + }, + aggs: { actionOutcomes: { terms: { field: 'event.outcome', size: 2 } } }, + }, + ruleExecution: { + filter: { + bool: { + must: [ + { match: { 'event.action': 'execute' } }, + { match: { 'event.provider': 'alerting' } }, + ], + }, + }, + aggs: { + executeStartTime: { min: { field: 'event.start' } }, + totalSearchDuration: { + max: { field: 'kibana.alert.rule.execution.metrics.total_search_duration_ms' }, + }, + esSearchDuration: { + max: { field: 'kibana.alert.rule.execution.metrics.es_search_duration_ms' }, + }, + numTriggeredActions: { + max: { field: 'kibana.alert.rule.execution.metrics.number_of_triggered_actions' }, + }, + executionDuration: { max: { field: 'event.duration' } }, + outcomeAndMessage: { + top_hits: { size: 1, _source: { includes: ['event.outcome', 'message'] } }, + }, + }, + }, + timeoutMessage: { + filter: { + bool: { + must: [ + { match: { 'event.action': 'execute-timeout' } }, + { match: { 'event.provider': 'alerting' } }, + ], + }, + }, + }, + }, + }, + }); + }); +}); + +describe('formatExecutionLogResult', () => { + test('should format results correctly', () => { + const results = { + aggregateResults: { + executionUuid: { + meta: {}, + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: '6705da7d-2635-499d-a6a8-1aee1ae1eac9', + doc_count: 27, + timeoutMessage: { + meta: {}, + doc_count: 0, + }, + alertCounts: { + meta: {}, + buckets: { + activeAlerts: { + doc_count: 5, + }, + newAlerts: { + doc_count: 5, + }, + recoveredAlerts: { + doc_count: 0, + }, + }, + }, + ruleExecution: { + meta: {}, + doc_count: 1, + numTriggeredActions: { + value: 5.0, + }, + outcomeAndMessage: { + hits: { + total: { + value: 1, + relation: 'eq', + }, + max_score: 1.0, + hits: [ + { + _index: '.kibana-event-log-8.2.0-000001', + _id: 'S4wIZX8B8TGQpG7XQZns', + _score: 1.0, + _source: { + event: { + outcome: 'success', + }, + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + }, + }, + ], + }, + }, + totalSearchDuration: { + value: 0.0, + }, + esSearchDuration: { + value: 0.0, + }, + executionDuration: { + value: 1.056e9, + }, + executeStartTime: { + value: 1.646667512617e12, + value_as_string: '2022-03-07T15:38:32.617Z', + }, + }, + actionExecution: { + meta: {}, + doc_count: 5, + actionOutcomes: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: 'success', + doc_count: 5, + }, + ], + }, + }, + }, + { + key: '41b2755e-765a-4044-9745-b03875d5e79a', + doc_count: 32, + timeoutMessage: { + meta: {}, + doc_count: 0, + }, + alertCounts: { + meta: {}, + buckets: { + activeAlerts: { + doc_count: 5, + }, + newAlerts: { + doc_count: 5, + }, + recoveredAlerts: { + doc_count: 5, + }, + }, + }, + ruleExecution: { + meta: {}, + doc_count: 1, + numTriggeredActions: { + value: 5.0, + }, + outcomeAndMessage: { + hits: { + total: { + value: 1, + relation: 'eq', + }, + max_score: 1.0, + hits: [ + { + _index: '.kibana-event-log-8.2.0-000001', + _id: 'a4wIZX8B8TGQpG7Xwpnz', + _score: 1.0, + _source: { + event: { + outcome: 'success', + }, + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + }, + }, + ], + }, + }, + totalSearchDuration: { + value: 0.0, + }, + esSearchDuration: { + value: 0.0, + }, + executionDuration: { + value: 1.165e9, + }, + executeStartTime: { + value: 1.646667545604e12, + value_as_string: '2022-03-07T15:39:05.604Z', + }, + }, + actionExecution: { + meta: {}, + doc_count: 5, + actionOutcomes: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: 'success', + doc_count: 5, + }, + ], + }, + }, + }, + ], + }, + executionUuidCardinality: { + value: 374, + }, + }, + }; + expect(formatExecutionLogResult(results)).toEqual({ + total: 374, + data: [ + { + id: '6705da7d-2635-499d-a6a8-1aee1ae1eac9', + timestamp: '2022-03-07T15:38:32.617Z', + duration_ms: 1056, + status: 'success', + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + num_active_alerts: 5, + num_new_alerts: 5, + num_recovered_alerts: 0, + num_triggered_actions: 5, + num_succeeded_actions: 5, + num_errored_actions: 0, + total_search_duration_ms: 0, + es_search_duration_ms: 0, + timed_out: false, + }, + { + id: '41b2755e-765a-4044-9745-b03875d5e79a', + timestamp: '2022-03-07T15:39:05.604Z', + duration_ms: 1165, + status: 'success', + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + num_active_alerts: 5, + num_new_alerts: 5, + num_recovered_alerts: 5, + num_triggered_actions: 5, + num_succeeded_actions: 5, + num_errored_actions: 0, + total_search_duration_ms: 0, + es_search_duration_ms: 0, + timed_out: false, + }, + ], + }); + }); + + test('should format results correctly when execution timeouts occur', () => { + const results = { + aggregateResults: { + executionUuid: { + meta: {}, + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: '09b5aeab-d50d-43b2-88e7-f1a20f682b3f', + doc_count: 3, + timeoutMessage: { + meta: {}, + doc_count: 1, + }, + alertCounts: { + meta: {}, + buckets: { + activeAlerts: { + doc_count: 0, + }, + newAlerts: { + doc_count: 0, + }, + recoveredAlerts: { + doc_count: 0, + }, + }, + }, + ruleExecution: { + meta: {}, + doc_count: 1, + numTriggeredActions: { + value: 0.0, + }, + outcomeAndMessage: { + hits: { + total: { + value: 1, + relation: 'eq', + }, + max_score: 1.0, + hits: [ + { + _index: '.kibana-event-log-8.2.0-000001', + _id: 'dJkWa38B1ylB1EvsAckB', + _score: 1.0, + _source: { + event: { + outcome: 'success', + }, + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + }, + }, + ], + }, + }, + totalSearchDuration: { + value: 0.0, + }, + esSearchDuration: { + value: 0.0, + }, + executionDuration: { + value: 1.0279e10, + }, + executeStartTime: { + value: 1.646769067607e12, + value_as_string: '2022-03-08T19:51:07.607Z', + }, + }, + actionExecution: { + meta: {}, + doc_count: 0, + actionOutcomes: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [], + }, + }, + }, + { + key: '41b2755e-765a-4044-9745-b03875d5e79a', + doc_count: 32, + timeoutMessage: { + meta: {}, + doc_count: 0, + }, + alertCounts: { + meta: {}, + buckets: { + activeAlerts: { + doc_count: 5, + }, + newAlerts: { + doc_count: 5, + }, + recoveredAlerts: { + doc_count: 5, + }, + }, + }, + ruleExecution: { + meta: {}, + doc_count: 1, + numTriggeredActions: { + value: 5.0, + }, + outcomeAndMessage: { + hits: { + total: { + value: 1, + relation: 'eq', + }, + max_score: 1.0, + hits: [ + { + _index: '.kibana-event-log-8.2.0-000001', + _id: 'a4wIZX8B8TGQpG7Xwpnz', + _score: 1.0, + _source: { + event: { + outcome: 'success', + }, + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + }, + }, + ], + }, + }, + totalSearchDuration: { + value: 0.0, + }, + esSearchDuration: { + value: 0.0, + }, + executionDuration: { + value: 1.165e9, + }, + executeStartTime: { + value: 1.646667545604e12, + value_as_string: '2022-03-07T15:39:05.604Z', + }, + }, + actionExecution: { + meta: {}, + doc_count: 5, + actionOutcomes: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: 'success', + doc_count: 5, + }, + ], + }, + }, + }, + ], + }, + executionUuidCardinality: { + value: 374, + }, + }, + }; + expect(formatExecutionLogResult(results)).toEqual({ + total: 374, + data: [ + { + id: '09b5aeab-d50d-43b2-88e7-f1a20f682b3f', + timestamp: '2022-03-08T19:51:07.607Z', + duration_ms: 10279, + status: 'success', + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + num_active_alerts: 0, + num_new_alerts: 0, + num_recovered_alerts: 0, + num_triggered_actions: 0, + num_succeeded_actions: 0, + num_errored_actions: 0, + total_search_duration_ms: 0, + es_search_duration_ms: 0, + timed_out: true, + }, + { + id: '41b2755e-765a-4044-9745-b03875d5e79a', + timestamp: '2022-03-07T15:39:05.604Z', + duration_ms: 1165, + status: 'success', + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + num_active_alerts: 5, + num_new_alerts: 5, + num_recovered_alerts: 5, + num_triggered_actions: 5, + num_succeeded_actions: 5, + num_errored_actions: 0, + total_search_duration_ms: 0, + es_search_duration_ms: 0, + timed_out: false, + }, + ], + }); + }); + + test('should format results correctly when action errors occur', () => {}); +}); diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts new file mode 100644 index 0000000000000..ab4ed19fcce86 --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts @@ -0,0 +1,261 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import Boom from '@hapi/boom'; +import { AggregateEventsBySavedObjectResult } from '../../../event_log/server'; + +const PROVIDER_FIELD = 'event.provider'; +const START_FIELD = 'event.start'; +const ACTION_FIELD = 'event.action'; +const OUTCOME_FIELD = 'event.outcome'; +const DURATION_FIELD = 'event.duration'; +const MESSAGE_FIELD = 'message'; +const ES_SEARCH_DURATION_FIELD = 'kibana.alert.rule.execution.metrics.es_search_duration_ms'; +const TOTAL_SEARCH_DURATION_FIELD = 'kibana.alert.rule.execution.metrics.total_search_duration_ms'; +const NUMBER_OF_TRIGGERED_ACTIONS_FIELD = + 'kibana.alert.rule.execution.metrics.number_of_triggered_actions'; +const EXECUTION_UUID_FIELD = 'kibana.alert.rule.execution.uuid'; + +const Millis2Nanos = 1000 * 1000; + +export interface IExecutionLog { + id: string; + timestamp: string; + duration_ms: number | null; + status: string; + message: string; + num_active_alerts: number; + num_new_alerts: number; + num_recovered_alerts: number; + num_triggered_actions: number | null; + num_succeeded_actions: number; + num_errored_actions: number; + total_search_duration_ms: number | null; + es_search_duration_ms: number | null; + timed_out: boolean; +} + +interface IAlertCounts extends estypes.AggregationsMultiBucketAggregateBase { + buckets: { + activeAlerts: estypes.AggregationsSingleBucketAggregateBase; + newAlerts: estypes.AggregationsSingleBucketAggregateBase; + recoveredAlerts: estypes.AggregationsSingleBucketAggregateBase; + }; +} + +interface IActionExecution + extends estypes.AggregationsTermsAggregateBase<{ key: string; doc_count: number }> { + buckets: Array<{ key: string; doc_count: number }>; +} + +interface IBucketAggregationResult extends estypes.AggregationsStringTermsBucketKeys { + timeoutMessage: estypes.AggregationsMultiBucketBase; + ruleExecution: { + executeStartTime: estypes.AggregationsMinAggregate; + executionDuration: estypes.AggregationsMaxAggregate; + esSearchDuration: estypes.AggregationsMaxAggregate; + totalSearchDuration: estypes.AggregationsMaxAggregate; + numTriggeredActions: estypes.AggregationsMaxAggregate; + outcomeAndMessage: estypes.AggregationsTopHitsAggregate; + }; + alertCounts: IAlertCounts; + actionExecution: { + actionOutcomes: IActionExecution; + }; +} + +export interface IExecutionLogAggOptions { + numExecutions: number; + page: number; + perPage: number; + sortField: string; + sortOrder: estypes.SortOrder; +} + +const ExecutionLogSortFields: Record = { + timestamp: 'ruleExecution>executeStartTime', + duration: 'ruleExecution>executionDuration', +}; + +export function getExecutionLogAggregation({ + numExecutions, + page, + perPage, + sortField, + sortOrder, +}: IExecutionLogAggOptions) { + // Check if valid sort field + if (!Object.keys(ExecutionLogSortFields).includes(sortField)) { + throw Boom.badRequest( + `Invalid sort field "${sortField}" - must be one of [${Object.keys( + ExecutionLogSortFields + ).join(',')}]` + ); + } + + // Check if valid page value + if (page <= 0) { + throw Boom.badRequest(`Invalid page field "${page}" - must be greater than 0`); + } + + return { + // Get total number of executions + executionUuidCardinality: { + cardinality: { + field: EXECUTION_UUID_FIELD, + }, + }, + executionUuid: { + // Bucket by execution UUID + terms: { + field: EXECUTION_UUID_FIELD, + size: numExecutions, + }, + aggs: { + // Bucket sort to allow paging through executions + executionUuidSorted: { + bucket_sort: { + sort: [ + { + [ExecutionLogSortFields[sortField]]: { + order: sortOrder, + }, + }, + ], + from: (page - 1) * perPage, + size: perPage, + }, + }, + // Get counts for types of alerts and whether there was an execution timeout + alertCounts: { + filters: { + filters: { + newAlerts: { match: { [ACTION_FIELD]: 'new-instance' } }, + activeAlerts: { match: { [ACTION_FIELD]: 'active-instance' } }, + recoveredAlerts: { match: { [ACTION_FIELD]: 'recovered-instance' } }, + }, + }, + }, + // Filter by action execute doc and get information from this event + actionExecution: { + filter: getProviderAndActionFilter('actions', 'execute'), + aggs: { + actionOutcomes: { + terms: { + field: OUTCOME_FIELD, + size: 2, + }, + }, + }, + }, + // Filter by rule execute doc and get information from this event + ruleExecution: { + filter: getProviderAndActionFilter('alerting', 'execute'), + aggs: { + executeStartTime: { + min: { + field: START_FIELD, + }, + }, + totalSearchDuration: { + max: { + field: TOTAL_SEARCH_DURATION_FIELD, + }, + }, + esSearchDuration: { + max: { + field: ES_SEARCH_DURATION_FIELD, + }, + }, + numTriggeredActions: { + max: { + field: NUMBER_OF_TRIGGERED_ACTIONS_FIELD, + }, + }, + executionDuration: { + max: { + field: DURATION_FIELD, + }, + }, + outcomeAndMessage: { + top_hits: { + size: 1, + _source: { + includes: [OUTCOME_FIELD, MESSAGE_FIELD], + }, + }, + }, + }, + }, + // If there was a timeout, this filter will return non-zero + timeoutMessage: { + filter: getProviderAndActionFilter('alerting', 'execute-timeout'), + }, + }, + }, + }; +} + +function getProviderAndActionFilter(provider: string, action: string) { + return { + bool: { + must: [ + { + match: { + [ACTION_FIELD]: action, + }, + }, + { + match: { + [PROVIDER_FIELD]: provider, + }, + }, + ], + }, + }; +} + +export function formatExecutionLogAggBucket(bucketVal: IBucketAggregationResult): IExecutionLog { + const durationUs = bucketVal.ruleExecution.executionDuration.value + ? bucketVal.ruleExecution.executionDuration.value + : 0; + const timedOut = bucketVal.timeoutMessage.doc_count > 0; + + const actionExecutionOutcomes = bucketVal.actionExecution.actionOutcomes.buckets; + const actionExecutionSuccess = + actionExecutionOutcomes.find((bucket) => bucket.key === 'success')?.doc_count ?? 0; + const actionExecutionError = + actionExecutionOutcomes.find((bucket) => bucket.key === 'failure')?.doc_count ?? 0; + return { + id: bucketVal.key, + timestamp: bucketVal.ruleExecution.executeStartTime.value_as_string!, + duration_ms: durationUs / Millis2Nanos, + status: bucketVal.ruleExecution.outcomeAndMessage.hits.hits[0]?._source?.event?.outcome, + message: bucketVal.ruleExecution.outcomeAndMessage.hits.hits[0]?._source?.message, + num_active_alerts: bucketVal.alertCounts.buckets.activeAlerts.doc_count, + num_new_alerts: bucketVal.alertCounts.buckets.newAlerts.doc_count, + num_recovered_alerts: bucketVal.alertCounts.buckets.recoveredAlerts.doc_count, + num_triggered_actions: bucketVal.ruleExecution.numTriggeredActions.value, + num_succeeded_actions: actionExecutionSuccess, + num_errored_actions: actionExecutionError, + total_search_duration_ms: bucketVal.ruleExecution.totalSearchDuration.value, + es_search_duration_ms: bucketVal.ruleExecution.esSearchDuration.value, + timed_out: timedOut, + }; +} + +export function formatExecutionLogResult(results: AggregateEventsBySavedObjectResult) { + const { aggregateResults } = results; + const total = aggregateResults.executionUuidCardinality.value; + const buckets: IBucketAggregationResult[] = aggregateResults.executionUuid.buckets; + + return { + total, + data: buckets.map((bucket: IBucketAggregationResult) => formatExecutionLogAggBucket(bucket)), + }; +} From 08d52cac14766e9a1d362d846176c19b304e6890 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Tue, 8 Mar 2022 15:05:23 -0500 Subject: [PATCH 13/39] Cleanup --- .../plugins/event_log/server/es/cluster_client_adapter.ts | 8 +++++--- x-pack/plugins/event_log/server/index.ts | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index d7fe22d6b4ca3..4249274816ba4 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -58,6 +58,10 @@ export type AggregateEventsOptionsBySavedObjectFilter = QueryOptionsEventsBySave aggregateOptions: AggregateOptionsType; }; +export interface AggregateEventsBySavedObjectResult { + aggregateResults: Record | undefined; +} + // eslint-disable-next-line @typescript-eslint/no-explicit-any type AliasAny = any; @@ -379,9 +383,7 @@ export class ClusterClientAdapter | undefined; - }> { + ): Promise { const { index, type, ids, aggregateOptions } = queryOptions; const { aggs } = aggregateOptions; diff --git a/x-pack/plugins/event_log/server/index.ts b/x-pack/plugins/event_log/server/index.ts index 877c39a02edc5..42fc2e9792014 100644 --- a/x-pack/plugins/event_log/server/index.ts +++ b/x-pack/plugins/event_log/server/index.ts @@ -17,6 +17,7 @@ export type { IValidatedEvent, IEventLogClient, QueryEventsBySavedObjectResult, + AggregateEventsBySavedObjectResult, } from './types'; export { SAVED_OBJECT_REL_PRIMARY } from './types'; From bf45ff29a203881d82892cd2a3d401fa0f239e50 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Tue, 8 Mar 2022 15:07:07 -0500 Subject: [PATCH 14/39] Changing api to internal --- x-pack/plugins/event_log/common/index.ts | 2 +- x-pack/plugins/event_log/server/routes/find.test.ts | 2 +- x-pack/plugins/event_log/server/routes/find_by_ids.test.ts | 2 +- .../test/alerting_api_integration/common/lib/get_event_log.ts | 2 +- .../test_suites/event_log/public_api_integration.ts | 4 ++-- .../test_suites/event_log/service_api_integration.ts | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/event_log/common/index.ts b/x-pack/plugins/event_log/common/index.ts index 79ecd47628712..5910dbe2c5ad7 100644 --- a/x-pack/plugins/event_log/common/index.ts +++ b/x-pack/plugins/event_log/common/index.ts @@ -5,4 +5,4 @@ * 2.0. */ -export const BASE_EVENT_LOG_API_PATH = '/api/event_log'; +export const BASE_EVENT_LOG_API_PATH = '/internal/event_log'; diff --git a/x-pack/plugins/event_log/server/routes/find.test.ts b/x-pack/plugins/event_log/server/routes/find.test.ts index b823d21a6c1f7..c51c8f3adf1e5 100644 --- a/x-pack/plugins/event_log/server/routes/find.test.ts +++ b/x-pack/plugins/event_log/server/routes/find.test.ts @@ -26,7 +26,7 @@ describe('find', () => { const [config, handler] = router.get.mock.calls[0]; - expect(config.path).toMatchInlineSnapshot(`"/api/event_log/{type}/{id}/_find"`); + expect(config.path).toMatchInlineSnapshot(`"/internal/event_log/{type}/{id}/_find"`); const events = [fakeEvent(), fakeEvent()]; const result = { diff --git a/x-pack/plugins/event_log/server/routes/find_by_ids.test.ts b/x-pack/plugins/event_log/server/routes/find_by_ids.test.ts index 4685306e869da..065174abcd9fd 100644 --- a/x-pack/plugins/event_log/server/routes/find_by_ids.test.ts +++ b/x-pack/plugins/event_log/server/routes/find_by_ids.test.ts @@ -26,7 +26,7 @@ describe('find_by_ids', () => { const [config, handler] = router.post.mock.calls[0]; - expect(config.path).toMatchInlineSnapshot(`"/api/event_log/{type}/_find"`); + expect(config.path).toMatchInlineSnapshot(`"/internal/event_log/{type}/_find"`); const events = [fakeEvent(), fakeEvent()]; const result = { diff --git a/x-pack/test/alerting_api_integration/common/lib/get_event_log.ts b/x-pack/test/alerting_api_integration/common/lib/get_event_log.ts index 3ee7929170338..194be6d184692 100644 --- a/x-pack/test/alerting_api_integration/common/lib/get_event_log.ts +++ b/x-pack/test/alerting_api_integration/common/lib/get_event_log.ts @@ -39,7 +39,7 @@ export async function getEventLog(params: GetEventLogParams): Promise = {} ) { const urlPrefix = urlPrefixFromNamespace(namespace); - const url = `${urlPrefix}/api/event_log/event_log_test/${id}/_find${ + const url = `${urlPrefix}/internal/event_log/event_log_test/${id}/_find${ isEmpty(query) ? '' : `?${Object.entries(query) @@ -260,7 +260,7 @@ export default function ({ getService }: FtrProviderContext) { legacyIds: string[] = [] ) { const urlPrefix = urlPrefixFromNamespace(namespace); - const url = `${urlPrefix}/api/event_log/event_log_test/_find${ + const url = `${urlPrefix}/internal/event_log/event_log_test/_find${ isEmpty(query) ? '' : `?${Object.entries(query) diff --git a/x-pack/test/plugin_api_integration/test_suites/event_log/service_api_integration.ts b/x-pack/test/plugin_api_integration/test_suites/event_log/service_api_integration.ts index 267df365427a0..f317ad2dcff13 100644 --- a/x-pack/test/plugin_api_integration/test_suites/event_log/service_api_integration.ts +++ b/x-pack/test/plugin_api_integration/test_suites/event_log/service_api_integration.ts @@ -259,7 +259,7 @@ export default function ({ getService }: FtrProviderContext) { async function fetchEvents(savedObjectType: string, savedObjectId: string) { log.debug(`Fetching events of Saved Object ${savedObjectId}`); return await supertest - .get(`/api/event_log/${savedObjectType}/${savedObjectId}/_find`) + .get(`/internal/event_log/${savedObjectType}/${savedObjectId}/_find`) .set('kbn-xsrf', 'foo') .expect(200); } From f68a4691f83a5ff4f5329f022114263f5530c83f Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Tue, 8 Mar 2022 16:19:21 -0500 Subject: [PATCH 15/39] Fixing types --- x-pack/plugins/event_log/server/types.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/event_log/server/types.ts b/x-pack/plugins/event_log/server/types.ts index df4dec55b069a..ea36bc1d4ed13 100644 --- a/x-pack/plugins/event_log/server/types.ts +++ b/x-pack/plugins/event_log/server/types.ts @@ -14,7 +14,10 @@ export { EventSchema, ECS_VERSION } from '../generated/schemas'; import { IEvent } from '../generated/schemas'; import { AggregateOptionsType, FindOptionsType } from './event_log_client'; import { QueryEventsBySavedObjectResult } from './es/cluster_client_adapter'; -export type { QueryEventsBySavedObjectResult } from './es/cluster_client_adapter'; +export type { + QueryEventsBySavedObjectResult, + AggregateEventsBySavedObjectResult, +} from './es/cluster_client_adapter'; import { SavedObjectProvider } from './saved_object_provider_registry'; export const SAVED_OBJECT_REL_PRIMARY = 'primary'; From 45ea81cfc0ccef73fcfde6d7b6e11d8e1a085f0a Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 9 Mar 2022 08:08:30 -0500 Subject: [PATCH 16/39] PR feedback --- .../event_log/server/es/cluster_client_adapter.test.ts | 2 +- .../event_log/server/es/cluster_client_adapter.ts | 4 ++-- x-pack/plugins/event_log/server/types.ts | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts index 435f1680f9fd6..56a708ef51b67 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts @@ -700,7 +700,7 @@ describe('aggregateEventsBySavedObject', () => { }, }); expect(result).toEqual({ - aggregateResults: { + aggregations: { genericAgg: { buckets: [ { diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index 4249274816ba4..502e48795f0cc 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -59,7 +59,7 @@ export type AggregateEventsOptionsBySavedObjectFilter = QueryOptionsEventsBySave }; export interface AggregateEventsBySavedObjectResult { - aggregateResults: Record | undefined; + aggregations: Record | undefined; } // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -407,7 +407,7 @@ export class ClusterClientAdapter, legacyIds?: string[] - ): Promise<{ - aggregateResults: Record | undefined; - }>; + ): Promise; } export interface IEventLogger { From ec23e2c5246b16689dfb557049ea0017d6c871de Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 9 Mar 2022 10:15:40 -0500 Subject: [PATCH 17/39] omg types --- .../lib/get_execution_log_aggregation.test.ts | 5 ++- .../lib/get_execution_log_aggregation.ts | 31 +++++++++++++------ 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts index 1585edf192bca..692df30d47306 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts @@ -9,7 +9,6 @@ import { getExecutionLogAggregation, formatExecutionLogResult, } from './get_execution_log_aggregation'; -import { AggregateEventsBySavedObjectResult } from '../../../event_log/server'; describe('getExecutionLogAggregation', () => { test('should throw error when given bad sort field', () => { @@ -124,7 +123,7 @@ describe('getExecutionLogAggregation', () => { describe('formatExecutionLogResult', () => { test('should format results correctly', () => { const results = { - aggregateResults: { + aggregations: { executionUuid: { meta: {}, doc_count_error_upper_bound: 0, @@ -338,7 +337,7 @@ describe('formatExecutionLogResult', () => { test('should format results correctly when execution timeouts occur', () => { const results = { - aggregateResults: { + aggregations: { executionUuid: { meta: {}, doc_count_error_upper_bound: 0, diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts index ab4ed19fcce86..5d949f4e2b290 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts @@ -26,17 +26,17 @@ const Millis2Nanos = 1000 * 1000; export interface IExecutionLog { id: string; timestamp: string; - duration_ms: number | null; + duration_ms: number; status: string; message: string; num_active_alerts: number; num_new_alerts: number; num_recovered_alerts: number; - num_triggered_actions: number | null; + num_triggered_actions: number; num_succeeded_actions: number; num_errored_actions: number; - total_search_duration_ms: number | null; - es_search_duration_ms: number | null; + total_search_duration_ms: number; + es_search_duration_ms: number; timed_out: boolean; } @@ -240,19 +240,30 @@ export function formatExecutionLogAggBucket(bucketVal: IBucketAggregationResult) num_active_alerts: bucketVal.alertCounts.buckets.activeAlerts.doc_count, num_new_alerts: bucketVal.alertCounts.buckets.newAlerts.doc_count, num_recovered_alerts: bucketVal.alertCounts.buckets.recoveredAlerts.doc_count, - num_triggered_actions: bucketVal.ruleExecution.numTriggeredActions.value, + num_triggered_actions: bucketVal.ruleExecution.numTriggeredActions.value ?? 0, num_succeeded_actions: actionExecutionSuccess, num_errored_actions: actionExecutionError, - total_search_duration_ms: bucketVal.ruleExecution.totalSearchDuration.value, - es_search_duration_ms: bucketVal.ruleExecution.esSearchDuration.value, + total_search_duration_ms: bucketVal.ruleExecution.totalSearchDuration.value ?? 0, + es_search_duration_ms: bucketVal.ruleExecution.esSearchDuration.value ?? 0, timed_out: timedOut, }; } export function formatExecutionLogResult(results: AggregateEventsBySavedObjectResult) { - const { aggregateResults } = results; - const total = aggregateResults.executionUuidCardinality.value; - const buckets: IBucketAggregationResult[] = aggregateResults.executionUuid.buckets; + const { aggregations } = results; + + if (!aggregations) { + return { + total: 0, + data: [], + }; + } + + const total = (aggregations.executionUuidCardinality as estypes.AggregationsCardinalityAggregate) + .value; + const buckets = ( + aggregations.executionUuid as estypes.AggregationsMultiBucketAggregateBase + ).buckets as IBucketAggregationResult[]; return { total, From 95b3fdf0467472794ade893812db3e55a9371812 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 9 Mar 2022 10:23:22 -0500 Subject: [PATCH 18/39] types and optional accessors --- .../lib/get_execution_log_aggregation.ts | 47 ++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts index 5d949f4e2b290..9236c310a19b0 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts @@ -53,7 +53,7 @@ interface IActionExecution buckets: Array<{ key: string; doc_count: number }>; } -interface IBucketAggregationResult extends estypes.AggregationsStringTermsBucketKeys { +interface IExecutionUuidAggBucket extends estypes.AggregationsStringTermsBucketKeys { timeoutMessage: estypes.AggregationsMultiBucketBase; ruleExecution: { executeStartTime: estypes.AggregationsMinAggregate; @@ -69,6 +69,10 @@ interface IBucketAggregationResult extends estypes.AggregationsStringTermsBucket }; } +interface ExecutionUuidAggResult + extends estypes.AggregationsAggregateBase { + buckets: TBucket[]; +} export interface IExecutionLogAggOptions { numExecutions: number; page: number; @@ -220,31 +224,32 @@ function getProviderAndActionFilter(provider: string, action: string) { }; } -export function formatExecutionLogAggBucket(bucketVal: IBucketAggregationResult): IExecutionLog { - const durationUs = bucketVal.ruleExecution.executionDuration.value - ? bucketVal.ruleExecution.executionDuration.value +export function formatExecutionLogAggBucket(bucket: IExecutionUuidAggBucket): IExecutionLog { + const durationUs = bucket?.ruleExecution?.executionDuration?.value + ? bucket.ruleExecution.executionDuration.value : 0; - const timedOut = bucketVal.timeoutMessage.doc_count > 0; + const timedOut = (bucket?.timeoutMessage?.doc_count ?? 0) > 0; - const actionExecutionOutcomes = bucketVal.actionExecution.actionOutcomes.buckets; + const actionExecutionOutcomes = bucket?.actionExecution?.actionOutcomes?.buckets ?? []; const actionExecutionSuccess = - actionExecutionOutcomes.find((bucket) => bucket.key === 'success')?.doc_count ?? 0; + actionExecutionOutcomes.find((subBucket) => subBucket?.key === 'success')?.doc_count ?? 0; const actionExecutionError = - actionExecutionOutcomes.find((bucket) => bucket.key === 'failure')?.doc_count ?? 0; + actionExecutionOutcomes.find((subBucket) => subBucket?.key === 'failure')?.doc_count ?? 0; + return { - id: bucketVal.key, - timestamp: bucketVal.ruleExecution.executeStartTime.value_as_string!, + id: bucket?.key ?? '', + timestamp: bucket?.ruleExecution?.executeStartTime.value_as_string ?? '', duration_ms: durationUs / Millis2Nanos, - status: bucketVal.ruleExecution.outcomeAndMessage.hits.hits[0]?._source?.event?.outcome, - message: bucketVal.ruleExecution.outcomeAndMessage.hits.hits[0]?._source?.message, - num_active_alerts: bucketVal.alertCounts.buckets.activeAlerts.doc_count, - num_new_alerts: bucketVal.alertCounts.buckets.newAlerts.doc_count, - num_recovered_alerts: bucketVal.alertCounts.buckets.recoveredAlerts.doc_count, - num_triggered_actions: bucketVal.ruleExecution.numTriggeredActions.value ?? 0, + status: bucket?.ruleExecution?.outcomeAndMessage?.hits?.hits[0]?._source?.event?.outcome, + message: bucket?.ruleExecution?.outcomeAndMessage?.hits?.hits[0]?._source?.message, + num_active_alerts: bucket?.alertCounts?.buckets?.activeAlerts?.doc_count ?? 0, + num_new_alerts: bucket?.alertCounts?.buckets?.newAlerts?.doc_count ?? 0, + num_recovered_alerts: bucket?.alertCounts?.buckets?.recoveredAlerts?.doc_count ?? 0, + num_triggered_actions: bucket?.ruleExecution?.numTriggeredActions?.value ?? 0, num_succeeded_actions: actionExecutionSuccess, num_errored_actions: actionExecutionError, - total_search_duration_ms: bucketVal.ruleExecution.totalSearchDuration.value ?? 0, - es_search_duration_ms: bucketVal.ruleExecution.esSearchDuration.value ?? 0, + total_search_duration_ms: bucket?.ruleExecution?.totalSearchDuration?.value ?? 0, + es_search_duration_ms: bucket?.ruleExecution?.esSearchDuration?.value ?? 0, timed_out: timedOut, }; } @@ -261,12 +266,10 @@ export function formatExecutionLogResult(results: AggregateEventsBySavedObjectRe const total = (aggregations.executionUuidCardinality as estypes.AggregationsCardinalityAggregate) .value; - const buckets = ( - aggregations.executionUuid as estypes.AggregationsMultiBucketAggregateBase - ).buckets as IBucketAggregationResult[]; + const buckets = (aggregations.executionUuid as ExecutionUuidAggResult).buckets; return { total, - data: buckets.map((bucket: IBucketAggregationResult) => formatExecutionLogAggBucket(bucket)), + data: buckets.map((bucket: IExecutionUuidAggBucket) => formatExecutionLogAggBucket(bucket)), }; } From 18e2e62e0ecc9c36c3f668d646ca305d1da1e0c2 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 9 Mar 2022 13:06:15 -0500 Subject: [PATCH 19/39] Adding fn to calculate num executions based on date range --- .../lib/get_execution_log_aggregation.test.ts | 253 +++++++++++++++++- .../lib/get_execution_log_aggregation.ts | 23 +- 2 files changed, 273 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts index 692df30d47306..a6848be8dd505 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts @@ -6,10 +6,43 @@ */ import { + getNumExecutions, getExecutionLogAggregation, formatExecutionLogResult, } from './get_execution_log_aggregation'; +describe('getNumExecutions', () => { + test('should calculate the expected number of executions in a given date range with a given schedule interval', () => { + expect( + getNumExecutions( + new Date('2020-12-01T00:00:00.000Z'), + new Date('2020-12-02T00:00:00.000Z'), + '1h' + ) + ).toEqual(24); + }); + + test('should return 0 if dateEnd is less that dateStart', () => { + expect( + getNumExecutions( + new Date('2020-12-02T00:00:00.000Z'), + new Date('2020-12-01T00:00:00.000Z'), + '1h' + ) + ).toEqual(0); + }); + + test('should cap numExecutions at default max buckets limit', () => { + expect( + getNumExecutions( + new Date('2020-12-01T00:00:00.000Z'), + new Date('2020-12-02T00:00:00.000Z'), + '1s' + ) + ).toEqual(65535); + }); +}); + describe('getExecutionLogAggregation', () => { test('should throw error when given bad sort field', () => { expect(() => { @@ -121,6 +154,12 @@ describe('getExecutionLogAggregation', () => { }); describe('formatExecutionLogResult', () => { + test('should return empty results if aggregations are undefined', () => { + expect(formatExecutionLogResult({ aggregations: undefined })).toEqual({ + total: 0, + data: [], + }); + }); test('should format results correctly', () => { const results = { aggregations: { @@ -544,5 +583,217 @@ describe('formatExecutionLogResult', () => { }); }); - test('should format results correctly when action errors occur', () => {}); + test('should format results correctly when action errors occur', () => { + const results = { + aggregations: { + executionUuid: { + meta: {}, + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: 'ecf7ac4c-1c15-4a1d-818a-cacbf57f6158', + doc_count: 32, + timeoutMessage: { + meta: {}, + doc_count: 0, + }, + alertCounts: { + meta: {}, + buckets: { + activeAlerts: { + doc_count: 5, + }, + newAlerts: { + doc_count: 5, + }, + recoveredAlerts: { + doc_count: 5, + }, + }, + }, + ruleExecution: { + meta: {}, + doc_count: 1, + numTriggeredActions: { + value: 5.0, + }, + outcomeAndMessage: { + hits: { + total: { + value: 1, + relation: 'eq', + }, + max_score: 1.0, + hits: [ + { + _index: '.kibana-event-log-8.2.0-000001', + _id: '7xKcb38BcntAq5ycFwiu', + _score: 1.0, + _source: { + event: { + outcome: 'success', + }, + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + }, + }, + ], + }, + }, + totalSearchDuration: { + value: 0.0, + }, + esSearchDuration: { + value: 0.0, + }, + executionDuration: { + value: 1.374e9, + }, + executeStartTime: { + value: 1.646844973039e12, + value_as_string: '2022-03-09T16:56:13.039Z', + }, + }, + actionExecution: { + meta: {}, + doc_count: 5, + actionOutcomes: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: 'failure', + doc_count: 5, + }, + ], + }, + }, + }, + { + key: '61bb867b-661a-471f-bf92-23471afa10b3', + doc_count: 32, + timeoutMessage: { + meta: {}, + doc_count: 0, + }, + alertCounts: { + meta: {}, + buckets: { + activeAlerts: { + doc_count: 5, + }, + newAlerts: { + doc_count: 5, + }, + recoveredAlerts: { + doc_count: 5, + }, + }, + }, + ruleExecution: { + meta: {}, + doc_count: 1, + numTriggeredActions: { + value: 5.0, + }, + outcomeAndMessage: { + hits: { + total: { + value: 1, + relation: 'eq', + }, + max_score: 1.0, + hits: [ + { + _index: '.kibana-event-log-8.2.0-000001', + _id: 'zRKbb38BcntAq5ycOwgk', + _score: 1.0, + _source: { + event: { + outcome: 'success', + }, + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + }, + }, + ], + }, + }, + totalSearchDuration: { + value: 0.0, + }, + esSearchDuration: { + value: 0.0, + }, + executionDuration: { + value: 4.18e8, + }, + executeStartTime: { + value: 1.646844917518e12, + value_as_string: '2022-03-09T16:55:17.518Z', + }, + }, + actionExecution: { + meta: {}, + doc_count: 5, + actionOutcomes: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: 'success', + doc_count: 5, + }, + ], + }, + }, + }, + ], + }, + executionUuidCardinality: { + value: 417, + }, + }, + }; + expect(formatExecutionLogResult(results)).toEqual({ + total: 417, + data: [ + { + id: 'ecf7ac4c-1c15-4a1d-818a-cacbf57f6158', + timestamp: '2022-03-09T16:56:13.039Z', + duration_ms: 1374, + status: 'success', + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + num_active_alerts: 5, + num_new_alerts: 5, + num_recovered_alerts: 5, + num_triggered_actions: 5, + num_succeeded_actions: 0, + num_errored_actions: 5, + total_search_duration_ms: 0, + es_search_duration_ms: 0, + timed_out: false, + }, + { + id: '61bb867b-661a-471f-bf92-23471afa10b3', + timestamp: '2022-03-09T16:55:17.518Z', + duration_ms: 418, + status: 'success', + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + num_active_alerts: 5, + num_new_alerts: 5, + num_recovered_alerts: 5, + num_triggered_actions: 5, + num_succeeded_actions: 5, + num_errored_actions: 0, + total_search_duration_ms: 0, + es_search_duration_ms: 0, + timed_out: false, + }, + ], + }); + }); }); diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts index 9236c310a19b0..92e48ea6311bb 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts @@ -7,8 +7,11 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import Boom from '@hapi/boom'; +import { parseDuration } from '.'; import { AggregateEventsBySavedObjectResult } from '../../../event_log/server'; +const DEFAULT_MAX_BUCKETS_LIMIT = 65535; + const PROVIDER_FIELD = 'event.provider'; const START_FIELD = 'event.start'; const ACTION_FIELD = 'event.action'; @@ -40,6 +43,11 @@ export interface IExecutionLog { timed_out: boolean; } +export interface IExecutionLogResult { + total: number; + data: IExecutionLog[]; +} + interface IAlertCounts extends estypes.AggregationsMultiBucketAggregateBase { buckets: { activeAlerts: estypes.AggregationsSingleBucketAggregateBase; @@ -196,7 +204,7 @@ export function getExecutionLogAggregation({ }, }, }, - // If there was a timeout, this filter will return non-zero + // If there was a timeout, this filter will return non-zero doc count timeoutMessage: { filter: getProviderAndActionFilter('alerting', 'execute-timeout'), }, @@ -254,7 +262,9 @@ export function formatExecutionLogAggBucket(bucket: IExecutionUuidAggBucket): IE }; } -export function formatExecutionLogResult(results: AggregateEventsBySavedObjectResult) { +export function formatExecutionLogResult( + results: AggregateEventsBySavedObjectResult +): IExecutionLogResult { const { aggregations } = results; if (!aggregations) { @@ -273,3 +283,12 @@ export function formatExecutionLogResult(results: AggregateEventsBySavedObjectRe data: buckets.map((bucket: IExecutionUuidAggBucket) => formatExecutionLogAggBucket(bucket)), }; } + +export function getNumExecutions(dateStart: Date, dateEnd: Date, ruleSchedule: string) { + const durationInMillis = dateEnd.getTime() - dateStart.getTime(); + const scheduleMillis = parseDuration(ruleSchedule); + + const numExecutions = Math.ceil(durationInMillis / scheduleMillis); + + return Math.min(numExecutions < 0 ? 0 : numExecutions, DEFAULT_MAX_BUCKETS_LIMIT); +} From c4fca5eab16022992413954e70e7caf186fe45b4 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 9 Mar 2022 13:58:03 -0500 Subject: [PATCH 20/39] Fleshing out rules client function and tests --- .../lib/get_execution_log_aggregation.test.ts | 14 + .../lib/get_execution_log_aggregation.ts | 7 +- .../server/rules_client/rules_client.ts | 105 ++-- .../tests/get_execution_log.test.ts | 522 ++++++++++++++++++ 4 files changed, 586 insertions(+), 62 deletions(-) create mode 100644 x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts index a6848be8dd505..de33bea29a8f6 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts @@ -70,6 +70,20 @@ describe('getExecutionLogAggregation', () => { }).toThrowErrorMatchingInlineSnapshot(`"Invalid page field \\"0\\" - must be greater than 0"`); }); + test('should throw error when given bad perPage field', () => { + expect(() => { + getExecutionLogAggregation({ + numExecutions: 5, + page: 1, + perPage: 0, + sortField: 'timestamp', + sortOrder: 'asc', + }); + }).toThrowErrorMatchingInlineSnapshot( + `"Invalid perPage field \\"0\\" - must be greater than 0"` + ); + }); + test('should correctly generate aggregation', () => { expect( getExecutionLogAggregation({ diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts index 92e48ea6311bb..eb3d168f4384e 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts @@ -115,6 +115,11 @@ export function getExecutionLogAggregation({ throw Boom.badRequest(`Invalid page field "${page}" - must be greater than 0`); } + // Check if valid page value + if (perPage <= 0) { + throw Boom.badRequest(`Invalid perPage field "${perPage}" - must be greater than 0`); + } + return { // Get total number of executions executionUuidCardinality: { @@ -232,7 +237,7 @@ function getProviderAndActionFilter(provider: string, action: string) { }; } -export function formatExecutionLogAggBucket(bucket: IExecutionUuidAggBucket): IExecutionLog { +function formatExecutionLogAggBucket(bucket: IExecutionUuidAggBucket): IExecutionLog { const durationUs = bucket?.ruleExecution?.executionDuration?.value ? bucket.ruleExecution.executionDuration.value : 0; diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index bc0a7c82119e0..006c7b97507f3 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -85,6 +85,12 @@ import { getModifiedSearch, modifyFilterKueryNode, } from './lib/mapped_params_utils'; +import { + formatExecutionLogResult, + getExecutionLogAggregation, + getNumExecutions, + IExecutionLogResult, +} from '../lib/get_execution_log_aggregation'; export interface RegistryAlertTypeWithAuth extends RegistryRuleType { authorizedConsumers: string[]; @@ -231,18 +237,13 @@ export interface GetAlertSummaryParams { export interface GetExecutionLogByIdParams { id: string; - dateStart?: string; - // dateEnd? -} - -export interface ExecutionLog { - count: number; - values: Array<{ - timestamp: number; - duration: number; - status: 'failed' | 'succeeded'; - message: string; - }>; + dateStart: string; + dateEnd?: string; + filter?: string; + page: number; + perPage: number; + sortField: string; + sortOrder: estypes.SortOrder; } // NOTE: Changing this prefix will require a migration to update the prefix in all existing `rule` saved objects @@ -650,8 +651,13 @@ export class RulesClient { public async getExecutionLogForRule({ id, dateStart, - }: // dateEnd? - GetExecutionLogByIdParams): Promise { + dateEnd, + filter, + page, + perPage, + sortField, + sortOrder, + }: GetExecutionLogByIdParams): Promise { this.logger.debug(`getExecutionLogForRule(): getting execution log for rule ${id}`); const rule = (await this.get({ id, includeLegacyId: true })) as SanitizedRuleWithLegacyId; @@ -659,62 +665,39 @@ export class RulesClient { await this.authorization.ensureAuthorized({ ruleTypeId: rule.alertTypeId, consumer: rule.consumer, - operation: ReadOperations.GetAlertSummary, + operation: ReadOperations.GetExecutionLog, entity: AlertingAuthorizationEntity.Rule, }); // default duration of instance summary is 60 * rule interval const dateNow = new Date(); - const durationMillis = parseDuration(rule.schedule.interval) * (numberOfExecutions ?? 60); - const defaultDateStart = new Date(dateNow.valueOf() - durationMillis); - const parsedDateStart = parseDate(dateStart, 'dateStart', defaultDateStart); + const parsedDateStart = parseDate(dateStart, 'dateStart', dateNow); + const parsedDateEnd = parseDate(dateEnd, 'dateEnd', dateNow); const eventLogClient = await this.getEventLogClient(); - let events: IEvent[]; - let executionEvents: IEvent[]; - - try { - const results = await eventLogClient.aggregateEventsBySavedObjectIds( - 'alert', - [id], - {}, - rule.legacyId !== null ? [rule.legacyId] : undefined - ); - } catch (err) { - this.logger.debug( - `rulesClient.getExecutionLogForRule(): error searching event log for rule ${id}: ${err.message}` - ); - } + const results = await eventLogClient.aggregateEventsBySavedObjectIds( + 'alert', + [id], + { + start: parsedDateStart.toISOString(), + end: parsedDateEnd.toISOString(), + filter, + aggs: getExecutionLogAggregation({ + // determine the number of executions to request + // based on the date interval and the rule schedule interval + // this value is capped by the max number of terms allowed in a term aggregation + numExecutions: getNumExecutions(parsedDateStart, parsedDateEnd, rule.schedule.interval), + page, + perPage, + sortField, + sortOrder, + }), + }, + rule.legacyId !== null ? [rule.legacyId] : undefined + ); - // desired output - // { - // count: number; - // values: [ - // { - // // this is shown in the screenshot - // executionTime: timestamp, - // duration: millis, - // status: failed/succeeded, - // message: log message - execution timeout is separate event log entry - - // // other stuff that might be useful that we could probably get - // number of active alerts - // number of new alerts - // number of recovered alerts - // number of triggered actions - // any errors in the actions - - // es search duration - // total search duration - - // } - // ] - // } - return { - count: 0, - values: [], - }; + return formatExecutionLogResult(results); } public async find({ diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts new file mode 100644 index 0000000000000..140431e1c7362 --- /dev/null +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts @@ -0,0 +1,522 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { RulesClient, ConstructorOptions } from '../rules_client'; +import { savedObjectsClientMock, loggingSystemMock } from '../../../../../../src/core/server/mocks'; +import { taskManagerMock } from '../../../../task_manager/server/mocks'; +import { ruleTypeRegistryMock } from '../../rule_type_registry.mock'; +import { alertingAuthorizationMock } from '../../authorization/alerting_authorization.mock'; +import { encryptedSavedObjectsMock } from '../../../../encrypted_saved_objects/server/mocks'; +import { actionsAuthorizationMock } from '../../../../actions/server/mocks'; +import { AlertingAuthorization } from '../../authorization/alerting_authorization'; +import { ActionsAuthorization } from '../../../../actions/server'; +import { eventLogClientMock } from '../../../../event_log/server/mocks'; +import { SavedObject } from 'kibana/server'; +import { RawRule } from '../../types'; +import { getBeforeSetup, mockedDateString, setGlobalDate } from './lib'; +import { getExecutionLogAggregation } from '../../lib/get_execution_log_aggregation'; + +const taskManager = taskManagerMock.createStart(); +const ruleTypeRegistry = ruleTypeRegistryMock.create(); +const unsecuredSavedObjectsClient = savedObjectsClientMock.create(); +const eventLogClient = eventLogClientMock.create(); + +const encryptedSavedObjects = encryptedSavedObjectsMock.createClient(); +const authorization = alertingAuthorizationMock.create(); +const actionsAuthorization = actionsAuthorizationMock.create(); + +const kibanaVersion = 'v7.10.0'; +const rulesClientParams: jest.Mocked = { + taskManager, + ruleTypeRegistry, + unsecuredSavedObjectsClient, + authorization: authorization as unknown as AlertingAuthorization, + actionsAuthorization: actionsAuthorization as unknown as ActionsAuthorization, + spaceId: 'default', + namespace: 'default', + minimumScheduleInterval: '1m', + getUserName: jest.fn(), + createAPIKey: jest.fn(), + logger: loggingSystemMock.create().get(), + encryptedSavedObjectsClient: encryptedSavedObjects, + getActionsClient: jest.fn(), + getEventLogClient: jest.fn(), + kibanaVersion, +}; + +beforeEach(() => { + getBeforeSetup(rulesClientParams, taskManager, ruleTypeRegistry, eventLogClient); +}); + +setGlobalDate(); + +const RuleIntervalSeconds = 1; + +const BaseRuleSavedObject: SavedObject = { + id: '1', + type: 'alert', + attributes: { + enabled: true, + name: 'rule-name', + tags: ['tag-1', 'tag-2'], + alertTypeId: '123', + consumer: 'rule-consumer', + legacyId: null, + schedule: { interval: `${RuleIntervalSeconds}s` }, + actions: [], + params: {}, + createdBy: null, + updatedBy: null, + createdAt: mockedDateString, + updatedAt: mockedDateString, + apiKey: null, + apiKeyOwner: null, + throttle: null, + notifyWhen: null, + muteAll: false, + mutedInstanceIds: [], + executionStatus: { + status: 'unknown', + lastExecutionDate: '2020-08-20T19:23:38Z', + error: null, + }, + }, + references: [], +}; + +const aggregateResults = { + aggregations: { + executionUuid: { + meta: {}, + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: '6705da7d-2635-499d-a6a8-1aee1ae1eac9', + doc_count: 27, + timeoutMessage: { + meta: {}, + doc_count: 0, + }, + alertCounts: { + meta: {}, + buckets: { + activeAlerts: { + doc_count: 5, + }, + newAlerts: { + doc_count: 5, + }, + recoveredAlerts: { + doc_count: 0, + }, + }, + }, + ruleExecution: { + meta: {}, + doc_count: 1, + numTriggeredActions: { + value: 5.0, + }, + outcomeAndMessage: { + hits: { + total: { + value: 1, + relation: 'eq', + }, + max_score: 1.0, + hits: [ + { + _index: '.kibana-event-log-8.2.0-000001', + _id: 'S4wIZX8B8TGQpG7XQZns', + _score: 1.0, + _source: { + event: { + outcome: 'success', + }, + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + }, + }, + ], + }, + }, + totalSearchDuration: { + value: 0.0, + }, + esSearchDuration: { + value: 0.0, + }, + executionDuration: { + value: 1.056e9, + }, + executeStartTime: { + value: 1.646667512617e12, + value_as_string: '2022-03-07T15:38:32.617Z', + }, + }, + actionExecution: { + meta: {}, + doc_count: 5, + actionOutcomes: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: 'success', + doc_count: 5, + }, + ], + }, + }, + }, + { + key: '41b2755e-765a-4044-9745-b03875d5e79a', + doc_count: 32, + timeoutMessage: { + meta: {}, + doc_count: 0, + }, + alertCounts: { + meta: {}, + buckets: { + activeAlerts: { + doc_count: 5, + }, + newAlerts: { + doc_count: 5, + }, + recoveredAlerts: { + doc_count: 5, + }, + }, + }, + ruleExecution: { + meta: {}, + doc_count: 1, + numTriggeredActions: { + value: 5.0, + }, + outcomeAndMessage: { + hits: { + total: { + value: 1, + relation: 'eq', + }, + max_score: 1.0, + hits: [ + { + _index: '.kibana-event-log-8.2.0-000001', + _id: 'a4wIZX8B8TGQpG7Xwpnz', + _score: 1.0, + _source: { + event: { + outcome: 'success', + }, + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + }, + }, + ], + }, + }, + totalSearchDuration: { + value: 0.0, + }, + esSearchDuration: { + value: 0.0, + }, + executionDuration: { + value: 1.165e9, + }, + executeStartTime: { + value: 1.646667545604e12, + value_as_string: '2022-03-07T15:39:05.604Z', + }, + }, + actionExecution: { + meta: {}, + doc_count: 5, + actionOutcomes: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: 'success', + doc_count: 5, + }, + ], + }, + }, + }, + ], + }, + executionUuidCardinality: { + value: 374, + }, + }, +}; + +function getRuleSavedObject(attributes: Partial = {}): SavedObject { + return { + ...BaseRuleSavedObject, + attributes: { ...BaseRuleSavedObject.attributes, ...attributes }, + }; +} + +function getExecutionLogByIdParams(overwrites = {}) { + return { + id: '1', + dateStart: new Date(Date.now() - 3600000).toISOString(), + page: 1, + perPage: 10, + sortField: 'timestamp', + sortOrder: 'desc' as estypes.SortOrder, + ...overwrites, + }; +} +describe('getExecutionLogForRule()', () => { + let rulesClient: RulesClient; + + beforeEach(() => { + rulesClient = new RulesClient(rulesClientParams); + }); + + test('runs as expected with some event log aggregation data', async () => { + const ruleSO = getRuleSavedObject({}); + unsecuredSavedObjectsClient.get.mockResolvedValueOnce(ruleSO); + eventLogClient.aggregateEventsBySavedObjectIds.mockResolvedValueOnce(aggregateResults); + + const result = await rulesClient.getExecutionLogForRule(getExecutionLogByIdParams()); + expect(result).toEqual({ + total: 374, + data: [ + { + id: '6705da7d-2635-499d-a6a8-1aee1ae1eac9', + timestamp: '2022-03-07T15:38:32.617Z', + duration_ms: 1056, + status: 'success', + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + num_active_alerts: 5, + num_new_alerts: 5, + num_recovered_alerts: 0, + num_triggered_actions: 5, + num_succeeded_actions: 5, + num_errored_actions: 0, + total_search_duration_ms: 0, + es_search_duration_ms: 0, + timed_out: false, + }, + { + id: '41b2755e-765a-4044-9745-b03875d5e79a', + timestamp: '2022-03-07T15:39:05.604Z', + duration_ms: 1165, + status: 'success', + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + num_active_alerts: 5, + num_new_alerts: 5, + num_recovered_alerts: 5, + num_triggered_actions: 5, + num_succeeded_actions: 5, + num_errored_actions: 0, + total_search_duration_ms: 0, + es_search_duration_ms: 0, + timed_out: false, + }, + ], + }); + }); + + // Further tests don't check the result of `getExecutionLogForRule()`, as the result + // is just the result from the `formatExecutionLogResult()`, which itself + // has a complete set of tests. + + test('calls saved objects and event log client with default params', async () => { + unsecuredSavedObjectsClient.get.mockResolvedValueOnce(getRuleSavedObject()); + eventLogClient.aggregateEventsBySavedObjectIds.mockResolvedValueOnce(aggregateResults); + + await rulesClient.getExecutionLogForRule(getExecutionLogByIdParams()); + + expect(unsecuredSavedObjectsClient.get).toHaveBeenCalledTimes(1); + expect(eventLogClient.aggregateEventsBySavedObjectIds).toHaveBeenCalledTimes(1); + expect(eventLogClient.aggregateEventsBySavedObjectIds.mock.calls[0]).toEqual([ + 'alert', + ['1'], + { + aggs: getExecutionLogAggregation({ + numExecutions: 3600, + page: 1, + perPage: 10, + sortField: 'timestamp', + sortOrder: 'desc', + }), + end: mockedDateString, + start: '2019-02-12T20:01:22.479Z', + }, + undefined, + ]); + }); + + test('calls event log client with legacy ids param', async () => { + unsecuredSavedObjectsClient.get.mockResolvedValueOnce( + getRuleSavedObject({ legacyId: '99999' }) + ); + eventLogClient.aggregateEventsBySavedObjectIds.mockResolvedValueOnce(aggregateResults); + + await rulesClient.getExecutionLogForRule(getExecutionLogByIdParams()); + + expect(unsecuredSavedObjectsClient.get).toHaveBeenCalledTimes(1); + expect(eventLogClient.aggregateEventsBySavedObjectIds).toHaveBeenCalledTimes(1); + expect(eventLogClient.aggregateEventsBySavedObjectIds.mock.calls[0]).toEqual([ + 'alert', + ['1'], + { + aggs: getExecutionLogAggregation({ + numExecutions: 3600, + page: 1, + perPage: 10, + sortField: 'timestamp', + sortOrder: 'desc', + }), + end: mockedDateString, + start: '2019-02-12T20:01:22.479Z', + }, + ['99999'], + ]); + }); + + test('calls event log client with end date if specified', async () => { + unsecuredSavedObjectsClient.get.mockResolvedValueOnce(getRuleSavedObject()); + eventLogClient.aggregateEventsBySavedObjectIds.mockResolvedValueOnce(aggregateResults); + + await rulesClient.getExecutionLogForRule( + getExecutionLogByIdParams({ dateEnd: new Date(Date.now() - 1800000).toISOString() }) + ); + + expect(unsecuredSavedObjectsClient.get).toHaveBeenCalledTimes(1); + expect(eventLogClient.aggregateEventsBySavedObjectIds).toHaveBeenCalledTimes(1); + expect(eventLogClient.aggregateEventsBySavedObjectIds.mock.calls[0]).toEqual([ + 'alert', + ['1'], + { + aggs: getExecutionLogAggregation({ + numExecutions: 1800, + page: 1, + perPage: 10, + sortField: 'timestamp', + sortOrder: 'desc', + }), + end: '2019-02-12T20:31:22.479Z', + start: '2019-02-12T20:01:22.479Z', + }, + undefined, + ]); + }); + + test('calls event log client with filter if specified', async () => { + unsecuredSavedObjectsClient.get.mockResolvedValueOnce(getRuleSavedObject()); + eventLogClient.aggregateEventsBySavedObjectIds.mockResolvedValueOnce(aggregateResults); + + await rulesClient.getExecutionLogForRule( + getExecutionLogByIdParams({ filter: 'event.outcome: success' }) + ); + + expect(unsecuredSavedObjectsClient.get).toHaveBeenCalledTimes(1); + expect(eventLogClient.aggregateEventsBySavedObjectIds).toHaveBeenCalledTimes(1); + expect(eventLogClient.aggregateEventsBySavedObjectIds.mock.calls[0]).toEqual([ + 'alert', + ['1'], + { + aggs: getExecutionLogAggregation({ + numExecutions: 3600, + page: 1, + perPage: 10, + sortField: 'timestamp', + sortOrder: 'desc', + }), + filter: 'event.outcome: success', + end: mockedDateString, + start: '2019-02-12T20:01:22.479Z', + }, + undefined, + ]); + }); + + test('invalid start date throws an error', async () => { + unsecuredSavedObjectsClient.get.mockResolvedValueOnce(getRuleSavedObject()); + eventLogClient.aggregateEventsBySavedObjectIds.mockResolvedValueOnce(aggregateResults); + + const dateStart = 'ain"t no way this will get parsed as a date'; + expect( + rulesClient.getExecutionLogForRule(getExecutionLogByIdParams({ dateStart })) + ).rejects.toMatchInlineSnapshot( + `[Error: Invalid date for parameter dateStart: "ain"t no way this will get parsed as a date"]` + ); + }); + + test('invalid end date throws an error', async () => { + unsecuredSavedObjectsClient.get.mockResolvedValueOnce(getRuleSavedObject()); + eventLogClient.aggregateEventsBySavedObjectIds.mockResolvedValueOnce(aggregateResults); + + const dateEnd = 'ain"t no way this will get parsed as a date'; + expect( + rulesClient.getExecutionLogForRule(getExecutionLogByIdParams({ dateEnd })) + ).rejects.toMatchInlineSnapshot( + `[Error: Invalid date for parameter dateEnd: "ain"t no way this will get parsed as a date"]` + ); + }); + + test('invalid page value throws an error', async () => { + unsecuredSavedObjectsClient.get.mockResolvedValueOnce(getRuleSavedObject()); + eventLogClient.aggregateEventsBySavedObjectIds.mockResolvedValueOnce(aggregateResults); + + expect( + rulesClient.getExecutionLogForRule(getExecutionLogByIdParams({ page: -3 })) + ).rejects.toMatchInlineSnapshot(`[Error: Invalid page field "-3" - must be greater than 0]`); + }); + + test('invalid perPage value throws an error', async () => { + unsecuredSavedObjectsClient.get.mockResolvedValueOnce(getRuleSavedObject()); + eventLogClient.aggregateEventsBySavedObjectIds.mockResolvedValueOnce(aggregateResults); + + expect( + rulesClient.getExecutionLogForRule(getExecutionLogByIdParams({ perPage: -3 })) + ).rejects.toMatchInlineSnapshot(`[Error: Invalid perPage field "-3" - must be greater than 0]`); + }); + + test('invalid sortField value throws an error', async () => { + unsecuredSavedObjectsClient.get.mockResolvedValueOnce(getRuleSavedObject()); + eventLogClient.aggregateEventsBySavedObjectIds.mockResolvedValueOnce(aggregateResults); + + expect( + rulesClient.getExecutionLogForRule(getExecutionLogByIdParams({ sortField: 'foo' })) + ).rejects.toMatchInlineSnapshot( + `[Error: Invalid sort field "foo" - must be one of [timestamp,duration]]` + ); + }); + + test('throws error when saved object get throws an error', async () => { + unsecuredSavedObjectsClient.get.mockRejectedValueOnce(new Error('OMG!')); + eventLogClient.aggregateEventsBySavedObjectIds.mockResolvedValueOnce(aggregateResults); + + expect( + rulesClient.getExecutionLogForRule(getExecutionLogByIdParams()) + ).rejects.toMatchInlineSnapshot(`[Error: OMG!]`); + }); + + test('throws error when eventLog.aggregateEventsBySavedObjectIds throws an error', async () => { + unsecuredSavedObjectsClient.get.mockResolvedValueOnce(getRuleSavedObject()); + eventLogClient.aggregateEventsBySavedObjectIds.mockRejectedValueOnce(new Error('OMG 2!')); + + expect( + rulesClient.getExecutionLogForRule(getExecutionLogByIdParams()) + ).rejects.toMatchInlineSnapshot(`[Error: OMG 2!]`); + }); +}); From adf038480757c020a4c1f6491f7c589f9ee5774e Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 9 Mar 2022 14:15:26 -0500 Subject: [PATCH 21/39] http api --- .../routes/get_rule_execution_log.test.ts | 105 +++++++++++------- .../server/routes/get_rule_execution_log.ts | 61 +++++----- .../alerting/server/rules_client.mock.ts | 1 + 3 files changed, 90 insertions(+), 77 deletions(-) diff --git a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts index 4193e769dc513..a9750b15a0d11 100644 --- a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts +++ b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts @@ -5,13 +5,13 @@ * 2.0. */ -import { getRuleAlertSummaryRoute } from './get_rule_alert_summary'; +import { getRuleExecutionLogRoute } from './get_rule_execution_log'; import { httpServiceMock } from 'src/core/server/mocks'; import { licenseStateMock } from '../lib/license_state.mock'; import { mockHandlerArguments } from './_mock_handler_arguments'; import { SavedObjectsErrorHelpers } from 'src/core/server'; import { rulesClientMock } from '../rules_client.mock'; -import { AlertSummary } from '../types'; +import { IExecutionLogResult } from '../lib/get_execution_log_aggregation'; const rulesClient = rulesClientMock.create(); jest.mock('../lib/license_api_access.ts', () => ({ @@ -22,43 +22,59 @@ beforeEach(() => { jest.resetAllMocks(); }); -describe('getRuleAlertSummaryRoute', () => { +describe('getRuleExecutionLogRoute', () => { const dateString = new Date().toISOString(); - const mockedAlertSummary: AlertSummary = { - id: '', - name: '', - tags: [], - ruleTypeId: '', - consumer: '', - muteAll: false, - throttle: null, - enabled: false, - statusStartDate: dateString, - statusEndDate: dateString, - status: 'OK', - errorMessages: [], - alerts: {}, - executionDuration: { - average: 1, - valuesWithTimestamp: { - '17 Nov 2021 @ 19:19:17': 3, - '18 Nov 2021 @ 19:19:17': 5, - '19 Nov 2021 @ 19:19:17': 5, + const mockedExecutionLog: IExecutionLogResult = { + total: 374, + data: [ + { + id: '6705da7d-2635-499d-a6a8-1aee1ae1eac9', + timestamp: '2022-03-07T15:38:32.617Z', + duration_ms: 1056, + status: 'success', + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + num_active_alerts: 5, + num_new_alerts: 5, + num_recovered_alerts: 0, + num_triggered_actions: 5, + num_succeeded_actions: 5, + num_errored_actions: 0, + total_search_duration_ms: 0, + es_search_duration_ms: 0, + timed_out: false, + }, + { + id: '41b2755e-765a-4044-9745-b03875d5e79a', + timestamp: '2022-03-07T15:39:05.604Z', + duration_ms: 1165, + status: 'success', + message: + "rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'", + num_active_alerts: 5, + num_new_alerts: 5, + num_recovered_alerts: 5, + num_triggered_actions: 5, + num_succeeded_actions: 5, + num_errored_actions: 0, + total_search_duration_ms: 0, + es_search_duration_ms: 0, + timed_out: false, }, - }, + ], }; - it('gets rule alert summary', async () => { + it('gets rule execution log', async () => { const licenseState = licenseStateMock.create(); const router = httpServiceMock.createRouter(); - getRuleAlertSummaryRoute(router, licenseState); + getRuleExecutionLogRoute(router, licenseState); const [config, handler] = router.get.mock.calls[0]; - expect(config.path).toMatchInlineSnapshot(`"/internal/alerting/rule/{id}/_alert_summary"`); + expect(config.path).toMatchInlineSnapshot(`"/internal/alerting/rule/{id}/_execution_log"`); - rulesClient.getAlertSummary.mockResolvedValueOnce(mockedAlertSummary); + rulesClient.getExecutionLogForRule.mockResolvedValue(mockedExecutionLog); const [context, req, res] = mockHandlerArguments( { rulesClient }, @@ -66,23 +82,30 @@ describe('getRuleAlertSummaryRoute', () => { params: { id: '1', }, - query: {}, + query: { + date_start: dateString, + per_page: 10, + page: 1, + sort_field: 'timestamp', + sort_order: 'desc', + }, }, ['ok'] ); await handler(context, req, res); - expect(rulesClient.getAlertSummary).toHaveBeenCalledTimes(1); - expect(rulesClient.getAlertSummary.mock.calls[0]).toMatchInlineSnapshot(` - Array [ - Object { - "dateStart": undefined, - "id": "1", - "numberOfExecutions": undefined, - }, - ] - `); + expect(rulesClient.getExecutionLogForRule).toHaveBeenCalledTimes(1); + expect(rulesClient.getExecutionLogForRule.mock.calls[0]).toEqual([ + { + dateStart: dateString, + id: '1', + page: 1, + perPage: 10, + sortField: 'timestamp', + sortOrder: 'desc', + }, + ]); expect(res.ok).toHaveBeenCalled(); }); @@ -91,11 +114,11 @@ describe('getRuleAlertSummaryRoute', () => { const licenseState = licenseStateMock.create(); const router = httpServiceMock.createRouter(); - getRuleAlertSummaryRoute(router, licenseState); + getRuleExecutionLogRoute(router, licenseState); const [, handler] = router.get.mock.calls[0]; - rulesClient.getAlertSummary = jest + rulesClient.getExecutionLogForRule = jest .fn() .mockRejectedValueOnce(SavedObjectsErrorHelpers.createGenericNotFoundError('alert', '1')); diff --git a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts index ef8458f97231b..0a35676a569fd 100644 --- a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts +++ b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts @@ -8,54 +8,42 @@ import { IRouter } from 'kibana/server'; import { schema } from '@kbn/config-schema'; import { ILicenseState } from '../lib'; -import { GetAlertSummaryParams } from '../rules_client'; -import { RewriteRequestCase, RewriteResponseCase, verifyAccessAndContext } from './lib'; -import { - AlertingRequestHandlerContext, - INTERNAL_BASE_ALERTING_API_PATH, - AlertSummary, -} from '../types'; +import { GetExecutionLogByIdParams } from '../rules_client'; +import { RewriteRequestCase, verifyAccessAndContext } from './lib'; +import { AlertingRequestHandlerContext, INTERNAL_BASE_ALERTING_API_PATH } from '../types'; const paramSchema = schema.object({ id: schema.string(), }); const querySchema = schema.object({ - date_start: schema.maybe(schema.string()), + date_start: schema.string(), date_end: schema.maybe(schema.string()), + filter: schema.maybe(schema.string()), + per_page: schema.number({ defaultValue: 10, min: 1 }), + page: schema.number({ defaultValue: 1, min: 1 }), + sort_field: schema.oneOf([schema.literal('timestamp'), schema.literal('duration')], { + defaultValue: 'timestamp', + }), + sort_order: schema.oneOf([schema.literal('asc'), schema.literal('desc')], { + defaultValue: 'desc', + }), }); -const rewriteReq: RewriteRequestCase = ({ +const rewriteReq: RewriteRequestCase = ({ date_start: dateStart, - number_of_executions: numberOfExecutions, + date_end: dateEnd, + per_page: perPage, + sort_field: sortField, + sort_order: sortOrder, ...rest }) => ({ ...rest, - numberOfExecutions, dateStart, -}); - -const rewriteBodyRes: RewriteResponseCase = ({ - ruleTypeId, - muteAll, - statusStartDate, - statusEndDate, - errorMessages, - lastRun, - executionDuration: { valuesWithTimestamp, ...executionDuration }, - ...rest -}) => ({ - ...rest, - rule_type_id: ruleTypeId, - mute_all: muteAll, - status_start_date: statusStartDate, - status_end_date: statusEndDate, - error_messages: errorMessages, - last_run: lastRun, - execution_duration: { - ...executionDuration, - values_with_timestamp: valuesWithTimestamp, - }, + dateEnd, + perPage, + sortField, + sortOrder, }); export const getRuleExecutionLogRoute = ( @@ -74,8 +62,9 @@ export const getRuleExecutionLogRoute = ( verifyAccessAndContext(licenseState, async function (context, req, res) { const rulesClient = context.alerting.getRulesClient(); const { id } = req.params; - const execLog = await rulesClient.getExecutionLogForRule(rewriteReq({ id, ...req.query })); - return res.ok({ body: rewriteBodyRes(execLog) }); + return res.ok({ + body: await rulesClient.getExecutionLogForRule(rewriteReq({ id, ...req.query })), + }); }) ) ); diff --git a/x-pack/plugins/alerting/server/rules_client.mock.ts b/x-pack/plugins/alerting/server/rules_client.mock.ts index 2395e7f041846..c86ca91cc1f98 100644 --- a/x-pack/plugins/alerting/server/rules_client.mock.ts +++ b/x-pack/plugins/alerting/server/rules_client.mock.ts @@ -30,6 +30,7 @@ const createRulesClientMock = () => { unmuteInstance: jest.fn(), listAlertTypes: jest.fn(), getAlertSummary: jest.fn(), + getExecutionLogForRule: jest.fn(), getSpaceId: jest.fn(), }; return mocked; From 47623f30001bd96a95291501c2f42477756d5c78 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 9 Mar 2022 14:35:30 -0500 Subject: [PATCH 22/39] Cleanup --- .../event_log/server/event_log_client.test.ts | 45 +++++++++++++++++-- .../event_log/server/event_log_client.ts | 11 ++++- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/event_log/server/event_log_client.test.ts b/x-pack/plugins/event_log/server/event_log_client.test.ts index 2d53a2c1897e4..9ee75dd97ed19 100644 --- a/x-pack/plugins/event_log/server/event_log_client.test.ts +++ b/x-pack/plugins/event_log/server/event_log_client.test.ts @@ -191,17 +191,54 @@ describe('EventLogStart', () => { describe('aggregateEventsBySavedObjectIds', () => { test('verifies that the user can access the specified saved object', async () => { savedObjectGetter.mockResolvedValueOnce(expectedSavedObject); - await eventLogClient.aggregateEventsBySavedObjectIds('saved-object-type', [ - 'saved-object-id', - ]); + await eventLogClient.aggregateEventsBySavedObjectIds( + 'saved-object-type', + ['saved-object-id'], + { aggs: {} } + ); expect(savedObjectGetter).toHaveBeenCalledWith('saved-object-type', ['saved-object-id']); }); + test('throws when no aggregation is defined in options', async () => { + savedObjectGetter.mockResolvedValueOnce(expectedSavedObject); + await expect( + eventLogClient.aggregateEventsBySavedObjectIds('saved-object-type', ['saved-object-id']) + ).rejects.toMatchInlineSnapshot(`[Error: No aggregation defined!]`); + }); test('throws when the user doesnt have permission to access the specified saved object', async () => { savedObjectGetter.mockRejectedValue(new Error('Fail')); await expect( - eventLogClient.aggregateEventsBySavedObjectIds('saved-object-type', ['saved-object-id']) + eventLogClient.aggregateEventsBySavedObjectIds('saved-object-type', ['saved-object-id'], { + aggs: {}, + }) ).rejects.toMatchInlineSnapshot(`[Error: Fail]`); }); + test('calls aggregateEventsBySavedObjects with given aggregation', async () => { + savedObjectGetter.mockResolvedValueOnce(expectedSavedObject); + await eventLogClient.aggregateEventsBySavedObjectIds( + 'saved-object-type', + ['saved-object-id'], + { aggs: { myAgg: {} } } + ); + expect(savedObjectGetter).toHaveBeenCalledWith('saved-object-type', ['saved-object-id']); + expect(esContext.esAdapter.aggregateEventsBySavedObjects).toHaveBeenCalledWith({ + index: esContext.esNames.indexPattern, + namespace: undefined, + type: 'saved-object-type', + ids: ['saved-object-id'], + aggregateOptions: { + aggs: { myAgg: {} }, + page: 1, + per_page: 10, + sort: [ + { + sort_field: '@timestamp', + sort_order: 'asc', + }, + ], + }, + legacyIds: undefined, + }); + }); }); }); diff --git a/x-pack/plugins/event_log/server/event_log_client.ts b/x-pack/plugins/event_log/server/event_log_client.ts index ceaed92e103d1..a2602a2b24adf 100644 --- a/x-pack/plugins/event_log/server/event_log_client.ts +++ b/x-pack/plugins/event_log/server/event_log_client.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { omit } from 'lodash'; import { Observable } from 'rxjs'; import { schema, TypeOf } from '@kbn/config-schema'; import { IClusterClient, KibanaRequest } from 'src/core/server'; @@ -117,7 +118,13 @@ export class EventLogClient implements IEventLogClient { options?: AggregateOptionsType, legacyIds?: string[] ) { - const aggregateOptions = queryOptionsSchema.validate(options ?? {}); + const aggs = options?.aggs; + if (!aggs) { + throw new Error('No aggregation defined!'); + } + + // validate other query options separately from + const aggregateOptions = queryOptionsSchema.validate(omit(options, 'aggs') ?? {}); // verify the user has the required permissions to view this saved object await this.savedObjectGetter(type, ids); @@ -127,7 +134,7 @@ export class EventLogClient implements IEventLogClient { namespace: await this.getNamespace(), type, ids, - aggregateOptions: aggregateOptions as AggregateOptionsType, + aggregateOptions: { ...aggregateOptions, aggs } as AggregateOptionsType, legacyIds, }); } From f87b4fef49ec603aefb7439add5f8730bc721d31 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 9 Mar 2022 16:06:01 -0500 Subject: [PATCH 23/39] Adding schedule delay --- .../lib/get_execution_log_aggregation.test.ts | 29 +++++++++++++++++++ .../lib/get_execution_log_aggregation.ts | 12 ++++++++ .../routes/get_rule_execution_log.test.ts | 2 ++ .../tests/get_execution_log.test.ts | 8 +++++ 4 files changed, 51 insertions(+) diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts index de33bea29a8f6..bdc1eeafde613 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts @@ -136,6 +136,11 @@ describe('getExecutionLogAggregation', () => { }, aggs: { executeStartTime: { min: { field: 'event.start' } }, + scheduleDelay: { + max: { + field: 'kibana.task.schedule_delay', + }, + }, totalSearchDuration: { max: { field: 'kibana.alert.rule.execution.metrics.total_search_duration_ms' }, }, @@ -232,6 +237,9 @@ describe('formatExecutionLogResult', () => { ], }, }, + scheduleDelay: { + value: 3.074e9, + }, totalSearchDuration: { value: 0.0, }, @@ -311,6 +319,9 @@ describe('formatExecutionLogResult', () => { ], }, }, + scheduleDelay: { + value: 3.126e9, + }, totalSearchDuration: { value: 0.0, }, @@ -366,6 +377,7 @@ describe('formatExecutionLogResult', () => { total_search_duration_ms: 0, es_search_duration_ms: 0, timed_out: false, + schedule_delay_ms: 3074, }, { id: '41b2755e-765a-4044-9745-b03875d5e79a', @@ -383,6 +395,7 @@ describe('formatExecutionLogResult', () => { total_search_duration_ms: 0, es_search_duration_ms: 0, timed_out: false, + schedule_delay_ms: 3126, }, ], }); @@ -446,6 +459,9 @@ describe('formatExecutionLogResult', () => { ], }, }, + scheduleDelay: { + value: 3.074e9, + }, totalSearchDuration: { value: 0.0, }, @@ -520,6 +536,9 @@ describe('formatExecutionLogResult', () => { ], }, }, + scheduleDelay: { + value: 3.126e9, + }, totalSearchDuration: { value: 0.0, }, @@ -575,6 +594,7 @@ describe('formatExecutionLogResult', () => { total_search_duration_ms: 0, es_search_duration_ms: 0, timed_out: true, + schedule_delay_ms: 3074, }, { id: '41b2755e-765a-4044-9745-b03875d5e79a', @@ -592,6 +612,7 @@ describe('formatExecutionLogResult', () => { total_search_duration_ms: 0, es_search_duration_ms: 0, timed_out: false, + schedule_delay_ms: 3126, }, ], }); @@ -655,6 +676,9 @@ describe('formatExecutionLogResult', () => { ], }, }, + scheduleDelay: { + value: 3.126e9, + }, totalSearchDuration: { value: 0.0, }, @@ -734,6 +758,9 @@ describe('formatExecutionLogResult', () => { ], }, }, + scheduleDelay: { + value: 3.133e9, + }, totalSearchDuration: { value: 0.0, }, @@ -789,6 +816,7 @@ describe('formatExecutionLogResult', () => { total_search_duration_ms: 0, es_search_duration_ms: 0, timed_out: false, + schedule_delay_ms: 3126, }, { id: '61bb867b-661a-471f-bf92-23471afa10b3', @@ -806,6 +834,7 @@ describe('formatExecutionLogResult', () => { total_search_duration_ms: 0, es_search_duration_ms: 0, timed_out: false, + schedule_delay_ms: 3133, }, ], }); diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts index eb3d168f4384e..ad6ab8633cf41 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts @@ -18,6 +18,7 @@ const ACTION_FIELD = 'event.action'; const OUTCOME_FIELD = 'event.outcome'; const DURATION_FIELD = 'event.duration'; const MESSAGE_FIELD = 'message'; +const SCHEDULE_DELAY_FIELD = 'kibana.task.schedule_delay'; const ES_SEARCH_DURATION_FIELD = 'kibana.alert.rule.execution.metrics.es_search_duration_ms'; const TOTAL_SEARCH_DURATION_FIELD = 'kibana.alert.rule.execution.metrics.total_search_duration_ms'; const NUMBER_OF_TRIGGERED_ACTIONS_FIELD = @@ -40,6 +41,7 @@ export interface IExecutionLog { num_errored_actions: number; total_search_duration_ms: number; es_search_duration_ms: number; + schedule_delay_ms: number; timed_out: boolean; } @@ -66,6 +68,7 @@ interface IExecutionUuidAggBucket extends estypes.AggregationsStringTermsBucketK ruleExecution: { executeStartTime: estypes.AggregationsMinAggregate; executionDuration: estypes.AggregationsMaxAggregate; + scheduleDelay: estypes.AggregationsMaxAggregate; esSearchDuration: estypes.AggregationsMaxAggregate; totalSearchDuration: estypes.AggregationsMaxAggregate; numTriggeredActions: estypes.AggregationsMaxAggregate; @@ -179,6 +182,11 @@ export function getExecutionLogAggregation({ field: START_FIELD, }, }, + scheduleDelay: { + max: { + field: SCHEDULE_DELAY_FIELD, + }, + }, totalSearchDuration: { max: { field: TOTAL_SEARCH_DURATION_FIELD, @@ -241,6 +249,9 @@ function formatExecutionLogAggBucket(bucket: IExecutionUuidAggBucket): IExecutio const durationUs = bucket?.ruleExecution?.executionDuration?.value ? bucket.ruleExecution.executionDuration.value : 0; + const scheduleDelayUs = bucket?.ruleExecution?.scheduleDelay?.value + ? bucket.ruleExecution.scheduleDelay.value + : 0; const timedOut = (bucket?.timeoutMessage?.doc_count ?? 0) > 0; const actionExecutionOutcomes = bucket?.actionExecution?.actionOutcomes?.buckets ?? []; @@ -263,6 +274,7 @@ function formatExecutionLogAggBucket(bucket: IExecutionUuidAggBucket): IExecutio num_errored_actions: actionExecutionError, total_search_duration_ms: bucket?.ruleExecution?.totalSearchDuration?.value ?? 0, es_search_duration_ms: bucket?.ruleExecution?.esSearchDuration?.value ?? 0, + schedule_delay_ms: scheduleDelayUs / Millis2Nanos, timed_out: timedOut, }; } diff --git a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts index a9750b15a0d11..6f095cceab347 100644 --- a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts +++ b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts @@ -43,6 +43,7 @@ describe('getRuleExecutionLogRoute', () => { total_search_duration_ms: 0, es_search_duration_ms: 0, timed_out: false, + schedule_delay_ms: 3126, }, { id: '41b2755e-765a-4044-9745-b03875d5e79a', @@ -60,6 +61,7 @@ describe('getRuleExecutionLogRoute', () => { total_search_duration_ms: 0, es_search_duration_ms: 0, timed_out: false, + schedule_delay_ms: 3008, }, ], }; diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts index 140431e1c7362..30b32d1bf6e3e 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts @@ -146,6 +146,9 @@ const aggregateResults = { ], }, }, + scheduleDelay: { + value: 3.126e9, + }, totalSearchDuration: { value: 0.0, }, @@ -225,6 +228,9 @@ const aggregateResults = { ], }, }, + scheduleDelay: { + value: 3.345e9, + }, totalSearchDuration: { value: 0.0, }, @@ -312,6 +318,7 @@ describe('getExecutionLogForRule()', () => { total_search_duration_ms: 0, es_search_duration_ms: 0, timed_out: false, + schedule_delay_ms: 3126, }, { id: '41b2755e-765a-4044-9745-b03875d5e79a', @@ -329,6 +336,7 @@ describe('getExecutionLogForRule()', () => { total_search_duration_ms: 0, es_search_duration_ms: 0, timed_out: false, + schedule_delay_ms: 3345, }, ], }); From ed5288316b5f885c49ca7cdea947d4e265cafece Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 9 Mar 2022 16:07:34 -0500 Subject: [PATCH 24/39] Limit to 1000 logs --- .../alerting/server/lib/get_execution_log_aggregation.test.ts | 2 +- .../alerting/server/lib/get_execution_log_aggregation.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts index bdc1eeafde613..5c6d172eca33c 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts @@ -39,7 +39,7 @@ describe('getNumExecutions', () => { new Date('2020-12-02T00:00:00.000Z'), '1s' ) - ).toEqual(65535); + ).toEqual(1000); }); }); diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts index ad6ab8633cf41..bf1cef603dcd3 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts @@ -10,7 +10,7 @@ import Boom from '@hapi/boom'; import { parseDuration } from '.'; import { AggregateEventsBySavedObjectResult } from '../../../event_log/server'; -const DEFAULT_MAX_BUCKETS_LIMIT = 65535; +const DEFAULT_MAX_BUCKETS_LIMIT = 1000; // do not retrieve more than this number of executions const PROVIDER_FIELD = 'event.provider'; const START_FIELD = 'event.start'; From 9d706ca0ef2a87c729565e709ed5cd6171b37cdd Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 9 Mar 2022 16:13:39 -0500 Subject: [PATCH 25/39] Fixing security tests --- .../privileges/feature_privilege_builder/alerting.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.test.ts b/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.test.ts index 861f6900fda58..a3b4b76980cd3 100644 --- a/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.test.ts +++ b/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/alerting.test.ts @@ -87,6 +87,7 @@ describe(`feature_privilege_builder`, () => { "alerting:1.0.0-zeta1:alert-type/my-feature/rule/get", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getRuleState", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getAlertSummary", + "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getExecutionLog", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/find", ] `); @@ -169,6 +170,7 @@ describe(`feature_privilege_builder`, () => { "alerting:1.0.0-zeta1:alert-type/my-feature/rule/get", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getRuleState", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getAlertSummary", + "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getExecutionLog", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/find", "alerting:1.0.0-zeta1:alert-type/my-feature/alert/get", "alerting:1.0.0-zeta1:alert-type/my-feature/alert/find", @@ -211,6 +213,7 @@ describe(`feature_privilege_builder`, () => { "alerting:1.0.0-zeta1:alert-type/my-feature/rule/get", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getRuleState", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getAlertSummary", + "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getExecutionLog", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/find", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/create", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/delete", @@ -304,6 +307,7 @@ describe(`feature_privilege_builder`, () => { "alerting:1.0.0-zeta1:alert-type/my-feature/rule/get", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getRuleState", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getAlertSummary", + "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getExecutionLog", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/find", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/create", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/delete", @@ -357,6 +361,7 @@ describe(`feature_privilege_builder`, () => { "alerting:1.0.0-zeta1:alert-type/my-feature/rule/get", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getRuleState", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getAlertSummary", + "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getExecutionLog", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/find", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/create", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/delete", @@ -371,6 +376,7 @@ describe(`feature_privilege_builder`, () => { "alerting:1.0.0-zeta1:readonly-alert-type/my-feature/rule/get", "alerting:1.0.0-zeta1:readonly-alert-type/my-feature/rule/getRuleState", "alerting:1.0.0-zeta1:readonly-alert-type/my-feature/rule/getAlertSummary", + "alerting:1.0.0-zeta1:readonly-alert-type/my-feature/rule/getExecutionLog", "alerting:1.0.0-zeta1:readonly-alert-type/my-feature/rule/find", ] `); @@ -456,6 +462,7 @@ describe(`feature_privilege_builder`, () => { "alerting:1.0.0-zeta1:alert-type/my-feature/rule/get", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getRuleState", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getAlertSummary", + "alerting:1.0.0-zeta1:alert-type/my-feature/rule/getExecutionLog", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/find", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/create", "alerting:1.0.0-zeta1:alert-type/my-feature/rule/delete", @@ -470,6 +477,7 @@ describe(`feature_privilege_builder`, () => { "alerting:1.0.0-zeta1:readonly-alert-type/my-feature/rule/get", "alerting:1.0.0-zeta1:readonly-alert-type/my-feature/rule/getRuleState", "alerting:1.0.0-zeta1:readonly-alert-type/my-feature/rule/getAlertSummary", + "alerting:1.0.0-zeta1:readonly-alert-type/my-feature/rule/getExecutionLog", "alerting:1.0.0-zeta1:readonly-alert-type/my-feature/rule/find", "alerting:1.0.0-zeta1:another-alert-type/my-feature/alert/get", "alerting:1.0.0-zeta1:another-alert-type/my-feature/alert/find", From 6822a79fddfeac4bec26c401b8c5430b9bd386b3 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 9 Mar 2022 20:14:58 -0500 Subject: [PATCH 26/39] Fixing unit tests --- .../rules_client/tests/get_execution_log.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts index 30b32d1bf6e3e..bda21f90e2324 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts @@ -359,7 +359,7 @@ describe('getExecutionLogForRule()', () => { ['1'], { aggs: getExecutionLogAggregation({ - numExecutions: 3600, + numExecutions: 1000, page: 1, perPage: 10, sortField: 'timestamp', @@ -387,7 +387,7 @@ describe('getExecutionLogForRule()', () => { ['1'], { aggs: getExecutionLogAggregation({ - numExecutions: 3600, + numExecutions: 1000, page: 1, perPage: 10, sortField: 'timestamp', @@ -405,7 +405,7 @@ describe('getExecutionLogForRule()', () => { eventLogClient.aggregateEventsBySavedObjectIds.mockResolvedValueOnce(aggregateResults); await rulesClient.getExecutionLogForRule( - getExecutionLogByIdParams({ dateEnd: new Date(Date.now() - 1800000).toISOString() }) + getExecutionLogByIdParams({ dateEnd: new Date(Date.now() - 2700000).toISOString() }) ); expect(unsecuredSavedObjectsClient.get).toHaveBeenCalledTimes(1); @@ -415,13 +415,13 @@ describe('getExecutionLogForRule()', () => { ['1'], { aggs: getExecutionLogAggregation({ - numExecutions: 1800, + numExecutions: 900, page: 1, perPage: 10, sortField: 'timestamp', sortOrder: 'desc', }), - end: '2019-02-12T20:31:22.479Z', + end: '2019-02-12T20:16:22.479Z', start: '2019-02-12T20:01:22.479Z', }, undefined, @@ -443,7 +443,7 @@ describe('getExecutionLogForRule()', () => { ['1'], { aggs: getExecutionLogAggregation({ - numExecutions: 3600, + numExecutions: 1000, page: 1, perPage: 10, sortField: 'timestamp', From eae7c6a6954038d2827039ef13573140b6da8c89 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 9 Mar 2022 20:17:29 -0500 Subject: [PATCH 27/39] Validating numExecutions --- .../lib/get_execution_log_aggregation.test.ts | 14 ++++++++++++++ .../server/lib/get_execution_log_aggregation.ts | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts index 5c6d172eca33c..d747768a02f43 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts @@ -70,6 +70,20 @@ describe('getExecutionLogAggregation', () => { }).toThrowErrorMatchingInlineSnapshot(`"Invalid page field \\"0\\" - must be greater than 0"`); }); + test('should throw error when given bad numExecutions field', () => { + expect(() => { + getExecutionLogAggregation({ + numExecutions: 1001, + page: 1, + perPage: 10, + sortField: 'timestamp', + sortOrder: 'asc', + }); + }).toThrowErrorMatchingInlineSnapshot( + `"Invalid numExecutions requested \\"1001\\" - must be less than 1000"` + ); + }); + test('should throw error when given bad perPage field', () => { expect(() => { getExecutionLogAggregation({ diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts index bf1cef603dcd3..c7d315cbce8d0 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts @@ -123,6 +123,13 @@ export function getExecutionLogAggregation({ throw Boom.badRequest(`Invalid perPage field "${perPage}" - must be greater than 0`); } + // Check if valid num executions + if (numExecutions > DEFAULT_MAX_BUCKETS_LIMIT) { + throw Boom.badRequest( + `Invalid numExecutions requested "${numExecutions}" - must be less than ${DEFAULT_MAX_BUCKETS_LIMIT}` + ); + } + return { // Get total number of executions executionUuidCardinality: { From 8636b8f551820ddb2ae9e11f5dd41676a3fd5d39 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 9 Mar 2022 21:21:16 -0500 Subject: [PATCH 28/39] Changing sort input format --- .../lib/get_execution_log_aggregation.test.ts | 33 +++++++++++------ .../lib/get_execution_log_aggregation.ts | 37 ++++++++++--------- .../routes/get_rule_execution_log.test.ts | 6 +-- .../server/routes/get_rule_execution_log.ts | 22 ++++++----- .../server/rules_client/rules_client.ts | 9 ++--- .../tests/get_execution_log.test.ts | 21 +++++------ 6 files changed, 67 insertions(+), 61 deletions(-) diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts index d747768a02f43..d5b5ef9d064ba 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts @@ -50,8 +50,20 @@ describe('getExecutionLogAggregation', () => { numExecutions: 5, page: 1, perPage: 10, - sortField: 'notsortable', - sortOrder: 'asc', + sort: [{ notsortable: { order: 'asc' } }], + }); + }).toThrowErrorMatchingInlineSnapshot( + `"Invalid sort field \\"notsortable\\" - must be one of [timestamp,duration]"` + ); + }); + + test('should throw error when given one bad sort field', () => { + expect(() => { + getExecutionLogAggregation({ + numExecutions: 5, + page: 1, + perPage: 10, + sort: [{ notsortable: { order: 'asc' } }, { timestamp: { order: 'asc' } }], }); }).toThrowErrorMatchingInlineSnapshot( `"Invalid sort field \\"notsortable\\" - must be one of [timestamp,duration]"` @@ -64,8 +76,7 @@ describe('getExecutionLogAggregation', () => { numExecutions: 5, page: 0, perPage: 10, - sortField: 'timestamp', - sortOrder: 'asc', + sort: [{ timestamp: { order: 'asc' } }], }); }).toThrowErrorMatchingInlineSnapshot(`"Invalid page field \\"0\\" - must be greater than 0"`); }); @@ -76,8 +87,7 @@ describe('getExecutionLogAggregation', () => { numExecutions: 1001, page: 1, perPage: 10, - sortField: 'timestamp', - sortOrder: 'asc', + sort: [{ timestamp: { order: 'asc' } }], }); }).toThrowErrorMatchingInlineSnapshot( `"Invalid numExecutions requested \\"1001\\" - must be less than 1000"` @@ -90,8 +100,7 @@ describe('getExecutionLogAggregation', () => { numExecutions: 5, page: 1, perPage: 0, - sortField: 'timestamp', - sortOrder: 'asc', + sort: [{ timestamp: { order: 'asc' } }], }); }).toThrowErrorMatchingInlineSnapshot( `"Invalid perPage field \\"0\\" - must be greater than 0"` @@ -104,8 +113,7 @@ describe('getExecutionLogAggregation', () => { numExecutions: 5, page: 2, perPage: 10, - sortField: 'timestamp', - sortOrder: 'asc', + sort: [{ timestamp: { order: 'asc' } }, { duration: { order: 'desc' } }], }) ).toEqual({ executionUuidCardinality: { cardinality: { field: 'kibana.alert.rule.execution.uuid' } }, @@ -114,7 +122,10 @@ describe('getExecutionLogAggregation', () => { aggs: { executionUuidSorted: { bucket_sort: { - sort: [{ 'ruleExecution>executeStartTime': { order: 'asc' } }], + sort: [ + { 'ruleExecution>executeStartTime': { order: 'asc' } }, + { 'ruleExecution>executionDuration': { order: 'desc' } }, + ], from: 10, size: 10, }, diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts index c7d315cbce8d0..64afa783c6b89 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts @@ -7,6 +7,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import Boom from '@hapi/boom'; +import { flatMap, get } from 'lodash'; import { parseDuration } from '.'; import { AggregateEventsBySavedObjectResult } from '../../../event_log/server'; @@ -88,8 +89,7 @@ export interface IExecutionLogAggOptions { numExecutions: number; page: number; perPage: number; - sortField: string; - sortOrder: estypes.SortOrder; + sort: estypes.Sort; } const ExecutionLogSortFields: Record = { @@ -101,16 +101,18 @@ export function getExecutionLogAggregation({ numExecutions, page, perPage, - sortField, - sortOrder, + sort, }: IExecutionLogAggOptions) { - // Check if valid sort field - if (!Object.keys(ExecutionLogSortFields).includes(sortField)) { - throw Boom.badRequest( - `Invalid sort field "${sortField}" - must be one of [${Object.keys( - ExecutionLogSortFields - ).join(',')}]` - ); + // Check if valid sort fields + const sortFields = flatMap(sort as estypes.SortCombinations[], (s) => Object.keys(s)); + for (const field of sortFields) { + if (!Object.keys(ExecutionLogSortFields).includes(field)) { + throw Boom.badRequest( + `Invalid sort field "${field}" - must be one of [${Object.keys(ExecutionLogSortFields).join( + ',' + )}]` + ); + } } // Check if valid page value @@ -147,13 +149,12 @@ export function getExecutionLogAggregation({ // Bucket sort to allow paging through executions executionUuidSorted: { bucket_sort: { - sort: [ - { - [ExecutionLogSortFields[sortField]]: { - order: sortOrder, - }, - }, - ], + sort: (sort as estypes.SortCombinations[]).map((s) => + Object.keys(s).reduce( + (acc, curr) => ({ ...acc, [ExecutionLogSortFields[curr]]: get(s, curr) }), + {} + ) + ), from: (page - 1) * perPage, size: perPage, }, diff --git a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts index 6f095cceab347..e359e9c52dda0 100644 --- a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts +++ b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.test.ts @@ -88,8 +88,7 @@ describe('getRuleExecutionLogRoute', () => { date_start: dateString, per_page: 10, page: 1, - sort_field: 'timestamp', - sort_order: 'desc', + sort: [{ timestamp: { order: 'desc' } }], }, }, ['ok'] @@ -104,8 +103,7 @@ describe('getRuleExecutionLogRoute', () => { id: '1', page: 1, perPage: 10, - sortField: 'timestamp', - sortOrder: 'desc', + sort: [{ timestamp: { order: 'desc' } }], }, ]); diff --git a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts index 0a35676a569fd..687be9e0f1869 100644 --- a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts +++ b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts @@ -16,34 +16,36 @@ const paramSchema = schema.object({ id: schema.string(), }); +const sortOrderSchema = schema.oneOf([schema.literal('asc'), schema.literal('desc')]); + +const sortFieldSchema = schema.oneOf([ + schema.object({ timestamp: schema.object({ order: sortOrderSchema }) }), + schema.object({ duration: schema.object({ order: sortOrderSchema }) }), +]); + +const sortFieldsSchema = schema.arrayOf(sortFieldSchema, { + defaultValue: [{ timestamp: { order: 'desc' } }], +}); + const querySchema = schema.object({ date_start: schema.string(), date_end: schema.maybe(schema.string()), filter: schema.maybe(schema.string()), per_page: schema.number({ defaultValue: 10, min: 1 }), page: schema.number({ defaultValue: 1, min: 1 }), - sort_field: schema.oneOf([schema.literal('timestamp'), schema.literal('duration')], { - defaultValue: 'timestamp', - }), - sort_order: schema.oneOf([schema.literal('asc'), schema.literal('desc')], { - defaultValue: 'desc', - }), + sort: sortFieldsSchema, }); const rewriteReq: RewriteRequestCase = ({ date_start: dateStart, date_end: dateEnd, per_page: perPage, - sort_field: sortField, - sort_order: sortOrder, ...rest }) => ({ ...rest, dateStart, dateEnd, perPage, - sortField, - sortOrder, }); export const getRuleExecutionLogRoute = ( diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index 006c7b97507f3..e1610d92423ce 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -242,8 +242,7 @@ export interface GetExecutionLogByIdParams { filter?: string; page: number; perPage: number; - sortField: string; - sortOrder: estypes.SortOrder; + sort: estypes.Sort; } // NOTE: Changing this prefix will require a migration to update the prefix in all existing `rule` saved objects @@ -655,8 +654,7 @@ export class RulesClient { filter, page, perPage, - sortField, - sortOrder, + sort, }: GetExecutionLogByIdParams): Promise { this.logger.debug(`getExecutionLogForRule(): getting execution log for rule ${id}`); const rule = (await this.get({ id, includeLegacyId: true })) as SanitizedRuleWithLegacyId; @@ -690,8 +688,7 @@ export class RulesClient { numExecutions: getNumExecutions(parsedDateStart, parsedDateEnd, rule.schedule.interval), page, perPage, - sortField, - sortOrder, + sort, }), }, rule.legacyId !== null ? [rule.legacyId] : undefined diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts index bda21f90e2324..2c552836eb4a1 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts @@ -281,8 +281,7 @@ function getExecutionLogByIdParams(overwrites = {}) { dateStart: new Date(Date.now() - 3600000).toISOString(), page: 1, perPage: 10, - sortField: 'timestamp', - sortOrder: 'desc' as estypes.SortOrder, + sort: [{ timestamp: { order: 'desc' } }] as estypes.Sort, ...overwrites, }; } @@ -362,8 +361,7 @@ describe('getExecutionLogForRule()', () => { numExecutions: 1000, page: 1, perPage: 10, - sortField: 'timestamp', - sortOrder: 'desc', + sort: [{ timestamp: { order: 'desc' } }], }), end: mockedDateString, start: '2019-02-12T20:01:22.479Z', @@ -390,8 +388,7 @@ describe('getExecutionLogForRule()', () => { numExecutions: 1000, page: 1, perPage: 10, - sortField: 'timestamp', - sortOrder: 'desc', + sort: [{ timestamp: { order: 'desc' } }], }), end: mockedDateString, start: '2019-02-12T20:01:22.479Z', @@ -418,8 +415,7 @@ describe('getExecutionLogForRule()', () => { numExecutions: 900, page: 1, perPage: 10, - sortField: 'timestamp', - sortOrder: 'desc', + sort: [{ timestamp: { order: 'desc' } }], }), end: '2019-02-12T20:16:22.479Z', start: '2019-02-12T20:01:22.479Z', @@ -446,8 +442,7 @@ describe('getExecutionLogForRule()', () => { numExecutions: 1000, page: 1, perPage: 10, - sortField: 'timestamp', - sortOrder: 'desc', + sort: [{ timestamp: { order: 'desc' } }], }), filter: 'event.outcome: success', end: mockedDateString, @@ -499,12 +494,14 @@ describe('getExecutionLogForRule()', () => { ).rejects.toMatchInlineSnapshot(`[Error: Invalid perPage field "-3" - must be greater than 0]`); }); - test('invalid sortField value throws an error', async () => { + test('invalid sort value throws an error', async () => { unsecuredSavedObjectsClient.get.mockResolvedValueOnce(getRuleSavedObject()); eventLogClient.aggregateEventsBySavedObjectIds.mockResolvedValueOnce(aggregateResults); expect( - rulesClient.getExecutionLogForRule(getExecutionLogByIdParams({ sortField: 'foo' })) + rulesClient.getExecutionLogForRule( + getExecutionLogByIdParams({ sort: [{ foo: { order: 'desc' } }] }) + ) ).rejects.toMatchInlineSnapshot( `[Error: Invalid sort field "foo" - must be one of [timestamp,duration]]` ); From dc24753802302da3523cf913620cbce72809d2e6 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 9 Mar 2022 21:27:54 -0500 Subject: [PATCH 29/39] Adding more sort fields --- .../server/lib/get_execution_log_aggregation.test.ts | 6 +++--- .../alerting/server/lib/get_execution_log_aggregation.ts | 6 +++++- .../alerting/server/routes/get_rule_execution_log.ts | 6 +++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts index d5b5ef9d064ba..91471d530c506 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts @@ -53,7 +53,7 @@ describe('getExecutionLogAggregation', () => { sort: [{ notsortable: { order: 'asc' } }], }); }).toThrowErrorMatchingInlineSnapshot( - `"Invalid sort field \\"notsortable\\" - must be one of [timestamp,duration]"` + `"Invalid sort field \\"notsortable\\" - must be one of [timestamp,execution_duration,total_search_duration,es_search_duration,schedule_delay,num_triggered_actions]"` ); }); @@ -66,7 +66,7 @@ describe('getExecutionLogAggregation', () => { sort: [{ notsortable: { order: 'asc' } }, { timestamp: { order: 'asc' } }], }); }).toThrowErrorMatchingInlineSnapshot( - `"Invalid sort field \\"notsortable\\" - must be one of [timestamp,duration]"` + `"Invalid sort field \\"notsortable\\" - must be one of [timestamp,execution_duration,total_search_duration,es_search_duration,schedule_delay,num_triggered_actions]"` ); }); @@ -113,7 +113,7 @@ describe('getExecutionLogAggregation', () => { numExecutions: 5, page: 2, perPage: 10, - sort: [{ timestamp: { order: 'asc' } }, { duration: { order: 'desc' } }], + sort: [{ timestamp: { order: 'asc' } }, { execution_duration: { order: 'desc' } }], }) ).toEqual({ executionUuidCardinality: { cardinality: { field: 'kibana.alert.rule.execution.uuid' } }, diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts index 64afa783c6b89..5775f5d2faec4 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts @@ -94,7 +94,11 @@ export interface IExecutionLogAggOptions { const ExecutionLogSortFields: Record = { timestamp: 'ruleExecution>executeStartTime', - duration: 'ruleExecution>executionDuration', + execution_duration: 'ruleExecution>executionDuration', + total_search_duration: 'ruleExecution>totalSearchDuration', + es_search_duration: 'ruleExecution>esSearchDuration', + schedule_delay: 'ruleExecution>scheduleDelay', + num_triggered_actions: 'ruleExecution>numTriggeredActions', }; export function getExecutionLogAggregation({ diff --git a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts index 687be9e0f1869..845c14ecf0ea4 100644 --- a/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts +++ b/x-pack/plugins/alerting/server/routes/get_rule_execution_log.ts @@ -20,7 +20,11 @@ const sortOrderSchema = schema.oneOf([schema.literal('asc'), schema.literal('des const sortFieldSchema = schema.oneOf([ schema.object({ timestamp: schema.object({ order: sortOrderSchema }) }), - schema.object({ duration: schema.object({ order: sortOrderSchema }) }), + schema.object({ execution_duration: schema.object({ order: sortOrderSchema }) }), + schema.object({ total_search_duration: schema.object({ order: sortOrderSchema }) }), + schema.object({ es_search_duration: schema.object({ order: sortOrderSchema }) }), + schema.object({ schedule_delay: schema.object({ order: sortOrderSchema }) }), + schema.object({ num_triggered_actions: schema.object({ order: sortOrderSchema }) }), ]); const sortFieldsSchema = schema.arrayOf(sortFieldSchema, { From 7a3f20f2014b1132d69c398ef29e104c28ebdee6 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Thu, 10 Mar 2022 07:48:50 -0500 Subject: [PATCH 30/39] Fixing unit tests --- .../server/rules_client/tests/get_execution_log.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts index 2c552836eb4a1..89ab7eb52ab83 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts @@ -503,7 +503,7 @@ describe('getExecutionLogForRule()', () => { getExecutionLogByIdParams({ sort: [{ foo: { order: 'desc' } }] }) ) ).rejects.toMatchInlineSnapshot( - `[Error: Invalid sort field "foo" - must be one of [timestamp,duration]]` + `[Error: Invalid sort field "foo" - must be one of [timestamp,execution_duration,total_search_duration,es_search_duration,schedule_delay,num_triggered_actions]]` ); }); From dfd1fb3e661c0bd51cc3626ab9385b559f5c3bfa Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Thu, 10 Mar 2022 10:49:25 -0500 Subject: [PATCH 31/39] Adding functional tests --- .../tests/alerting/get_execution_log.ts | 456 ++++++++++++++++++ .../spaces_only/tests/alerting/index.ts | 1 + 2 files changed, 457 insertions(+) create mode 100644 x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get_execution_log.ts diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get_execution_log.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get_execution_log.ts new file mode 100644 index 0000000000000..27a444dc3013c --- /dev/null +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get_execution_log.ts @@ -0,0 +1,456 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { Spaces } from '../../scenarios'; +import { + getUrlPrefix, + ObjectRemover, + getTestRuleData, + getEventLog, + ESTestIndexTool, +} from '../../../common/lib'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default function createGetExecutionLogTests({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const retry = getService('retry'); + const es = getService('es'); + const esTestIndexTool = new ESTestIndexTool(es, retry); + + const dateStart = new Date(Date.now() - 600000).toISOString(); + + describe('getExecutionLog', () => { + const objectRemover = new ObjectRemover(supertest); + + beforeEach(async () => { + await esTestIndexTool.destroy(); + await esTestIndexTool.setup(); + }); + + afterEach(() => objectRemover.removeAll()); + + it(`handles non-existent rule`, async () => { + await supertest + .get( + `${getUrlPrefix( + Spaces.space1.id + )}/internal/alerting/rule/1/_execution_log?date_start=${dateStart}` + ) + .expect(404, { + statusCode: 404, + error: 'Not Found', + message: 'Saved object [alert/1] not found', + }); + }); + + it('gets execution log for rule with executions', async () => { + const { body: createdRule } = await supertest + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send(getTestRuleData({ schedule: { interval: '15s' } })) + .expect(200); + objectRemover.add(Spaces.space1.id, createdRule.id, 'rule', 'alerting'); + + await waitForEvents(createdRule.id, 'alerting', new Map([['execute', { gte: 2 }]])); + const response = await supertest.get( + `${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ + createdRule.id + }/_execution_log?date_start=${dateStart}` + ); + + expect(response.status).to.eql(200); + + expect(response.body.total).to.eql(2); + + const execLogs = response.body.data; + expect(execLogs.length).to.eql(2); + + let previousTimestamp: string | null = null; + for (const log of execLogs) { + if (previousTimestamp) { + // default sort is `desc` by timestamp + expect(Date.parse(log.timestamp)).to.be.lessThan(Date.parse(previousTimestamp)); + } + previousTimestamp = log.timstamp; + expect(Date.parse(log.timestamp)).to.be.greaterThan(Date.parse(dateStart)); + expect(Date.parse(log.timestamp)).to.be.lessThan(Date.parse(new Date().toISOString())); + + expect(log.duration_ms).to.be.greaterThan(0); + expect(log.schedule_delay_ms).to.be.greaterThan(0); + expect(log.status).to.equal('success'); + expect(log.timed_out).to.equal(false); + + // no-op rule doesn't generate alerts + expect(log.num_active_alerts).to.equal(0); + expect(log.num_new_alerts).to.equal(0); + expect(log.num_recovered_alerts).to.equal(0); + expect(log.num_triggered_actions).to.equal(0); + expect(log.num_succeeded_actions).to.equal(0); + expect(log.num_errored_actions).to.equal(0); + + // no-op rule doesn't query ES + expect(log.total_search_duration_ms).to.equal(0); + expect(log.es_search_duration_ms).to.equal(0); + } + }); + + it('gets execution log for rule with no executions', async () => { + const { body: createdRule } = await supertest + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send(getTestRuleData({ schedule: { interval: '15s' } })) + .expect(200); + objectRemover.add(Spaces.space1.id, createdRule.id, 'rule', 'alerting'); + + const response = await supertest.get( + `${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ + createdRule.id + }/_execution_log?date_start=${dateStart}` + ); + + expect(response.status).to.eql(200); + + expect(response.body.total).to.eql(0); + expect(response.body.data).to.eql([]); + }); + + it('gets execution log for rule that performs ES searches', async () => { + const { body: createdRule } = await supertest + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send( + getTestRuleData({ + rule_type_id: 'test.multipleSearches', + params: { + numSearches: 2, + delay: `2s`, + }, + }) + ) + .expect(200); + objectRemover.add(Spaces.space1.id, createdRule.id, 'rule', 'alerting'); + + await waitForEvents(createdRule.id, 'alerting', new Map([['execute', { gte: 1 }]])); + const response = await supertest.get( + `${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ + createdRule.id + }/_execution_log?date_start=${dateStart}` + ); + + expect(response.status).to.eql(200); + + expect(response.body.total).to.eql(1); + + const execLogs = response.body.data; + expect(execLogs.length).to.eql(1); + + for (const log of execLogs) { + expect(log.duration_ms).to.be.greaterThan(0); + expect(log.schedule_delay_ms).to.be.greaterThan(0); + expect(log.status).to.equal('success'); + expect(log.timed_out).to.equal(false); + + // no-op rule doesn't generate alerts + expect(log.num_active_alerts).to.equal(0); + expect(log.num_new_alerts).to.equal(0); + expect(log.num_recovered_alerts).to.equal(0); + expect(log.num_triggered_actions).to.equal(0); + expect(log.num_succeeded_actions).to.equal(0); + expect(log.num_errored_actions).to.equal(0); + + // rule executes 2 searches with delay of 2 seconds each + // setting compare threshold lower to avoid flakiness + expect(log.total_search_duration_ms).to.be.greaterThan(2000); + expect(log.es_search_duration_ms).to.be.greaterThan(2000); + } + }); + + it('gets execution log for rule that errors', async () => { + const { body: createdRule } = await supertest + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send( + getTestRuleData({ + rule_type_id: 'test.throw', + }) + ) + .expect(200); + objectRemover.add(Spaces.space1.id, createdRule.id, 'rule', 'alerting'); + + await waitForEvents(createdRule.id, 'alerting', new Map([['execute', { gte: 1 }]])); + const response = await supertest.get( + `${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ + createdRule.id + }/_execution_log?date_start=${dateStart}` + ); + + expect(response.status).to.eql(200); + + expect(response.body.total).to.eql(1); + + const execLogs = response.body.data; + expect(execLogs.length).to.eql(1); + + for (const log of execLogs) { + expect(log.status).to.equal('failure'); + expect(log.timed_out).to.equal(false); + } + }); + + it('gets execution log for rule that times out', async () => { + const { body: createdRule } = await supertest + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send( + getTestRuleData({ + rule_type_id: 'test.patternLongRunning', + params: { + pattern: [true, true, true, true], + }, + }) + ) + .expect(200); + objectRemover.add(Spaces.space1.id, createdRule.id, 'rule', 'alerting'); + + await waitForEvents(createdRule.id, 'alerting', new Map([['execute', { gte: 1 }]])); + const response = await supertest.get( + `${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ + createdRule.id + }/_execution_log?date_start=${dateStart}` + ); + + expect(response.status).to.eql(200); + + expect(response.body.total).to.eql(1); + + const execLogs = response.body.data; + expect(execLogs.length).to.eql(1); + + for (const log of execLogs) { + expect(log.status).to.equal('success'); + expect(log.timed_out).to.equal(true); + } + }); + + it('gets execution log for rule that triggers actions', async () => { + const { body: createdConnector } = await supertest + .post(`${getUrlPrefix(Spaces.space1.id)}/api/actions/connector`) + .set('kbn-xsrf', 'foo') + .send({ + name: 'noop connector', + connector_type_id: 'test.noop', + config: {}, + secrets: {}, + }) + .expect(200); + objectRemover.add(Spaces.space1.id, createdConnector.id, 'action', 'actions'); + const { body: createdRule } = await supertest + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send( + getTestRuleData({ + rule_type_id: 'test.cumulative-firing', + actions: [ + { + id: createdConnector.id, + group: 'default', + params: {}, + }, + ], + }) + ) + .expect(200); + objectRemover.add(Spaces.space1.id, createdRule.id, 'rule', 'alerting'); + + await waitForEvents(createdRule.id, 'alerting', new Map([['execute', { gte: 1 }]])); + await waitForEvents(createdRule.id, 'actions', new Map([['execute', { gte: 1 }]])); + const response = await supertest.get( + `${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ + createdRule.id + }/_execution_log?date_start=${dateStart}` + ); + + expect(response.status).to.eql(200); + + expect(response.body.total).to.eql(1); + + const execLogs = response.body.data; + expect(execLogs.length).to.eql(1); + + for (const log of execLogs) { + expect(log.status).to.equal('success'); + + expect(log.num_active_alerts).to.equal(1); + expect(log.num_new_alerts).to.equal(1); + expect(log.num_recovered_alerts).to.equal(0); + expect(log.num_triggered_actions).to.equal(1); + expect(log.num_succeeded_actions).to.equal(1); + expect(log.num_errored_actions).to.equal(0); + } + }); + + it('gets execution log for rule that has failed actions', async () => { + const { body: createdConnector } = await supertest + .post(`${getUrlPrefix(Spaces.space1.id)}/api/actions/connector`) + .set('kbn-xsrf', 'foo') + .send({ + name: 'connector that throws', + connector_type_id: 'test.throw', + config: {}, + secrets: {}, + }) + .expect(200); + objectRemover.add(Spaces.space1.id, createdConnector.id, 'action', 'actions'); + const { body: createdRule } = await supertest + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send( + getTestRuleData({ + rule_type_id: 'test.cumulative-firing', + actions: [ + { + id: createdConnector.id, + group: 'default', + params: {}, + }, + ], + }) + ) + .expect(200); + objectRemover.add(Spaces.space1.id, createdRule.id, 'rule', 'alerting'); + + await waitForEvents(createdRule.id, 'alerting', new Map([['execute', { gte: 1 }]])); + await waitForEvents(createdRule.id, 'actions', new Map([['execute', { gte: 1 }]])); + const response = await supertest.get( + `${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ + createdRule.id + }/_execution_log?date_start=${dateStart}` + ); + + expect(response.status).to.eql(200); + + expect(response.body.total).to.eql(1); + + const execLogs = response.body.data; + expect(execLogs.length).to.eql(1); + + for (const log of execLogs) { + expect(log.status).to.equal('success'); + + expect(log.num_active_alerts).to.equal(1); + expect(log.num_new_alerts).to.equal(1); + expect(log.num_recovered_alerts).to.equal(0); + expect(log.num_triggered_actions).to.equal(1); + expect(log.num_succeeded_actions).to.equal(0); + expect(log.num_errored_actions).to.equal(1); + } + }); + + it('handles date_end if specified', async () => { + const { body: createdRule } = await supertest + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send(getTestRuleData({ schedule: { interval: '10s' } })) + .expect(200); + objectRemover.add(Spaces.space1.id, createdRule.id, 'rule', 'alerting'); + + await waitForEvents(createdRule.id, 'alerting', new Map([['execute', { gte: 2 }]])); + + // set the date end to date start - should filter out all execution logs + const earlierDateStart = new Date(Date.now() - 900000).toISOString(); + const response = await supertest.get( + `${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ + createdRule.id + }/_execution_log?date_start=${earlierDateStart}&date_end=${dateStart}` + ); + + expect(response.status).to.eql(200); + + expect(response.body.total).to.eql(0); + expect(response.body.data.length).to.eql(0); + }); + + it('handles sort query parameter', async () => { + const { body: createdRule } = await supertest + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send(getTestRuleData({ schedule: { interval: '5s' } })) + .expect(200); + objectRemover.add(Spaces.space1.id, createdRule.id, 'rule', 'alerting'); + + await waitForEvents(createdRule.id, 'alerting', new Map([['execute', { gte: 3 }]])); + const response = await supertest.get( + `${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ + createdRule.id + }/_execution_log?date_start=${dateStart}&sort=[{"timestamp":{"order":"asc"}}]` + ); + + expect(response.status).to.eql(200); + + expect(response.body.total).to.eql(3); + + const execLogs = response.body.data; + expect(execLogs.length).to.eql(3); + + let previousTimestamp: string | null = null; + for (const log of execLogs) { + if (previousTimestamp) { + // sorting by `asc` timestamp + expect(Date.parse(log.timestamp)).to.be.greaterThan(Date.parse(previousTimestamp)); + } + previousTimestamp = log.timstamp; + } + }); + + it(`handles invalid date_start`, async () => { + const { body: createdRule } = await supertest + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send(getTestRuleData({ schedule: { interval: '10s' } })) + .expect(200); + objectRemover.add(Spaces.space1.id, createdRule.id, 'rule', 'alerting'); + + await waitForEvents(createdRule.id, 'alerting', new Map([['execute', { gte: 2 }]])); + await supertest + .get( + `${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ + createdRule.id + }/_execution_log?date_start=X0X0-08-08T08:08:08.008Z` + ) + .expect(400, { + statusCode: 400, + error: 'Bad Request', + message: 'Invalid date for parameter dateStart: "X0X0-08-08T08:08:08.008Z"', + }); + }); + }); + + async function waitForEvents( + id: string, + provider: string, + actions: Map< + string, + { + gte: number; + } + > + ) { + await retry.try(async () => { + return await getEventLog({ + getService, + spaceId: Spaces.space1.id, + type: 'alert', + id, + provider, + actions, + }); + }); + } +} diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/index.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/index.ts index 242c6ffcba10f..14c8268ce80e0 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/index.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/index.ts @@ -23,6 +23,7 @@ export default function alertingTests({ loadTestFile, getService }: FtrProviderC loadTestFile(require.resolve('./get')); loadTestFile(require.resolve('./get_alert_state')); loadTestFile(require.resolve('./get_alert_summary')); + loadTestFile(require.resolve('./get_execution_log')); loadTestFile(require.resolve('./rule_types')); loadTestFile(require.resolve('./event_log')); loadTestFile(require.resolve('./execution_status')); From 73b7278ce7ddae50b1958459b28f6143222fde9b Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Thu, 10 Mar 2022 13:50:25 -0500 Subject: [PATCH 32/39] Adding sort to terms aggregation --- .../lib/get_execution_log_aggregation.test.ts | 39 ++++++++++++++++++- .../lib/get_execution_log_aggregation.ts | 26 ++++++++++--- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts index 91471d530c506..d364e64efed53 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts @@ -9,8 +9,38 @@ import { getNumExecutions, getExecutionLogAggregation, formatExecutionLogResult, + formatSortForBucketSort, + formatSortForTermSort, } from './get_execution_log_aggregation'; +describe('formatSortForBucketSort', () => { + test('should correctly format array of sort combinations for bucket sorting', () => { + expect( + formatSortForBucketSort([ + { timestamp: { order: 'desc' } }, + { execution_duration: { order: 'asc' } }, + ]) + ).toEqual([ + { 'ruleExecution>executeStartTime': { order: 'desc' } }, + { 'ruleExecution>executionDuration': { order: 'asc' } }, + ]); + }); +}); + +describe('formatSortForTermSort', () => { + test('should correctly format array of sort combinations for bucket sorting', () => { + expect( + formatSortForTermSort([ + { timestamp: { order: 'desc' } }, + { execution_duration: { order: 'asc' } }, + ]) + ).toEqual([ + { 'ruleExecution>executeStartTime': 'desc' }, + { 'ruleExecution>executionDuration': 'asc' }, + ]); + }); +}); + describe('getNumExecutions', () => { test('should calculate the expected number of executions in a given date range with a given schedule interval', () => { expect( @@ -118,7 +148,14 @@ describe('getExecutionLogAggregation', () => { ).toEqual({ executionUuidCardinality: { cardinality: { field: 'kibana.alert.rule.execution.uuid' } }, executionUuid: { - terms: { field: 'kibana.alert.rule.execution.uuid', size: 5 }, + terms: { + field: 'kibana.alert.rule.execution.uuid', + size: 5, + order: [ + { 'ruleExecution>executeStartTime': 'asc' }, + { 'ruleExecution>executionDuration': 'desc' }, + ], + }, aggs: { executionUuidSorted: { bucket_sort: { diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts index 5775f5d2faec4..2aab07284910a 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts @@ -148,17 +148,13 @@ export function getExecutionLogAggregation({ terms: { field: EXECUTION_UUID_FIELD, size: numExecutions, + order: formatSortForTermSort(sort), }, aggs: { // Bucket sort to allow paging through executions executionUuidSorted: { bucket_sort: { - sort: (sort as estypes.SortCombinations[]).map((s) => - Object.keys(s).reduce( - (acc, curr) => ({ ...acc, [ExecutionLogSortFields[curr]]: get(s, curr) }), - {} - ) - ), + sort: formatSortForBucketSort(sort), from: (page - 1) * perPage, size: perPage, }, @@ -321,3 +317,21 @@ export function getNumExecutions(dateStart: Date, dateEnd: Date, ruleSchedule: s return Math.min(numExecutions < 0 ? 0 : numExecutions, DEFAULT_MAX_BUCKETS_LIMIT); } + +export function formatSortForBucketSort(sort: estypes.Sort) { + return (sort as estypes.SortCombinations[]).map((s) => + Object.keys(s).reduce( + (acc, curr) => ({ ...acc, [ExecutionLogSortFields[curr]]: get(s, curr) }), + {} + ) + ); +} + +export function formatSortForTermSort(sort: estypes.Sort) { + return (sort as estypes.SortCombinations[]).map((s) => + Object.keys(s).reduce( + (acc, curr) => ({ ...acc, [ExecutionLogSortFields[curr]]: get(s, `${curr}.order`) }), + {} + ) + ); +} From ce98738bc7ce2342ef6374121dca2c01c8deadac Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Thu, 10 Mar 2022 15:16:06 -0500 Subject: [PATCH 33/39] Fixing functional test --- .../spaces_only/tests/alerting/get_execution_log.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get_execution_log.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get_execution_log.ts index 27a444dc3013c..55d4a72643c86 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get_execution_log.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get_execution_log.ts @@ -364,7 +364,7 @@ export default function createGetExecutionLogTests({ getService }: FtrProviderCo await waitForEvents(createdRule.id, 'alerting', new Map([['execute', { gte: 2 }]])); // set the date end to date start - should filter out all execution logs - const earlierDateStart = new Date(Date.now() - 900000).toISOString(); + const earlierDateStart = new Date(new Date(dateStart).getTime() - 900000).toISOString(); const response = await supertest.get( `${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ createdRule.id From 1fad8678bf566dbc00b78d1e043c3d95e331f5b0 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Mon, 14 Mar 2022 13:26:45 -0400 Subject: [PATCH 34/39] Adding audit event for rule GET --- x-pack/plugins/alerting/README.md | 1 + .../server/rules_client/rules_client.ts | 32 +++++-- .../tests/get_execution_log.test.ts | 83 +++++++++++++++++++ 3 files changed, 109 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/alerting/README.md b/x-pack/plugins/alerting/README.md index 903d190a216bd..6dde7de84aab4 100644 --- a/x-pack/plugins/alerting/README.md +++ b/x-pack/plugins/alerting/README.md @@ -643,6 +643,7 @@ When a user is granted the `read` role in the Alerting Framework, they will be a - `get` - `getRuleState` - `getAlertSummary` +- `getExecutionLog` - `find` When a user is granted the `all` role in the Alerting Framework, they will be able to execute all of the `read` privileged api calls, but in addition they'll be granted the following calls: diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index e1610d92423ce..e4c2e197c847c 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -659,13 +659,31 @@ export class RulesClient { this.logger.debug(`getExecutionLogForRule(): getting execution log for rule ${id}`); const rule = (await this.get({ id, includeLegacyId: true })) as SanitizedRuleWithLegacyId; - // Make sure user has access to this rule - await this.authorization.ensureAuthorized({ - ruleTypeId: rule.alertTypeId, - consumer: rule.consumer, - operation: ReadOperations.GetExecutionLog, - entity: AlertingAuthorizationEntity.Rule, - }); + try { + // Make sure user has access to this rule + await this.authorization.ensureAuthorized({ + ruleTypeId: rule.alertTypeId, + consumer: rule.consumer, + operation: ReadOperations.GetExecutionLog, + entity: AlertingAuthorizationEntity.Rule, + }); + } catch (error) { + this.auditLogger?.log( + ruleAuditEvent({ + action: RuleAuditAction.GET, + savedObject: { type: 'alert', id }, + error, + }) + ); + throw error; + } + + this.auditLogger?.log( + ruleAuditEvent({ + action: RuleAuditAction.GET, + savedObject: { type: 'alert', id }, + }) + ); // default duration of instance summary is 60 * rule interval const dateNow = new Date(); diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts index 89ab7eb52ab83..6768ed2492431 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts @@ -18,6 +18,7 @@ import { ActionsAuthorization } from '../../../../actions/server'; import { eventLogClientMock } from '../../../../event_log/server/mocks'; import { SavedObject } from 'kibana/server'; import { RawRule } from '../../types'; +import { auditLoggerMock } from '../../../../security/server/audit/mocks'; import { getBeforeSetup, mockedDateString, setGlobalDate } from './lib'; import { getExecutionLogAggregation } from '../../lib/get_execution_log_aggregation'; @@ -29,6 +30,7 @@ const eventLogClient = eventLogClientMock.create(); const encryptedSavedObjects = encryptedSavedObjectsMock.createClient(); const authorization = alertingAuthorizationMock.create(); const actionsAuthorization = actionsAuthorizationMock.create(); +const auditLogger = auditLoggerMock.create(); const kibanaVersion = 'v7.10.0'; const rulesClientParams: jest.Mocked = { @@ -47,10 +49,12 @@ const rulesClientParams: jest.Mocked = { getActionsClient: jest.fn(), getEventLogClient: jest.fn(), kibanaVersion, + auditLogger, }; beforeEach(() => { getBeforeSetup(rulesClientParams, taskManager, ruleTypeRegistry, eventLogClient); + (auditLogger.log as jest.Mock).mockClear(); }); setGlobalDate(); @@ -524,4 +528,83 @@ describe('getExecutionLogForRule()', () => { rulesClient.getExecutionLogForRule(getExecutionLogByIdParams()) ).rejects.toMatchInlineSnapshot(`[Error: OMG 2!]`); }); + + describe('authorization', () => { + beforeEach(() => { + const ruleSO = getRuleSavedObject({}); + unsecuredSavedObjectsClient.get.mockResolvedValueOnce(ruleSO); + }); + + test('ensures user is authorised to get this type of alert under the consumer', async () => { + await rulesClient.get({ id: '1' }); + + expect(authorization.ensureAuthorized).toHaveBeenCalledWith({ + entity: 'rule', + consumer: 'rule-consumer', + operation: 'get', + ruleTypeId: '123', + }); + }); + + test('throws when user is not authorised to get this type of alert', async () => { + authorization.ensureAuthorized.mockRejectedValue( + new Error(`Unauthorized to get a "myType" alert for "myApp"`) + ); + + await expect(rulesClient.get({ id: '1' })).rejects.toMatchInlineSnapshot( + `[Error: Unauthorized to get a "myType" alert for "myApp"]` + ); + + expect(authorization.ensureAuthorized).toHaveBeenCalledWith({ + entity: 'rule', + consumer: 'rule-consumer', + operation: 'get', + ruleTypeId: '123', + }); + }); + }); + + describe('auditLogger', () => { + beforeEach(() => { + const ruleSO = getRuleSavedObject({}); + unsecuredSavedObjectsClient.get.mockResolvedValueOnce(ruleSO); + }); + + test('logs audit event when getting a rule execution log', async () => { + await rulesClient.get({ id: '1' }); + expect(auditLogger.log).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + action: 'rule_get', + outcome: 'success', + }), + kibana: { saved_object: { id: '1', type: 'alert' } }, + }) + ); + }); + + test('logs audit event when not authorised to get a rule', async () => { + authorization.ensureAuthorized.mockRejectedValue(new Error('Unauthorized')); + + await expect(rulesClient.get({ id: '1' })).rejects.toThrow(); + expect(auditLogger.log).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + action: 'rule_get', + outcome: 'failure', + }), + kibana: { + saved_object: { + id: '1', + type: 'alert', + }, + }, + error: { + code: 'Error', + message: 'Unauthorized', + }, + }) + ); + }); + }); }); From 75010576c1fcbd9bf9cc1b9098fd22620e9f33e9 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Mon, 14 Mar 2022 14:35:49 -0400 Subject: [PATCH 35/39] Adding audit event for rule execution log GET --- .../server/rules_client/audit_events.ts | 7 +++++ .../server/rules_client/rules_client.ts | 4 +-- .../tests/get_execution_log.test.ts | 26 ++++++++++++------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/alerting/server/rules_client/audit_events.ts b/x-pack/plugins/alerting/server/rules_client/audit_events.ts index 487de19691503..a789a5ae6bfca 100644 --- a/x-pack/plugins/alerting/server/rules_client/audit_events.ts +++ b/x-pack/plugins/alerting/server/rules_client/audit_events.ts @@ -23,6 +23,7 @@ export enum RuleAuditAction { MUTE_ALERT = 'rule_alert_mute', UNMUTE_ALERT = 'rule_alert_unmute', AGGREGATE = 'rule_aggregate', + GET_EXECUTION_LOG = 'rule_get_execution_log', } type VerbsTuple = [string, string, string]; @@ -42,6 +43,11 @@ const eventVerbs: Record = { rule_alert_mute: ['mute alert of', 'muting alert of', 'muted alert of'], rule_alert_unmute: ['unmute alert of', 'unmuting alert of', 'unmuted alert of'], rule_aggregate: ['access', 'accessing', 'accessed'], + rule_get_execution_log: [ + 'access execution log for', + 'accessing executiog log for', + 'accessed execution log for', + ], }; const eventTypes: Record = { @@ -59,6 +65,7 @@ const eventTypes: Record = { rule_alert_mute: 'change', rule_alert_unmute: 'change', rule_aggregate: 'access', + rule_get_execution_log: 'access', }; export interface RuleAuditEventParams { diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index e4c2e197c847c..0a1ea61e9b1d9 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -670,7 +670,7 @@ export class RulesClient { } catch (error) { this.auditLogger?.log( ruleAuditEvent({ - action: RuleAuditAction.GET, + action: RuleAuditAction.GET_EXECUTION_LOG, savedObject: { type: 'alert', id }, error, }) @@ -680,7 +680,7 @@ export class RulesClient { this.auditLogger?.log( ruleAuditEvent({ - action: RuleAuditAction.GET, + action: RuleAuditAction.GET_EXECUTION_LOG, savedObject: { type: 'alert', id }, }) ); diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts index 6768ed2492431..7ffb0c8686df8 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts @@ -536,7 +536,8 @@ describe('getExecutionLogForRule()', () => { }); test('ensures user is authorised to get this type of alert under the consumer', async () => { - await rulesClient.get({ id: '1' }); + eventLogClient.aggregateEventsBySavedObjectIds.mockResolvedValueOnce(aggregateResults); + await rulesClient.getExecutionLogForRule(getExecutionLogByIdParams()); expect(authorization.ensureAuthorized).toHaveBeenCalledWith({ entity: 'rule', @@ -547,13 +548,13 @@ describe('getExecutionLogForRule()', () => { }); test('throws when user is not authorised to get this type of alert', async () => { - authorization.ensureAuthorized.mockRejectedValue( + authorization.ensureAuthorized.mockRejectedValueOnce( new Error(`Unauthorized to get a "myType" alert for "myApp"`) ); - await expect(rulesClient.get({ id: '1' })).rejects.toMatchInlineSnapshot( - `[Error: Unauthorized to get a "myType" alert for "myApp"]` - ); + await expect( + rulesClient.getExecutionLogForRule(getExecutionLogByIdParams()) + ).rejects.toMatchInlineSnapshot(`[Error: Unauthorized to get a "myType" alert for "myApp"]`); expect(authorization.ensureAuthorized).toHaveBeenCalledWith({ entity: 'rule', @@ -571,11 +572,12 @@ describe('getExecutionLogForRule()', () => { }); test('logs audit event when getting a rule execution log', async () => { - await rulesClient.get({ id: '1' }); + eventLogClient.aggregateEventsBySavedObjectIds.mockResolvedValueOnce(aggregateResults); + await rulesClient.getExecutionLogForRule(getExecutionLogByIdParams()); expect(auditLogger.log).toHaveBeenCalledWith( expect.objectContaining({ event: expect.objectContaining({ - action: 'rule_get', + action: 'rule_get_execution_log', outcome: 'success', }), kibana: { saved_object: { id: '1', type: 'alert' } }, @@ -584,13 +586,17 @@ describe('getExecutionLogForRule()', () => { }); test('logs audit event when not authorised to get a rule', async () => { - authorization.ensureAuthorized.mockRejectedValue(new Error('Unauthorized')); + // first call occurs during rule SO get + authorization.ensureAuthorized.mockResolvedValueOnce(); + authorization.ensureAuthorized.mockRejectedValueOnce(new Error('Unauthorized')); - await expect(rulesClient.get({ id: '1' })).rejects.toThrow(); + await expect( + rulesClient.getExecutionLogForRule(getExecutionLogByIdParams()) + ).rejects.toMatchInlineSnapshot(`[Error: Unauthorized]`); expect(auditLogger.log).toHaveBeenCalledWith( expect.objectContaining({ event: expect.objectContaining({ - action: 'rule_get', + action: 'rule_get_execution_log', outcome: 'failure', }), kibana: { From 2a9a2842d4a3759e17ddbcdd224a96fc0cae7152 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Mon, 14 Mar 2022 14:59:55 -0400 Subject: [PATCH 36/39] PR feedback --- docs/user/security/audit-logging.asciidoc | 4 ++++ x-pack/plugins/alerting/server/rules_client/audit_events.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/user/security/audit-logging.asciidoc b/docs/user/security/audit-logging.asciidoc index 58f61b79f3ba6..58677141ab0c8 100644 --- a/docs/user/security/audit-logging.asciidoc +++ b/docs/user/security/audit-logging.asciidoc @@ -213,6 +213,10 @@ Refer to the corresponding {es} logs for potential write errors. | `success` | User has accessed a rule. | `failure` | User is not authorized to access a rule. +.2+| `rule_get_execution_log` +| `success` | User has accessed execution log for a rule. +| `failure` | User is not authorized to access execution log for a rule. + .2+| `rule_find` | `success` | User has accessed a rule as part of a search operation. | `failure` | User is not authorized to search for rules. diff --git a/x-pack/plugins/alerting/server/rules_client/audit_events.ts b/x-pack/plugins/alerting/server/rules_client/audit_events.ts index a789a5ae6bfca..22820a162db54 100644 --- a/x-pack/plugins/alerting/server/rules_client/audit_events.ts +++ b/x-pack/plugins/alerting/server/rules_client/audit_events.ts @@ -45,7 +45,7 @@ const eventVerbs: Record = { rule_aggregate: ['access', 'accessing', 'accessed'], rule_get_execution_log: [ 'access execution log for', - 'accessing executiog log for', + 'accessing execution log for', 'accessed execution log for', ], }; From 8ca934de33f91af9b181b34fac5f053313c5d471 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Thu, 17 Mar 2022 21:36:59 -0400 Subject: [PATCH 37/39] Adding gap policy and using static num buckets --- .../lib/get_execution_log_aggregation.test.ts | 21 ++----------------- .../lib/get_execution_log_aggregation.ts | 18 +++------------- .../server/rules_client/rules_client.ts | 5 ----- 3 files changed, 5 insertions(+), 39 deletions(-) diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts index d364e64efed53..92999a80f6b99 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.test.ts @@ -77,7 +77,6 @@ describe('getExecutionLogAggregation', () => { test('should throw error when given bad sort field', () => { expect(() => { getExecutionLogAggregation({ - numExecutions: 5, page: 1, perPage: 10, sort: [{ notsortable: { order: 'asc' } }], @@ -90,7 +89,6 @@ describe('getExecutionLogAggregation', () => { test('should throw error when given one bad sort field', () => { expect(() => { getExecutionLogAggregation({ - numExecutions: 5, page: 1, perPage: 10, sort: [{ notsortable: { order: 'asc' } }, { timestamp: { order: 'asc' } }], @@ -103,7 +101,6 @@ describe('getExecutionLogAggregation', () => { test('should throw error when given bad page field', () => { expect(() => { getExecutionLogAggregation({ - numExecutions: 5, page: 0, perPage: 10, sort: [{ timestamp: { order: 'asc' } }], @@ -111,23 +108,9 @@ describe('getExecutionLogAggregation', () => { }).toThrowErrorMatchingInlineSnapshot(`"Invalid page field \\"0\\" - must be greater than 0"`); }); - test('should throw error when given bad numExecutions field', () => { - expect(() => { - getExecutionLogAggregation({ - numExecutions: 1001, - page: 1, - perPage: 10, - sort: [{ timestamp: { order: 'asc' } }], - }); - }).toThrowErrorMatchingInlineSnapshot( - `"Invalid numExecutions requested \\"1001\\" - must be less than 1000"` - ); - }); - test('should throw error when given bad perPage field', () => { expect(() => { getExecutionLogAggregation({ - numExecutions: 5, page: 1, perPage: 0, sort: [{ timestamp: { order: 'asc' } }], @@ -140,7 +123,6 @@ describe('getExecutionLogAggregation', () => { test('should correctly generate aggregation', () => { expect( getExecutionLogAggregation({ - numExecutions: 5, page: 2, perPage: 10, sort: [{ timestamp: { order: 'asc' } }, { execution_duration: { order: 'desc' } }], @@ -150,7 +132,7 @@ describe('getExecutionLogAggregation', () => { executionUuid: { terms: { field: 'kibana.alert.rule.execution.uuid', - size: 5, + size: 1000, order: [ { 'ruleExecution>executeStartTime': 'asc' }, { 'ruleExecution>executionDuration': 'desc' }, @@ -165,6 +147,7 @@ describe('getExecutionLogAggregation', () => { ], from: 10, size: 10, + gap_policy: 'insert_zeros', }, }, alertCounts: { diff --git a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts index 2aab07284910a..445cec6ad8412 100644 --- a/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts +++ b/x-pack/plugins/alerting/server/lib/get_execution_log_aggregation.ts @@ -86,7 +86,6 @@ interface ExecutionUuidAggResult buckets: TBucket[]; } export interface IExecutionLogAggOptions { - numExecutions: number; page: number; perPage: number; sort: estypes.Sort; @@ -101,12 +100,7 @@ const ExecutionLogSortFields: Record = { num_triggered_actions: 'ruleExecution>numTriggeredActions', }; -export function getExecutionLogAggregation({ - numExecutions, - page, - perPage, - sort, -}: IExecutionLogAggOptions) { +export function getExecutionLogAggregation({ page, perPage, sort }: IExecutionLogAggOptions) { // Check if valid sort fields const sortFields = flatMap(sort as estypes.SortCombinations[], (s) => Object.keys(s)); for (const field of sortFields) { @@ -129,13 +123,6 @@ export function getExecutionLogAggregation({ throw Boom.badRequest(`Invalid perPage field "${perPage}" - must be greater than 0`); } - // Check if valid num executions - if (numExecutions > DEFAULT_MAX_BUCKETS_LIMIT) { - throw Boom.badRequest( - `Invalid numExecutions requested "${numExecutions}" - must be less than ${DEFAULT_MAX_BUCKETS_LIMIT}` - ); - } - return { // Get total number of executions executionUuidCardinality: { @@ -147,7 +134,7 @@ export function getExecutionLogAggregation({ // Bucket by execution UUID terms: { field: EXECUTION_UUID_FIELD, - size: numExecutions, + size: DEFAULT_MAX_BUCKETS_LIMIT, order: formatSortForTermSort(sort), }, aggs: { @@ -157,6 +144,7 @@ export function getExecutionLogAggregation({ sort: formatSortForBucketSort(sort), from: (page - 1) * perPage, size: perPage, + gap_policy: 'insert_zeros' as estypes.AggregationsGapPolicy, }, }, // Get counts for types of alerts and whether there was an execution timeout diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index 0a1ea61e9b1d9..f1599fe940e2c 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -88,7 +88,6 @@ import { import { formatExecutionLogResult, getExecutionLogAggregation, - getNumExecutions, IExecutionLogResult, } from '../lib/get_execution_log_aggregation'; @@ -700,10 +699,6 @@ export class RulesClient { end: parsedDateEnd.toISOString(), filter, aggs: getExecutionLogAggregation({ - // determine the number of executions to request - // based on the date interval and the rule schedule interval - // this value is capped by the max number of terms allowed in a term aggregation - numExecutions: getNumExecutions(parsedDateStart, parsedDateEnd, rule.schedule.interval), page, perPage, sort, From 3f3c7fd94fea208ef0b9670c84e8c3f56fd4572b Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Thu, 17 Mar 2022 22:41:32 -0400 Subject: [PATCH 38/39] Fixing checks --- .../server/rules_client/tests/get_execution_log.test.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts index 7ffb0c8686df8..0938dfe49bf93 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts @@ -362,7 +362,6 @@ describe('getExecutionLogForRule()', () => { ['1'], { aggs: getExecutionLogAggregation({ - numExecutions: 1000, page: 1, perPage: 10, sort: [{ timestamp: { order: 'desc' } }], @@ -389,7 +388,6 @@ describe('getExecutionLogForRule()', () => { ['1'], { aggs: getExecutionLogAggregation({ - numExecutions: 1000, page: 1, perPage: 10, sort: [{ timestamp: { order: 'desc' } }], @@ -416,7 +414,6 @@ describe('getExecutionLogForRule()', () => { ['1'], { aggs: getExecutionLogAggregation({ - numExecutions: 900, page: 1, perPage: 10, sort: [{ timestamp: { order: 'desc' } }], @@ -443,7 +440,6 @@ describe('getExecutionLogForRule()', () => { ['1'], { aggs: getExecutionLogAggregation({ - numExecutions: 1000, page: 1, perPage: 10, sort: [{ timestamp: { order: 'desc' } }], From b52adb1779faf824cb3305025647fedd2aa631c8 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Fri, 18 Mar 2022 16:50:27 -0400 Subject: [PATCH 39/39] Fixing checks --- .../alerting/server/rules_client/tests/get_execution_log.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts index 0938dfe49bf93..a55a3e57428bb 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_execution_log.test.ts @@ -88,6 +88,7 @@ const BaseRuleSavedObject: SavedObject = { status: 'unknown', lastExecutionDate: '2020-08-20T19:23:38Z', error: null, + warning: null, }, }, references: [],