From d300657e1c5572d71a5b439c40f3d335ff605cf7 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 4 Nov 2021 13:28:33 +0100 Subject: [PATCH 1/5] [Alerting] Add more rule execution context Closes #113506. --- .../server/task_runner/task_runner.ts | 42 ++++++++++++++++++- .../task_manager/server/task_scheduling.ts | 8 +++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 8b93d3fa17211..f8d84a0d16ec7 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import apm from 'elastic-apm-node'; import type { PublicMethodsOf } from '@kbn/utility-types'; import { Dictionary, pickBy, mapValues, without, cloneDeep } from 'lodash'; import type { Request } from '@hapi/hapi'; @@ -483,6 +483,17 @@ export class TaskRunner< // Ensure API key is still valid and user has access try { alert = await rulesClient.get({ id: alertId }); + + if (apm.currentTransaction) { + apm.currentTransaction.name = `Execute Alerting Rule: "${alert.name}"`; + apm.currentTransaction.addLabels({ + alerting_rule_consumer: alert.consumer, + alerting_rule_name: alert.name, + alerting_rule_tags: alert.tags.join(', '), + alerting_rule_type_id: alert.alertTypeId, + alerting_rule_params: JSON.stringify(alert.params), + }); + } } catch (err) { throw new ErrorWithReason(AlertExecutionStatusErrorReasons.Read, err); } @@ -512,6 +523,13 @@ export class TaskRunner< schedule: taskSchedule, } = this.taskInstance; + if (apm.currentTransaction) { + apm.currentTransaction.name = `Execute Alerting Rule`; + apm.currentTransaction.addLabels({ + alerting_rule_id: alertId, + }); + } + const runDate = new Date(); const runDateString = runDate.toISOString(); this.logger.debug(`executing alert ${this.alertType.id}:${alertId} at ${runDateString}`); @@ -567,6 +585,14 @@ export class TaskRunner< executionStatus.lastExecutionDate = new Date(event.event.start); } + if (apm.currentTransaction) { + if (executionStatus.status === 'ok') { + apm.currentTransaction.setOutcome('success'); + } else if (executionStatus.status === 'error') { + apm.currentTransaction.setOutcome('failure'); + } + } + this.logger.debug( `alertExecutionStatus for ${this.alertType.id}:${alertId}: ${JSON.stringify(executionStatus)}` ); @@ -749,6 +775,12 @@ function generateNewAndRecoveredInstanceEvents< const recoveredAlertInstanceIds = Object.keys(recoveredAlertInstances); const newIds = without(currentAlertInstanceIds, ...originalAlertInstanceIds); + if (apm.currentTransaction) { + apm.currentTransaction.addLabels({ + alerting_new_instances: newIds.length, + }); + } + for (const id of recoveredAlertInstanceIds) { const { group: actionGroup, subgroup: actionSubgroup } = recoveredAlertInstances[id].getLastScheduledActions() ?? {}; @@ -929,6 +961,14 @@ function logActiveAndRecoveredInstances< const { logger, activeAlertInstances, recoveredAlertInstances, alertLabel } = params; const activeInstanceIds = Object.keys(activeAlertInstances); const recoveredInstanceIds = Object.keys(recoveredAlertInstances); + + if (apm.currentTransaction) { + apm.currentTransaction.addLabels({ + alerting_active_instances: activeInstanceIds.length, + alerting_recovered_instances: recoveredInstanceIds.length, + }); + } + if (activeInstanceIds.length > 0) { logger.debug( `alert ${alertLabel} has ${activeInstanceIds.length} active alert instances: ${JSON.stringify( diff --git a/x-pack/plugins/task_manager/server/task_scheduling.ts b/x-pack/plugins/task_manager/server/task_scheduling.ts index a89f66d9c772b..abf1ea0f50eda 100644 --- a/x-pack/plugins/task_manager/server/task_scheduling.ts +++ b/x-pack/plugins/task_manager/server/task_scheduling.ts @@ -99,9 +99,15 @@ export class TaskScheduling { ...options, taskInstance: ensureDeprecatedFieldsAreCorrected(taskInstance, this.logger), }); + + const traceparent = + agent.currentTransaction && agent.currentTransaction.type !== 'request' + ? agent.currentTraceparent + : ''; + return await this.store.schedule({ ...modifiedTask, - traceparent: agent.currentTraceparent ?? '', + traceparent: traceparent || '', }); } From 6ac6e4ab93009b144dd287464be42958fa383b07 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 5 Nov 2021 15:48:37 +0100 Subject: [PATCH 2/5] Fix TM jest test --- x-pack/plugins/task_manager/server/task_scheduling.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugins/task_manager/server/task_scheduling.test.ts b/x-pack/plugins/task_manager/server/task_scheduling.test.ts index 41a172bfb2f8e..f593363c53bcf 100644 --- a/x-pack/plugins/task_manager/server/task_scheduling.test.ts +++ b/x-pack/plugins/task_manager/server/task_scheduling.test.ts @@ -35,6 +35,9 @@ jest.mock('uuid', () => ({ jest.mock('elastic-apm-node', () => ({ currentTraceparent: 'parent', + currentTransaction: { + type: 'taskManager run', + }, })); describe('TaskScheduling', () => { From b99d0c203df8a436ba42017e77d1c24c47a18e7f Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Mon, 8 Nov 2021 13:07:43 +0100 Subject: [PATCH 3/5] Consistent label formatting --- x-pack/plugins/actions/server/lib/action_executor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/actions/server/lib/action_executor.ts b/x-pack/plugins/actions/server/lib/action_executor.ts index 518d4582de2bc..f8a732212b323 100644 --- a/x-pack/plugins/actions/server/lib/action_executor.ts +++ b/x-pack/plugins/actions/server/lib/action_executor.ts @@ -105,7 +105,7 @@ export class ActionExecutor { name: `execute_action`, type: 'actions', labels: { - actionId, + actions_action_id: actionId, }, }, async (span) => { @@ -135,7 +135,7 @@ export class ActionExecutor { if (span) { span.name = `execute_action ${actionTypeId}`; span.addLabels({ - actionTypeId, + actions_action_type_id: actionTypeId, }); } From 04a0f6de82ee8027dcb7ee5baecf7850feb67e24 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 16 Nov 2021 10:26:34 +0100 Subject: [PATCH 4/5] Handle active/unknown state --- x-pack/plugins/alerting/server/task_runner/task_runner.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 762964c0a3e7b..f98f2df231545 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -634,9 +634,9 @@ export class TaskRunner< } if (apm.currentTransaction) { - if (executionStatus.status === 'ok') { + if (executionStatus.status === 'ok' || executionStatus.status === 'active') { apm.currentTransaction.setOutcome('success'); - } else if (executionStatus.status === 'error') { + } else if (executionStatus.status === 'error' || executionStatus.status === 'unknown') { apm.currentTransaction.setOutcome('failure'); } } From edf303d912004d11ab2792e6086ce136fa21ef3d Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sun, 28 Nov 2021 11:58:09 +0100 Subject: [PATCH 5/5] Review feedback --- x-pack/plugins/actions/server/lib/action_executor.ts | 4 ++-- x-pack/plugins/alerting/server/task_runner/task_runner.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/actions/server/lib/action_executor.ts b/x-pack/plugins/actions/server/lib/action_executor.ts index f8a732212b323..9458180fdd220 100644 --- a/x-pack/plugins/actions/server/lib/action_executor.ts +++ b/x-pack/plugins/actions/server/lib/action_executor.ts @@ -105,7 +105,7 @@ export class ActionExecutor { name: `execute_action`, type: 'actions', labels: { - actions_action_id: actionId, + actions_connector_id: actionId, }, }, async (span) => { @@ -135,7 +135,7 @@ export class ActionExecutor { if (span) { span.name = `execute_action ${actionTypeId}`; span.addLabels({ - actions_action_type_id: actionTypeId, + actions_connector_type_id: actionTypeId, }); } diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index f98f2df231545..fe95ec646387d 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -883,7 +883,7 @@ function generateNewAndRecoveredInstanceEvents< if (apm.currentTransaction) { apm.currentTransaction.addLabels({ - alerting_new_instances: newIds.length, + alerting_new_alerts: newIds.length, }); } @@ -1070,8 +1070,8 @@ function logActiveAndRecoveredInstances< if (apm.currentTransaction) { apm.currentTransaction.addLabels({ - alerting_active_instances: activeInstanceIds.length, - alerting_recovered_instances: recoveredInstanceIds.length, + alerting_active_alerts: activeInstanceIds.length, + alerting_recovered_alerts: recoveredInstanceIds.length, }); }