From caa36545b2662b97cf4ededd0987cfd5d6a682d2 Mon Sep 17 00:00:00 2001 From: chrisronline Date: Thu, 2 Sep 2021 16:07:09 -0400 Subject: [PATCH 01/16] Support excluding certain action types --- x-pack/plugins/task_manager/server/config.ts | 4 ++ .../task_manager/server/polling_lifecycle.ts | 1 + .../server/queries/task_claiming.test.ts | 14 +++++ .../server/queries/task_claiming.ts | 62 +++++++++++++------ 4 files changed, 63 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/task_manager/server/config.ts b/x-pack/plugins/task_manager/server/config.ts index 9b4f4856bf8a9..f21e69b95af36 100644 --- a/x-pack/plugins/task_manager/server/config.ts +++ b/x-pack/plugins/task_manager/server/config.ts @@ -128,6 +128,10 @@ export const configSchema = schema.object( max: DEFAULT_MAX_EPHEMERAL_REQUEST_CAPACITY, }), }), + /* These are not designed to be used by most users. Please use caution when changing these */ + internal: schema.object({ + exclude_task_types: schema.arrayOf(schema.string(), { defaultValue: [] }), + }), }, { validate: (config) => { diff --git a/x-pack/plugins/task_manager/server/polling_lifecycle.ts b/x-pack/plugins/task_manager/server/polling_lifecycle.ts index 847e1f32e8f2a..dedef7e8d303d 100644 --- a/x-pack/plugins/task_manager/server/polling_lifecycle.ts +++ b/x-pack/plugins/task_manager/server/polling_lifecycle.ts @@ -126,6 +126,7 @@ export class TaskPollingLifecycle { this.taskClaiming = new TaskClaiming({ taskStore, maxAttempts: config.max_attempts, + excludedTaskTypes: config.internal.exclude_task_types, definitions, logger: this.logger, getCapacity: (taskType?: string) => diff --git a/x-pack/plugins/task_manager/server/queries/task_claiming.test.ts b/x-pack/plugins/task_manager/server/queries/task_claiming.test.ts index ee185ac6ce6bb..1ddafbda71e14 100644 --- a/x-pack/plugins/task_manager/server/queries/task_claiming.test.ts +++ b/x-pack/plugins/task_manager/server/queries/task_claiming.test.ts @@ -102,6 +102,7 @@ describe('TaskClaiming', () => { new TaskClaiming({ logger: taskManagerLogger, definitions, + excludedTaskTypes: [], taskStore: taskStoreMock.create({ taskManagerId: '' }), maxAttempts: 2, getCapacity: () => 10, @@ -119,11 +120,13 @@ describe('TaskClaiming', () => { taskClaimingOpts = {}, hits = [generateFakeTasks(1)], versionConflicts = 2, + excludedTaskTypes = [], }: { storeOpts: Partial; taskClaimingOpts: Partial; hits?: ConcreteTaskInstance[][]; versionConflicts?: number; + excludedTaskTypes?: string[]; }) { const definitions = storeOpts.definitions ?? taskDefinitions; const store = taskStoreMock.create({ taskManagerId: storeOpts.taskManagerId }); @@ -151,6 +154,7 @@ describe('TaskClaiming', () => { logger: taskManagerLogger, definitions, taskStore: store, + excludedTaskTypes, maxAttempts: taskClaimingOpts.maxAttempts ?? 2, getCapacity: taskClaimingOpts.getCapacity ?? (() => 10), ...taskClaimingOpts, @@ -165,17 +169,20 @@ describe('TaskClaiming', () => { claimingOpts, hits = [generateFakeTasks(1)], versionConflicts = 2, + excludedTaskTypes = [], }: { storeOpts: Partial; taskClaimingOpts: Partial; claimingOpts: Omit; hits?: ConcreteTaskInstance[][]; versionConflicts?: number; + excludedTaskTypes?: string[]; }) { const getCapacity = taskClaimingOpts.getCapacity ?? (() => 10); const { taskClaiming, store } = initialiseTestClaiming({ storeOpts, taskClaimingOpts, + excludedTaskTypes, hits, versionConflicts, }); @@ -264,6 +271,11 @@ describe('TaskClaiming', () => { maxAttempts: customMaxAttempts, createTaskRunner: jest.fn(), }, + foobar: { + title: 'foobar', + maxAttempts: customMaxAttempts, + createTaskRunner: jest.fn(), + }, }); const [ @@ -282,6 +294,7 @@ describe('TaskClaiming', () => { claimingOpts: { claimOwnershipUntil: new Date(), }, + excludedTaskTypes: ['foobar'], }); expect(query).toMatchObject({ bool: { @@ -1241,6 +1254,7 @@ if (doc['task.runAt'].size()!=0) { const taskClaiming = new TaskClaiming({ logger: taskManagerLogger, definitions, + excludedTaskTypes: [], taskStore, maxAttempts: 2, getCapacity, diff --git a/x-pack/plugins/task_manager/server/queries/task_claiming.ts b/x-pack/plugins/task_manager/server/queries/task_claiming.ts index 20a0275d8fa07..326a07525d17f 100644 --- a/x-pack/plugins/task_manager/server/queries/task_claiming.ts +++ b/x-pack/plugins/task_manager/server/queries/task_claiming.ts @@ -57,6 +57,7 @@ export interface TaskClaimingOpts { definitions: TaskTypeDictionary; taskStore: TaskStore; maxAttempts: number; + excludedTaskTypes: string[]; getCapacity: (taskType?: string) => number; } @@ -115,6 +116,7 @@ export class TaskClaiming { private logger: Logger; private readonly taskClaimingBatchesByType: TaskClaimingBatches; private readonly taskMaxAttempts: Record; + private readonly excludedTaskTypes: string[]; /** * Constructs a new TaskStore. @@ -130,6 +132,7 @@ export class TaskClaiming { this.logger = opts.logger; this.taskClaimingBatchesByType = this.partitionIntoClaimingBatches(this.definitions); this.taskMaxAttempts = Object.fromEntries(this.normalizeMaxAttempts(this.definitions)); + this.excludedTaskTypes = opts.excludedTaskTypes; this.events$ = new Subject(); } @@ -354,6 +357,24 @@ export class TaskClaiming { }; }; + private isTaskTypeExcluded(taskType: string) { + for (const excludedType of this.excludedTaskTypes) { + const indexOfWildcard = excludedType.indexOf('*'); + if (indexOfWildcard > -1) { + const safeStr = `${excludedType.slice(0, indexOfWildcard)}*${excludedType.slice( + indexOfWildcard + 1 + )}`; + const regexp = new RegExp(safeStr); + if (regexp.test(taskType)) { + return true; + } + } else if (excludedType === taskType) { + return true; + } + } + return false; + } + private async markAvailableTasksAsClaimed({ claimOwnershipUntil, claimTasksById, @@ -362,9 +383,11 @@ export class TaskClaiming { }: OwnershipClaimingOpts): Promise { const { taskTypesToSkip = [], taskTypesToClaim = [] } = groupBy( this.definitions.getAllTypes(), - (type) => (taskTypes.has(type) ? 'taskTypesToClaim' : 'taskTypesToSkip') + (type) => + taskTypes.has(type) && !this.isTaskTypeExcluded(type) + ? 'taskTypesToClaim' + : 'taskTypesToSkip' ); - const queryForScheduledTasks = mustBeAllOf( // Either a task with idle status and runAt <= now or // status running or claiming with a retryAt <= now. @@ -382,6 +405,23 @@ export class TaskClaiming { sort.unshift('_score'); } + const query = matchesClauses( + claimTasksById && claimTasksById.length + ? mustBeAllOf(asPinnedQuery(claimTasksById, queryForScheduledTasks)) + : queryForScheduledTasks, + filterDownBy(InactiveTasks) + ); + const script = updateFieldsAndMarkAsFailed( + { + ownerId: this.taskStore.taskManagerId, + retryAt: claimOwnershipUntil, + }, + claimTasksById || [], + taskTypesToClaim, + taskTypesToSkip, + pick(this.taskMaxAttempts, taskTypesToClaim) + ); + const apmTrans = apm.startTransaction( 'markAvailableTasksAsClaimed', `taskManager markAvailableTasksAsClaimed` @@ -389,22 +429,8 @@ export class TaskClaiming { try { const result = await this.taskStore.updateByQuery( { - query: matchesClauses( - claimTasksById && claimTasksById.length - ? mustBeAllOf(asPinnedQuery(claimTasksById, queryForScheduledTasks)) - : queryForScheduledTasks, - filterDownBy(InactiveTasks) - ), - script: updateFieldsAndMarkAsFailed( - { - ownerId: this.taskStore.taskManagerId, - retryAt: claimOwnershipUntil, - }, - claimTasksById || [], - taskTypesToClaim, - taskTypesToSkip, - pick(this.taskMaxAttempts, taskTypesToClaim) - ), + query, + script, sort, }, { From a185b7a504aea57dd7ad9472dfada001f0af3752 Mon Sep 17 00:00:00 2001 From: chrisronline Date: Thu, 2 Sep 2021 16:53:08 -0400 Subject: [PATCH 02/16] Fix types --- .../server/ephemeral_task_lifecycle.test.ts | 3 +++ .../integration_tests/managed_configuration.test.ts | 3 +++ .../monitoring/configuration_statistics.test.ts | 3 +++ .../monitoring/monitoring_stats_stream.test.ts | 3 +++ x-pack/plugins/task_manager/server/plugin.test.ts | 6 ++++++ .../task_manager/server/polling_lifecycle.test.ts | 3 +++ .../task_manager/server/queries/task_claiming.ts | 12 ++---------- 7 files changed, 23 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/task_manager/server/ephemeral_task_lifecycle.test.ts b/x-pack/plugins/task_manager/server/ephemeral_task_lifecycle.test.ts index 5dbea8e2f4dee..500697ebbe4f2 100644 --- a/x-pack/plugins/task_manager/server/ephemeral_task_lifecycle.test.ts +++ b/x-pack/plugins/task_manager/server/ephemeral_task_lifecycle.test.ts @@ -68,6 +68,9 @@ describe('EphemeralTaskLifecycle', () => { enabled: true, request_capacity: 10, }, + internal: { + exclude_task_types: [], + }, ...config, }, elasticsearchAndSOAvailability$, diff --git a/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts b/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts index 496c0138cb1e5..765a5742eb05f 100644 --- a/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts +++ b/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts @@ -55,6 +55,9 @@ describe('managed configuration', () => { enabled: true, request_capacity: 10, }, + internal: { + exclude_task_types: [], + }, }); logger = context.logger.get('taskManager'); diff --git a/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts b/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts index e477edf3b9aed..1e4d3bdb55078 100644 --- a/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts +++ b/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts @@ -39,6 +39,9 @@ describe('Configuration Statistics Aggregator', () => { enabled: true, request_capacity: 10, }, + internal: { + exclude_task_types: [], + }, }; const managedConfig = { diff --git a/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts b/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts index 50d4b6af9a4cf..f8dda12da3e9d 100644 --- a/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts +++ b/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts @@ -43,6 +43,9 @@ describe('createMonitoringStatsStream', () => { enabled: true, request_capacity: 10, }, + internal: { + exclude_task_types: [], + }, }; it('returns the initial config used to configure Task Manager', async () => { diff --git a/x-pack/plugins/task_manager/server/plugin.test.ts b/x-pack/plugins/task_manager/server/plugin.test.ts index dff94259dbe62..82d42d52380a0 100644 --- a/x-pack/plugins/task_manager/server/plugin.test.ts +++ b/x-pack/plugins/task_manager/server/plugin.test.ts @@ -42,6 +42,9 @@ describe('TaskManagerPlugin', () => { enabled: false, request_capacity: 10, }, + internal: { + exclude_task_types: [], + }, }); pluginInitializerContext.env.instanceUuid = ''; @@ -82,6 +85,9 @@ describe('TaskManagerPlugin', () => { enabled: true, request_capacity: 10, }, + internal: { + exclude_task_types: [], + }, }); const taskManagerPlugin = new TaskManagerPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts b/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts index 42bccbcbaba89..ad3276dbd99a9 100644 --- a/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts +++ b/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts @@ -64,6 +64,9 @@ describe('TaskPollingLifecycle', () => { enabled: true, request_capacity: 10, }, + internal: { + exclude_task_types: [], + }, }, taskStore: mockTaskStore, logger: taskManagerLogger, diff --git a/x-pack/plugins/task_manager/server/queries/task_claiming.ts b/x-pack/plugins/task_manager/server/queries/task_claiming.ts index 326a07525d17f..b587b89b56d76 100644 --- a/x-pack/plugins/task_manager/server/queries/task_claiming.ts +++ b/x-pack/plugins/task_manager/server/queries/task_claiming.ts @@ -9,6 +9,7 @@ * This module contains helpers for managing the task manager storage layer. */ import apm from 'elastic-apm-node'; +import minimatch from 'minimatch'; import { Subject, Observable, from, of } from 'rxjs'; import { map, mergeScan } from 'rxjs/operators'; import { difference, partition, groupBy, mapValues, countBy, pick, isPlainObject } from 'lodash'; @@ -359,16 +360,7 @@ export class TaskClaiming { private isTaskTypeExcluded(taskType: string) { for (const excludedType of this.excludedTaskTypes) { - const indexOfWildcard = excludedType.indexOf('*'); - if (indexOfWildcard > -1) { - const safeStr = `${excludedType.slice(0, indexOfWildcard)}*${excludedType.slice( - indexOfWildcard + 1 - )}`; - const regexp = new RegExp(safeStr); - if (regexp.test(taskType)) { - return true; - } - } else if (excludedType === taskType) { + if (minimatch(excludedType, taskType)) { return true; } } From 2ede415d3d4d398556a90f260418c12e8e9c5eb6 Mon Sep 17 00:00:00 2001 From: chrisronline Date: Thu, 2 Sep 2021 21:37:08 -0400 Subject: [PATCH 03/16] Fix jest tests --- x-pack/plugins/task_manager/server/config.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/x-pack/plugins/task_manager/server/config.test.ts b/x-pack/plugins/task_manager/server/config.test.ts index 14d95e3fd2226..de69a527a7733 100644 --- a/x-pack/plugins/task_manager/server/config.test.ts +++ b/x-pack/plugins/task_manager/server/config.test.ts @@ -18,6 +18,9 @@ describe('config validation', () => { "request_capacity": 10, }, "index": ".kibana_task_manager", + "internal": Object { + "exclude_task_types": Array [], + }, "max_attempts": 3, "max_poll_inactivity_cycles": 10, "max_workers": 10, @@ -74,6 +77,9 @@ describe('config validation', () => { "request_capacity": 10, }, "index": ".kibana_task_manager", + "internal": Object { + "exclude_task_types": Array [], + }, "max_attempts": 3, "max_poll_inactivity_cycles": 10, "max_workers": 10, @@ -117,6 +123,9 @@ describe('config validation', () => { "request_capacity": 10, }, "index": ".kibana_task_manager", + "internal": Object { + "exclude_task_types": Array [], + }, "max_attempts": 3, "max_poll_inactivity_cycles": 10, "max_workers": 10, From ef00530508e18ae7c2704f6aa6be436e3ffc154b Mon Sep 17 00:00:00 2001 From: chrisronline Date: Fri, 3 Sep 2021 12:49:56 -0400 Subject: [PATCH 04/16] Flip this --- x-pack/plugins/task_manager/server/queries/task_claiming.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/task_manager/server/queries/task_claiming.ts b/x-pack/plugins/task_manager/server/queries/task_claiming.ts index b587b89b56d76..8380b8fdedb1d 100644 --- a/x-pack/plugins/task_manager/server/queries/task_claiming.ts +++ b/x-pack/plugins/task_manager/server/queries/task_claiming.ts @@ -360,10 +360,11 @@ export class TaskClaiming { private isTaskTypeExcluded(taskType: string) { for (const excludedType of this.excludedTaskTypes) { - if (minimatch(excludedType, taskType)) { + if (minimatch(taskType, excludedType)) { return true; } } + return false; } From 88ce057976fda684264c0b10a5cff6e4e6671715 Mon Sep 17 00:00:00 2001 From: chrisronline Date: Tue, 7 Sep 2021 13:28:22 -0400 Subject: [PATCH 05/16] Add functional test --- .../alerting_api_integration/common/config.ts | 4 + .../plugins/alerts/server/action_types.ts | 13 +++ .../tests/alerting/excluded.ts | 107 ++++++++++++++++++ .../tests/alerting/index.ts | 1 + 4 files changed, 125 insertions(+) create mode 100644 x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/excluded.ts diff --git a/x-pack/test/alerting_api_integration/common/config.ts b/x-pack/test/alerting_api_integration/common/config.ts index dd43606cc79b7..80b4ce5c6f2df 100644 --- a/x-pack/test/alerting_api_integration/common/config.ts +++ b/x-pack/test/alerting_api_integration/common/config.ts @@ -45,6 +45,7 @@ const enabledActionTypes = [ 'test.delayed', 'test.rate-limit', 'test.throw', + 'test.excluded', ]; export function createTestConfig(name: string, options: CreateTestConfigOptions) { @@ -164,6 +165,9 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions) ...customHostSettings, '--xpack.eventLog.logEntries=true', '--xpack.task_manager.ephemeral_tasks.enabled=false', + `--xpack.task_manager.internal.exclude_task_types=${JSON.stringify([ + 'actions:test.excluded', + ])}`, `--xpack.actions.preconfiguredAlertHistoryEsIndex=${preconfiguredAlertHistoryEsIndex}`, `--xpack.actions.preconfigured=${JSON.stringify({ 'my-slack1': { diff --git a/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts/server/action_types.ts b/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts/server/action_types.ts index a848207bf1b70..6df0949fd6ba2 100644 --- a/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts/server/action_types.ts +++ b/x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts/server/action_types.ts @@ -38,6 +38,7 @@ export function defineActionTypes( actions.registerType(getFailingActionType()); actions.registerType(getRateLimitedActionType()); actions.registerType(getAuthorizationActionType(core)); + actions.registerType(getExcludedActionType()); } function getIndexRecordActionType() { @@ -269,3 +270,15 @@ function getAuthorizationActionType(core: CoreSetup) { }; return result; } + +function getExcludedActionType() { + const result: ActionType<{}, {}, {}, void> = { + id: 'test.excluded', + name: 'Test: Excluded', + minimumLicenseRequired: 'gold', + async executor({ actionId }) { + return { status: 'ok', actionId }; + }, + }; + return result; +} diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/excluded.ts b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/excluded.ts new file mode 100644 index 0000000000000..add30b178c7e6 --- /dev/null +++ b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/excluded.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { UserAtSpaceScenarios } from '../../scenarios'; +import { + getTestAlertData, + getUrlPrefix, + ObjectRemover, + getEventLog, + AlertUtils, + ES_TEST_INDEX_NAME, +} from '../../../common/lib'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default function createAlertTests({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const retry = getService('retry'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); + + describe('excluded', () => { + const objectRemover = new ObjectRemover(supertest); + const scenario = UserAtSpaceScenarios[1]; + + let alertUtils: AlertUtils; + before(async () => { + alertUtils = new AlertUtils({ + user: scenario.user, + space: scenario.space, + supertestWithoutAuth, + objectRemover, + }); + }); + after(() => objectRemover.removeAll()); + + it('should handle create alert request appropriately', async () => { + const { body: createdAction } = await supertest + .post(`${getUrlPrefix(scenario.space.id)}/api/actions/connector`) + .set('kbn-xsrf', 'foo') + .send({ + name: 'MY action', + connector_type_id: 'test.excluded', + config: {}, + secrets: {}, + }) + .expect(200); + objectRemover.add(scenario.space.id, createdAction.id, 'connector', 'actions'); + + const reference = alertUtils.generateReference(); + const { body: rule } = await supertestWithoutAuth + .post(`${getUrlPrefix(scenario.space.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .auth(scenario.user.username, scenario.user.password) + .send( + getTestAlertData({ + rule_type_id: 'test.always-firing', + schedule: { interval: '1s' }, + throttle: '1s', + notify_when: 'onActiveAlert', + params: { + index: ES_TEST_INDEX_NAME, + reference, + }, + actions: [ + { + id: createdAction.id, + group: 'default', + params: {}, + }, + ], + }) + ) + .expect(200); + + objectRemover.add(scenario.space.id, rule.id, 'rule', 'alerting'); + await retry.try(async () => { + return await getEventLog({ + getService, + spaceId: scenario.space.id, + type: 'alert', + id: rule.id, + provider: 'alerting', + actions: new Map([['execute-action', { gte: 3 }]]), + }); + }); + let message; + try { + await getEventLog({ + getService, + spaceId: scenario.space.id, + type: 'action', + id: createdAction.id, + provider: 'actions', + actions: new Map([['execute-start', { gte: 0 }]]), + }); + } catch (err) { + message = err.message; + } + expect(message).to.eql('no events found yet'); + }); + }); +} diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/index.ts b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/index.ts index 6ca68bd188124..114e53f51ec28 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/index.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/index.ts @@ -52,6 +52,7 @@ export default function alertingTests({ loadTestFile, getService }: FtrProviderC loadTestFile(require.resolve('./event_log')); loadTestFile(require.resolve('./mustache_templates')); loadTestFile(require.resolve('./health')); + loadTestFile(require.resolve('./excluded')); }); }); } From 36c20c48eff60e0dfff5d02b6042f57369e05044 Mon Sep 17 00:00:00 2001 From: chrisronline Date: Tue, 7 Sep 2021 13:33:29 -0400 Subject: [PATCH 06/16] Add to README --- x-pack/plugins/task_manager/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/task_manager/README.md b/x-pack/plugins/task_manager/README.md index 400636824a764..bdfc7573bac92 100644 --- a/x-pack/plugins/task_manager/README.md +++ b/x-pack/plugins/task_manager/README.md @@ -55,6 +55,7 @@ The task_manager can be configured via `taskManager` config options (e.g. `taskM - `monitored_stats_running_average_window`- Dictates the size of the window used to calculate the running average of various "Hot" stats. Learn More: [./MONITORING](./MONITORING.MD) - `monitored_stats_required_freshness` - Dictates the _required freshness_ of critical "Hot" stats. Learn More: [./MONITORING](./MONITORING.MD) - `monitored_task_execution_thresholds`- Dictates the threshold of failed task executions. Learn More: [./MONITORING](./MONITORING.MD) +- `internal.exclude_task_types` - Which task types to exclude from execution by ensuring they are never claimed. This is intended for debugging use only ## Task definitions From 76d15f31f85308022987edd95fdc587ffd7e8788 Mon Sep 17 00:00:00 2001 From: chrisronline Date: Tue, 7 Sep 2021 13:34:13 -0400 Subject: [PATCH 07/16] Updated README --- x-pack/plugins/task_manager/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/task_manager/README.md b/x-pack/plugins/task_manager/README.md index bdfc7573bac92..c7e0e505b082c 100644 --- a/x-pack/plugins/task_manager/README.md +++ b/x-pack/plugins/task_manager/README.md @@ -55,7 +55,7 @@ The task_manager can be configured via `taskManager` config options (e.g. `taskM - `monitored_stats_running_average_window`- Dictates the size of the window used to calculate the running average of various "Hot" stats. Learn More: [./MONITORING](./MONITORING.MD) - `monitored_stats_required_freshness` - Dictates the _required freshness_ of critical "Hot" stats. Learn More: [./MONITORING](./MONITORING.MD) - `monitored_task_execution_thresholds`- Dictates the threshold of failed task executions. Learn More: [./MONITORING](./MONITORING.MD) -- `internal.exclude_task_types` - Which task types to exclude from execution by ensuring they are never claimed. This is intended for debugging use only +- `internal.exclude_task_types` - Which task types to exclude from execution by ensuring they are never claimed. Supports wildcard usage (such as `actions:*`). This is intended for debugging use only ## Task definitions From 3002853d3ca4e55aaa8bcab467c8d2a3b24b3d88 Mon Sep 17 00:00:00 2001 From: chrisronline Date: Tue, 7 Sep 2021 13:41:48 -0400 Subject: [PATCH 08/16] Add startup log --- x-pack/plugins/task_manager/server/plugin.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/plugins/task_manager/server/plugin.ts b/x-pack/plugins/task_manager/server/plugin.ts index 35dec38b43df5..724dddc3fade7 100644 --- a/x-pack/plugins/task_manager/server/plugin.ts +++ b/x-pack/plugins/task_manager/server/plugin.ts @@ -121,6 +121,12 @@ export class TaskManagerPlugin ); } + if (this.config.internal.exclude_task_types.length) { + this.logger.debug( + `Excluding task types from execution: ${this.config.internal.exclude_task_types.join(', ')}` + ); + } + return { index: this.config.index, addMiddleware: (middleware: Middleware) => { From b2e062927aaf709dce946cec61385b3009977b2f Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Wed, 8 Sep 2021 10:57:52 -0400 Subject: [PATCH 09/16] Update x-pack/plugins/task_manager/README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mike Côté --- x-pack/plugins/task_manager/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/task_manager/README.md b/x-pack/plugins/task_manager/README.md index c7e0e505b082c..0608a4a601260 100644 --- a/x-pack/plugins/task_manager/README.md +++ b/x-pack/plugins/task_manager/README.md @@ -55,7 +55,7 @@ The task_manager can be configured via `taskManager` config options (e.g. `taskM - `monitored_stats_running_average_window`- Dictates the size of the window used to calculate the running average of various "Hot" stats. Learn More: [./MONITORING](./MONITORING.MD) - `monitored_stats_required_freshness` - Dictates the _required freshness_ of critical "Hot" stats. Learn More: [./MONITORING](./MONITORING.MD) - `monitored_task_execution_thresholds`- Dictates the threshold of failed task executions. Learn More: [./MONITORING](./MONITORING.MD) -- `internal.exclude_task_types` - Which task types to exclude from execution by ensuring they are never claimed. Supports wildcard usage (such as `actions:*`). This is intended for debugging use only +- `internal.exclude_task_types` - A list of task types to exclude from running. Supports wildcard usage (such as `namespace:*`). This configuration is experimental, unsupported and can only be used for temporary debugging purposes as it will cause Kibana to behave in unexpected ways. ## Task definitions From 7cc369b61411d945e729b085b2cc6d7f821ed1a2 Mon Sep 17 00:00:00 2001 From: chrisronline Date: Wed, 8 Sep 2021 16:55:58 -0400 Subject: [PATCH 10/16] Add telemetry --- x-pack/plugins/task_manager/server/plugin.ts | 3 ++- .../server/usage/task_manager_usage_collector.ts | 11 ++++++++--- x-pack/plugins/task_manager/server/usage/types.ts | 1 + .../schema/xpack_plugins.json | 6 ++++++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/task_manager/server/plugin.ts b/x-pack/plugins/task_manager/server/plugin.ts index 724dddc3fade7..c47a5154587eb 100644 --- a/x-pack/plugins/task_manager/server/plugin.ts +++ b/x-pack/plugins/task_manager/server/plugin.ts @@ -117,7 +117,8 @@ export class TaskManagerPlugin usageCollection, monitoredHealth$, this.config.ephemeral_tasks.enabled, - this.config.ephemeral_tasks.request_capacity + this.config.ephemeral_tasks.request_capacity, + this.config.internal.exclude_task_types ); } diff --git a/x-pack/plugins/task_manager/server/usage/task_manager_usage_collector.ts b/x-pack/plugins/task_manager/server/usage/task_manager_usage_collector.ts index 3eff2370ec0cb..6e9891ecd6d65 100644 --- a/x-pack/plugins/task_manager/server/usage/task_manager_usage_collector.ts +++ b/x-pack/plugins/task_manager/server/usage/task_manager_usage_collector.ts @@ -13,7 +13,8 @@ export function createTaskManagerUsageCollector( usageCollection: UsageCollectionSetup, monitoringStats$: Observable, ephemeralTasksEnabled: boolean, - ephemeralRequestCapacity: number + ephemeralRequestCapacity: number, + excludeTaskTypes: string[] ) { let lastMonitoredHealth: MonitoredHealth | null = null; monitoringStats$.subscribe((health) => { @@ -50,6 +51,7 @@ export function createTaskManagerUsageCollector( p99: lastMonitoredHealth?.stats.ephemeral?.value.executionsPerCycle.p99 ?? 0, }, }, + task_type_exclusion: excludeTaskTypes, }; }, schema: { @@ -76,6 +78,7 @@ export function createTaskManagerUsageCollector( p99: { type: 'long' }, }, }, + task_type_exclusion: { type: 'array', items: { type: 'keyword' } }, }, }); } @@ -84,13 +87,15 @@ export function registerTaskManagerUsageCollector( usageCollection: UsageCollectionSetup, monitoringStats$: Observable, ephemeralTasksEnabled: boolean, - ephemeralRequestCapacity: number + ephemeralRequestCapacity: number, + excludeTaskTypes: string[] ) { const collector = createTaskManagerUsageCollector( usageCollection, monitoringStats$, ephemeralTasksEnabled, - ephemeralRequestCapacity + ephemeralRequestCapacity, + excludeTaskTypes ); usageCollection.registerCollector(collector); } diff --git a/x-pack/plugins/task_manager/server/usage/types.ts b/x-pack/plugins/task_manager/server/usage/types.ts index 78e948e21d0aa..0acbfd1d4fab9 100644 --- a/x-pack/plugins/task_manager/server/usage/types.ts +++ b/x-pack/plugins/task_manager/server/usage/types.ts @@ -6,6 +6,7 @@ */ export interface TaskManagerUsage { + task_type_exclusion: string[]; ephemeral_tasks_enabled: boolean; ephemeral_request_capacity: number; ephemeral_stats: { diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 5910de62271e9..b6730a1516361 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -6191,6 +6191,12 @@ } } } + }, + "task_type_exclusion": { + "type": "array", + "items": { + "type": "keyword" + } } } }, From 81b4f4143e8e558a309d9bcfa7fa4f2382686fff Mon Sep 17 00:00:00 2001 From: chrisronline Date: Thu, 9 Sep 2021 09:08:38 -0400 Subject: [PATCH 11/16] Add test --- .../task_manager_usage_collector.test.ts | 43 ++++++++++++++----- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/task_manager/server/usage/task_manager_usage_collector.test.ts b/x-pack/plugins/task_manager/server/usage/task_manager_usage_collector.test.ts index 4b993a4e0629d..8e050e31b6cad 100644 --- a/x-pack/plugins/task_manager/server/usage/task_manager_usage_collector.test.ts +++ b/x-pack/plugins/task_manager/server/usage/task_manager_usage_collector.test.ts @@ -17,6 +17,7 @@ import { MonitoredHealth } from '../routes/health'; import { TaskPersistence } from '../task_events'; import { registerTaskManagerUsageCollector } from './task_manager_usage_collector'; import { sleep } from '../test_utils'; +import { TaskManagerUsage } from './types'; describe('registerTaskManagerUsageCollector', () => { let collector: Collector; @@ -31,25 +32,45 @@ describe('registerTaskManagerUsageCollector', () => { return createUsageCollectionSetupMock().makeUsageCollector(config); }); - registerTaskManagerUsageCollector(usageCollectionMock, monitoringStats$, true, 10); + registerTaskManagerUsageCollector(usageCollectionMock, monitoringStats$, true, 10, []); const mockHealth = getMockMonitoredHealth(); monitoringStats$.next(mockHealth); await sleep(1001); expect(usageCollectionMock.makeUsageCollector).toBeCalled(); - const telemetry = await collector.fetch(fetchContext); - expect(telemetry).toMatchObject({ - ephemeral_tasks_enabled: true, - ephemeral_request_capacity: 10, - ephemeral_stats: { - status: mockHealth.stats.ephemeral?.status, - load: mockHealth.stats.ephemeral?.value.load, - executions_per_cycle: mockHealth.stats.ephemeral?.value.executionsPerCycle, - queued_tasks: mockHealth.stats.ephemeral?.value.queuedTasks, - }, + const telemetry: TaskManagerUsage = (await collector.fetch(fetchContext)) as TaskManagerUsage; + expect(telemetry.ephemeral_tasks_enabled).toBe(true); + expect(telemetry.ephemeral_request_capacity).toBe(10); + expect(telemetry.ephemeral_stats).toMatchObject({ + status: mockHealth.stats.ephemeral?.status, + load: mockHealth.stats.ephemeral?.value.load, + executions_per_cycle: mockHealth.stats.ephemeral?.value.executionsPerCycle, + queued_tasks: mockHealth.stats.ephemeral?.value.queuedTasks, }); }); + + it('should report telemetry on the excluded task types', async () => { + const monitoringStats$ = new Subject(); + const usageCollectionMock = createUsageCollectionSetupMock(); + const fetchContext = createCollectorFetchContextWithKibanaMock(); + usageCollectionMock.makeUsageCollector.mockImplementation((config) => { + collector = new Collector(logger, config); + return createUsageCollectionSetupMock().makeUsageCollector(config); + }); + + registerTaskManagerUsageCollector(usageCollectionMock, monitoringStats$, true, 10, [ + 'actions:*', + ]); + + const mockHealth = getMockMonitoredHealth(); + monitoringStats$.next(mockHealth); + await sleep(1001); + + expect(usageCollectionMock.makeUsageCollector).toBeCalled(); + const telemetry: TaskManagerUsage = (await collector.fetch(fetchContext)) as TaskManagerUsage; + expect(telemetry.task_type_exclusion).toEqual(['actions:*']); + }); }); function getMockMonitoredHealth(overrides = {}): MonitoredHealth { From 48529eddbe83b071af9ed01dec7cb2f5708df2ca Mon Sep 17 00:00:00 2001 From: chrisronline Date: Thu, 9 Sep 2021 15:09:04 -0400 Subject: [PATCH 12/16] Rename internal to unsafe --- x-pack/plugins/task_manager/README.md | 2 +- x-pack/plugins/task_manager/server/config.test.ts | 6 +++--- x-pack/plugins/task_manager/server/config.ts | 2 +- .../task_manager/server/ephemeral_task_lifecycle.test.ts | 2 +- .../server/integration_tests/managed_configuration.test.ts | 2 +- .../server/monitoring/configuration_statistics.test.ts | 2 +- .../server/monitoring/monitoring_stats_stream.test.ts | 2 +- x-pack/plugins/task_manager/server/plugin.test.ts | 4 ++-- x-pack/plugins/task_manager/server/plugin.ts | 6 +++--- .../plugins/task_manager/server/polling_lifecycle.test.ts | 2 +- x-pack/plugins/task_manager/server/polling_lifecycle.ts | 2 +- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/task_manager/README.md b/x-pack/plugins/task_manager/README.md index 0608a4a601260..22a49db297c66 100644 --- a/x-pack/plugins/task_manager/README.md +++ b/x-pack/plugins/task_manager/README.md @@ -55,7 +55,7 @@ The task_manager can be configured via `taskManager` config options (e.g. `taskM - `monitored_stats_running_average_window`- Dictates the size of the window used to calculate the running average of various "Hot" stats. Learn More: [./MONITORING](./MONITORING.MD) - `monitored_stats_required_freshness` - Dictates the _required freshness_ of critical "Hot" stats. Learn More: [./MONITORING](./MONITORING.MD) - `monitored_task_execution_thresholds`- Dictates the threshold of failed task executions. Learn More: [./MONITORING](./MONITORING.MD) -- `internal.exclude_task_types` - A list of task types to exclude from running. Supports wildcard usage (such as `namespace:*`). This configuration is experimental, unsupported and can only be used for temporary debugging purposes as it will cause Kibana to behave in unexpected ways. +- `unsafe.exclude_task_types` - A list of task types to exclude from running. Supports wildcard usage (such as `namespace:*`). This configuration is experimental, unsupported and can only be used for temporary debugging purposes as it will cause Kibana to behave in unexpected ways. ## Task definitions diff --git a/x-pack/plugins/task_manager/server/config.test.ts b/x-pack/plugins/task_manager/server/config.test.ts index de69a527a7733..8bb90539db318 100644 --- a/x-pack/plugins/task_manager/server/config.test.ts +++ b/x-pack/plugins/task_manager/server/config.test.ts @@ -18,7 +18,7 @@ describe('config validation', () => { "request_capacity": 10, }, "index": ".kibana_task_manager", - "internal": Object { + "unsafe": Object { "exclude_task_types": Array [], }, "max_attempts": 3, @@ -77,7 +77,7 @@ describe('config validation', () => { "request_capacity": 10, }, "index": ".kibana_task_manager", - "internal": Object { + "unsafe": Object { "exclude_task_types": Array [], }, "max_attempts": 3, @@ -123,7 +123,7 @@ describe('config validation', () => { "request_capacity": 10, }, "index": ".kibana_task_manager", - "internal": Object { + "unsafe": Object { "exclude_task_types": Array [], }, "max_attempts": 3, diff --git a/x-pack/plugins/task_manager/server/config.ts b/x-pack/plugins/task_manager/server/config.ts index f21e69b95af36..286a9feaa1b5e 100644 --- a/x-pack/plugins/task_manager/server/config.ts +++ b/x-pack/plugins/task_manager/server/config.ts @@ -129,7 +129,7 @@ export const configSchema = schema.object( }), }), /* These are not designed to be used by most users. Please use caution when changing these */ - internal: schema.object({ + unsafe: schema.object({ exclude_task_types: schema.arrayOf(schema.string(), { defaultValue: [] }), }), }, diff --git a/x-pack/plugins/task_manager/server/ephemeral_task_lifecycle.test.ts b/x-pack/plugins/task_manager/server/ephemeral_task_lifecycle.test.ts index 500697ebbe4f2..558aa108c2462 100644 --- a/x-pack/plugins/task_manager/server/ephemeral_task_lifecycle.test.ts +++ b/x-pack/plugins/task_manager/server/ephemeral_task_lifecycle.test.ts @@ -68,7 +68,7 @@ describe('EphemeralTaskLifecycle', () => { enabled: true, request_capacity: 10, }, - internal: { + unsafe: { exclude_task_types: [], }, ...config, diff --git a/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts b/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts index 765a5742eb05f..f714fd36c2658 100644 --- a/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts +++ b/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts @@ -55,7 +55,7 @@ describe('managed configuration', () => { enabled: true, request_capacity: 10, }, - internal: { + unsafe: { exclude_task_types: [], }, }); diff --git a/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts b/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts index 1e4d3bdb55078..6e88e9803add2 100644 --- a/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts +++ b/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts @@ -39,7 +39,7 @@ describe('Configuration Statistics Aggregator', () => { enabled: true, request_capacity: 10, }, - internal: { + unsafe: { exclude_task_types: [], }, }; diff --git a/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts b/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts index f8dda12da3e9d..ec94d9df1a4dc 100644 --- a/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts +++ b/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts @@ -43,7 +43,7 @@ describe('createMonitoringStatsStream', () => { enabled: true, request_capacity: 10, }, - internal: { + unsafe: { exclude_task_types: [], }, }; diff --git a/x-pack/plugins/task_manager/server/plugin.test.ts b/x-pack/plugins/task_manager/server/plugin.test.ts index 82d42d52380a0..efa1eabdf7392 100644 --- a/x-pack/plugins/task_manager/server/plugin.test.ts +++ b/x-pack/plugins/task_manager/server/plugin.test.ts @@ -42,7 +42,7 @@ describe('TaskManagerPlugin', () => { enabled: false, request_capacity: 10, }, - internal: { + unsafe: { exclude_task_types: [], }, }); @@ -85,7 +85,7 @@ describe('TaskManagerPlugin', () => { enabled: true, request_capacity: 10, }, - internal: { + unsafe: { exclude_task_types: [], }, }); diff --git a/x-pack/plugins/task_manager/server/plugin.ts b/x-pack/plugins/task_manager/server/plugin.ts index c47a5154587eb..802c8867e3591 100644 --- a/x-pack/plugins/task_manager/server/plugin.ts +++ b/x-pack/plugins/task_manager/server/plugin.ts @@ -118,13 +118,13 @@ export class TaskManagerPlugin monitoredHealth$, this.config.ephemeral_tasks.enabled, this.config.ephemeral_tasks.request_capacity, - this.config.internal.exclude_task_types + this.config.unsafe.exclude_task_types ); } - if (this.config.internal.exclude_task_types.length) { + if (this.config.unsafe.exclude_task_types.length) { this.logger.debug( - `Excluding task types from execution: ${this.config.internal.exclude_task_types.join(', ')}` + `Excluding task types from execution: ${this.config.unsafe.exclude_task_types.join(', ')}` ); } diff --git a/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts b/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts index ad3276dbd99a9..b6a93b14f578b 100644 --- a/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts +++ b/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts @@ -64,7 +64,7 @@ describe('TaskPollingLifecycle', () => { enabled: true, request_capacity: 10, }, - internal: { + unsafe: { exclude_task_types: [], }, }, diff --git a/x-pack/plugins/task_manager/server/polling_lifecycle.ts b/x-pack/plugins/task_manager/server/polling_lifecycle.ts index dedef7e8d303d..26986fc4709f5 100644 --- a/x-pack/plugins/task_manager/server/polling_lifecycle.ts +++ b/x-pack/plugins/task_manager/server/polling_lifecycle.ts @@ -126,7 +126,7 @@ export class TaskPollingLifecycle { this.taskClaiming = new TaskClaiming({ taskStore, maxAttempts: config.max_attempts, - excludedTaskTypes: config.internal.exclude_task_types, + excludedTaskTypes: config.unsafe.exclude_task_types, definitions, logger: this.logger, getCapacity: (taskType?: string) => From a6d9461c9a3909663568e4c8c6937cd3d610c618 Mon Sep 17 00:00:00 2001 From: chrisronline Date: Thu, 9 Sep 2021 15:44:37 -0400 Subject: [PATCH 13/16] Update test config --- x-pack/test/alerting_api_integration/common/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/alerting_api_integration/common/config.ts b/x-pack/test/alerting_api_integration/common/config.ts index fb49260abfca0..780576d17a5af 100644 --- a/x-pack/test/alerting_api_integration/common/config.ts +++ b/x-pack/test/alerting_api_integration/common/config.ts @@ -166,7 +166,7 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions) ...customHostSettings, '--xpack.eventLog.logEntries=true', '--xpack.task_manager.ephemeral_tasks.enabled=false', - `--xpack.task_manager.internal.exclude_task_types=${JSON.stringify([ + `--xpack.task_manager.unsafe.exclude_task_types=${JSON.stringify([ 'actions:test.excluded', ])}`, `--xpack.actions.preconfiguredAlertHistoryEsIndex=${preconfiguredAlertHistoryEsIndex}`, From ca6b2be0f1435944909299014e39a6913df4fca8 Mon Sep 17 00:00:00 2001 From: chrisronline Date: Thu, 9 Sep 2021 21:55:08 -0400 Subject: [PATCH 14/16] Fix tests --- .../plugins/task_manager/server/config.test.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/task_manager/server/config.test.ts b/x-pack/plugins/task_manager/server/config.test.ts index 8bb90539db318..e9701fc3e7c05 100644 --- a/x-pack/plugins/task_manager/server/config.test.ts +++ b/x-pack/plugins/task_manager/server/config.test.ts @@ -18,9 +18,6 @@ describe('config validation', () => { "request_capacity": 10, }, "index": ".kibana_task_manager", - "unsafe": Object { - "exclude_task_types": Array [], - }, "max_attempts": 3, "max_poll_inactivity_cycles": 10, "max_workers": 10, @@ -40,6 +37,9 @@ describe('config validation', () => { }, "poll_interval": 3000, "request_capacity": 1000, + "unsafe": Object { + "exclude_task_types": Array [], + }, "version_conflict_threshold": 80, } `); @@ -77,9 +77,6 @@ describe('config validation', () => { "request_capacity": 10, }, "index": ".kibana_task_manager", - "unsafe": Object { - "exclude_task_types": Array [], - }, "max_attempts": 3, "max_poll_inactivity_cycles": 10, "max_workers": 10, @@ -99,6 +96,9 @@ describe('config validation', () => { }, "poll_interval": 3000, "request_capacity": 1000, + "unsafe": Object { + "exclude_task_types": Array [], + }, "version_conflict_threshold": 80, } `); @@ -123,9 +123,6 @@ describe('config validation', () => { "request_capacity": 10, }, "index": ".kibana_task_manager", - "unsafe": Object { - "exclude_task_types": Array [], - }, "max_attempts": 3, "max_poll_inactivity_cycles": 10, "max_workers": 10, @@ -150,6 +147,9 @@ describe('config validation', () => { }, "poll_interval": 3000, "request_capacity": 1000, + "unsafe": Object { + "exclude_task_types": Array [], + }, "version_conflict_threshold": 80, } `); From 75496e13f832e95907694f8b31832d845acb356a Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Fri, 10 Sep 2021 16:12:18 -0400 Subject: [PATCH 15/16] Update x-pack/plugins/task_manager/README.md Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> --- x-pack/plugins/task_manager/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/task_manager/README.md b/x-pack/plugins/task_manager/README.md index 22a49db297c66..350a53c660bc7 100644 --- a/x-pack/plugins/task_manager/README.md +++ b/x-pack/plugins/task_manager/README.md @@ -55,7 +55,7 @@ The task_manager can be configured via `taskManager` config options (e.g. `taskM - `monitored_stats_running_average_window`- Dictates the size of the window used to calculate the running average of various "Hot" stats. Learn More: [./MONITORING](./MONITORING.MD) - `monitored_stats_required_freshness` - Dictates the _required freshness_ of critical "Hot" stats. Learn More: [./MONITORING](./MONITORING.MD) - `monitored_task_execution_thresholds`- Dictates the threshold of failed task executions. Learn More: [./MONITORING](./MONITORING.MD) -- `unsafe.exclude_task_types` - A list of task types to exclude from running. Supports wildcard usage (such as `namespace:*`). This configuration is experimental, unsupported and can only be used for temporary debugging purposes as it will cause Kibana to behave in unexpected ways. +- `unsafe.exclude_task_types` - A list of task types to exclude from running. Supports wildcard usage, such as `namespace:*`. This configuration is experimental, unsupported, and can only be used for temporary debugging purposes because it causes Kibana to behave in unexpected ways. ## Task definitions From e709483fc3e788986f1f16d45ba8d88d97605b14 Mon Sep 17 00:00:00 2001 From: chrisronline Date: Mon, 13 Sep 2021 13:33:58 -0400 Subject: [PATCH 16/16] PR feedback --- .../task_manager/server/plugin.test.ts | 42 +++++++++++++++++++ x-pack/plugins/task_manager/server/plugin.ts | 2 +- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/task_manager/server/plugin.test.ts b/x-pack/plugins/task_manager/server/plugin.test.ts index efa1eabdf7392..c47f006eca415 100644 --- a/x-pack/plugins/task_manager/server/plugin.test.ts +++ b/x-pack/plugins/task_manager/server/plugin.test.ts @@ -128,6 +128,48 @@ describe('TaskManagerPlugin', () => { `"Cannot register task definitions after the task manager has started"` ); }); + + test('it logs a warning when the unsafe `exclude_task_types` config is used', async () => { + const pluginInitializerContext = coreMock.createPluginInitializerContext({ + enabled: true, + max_workers: 10, + index: 'foo', + max_attempts: 9, + poll_interval: 3000, + version_conflict_threshold: 80, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_health_verbose_log: { + enabled: false, + warn_delayed_task_start_in_seconds: 60, + }, + monitored_stats_required_freshness: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + ephemeral_tasks: { + enabled: false, + request_capacity: 10, + }, + unsafe: { + exclude_task_types: ['*'], + }, + }); + + const logger = pluginInitializerContext.logger.get(); + const taskManagerPlugin = new TaskManagerPlugin(pluginInitializerContext); + taskManagerPlugin.setup(coreMock.createSetup(), { usageCollection: undefined }); + expect((logger.warn as jest.Mock).mock.calls.length).toBe(1); + expect((logger.warn as jest.Mock).mock.calls[0][0]).toBe( + 'Excluding task types from execution: *' + ); + }); }); describe('getElasticsearchAndSOAvailability', () => { diff --git a/x-pack/plugins/task_manager/server/plugin.ts b/x-pack/plugins/task_manager/server/plugin.ts index 802c8867e3591..f91bb783a5c4c 100644 --- a/x-pack/plugins/task_manager/server/plugin.ts +++ b/x-pack/plugins/task_manager/server/plugin.ts @@ -123,7 +123,7 @@ export class TaskManagerPlugin } if (this.config.unsafe.exclude_task_types.length) { - this.logger.debug( + this.logger.warn( `Excluding task types from execution: ${this.config.unsafe.exclude_task_types.join(', ')}` ); }