Put the auto calculation of capacity behind a feature flag, for now (e…

…lastic#195390) In this PR, I'm preparing for the 8.16 release where we'd like to start rolling out the `mget` task claiming strategy separately from the added concurrency. To accomplish this, we need to put the capacity calculation behind a feature flag that is default to false for now, until we do a second rollout with an increased concurrency. The increased concurrency can be calculated and adjusted based on experiments of clusters setting `xpack.task_manager.capacity` to a higher value and observe the resource usage. PR to deploy to Cloud and verify that we always default to 10 normal tasks: elastic#195392
bryce-b · Oct 8, 2024 · 9c8f689 · 9c8f689
1 parent 16cd4bb
commit 9c8f689
Show file tree

Hide file tree

Showing 13 changed files with 88 additions and 2 deletions.
diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker
@@ -416,6 +416,7 @@ kibana_vars=(
     xpack.spaces.maxSpaces
     xpack.task_manager.capacity
     xpack.task_manager.claim_strategy
+    xpack.task_manager.auto_calculate_default_ech_capacity
     xpack.task_manager.discovery.active_nodes_lookback
     xpack.task_manager.discovery.interval
     xpack.task_manager.kibanas_per_partition

diff --git a/x-pack/plugins/task_manager/server/config.test.ts b/x-pack/plugins/task_manager/server/config.test.ts
@@ -13,6 +13,7 @@ describe('config validation', () => {
     expect(configSchema.validate(config)).toMatchInlineSnapshot(`
       Object {
         "allow_reading_invalid_state": true,
+        "auto_calculate_default_ech_capacity": false,
         "claim_strategy": "update_by_query",
         "discovery": Object {
           "active_nodes_lookback": "30s",
@@ -75,6 +76,7 @@ describe('config validation', () => {
     expect(configSchema.validate(config)).toMatchInlineSnapshot(`
       Object {
         "allow_reading_invalid_state": true,
+        "auto_calculate_default_ech_capacity": false,
         "claim_strategy": "update_by_query",
         "discovery": Object {
           "active_nodes_lookback": "30s",
@@ -135,6 +137,7 @@ describe('config validation', () => {
     expect(configSchema.validate(config)).toMatchInlineSnapshot(`
       Object {
         "allow_reading_invalid_state": true,
+        "auto_calculate_default_ech_capacity": false,
         "claim_strategy": "update_by_query",
         "discovery": Object {
           "active_nodes_lookback": "30s",

diff --git a/x-pack/plugins/task_manager/server/config.ts b/x-pack/plugins/task_manager/server/config.ts
@@ -204,6 +204,7 @@ export const configSchema = schema.object(
     }),
     claim_strategy: schema.string({ defaultValue: CLAIM_STRATEGY_UPDATE_BY_QUERY }),
     request_timeouts: requestTimeoutsConfig,
+    auto_calculate_default_ech_capacity: schema.boolean({ defaultValue: false }),
   },
   {
     validate: (config) => {

diff --git a/x-pack/plugins/task_manager/server/ephemeral_task_lifecycle.test.ts b/x-pack/plugins/task_manager/server/ephemeral_task_lifecycle.test.ts
@@ -88,6 +88,7 @@ describe('EphemeralTaskLifecycle', () => {
         request_timeouts: {
           update_by_query: 1000,
         },
+        auto_calculate_default_ech_capacity: false,
         ...config,
       },
       elasticsearchAndSOAvailability$,

diff --git a/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts b/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts
@@ -87,6 +87,7 @@ describe('managed configuration', () => {
         request_timeouts: {
           update_by_query: 1000,
         },
+        auto_calculate_default_ech_capacity: false,
       });
       logger = context.logger.get('taskManager');
 
@@ -209,6 +210,7 @@ describe('managed configuration', () => {
         request_timeouts: {
           update_by_query: 1000,
         },
+        auto_calculate_default_ech_capacity: false,
       });
       logger = context.logger.get('taskManager');
 
@@ -334,6 +336,7 @@ describe('managed configuration', () => {
         request_timeouts: {
           update_by_query: 1000,
         },
+        auto_calculate_default_ech_capacity: false,
       });
       logger = context.logger.get('taskManager');
 

diff --git a/x-pack/plugins/task_manager/server/lib/calculate_health_status.test.ts b/x-pack/plugins/task_manager/server/lib/calculate_health_status.test.ts
@@ -60,6 +60,7 @@ const config = {
   request_timeouts: {
     update_by_query: 1000,
   },
+  auto_calculate_default_ech_capacity: false,
 };
 
 const getStatsWithTimestamp = ({

diff --git a/x-pack/plugins/task_manager/server/lib/get_default_capacity.test.ts b/x-pack/plugins/task_manager/server/lib/get_default_capacity.test.ts
@@ -9,9 +9,56 @@ import { CLAIM_STRATEGY_UPDATE_BY_QUERY, CLAIM_STRATEGY_MGET, DEFAULT_CAPACITY }
 import { getDefaultCapacity } from './get_default_capacity';
 
 describe('getDefaultCapacity', () => {
+  it('returns default capacity when autoCalculateDefaultEchCapacity=false', () => {
+    expect(
+      getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: false,
+        heapSizeLimit: 851443712,
+        isCloud: false,
+        isServerless: false,
+        isBackgroundTaskNodeOnly: false,
+        claimStrategy: CLAIM_STRATEGY_MGET,
+      })
+    ).toBe(DEFAULT_CAPACITY);
+
+    expect(
+      getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: false,
+        heapSizeLimit: 851443712,
+        isCloud: false,
+        isServerless: true,
+        isBackgroundTaskNodeOnly: false,
+        claimStrategy: CLAIM_STRATEGY_MGET,
+      })
+    ).toBe(DEFAULT_CAPACITY);
+
+    expect(
+      getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: false,
+        heapSizeLimit: 851443712,
+        isCloud: false,
+        isServerless: false,
+        isBackgroundTaskNodeOnly: true,
+        claimStrategy: CLAIM_STRATEGY_MGET,
+      })
+    ).toBe(DEFAULT_CAPACITY);
+
+    expect(
+      getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: false,
+        heapSizeLimit: 851443712,
+        isCloud: false,
+        isServerless: true,
+        isBackgroundTaskNodeOnly: true,
+        claimStrategy: CLAIM_STRATEGY_MGET,
+      })
+    ).toBe(DEFAULT_CAPACITY);
+  });
+
   it('returns default capacity when not in cloud', () => {
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 851443712,
         isCloud: false,
         isServerless: false,
@@ -22,6 +69,7 @@ describe('getDefaultCapacity', () => {
 
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 851443712,
         isCloud: false,
         isServerless: true,
@@ -32,6 +80,7 @@ describe('getDefaultCapacity', () => {
 
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 851443712,
         isCloud: false,
         isServerless: false,
@@ -42,6 +91,7 @@ describe('getDefaultCapacity', () => {
 
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 851443712,
         isCloud: false,
         isServerless: true,
@@ -54,6 +104,7 @@ describe('getDefaultCapacity', () => {
   it('returns default capacity when default claim strategy', () => {
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 851443712,
         isCloud: true,
         isServerless: false,
@@ -64,6 +115,7 @@ describe('getDefaultCapacity', () => {
 
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 851443712,
         isCloud: true,
         isServerless: false,
@@ -76,6 +128,7 @@ describe('getDefaultCapacity', () => {
   it('returns default capacity when serverless', () => {
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 851443712,
         isCloud: false,
         isServerless: true,
@@ -86,6 +139,7 @@ describe('getDefaultCapacity', () => {
 
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 851443712,
         isCloud: false,
         isServerless: true,
@@ -96,6 +150,7 @@ describe('getDefaultCapacity', () => {
 
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 851443712,
         isCloud: true,
         isServerless: true,
@@ -106,6 +161,7 @@ describe('getDefaultCapacity', () => {
 
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 851443712,
         isCloud: true,
         isServerless: true,
@@ -119,6 +175,7 @@ describe('getDefaultCapacity', () => {
     // 1GB
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 851443712,
         isCloud: true,
         isServerless: false,
@@ -130,6 +187,7 @@ describe('getDefaultCapacity', () => {
     // 1GB but somehow background task node only is true
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 851443712,
         isCloud: true,
         isServerless: false,
@@ -141,6 +199,7 @@ describe('getDefaultCapacity', () => {
     // 2GB
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 1702887424,
         isCloud: true,
         isServerless: false,
@@ -152,6 +211,7 @@ describe('getDefaultCapacity', () => {
     // 2GB but somehow background task node only is true
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 1702887424,
         isCloud: true,
         isServerless: false,
@@ -163,6 +223,7 @@ describe('getDefaultCapacity', () => {
     // 4GB
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 3405774848,
         isCloud: true,
         isServerless: false,
@@ -174,6 +235,7 @@ describe('getDefaultCapacity', () => {
     // 4GB background task only
     expect(
       getDefaultCapacity({
+        autoCalculateDefaultEchCapacity: true,
         heapSizeLimit: 3405774848,
         isCloud: true,
         isServerless: false,

diff --git a/x-pack/plugins/task_manager/server/lib/get_default_capacity.ts b/x-pack/plugins/task_manager/server/lib/get_default_capacity.ts
@@ -8,6 +8,7 @@
 import { CLAIM_STRATEGY_MGET, DEFAULT_CAPACITY } from '../config';
 
 interface GetDefaultCapacityOpts {
+  autoCalculateDefaultEchCapacity: boolean;
   claimStrategy?: string;
   heapSizeLimit: number;
   isCloud: boolean;
@@ -24,14 +25,20 @@ const HEAP_TO_CAPACITY_MAP = [
 ];
 
 export function getDefaultCapacity({
+  autoCalculateDefaultEchCapacity,
   claimStrategy,
   heapSizeLimit: heapSizeLimitInBytes,
   isCloud,
   isServerless,
   isBackgroundTaskNodeOnly,
 }: GetDefaultCapacityOpts) {
   // perform heap size based calculations only in cloud
-  if (isCloud && !isServerless && claimStrategy === CLAIM_STRATEGY_MGET) {
+  if (
+    autoCalculateDefaultEchCapacity &&
+    isCloud &&
+    !isServerless &&
+    claimStrategy === CLAIM_STRATEGY_MGET
+  ) {
     // convert bytes to GB
     const heapSizeLimitInGB = heapSizeLimitInBytes / 1e9;
 

diff --git a/x-pack/plugins/task_manager/server/metrics/create_aggregator.test.ts b/x-pack/plugins/task_manager/server/metrics/create_aggregator.test.ts
@@ -78,6 +78,7 @@ const config: TaskManagerConfig = {
   request_timeouts: {
     update_by_query: 1000,
   },
+  auto_calculate_default_ech_capacity: false,
 };
 
 describe('createAggregator', () => {

diff --git a/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts b/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts
@@ -56,6 +56,7 @@ describe('Configuration Statistics Aggregator', () => {
       request_timeouts: {
         update_by_query: 1000,
       },
+      auto_calculate_default_ech_capacity: false,
     };
 
     const managedConfig = {

diff --git a/x-pack/plugins/task_manager/server/plugin.test.ts b/x-pack/plugins/task_manager/server/plugin.test.ts
@@ -87,6 +87,7 @@ const pluginInitializerContextParams = {
   request_timeouts: {
     update_by_query: 1000,
   },
+  auto_calculate_default_ech_capacity: false,
 };
 
 describe('TaskManagerPlugin', () => {

diff --git a/x-pack/plugins/task_manager/server/plugin.ts b/x-pack/plugins/task_manager/server/plugin.ts
@@ -286,6 +286,7 @@ export class TaskManagerPlugin
     const isServerless = this.initContext.env.packageInfo.buildFlavor === 'serverless';
 
     const defaultCapacity = getDefaultCapacity({
+      autoCalculateDefaultEchCapacity: this.config.auto_calculate_default_ech_capacity,
       claimStrategy: this.config?.claim_strategy,
       heapSizeLimit: this.heapSizeLimit,
       isCloud: cloud?.isCloudEnabled ?? false,
@@ -300,7 +301,9 @@ export class TaskManagerPlugin
         this.config!.claim_strategy
       } isBackgroundTaskNodeOnly=${this.isNodeBackgroundTasksOnly()} heapSizeLimit=${
         this.heapSizeLimit
-      } defaultCapacity=${defaultCapacity}`
+      } defaultCapacity=${defaultCapacity} autoCalculateDefaultEchCapacity=${
+        this.config.auto_calculate_default_ech_capacity
+      }`
     );
 
     const managedConfiguration = createManagedConfiguration({

diff --git a/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts b/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts
@@ -91,6 +91,7 @@ describe('TaskPollingLifecycle', () => {
       request_timeouts: {
         update_by_query: 1000,
       },
+      auto_calculate_default_ech_capacity: false,
     },
     taskStore: mockTaskStore,
     logger: taskManagerLogger,