From 3fc61d2df324e792e7ff38f0b5ac370dbe2eacb2 Mon Sep 17 00:00:00 2001 From: William McConaghy Date: Wed, 6 Jan 2021 08:53:26 -0500 Subject: [PATCH 1/3] improving production documentation for task manager and alerting --- docs/settings/settings-xkb.asciidoc | 1 + docs/settings/task-manager-settings.asciidoc | 35 +++++++++++++++++++ .../alerting-getting-started.asciidoc | 2 +- ...erting-production-considerations.asciidoc} | 14 ++++---- docs/user/alerting/defining-alerts.asciidoc | 2 +- docs/user/alerting/index.asciidoc | 2 +- 6 files changed, 47 insertions(+), 9 deletions(-) create mode 100644 docs/settings/task-manager-settings.asciidoc rename docs/user/alerting/{alerting-scale-performance.asciidoc => alerting-production-considerations.asciidoc} (66%) diff --git a/docs/settings/settings-xkb.asciidoc b/docs/settings/settings-xkb.asciidoc index 9d9cc92401896..4a211976be8cf 100644 --- a/docs/settings/settings-xkb.asciidoc +++ b/docs/settings/settings-xkb.asciidoc @@ -19,5 +19,6 @@ include::logs-ui-settings.asciidoc[] include::ml-settings.asciidoc[] include::reporting-settings.asciidoc[] include::spaces-settings.asciidoc[] +include::task-manager-settings.asciidoc[] include::i18n-settings.asciidoc[] include::fleet-settings.asciidoc[] diff --git a/docs/settings/task-manager-settings.asciidoc b/docs/settings/task-manager-settings.asciidoc new file mode 100644 index 0000000000000..698f6f5b1440c --- /dev/null +++ b/docs/settings/task-manager-settings.asciidoc @@ -0,0 +1,35 @@ +[role="xpack"] +[[task-manager-settings-kb]] +=== Task Manager settings in {kib} +++++ +Task Manager settings +++++ + +Task Manager runs background tasks by polling for work on an interval. You can configure its behavior to tune for performance and throughput. + +[float] +[[task-manager-settings]] +==== Task Manager settings + +[cols="2*<"] +|=== +| `xpack.task_manager.max_attempts` + | The maximum number of times a task will be attempted before being abandoned as failed. Defaults to 3. + +| `xpack.task_manager.poll_interval` + | How often, in milliseconds, the task manager will look for more work. Defaults to 3000 and cannot be lower than 100. + +| `xpack.task_manager.max_poll_inactivity_cycles` + | How many poll interval cycles can work take before it's timed out. Defaults to 10. + +| `xpack.task_manager.request_capacity` + | How many requests can Task Manager buffer before it rejects new requests. Defaults to 1000. + +| `xpack.task_manager.index` + | The name of the index used to store task information. Defaults to `.kibana_task_manager`. + + | `xpack.task_manager.max_workers` + | The maximum number of tasks that this Kibana instance will run simultaneously. Defaults to 10. + + +|=== diff --git a/docs/user/alerting/alerting-getting-started.asciidoc b/docs/user/alerting/alerting-getting-started.asciidoc index cb2b9b19a0726..06370c64aedf8 100644 --- a/docs/user/alerting/alerting-getting-started.asciidoc +++ b/docs/user/alerting/alerting-getting-started.asciidoc @@ -57,7 +57,7 @@ Alert schedules are defined as an interval between subsequent checks, and can ra [IMPORTANT] ============================================== -The intervals of alert checks in {kib} are approximate, their timing of their execution is affected by factors such as the frequency at which tasks are claimed and the task load on the system. See <> for more information. +The intervals of alert checks in {kib} are approximate, their timing of their execution is affected by factors such as the frequency at which tasks are claimed and the task load on the system. See <> for more information. ============================================== [float] diff --git a/docs/user/alerting/alerting-scale-performance.asciidoc b/docs/user/alerting/alerting-production-considerations.asciidoc similarity index 66% rename from docs/user/alerting/alerting-scale-performance.asciidoc rename to docs/user/alerting/alerting-production-considerations.asciidoc index 644a7143f8278..798755d52dba1 100644 --- a/docs/user/alerting/alerting-scale-performance.asciidoc +++ b/docs/user/alerting/alerting-production-considerations.asciidoc @@ -1,10 +1,10 @@ [role="xpack"] -[[alerting-scale-performance]] -== Scale and performance +[[alerting-production-considerations]] +== Production considerations -{kib} alerting run both alert checks and actions as persistent background tasks. This has two major benefits: +{kib} alerting run both alert checks and actions as persistent background tasks managed by the Task Manager. This has two major benefits: -* *Persistence*: all task state and scheduling is stored in {es}, so if {kib} is restarted, alerts and actions will pick up where they left off. +* *Persistence*: all task state and scheduling is stored in {es}, so if {kib} is restarted, alerts and actions will pick up where they left off. Task definitions for alerts and actions are stored in the index specified by `xpack.task_manager.index` (defaults to `.kibana_task_manager`). It is important to have at least 1 replica of this index for production deployments, since if you lose this index all scheduled alerts and actions are also lost. * *Scaling*: multiple {kib} instances can read from and update the same task queue in {es}, allowing the alerting and action load to be distributed across instances. In cases where a {kib} instance no longer has capacity to run alert checks or actions, capacity can be increased by adding additional {kib} instances. [float] @@ -12,17 +12,19 @@ {kib} background tasks are managed by: -* Polling an {es} task index for overdue tasks at 3 second intervals. +* Polling an {es} task index for overdue tasks at 3 second intervals. This interval can be changed using the * Tasks are then claiming them by updating them in the {es} index, using optimistic concurrency control to prevent conflicts. Each {kib} instance can run a maximum of 10 concurrent tasks, so a maximum of 10 tasks are claimed each interval. * Tasks are run on the {kib} server. * In the case of alerts which are recurring background checks, upon completion the task is scheduled again according to the <>. [IMPORTANT] ============================================== -Because tasks are polled at 3 second intervals and only 10 tasks can run concurrently per {kib} instance, it is possible for alert and action tasks to be run late. This can happen if: +Because by default tasks are polled at 3 second intervals and only 10 tasks can run concurrently per {kib} instance, it is possible for alert and action tasks to be run late. This can happen if: * Alerts use a small *check interval*. The lowest interval possible is 3 seconds, though intervals of 30 seconds or higher are recommended. * Many alerts or actions must be *run at once*. In this case pending tasks will queue in {es}, and be pulled 10 at a time from the queue at 3 second intervals. * *Long running tasks* occupy slots for an extended time, leaving fewer slots for other tasks. +For details on the settings that can influence the performance and throughput of Task Manager, see {task-manager-settings}. + ============================================== \ No newline at end of file diff --git a/docs/user/alerting/defining-alerts.asciidoc b/docs/user/alerting/defining-alerts.asciidoc index ffd72cc824336..94cca7f91494e 100644 --- a/docs/user/alerting/defining-alerts.asciidoc +++ b/docs/user/alerting/defining-alerts.asciidoc @@ -26,7 +26,7 @@ image::images/alert-flyout-general-details.png[alt='All alerts have name, tags, Name:: The name of the alert. While this name does not have to be unique, the name can be referenced in actions and also appears in the searchable alert listing in the management UI. A distinctive name can help identify and find an alert. Tags:: A list of tag names that can be applied to an alert. Tags can help you organize and find alerts, because tags appear in the alert listing in the management UI which is searchable by tag. -Check every:: This value determines how frequently the alert conditions below are checked. Note that the timing of background alert checks are not guaranteed, particularly for intervals of less than 10 seconds. See <> for more information. +Check every:: This value determines how frequently the alert conditions below are checked. Note that the timing of background alert checks are not guaranteed, particularly for intervals of less than 10 seconds. See <> for more information. Notify every:: This value limits how often actions are repeated when an alert instance remains active across alert checks. See <> for more information. [float] diff --git a/docs/user/alerting/index.asciidoc b/docs/user/alerting/index.asciidoc index 56404d9a33b80..caef0c6e7332d 100644 --- a/docs/user/alerting/index.asciidoc +++ b/docs/user/alerting/index.asciidoc @@ -2,4 +2,4 @@ include::alerting-getting-started.asciidoc[] include::defining-alerts.asciidoc[] include::action-types.asciidoc[] include::alert-types.asciidoc[] -include::alerting-scale-performance.asciidoc[] +include::alerting-production-considerations.asciidoc[] From 8c995a0f5ca1b17c18c008038ee11a6505fbcea3 Mon Sep 17 00:00:00 2001 From: William McConaghy Date: Wed, 6 Jan 2021 09:17:37 -0500 Subject: [PATCH 2/3] calling it Kibana Task Manager to be more explicit --- docs/user/alerting/alerting-production-considerations.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/alerting/alerting-production-considerations.asciidoc b/docs/user/alerting/alerting-production-considerations.asciidoc index 798755d52dba1..5498153df34e9 100644 --- a/docs/user/alerting/alerting-production-considerations.asciidoc +++ b/docs/user/alerting/alerting-production-considerations.asciidoc @@ -2,7 +2,7 @@ [[alerting-production-considerations]] == Production considerations -{kib} alerting run both alert checks and actions as persistent background tasks managed by the Task Manager. This has two major benefits: +{kib} alerting run both alert checks and actions as persistent background tasks managed by the Kibana Task Manager. This has two major benefits: * *Persistence*: all task state and scheduling is stored in {es}, so if {kib} is restarted, alerts and actions will pick up where they left off. Task definitions for alerts and actions are stored in the index specified by `xpack.task_manager.index` (defaults to `.kibana_task_manager`). It is important to have at least 1 replica of this index for production deployments, since if you lose this index all scheduled alerts and actions are also lost. * *Scaling*: multiple {kib} instances can read from and update the same task queue in {es}, allowing the alerting and action load to be distributed across instances. In cases where a {kib} instance no longer has capacity to run alert checks or actions, capacity can be increased by adding additional {kib} instances. From d17f3af02ca72008faaec1b4be5d2ee397f4fcaa Mon Sep 17 00:00:00 2001 From: William McConaghy Date: Wed, 6 Jan 2021 09:38:57 -0500 Subject: [PATCH 3/3] addressing PR feedback --- docs/settings/task-manager-settings.asciidoc | 3 --- docs/user/alerting/alerting-production-considerations.asciidoc | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/settings/task-manager-settings.asciidoc b/docs/settings/task-manager-settings.asciidoc index 698f6f5b1440c..507e54349276b 100644 --- a/docs/settings/task-manager-settings.asciidoc +++ b/docs/settings/task-manager-settings.asciidoc @@ -19,9 +19,6 @@ Task Manager runs background tasks by polling for work on an interval. You can | `xpack.task_manager.poll_interval` | How often, in milliseconds, the task manager will look for more work. Defaults to 3000 and cannot be lower than 100. -| `xpack.task_manager.max_poll_inactivity_cycles` - | How many poll interval cycles can work take before it's timed out. Defaults to 10. - | `xpack.task_manager.request_capacity` | How many requests can Task Manager buffer before it rejects new requests. Defaults to 1000. diff --git a/docs/user/alerting/alerting-production-considerations.asciidoc b/docs/user/alerting/alerting-production-considerations.asciidoc index 5498153df34e9..3a68e81879e24 100644 --- a/docs/user/alerting/alerting-production-considerations.asciidoc +++ b/docs/user/alerting/alerting-production-considerations.asciidoc @@ -12,7 +12,7 @@ {kib} background tasks are managed by: -* Polling an {es} task index for overdue tasks at 3 second intervals. This interval can be changed using the +* Polling an {es} task index for overdue tasks at 3 second intervals. This interval can be changed using the `xpack.task_manager.poll_interval` setting. * Tasks are then claiming them by updating them in the {es} index, using optimistic concurrency control to prevent conflicts. Each {kib} instance can run a maximum of 10 concurrent tasks, so a maximum of 10 tasks are claimed each interval. * Tasks are run on the {kib} server. * In the case of alerts which are recurring background checks, upon completion the task is scheduled again according to the <>.