[actions] adds config allowing per-host networking options (#96630)

resolves: #80120

Adds a new Kibana configuration key xpack.actions.customHostSettings which
allows per-host configuration of connection settings for https and smtp for
alerting actions. Initially this is just for TLS settings, expandable to other
settings in the future.

The purpose of these is to allow customers to provide server certificates for
servers accessed by actions, whose certificate authority is not available
publicly. Alternatively, a per-server rejectUnauthorized: false configuration
may be used to bypass the verification step for specific servers, but require it
for other servers that do not have per-host customization.

Support was also added to allow per-host customization of ignoreTLS and
requireTLS flags for use with the email action.

docs/settings/alert-action-settings.asciidoc

@@ -47,6 +47,88 @@ You can configure the following settings in the `kibana.yml` file.
   | A list of hostnames that {kib} is allowed to connect to when built-in actions are triggered. It defaults to `[*]`, allowing any host, but keep in mind the potential for SSRF attacks when hosts are not explicitly added to the allowed hosts. An empty list `[]` can be used to block built-in actions from making any external connections. +
   +
   Note that hosts associated with built-in actions, such as Slack and PagerDuty, are not automatically added to allowed hosts. If you are not using the default `[*]` setting, you must ensure that the corresponding endpoints are added to the allowed hosts as well.
+
+| `xpack.actions.customHostSettings` {ess-icon} 
+  | A list of custom host settings to override existing global settings.
+  Defaults to an empty list. +
+  +
+  Each entry in the list must have a `url` property, to associate a connection
+  type (mail or https), hostname and port with the remaining options in the
+  entry.
+  +
+  In the following example, two custom host settings
+  are defined.  The first provides a custom host setting for mail server
+  `mail.example.com` using port 465 that supplies server certificate authorization
+  data from both a file and inline, and requires TLS for the
+  connection.  The second provides a custom host setting for https server
+  `webhook.example.com` which turns off server certificate authorization.
+
+|===
+
+[source,yaml]
+--
+xpack.actions.customHostSettings:
+  - url: smtp://mail.example.com:465
+    tls: 
+      certificateAuthoritiesFiles: [ 'one.crt' ]
+      certificateAuthoritiesData: |
+          -----BEGIN CERTIFICATE-----
+          ... multiple lines of certificate data here ...
+          -----END CERTIFICATE-----
+    smtp:
+      requireTLS: true
+  - url: https://webhook.example.com
+    tls: 
+      rejectUnauthorized: false
+--
+
+[cols="2*<"]
+|===
+
+| `xpack.actions.customHostSettings[n]`
+`.url` {ess-icon}
+  | A URL associated with this custom host setting.  Should be in the form of
+  `protocol://hostname:port`, where `protocol` is `https` or `smtp`.  If the
+  port is not provided, 443 is used for `https` and 25 is used for
+  `smtp`.  The `smtp` URLs are used for the Email actions that use this
+  server, and the `https` URLs are used for actions which use `https` to
+  connect to services. +
+  +
+  Entries with `https` URLs can use the `tls` options, and entries with `smtp`
+  URLs can use both the `tls` and `smtp` options. +
+  +
+  No other URL values should be part of this URL, including paths,
+  query strings, and authentication information.  When an http or smtp request
+  is made as part of executing an action, only the protocol, hostname, and
+  port of the URL for that request are used to look up these configuration
+  values.
+
+| `xpack.actions.customHostSettings[n]`
+`.smtp.ignoreTLS` {ess-icon}
+  | A boolean value indicating that TLS must not be used for this connection.
+  The options `smtp.ignoreTLS` and `smtp.requireTLS` can not both be set to true.
+
+| `xpack.actions.customHostSettings[n]`
+`.smtp.requireTLS` {ess-icon}
+  | A boolean value indicating that TLS must be used for this connection.
+  The options `smtp.ignoreTLS` and `smtp.requireTLS` can not both be set to true.
+
+| `xpack.actions.customHostSettings[n]`
+`.tls.rejectUnauthorized` {ess-icon}
+  | A boolean value indicating whether to bypass server certificate validation.
+  Overrides the general `xpack.actions.rejectUnauthorized` configuration
+  for requests made for this hostname/port.
+
+| `xpack.actions.customHostSettings[n]`
+`.tls.certificateAuthoritiesFiles`
+  | A file name or list of file names of PEM-encoded certificate files to use
+  to validate the server.
+
+| `xpack.actions.customHostSettings[n]`
+`.tls.certificateAuthoritiesData` {ess-icon}
+  | The contents of a PEM-encoded certificate file, or multiple files appended
+  into a single string.  This configuration can be used for environments where
+  the files cannot be made available.  
 
 | `xpack.actions.enabledActionTypes` {ess-icon}
   | A list of action types that are enabled. It defaults to `[*]`, enabling all types. The names for built-in {kib} action types are prefixed with a `.` and include: `.server-log`, `.slack`, `.email`, `.index`, `.pagerduty`, and `.webhook`. An empty list `[]` will disable all action types. +
@@ -79,13 +161,18 @@ a|`xpack.actions.`
 | `xpack.actions.rejectUnauthorized` {ess-icon}
   | Set to `false` to bypass certificate validation for actions. Defaults to `true`. +
   +
-  As an alternative to setting both `xpack.actions.proxyRejectUnauthorizedCertificates` and `xpack.actions.rejectUnauthorized`, you can point the OS level environment variable `NODE_EXTRA_CA_CERTS` to a file that contains the root CAs needed to trust certificates.
+  As an alternative to setting `xpack.actions.rejectUnauthorized`, you can use the setting
+  `xpack.actions.customHostSettings` to set TLS options for specific servers.
 
 | `xpack.actions.maxResponseContentLength` {ess-icon}
   | Specifies the max number of bytes of the http response for requests to external resources. Defaults to 1000000 (1MB).
 
 | `xpack.actions.responseTimeout` {ess-icon}
-  | Specifies the time allowed for requests to external resources. Requests that take longer are aborted. The time is formatted as <count>[ms|s|m|h|d|w|M|Y], for example, '20m', '24h', '7d', '1w'. Defaults to 60s.
+  | Specifies the time allowed for requests to external resources. Requests that take longer are aborted. The time is formatted as: +
+  +
+  `<count>[ms,s,m,h,d,w,M,Y]` +
+  +
+  For example, `20m`, `24h`, `7d`, `1w`. Defaults to `60s`.
 
 
 |===

docs/user/alerting/alerting-troubleshooting.asciidoc

@@ -53,3 +53,19 @@ Alerting and action tasks are identified by their type.
 When diagnosing issues related to Alerting, focus on the tasks that begin with `alerting:` and `actions:`.
 
 For more details on monitoring and diagnosing task execution in Task Manager, see <<task-manager-health-monitoring>>.
+
+[float]
+[[connector-tls-settings]]
+=== Connectors have TLS errors when executing actions
+
+*Problem*:
+
+When executing actions, a connector gets a TLS socket error when connecting to
+the server.
+
+*Resolution*:
+
+Configuration options are available to specialize connections to TLS servers,
+including ignoring server certificate validation, and providing certificate
+authority data to verify servers using custom certificates.  For more details, 
+see <<action-settings,Action settings>>.

src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker

@@ -162,6 +162,7 @@ kibana_vars=(
     timelion.enabled
     vega.enableExternalUrls
     xpack.actions.allowedHosts
+    xpack.actions.customHostSettings
     xpack.actions.enabled
     xpack.actions.enabledActionTypes
     xpack.actions.preconfiguredAlertHistoryEsIndex

x-pack/plugins/actions/server/actions_config.mock.ts

@@ -21,6 +21,7 @@ const createActionsConfigMock = () => {
       maxContentLength: 1000000,
       timeout: 360000,
     }),
+    getCustomHostSettings: jest.fn().mockReturnValue(undefined),
   };
   return mocked;
 };

x-pack/plugins/actions/server/actions_config.test.ts

@@ -13,8 +13,14 @@ import {
   AllowedHosts,
   EnabledActionTypes,
 } from './actions_config';
+import { resolveCustomHosts } from './lib/custom_host_settings';
+import { Logger } from '../../../../src/core/server';
+import { loggingSystemMock } from '../../../../src/core/server/mocks';
+
 import moment from 'moment';
 
+const mockLogger = loggingSystemMock.create().get() as jest.Mocked<Logger>;
+
 const defaultActionsConfig: ActionsConfig = {
   enabled: false,
   allowedHosts: [],
@@ -355,4 +361,79 @@ describe('getProxySettings', () => {
     const proxySettings = getActionsConfigurationUtilities(config).getProxySettings();
     expect(proxySettings?.proxyOnlyHosts).toEqual(new Set(proxyOnlyHosts));
   });
+
+  test('getCustomHostSettings() returns undefined when no matching config', () => {
+    const httpsUrl = 'https://elastic.co/foo/bar';
+    const smtpUrl = 'smtp://elastic.co';
+    let config: ActionsConfig = resolveCustomHosts(mockLogger, {
+      ...defaultActionsConfig,
+    });
+
+    let chs = getActionsConfigurationUtilities(config).getCustomHostSettings(httpsUrl);
+    expect(chs).toEqual(undefined);
+    chs = getActionsConfigurationUtilities(config).getCustomHostSettings(smtpUrl);
+    expect(chs).toEqual(undefined);
+
+    config = resolveCustomHosts(mockLogger, {
+      ...defaultActionsConfig,
+      customHostSettings: [],
+    });
+    chs = getActionsConfigurationUtilities(config).getCustomHostSettings(httpsUrl);
+    expect(chs).toEqual(undefined);
+    chs = getActionsConfigurationUtilities(config).getCustomHostSettings(smtpUrl);
+    expect(chs).toEqual(undefined);
+
+    config = resolveCustomHosts(mockLogger, {
+      ...defaultActionsConfig,
+      customHostSettings: [
+        {
+          url: 'https://www.elastic.co:443',
+        },
+      ],
+    });
+    chs = getActionsConfigurationUtilities(config).getCustomHostSettings(httpsUrl);
+    expect(chs).toEqual(undefined);
+    chs = getActionsConfigurationUtilities(config).getCustomHostSettings(smtpUrl);
+    expect(chs).toEqual(undefined);
+  });
+
+  test('getCustomHostSettings() returns matching config', () => {
+    const httpsUrl = 'https://elastic.co/ignoring/paths/here';
+    const smtpUrl = 'smtp://elastic.co:123';
+    const config: ActionsConfig = resolveCustomHosts(mockLogger, {
+      ...defaultActionsConfig,
+      customHostSettings: [
+        {
+          url: 'https://elastic.co',
+          tls: {
+            rejectUnauthorized: true,
+          },
+        },
+        {
+          url: 'smtp://elastic.co:123',
+          tls: {
+            rejectUnauthorized: false,
+          },
+          smtp: {
+            ignoreTLS: true,
+          },
+        },
+      ],
+    });
+
+    let chs = getActionsConfigurationUtilities(config).getCustomHostSettings(httpsUrl);
+    expect(chs).toEqual(config.customHostSettings![0]);
+    chs = getActionsConfigurationUtilities(config).getCustomHostSettings(smtpUrl);
+    expect(chs).toEqual(config.customHostSettings![1]);
+  });
+
+  test('getCustomHostSettings() returns undefined when bad url is passed in', () => {
+    const badUrl = 'https://elastic.co/foo/bar';
+    const config: ActionsConfig = resolveCustomHosts(mockLogger, {
+      ...defaultActionsConfig,
+    });
+
+    const chs = getActionsConfigurationUtilities(config).getCustomHostSettings(badUrl);
+    expect(chs).toEqual(undefined);
+  });
 });

x-pack/plugins/actions/server/actions_config.ts

@@ -11,7 +11,8 @@ import url from 'url';
 import { curry } from 'lodash';
 import { pipe } from 'fp-ts/lib/pipeable';
 
-import { ActionsConfig, AllowedHosts, EnabledActionTypes } from './config';
+import { ActionsConfig, AllowedHosts, EnabledActionTypes, CustomHostSettings } from './config';
+import { getCanonicalCustomHostUrl } from './lib/custom_host_settings';
 import { ActionTypeDisabledError } from './lib';
 import { ProxySettings, ResponseSettings } from './types';
 
@@ -32,6 +33,7 @@ export interface ActionsConfigurationUtilities {
   isRejectUnauthorizedCertificatesEnabled: () => boolean;
   getProxySettings: () => undefined | ProxySettings;
   getResponseSettings: () => ResponseSettings;
+  getCustomHostSettings: (targetUrl: string) => CustomHostSettings | undefined;
 }
 
 function allowListErrorMessage(field: AllowListingField, value: string) {
@@ -107,6 +109,27 @@ function getResponseSettingsFromConfig(config: ActionsConfig): ResponseSettings
   };
 }
 
+function getCustomHostSettings(
+  config: ActionsConfig,
+  targetUrl: string
+): CustomHostSettings | undefined {
+  const customHostSettings = config.customHostSettings;
+  if (!customHostSettings) {
+    return;
+  }
+
+  let parsedUrl: URL | undefined;
+  try {
+    parsedUrl = new URL(targetUrl);
+  } catch (err) {
+    // presumably this bad URL is reported elsewhere
+    return;
+  }
+
+  const canonicalUrl = getCanonicalCustomHostUrl(parsedUrl);
+  return customHostSettings.find((settings) => settings.url === canonicalUrl);
+}
+
 export function getActionsConfigurationUtilities(
   config: ActionsConfig
 ): ActionsConfigurationUtilities {
@@ -119,6 +142,7 @@ export function getActionsConfigurationUtilities(
     isActionTypeEnabled,
     getProxySettings: () => getProxySettingsFromConfig(config),
     getResponseSettings: () => getResponseSettingsFromConfig(config),
+    // returns the global rejectUnauthorized setting
     isRejectUnauthorizedCertificatesEnabled: () => config.rejectUnauthorized,
     ensureUriAllowed(uri: string) {
       if (!isUriAllowed(uri)) {
@@ -135,5 +159,6 @@ export function getActionsConfigurationUtilities(
         throw new ActionTypeDisabledError(disabledActionTypeErrorMessage(actionType), 'config');
       }
     },
+    getCustomHostSettings: (targetUrl: string) => getCustomHostSettings(config, targetUrl),
   };
 }

x-pack/plugins/actions/server/builtin_action_types/email.test.ts

@@ -282,6 +282,7 @@ describe('execute()', () => {
           "ensureActionTypeEnabled": [MockFunction],
           "ensureHostnameAllowed": [MockFunction],
           "ensureUriAllowed": [MockFunction],
+          "getCustomHostSettings": [MockFunction],
           "getProxySettings": [MockFunction],
           "getResponseSettings": [MockFunction],
           "isActionTypeEnabled": [MockFunction],
@@ -342,6 +343,7 @@ describe('execute()', () => {
           "ensureActionTypeEnabled": [MockFunction],
           "ensureHostnameAllowed": [MockFunction],
           "ensureUriAllowed": [MockFunction],
+          "getCustomHostSettings": [MockFunction],
           "getProxySettings": [MockFunction],
           "getResponseSettings": [MockFunction],
           "isActionTypeEnabled": [MockFunction],