From c677f75fbe6b081c7446f9f279ae9618cc70b6fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D1=83=D1=80=D1=86=D0=B5=D0=B2=20=D0=AE=D1=80=D0=B8?= =?UTF-8?q?=D0=B9?= Date: Mon, 8 Nov 2021 01:38:21 +0300 Subject: [PATCH 01/14] Support for multi es instances --- chart/elastalert2/Chart.yaml | 4 +-- chart/elastalert2/README.md | 2 +- chart/elastalert2/values.yaml | 2 +- docs/source/running_elastalert.rst | 13 +++------ elastalert/__init__.py | 2 +- elastalert/util.py | 43 ++++++++++++++++++++++-------- setup.py | 2 +- tests/util_test.py | 36 ++++++++++++++++++++++--- 8 files changed, 74 insertions(+), 30 deletions(-) diff --git a/chart/elastalert2/Chart.yaml b/chart/elastalert2/Chart.yaml index a4a274d8..91f5a063 100644 --- a/chart/elastalert2/Chart.yaml +++ b/chart/elastalert2/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: Automated rule-based alerting for Elasticsearch name: elastalert2 -version: 2.2.2 -appVersion: 2.2.2 +version: 2.3.0 +appVersion: 2.3.0 home: https://github.com/jertel/elastalert2 sources: - https://github.com/jertel/elastalert2 diff --git a/chart/elastalert2/README.md b/chart/elastalert2/README.md index 38128794..2621d1ee 100644 --- a/chart/elastalert2/README.md +++ b/chart/elastalert2/README.md @@ -47,7 +47,7 @@ The command removes all the Kubernetes components associated with the chart and | Parameter | Description | Default | |----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------| | `image.repository` | docker image | jertel/elastalert2 | -| `image.tag` | docker image tag | 2.2.2 | +| `image.tag` | docker image tag | 2.3.0 | | `image.pullPolicy` | image pull policy | IfNotPresent | | `image.pullSecret` | image pull secret | "" | | `podAnnotations` | Annotations to be added to pods | {} | diff --git a/chart/elastalert2/values.yaml b/chart/elastalert2/values.yaml index 997aa107..4c1d9111 100644 --- a/chart/elastalert2/values.yaml +++ b/chart/elastalert2/values.yaml @@ -25,7 +25,7 @@ image: # docker image repository: jertel/elastalert2 # docker image tag - tag: 2.2.2 + tag: 2.3.0 pullPolicy: IfNotPresent pullSecret: "" diff --git a/docs/source/running_elastalert.rst b/docs/source/running_elastalert.rst index 8dab1f52..c91eea62 100644 --- a/docs/source/running_elastalert.rst +++ b/docs/source/running_elastalert.rst @@ -76,11 +76,12 @@ As a Docker container ===================== If you're interested in a pre-built Docker image check out the -elastalert2 container image on `Docker Hub `_ or `GitHub Container Registry `_. Both images are published for each release. Use GitHub Container Registry if you are running into Docker Hub usage limits. +`elastalert2 +`_ project on Docker Hub. Be aware that the ``latest`` tag of the image represents the latest commit into the master branch. If you prefer to upgrade more slowly you will need utilize a -versioned tag, such as ``2.2.2`` instead, or ``2`` if you are comfortable with +versioned tag, such as ``2.3.2`` instead, or ``2`` if you are comfortable with always using the latest released version of ElastAlert 2. A properly configured config.yaml file must be mounted into the container during @@ -89,18 +90,10 @@ startup of the container. Use the `example file provided as a template, and once saved locally to a file such as ``/tmp/elastalert.yaml``, run the container as follows: -via Docker Hub (hub.docker.com) - .. code-block:: docker run -d -v /tmp/elastalert.yaml:/opt/elastalert/config.yaml jertel/elastalert2 -via GitHub Container Registry (ghcr.io) - -.. code-block:: - - docker run -d -v /tmp/elastalert.yaml:/opt/elastalert/config.yaml ghcr.io/jertel/elastalert2/elastalert2 - To build the image locally run the following command: .. code-block:: diff --git a/elastalert/__init__.py b/elastalert/__init__.py index 40d4397f..8c9fb9ec 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -16,7 +16,7 @@ def __init__(self, conf): """ :arg conf: es_conn_config dictionary. Ref. :func:`~util.build_es_conn_config` """ - super(ElasticSearchClient, self).__init__(host=conf['es_host'], + super(ElasticSearchClient, self).__init__(hosts=conf['es_host'], port=conf['es_port'], url_prefix=conf['es_url_prefix'], use_ssl=conf['use_ssl'], diff --git a/elastalert/util.py b/elastalert/util.py index 42e23ee6..6ecac238 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -363,6 +363,11 @@ def build_es_conn_config(conf): parsed_conf['headers'] = None parsed_conf['es_host'] = os.environ.get('ES_HOST', conf['es_host']) parsed_conf['es_port'] = int(os.environ.get('ES_PORT', conf['es_port'])) + es_host = os.environ.get('ES_HOST', conf['es_host']) + es_port = int(os.environ.get('ES_PORT', conf['es_port'])) + parsed_conf['es_host'] = parse_host(es_host, es_port) + parsed_conf['es_port'] = es_port + parsed_conf['es_url_prefix'] = '' parsed_conf['es_conn_timeout'] = conf.get('es_conn_timeout', 20) parsed_conf['send_get_body_as'] = conf.get('es_send_get_body_as', 'GET') @@ -486,34 +491,34 @@ def should_scrolling_continue(rule_conf): return not stop_the_scroll -def _expand_string_into_dict(string, value, sep='.'): +def _expand_string_into_dict(string, value, sep='.'): """ Converts a encapsulated string-dict to a sequence of dict. Use separator (default '.') to split the string. - Example: + Example: string1.string2.stringN : value -> {string1: {string2: {string3: value}} - + :param string: The encapsulated "string-dict" :param value: Value associated to the last field of the "string-dict" :param sep: Separator character. Default: '.' :rtype: dict """ if sep not in string: - return {string : value} + return {string: value} key, val = string.split(sep, 1) return {key: _expand_string_into_dict(val, value)} - - -def expand_string_into_dict(dictionary, string , value, sep='.'): + + +def expand_string_into_dict(dictionary, string, value, sep='.'): """ Useful function to "compile" a string-dict string used in metric and percentage rules into a dictionary sequence. - + :param dictionary: The dictionary dict - :param string: String Key + :param string: String Key :param value: String Value :param sep: Separator character. Default: '.' :rtype: dict """ - + if sep not in string: dictionary[string] = value return dictionary @@ -526,7 +531,7 @@ def expand_string_into_dict(dictionary, string , value, sep='.'): def format_string(format_config, target_value): """ Formats number, supporting %-format and str.format() syntax. - + :param format_config: string format syntax, for example '{:.2%}' or '%.2f' :param target_value: number to format :rtype: string @@ -536,3 +541,19 @@ def format_string(format_config, target_value): else: return format_config % (target_value) + +def parse_host(host, port=9200): + """ + Convet host str like "host1:port1, host2:port2" to list + :param host str: hostnames (separated with comma ) or single host name + :param port: default to 9200 + :return: list of hosts + + """ + if "," in host: + host_list = host.split(",") + host_list = [("{host}:{port}".format(host=x.strip(), port=port) + if ":" not in x else x.strip()) for x in host_list] + return host_list + else: + return ["{host}:{port}".format(host=host, port=port)] diff --git a/setup.py b/setup.py index c52551bb..a1af7056 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert2', - version='2.2.2', + version='2.3.0', description='Automated rule-based alerting for Elasticsearch', long_description=open('README.md').read(), long_description_content_type="text/markdown", diff --git a/tests/util_test.py b/tests/util_test.py index 16a7ca28..b29ea6b7 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -37,6 +37,7 @@ from elastalert.util import unixms_to_dt from elastalert.util import format_string from elastalert.util import pretty_ts +from elastalert.util import parse_host @pytest.mark.parametrize('spec, expected_delta', [ @@ -337,7 +338,7 @@ def test_ts_utc_to_tz(): 'aws_region': None, 'profile': None, 'headers': None, - 'es_host': 'localhost', + 'es_host': ['localhost:9200'], 'es_port': 9200, 'es_url_prefix': '', 'es_conn_timeout': 20, @@ -360,7 +361,7 @@ def test_ts_utc_to_tz(): 'aws_region': 'us-east-1', 'profile': 'default', 'headers': None, - 'es_host': 'localhost', + 'es_host': ['localhost:9200'], 'es_port': 9200, 'es_url_prefix': 'elasticsearch', 'es_conn_timeout': 30, @@ -435,7 +436,7 @@ def test_build_es_conn_config2(): 'aws_region': None, 'profile': None, 'headers': None, - 'es_host': 'localhost', + 'es_host': ['localhost:9200'], 'es_port': 9200, 'es_url_prefix': '', 'es_conn_timeout': 20, @@ -519,3 +520,32 @@ def test_pretty_ts(): assert '2021-08-16 16:35 UTC' == pretty_ts(ts) assert '2021-08-16 16:35 ' == pretty_ts(ts, False) assert '2021-08-16 16:35 +0000' == pretty_ts(ts, ts_format='%Y-%m-%d %H:%M %z') + + +def test_parse_host(): + assert parse_host("localhost", port=9200) == ["localhost:9200"] + assert parse_host("host1:9200, host2:9200, host3:9300") == ["host1:9200", + "host2:9200", + "host3:9300"] + + +def test_build_cofig_for_multi(): + assert build_es_conn_config({ + "es_host": "localhost", + "es_port": 9200 + })['es_host'] == ['localhost:9200'] + + assert build_es_conn_config({ + "es_host": "host1:9200, host2:9200, host3:9300", + "es_port": 9200 + })['es_host'] == ["host1:9200", "host2:9200", "host3:9300"] + + assert build_es_conn_config({ + "es_host": "host1, host2, host3", + "es_port": 9200 + })['es_host'] == ["host1:9200", "host2:9200", "host3:9200"] + + assert build_es_conn_config({ + "es_host": "host1, host2:9200, host3:9300", + "es_port": 9200 + })['es_host'] == ["host1:9200", "host2:9200", "host3:9300"] From 33d3bdf03dddcda6478b9c743899954bf17a9b04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D1=83=D1=80=D1=86=D0=B5=D0=B2=20=D0=AE=D1=80=D0=B8?= =?UTF-8?q?=D0=B9?= Date: Mon, 8 Nov 2021 16:01:21 +0300 Subject: [PATCH 02/14] Support for multi es instances --- chart/elastalert2/Chart.yaml | 4 ++-- chart/elastalert2/README.md | 2 +- chart/elastalert2/values.yaml | 2 +- docs/source/ruletypes.rst | 7 ++++++- docs/source/running_elastalert.rst | 13 ++++++++++--- elastalert/util.py | 2 -- setup.py | 2 +- 7 files changed, 21 insertions(+), 11 deletions(-) diff --git a/chart/elastalert2/Chart.yaml b/chart/elastalert2/Chart.yaml index 91f5a063..a4a274d8 100644 --- a/chart/elastalert2/Chart.yaml +++ b/chart/elastalert2/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: Automated rule-based alerting for Elasticsearch name: elastalert2 -version: 2.3.0 -appVersion: 2.3.0 +version: 2.2.2 +appVersion: 2.2.2 home: https://github.com/jertel/elastalert2 sources: - https://github.com/jertel/elastalert2 diff --git a/chart/elastalert2/README.md b/chart/elastalert2/README.md index 2621d1ee..38128794 100644 --- a/chart/elastalert2/README.md +++ b/chart/elastalert2/README.md @@ -47,7 +47,7 @@ The command removes all the Kubernetes components associated with the chart and | Parameter | Description | Default | |----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------| | `image.repository` | docker image | jertel/elastalert2 | -| `image.tag` | docker image tag | 2.3.0 | +| `image.tag` | docker image tag | 2.2.2 | | `image.pullPolicy` | image pull policy | IfNotPresent | | `image.pullSecret` | image pull secret | "" | | `podAnnotations` | Annotations to be added to pods | {} | diff --git a/chart/elastalert2/values.yaml b/chart/elastalert2/values.yaml index 4c1d9111..997aa107 100644 --- a/chart/elastalert2/values.yaml +++ b/chart/elastalert2/values.yaml @@ -25,7 +25,7 @@ image: # docker image repository: jertel/elastalert2 # docker image tag - tag: 2.3.0 + tag: 2.2.2 pullPolicy: IfNotPresent pullSecret: "" diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 2f5da6fc..06960f8e 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -225,8 +225,13 @@ Required Settings es_host ^^^^^^^ -``es_host``: The hostname of the Elasticsearch cluster the rule will use to query. (Required, string, no default) +``es_host``: The hostname (or comma separated list of hostnames) of the Elasticsearch cluster the rule will use to query. (Required, string, no default) The environment variable ``ES_HOST`` will override this field. +Definition options: + * hostname + * hostname0, hostname1, hostname2 + * hostname0:9200, hostname1, hostname2:9201 - Nodes with a specified port will be used as is, nodes without a port will be used with the port from es_port + es_port ^^^^^^^ diff --git a/docs/source/running_elastalert.rst b/docs/source/running_elastalert.rst index c91eea62..8dab1f52 100644 --- a/docs/source/running_elastalert.rst +++ b/docs/source/running_elastalert.rst @@ -76,12 +76,11 @@ As a Docker container ===================== If you're interested in a pre-built Docker image check out the -`elastalert2 -`_ project on Docker Hub. +elastalert2 container image on `Docker Hub `_ or `GitHub Container Registry `_. Both images are published for each release. Use GitHub Container Registry if you are running into Docker Hub usage limits. Be aware that the ``latest`` tag of the image represents the latest commit into the master branch. If you prefer to upgrade more slowly you will need utilize a -versioned tag, such as ``2.3.2`` instead, or ``2`` if you are comfortable with +versioned tag, such as ``2.2.2`` instead, or ``2`` if you are comfortable with always using the latest released version of ElastAlert 2. A properly configured config.yaml file must be mounted into the container during @@ -90,10 +89,18 @@ startup of the container. Use the `example file provided as a template, and once saved locally to a file such as ``/tmp/elastalert.yaml``, run the container as follows: +via Docker Hub (hub.docker.com) + .. code-block:: docker run -d -v /tmp/elastalert.yaml:/opt/elastalert/config.yaml jertel/elastalert2 +via GitHub Container Registry (ghcr.io) + +.. code-block:: + + docker run -d -v /tmp/elastalert.yaml:/opt/elastalert/config.yaml ghcr.io/jertel/elastalert2/elastalert2 + To build the image locally run the following command: .. code-block:: diff --git a/elastalert/util.py b/elastalert/util.py index 6ecac238..2a00f5db 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -361,8 +361,6 @@ def build_es_conn_config(conf): parsed_conf['aws_region'] = None parsed_conf['profile'] = None parsed_conf['headers'] = None - parsed_conf['es_host'] = os.environ.get('ES_HOST', conf['es_host']) - parsed_conf['es_port'] = int(os.environ.get('ES_PORT', conf['es_port'])) es_host = os.environ.get('ES_HOST', conf['es_host']) es_port = int(os.environ.get('ES_PORT', conf['es_port'])) parsed_conf['es_host'] = parse_host(es_host, es_port) diff --git a/setup.py b/setup.py index a1af7056..c52551bb 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert2', - version='2.3.0', + version='2.2.2', description='Automated rule-based alerting for Elasticsearch', long_description=open('README.md').read(), long_description_content_type="text/markdown", From 856253957b087906ad1e57e54d1175635bd62e3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D1=83=D1=80=D1=86=D0=B5=D0=B2=20=D0=AE=D1=80=D0=B8?= =?UTF-8?q?=D0=B9?= Date: Mon, 8 Nov 2021 16:05:07 +0300 Subject: [PATCH 03/14] Support for multi es instances --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 537b13ba..d0b261a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Added support for shortening Kibana Discover URLs using Kibana Shorten URL API - [#512](https://github.com/jertel/elastalert2/pull/512) - @JeffAshton - Added new alerter `HTTP Post 2` which allow more flexibility to build the body/headers of the request. - [#530](https://github.com/jertel/elastalert2/pull/530) - @lepouletsuisse - [Slack] Added new option to include url to jira ticket if it is created in the same pipeline. - [#547](https://github.com/jertel/elastalert2/pull/547) - @hugefarsen +- Added support for multi ElasticSearch instances. - [#548](https://github.com/jertel/elastalert2/pull/548) - @buratinopy ## Other changes - [Docs] Add exposed metrics documentation - [#498](https://github.com/jertel/elastalert2/pull/498) - @thisisxgp From 379990a4c49bbf1454a81a93dec1a0fbdba5b35e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D1=83=D1=80=D1=86=D0=B5=D0=B2=20=D0=AE=D1=80=D0=B8?= =?UTF-8?q?=D0=B9?= Date: Mon, 8 Nov 2021 16:45:31 +0300 Subject: [PATCH 04/14] Support for multi es instances --- docs/source/ruletypes.rst | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 06960f8e..d93f1da2 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -227,10 +227,20 @@ es_host ``es_host``: The hostname (or comma separated list of hostnames) of the Elasticsearch cluster the rule will use to query. (Required, string, no default) The environment variable ``ES_HOST`` will override this field. -Definition options: - * hostname - * hostname0, hostname1, hostname2 - * hostname0:9200, hostname1, hostname2:9201 - Nodes with a specified port will be used as is, nodes without a port will be used with the port from es_port + +Example of working with one host:: + + es_host: hostname + +Example of working with multiple hosts:: + + hostname0, hostname1, hostname2 + +Example of working with multiple hosts and specific ports:: + + hostname0:9200, hostname1, hostname2:9201 + +Nodes with a specified port will be used as is, nodes without a port will be used with the port from es_port es_port From 6fd85773c1d09cd31b9077106c34cc92c48688b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D1=83=D1=80=D1=86=D0=B5=D0=B2=20=D0=AE=D1=80=D0=B8?= =?UTF-8?q?=D0=B9?= Date: Fri, 12 Nov 2021 00:14:57 +0300 Subject: [PATCH 05/14] The method of connecting to the ES cluster has been changed, while the method of connecting to the ec_host is preserved --- docs/source/elastalert.rst | 5 +++++ docs/source/ruletypes.rst | 24 +++++++------------- docs/source/running_elastalert.rst | 5 +++++ elastalert/__init__.py | 3 ++- elastalert/config.py | 2 +- elastalert/util.py | 25 +++------------------ tests/util_test.py | 36 +++--------------------------- 7 files changed, 27 insertions(+), 73 deletions(-) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index bcec23b4..e8746f70 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -130,6 +130,11 @@ The environment variable ``ES_HOST`` will override this field. ``es_port``: The port corresponding to ``es_host``. The environment variable ``ES_PORT`` will override this field. +``es_hosts``: The list of addresses of the nodes of the Elasticsearch cluster where ElastAlert 2 records metadata about its searches. +When ElastAlert 2 is started, it will query for information about the time that it was last run. This way, +even if ElastAlert 2 is stopped and restarted, it will never miss data or look at the same events twice. It will also specify the default cluster for each rule to run on. +The environment variable ``ES_HOST`` will override this field. + ``use_ssl``: Optional; whether or not to connect to ``es_host`` using TLS; set to ``True`` or ``False``. The environment variable ``ES_USE_SSL`` will override this field. diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index d93f1da2..b76886e0 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -26,6 +26,8 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``alert`` (string or list) | | +--------------------------------------------------------------+-----------+ +| ``hosts`` (list, no default) | | ++--------------------------------------------------------------+ | | ``name`` (string, defaults to the filename) | | +--------------------------------------------------------------+ | | ``use_strftime_index`` (boolean, default False) | Optional | @@ -225,24 +227,9 @@ Required Settings es_host ^^^^^^^ -``es_host``: The hostname (or comma separated list of hostnames) of the Elasticsearch cluster the rule will use to query. (Required, string, no default) +``es_host``: The hostname of the Elasticsearch cluster the rule will use to query. (Required, string, no default) The environment variable ``ES_HOST`` will override this field. -Example of working with one host:: - - es_host: hostname - -Example of working with multiple hosts:: - - hostname0, hostname1, hostname2 - -Example of working with multiple hosts and specific ports:: - - hostname0:9200, hostname1, hostname2:9201 - -Nodes with a specified port will be used as is, nodes without a port will be used with the port from es_port - - es_port ^^^^^^^ @@ -276,6 +263,11 @@ or loaded from a module. For loading from a module, the alert should be specifie Optional Settings ~~~~~~~~~~~~~~~~~ +es_hosts +^^^^^^^^ + +``es_host``: The list of nodes of the Elasticsearch cluster that the rule will use for the request. (Optional, list, default none) +The environment variable ``ES_HOSTS`` will override this field. import ^^^^^^ diff --git a/docs/source/running_elastalert.rst b/docs/source/running_elastalert.rst index 8dab1f52..a022e832 100644 --- a/docs/source/running_elastalert.rst +++ b/docs/source/running_elastalert.rst @@ -171,6 +171,11 @@ use a different Elasticsearch host to query against. ``es_port`` is the port corresponding to ``es_host``. +``es_hosts`` is the list of addresses of the nodes of the Elasticsearch cluster where +ElastAlert 2 will store data about its state, queries run, alerts, and errors. +(If you want to half connect to a cluster without a load balancer) +Each rule may also use a different Elasticsearch host to query against. + ``use_ssl``: Optional; whether or not to connect to ``es_host`` using TLS; set to ``True`` or ``False``. diff --git a/elastalert/__init__.py b/elastalert/__init__.py index 8c9fb9ec..e0892a11 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -16,7 +16,8 @@ def __init__(self, conf): """ :arg conf: es_conn_config dictionary. Ref. :func:`~util.build_es_conn_config` """ - super(ElasticSearchClient, self).__init__(hosts=conf['es_host'], + super(ElasticSearchClient, self).__init__(host=conf.get('es_host'), + hosts=conf.get('es_hosts'), port=conf['es_port'], url_prefix=conf['es_url_prefix'], use_ssl=conf['use_ssl'], diff --git a/elastalert/config.py b/elastalert/config.py index 381bb759..61aea787 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -21,6 +21,7 @@ 'ES_USERNAME': 'es_username', 'ES_API_KEY': 'es_api_key', 'ES_HOST': 'es_host', + 'ES_HOSTS': 'es_hosts', 'ES_PORT': 'es_port', 'ES_URL_PREFIX': 'es_url_prefix', 'STATSD_INSTANCE_TAG': 'statsd_instance_tag', @@ -28,7 +29,6 @@ env = Env(ES_USE_SSL=bool) - # Used to map the names of rule loaders to their classes loader_mapping = { 'file': loaders.FileRulesLoader, diff --git a/elastalert/util.py b/elastalert/util.py index 2a00f5db..3b773d83 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -361,11 +361,9 @@ def build_es_conn_config(conf): parsed_conf['aws_region'] = None parsed_conf['profile'] = None parsed_conf['headers'] = None - es_host = os.environ.get('ES_HOST', conf['es_host']) - es_port = int(os.environ.get('ES_PORT', conf['es_port'])) - parsed_conf['es_host'] = parse_host(es_host, es_port) - parsed_conf['es_port'] = es_port - + parsed_conf['es_host'] = os.environ.get('ES_HOST', conf['es_host']) + parsed_conf['es_hosts'] = os.environ.get('ES_HOSTS', conf['es_hosts']) + parsed_conf['es_port'] = int(os.environ.get('ES_PORT', conf['es_port'])) parsed_conf['es_url_prefix'] = '' parsed_conf['es_conn_timeout'] = conf.get('es_conn_timeout', 20) parsed_conf['send_get_body_as'] = conf.get('es_send_get_body_as', 'GET') @@ -538,20 +536,3 @@ def format_string(format_config, target_value): return format_config.format(target_value) else: return format_config % (target_value) - - -def parse_host(host, port=9200): - """ - Convet host str like "host1:port1, host2:port2" to list - :param host str: hostnames (separated with comma ) or single host name - :param port: default to 9200 - :return: list of hosts - - """ - if "," in host: - host_list = host.split(",") - host_list = [("{host}:{port}".format(host=x.strip(), port=port) - if ":" not in x else x.strip()) for x in host_list] - return host_list - else: - return ["{host}:{port}".format(host=host, port=port)] diff --git a/tests/util_test.py b/tests/util_test.py index b29ea6b7..16a7ca28 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -37,7 +37,6 @@ from elastalert.util import unixms_to_dt from elastalert.util import format_string from elastalert.util import pretty_ts -from elastalert.util import parse_host @pytest.mark.parametrize('spec, expected_delta', [ @@ -338,7 +337,7 @@ def test_ts_utc_to_tz(): 'aws_region': None, 'profile': None, 'headers': None, - 'es_host': ['localhost:9200'], + 'es_host': 'localhost', 'es_port': 9200, 'es_url_prefix': '', 'es_conn_timeout': 20, @@ -361,7 +360,7 @@ def test_ts_utc_to_tz(): 'aws_region': 'us-east-1', 'profile': 'default', 'headers': None, - 'es_host': ['localhost:9200'], + 'es_host': 'localhost', 'es_port': 9200, 'es_url_prefix': 'elasticsearch', 'es_conn_timeout': 30, @@ -436,7 +435,7 @@ def test_build_es_conn_config2(): 'aws_region': None, 'profile': None, 'headers': None, - 'es_host': ['localhost:9200'], + 'es_host': 'localhost', 'es_port': 9200, 'es_url_prefix': '', 'es_conn_timeout': 20, @@ -520,32 +519,3 @@ def test_pretty_ts(): assert '2021-08-16 16:35 UTC' == pretty_ts(ts) assert '2021-08-16 16:35 ' == pretty_ts(ts, False) assert '2021-08-16 16:35 +0000' == pretty_ts(ts, ts_format='%Y-%m-%d %H:%M %z') - - -def test_parse_host(): - assert parse_host("localhost", port=9200) == ["localhost:9200"] - assert parse_host("host1:9200, host2:9200, host3:9300") == ["host1:9200", - "host2:9200", - "host3:9300"] - - -def test_build_cofig_for_multi(): - assert build_es_conn_config({ - "es_host": "localhost", - "es_port": 9200 - })['es_host'] == ['localhost:9200'] - - assert build_es_conn_config({ - "es_host": "host1:9200, host2:9200, host3:9300", - "es_port": 9200 - })['es_host'] == ["host1:9200", "host2:9200", "host3:9300"] - - assert build_es_conn_config({ - "es_host": "host1, host2, host3", - "es_port": 9200 - })['es_host'] == ["host1:9200", "host2:9200", "host3:9200"] - - assert build_es_conn_config({ - "es_host": "host1, host2:9200, host3:9300", - "es_port": 9200 - })['es_host'] == ["host1:9200", "host2:9200", "host3:9300"] From 0aac62c3cd506485d9fa7c491136781baa62b1ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D1=83=D1=80=D1=86=D0=B5=D0=B2=20=D0=AE=D1=80=D0=B8?= =?UTF-8?q?=D0=B9?= Date: Fri, 12 Nov 2021 00:18:45 +0300 Subject: [PATCH 06/14] small fix --- docs/source/elastalert.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index e8746f70..9c0bcb7a 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -133,7 +133,7 @@ The environment variable ``ES_HOST`` will override this field. ``es_hosts``: The list of addresses of the nodes of the Elasticsearch cluster where ElastAlert 2 records metadata about its searches. When ElastAlert 2 is started, it will query for information about the time that it was last run. This way, even if ElastAlert 2 is stopped and restarted, it will never miss data or look at the same events twice. It will also specify the default cluster for each rule to run on. -The environment variable ``ES_HOST`` will override this field. +The environment variable ``ES_HOSTS`` will override this field. ``use_ssl``: Optional; whether or not to connect to ``es_host`` using TLS; set to ``True`` or ``False``. The environment variable ``ES_USE_SSL`` will override this field. From 8e72492c20b888a4b94653a8cf40f420e1d494c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D1=83=D1=80=D1=86=D0=B5=D0=B2=20=D0=AE=D1=80=D0=B8?= =?UTF-8?q?=D0=B9?= Date: Fri, 12 Nov 2021 01:49:36 +0300 Subject: [PATCH 07/14] fix KeyError es_hosts --- elastalert/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/util.py b/elastalert/util.py index 3b773d83..be6cc844 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -362,7 +362,7 @@ def build_es_conn_config(conf): parsed_conf['profile'] = None parsed_conf['headers'] = None parsed_conf['es_host'] = os.environ.get('ES_HOST', conf['es_host']) - parsed_conf['es_hosts'] = os.environ.get('ES_HOSTS', conf['es_hosts']) + parsed_conf['es_hosts'] = os.environ.get('ES_HOSTS', conf.get('es_hosts')) parsed_conf['es_port'] = int(os.environ.get('ES_PORT', conf['es_port'])) parsed_conf['es_url_prefix'] = '' parsed_conf['es_conn_timeout'] = conf.get('es_conn_timeout', 20) From 7dc78eaedaa3b0980cbdfb241884381bfa90e4e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D1=83=D1=80=D1=86=D0=B5=D0=B2=20=D0=AE=D1=80=D0=B8?= =?UTF-8?q?=D0=B9?= Date: Fri, 12 Nov 2021 16:29:01 +0300 Subject: [PATCH 08/14] https://github.com/jertel/elastalert2/pull/548#discussion_r747905875 --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index b76886e0..03587292 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -26,7 +26,7 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``alert`` (string or list) | | +--------------------------------------------------------------+-----------+ -| ``hosts`` (list, no default) | | +| ``es_hosts`` (list, no default) | | +--------------------------------------------------------------+ | | ``name`` (string, defaults to the filename) | | +--------------------------------------------------------------+ | From d14b5af542fdd35d9be7978979f91b252faff4ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D1=83=D1=80=D1=86=D0=B5=D0=B2=20=D0=AE=D1=80=D0=B8?= =?UTF-8?q?=D0=B9?= Date: Fri, 12 Nov 2021 16:29:37 +0300 Subject: [PATCH 09/14] https://github.com/jertel/elastalert2/pull/548#discussion_r747905999 --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 03587292..ce89e212 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -266,7 +266,7 @@ Optional Settings es_hosts ^^^^^^^^ -``es_host``: The list of nodes of the Elasticsearch cluster that the rule will use for the request. (Optional, list, default none) +``es_hosts``: The list of nodes of the Elasticsearch cluster that the rule will use for the request. (Optional, list, default none) The environment variable ``ES_HOSTS`` will override this field. import From 2b6fe98c2f6283d7a8273a2eb6dfcf81c469f0ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D1=83=D1=80=D1=86=D0=B5=D0=B2=20=D0=AE=D1=80=D0=B8?= =?UTF-8?q?=D0=B9?= Date: Fri, 12 Nov 2021 16:56:48 +0300 Subject: [PATCH 10/14] https://github.com/jertel/elastalert2/pull/548#discussion_r747906790 --- elastalert/util.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/elastalert/util.py b/elastalert/util.py index be6cc844..ae073f02 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -362,8 +362,12 @@ def build_es_conn_config(conf): parsed_conf['profile'] = None parsed_conf['headers'] = None parsed_conf['es_host'] = os.environ.get('ES_HOST', conf['es_host']) - parsed_conf['es_hosts'] = os.environ.get('ES_HOSTS', conf.get('es_hosts')) parsed_conf['es_port'] = int(os.environ.get('ES_PORT', conf['es_port'])) + + es_hosts = os.environ.get('ES_HOSTS') + es_hosts = parse_hosts(es_hosts, parsed_conf.get('es_port')) if es_hosts else conf.get('es_hosts') + parsed_conf['es_hosts'] = es_hosts + parsed_conf['es_url_prefix'] = '' parsed_conf['es_conn_timeout'] = conf.get('es_conn_timeout', 20) parsed_conf['send_get_body_as'] = conf.get('es_send_get_body_as', 'GET') @@ -536,3 +540,19 @@ def format_string(format_config, target_value): return format_config.format(target_value) else: return format_config % (target_value) + + +def parse_hosts(host, port=9200): + """ + Convet host str like "host1:port1, host2:port2" to list + :param host str: hostnames (separated with comma ) or single host name + :param port: default to 9200 + :return: list of hosts + """ + if "," in host: + host_list = host.split(",") + host_list = [("{host}:{port}".format(host=x.strip(), port=port) + if ":" not in x else x.strip()) for x in host_list] + return host_list + else: + return ["{host}:{port}".format(host=host, port=port)] From b51d947c9a22ea43bc7e04981f68aa309c535ed9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D1=83=D1=80=D1=86=D0=B5=D0=B2=20=D0=AE=D1=80=D0=B8?= =?UTF-8?q?=D0=B9?= Date: Fri, 12 Nov 2021 17:21:28 +0300 Subject: [PATCH 11/14] https://github.com/jertel/elastalert2/pull/548#discussion_r747906790 --- tests/util_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/util_test.py b/tests/util_test.py index 16a7ca28..76c37249 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -37,6 +37,7 @@ from elastalert.util import unixms_to_dt from elastalert.util import format_string from elastalert.util import pretty_ts +from elastalert.util import parse_hosts @pytest.mark.parametrize('spec, expected_delta', [ @@ -519,3 +520,10 @@ def test_pretty_ts(): assert '2021-08-16 16:35 UTC' == pretty_ts(ts) assert '2021-08-16 16:35 ' == pretty_ts(ts, False) assert '2021-08-16 16:35 +0000' == pretty_ts(ts, ts_format='%Y-%m-%d %H:%M %z') + + +def test_parse_host(): + assert parse_hosts("localhost", port=9200) == ["localhost:9200"] + assert parse_hosts("host1:9200, host2:9200, host3:9300") == ["host1:9200", + "host2:9200", + "host3:9300"] From aff0b35b90bfb6a10ddfd1d34d2c3dffced886c8 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 13 Nov 2021 12:03:38 -0500 Subject: [PATCH 12/14] Improve documentation; fix broken unit tests; add missing unit tests; add new es_hosts param to schema --- docs/source/elastalert.rst | 9 ++-- docs/source/recipes/faq-md.md | 8 ++-- docs/source/ruletypes.rst | 4 +- docs/source/running_elastalert.rst | 14 +++--- elastalert/schema.yaml | 1 + elastalert/util.py | 3 +- tests/util_test.py | 73 ++++++++++++++++++++++++++++++ 7 files changed, 96 insertions(+), 16 deletions(-) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 9c0bcb7a..72ab99f7 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -130,10 +130,11 @@ The environment variable ``ES_HOST`` will override this field. ``es_port``: The port corresponding to ``es_host``. The environment variable ``ES_PORT`` will override this field. -``es_hosts``: The list of addresses of the nodes of the Elasticsearch cluster where ElastAlert 2 records metadata about its searches. -When ElastAlert 2 is started, it will query for information about the time that it was last run. This way, -even if ElastAlert 2 is stopped and restarted, it will never miss data or look at the same events twice. It will also specify the default cluster for each rule to run on. -The environment variable ``ES_HOSTS`` will override this field. +``es_hosts`` is the list of addresses of the nodes of the Elasticsearch cluster. This +parameter can be used for high availability purposes, but the primary host must also +be specified in the ``es_host`` parameter. The ``es_hosts`` parameter can be overridden +within each rule. This value can be specified as ``host:port`` if overriding the default port. +The environment variable ``ES_HOSTS`` will override this field, and can be specified as a comma-separated value to denote multiple hosts. ``use_ssl``: Optional; whether or not to connect to ``es_host`` using TLS; set to ``True`` or ``False``. The environment variable ``ES_USE_SSL`` will override this field. diff --git a/docs/source/recipes/faq-md.md b/docs/source/recipes/faq-md.md index 3cb8dc1b..a9e52da4 100644 --- a/docs/source/recipes/faq-md.md +++ b/docs/source/recipes/faq-md.md @@ -374,13 +374,15 @@ alert_text: | Does the alert notification destination support Alertmanager? ========== -Not supported. +Now supported as of ElastAlert 2.2.3. The es_host parameter seems to use only one host. Is it possible to specify multiple nodes? ========== -Only one can be set in es_host. -Please use haproxy in front of elasticsearch to support multiple hosts. +There are two options: + +1. Use haproxy in front of elasticsearch to support multiple hosts. +2. Use the new ``es_hosts`` parameter introduced in ElastAlert 2.2.3. See :ref:`Configuration `. Is there any plan to implement a REST API into this project? ========== diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index ce89e212..c4ebd8fc 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -266,8 +266,8 @@ Optional Settings es_hosts ^^^^^^^^ -``es_hosts``: The list of nodes of the Elasticsearch cluster that the rule will use for the request. (Optional, list, default none) -The environment variable ``ES_HOSTS`` will override this field. +``es_hosts``: The list of nodes of the Elasticsearch cluster that the rule will use for the request. (Optional, list, default none). Values can be specified as ``host:port`` if overriding the default port. +The environment variable ``ES_HOSTS`` will override this field, and can be specified as a comma-separated value. Note that the ``es_host`` parameter must still be specified in order to identify a primary Elasticsearch host. import ^^^^^^ diff --git a/docs/source/running_elastalert.rst b/docs/source/running_elastalert.rst index a022e832..3e9fb820 100644 --- a/docs/source/running_elastalert.rst +++ b/docs/source/running_elastalert.rst @@ -165,16 +165,18 @@ For this tutorial, we will use the ``examples/rules`` folder. time each query is run. This value is ignored for rules where ``use_count_query`` or ``use_terms_query`` is set to true. -``es_host`` is the address of an Elasticsearch cluster where ElastAlert 2 will +``es_host`` is the primary address of an Elasticsearch cluster where ElastAlert 2 will store data about its state, queries run, alerts, and errors. Each rule may also -use a different Elasticsearch host to query against. +use a different Elasticsearch host to query against. For multiple host Elasticsearch +clusters see ``es_hosts`` parameter. ``es_port`` is the port corresponding to ``es_host``. -``es_hosts`` is the list of addresses of the nodes of the Elasticsearch cluster where -ElastAlert 2 will store data about its state, queries run, alerts, and errors. -(If you want to half connect to a cluster without a load balancer) -Each rule may also use a different Elasticsearch host to query against. +``es_hosts`` is the list of addresses of the nodes of the Elasticsearch cluster. This +parameter can be used for high availability purposes, but the primary host must also +be specified in the ``es_host`` parameter. The ``es_hosts`` parameter can be overridden +within each rule. This value can be specified as ``host:port`` if overriding the default +port. ``use_ssl``: Optional; whether or not to connect to ``es_host`` using TLS; set to ``True`` or ``False``. diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index d972b8ac..eeac1c69 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -200,6 +200,7 @@ properties: use_strftime_index: {type: boolean} # Optional Settings + es_hosts: {type: array, items: {type: string}} import: anyOf: - type: array diff --git a/elastalert/util.py b/elastalert/util.py index ae073f02..4afe45f1 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -544,7 +544,7 @@ def format_string(format_config, target_value): def parse_hosts(host, port=9200): """ - Convet host str like "host1:port1, host2:port2" to list + Convert host str like "host1:port1, host2:port2" to list :param host str: hostnames (separated with comma ) or single host name :param port: default to 9200 :return: list of hosts @@ -556,3 +556,4 @@ def parse_hosts(host, port=9200): return host_list else: return ["{host}:{port}".format(host=host, port=port)] + diff --git a/tests/util_test.py b/tests/util_test.py index 76c37249..fc4a9d12 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -339,6 +339,7 @@ def test_ts_utc_to_tz(): 'profile': None, 'headers': None, 'es_host': 'localhost', + 'es_hosts': None, 'es_port': 9200, 'es_url_prefix': '', 'es_conn_timeout': 20, @@ -362,6 +363,7 @@ def test_ts_utc_to_tz(): 'profile': 'default', 'headers': None, 'es_host': 'localhost', + 'es_hosts': None, 'es_port': 9200, 'es_url_prefix': 'elasticsearch', 'es_conn_timeout': 30, @@ -437,6 +439,77 @@ def test_build_es_conn_config2(): 'profile': None, 'headers': None, 'es_host': 'localhost', + 'es_hosts': None, + 'es_port': 9200, + 'es_url_prefix': '', + 'es_conn_timeout': 20, + 'send_get_body_as': 'GET', + 'ssl_show_warn': True + } + actual = build_es_conn_config(conf) + assert expected == actual + + +@mock.patch.dict(os.environ, {'ES_USERNAME': 'USER', + 'ES_PASSWORD': 'PASS', + 'ES_API_KEY': 'KEY', + 'ES_BEARER': 'BEARE'}) +def test_build_es_conn_config_es_hosts_list(): + conf = {} + conf['es_host'] = 'localhost' + conf['es_port'] = 9200 + conf['es_hosts'] = ['host1:123', 'host2'] + expected = { + 'use_ssl': False, + 'verify_certs': True, + 'ca_certs': None, + 'client_cert': None, + 'client_key': None, + 'http_auth': None, + 'es_username': 'USER', + 'es_password': 'PASS', + 'es_api_key': 'KEY', + 'es_bearer': 'BEARE', + 'aws_region': None, + 'profile': None, + 'headers': None, + 'es_host': 'localhost', + 'es_hosts': ['host1:123', 'host2'], + 'es_port': 9200, + 'es_url_prefix': '', + 'es_conn_timeout': 20, + 'send_get_body_as': 'GET', + 'ssl_show_warn': True + } + actual = build_es_conn_config(conf) + assert expected == actual + + +@mock.patch.dict(os.environ, {'ES_USERNAME': 'USER', + 'ES_PASSWORD': 'PASS', + 'ES_API_KEY': 'KEY', + 'ES_HOSTS': 'host1:123,host2', + 'ES_BEARER': 'BEARE'}) +def test_build_es_conn_config_es_hosts_csv(): + conf = {} + conf['es_host'] = 'localhost' + conf['es_port'] = 9200 + expected = { + 'use_ssl': False, + 'verify_certs': True, + 'ca_certs': None, + 'client_cert': None, + 'client_key': None, + 'http_auth': None, + 'es_username': 'USER', + 'es_password': 'PASS', + 'es_api_key': 'KEY', + 'es_bearer': 'BEARE', + 'aws_region': None, + 'profile': None, + 'headers': None, + 'es_host': 'localhost', + 'es_hosts': ['host1:123', 'host2:9200'], 'es_port': 9200, 'es_url_prefix': '', 'es_conn_timeout': 20, From a2eb912378f935a4ef6786fb7a6f4b68441a293b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 13 Nov 2021 12:29:48 -0500 Subject: [PATCH 13/14] Further doc clarifications --- docs/source/elastalert.rst | 1 + docs/source/ruletypes.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 72ab99f7..05fa5551 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -127,6 +127,7 @@ is set to true. Note that back filled data may not always trigger count based al When ElastAlert 2 is started, it will query for information about the time that it was last run. This way, even if ElastAlert 2 is stopped and restarted, it will never miss data or look at the same events twice. It will also specify the default cluster for each rule to run on. The environment variable ``ES_HOST`` will override this field. +For multiple host Elasticsearch clusters see ``es_hosts`` parameter. ``es_port``: The port corresponding to ``es_host``. The environment variable ``ES_PORT`` will override this field. diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index c4ebd8fc..4d62c506 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -229,6 +229,7 @@ es_host ``es_host``: The hostname of the Elasticsearch cluster the rule will use to query. (Required, string, no default) The environment variable ``ES_HOST`` will override this field. +For multiple host Elasticsearch clusters see ``es_hosts`` parameter. es_port ^^^^^^^ From 7f02cff331126690e248528c1f6697683978789b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 14 Nov 2021 11:50:54 -0500 Subject: [PATCH 14/14] Clean up parse_hosts() function; add more unit test coverage --- elastalert/util.py | 17 ++++++++++------- tests/util_test.py | 10 +++++++--- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/elastalert/util.py b/elastalert/util.py index 4afe45f1..d4ad4c71 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -542,6 +542,13 @@ def format_string(format_config, target_value): return format_config % (target_value) +def format_host_port(host, port): + host = host.strip() + if ":" not in host: + return "{host}:{port}".format(host=host, port=port) + return host + + def parse_hosts(host, port=9200): """ Convert host str like "host1:port1, host2:port2" to list @@ -549,11 +556,7 @@ def parse_hosts(host, port=9200): :param port: default to 9200 :return: list of hosts """ - if "," in host: - host_list = host.split(",") - host_list = [("{host}:{port}".format(host=x.strip(), port=port) - if ":" not in x else x.strip()) for x in host_list] - return host_list - else: - return ["{host}:{port}".format(host=host, port=port)] + host_list = host.split(",") + host_list = [format_host_port(x, port) for x in host_list] + return host_list diff --git a/tests/util_test.py b/tests/util_test.py index fc4a9d12..3101cfc0 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -597,6 +597,10 @@ def test_pretty_ts(): def test_parse_host(): assert parse_hosts("localhost", port=9200) == ["localhost:9200"] - assert parse_hosts("host1:9200, host2:9200, host3:9300") == ["host1:9200", - "host2:9200", - "host3:9300"] + assert parse_hosts("localhost:9201", port=9200) == ["localhost:9201"] + assert parse_hosts("host1, host2, host3.foo") == ["host1:9200", + "host2:9200", + "host3.foo:9200"] + assert parse_hosts("host1, host2:9200, host3:9300") == ["host1:9200", + "host2:9200", + "host3:9300"]