diff --git a/filebeat/docker-compose.yml b/filebeat/docker-compose.yml index c2349a8d0039..c0bd070decc7 100644 --- a/filebeat/docker-compose.yml +++ b/filebeat/docker-compose.yml @@ -28,12 +28,12 @@ services: elasticsearch: extends: - file: ../testing/environments/${TESTING_ENVIRONMENT}.yml + file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: elasticsearch kibana: extends: - file: ../testing/environments/${TESTING_ENVIRONMENT}.yml + file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: kibana redis: diff --git a/filebeat/module/nginx/access/machine_learning/low_request_rate.json b/filebeat/module/nginx/access/machine_learning/low_request_rate.json index 74fe31b2c66c..3af64d344e7f 100644 --- a/filebeat/module/nginx/access/machine_learning/low_request_rate.json +++ b/filebeat/module/nginx/access/machine_learning/low_request_rate.json @@ -6,8 +6,7 @@ "detectors": [ { "detector_description": "nginx_access_low_request_rate", - "function": "low_count", - "detector_rules": [] + "function": "low_count" } ], "influencers": [] diff --git a/filebeat/module/nginx/access/machine_learning/remote_ip_request_rate.json b/filebeat/module/nginx/access/machine_learning/remote_ip_request_rate.json index 53ea56c264e9..eaaf9fa4624b 100644 --- a/filebeat/module/nginx/access/machine_learning/remote_ip_request_rate.json +++ b/filebeat/module/nginx/access/machine_learning/remote_ip_request_rate.json @@ -6,8 +6,7 @@ { "detector_description": "nginx_access_remote_ip_high_count", "function": "high_count", - "over_field_name": "nginx.access.remote_ip", - "detector_rules": [] + "over_field_name": "nginx.access.remote_ip" } ], "influencers": [ diff --git a/filebeat/module/nginx/access/machine_learning/remote_ip_url_count.json b/filebeat/module/nginx/access/machine_learning/remote_ip_url_count.json index 28ccd5a19373..d2572547a64a 100644 --- a/filebeat/module/nginx/access/machine_learning/remote_ip_url_count.json +++ b/filebeat/module/nginx/access/machine_learning/remote_ip_url_count.json @@ -7,8 +7,7 @@ "detector_description": "nginx_access_remote_ip_high_dc_url", "function": "high_distinct_count", "field_name": "nginx.access.url", - "over_field_name": "nginx.access.remote_ip", - "detector_rules": [] + "over_field_name": "nginx.access.remote_ip" } ], "influencers": [ diff --git a/filebeat/module/nginx/access/machine_learning/response_code.json b/filebeat/module/nginx/access/machine_learning/response_code.json index ee4a71cddf16..13976622fd96 100644 --- a/filebeat/module/nginx/access/machine_learning/response_code.json +++ b/filebeat/module/nginx/access/machine_learning/response_code.json @@ -6,8 +6,7 @@ { "detector_description": "nginx_access_response_code_rate", "function": "count", - "partition_field_name": "nginx.access.response_code", - "detector_rules": [] + "partition_field_name": "nginx.access.response_code" } ], "influencers": [ diff --git a/filebeat/module/nginx/access/machine_learning/visitor_rate.json b/filebeat/module/nginx/access/machine_learning/visitor_rate.json index bc5341e55b86..15871e061917 100644 --- a/filebeat/module/nginx/access/machine_learning/visitor_rate.json +++ b/filebeat/module/nginx/access/machine_learning/visitor_rate.json @@ -6,8 +6,7 @@ "detectors": [ { "detector_description": "nginx_access_visitor_rate", - "function": "non_zero_count", - "detector_rules": [] + "function": "non_zero_count" } ], "influencers": [] diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json index 9b31bf3fa709..f7e93a36eecf 100644 --- a/filebeat/module/nginx/access/test/test.log-expected.json +++ b/filebeat/module/nginx/access/test/test.log-expected.json @@ -16,9 +16,11 @@ "continent_name" : "Europe", "country_iso_code" : "DE", "location" : { - "lon" : 9.0, - "lat" : 51.0 - } + "lon" : 13.4, + "lat" : 52.5167 + }, + "region_name": "Land Berlin", + "city_name": "Berlin" }, "method" : "GET", "user_name" : "-", @@ -127,9 +129,11 @@ "continent_name" : "Europe", "country_iso_code" : "DE", "location" : { - "lon" : 9.0, - "lat" : 51.0 - } + "lon" : 13.4, + "lat" : 52.5167 + }, + "region_name": "Land Berlin", + "city_name": "Berlin" }, "method" : "GET", "user_name" : "-", @@ -299,8 +303,8 @@ "continent_name" : "Europe", "country_iso_code" : "PT", "location" : { - "lon" : -8.13057, - "lat" : 39.6945 + "lon" : -8.0, + "lat" : 39.5 } }, "method" : "GET", diff --git a/filebeat/module/system/auth/test/test.log-expected.json b/filebeat/module/system/auth/test/test.log-expected.json index f601ddb62773..61544f5d0fcf 100644 --- a/filebeat/module/system/auth/test/test.log-expected.json +++ b/filebeat/module/system/auth/test/test.log-expected.json @@ -299,12 +299,12 @@ "ssh": { "geoip": { "continent_name": "Asia", - "city_name": "Guangzhou", + "city_name": "Shenzhen", "country_iso_code": "CN", "region_name": "Guangdong", "location": { - "lon": 113.25, - "lat": 23.1167 + "lon": 114.1333, + "lat": 22.5333 } }, "method": "password", diff --git a/filebeat/module/traefik/access/machine_learning/low_request_rate.json b/filebeat/module/traefik/access/machine_learning/low_request_rate.json index 682af7cda766..ef1475a1fb3a 100644 --- a/filebeat/module/traefik/access/machine_learning/low_request_rate.json +++ b/filebeat/module/traefik/access/machine_learning/low_request_rate.json @@ -6,8 +6,7 @@ "detectors": [ { "detector_description": "traefik_access_low_request_rate", - "function": "low_count", - "detector_rules": [] + "function": "low_count" } ], "influencers": [] diff --git a/filebeat/module/traefik/access/machine_learning/remote_ip_request_rate.json b/filebeat/module/traefik/access/machine_learning/remote_ip_request_rate.json index 86e90358b021..0ffcc2d8eba8 100644 --- a/filebeat/module/traefik/access/machine_learning/remote_ip_request_rate.json +++ b/filebeat/module/traefik/access/machine_learning/remote_ip_request_rate.json @@ -6,8 +6,7 @@ { "detector_description": "traefik_access_remote_ip_high_count", "function": "high_count", - "over_field_name": "traefik.access.remote_ip", - "detector_rules": [] + "over_field_name": "traefik.access.remote_ip" } ], "influencers": [ diff --git a/filebeat/module/traefik/access/machine_learning/remote_ip_url_count.json b/filebeat/module/traefik/access/machine_learning/remote_ip_url_count.json index 8faa56012167..82be4d031fc1 100644 --- a/filebeat/module/traefik/access/machine_learning/remote_ip_url_count.json +++ b/filebeat/module/traefik/access/machine_learning/remote_ip_url_count.json @@ -7,8 +7,7 @@ "detector_description": "traefik_access_remote_ip_high_dc_url", "function": "high_distinct_count", "field_name": "traefik.access.url", - "over_field_name": "traefik.access.remote_ip", - "detector_rules": [] + "over_field_name": "traefik.access.remote_ip" } ], "influencers": [ diff --git a/filebeat/module/traefik/access/machine_learning/response_code.json b/filebeat/module/traefik/access/machine_learning/response_code.json index 3785770dff06..c0662c206346 100644 --- a/filebeat/module/traefik/access/machine_learning/response_code.json +++ b/filebeat/module/traefik/access/machine_learning/response_code.json @@ -6,8 +6,7 @@ { "detector_description": "traefik_access_response_code_rate", "function": "count", - "partition_field_name": "traefik.access.response_code", - "detector_rules": [] + "partition_field_name": "traefik.access.response_code" } ], "influencers": [ diff --git a/filebeat/module/traefik/access/machine_learning/visitor_rate.json b/filebeat/module/traefik/access/machine_learning/visitor_rate.json index 8a59c823c240..794f07a1f905 100644 --- a/filebeat/module/traefik/access/machine_learning/visitor_rate.json +++ b/filebeat/module/traefik/access/machine_learning/visitor_rate.json @@ -6,8 +6,7 @@ "detectors": [ { "detector_description": "traefik_access_visitor_rate", - "function": "non_zero_count", - "detector_rules": [] + "function": "non_zero_count" } ], "influencers": [] diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index dbe86ec08a6a..9ad002624d25 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -89,7 +89,9 @@ def _test_expected_events(self, module, test_file, res, objects): if ev["_source"][module] == obj[module]: found = True break - assert found, "The following expected object was not found: {}".format(obj) + + assert found, "The following expected object was not found:\n {}\nSearched in: \n{}".format( + ev["_source"][module], objects) def run_on_file(self, module, fileset, test_file, cfgfile): print("Testing {}/{} on {}".format(module, fileset, test_file)) diff --git a/libbeat/docker-compose.yml b/libbeat/docker-compose.yml index dd92d1b32275..745fbc56c134 100644 --- a/libbeat/docker-compose.yml +++ b/libbeat/docker-compose.yml @@ -14,8 +14,6 @@ services: - LS_TLS_PORT=5055 # Setup work environment - LIBBEAT_PATH=/go/src/github.com/elastic/beats/libbeat - # TODO: Set to true as soon as stable release is out - - SHIELD=false - KAFKA_HOST=kafka - KAFKA_PORT=9092 - KIBANA_HOST=kibana @@ -39,18 +37,20 @@ services: kafka: { condition: service_healthy } redis: { condition: service_healthy } sredis: { condition: service_healthy } - kafka: { condition: service_healthy } kibana: { condition: service_healthy } + healthcheck: + interval: 1s + retries: 600 elasticsearch: extends: - file: ../testing/environments/${TESTING_ENVIRONMENT}.yml + file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: elasticsearch # This host name is static because of the certificate. logstash: extends: - file: ../testing/environments/${TESTING_ENVIRONMENT}.yml + file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml service: logstash env_file: - ${PWD}/build/test.env @@ -59,11 +59,11 @@ services: condition: service_healthy redis: - build: ${PWD}/../testing/environments/docker/redis + build: ${ES_BEATS}/testing/environments/docker/redis # This host name is static because of the certificate. sredis: # stunnel proxy for redis - build: ${PWD}/../testing/environments/docker/sredis + build: ${ES_BEATS}/testing/environments/docker/sredis expose: - 6380 depends_on: @@ -76,7 +76,7 @@ services: - ${PWD}/build/test.env kafka: - build: ${PWD}/../testing/environments/docker/kafka + build: ${ES_BEATS}/testing/environments/docker/kafka expose: - 9092 - 2181 diff --git a/libbeat/ml-importer/importer_integration_test.go b/libbeat/ml-importer/importer_integration_test.go index 5daa58e5d685..d13a770b7001 100644 --- a/libbeat/ml-importer/importer_integration_test.go +++ b/libbeat/ml-importer/importer_integration_test.go @@ -24,7 +24,6 @@ const sampleJob = ` { "detector_description": "Event rate for nginx.access.response_code", "function": "count", - "detector_rules": [], "partition_field_name": "nginx.access.response_code" } ], diff --git a/testing/environments/args.yml b/testing/environments/args.yml index e03279e87bf1..aba0f6b7a997 100644 --- a/testing/environments/args.yml +++ b/testing/environments/args.yml @@ -7,4 +7,4 @@ services: args: DOWNLOAD_URL: https://snapshots.elastic.co/downloads ELASTIC_VERSION: 7.0.0-alpha1-SNAPSHOT - CACHE_BUST: 20171201 + CACHE_BUST: 20180103 diff --git a/testing/environments/docker/elasticsearch/Dockerfile-snapshot b/testing/environments/docker/elasticsearch/Dockerfile-snapshot index ecc408805eed..2920021d9ede 100644 --- a/testing/environments/docker/elasticsearch/Dockerfile-snapshot +++ b/testing/environments/docker/elasticsearch/Dockerfile-snapshot @@ -55,6 +55,6 @@ RUN chown elasticsearch:elasticsearch config/elasticsearch.yml config/log4j2.pro USER elasticsearch CMD ["/bin/bash", "bin/es-docker"] -HEALTHCHECK CMD curl -f http://localhost:9200 +HEALTHCHECK --interval=1s --retries=600 CMD curl -f http://localhost:9200 EXPOSE 9200 9300 diff --git a/testing/environments/docker/kafka/Dockerfile b/testing/environments/docker/kafka/Dockerfile index 4a49c5d69c9c..99d01f83d082 100644 --- a/testing/environments/docker/kafka/Dockerfile +++ b/testing/environments/docker/kafka/Dockerfile @@ -21,6 +21,6 @@ EXPOSE 9092 EXPOSE 2181 # Healtcheck creates an empty topic foo. As soon as a topic is created, it assumes broke is available -HEALTHCHECK --interval=1s --retries=90 CMD /healthcheck.sh +HEALTHCHECK --interval=1s --retries=600 CMD /healthcheck.sh ENTRYPOINT ["/run.sh"] diff --git a/testing/environments/docker/kibana/Dockerfile-snapshot b/testing/environments/docker/kibana/Dockerfile-snapshot index 1df971dec235..cf8ce1823f74 100644 --- a/testing/environments/docker/kibana/Dockerfile-snapshot +++ b/testing/environments/docker/kibana/Dockerfile-snapshot @@ -1,19 +1,22 @@ # Copy of https://github.com/elastic/kibana-docker/blob/master/build/kibana/Dockerfile FROM centos:7 LABEL maintainer "Elastic Docker Team " - EXPOSE 5601 -ARG DOWNLOAD_URL -ARG ELASTIC_VERSION + +### Beats specific args #### +ARG DOWNLOAD_URL=https://snapshots.elastic.co/downloads +ARG ELASTIC_VERSION=7.0.0-alpha1-SNAPSHOT ARG CACHE_BUST=1 -ARG XPACK +ARG XPACK=1 -# Add Reporting dependencies + healthcheck tool -RUN yum update -y && yum install -y epel-release -RUN yum install -y fontconfig freetype jq && yum clean all +# Healthcheck create by beats team +RUN yum install update -y epel-release && yum install -y jq +HEALTHCHECK --interval=1s --retries=600 CMD curl -f http://localhost:5601/api/status | jq '. | .status.overall.state' | grep -q green +### Beats ### -HEALTHCHECK --interval=1s --retries=300 CMD curl -f http://localhost:5601/api/status | jq '. | .status.overall.state' | grep -q green +# Add Reporting dependencies + healthcheck tool +RUN yum update -y && yum install -y fontconfig freetype && yum clean all WORKDIR /usr/share/kibana RUN curl -Ls ${DOWNLOAD_URL}/kibana/kibana-${ELASTIC_VERSION}-linux-x86_64.tar.gz?c=${CACHE_BUST} | tar --strip-components=1 -zxf - && \ @@ -39,6 +42,8 @@ RUN groupadd --gid 1000 kibana && \ kibana USER kibana -RUN if [ ${XPACK} = "1" ]; then bin/kibana-plugin install ${DOWNLOAD_URL}/kibana-plugins/x-pack/x-pack-${ELASTIC_VERSION}.zip?c=${CACHE_BUST}; fi +# Beats specific check for XPACK to have both variables in one +RUN if [ ${XPACK} = "1" ]; then NODE_OPTIONS="--max-old-space-size=4096" bin/kibana-plugin install ${DOWNLOAD_URL}/kibana-plugins/x-pack/x-pack-${ELASTIC_VERSION}.zip?c=${CACHE_BUST}; fi CMD ["/bin/bash", "/usr/local/bin/kibana-docker"] + diff --git a/testing/environments/docker/kibana/config/kibana-x-pack.yml b/testing/environments/docker/kibana/config/kibana-x-pack.yml index b28b761aa288..b1ff4d106c42 100644 --- a/testing/environments/docker/kibana/config/kibana-x-pack.yml +++ b/testing/environments/docker/kibana/config/kibana-x-pack.yml @@ -6,4 +6,15 @@ server.host: "0" elasticsearch.url: http://elasticsearch:9200 elasticsearch.username: elastic elasticsearch.password: changeme -xpack.monitoring.ui.container.elasticsearch.enabled: true + +# The following config options are currently commented out as they would cause a rebuild of kibana which takes a long time. + +#xpack.monitoring.ui.container.elasticsearch.enabled: true + +# Potentially disabled for beats testing when rebuilding is not needed +#xpack.apm.enabled: false +#xpack.graph.enabled: false +#xpack.ml.enabled: false +#xpack.monitoring.enabled: false +#xpack.reporting.enabled: false +#xpack.watcher.enabled: false diff --git a/testing/environments/docker/logstash/Dockerfile b/testing/environments/docker/logstash/Dockerfile index 0e281028b31b..eaa88ed413b2 100644 --- a/testing/environments/docker/logstash/Dockerfile +++ b/testing/environments/docker/logstash/Dockerfile @@ -22,7 +22,7 @@ COPY docker-entrypoint.sh /entrypoint.sh COPY pki /etc/pki -HEALTHCHECK CMD nc -z localhost 5044 +HEALTHCHECK --interval=1s --retries=600 CMD nc -z localhost 5044 ENTRYPOINT ["/entrypoint.sh"] diff --git a/testing/environments/docker/redis/Dockerfile b/testing/environments/docker/redis/Dockerfile index 368644bc987d..32fef55e011c 100644 --- a/testing/environments/docker/redis/Dockerfile +++ b/testing/environments/docker/redis/Dockerfile @@ -1,2 +1,2 @@ FROM redis:3.2.4-alpine -HEALTHCHECK CMD nc -z localhost 6379 +HEALTHCHECK --interval=1s --retries=600 CMD nc -z localhost 6379 diff --git a/testing/environments/docker/sredis/Dockerfile b/testing/environments/docker/sredis/Dockerfile index 65b6801d2cee..5abbc29468b6 100644 --- a/testing/environments/docker/sredis/Dockerfile +++ b/testing/environments/docker/sredis/Dockerfile @@ -9,7 +9,7 @@ RUN chmod 600 /etc/stunnel/stunnel.conf; \ chmod 600 /etc/pki/tls/certs/*; \ chmod 600 /etc/pki/tls/private/*; -HEALTHCHECK CMD nc -z localhost 6380 +HEALTHCHECK --interval=1s --retries=600 CMD nc -z localhost 6380 EXPOSE 6380 CMD ["stunnel"]