diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/dashboard/ml_http_access_explorer_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/dashboard/ml_http_access_explorer_ecs.json new file mode 100644 index 0000000000000..681ad572417f6 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/dashboard/ml_http_access_explorer_ecs.json @@ -0,0 +1,13 @@ +{ + "hits": 0, + "timeRestore": false, + "description": "", + "title": "ML HTTP Access Explorer (ECS)", + "uiStateJSON": "{\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", + "panelsJSON": "[{\"size_x\":6,\"size_y\":3,\"panelIndex\":1,\"type\":\"visualization\",\"id\":\"ml_http_access_events_timechart_ecs\",\"col\":1,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":2,\"type\":\"visualization\",\"id\":\"ml_http_access_unique_count_url_timechart_ecs\",\"col\":7,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":3,\"type\":\"visualization\",\"id\":\"ml_http_access_status_code_timechart_ecs\",\"col\":1,\"row\":4},{\"size_x\":6,\"size_y\":3,\"panelIndex\":4,\"type\":\"visualization\",\"id\":\"ml_http_access_source_ip_timechart_ecs\",\"col\":7,\"row\":4},{\"size_x\":6,\"size_y\":3,\"panelIndex\":5,\"type\":\"visualization\",\"id\":\"ml_http_access_top_source_ips_table_ecs\",\"col\":1,\"row\":8},{\"size_x\":6,\"size_y\":3,\"panelIndex\":6,\"type\":\"visualization\",\"id\":\"ml_http_access_map_ecs\",\"col\":7,\"row\":8},{\"size_x\":12,\"size_y\":9,\"panelIndex\":7,\"type\":\"visualization\",\"id\":\"ml_http_access_top_urls_table_ecs\",\"col\":1,\"row\":11}]", + "optionsJSON": "{\"darkTheme\":false}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\n \"filter\": [],\n \"highlightAll\": true,\n \"version\": true,\n \"query\": {\n \"query\": \"\",\n \"language\": \"lucene\"\n }\n}" + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/search/ml_http_access_filebeat_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/search/ml_http_access_filebeat_ecs.json new file mode 100644 index 0000000000000..6575c7e65d78d --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/search/ml_http_access_filebeat_ecs.json @@ -0,0 +1,16 @@ +{ + "sort": [ + "@timestamp", + "desc" + ], + "hits": 0, + "description": "Filebeat HTTP Access Data (ECS)", + "title": "ML HTTP Access Data (ECS)", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"INDEX_PATTERN_ID\",\"query\":{\"query_string\":{\"query\":\"fileset.name:access\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" + }, + "columns": [ + "_source" + ] +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_events_timechart_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_events_timechart_ecs.json new file mode 100644 index 0000000000000..ff507da41d1a5 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_events_timechart_ecs.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML HTTP Access Event Timechart (ECS)\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"event.module\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"event.dataset\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "ML HTTP Access Event Timechart (ECS)", + "uiStateJSON": "{\"vis\":{\"colors\":{\"apache - access\":\"#629E51\",\"nginx - access\":\"#1F78C1\"}}}", + "version": 1, + "savedSearchId": "ml_http_access_filebeat_ecs", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } + } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_map_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_map_ecs.json new file mode 100644 index 0000000000000..4cf3982f5ac6a --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_map_ecs.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"autoPrecision\":true,\"field\":\"source.geo.location\"},\"schema\":\"segment\",\"type\":\"geohash_grid\"}],\"listeners\":{},\"params\":{\"addTooltip\":true,\"heatBlur\":15,\"heatMaxZoom\":16,\"heatMinOpacity\":0.1,\"heatNormalizeData\":true,\"heatRadius\":25,\"isDesaturated\":true,\"legendPosition\":\"bottomright\",\"mapCenter\":[15,5],\"mapType\":\"Scaled Circle Markers\",\"mapZoom\":2,\"wms\":{\"enabled\":false,\"options\":{\"attribution\":\"Maps provided by USGS\",\"format\":\"image/png\",\"layers\":\"0\",\"styles\":\"\",\"transparent\":true,\"version\":\"1.3.0\"},\"url\":\"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\"}},\"title\":\"ML HTTP Access Map (ECS)\",\"type\":\"tile_map\"}", + "description": "", + "title": "ML HTTP Access Map (ECS)", + "uiStateJSON": "{\n \"mapCenter\": [\n 12.039320557540572,\n -0.17578125\n ]\n}", + "version": 1, + "savedSearchId": "ml_http_access_filebeat_ecs", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_source_ip_timechart_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_source_ip_timechart_ecs.json new file mode 100644 index 0000000000000..7525752abb053 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_source_ip_timechart_ecs.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML HTTP Access Source IP Timechart (ECS)\",\"type\":\"area\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per 5 minutes\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"area\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"source.address\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "ML HTTP Access Source IP Timechart (ECS)", + "uiStateJSON": "{}", + "version": 1, + "savedSearchId": "ml_http_access_filebeat_ecs", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_status_code_timechart_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_status_code_timechart_ecs.json new file mode 100644 index 0000000000000..e104b047f924b --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_status_code_timechart_ecs.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML HTTP Access Status Code Timechart (ECS)\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"http.response.status_code\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "ML HTTP Access Status Code Timechart (ECS)", + "uiStateJSON": "{\n \"vis\": {\n \"colors\": {\n \"200\": \"#7EB26D\",\n \"404\": \"#614D93\"\n }\n }\n}", + "version": 1, + "savedSearchId": "ml_http_access_filebeat_ecs", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_top_source_ips_table_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_top_source_ips_table_ecs.json new file mode 100644 index 0000000000000..c9afd66ea39b1 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_top_source_ips_table_ecs.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML HTTP Access Top Source IPs Table (ECS)\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source.address\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "ML HTTP Access Top Source IPs Table (ECS)", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "savedSearchId": "ml_http_access_filebeat_ecs", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_top_urls_table_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_top_urls_table_ecs.json new file mode 100644 index 0000000000000..34fbcb621085c --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_top_urls_table_ecs.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML HTTP Access Top URLs Table (ECS)\",\"type\":\"table\",\"params\":{\"perPage\":100,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"url.original\",\"size\":1000,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "ML HTTP Access Top URLs Table (ECS)", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "savedSearchId": "ml_http_access_filebeat_ecs", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_unique_count_url_timechart_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_unique_count_url_timechart_ecs.json new file mode 100644 index 0000000000000..d14592e68bb6e --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/kibana/visualization/ml_http_access_unique_count_url_timechart_ecs.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML HTTP Access Unique Count URL Timechart (ECS)\",\"type\":\"line\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"@timestamp per day\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Unique count of url.original\"}}],\"seriesParams\":[{\"show\":true,\"mode\":\"normal\",\"type\":\"line\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"data\":{\"id\":\"1\",\"label\":\"Unique count of url.original\"},\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"showCircles\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"url.original\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", + "description": "", + "title": "ML HTTP Access Unique Count URL Timechart (ECS)", + "uiStateJSON": "{}", + "version": 1, + "savedSearchId": "ml_http_access_filebeat_ecs", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/logo.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/logo.json new file mode 100644 index 0000000000000..e15620ed29fcc --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/logo.json @@ -0,0 +1,6 @@ +{ + "src": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAA5CAIAAADYwb7wAAAHTUlEQVR4AbXTa1iT5xkH8Lvt7BTFtXioxjcJggIiHkQFUQQFJgdEUGB27arOQxHdCgNdUStFC1OrbR3latU5kIAogoKUg1JQAjkgkIQEAjSExBBCSAIkkDM5sMf5rR/64ZU81/vhzvX+r/v33s/zBKZnYtlVPHOhl/EGwabo+dUrmInu3aYrYM4Dwzcw+QXYdaoZBTQi/eVX3c233zbdnKfNAkNxzMwBZo3+azD++1V3c76T6SYYrsPkFZg2js0EYLOidsbvwHQLpu76ThV5mhHwPWgvARrLbre/KTCRG69Jh6kCMJUGTlXuM99Cc4DpRzBfhWm9wmq1cbg8iUSCEzAz78oTYfQ0mAq9ppnpZoqTpQCm8sH4AxivoS1So8zYuFqhUOIBrAqxbB8oU2cp/gbG6kxjUfhENkzkOKuznEfTQH01eNpux38GdptVdgBkB0F+BLR3UvTlWYpjoEpbqEpzUZ1aOHIAjI15b3SL1DfTxBEwfGSJMiPQ0ERRnnZXpswfPb1g7HOX8Qxn5TGwjUnxA+Z+jsAfhg57SBJA/zRfczt16ACMpBAVqUtUp5YqjsJE7m78/2S71SIMf1e0e554z2zVt8n65/cG/wSypOXDyST5SeLIZ5gsAaxDvfgBVe65nrUg3u8zEOViaH6szP5Ikug0dHiF7Ijb8HG3wXgYzY5GMZyARSrikUG427tvy9vjd67qfi4VhsDgXzyln6yQHnIfOkQWh4FVJcUL2O09fht6NswV7FrVH71Gz2iQJkcKo4niRK+X+z0HD3j3B4D6zpcoiBPQPKigAfQGb+ryfEtdckvzuLhnHQzE+ojiVokTVgsCYfDgGvQROAGb0dgGi1ngwVu++pfIAH1ri2h/iGCHe3+UtzDGqz+CMJJ10KpWoSROYPh8Dh2At8L/BThr7t+frCnnEqBv59q+nT59OzwHj+8fu5E3WVdnt1rxABbVKANmsz9Yx17k07V2p57G7A0K5WLePLf13OUbuO6+Pf6hDICJigqcE7xMTmeCU6e7Xyu8P1Zwd/zeQwbM78Q2dy7x5RJ8uZgvC1w5Lp7TFgsewCIbocHvOYQN7KXrOkkbtejzQ2NZ8705mC8H29BJ8uWSN9IAtPWNKIwHEB/9jA7OHDd/JixQ3aZM1NYz0ee7bkQA8tC+obFkZy6gJB5galBGhfdYyzaxl25sn+OtY3YIEw93zPLoxF4BnCXr+JvCRH89ae4X4QQkGTlNQGgnb2UAceSbHyYbqQxw4ZD8ONgm9KB94/v9kesRYJHJ8QDWSe1zWE13CXhB2t4M7loqU5KW2QrubDQQesibO+Z602G2rqkFhfEA8vzyOnClk0OaYJ0wJUvLYLXAyjZSQAfmx1q2uWPxen5wrDLvNkriAew2W8vK6EbY0kIKeworNc+ZwzeLnsGKVuL2F9jWNmxLG2Ezb3OE+PgpdDvxANouQRX4NGHhjfOCWFGfalndNCyk5YMgOhbEwIJalwW2uQY+gzljxWU4JxBcuFEB6xtdo6tgjbKyQVXVWANezeSwFmIIDdtJx4LR2VDB26bV4QFsFkuda1z1nNCnhOgnfwjTvOCx4v7e4BTchO2iYmHNy0L/v29uA2k5KIwHmOgSlsG2WmLcI9gmvEZR0zmV4NtAjmokRjzDwhHTRAitgZVmqRwnMHCjsgS2V5MTSmHrOJMnvFZYCdvqSTH1pKifiZHP3HdXgkdv2mWUxAlQYzNKIfKhczQj4aya3ff4vegawp46YuwTLKaevKfmnR2VsNZmMOEErFpD+eL4soXxFNg+/BNdXk27CwHVpPhq0t5aclwtIaYIlqtpHJTECYzypXcg7uHSxPtOcWMdvfQ/Z5XNiakkJlSREquW7i0Ar4HcEhTDD3Q/Yn0HH5dAJPcrymh7HwV2PSJ9WOH64YM5MRTwF+WVogwewGa1qfrkqGAXtFx8N+lHiBxl8oUFTwog/P77CRQIfbRgr/YXyeskzgmEDfzWvAZm7tMv3kku3Hp+omvgJ5+jFa6f0D/+Sl7bajNbNNJxLoWOH0DrvxHfZs46cQ4+5dxrk9F6L8G+yvhr/GKatE30+GTxxd8lTemMKIYfUPWNpMOxc28lyzsHGy7VZcz9x79WZmYtSj8Fh66vOmPU6FEGJ2C32V//uLLqy+8DLsu40gvY5zluZ3NIGdfXZ1G/rhvqEKO3+IEpk0XS/lLGHcrEzrRTmP1UQSqcyPY4f5GYcXZu6vWN2Qq+7I0AtPi13Ulw4iScUAmU+R/lJ8HxM4v+eX5ZRunRwmHOIArgB9DFkHXLO8rYvfW9jHwm+yGnNKUM1cImgVYx+es4PkA3bnh9EkK6SDeuk7ClOJv99i0SNA/waviosE5ZTTrTzAPNBW1olN9O4wdGpZqmgnZUOArg1gu6GoQOBJpLuNpxg6OAyTED9QEPFY4CWM2CDqrAgUD5vRaVcsJRgFZrKC5qnHbYAiazt6623YHAf27ViUQjDgTo9J5pR67/AfRrVpVr6wheAAAAAElFTkSuQmCC", + "height": 25, + "width": 125 +} + diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/manifest.json new file mode 100644 index 0000000000000..031cf05ca39cd --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/manifest.json @@ -0,0 +1,111 @@ +{ + "id": "apache_ecs", + "title": "Apache access logs", + "description": "Find unusual activity in HTTP access logs from filebeat (ECS)", + "type": "Web Access Logs", + "logoFile": "logo.json", + "defaultIndexPattern": "filebeat-*", + "query": { + "bool": { + "filter": [ + { "term": { "event.dataset": "apache.access" } }, + { "exists": { "field": "source.address" } }, + { "exists": { "field": "url.original" } }, + { "exists": { "field": "http.response.status_code" } } + ] + } + }, + "jobs": [ + { + "id": "visitor_rate_ecs", + "file": "visitor_rate_ecs.json" + }, + { + "id": "status_code_rate_ecs", + "file": "status_code_rate_ecs.json" + }, + { + "id": "source_ip_url_count_ecs", + "file": "source_ip_url_count_ecs.json" + }, + { + "id": "source_ip_request_rate_ecs", + "file": "source_ip_request_rate_ecs.json" + }, + { + "id": "low_request_rate_ecs", + "file": "low_request_rate_ecs.json" + } + ], + "datafeeds": [ + { + "id": "datafeed-visitor_rate_ecs", + "file": "datafeed_visitor_rate_ecs.json", + "job_id": "visitor_rate_ecs" + }, + { + "id": "datafeed-status_code_rate_ecs", + "file": "datafeed_status_code_rate_ecs.json", + "job_id": "status_code_rate_ecs" + }, + { + "id": "datafeed-source_ip_url_count_ecs", + "file": "datafeed_source_ip_url_count_ecs.json", + "job_id": "source_ip_url_count_ecs" + }, + { + "id": "datafeed-source_ip_request_rate_ecs", + "file": "datafeed_source_ip_request_rate_ecs.json", + "job_id": "source_ip_request_rate_ecs" + }, + { + "id": "datafeed-low_request_rate_ecs", + "file": "datafeed_low_request_rate_ecs.json", + "job_id": "low_request_rate_ecs" + } + ], + "kibana": { + "dashboard": [ + { + "id": "ml_http_access_explorer_ecs", + "file": "ml_http_access_explorer_ecs.json" + } + ], + "search": [ + { + "id": "ml_http_access_filebeat_ecs", + "file": "ml_http_access_filebeat_ecs.json" + } + ], + "visualization": [ + { + "id": "ml_http_access_map_ecs", + "file": "ml_http_access_map_ecs.json" + }, + { + "id": "ml_http_access_source_ip_timechart_ecs", + "file": "ml_http_access_source_ip_timechart_ecs.json" + }, + { + "id": "ml_http_access_status_code_timechart_ecs", + "file": "ml_http_access_status_code_timechart_ecs.json" + }, + { + "id": "ml_http_access_top_source_ips_table_ecs", + "file": "ml_http_access_top_source_ips_table_ecs.json" + }, + { + "id": "ml_http_access_top_urls_table_ecs", + "file": "ml_http_access_top_urls_table_ecs.json" + }, + { + "id": "ml_http_access_unique_count_url_timechart_ecs", + "file": "ml_http_access_unique_count_url_timechart_ecs.json" + }, + { + "id": "ml_http_access_events_timechart_ecs", + "file": "ml_http_access_events_timechart_ecs.json" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/datafeed_low_request_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/datafeed_low_request_rate_ecs.json new file mode 100644 index 0000000000000..b08a8526b17e5 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/datafeed_low_request_rate_ecs.json @@ -0,0 +1,34 @@ +{ + "job_id": "JOB_ID", + "indexes": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { "term": { "event.dataset": "apache.access" } } + ] + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "interval": 900000, + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + } + } + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/datafeed_source_ip_request_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/datafeed_source_ip_request_rate_ecs.json new file mode 100644 index 0000000000000..824d6a934d865 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/datafeed_source_ip_request_rate_ecs.json @@ -0,0 +1,13 @@ +{ + "job_id": "JOB_ID", + "indexes": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { "term": { "event.dataset": "apache.access" } } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/datafeed_source_ip_url_count_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/datafeed_source_ip_url_count_ecs.json new file mode 100644 index 0000000000000..824d6a934d865 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/datafeed_source_ip_url_count_ecs.json @@ -0,0 +1,13 @@ +{ + "job_id": "JOB_ID", + "indexes": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { "term": { "event.dataset": "apache.access" } } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/datafeed_status_code_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/datafeed_status_code_rate_ecs.json new file mode 100644 index 0000000000000..824d6a934d865 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/datafeed_status_code_rate_ecs.json @@ -0,0 +1,13 @@ +{ + "job_id": "JOB_ID", + "indexes": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { "term": { "event.dataset": "apache.access" } } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/datafeed_visitor_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/datafeed_visitor_rate_ecs.json new file mode 100644 index 0000000000000..b88b66f15b329 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/datafeed_visitor_rate_ecs.json @@ -0,0 +1,39 @@ +{ + "job_id": "JOB_ID", + "indexes": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { "term": { "event.dataset": "apache.access" } } + ] + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "interval": 900000, + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + }, + "dc_source_address": { + "cardinality": { + "field": "source.address" + } + } + } + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/low_request_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/low_request_rate_ecs.json new file mode 100644 index 0000000000000..1bfb864bef9ee --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/low_request_rate_ecs.json @@ -0,0 +1,34 @@ +{ + "groups": ["apache"], + "description": "HTTP Access Logs: Detect low request rate (ECS)", + "analysis_config" : { + "bucket_span": "15m", + "summary_count_field_name": "doc_count", + "detectors": [ + { + "detector_description": "apache_access_low_request_rate", + "function": "low_count" + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-apache-access", + "custom_urls": [ + { + "url_name": "Raw data", + "url_value": "kibana#/discover/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027,type:phrase),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:lucene,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/source_ip_request_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/source_ip_request_rate_ecs.json new file mode 100644 index 0000000000000..25c51caec8727 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/source_ip_request_rate_ecs.json @@ -0,0 +1,34 @@ +{ + "groups": ["apache"], + "description": "HTTP Access Logs: Detect unusual source ips - high request rates (ECS)", + "analysis_config" : { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "apache_access_source_ip_high_count", + "function": "high_count", + "over_field_name": "source.address" + } + ], + "influencers": [ + "source.address" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-apache-access", + "custom_urls": [ + { + "url_name": "Investigate source IP", + "url_value": "kibana#/dashboard/ml_http_access_explorer_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027,type:phrase),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),query:(language:lucene,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "kibana#/discover/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027,type:phrase),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:lucene,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/source_ip_url_count_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/source_ip_url_count_ecs.json new file mode 100644 index 0000000000000..2982e04771086 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/source_ip_url_count_ecs.json @@ -0,0 +1,35 @@ +{ + "groups": ["apache"], + "description": "HTTP Access Logs: Detect unusual source ips - high distinct count of urls (ECS)", + "analysis_config" : { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "apache_access_source_ip_high_dc_url", + "function": "high_distinct_count", + "field_name": "url.original", + "over_field_name": "source.address" + } + ], + "influencers": [ + "source.address" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-apache-access", + "custom_urls": [ + { + "url_name": "Investigate source IP", + "url_value": "kibana#/dashboard/ml_http_access_explorer_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027,type:phrase),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),query:(language:lucene,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "kibana#/discover/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027,type:phrase),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:lucene,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/status_code_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/status_code_rate_ecs.json new file mode 100644 index 0000000000000..39bf67c71ae31 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/status_code_rate_ecs.json @@ -0,0 +1,41 @@ +{ + "groups": ["apache"], + "description": "HTTP Access Logs: Detect unusual status_code rates (ECS)", + "analysis_config" : { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "apache_access_status_code_rate", + "function": "count", + "partition_field_name": "http.response.status_code" + } + ], + "influencers": [ + "http.response.status_code", + "source.address" + ] + }, + "analysis_limits": { + "model_memory_limit": "100mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-apache-access", + "custom_urls": [ + { + "url_name": "Investigate status code", + "url_value": "kibana#/dashboard/ml_http_access_explorer_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027,type:phrase),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,params:(query:\u0027$http.response.status_code$\u0027,type:phrase),type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),query:(language:lucene,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "kibana#/discover/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027,type:phrase),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:lucene,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/visitor_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/visitor_rate_ecs.json new file mode 100644 index 0000000000000..66ffde6bccf84 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache_ecs/ml/visitor_rate_ecs.json @@ -0,0 +1,34 @@ +{ + "groups": ["apache"], + "description": "HTTP Access Logs: Detect unusual visitor rate (ECS)", + "analysis_config" : { + "bucket_span": "15m", + "summary_count_field_name": "dc_source_address", + "detectors": [ + { + "detector_description": "apache_access_visitor_rate", + "function": "non_zero_count" + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-apache-access", + "custom_urls": [ + { + "url_name": "Raw data", + "url_value": "kibana#/discover/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027,type:phrase),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:lucene,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/dashboard/ml_http_access_explorer_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/dashboard/ml_http_access_explorer_ecs.json new file mode 100644 index 0000000000000..681ad572417f6 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/dashboard/ml_http_access_explorer_ecs.json @@ -0,0 +1,13 @@ +{ + "hits": 0, + "timeRestore": false, + "description": "", + "title": "ML HTTP Access Explorer (ECS)", + "uiStateJSON": "{\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", + "panelsJSON": "[{\"size_x\":6,\"size_y\":3,\"panelIndex\":1,\"type\":\"visualization\",\"id\":\"ml_http_access_events_timechart_ecs\",\"col\":1,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":2,\"type\":\"visualization\",\"id\":\"ml_http_access_unique_count_url_timechart_ecs\",\"col\":7,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":3,\"type\":\"visualization\",\"id\":\"ml_http_access_status_code_timechart_ecs\",\"col\":1,\"row\":4},{\"size_x\":6,\"size_y\":3,\"panelIndex\":4,\"type\":\"visualization\",\"id\":\"ml_http_access_source_ip_timechart_ecs\",\"col\":7,\"row\":4},{\"size_x\":6,\"size_y\":3,\"panelIndex\":5,\"type\":\"visualization\",\"id\":\"ml_http_access_top_source_ips_table_ecs\",\"col\":1,\"row\":8},{\"size_x\":6,\"size_y\":3,\"panelIndex\":6,\"type\":\"visualization\",\"id\":\"ml_http_access_map_ecs\",\"col\":7,\"row\":8},{\"size_x\":12,\"size_y\":9,\"panelIndex\":7,\"type\":\"visualization\",\"id\":\"ml_http_access_top_urls_table_ecs\",\"col\":1,\"row\":11}]", + "optionsJSON": "{\"darkTheme\":false}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\n \"filter\": [],\n \"highlightAll\": true,\n \"version\": true,\n \"query\": {\n \"query\": \"\",\n \"language\": \"lucene\"\n }\n}" + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/search/ml_http_access_filebeat_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/search/ml_http_access_filebeat_ecs.json new file mode 100644 index 0000000000000..6575c7e65d78d --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/search/ml_http_access_filebeat_ecs.json @@ -0,0 +1,16 @@ +{ + "sort": [ + "@timestamp", + "desc" + ], + "hits": 0, + "description": "Filebeat HTTP Access Data (ECS)", + "title": "ML HTTP Access Data (ECS)", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"INDEX_PATTERN_ID\",\"query\":{\"query_string\":{\"query\":\"fileset.name:access\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" + }, + "columns": [ + "_source" + ] +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_events_timechart_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_events_timechart_ecs.json new file mode 100644 index 0000000000000..ff507da41d1a5 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_events_timechart_ecs.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML HTTP Access Event Timechart (ECS)\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"event.module\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"event.dataset\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "ML HTTP Access Event Timechart (ECS)", + "uiStateJSON": "{\"vis\":{\"colors\":{\"apache - access\":\"#629E51\",\"nginx - access\":\"#1F78C1\"}}}", + "version": 1, + "savedSearchId": "ml_http_access_filebeat_ecs", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } + } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_map_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_map_ecs.json new file mode 100644 index 0000000000000..4cf3982f5ac6a --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_map_ecs.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"autoPrecision\":true,\"field\":\"source.geo.location\"},\"schema\":\"segment\",\"type\":\"geohash_grid\"}],\"listeners\":{},\"params\":{\"addTooltip\":true,\"heatBlur\":15,\"heatMaxZoom\":16,\"heatMinOpacity\":0.1,\"heatNormalizeData\":true,\"heatRadius\":25,\"isDesaturated\":true,\"legendPosition\":\"bottomright\",\"mapCenter\":[15,5],\"mapType\":\"Scaled Circle Markers\",\"mapZoom\":2,\"wms\":{\"enabled\":false,\"options\":{\"attribution\":\"Maps provided by USGS\",\"format\":\"image/png\",\"layers\":\"0\",\"styles\":\"\",\"transparent\":true,\"version\":\"1.3.0\"},\"url\":\"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\"}},\"title\":\"ML HTTP Access Map (ECS)\",\"type\":\"tile_map\"}", + "description": "", + "title": "ML HTTP Access Map (ECS)", + "uiStateJSON": "{\n \"mapCenter\": [\n 12.039320557540572,\n -0.17578125\n ]\n}", + "version": 1, + "savedSearchId": "ml_http_access_filebeat_ecs", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_source_ip_timechart_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_source_ip_timechart_ecs.json new file mode 100644 index 0000000000000..8b2a9e14a74b8 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_source_ip_timechart_ecs.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML HTTP Access Source IP Timechart (ECS)\",\"type\":\"area\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per 5 minutes\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"area\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"source.address\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "ML HTTP Access Source IP Timechart (ECS)", + "uiStateJSON": "{}", + "version": 1, + "savedSearchId": "ml_http_access_filebeat_ecs", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_status_code_timechart_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_status_code_timechart_ecs.json new file mode 100644 index 0000000000000..e104b047f924b --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_status_code_timechart_ecs.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML HTTP Access Status Code Timechart (ECS)\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"http.response.status_code\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "ML HTTP Access Status Code Timechart (ECS)", + "uiStateJSON": "{\n \"vis\": {\n \"colors\": {\n \"200\": \"#7EB26D\",\n \"404\": \"#614D93\"\n }\n }\n}", + "version": 1, + "savedSearchId": "ml_http_access_filebeat_ecs", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_top_source_ips_table_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_top_source_ips_table_ecs.json new file mode 100644 index 0000000000000..c9afd66ea39b1 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_top_source_ips_table_ecs.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML HTTP Access Top Source IPs Table (ECS)\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source.address\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "ML HTTP Access Top Source IPs Table (ECS)", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "savedSearchId": "ml_http_access_filebeat_ecs", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_top_urls_table_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_top_urls_table_ecs.json new file mode 100644 index 0000000000000..34fbcb621085c --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_top_urls_table_ecs.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML HTTP Access Top URLs Table (ECS)\",\"type\":\"table\",\"params\":{\"perPage\":100,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"url.original\",\"size\":1000,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "ML HTTP Access Top URLs Table (ECS)", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "savedSearchId": "ml_http_access_filebeat_ecs", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_unique_count_url_timechart_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_unique_count_url_timechart_ecs.json new file mode 100644 index 0000000000000..d14592e68bb6e --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/kibana/visualization/ml_http_access_unique_count_url_timechart_ecs.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"ML HTTP Access Unique Count URL Timechart (ECS)\",\"type\":\"line\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"@timestamp per day\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Unique count of url.original\"}}],\"seriesParams\":[{\"show\":true,\"mode\":\"normal\",\"type\":\"line\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"data\":{\"id\":\"1\",\"label\":\"Unique count of url.original\"},\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"showCircles\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"url.original\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", + "description": "", + "title": "ML HTTP Access Unique Count URL Timechart (ECS)", + "uiStateJSON": "{}", + "version": 1, + "savedSearchId": "ml_http_access_filebeat_ecs", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{}" + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/logo.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/logo.json new file mode 100644 index 0000000000000..9372c36cbfa6d --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/logo.json @@ -0,0 +1,5 @@ +{ + "src": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAlCAMAAAAUaRt1AAACPVBMVEUAcQAAdgAAfQAAfgAAfwAAgAAAggAAhAAAhQAAhgAAhwIAhwUAiAcAiAoAiQkAiQwAiQ8Aig4AihEAixMAixUAixcAjBYAjBcAjBkAjRgAjRoAjRwAjh0Ajh8Ajx8AjyAAkCIAkCMAkSMAkSUAkSYAkiYAkicAkigAkikAkykAkyoAkywAlCsAlC0AlS4AlTAAli8AljEAljIAlzIAlzMAlzUAmDQAmDYAmDcAmTcAmTgAmToAmTsAmjgAmjkAmjsEmToLmz0TnD4TnEAYnEAanUEcnkMfnkMhn0Qhn0UnoUcroksvo0svo0w4pVI+qFY+qFdEq1xFq1xIrF5KrF9KrWFLrWBNrWFNrmJPr2NTsWhUsGdVsWdXsWlYsmtZs2xbtW5puXhvvH1vvH5zv4N2v4R3wId3wYd6w4p/w4uAw4uAxIyBxIyBxY6CxY6ExY+FxpCHyJSJyZaMypmOy5uQypmRy5uSzJyUzZ2b0aed0aig06yj06ul1K2p1rCq1rGs2LWu2reu2rix2rex3Ly84MS94MO+4MO+4cS+4ca/4sfB4cbC48nJ5c3L5s7L5s/M59DM6NLN6NLO6NLO6dXQ6tbR69jU69bY7t7Z7t3Z7t7a79/b8OHc8OHe8OLf8ePh8OTh8uXm9Oro8+no9Ovp9Orp9ezq9ezq9u3r9u3s9e3s9e/u9e/u9/Hv+PPw9/Lx+PPy+fTy+fXz+fXz+vb0+vb3+/n3/Pn4/Pr5+/n6/Pr9/v3+/v7+//7///+8pzoSAAACRElEQVR42nXU91/TQBQA8CiUpli4po22jaVNRevosulLzgQXuAruiVi3uPceqKgI4sK6994b9wD927ykkRJI3g/Ju8v3k1zu7h311xCPly58YOwxgE/7AiGe29NpBU6nXQmME+74yW4zcGMBimAggSNo3rV+4FUjG5BE0EKUgp51zwzgd1Mlm5KgJySB5Y/8KIALdWgMBkPgKJpxVgf3Vrt4LMbjUAgxERdxmFl5h4DP2/1+kIVSjqMjiv58opfl0DgM3JCNX6hZZUkMac/ajovbOC4vJO/M8x1ZBoM0FtVSUfXjzqz6uTZHhayCmO82aVUHSCpHqIxA7iUHtBE327Sxhmp+kcaKQSqemgf2Xfmf3lGUFgH48e9IXo96g536tCy3yQRUvSHpElPQPc2uWIIrr8nleRApVuDQdPUdOTo4wgKcodar4nixb5I5aC8deEwVmwdUvTUFbYOH0pdUMafkozlwiyjwlCTfaq9agORk+5Qukr18ZAVAsTVo82EJRCjaqoNFqO9atBMAUrS42fiGwmrmGAJArqBzGsjqq6ktcFk96TlIqxYUxN8krfdCOL8fZjuThHjWtOwfPkrfcO6RW1qO1rEy4FR5hvq6yceBDE6nY/T/LalU0k6Hb4Ikct4NnWRX313FhDGk0tArUoKIh7mW3dLr4lzGFe1bFzFXTWuhsn428axgrKzQ4e+G2nzR6DHW5pN+1X19fk91l8+9bHY+/DmVYhIYJ92xE11WJ8xeLsT7d3+wOmFIPGxYfN/Y8w+Jrert1P9yBgAAAABJRU5ErkJggg==", + "height": 25, + "width": 120 +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/manifest.json new file mode 100644 index 0000000000000..bd31fcd23f504 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/manifest.json @@ -0,0 +1,111 @@ +{ + "id": "nginx_ecs", + "title": "Nginx access logs", + "description": "Find unusual activity in HTTP access logs from filebeat (ECS)", + "type": "Web Access Logs", + "logoFile": "logo.json", + "defaultIndexPattern": "filebeat-*", + "query": { + "bool": { + "filter": [ + { "term": { "event.dataset": "nginx.access" } }, + { "exists": { "field": "source.address" } }, + { "exists": { "field": "url.original" } }, + { "exists": { "field": "http.response.status_code" } } + ] + } + }, + "jobs": [ + { + "id": "visitor_rate_ecs", + "file": "visitor_rate_ecs.json" + }, + { + "id": "status_code_rate_ecs", + "file": "status_code_rate_ecs.json" + }, + { + "id": "source_ip_url_count_ecs", + "file": "source_ip_url_count_ecs.json" + }, + { + "id": "source_ip_request_rate_ecs", + "file": "source_ip_request_rate_ecs.json" + }, + { + "id": "low_request_rate_ecs", + "file": "low_request_rate_ecs.json" + } + ], + "datafeeds": [ + { + "id": "datafeed-visitor_rate_ecs", + "file": "datafeed_visitor_rate_ecs.json", + "job_id": "visitor_rate_ecs" + }, + { + "id": "datafeed-status_code_rate_ecs", + "file": "datafeed_status_code_rate_ecs.json", + "job_id": "status_code_rate_ecs" + }, + { + "id": "datafeed-source_ip_url_count_ecs", + "file": "datafeed_source_ip_url_count_ecs.json", + "job_id": "source_ip_url_count_ecs" + }, + { + "id": "datafeed-source_ip_request_rate_ecs", + "file": "datafeed_source_ip_request_rate_ecs.json", + "job_id": "source_ip_request_rate_ecs" + }, + { + "id": "datafeed-low_request_rate_ecs", + "file": "datafeed_low_request_rate_ecs.json", + "job_id": "low_request_rate_ecs" + } + ], + "kibana": { + "dashboard": [ + { + "id": "ml_http_access_explorer_ecs", + "file": "ml_http_access_explorer_ecs.json" + } + ], + "search": [ + { + "id": "ml_http_access_filebeat_ecs", + "file": "ml_http_access_filebeat_ecs.json" + } + ], + "visualization": [ + { + "id": "ml_http_access_map_ecs", + "file": "ml_http_access_map_ecs.json" + }, + { + "id": "ml_http_access_source_ip_timechart_ecs", + "file": "ml_http_access_source_ip_timechart_ecs.json" + }, + { + "id": "ml_http_access_status_code_timechart_ecs", + "file": "ml_http_access_status_code_timechart_ecs.json" + }, + { + "id": "ml_http_access_top_source_ips_table_ecs", + "file": "ml_http_access_top_source_ips_table_ecs.json" + }, + { + "id": "ml_http_access_top_urls_table_ecs", + "file": "ml_http_access_top_urls_table_ecs.json" + }, + { + "id": "ml_http_access_unique_count_url_timechart_ecs", + "file": "ml_http_access_unique_count_url_timechart_ecs.json" + }, + { + "id": "ml_http_access_events_timechart_ecs", + "file": "ml_http_access_events_timechart_ecs.json" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/datafeed_low_request_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/datafeed_low_request_rate_ecs.json new file mode 100644 index 0000000000000..cf143071aa519 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/datafeed_low_request_rate_ecs.json @@ -0,0 +1,34 @@ +{ + "job_id": "JOB_ID", + "indexes": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { "term": { "event.dataset": "nginx.access" } } + ] + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "interval": 900000, + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + } + } + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/datafeed_source_ip_request_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/datafeed_source_ip_request_rate_ecs.json new file mode 100644 index 0000000000000..bccf1bd8de6d5 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/datafeed_source_ip_request_rate_ecs.json @@ -0,0 +1,13 @@ +{ + "job_id": "JOB_ID", + "indexes": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { "term": { "event.dataset": "nginx.access" } } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/datafeed_source_ip_url_count_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/datafeed_source_ip_url_count_ecs.json new file mode 100644 index 0000000000000..bccf1bd8de6d5 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/datafeed_source_ip_url_count_ecs.json @@ -0,0 +1,13 @@ +{ + "job_id": "JOB_ID", + "indexes": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { "term": { "event.dataset": "nginx.access" } } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/datafeed_status_code_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/datafeed_status_code_rate_ecs.json new file mode 100644 index 0000000000000..bccf1bd8de6d5 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/datafeed_status_code_rate_ecs.json @@ -0,0 +1,13 @@ +{ + "job_id": "JOB_ID", + "indexes": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { "term": { "event.dataset": "nginx.access" } } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/datafeed_visitor_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/datafeed_visitor_rate_ecs.json new file mode 100644 index 0000000000000..297bd16db4edf --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/datafeed_visitor_rate_ecs.json @@ -0,0 +1,39 @@ +{ + "job_id": "JOB_ID", + "indexes": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { "term": { "event.dataset": "nginx.access" } } + ] + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "interval": 900000, + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + }, + "dc_source_address": { + "cardinality": { + "field": "source.address" + } + } + } + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/low_request_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/low_request_rate_ecs.json new file mode 100644 index 0000000000000..22631813346dc --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/low_request_rate_ecs.json @@ -0,0 +1,34 @@ +{ + "groups": ["nginx"], + "description": "HTTP Access Logs: Detect low request rate (ECS)", + "analysis_config" : { + "bucket_span": "15m", + "summary_count_field_name": "doc_count", + "detectors": [ + { + "detector_description": "nginx_access_low_request_rate", + "function": "low_count" + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-nginx-access", + "custom_urls": [ + { + "url_name": "Raw data", + "url_value": "kibana#/discover/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027,type:phrase),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:lucene,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/source_ip_request_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/source_ip_request_rate_ecs.json new file mode 100644 index 0000000000000..7d91ec8a81cd2 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/source_ip_request_rate_ecs.json @@ -0,0 +1,34 @@ +{ + "groups": ["nginx"], + "description": "HTTP Access Logs: Detect unusual source ips - high request rates (ECS)", + "analysis_config" : { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "nginx_access_source_ip_high_count", + "function": "high_count", + "over_field_name": "source.address" + } + ], + "influencers": [ + "source.address" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-nginx-access", + "custom_urls": [ + { + "url_name": "Investigate source IP", + "url_value": "kibana#/dashboard/ml_http_access_explorer_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027,type:phrase),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),query:(language:lucene,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "kibana#/discover/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027,type:phrase),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:lucene,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/source_ip_url_count_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/source_ip_url_count_ecs.json new file mode 100644 index 0000000000000..dc965e7061b95 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/source_ip_url_count_ecs.json @@ -0,0 +1,35 @@ +{ + "groups": ["nginx"], + "description": "HTTP Access Logs: Detect unusual source ips - high distinct count of urls (ECS)", + "analysis_config" : { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "nginx_access_source_ip_high_dc_url", + "function": "high_distinct_count", + "field_name": "url.original", + "over_field_name": "source.address" + } + ], + "influencers": [ + "source.address" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-nginx-access", + "custom_urls": [ + { + "url_name": "Investigate source IP", + "url_value": "kibana#/dashboard/ml_http_access_explorer_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027,type:phrase),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),query:(language:lucene,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "kibana#/discover/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027,type:phrase),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:lucene,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/status_code_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/status_code_rate_ecs.json new file mode 100644 index 0000000000000..14cba15dd14f4 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/status_code_rate_ecs.json @@ -0,0 +1,41 @@ +{ + "groups": ["nginx"], + "description": "HTTP Access Logs: Detect unusual status_code rates (ECS)", + "analysis_config" : { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "nginx_access_status_code_rate", + "function": "count", + "partition_field_name": "http.response.status_code" + } + ], + "influencers": [ + "http.response.status_code", + "source.address" + ] + }, + "analysis_limits": { + "model_memory_limit": "100mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-nginx-access", + "custom_urls": [ + { + "url_name": "Investigate status code", + "url_value": "kibana#/dashboard/ml_http_access_explorer_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027,type:phrase),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,params:(query:\u0027$http.response.status_code$\u0027,type:phrase),type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),query:(language:lucene,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "kibana#/discover/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027,type:phrase),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:lucene,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/visitor_rate_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/visitor_rate_ecs.json new file mode 100644 index 0000000000000..e0ba814facb8d --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/nginx_ecs/ml/visitor_rate_ecs.json @@ -0,0 +1,34 @@ +{ + "groups": ["nginx"], + "description": "HTTP Access Logs: Detect unusual visitor rate (ECS)", + "analysis_config" : { + "bucket_span": "15m", + "summary_count_field_name": "dc_source_address", + "detectors": [ + { + "detector_description": "nginx_access_visitor_rate", + "function": "non_zero_count" + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-nginx-access", + "custom_urls": [ + { + "url_name": "Raw data", + "url_value": "kibana#/discover/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027,type:phrase),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:lucene,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } +}