Use import/export API for loading dashboards and index pattern (#4413)

If the Elasticsearch version is older than 6.x, then the dashboards are loaded directly into Elasticsearch. The old dashboards are moved to the 5.x directory.

.travis.yml

@@ -85,12 +85,6 @@ jobs:
       go: $GO_VERSION
       stage: test
 
-    # Dashboards
-    - os: linux
-      env: TARGETS="-C dev-tools/cmd/import_dashboards"
-      go: $GO_VERSION
-      stage: test
-
     # Generators
     - os: linux
       env: TARGETS="-C generator/metricbeat test"

auditbeat/auditbeat.reference.yml

@@ -619,7 +619,7 @@ output.elasticsearch:
 #============================== Dashboards =====================================
 # These settings control loading the sample dashboards to the Kibana index. Loading
 # the dashboards is disabled by default and can be enabled either by setting the
-# options here, or by using the `-setup` CLI flag.
+# options here, or by using the `-setup` CLI flag or the `setup` command.
 #setup.dashboards.enabled: false
 
 # The URL from where to download the dashboards archive. By default this URL
@@ -675,6 +675,25 @@ output.elasticsearch:
 # Overwrite existing template
 #setup.template.overwrite: false
 
+#============================== Kibana =====================================
+
+# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. 
+# This requires a Kibana endpoint configuration.
+setup.kibana:
+
+  # Kibana Host
+  # Scheme and port can be left out and will be set to the default (http and 5601)
+  # In case you specify and additional path, the scheme is required: http://localhost:5601/path
+  # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
+  #host: "localhost:5601"
+
+  # Optional protocol and basic auth crendentials.
+  #protocol: "https"
+  #username: "elastic"
+  #password: "changeme"
+
+  # Optional HTTP Path
+  #path: ""
 
 #================================ HTTP Endpoint ======================================
 # Each beat can expose internal data points through a http endpoint. For security

auditbeat/auditbeat.yml

@@ -38,6 +38,31 @@ auditbeat.modules:
 #fields:
 #  env: staging
 
+
+#============================== Dashboards =====================================
+# These settings control loading the sample dashboards to the Kibana index. Loading
+# the dashboards is disabled by default and can be enabled either by setting the
+# options here, or by using the `-setup` CLI flag or the `setup` command.
+#setup.dashboards.enabled: false
+
+# The URL from where to download the dashboards archive. By default this URL
+# has a value which is computed based on the Beat name and version. For released
+# versions, this URL points to the dashboard archive on the artifacts.elastic.co
+# website.
+#setup.dashboards.url:
+
+#============================== Kibana =====================================
+
+# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. 
+# This requires a Kibana endpoint configuration.
+setup.kibana:
+
+  # Kibana Host
+  # Scheme and port can be left out and will be set to the default (http and 5601)
+  # In case you specify and additional path, the scheme is required: http://localhost:5601/path
+  # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
+  #host: "localhost:5601"
+
 #================================ Outputs =====================================
 
 # Configure what outputs to use when sending the data collected by the beat.

auditbeat/module/audit/_meta/kibana/default/dashboard/auditbeat-audit.json

@@ -0,0 +1,132 @@
+{
+  "objects": [
+    {
+      "attributes": {
+        "description": "",
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\n  \"filter\": []\n}"
+        },
+        "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52",
+        "title": "HTTP ping times",
+        "uiStateJSON": "{}",
+        "version": 1,
+        "visState": "{\n  \"title\": \"HTTP ping times\",\n  \"type\": \"area\",\n  \"params\": {\n    \"addTooltip\": true,\n    \"addLegend\": true,\n    \"legendPosition\": \"right\",\n    \"scale\": \"linear\",\n    \"interpolate\": \"linear\",\n    \"mode\": \"stacked\",\n    \"times\": [],\n    \"addTimeMarker\": false,\n    \"defaultYExtents\": false,\n    \"setYExtents\": false\n  },\n  \"aggs\": [\n    {\n      \"id\": \"1\",\n      \"enabled\": true,\n      \"type\": \"max\",\n      \"schema\": \"metric\",\n      \"params\": {\n        \"field\": \"resolve.rtt.us\",\n        \"customLabel\": \"\"\n      }\n    },\n    {\n      \"id\": \"3\",\n      \"enabled\": true,\n      \"type\": \"max\",\n      \"schema\": \"metric\",\n      \"params\": {\n        \"field\": \"tcp.rtt.connect.us\"\n      }\n    },\n    {\n      \"id\": \"5\",\n      \"enabled\": true,\n      \"type\": \"max\",\n      \"schema\": \"metric\",\n      \"params\": {\n        \"field\": \"tls.rtt.handshake.us\"\n      }\n    },\n    {\n      \"id\": \"4\",\n      \"enabled\": true,\n      \"type\": \"max\",\n      \"schema\": \"metric\",\n      \"params\": {\n        \"field\": \"http.rtt.response_header.us\"\n      }\n    },\n    {\n      \"id\": \"2\",\n      \"enabled\": true,\n      \"type\": \"date_histogram\",\n      \"schema\": \"segment\",\n      \"params\": {\n        \"field\": \"@timestamp\",\n        \"interval\": \"auto\",\n        \"customInterval\": \"2h\",\n        \"min_doc_count\": 1,\n        \"extended_bounds\": {}\n      }\n    }\n  ],\n  \"listeners\": {}\n}"
+      },
+      "id": "c65ef340-eb19-11e6-be20-559646f8b9ba",
+      "type": "visualization",
+      "version": 1
+    },
+    {
+      "attributes": {
+        "description": "",
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\n  \"filter\": []\n}"
+        },
+        "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52",
+        "title": "HTTP monitors status",
+        "uiStateJSON": "{\n  \"vis\": {\n    \"colors\": {\n      \"200\": \"#B7DBAB\",\n      \"monitor.status: up\": \"#629E51\",\n      \"monitor.status: down\": \"#E24D42\"\n    },\n    \"legendOpen\": true\n  }\n}",
+        "version": 1,
+        "visState": "{\n  \"title\": \"HTTP monitors status\",\n  \"type\": \"pie\",\n  \"params\": {\n    \"addTooltip\": true,\n    \"addLegend\": true,\n    \"legendPosition\": \"bottom\",\n    \"isDonut\": false\n  },\n  \"aggs\": [\n    {\n      \"id\": \"1\",\n      \"enabled\": true,\n      \"type\": \"cardinality\",\n      \"schema\": \"metric\",\n      \"params\": {\n        \"field\": \"monitor.id\"\n      }\n    },\n    {\n      \"id\": \"3\",\n      \"enabled\": true,\n      \"type\": \"filters\",\n      \"schema\": \"segment\",\n      \"params\": {\n        \"filters\": [\n          {\n            \"input\": {\n              \"query\": {\n                \"query_string\": {\n                  \"query\": \"monitor.status: up\",\n                  \"analyze_wildcard\": true\n                }\n              }\n            },\n            \"label\": \"\"\n          },\n          {\n            \"input\": {\n              \"query\": {\n                \"query_string\": {\n                  \"query\": \"monitor.status: down\",\n                  \"analyze_wildcard\": true\n                }\n              }\n            }\n          }\n        ]\n      }\n    },\n    {\n      \"id\": \"2\",\n      \"enabled\": true,\n      \"type\": \"terms\",\n      \"schema\": \"segment\",\n      \"params\": {\n        \"field\": \"http.response.status\",\n        \"size\": 5,\n        \"order\": \"desc\",\n        \"orderBy\": \"1\"\n      }\n    }\n  ],\n  \"listeners\": {}\n}"
+      },
+      "id": "920e8140-eb1a-11e6-be20-559646f8b9ba",
+      "type": "visualization",
+      "version": 1
+    },
+    {
+      "attributes": {
+        "description": "",
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\n  \"filter\": []\n}"
+        },
+        "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52",
+        "title": "HTTP monitors",
+        "uiStateJSON": "{\n  \"vis\": {\n    \"params\": {\n      \"sort\": {\n        \"columnIndex\": null,\n        \"direction\": null\n      }\n    }\n  }\n}",
+        "version": 1,
+        "visState": "{\n  \"title\": \"HTTP monitors\",\n  \"type\": \"table\",\n  \"params\": {\n    \"perPage\": 10,\n    \"showPartialRows\": false,\n    \"showMeticsAtAllLevels\": false,\n    \"sort\": {\n      \"columnIndex\": null,\n      \"direction\": null\n    },\n    \"showTotal\": false,\n    \"totalFunc\": \"sum\"\n  },\n  \"aggs\": [\n    {\n      \"id\": \"1\",\n      \"enabled\": true,\n      \"type\": \"max\",\n      \"schema\": \"metric\",\n      \"params\": {\n        \"field\": \"monitor.duration.us\"\n      }\n    },\n    {\n      \"id\": \"2\",\n      \"enabled\": true,\n      \"type\": \"terms\",\n      \"schema\": \"bucket\",\n      \"params\": {\n        \"field\": \"monitor.id\",\n        \"size\": 5,\n        \"order\": \"desc\",\n        \"orderBy\": \"1\"\n      }\n    },\n    {\n      \"id\": \"5\",\n      \"enabled\": true,\n      \"type\": \"max\",\n      \"schema\": \"metric\",\n      \"params\": {\n        \"field\": \"resolve.rtt.us\"\n      }\n    },\n    {\n      \"id\": \"6\",\n      \"enabled\": true,\n      \"type\": \"max\",\n      \"schema\": \"metric\",\n      \"params\": {\n        \"field\": \"tcp.rtt.connect.us\"\n      }\n    },\n    {\n      \"id\": \"7\",\n      \"enabled\": true,\n      \"type\": \"max\",\n      \"schema\": \"metric\",\n      \"params\": {\n        \"field\": \"tls.rtt.handshake.us\"\n      }\n    },\n    {\n      \"id\": \"8\",\n      \"enabled\": true,\n      \"type\": \"max\",\n      \"schema\": \"metric\",\n      \"params\": {\n        \"field\": \"http.rtt.response_header.us\"\n      }\n    }\n  ],\n  \"listeners\": {}\n}"
+      },
+      "id": "1738dbc0-eb1d-11e6-be20-559646f8b9ba",
+      "type": "visualization",
+      "version": 1
+    },
+    {
+      "attributes": {
+        "description": "",
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\n  \"filter\": []\n}"
+        },
+        "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52",
+        "title": "HTTP up status",
+        "uiStateJSON": "{\n  \"vis\": {\n    \"colors\": {\n      \"monitor.status: up\": \"#629E51\",\n      \"monitor.status: down\": \"#E24D42\"\n    }\n  }\n}",
+        "version": 1,
+        "visState": "{\n  \"title\": \"HTTP up status\",\n  \"type\": \"area\",\n  \"params\": {\n    \"addTooltip\": true,\n    \"addLegend\": true,\n    \"legendPosition\": \"right\",\n    \"scale\": \"linear\",\n    \"interpolate\": \"linear\",\n    \"mode\": \"percentage\",\n    \"times\": [],\n    \"addTimeMarker\": false,\n    \"defaultYExtents\": false,\n    \"setYExtents\": true,\n    \"yAxis\": {\n      \"max\": 100,\n      \"min\": 0\n    }\n  },\n  \"aggs\": [\n    {\n      \"id\": \"1\",\n      \"enabled\": true,\n      \"type\": \"count\",\n      \"schema\": \"metric\",\n      \"params\": {}\n    },\n    {\n      \"id\": \"2\",\n      \"enabled\": true,\n      \"type\": \"date_histogram\",\n      \"schema\": \"segment\",\n      \"params\": {\n        \"field\": \"@timestamp\",\n        \"interval\": \"auto\",\n        \"customInterval\": \"2h\",\n        \"min_doc_count\": 1,\n        \"extended_bounds\": {}\n      }\n    },\n    {\n      \"id\": \"3\",\n      \"enabled\": true,\n      \"type\": \"filters\",\n      \"schema\": \"group\",\n      \"params\": {\n        \"filters\": [\n          {\n            \"input\": {\n              \"query\": {\n                \"query_string\": {\n                  \"query\": \"monitor.status: down\",\n                  \"analyze_wildcard\": true\n                }\n              }\n            },\n            \"label\": \"\"\n          },\n          {\n            \"input\": {\n              \"query\": {\n                \"query_string\": {\n                  \"query\": \"monitor.status: up\",\n                  \"analyze_wildcard\": true\n                }\n              }\n            }\n          }\n        ]\n      }\n    }\n  ],\n  \"listeners\": {}\n}"
+      },
+      "id": "091c3a90-eb1e-11e6-be20-559646f8b9ba",
+      "type": "visualization",
+      "version": 1
+    },
+    {
+      "attributes": {
+        "description": "",
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\n  \"filter\": []\n}"
+        },
+        "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52",
+        "title": "HTTP duration heatmap",
+        "uiStateJSON": "{\n  \"vis\": {\n    \"defaultColors\": {\n      \"0 - 2\": \"rgb(247,251,255)\",\n      \"2 - 3\": \"rgb(227,238,249)\",\n      \"3 - 4\": \"rgb(208,225,242)\",\n      \"4 - 5\": \"rgb(182,212,233)\",\n      \"5 - 6\": \"rgb(148,196,223)\",\n      \"6 - 8\": \"rgb(107,174,214)\",\n      \"8 - 9\": \"rgb(74,152,201)\",\n      \"9 - 10\": \"rgb(46,126,188)\",\n      \"10 - 11\": \"rgb(23,100,171)\",\n      \"11 - 12\": \"rgb(8,74,145)\"\n    }\n  }\n}",
+        "version": 1,
+        "visState": "{\n  \"title\": \"HTTP duration heatmap\",\n  \"type\": \"heatmap\",\n  \"params\": {\n    \"addTooltip\": true,\n    \"addLegend\": true,\n    \"enableHover\": false,\n    \"legendPosition\": \"right\",\n    \"times\": [],\n    \"colorsNumber\": 10,\n    \"colorSchema\": \"Blues\",\n    \"setColorRange\": false,\n    \"colorsRange\": [],\n    \"invertColors\": false,\n    \"percentageMode\": false,\n    \"valueAxes\": [\n      {\n        \"show\": false,\n        \"id\": \"ValueAxis-1\",\n        \"type\": \"value\",\n        \"scale\": {\n          \"type\": \"linear\",\n          \"defaultYExtents\": false\n        },\n        \"labels\": {\n          \"show\": false,\n          \"rotate\": 0,\n          \"color\": \"#555\"\n        }\n      }\n    ]\n  },\n  \"aggs\": [\n    {\n      \"id\": \"1\",\n      \"enabled\": true,\n      \"type\": \"count\",\n      \"schema\": \"metric\",\n      \"params\": {}\n    },\n    {\n      \"id\": \"2\",\n      \"enabled\": true,\n      \"type\": \"date_histogram\",\n      \"schema\": \"segment\",\n      \"params\": {\n        \"field\": \"@timestamp\",\n        \"interval\": \"auto\",\n        \"customInterval\": \"2h\",\n        \"min_doc_count\": 1,\n        \"extended_bounds\": {}\n      }\n    },\n    {\n      \"id\": \"3\",\n      \"enabled\": true,\n      \"type\": \"histogram\",\n      \"schema\": \"group\",\n      \"params\": {\n        \"field\": \"monitor.duration.us\",\n        \"interval\": 50000,\n        \"extended_bounds\": {}\n      }\n    }\n  ],\n  \"listeners\": {}\n}"
+      },
+      "id": "0f4c0560-eb20-11e6-9f11-159ff202874a",
+      "type": "visualization",
+      "version": 1
+    },
+    {
+      "attributes": {
+        "columns": [
+          "monitor.id",
+          "http.url",
+          "monitor.status",
+          "http.response.status",
+          "monitor.duration.us",
+          "tcp.rtt.connect.us",
+          "tls.rtt.handshake.us",
+          "resolve.rtt.us",
+          "http.rtt.content.us"
+        ],
+        "description": "",
+        "hits": 0,
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\n  \"index\": \"heartbeat-*\",\n  \"highlightAll\": true,\n  \"query\": {\n    \"query_string\": {\n      \"analyze_wildcard\": true,\n      \"query\": \"*\"\n    }\n  },\n  \"filter\": [\n    {\n      \"$state\": {\n        \"store\": \"appState\"\n      },\n      \"meta\": {\n        \"alias\": null,\n        \"disabled\": false,\n        \"index\": \"heartbeat-*\",\n        \"key\": \"monitor.name\",\n        \"negate\": false,\n        \"value\": \"http\"\n      },\n      \"query\": {\n        \"match\": {\n          \"monitor.name\": {\n            \"query\": \"http\",\n            \"type\": \"phrase\"\n          }\n        }\n      }\n    }\n  ]\n}"
+        },
+        "sort": [
+          "@timestamp",
+          "desc"
+        ],
+        "title": "Heartbeat HTTP pings",
+        "version": 1
+      },
+      "id": "02014c80-29d2-11e7-a68f-bfaa2341cc52",
+      "type": "search",
+      "version": 5
+    },
+    {
+      "attributes": {
+        "description": "",
+        "hits": 0,
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}"
+        },
+        "optionsJSON": "{\"darkTheme\":false}",
+        "panelsJSON": "[{\"col\":1,\"id\":\"c65ef340-eb19-11e6-be20-559646f8b9ba\",\"panelIndex\":1,\"row\":7,\"size_x\":12,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"920e8140-eb1a-11e6-be20-559646f8b9ba\",\"panelIndex\":2,\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"1738dbc0-eb1d-11e6-be20-559646f8b9ba\",\"panelIndex\":3,\"row\":1,\"size_x\":8,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"091c3a90-eb1e-11e6-be20-559646f8b9ba\",\"panelIndex\":4,\"row\":5,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"0f4c0560-eb20-11e6-9f11-159ff202874a\",\"panelIndex\":5,\"row\":11,\"size_x\":12,\"size_y\":5,\"type\":\"visualization\"}]",
+        "timeRestore": false,
+        "title": "Heartbeat HTTP monitoring",
+        "uiStateJSON": "{\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-5\":{\"vis\":{\"defaultColors\":{\"0 - 2\":\"rgb(247,251,255)\",\"2 - 3\":\"rgb(227,238,249)\",\"3 - 4\":\"rgb(208,225,242)\",\"4 - 5\":\"rgb(182,212,233)\",\"5 - 6\":\"rgb(148,196,223)\",\"6 - 8\":\"rgb(107,174,214)\",\"8 - 9\":\"rgb(74,152,201)\",\"9 - 10\":\"rgb(46,126,188)\",\"10 - 11\":\"rgb(23,100,171)\",\"11 - 12\":\"rgb(8,74,145)\"}}}}",
+        "version": 1
+      },
+      "id": "f3e771c0-eb19-11e6-be20-559646f8b9ba",
+      "type": "dashboard",
+      "version": 1
+    }
+  ],
+  "version": "6.0.0-alpha3-SNAPSHOT"
+}

dev-tools/cmd/dashboards/.gitignore

@@ -0,0 +1,2 @@
+# Executable
+export_dashboards

dev-tools/cmd/dashboards/Makefile

@@ -0,0 +1,5 @@
+
+BEAT?=
+
+export_dashboards:
+	go build -ldflags "-X main.beat=${BEAT}" -o export_dashboards

dev-tools/cmd/dashboards/export_dashboards.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/elastic/beats/libbeat/common"
+)
+
+var exportAPI = "/api/kibana/dashboards/export"
+
+func makeURL(url, path string, params url.Values) string {
+
+	if len(params) == 0 {
+		return url + path
+	}
+
+	return strings.Join([]string{url, path, "?", params.Encode()}, "")
+}
+
+func ExtractIndexPattern(body []byte) ([]byte, error) {
+
+	var contents common.MapStr
+
+	err := json.Unmarshal(body, &contents)
+	if err != nil {
+		return nil, err
+	}
+
+	objects, ok := contents["objects"].([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("Key objects not found or wrong type")
+	}
+
+	var result []interface{}
+	for _, obj := range objects {
+		_type, ok := obj.(map[string]interface{})["type"].(string)
+		if !ok {
+			return nil, fmt.Errorf("type key not found or not string")
+		}
+		if _type != "index-pattern" {
+			result = append(result, obj)
+		}
+	}
+	contents["objects"] = result
+
+	newBody, err := json.MarshalIndent(contents, "", "  ")
+	if err != nil {
+		return nil, fmt.Errorf("Error mashaling: %v", err)
+	}
+
+	return newBody, nil
+
+}
+
+func Export(client *http.Client, conn string, dashboards []string, out string) error {
+
+	params := url.Values{}
+
+	for _, dashboard := range dashboards {
+		params.Add("dashboard", dashboard)
+	}
+
+	fullURL := makeURL(conn, exportAPI, params)
+	fmt.Printf("Calling HTTP GET %v\n", fullURL)
+
+	req, err := http.NewRequest("GET", fullURL, nil)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("GET HTTP request fails with: %v", err)
+	}
+
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("fail to read response %s", err)
+	}
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("HTTP GET %s fails with %s, %s", fullURL, resp.Status, body)
+	}
+
+	body, err = ExtractIndexPattern(body)
+	if err != nil {
+		return fmt.Errorf("fail to extract the index pattern: %v", err)
+	}
+
+	err = ioutil.WriteFile(out, body, 0666)
+
+	fmt.Printf("The dashboards were exported under the %s file\n", out)
+	return err
+}
+
+func main() {
+
+	kibanaURL := flag.String("kibana", "http://localhost:5601", "Kibana URL")
+	fileOutput := flag.String("output", "output.json", "Output file")
+
+	flag.Parse()
+
+	args := flag.Args()
+
+	transCfg := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // ignore expired SSL certificates
+	}
+
+	client := &http.Client{Transport: transCfg}
+
+	err := Export(client, *kibanaURL, args, *fileOutput)
+	if err != nil {
+		fmt.Printf("ERROR: fail to export the dashboards: %s\n", err)
+	}
+}