Add ssl support, username and password for connecting to Kibana (#4597)

CHANGELOG.asciidoc

@@ -17,6 +17,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha2...master[Check the HEAD d
 - Rename `kubernetes` processor to `add_kubernetes_metadata`. {pull}4473[4473]
 - Rename `*.full.yml` config files to `*.reference.yml`. {pull}4563[4563]
 - The `scripts/import_dashboards` is removed from packages. Use the `setup` command instead. {pull}4586[4586]
+- Change format of the saved kibana dashboards to have a single JSON file for each dashboard {pull}4413[4413]
 
 *Filebeat*
 

auditbeat/auditbeat.reference.yml

@@ -695,6 +695,40 @@ setup.kibana:
   # Optional HTTP Path
   #path: ""
 
+  # Use SSL settings for HTTPS. Default is true.
+  #ssl.enabled: true
+
+  # Configure SSL verification mode. If `none` is configured, all server hosts
+  # and certificates will be accepted. In this mode, SSL based connections are
+  # susceptible to man-in-the-middle attacks. Use only for testing. Default is
+  # `full`.
+  #ssl.verification_mode: full
+
+  # List of supported/valid TLS versions. By default all TLS versions 1.0 up to
+  # 1.2 are enabled.
+  #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
+
+  # SSL configuration. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
+  # Optional passphrase for decrypting the Certificate Key.
+  #ssl.key_passphrase: ''
+
+  # Configure cipher suites to be used for SSL connections
+  #ssl.cipher_suites: []
+
+  # Configure curve types for ECDHE based cipher suites
+  #ssl.curve_types: []
+
+
+
 #================================ HTTP Endpoint ======================================
 # Each beat can expose internal data points through a http endpoint. For security
 # reason the endpoint is disabled by default. This feature is currently in beta.

filebeat/filebeat.reference.yml

@@ -1077,6 +1077,40 @@ setup.kibana:
   # Optional HTTP Path
   #path: ""
 
+  # Use SSL settings for HTTPS. Default is true.
+  #ssl.enabled: true
+
+  # Configure SSL verification mode. If `none` is configured, all server hosts
+  # and certificates will be accepted. In this mode, SSL based connections are
+  # susceptible to man-in-the-middle attacks. Use only for testing. Default is
+  # `full`.
+  #ssl.verification_mode: full
+
+  # List of supported/valid TLS versions. By default all TLS versions 1.0 up to
+  # 1.2 are enabled.
+  #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
+
+  # SSL configuration. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
+  # Optional passphrase for decrypting the Certificate Key.
+  #ssl.key_passphrase: ''
+
+  # Configure cipher suites to be used for SSL connections
+  #ssl.cipher_suites: []
+
+  # Configure curve types for ECDHE based cipher suites
+  #ssl.curve_types: []
+
+
+
 #================================ HTTP Endpoint ======================================
 # Each beat can expose internal data points through a http endpoint. For security
 # reason the endpoint is disabled by default. This feature is currently in beta.

heartbeat/heartbeat.reference.yml

@@ -850,6 +850,40 @@ setup.kibana:
   # Optional HTTP Path
   #path: ""
 
+  # Use SSL settings for HTTPS. Default is true.
+  #ssl.enabled: true
+
+  # Configure SSL verification mode. If `none` is configured, all server hosts
+  # and certificates will be accepted. In this mode, SSL based connections are
+  # susceptible to man-in-the-middle attacks. Use only for testing. Default is
+  # `full`.
+  #ssl.verification_mode: full
+
+  # List of supported/valid TLS versions. By default all TLS versions 1.0 up to
+  # 1.2 are enabled.
+  #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
+
+  # SSL configuration. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
+  # Optional passphrase for decrypting the Certificate Key.
+  #ssl.key_passphrase: ''
+
+  # Configure cipher suites to be used for SSL connections
+  #ssl.cipher_suites: []
+
+  # Configure curve types for ECDHE based cipher suites
+  #ssl.curve_types: []
+
+
+
 #================================ HTTP Endpoint ======================================
 # Each beat can expose internal data points through a http endpoint. For security
 # reason the endpoint is disabled by default. This feature is currently in beta.

libbeat/_meta/config.reference.yml

@@ -636,6 +636,40 @@ setup.kibana:
   # Optional HTTP Path
   #path: ""
 
+  # Use SSL settings for HTTPS. Default is true.
+  #ssl.enabled: true
+
+  # Configure SSL verification mode. If `none` is configured, all server hosts
+  # and certificates will be accepted. In this mode, SSL based connections are
+  # susceptible to man-in-the-middle attacks. Use only for testing. Default is
+  # `full`.
+  #ssl.verification_mode: full
+
+  # List of supported/valid TLS versions. By default all TLS versions 1.0 up to
+  # 1.2 are enabled.
+  #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
+
+  # SSL configuration. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
+  # Optional passphrase for decrypting the Certificate Key.
+  #ssl.key_passphrase: ''
+
+  # Configure cipher suites to be used for SSL connections
+  #ssl.cipher_suites: []
+
+  # Configure curve types for ECDHE based cipher suites
+  #ssl.curve_types: []
+
+
+
 #================================ HTTP Endpoint ======================================
 # Each beat can expose internal data points through a http endpoint. For security
 # reason the endpoint is disabled by default. This feature is currently in beta.

libbeat/setup/kibana/client.go

@@ -1,7 +1,6 @@
 package kibana
 
 import (
-	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -12,11 +11,15 @@ import (
 
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/logp"
+	"github.com/elastic/beats/libbeat/outputs"
+	"github.com/elastic/beats/libbeat/outputs/transport"
 )
 
 type Connection struct {
-	URL     string
-	Headers map[string]string
+	URL      string
+	Username string
+	Password string
+	Headers  map[string]string
 
 	http    *http.Client
 	version string
@@ -48,15 +51,49 @@ func NewKibanaClient(cfg *common.Config) (*Client, error) {
 		return nil, fmt.Errorf("invalid Kibana host: %v", err)
 	}
 
-	logp.Debug("kibana", "Kibana url: %s", kibanaURL)
+	u, err := url.Parse(kibanaURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse the Kibana URL: %v", err)
+	}
+
+	username := config.Username
+	password := config.Password
+
+	if u.User != nil {
+		username = u.User.Username()
+		password, _ = u.User.Password()
+		u.User = nil
+
+		// Re-write URL without credentials.
+		kibanaURL = u.String()
+	}
+
+	logp.Info("Kibana url: %s", kibanaURL)
+
+	var dialer, tlsDialer transport.Dialer
+
+	tlsConfig, err := outputs.LoadTLSConfig(config.TLS)
+	if err != nil {
+		return nil, fmt.Errorf("fail to load the TLS config: %v", err)
+	}
+
+	dialer = transport.NetDialer(config.Timeout)
+	tlsDialer, err = transport.TLSDialer(dialer, tlsConfig, config.Timeout)
+	if err != nil {
+		return nil, err
+	}
 
 	client := &Client{
 		Connection: Connection{
-			URL: kibanaURL,
+			URL:      kibanaURL,
+			Username: username,
+			Password: password,
 			http: &http.Client{
 				Transport: &http.Transport{
-					TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // ignore expired SSL certificates
+					Dial:    dialer.Dial,
+					DialTLS: tlsDialer.Dial,
 				},
+				Timeout: config.Timeout,
 			},
 		},
 	}
@@ -76,14 +113,19 @@ func (conn *Connection) Request(method, extraPath string, params url.Values, bod
 	}
 
 	logp.Debug("kibana", "HTTP request URL: %s", reqURL)
-	logp.Debug("kibana", "Kibana version: %s", conn.version)
 
 	req, err := http.NewRequest(method, reqURL, body)
 	if err != nil {
 		return 0, nil, fmt.Errorf("fail to create the HTTP %s request: %v", method, err)
 	}
 
+	if conn.Username != "" || conn.Password != "" {
+		req.SetBasicAuth(conn.Username, conn.Password)
+	}
+
 	req.Header.Set("Content-Type", "application/json")
+	req.Header.Add("Accept", "application/json")
+
 	if method != "GET" {
 		req.Header.Set("kbn-version", conn.version)
 	}
@@ -95,8 +137,6 @@ func (conn *Connection) Request(method, extraPath string, params url.Values, bod
 
 	defer resp.Body.Close()
 
-	logp.Debug("kibana", "Response: %s", resp.Status)
-
 	var retError error
 	if resp.StatusCode >= 300 {
 		retError = fmt.Errorf("%v", resp.Status)
@@ -128,7 +168,7 @@ func (client *Client) SetVersion() error {
 	var kibanaVersion kibanaVersionResponse
 	err = json.Unmarshal(result, &kibanaVersion)
 	if err != nil {
-		return fmt.Errorf("fail to unmarshal the HTTP response from Kibana %s: %v", client.Connection.URL, err)
+		return fmt.Errorf("fail to unmarshal the response from GET %s/api/status: %v", client.Connection.URL, err)
 	}
 
 	client.version = kibanaVersion.Version.Number

libbeat/setup/kibana/config.go

@@ -1,15 +1,29 @@
 package kibana
 
+import (
+	"time"
+
+	"github.com/elastic/beats/libbeat/outputs"
+)
+
 type kibanaConfig struct {
-	Protocol string `config:"protocol"`
-	Host     string `config:"host"`
-	Path     string `config:"path"`
+	Protocol string             `config:"protocol"`
+	Host     string             `config:"host"`
+	Path     string             `config:"path"`
+	Username string             `config:"username"`
+	Password string             `config:"password"`
+	TLS      *outputs.TLSConfig `config:"ssl"`
+	Timeout  time.Duration      `config:"timeout"`
 }
 
 var (
 	defaultKibanaConfig = kibanaConfig{
 		Protocol: "http",
 		Host:     "",
 		Path:     "",
+		Username: "",
+		Password: "",
+		Timeout:  90 * time.Second,
+		TLS:      nil,
 	}
 )

metricbeat/metricbeat.reference.yml

@@ -1074,6 +1074,40 @@ setup.kibana:
   # Optional HTTP Path
   #path: ""
 
+  # Use SSL settings for HTTPS. Default is true.
+  #ssl.enabled: true
+
+  # Configure SSL verification mode. If `none` is configured, all server hosts
+  # and certificates will be accepted. In this mode, SSL based connections are
+  # susceptible to man-in-the-middle attacks. Use only for testing. Default is
+  # `full`.
+  #ssl.verification_mode: full
+
+  # List of supported/valid TLS versions. By default all TLS versions 1.0 up to
+  # 1.2 are enabled.
+  #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
+
+  # SSL configuration. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
+  # Optional passphrase for decrypting the Certificate Key.
+  #ssl.key_passphrase: ''
+
+  # Configure cipher suites to be used for SSL connections
+  #ssl.cipher_suites: []
+
+  # Configure curve types for ECDHE based cipher suites
+  #ssl.curve_types: []
+
+
+
 #================================ HTTP Endpoint ======================================
 # Each beat can expose internal data points through a http endpoint. For security
 # reason the endpoint is disabled by default. This feature is currently in beta.