Add flags to consul connect envoy for metrics merging.

Allows setting -prometheus-backend-port to configure the cluster
envoy_prometheus_bind_addr points to.

Allows setting -prometheus-scrape-path to configure which path
envoy_prometheus_bind_addr exposes metrics on.

-prometheus-backend-port is used by the consul-k8s metrics merging feature, to
configure envoy_prometheus_bind_addr to point to the merged metrics
endpoint that combines Envoy and service metrics so that one set of
annotations on a Pod can scrape metrics from the service and it's Envoy
sidecar.

-prometheus-scrape-path is used to allow configurability of the path
where prometheus metrics are exposed on envoy_prometheus_bind_addr.

.changelog/9768.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+connect: adds new flags `prometheus-backend-port` and `prometheus-scrape-port` to `consul connect envoy` to support envoy_prometheus_bind_addr pointing to the merged metrics port when using Consul Connect on K8s.
+```

command/connect/envoy/bootstrap_config.go

@@ -213,19 +213,19 @@ func (c *BootstrapConfig) ConfigureArgs(args *BootstrapTplArgs, omitDeprecatedTa
 	}
 	// Setup prometheus if needed. This MUST happen after the Static*JSON is set above
 	if c.PrometheusBindAddr != "" {
-		if err := c.generateListenerConfig(args, c.PrometheusBindAddr, "envoy_prometheus_metrics", "path", "/metrics", "/stats/prometheus"); err != nil {
+		if err := c.generateListenerConfig(args, c.PrometheusBindAddr, "envoy_prometheus_metrics", "path", args.PrometheusScrapePath, "/stats/prometheus", args.PrometheusBackendPort); err != nil {
 			return err
 		}
 	}
 	// Setup /stats proxy listener if needed. This MUST happen after the Static*JSON is set above
 	if c.StatsBindAddr != "" {
-		if err := c.generateListenerConfig(args, c.StatsBindAddr, "envoy_metrics", "prefix", "/stats", "/stats"); err != nil {
+		if err := c.generateListenerConfig(args, c.StatsBindAddr, "envoy_metrics", "prefix", "/stats", "/stats", ""); err != nil {
 			return err
 		}
 	}
 	// Setup /ready proxy listener if needed. This MUST happen after the Static*JSON is set above
 	if c.ReadyBindAddr != "" {
-		if err := c.generateListenerConfig(args, c.ReadyBindAddr, "envoy_ready", "path", "/ready", "/ready"); err != nil {
+		if err := c.generateListenerConfig(args, c.ReadyBindAddr, "envoy_ready", "path", "/ready", "/ready", ""); err != nil {
 			return err
 		}
 	}
@@ -549,20 +549,36 @@ func generateStatsTags(args *BootstrapTplArgs, initialTags []string, omitDepreca
 	return tagJSONs, nil
 }
 
-func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAddr, name, matchType, matchValue, prefixRewrite string) error {
+func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAddr, name, matchType, matchValue, prefixRewrite, prometheusBackendPort string) error {
 	host, port, err := net.SplitHostPort(bindAddr)
 	if err != nil {
 		return fmt.Errorf("invalid %s bind address: %s", name, err)
 	}
 
+	// If prometheusBackendPort is set (not empty string), create
+	// "prometheus_backend" cluster with the prometheusBackendPort that the
+	// listener will point to, rather than the "self_admin" cluster. This is for
+	// the merged metrics feature in consul-k8s, so the
+	// envoy_prometheus_bind_addr listener will point to the merged Envoy and
+	// service metrics endpoint rather than the Envoy admin endpoint for
+	// metrics. This cluster will only be created once since it's only created
+	// when prometheusBackendPort is set, and prometheusBackendPort is only set
+	// when calling this function if c.PrometheusBindAddr is set.
+	clusterPort := args.AdminBindPort
+	clusterName := selfAdminName
+	if prometheusBackendPort != "" {
+		clusterPort = prometheusBackendPort
+		clusterName = "prometheus_backend"
+	}
+
 	clusterJSON := `{
-		"name": "` + selfAdminName + `",
+		"name": "` + clusterName + `",
 		"ignore_health_on_host_removal": false,
 		"connect_timeout": "5s",
 		"type": "STATIC",
 		"http_protocol_options": {},
 		"loadAssignment": {
-			"clusterName": "` + selfAdminName + `",
+			"clusterName": "` + clusterName + `",
 			"endpoints": [
 				{
 					"lbEndpoints": [
@@ -571,7 +587,7 @@ func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAdd
 								"address": {
 									"socket_address": {
 										"address": "127.0.0.1",
-										"port_value": ` + args.AdminBindPort + `
+										"port_value": ` + clusterPort + `
 									}
 								}
 							}
@@ -612,7 +628,7 @@ func (c *BootstrapConfig) generateListenerConfig(args *BootstrapTplArgs, bindAdd
 													"` + matchType + `": "` + matchValue + `"
 												},
 												"route": {
-													"cluster": "self_admin",
+													"cluster": "` + clusterName + `",
 													"prefix_rewrite": "` + prefixRewrite + `"
 												}
 											},