Feature/istio metrics (#4253)

Metrics, dashboards and docs

packages/istio/_dev/build/docs/README.md

@@ -1,6 +1,6 @@
 # Istio Integration
 
-This integration ingest access logs created by the [Istio](https://istio.io/) service mesh.
+This integration ingest access logs and metrics created by the [Istio](https://istio.io/) service mesh.
 
 ## Compatibility
 
@@ -15,3 +15,61 @@ The `access_logs` data stream collects Istio access logs.
 {{event "access_logs"}}
 
 {{fields "access_logs"}}
+
+
+## Metrics
+
+### Istiod Metrics
+
+The `istiod_metrics` data stream collects Istiod metrics.
+
+{{event "istiod_metrics"}}
+
+{{fields "istiod_metrics"}}
+
+### Proxy Metrics
+
+The `proxy_metrics` data stream collects Istio proxy metrics.
+
+{{event "proxy_metrics"}}
+
+{{fields "proxy_metrics"}}
+
+
+## How to setup and test Istio locally
+
+1. Setup a Kubernetes cluster. Since the Istio sample app requires lots of RAM (> 10GB) it's preferable to use a managed Kubernetes cluster (any cloud provider will do).
+2. Setup a EK cluster on Elastic Cloud. For the same reason that Istio sample app requires a lot of RAM, it's unfeasible to run the Elastic cluster on your laptop via elastic-package. As an alternative ECK might be used as well.
+3. Start elastic agents on Kubernetes cluster. The easiest way to achieve this is by using Fleet Server. You can find instructions [here](https://www.elastic.co/guide/en/fleet/master/running-on-kubernetes-managed-by-fleet.html)
+4. Download Istio cli following the [instructions](https://istio.io/latest/docs/setup/getting-started/#download).
+5. Install Istio via [instructions](https://istio.io/latest/docs/setup/getting-started/#install). The namespace `default` is used with this basic installation. This is the same namespace where we are going to run the Istio sample app.
+6. Deploy the sample application via [instructions](https://istio.io/latest/docs/setup/getting-started/#bookinfo)
+7. Open the application to external traffic and determine the ingress IP and ports. This step is slightly different depending where Kubernetes is running. More info at [here](https://istio.io/latest/docs/setup/getting-started/#ip) and [here](https://istio.io/latest/docs/setup/getting-started/#determining-the-ingress-ip-and-ports). The following commands should be enough to get this working.
+
+```bash
+kubectl apply -f samples/bookinfo/networking/bookinfo-gateway.yaml
+istioctl analyze
+
+# since we are using a cloud environment with an external load balancer
+export INGRESS_HOST=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+export INGRESS_PORT=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="http2")].port}')
+export SECURE_INGRESS_PORT=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="https")].port}')
+export GATEWAY_URL=$INGRESS_HOST:$INGRESS_PORT
+```
+
+From the same terminal run the following command to open a browser to that link. This should verify that the sample application is reachable.
+
+```bash
+open "http://$GATEWAY_URL/productpage"
+```
+
+8. Generate some traffic to the sample application
+
+
+```bash
+for i in $(seq 1 100); do curl -s -o /dev/null "http://$GATEWAY_URL/productpage"; done
+```
+
+9. (Optional) You can visualize the graph of microservices in the sample app via [instructions](https://istio.io/latest/docs/setup/getting-started/#dashboard).
+9.  Add the Istio integration from the registry. 
+10. View logs and/or metrics from the Istio integration using the Discovery tab and selecting the right Data view

packages/istio/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,21 @@
+version: '2.3'
+services:
+  istio_is_ready:
+    image: tianon/true
+    depends_on:
+      istio:
+        condition: service_healthy
+  istio:
+    image: nginx:alpine
+    ports:
+      - 8080
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
+      - ./istiod.txt:/www/data/metrics/istiod.txt
+      - ./proxy.txt:/www/data/metrics/proxy.txt
+    healthcheck:
+      interval: 1s
+      retries: 120
+      timeout: 120s
+      test: |-
+        curl -f -s http://localhost:8080/metrics/ -o /dev/null

packages/istio/_dev/deploy/docker/nginx.conf

@@ -0,0 +1,17 @@
+worker_processes 1;
+
+events { worker_connections 1024; }
+
+http {
+    sendfile on;
+
+    server {
+        listen 8080;
+
+        root /www/data;
+
+        location /metrics {
+            autoindex on;
+        }
+    }
+}

packages/istio/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.2.0"
+  changes:
+    - description: Metrics for Istiod and Proxy sidecar container
+      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/integrations/pull/4253
 - version: "0.1.0"
   changes:
     - description: Initial release

packages/istio/data_stream/access_logs/manifest.yml

@@ -3,7 +3,7 @@ type: logs
 release: experimental
 streams:
   - input: filestream
-    title: Collect Istio access logs
+    title: Istio access logs
     description: Collect Istio access logs either in text or json format
     vars:
       - name: paths

packages/istio/data_stream/access_logs/sample_event.json

@@ -1,5 +1,10 @@
 {
     "@timestamp": "2022-07-20T09:52:24.955Z",
+    "data_stream": {
+        "namespace": "default",
+        "type": "logs",
+        "dataset": "istio.access_logs"
+    },
     "destination": {
         "address": "10.68.2.10:9080",
         "ip": "10.68.2.10",

packages/istio/data_stream/istiod_metrics/_dev/test/system/test-default-config.yml

@@ -0,0 +1,8 @@
+service: istio
+data_stream:
+  vars:
+    period: 1s
+    hosts:
+      - "http://{{Hostname}}:8080"
+    metrics_path: "/metrics/istiod.txt"
+    condition: "true"

packages/istio/data_stream/istiod_metrics/agent/stream/stream.yml.hbs

@@ -0,0 +1,20 @@
+metricsets: ["collector"]
+period: {{period}}
+hosts:
+{{#each hosts}}
+  - {{this}}
+{{/each}}
+condition: {{ condition }}
+{{#if metrics_path}}
+metrics_path: {{metrics_path}}
+{{/if}}
+metrics_filters.exclude:
+{{#each metrics_filters.exclude}}
+    - {{this}}
+{{/each}}
+metrics_filters.include:
+{{#each metrics_filters.include}}
+    - {{this}}
+{{/each}}
+use_types: true
+rate_counters: true

packages/istio/data_stream/istiod_metrics/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,34 @@
+---
+description: Pipeline for renaming object
+processors:
+  - remove:
+      field:
+        - metricset.name
+        - service.address
+        - service.type
+      ignore_missing: true
+  - set:
+      field: ecs.version
+      value: '8.4.0'
+  - set:
+      field: event.module
+      value: istio
+  - set:
+      field: event.kind
+      value: metric
+  - rename:
+      field: prometheus.labels
+      target_field: istio.istiod.labels
+      ignore_missing: true
+  - set:
+      field: istio.istiod.labels.job
+      value: istio
+      override: true
+  - rename:
+      field: prometheus
+      target_field: istio.istiod.metrics
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: '{{ _ingest.on_failure_message }}'

packages/istio/data_stream/istiod_metrics/fields/base-fields.yml

@@ -0,0 +1,18 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.
+- name: event.module
+  type: constant_keyword
+  description: Event module
+- name: event.dataset
+  type: constant_keyword
+  description: Event dataset

packages/istio/data_stream/istiod_metrics/fields/ecs.yml

@@ -0,0 +1,8 @@
+- name: ecs.version
+  external: ecs
+- name: error.message
+  external: ecs
+- name: event.ingested
+  external: ecs
+- name: event.kind
+  external: ecs

packages/istio/data_stream/istiod_metrics/fields/fields.yml

@@ -0,0 +1,35 @@
+- name: istio.istiod
+  type: group
+  fields:
+    - name: labels.*
+      type: object
+      object_type: keyword
+      description: |
+        Istiod metric labels
+- name: istio.istiod.metrics.*.value
+  type: object
+  object_type: double
+  object_type_mapping_type: "*"
+  description: >
+    Istiod gauge metric
+
+- name: istio.istiod.metrics.*.counter
+  type: object
+  object_type: double
+  object_type_mapping_type: "*"
+  description: >
+    Istiod counter metric
+
+- name: istio.istiod.metrics.*.rate
+  type: object
+  object_type: double
+  object_type_mapping_type: "*"
+  description: >
+    Istiod rated counter metric
+
+- name: istio.istiod.metrics.*.histogram
+  type: object
+  object_type: histogram
+  object_type_mapping_type: "*"
+  description: >-
+    Istiod histogram metric

packages/istio/data_stream/istiod_metrics/manifest.yml

@@ -0,0 +1,58 @@
+title: "Istiod Metrics"
+release: experimental
+type: metrics
+streams:
+  - input: prometheus/metrics
+    title: Istiod metrics
+    description: Collect Istiod metrics
+    vars:
+      - name: period
+        type: text
+        title: Period
+        multi: false
+        required: true
+        show_user: true
+        default: 10s
+      - name: hosts
+        type: text
+        title: Hosts
+        multi: true
+        required: true
+        show_user: true
+        default:
+          - ${kubernetes.pod.ip}:15014
+      - name: metrics_path
+        type: text
+        title: Metrics Path
+        multi: false
+        required: true
+        show_user: true
+        default:
+          - /metrics
+      - name: condition
+        title: Condition
+        description: Condition to filter when to apply this datastream
+        type: text
+        multi: false
+        required: true
+        show_user: true
+        default: ${kubernetes.labels.app} == 'istiod' and ${kubernetes.annotations.prometheus.io/scrape} == 'true'
+      - name: metrics_filters.exclude
+        type: text
+        title: Metrics Filters Exclude
+        multi: true
+        required: false
+        show_user: true
+        default:
+          - "^up$"
+      - name: metrics_filters.include
+        type: text
+        title: Metrics Filters Include
+        multi: true
+        required: false
+        show_user: true
+        default:
+          - "galley_*"
+          - "pilot_*"
+          - "citadel_*"
+          - "istio_*"

packages/istio/data_stream/istiod_metrics/sample_event.json

@@ -0,0 +1,55 @@
+{
+    "istio": {
+        "istiod": {
+            "metrics": {
+                "pilot_xds_config_size_bytes": {
+                    "histogram": {
+                        "counts": [
+                            0,
+                            0,
+                            0,
+                            0,
+                            0,
+                            0,
+                            0
+                        ],
+                        "values": [
+                            0.5,
+                            5000.5,
+                            505000,
+                            2500000,
+                            7000000,
+                            25000000,
+                            70000000
+                        ]
+                    }
+                }
+            },
+            "labels": {
+                "instance": "10.124.0.8:15014",
+                "type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
+                "job": "istio"
+            }
+        }
+    },
+    "@timestamp": "2022-09-23T09:30:56.055Z",
+    "ecs": {
+        "version": "8.4.0"
+    },
+    "data_stream": {
+        "namespace": "default",
+        "type": "metrics",
+        "dataset": "istio.istiod_metrics"
+    },
+    "metricset": {
+        "period": 10000
+    },
+    "event": {
+        "duration": 10806443,
+        "agent_id_status": "verified",
+        "kind": "metric",
+        "ingested": "2022-09-23T09:30:57Z",
+        "module": "istio",
+        "dataset": "istio.istiod_metrics"
+    }
+}

packages/istio/data_stream/proxy_metrics/_dev/test/system/test-default-config.yml

@@ -0,0 +1,8 @@
+service: istio
+data_stream:
+  vars:
+    period: 1s
+    hosts:
+      - "http://{{Hostname}}:8080"
+    metrics_path: "/metrics/proxy.txt"
+    condition: "true"