From 3320d21d7a420e93cb40436022a3944476bbe089 Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 10 Aug 2020 15:01:47 +0300 Subject: [PATCH 1/6] Add k8s manifest leveraging leaderelection Signed-off-by: chrismark --- .../metricbeat-leaderelection-kubernetes.yaml | 288 ++++++++++++++++++ ...at-leaderelection-daemonset-configmap.yaml | 126 ++++++++ .../metricbeat-leaderelection-daemonset.yaml | 97 ++++++ 3 files changed, 511 insertions(+) create mode 100644 deploy/kubernetes/metricbeat-leaderelection-kubernetes.yaml create mode 100644 deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset-configmap.yaml create mode 100644 deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset.yaml diff --git a/deploy/kubernetes/metricbeat-leaderelection-kubernetes.yaml b/deploy/kubernetes/metricbeat-leaderelection-kubernetes.yaml new file mode 100644 index 00000000000..ac6da3d2a45 --- /dev/null +++ b/deploy/kubernetes/metricbeat-leaderelection-kubernetes.yaml @@ -0,0 +1,288 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-leaderelection-daemonset-config + namespace: kube-system + labels: + k8s-app: metricbeat +data: + metricbeat.yml: |- + metricbeat.config.modules: + # Mounted `metricbeat-daemonset-modules` configmap: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + # To enable hints based autodiscover uncomment this: + #metricbeat.autodiscover: + # providers: + # - type: kubernetes + # node: ${NODE_NAME} + # hints.enabled: true + # Unique Autodiscover provider that handles singleton instance in the whole cluster for some unique data sources, + # like kube-state-metrics + metricbeat.autodiscover: + providers: + - type: kubernetes + scope: cluster + node: ${NODE_NAME} + unique: true + # identifier: + templates: + - config: + - module: kubernetes + hosts: ["kube-state-metrics:8080"] + period: 10s + add_metadata: true + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + - state_cronjob + - state_resourcequota + - state_statefulset + # Uncomment this to get k8s events: + #- event + #- module: kubernetes + # metricsets: + # - apiserver + # hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + # period: 30s + + processors: + - add_cloud_metadata: + + cloud.id: ${ELASTIC_CLOUD_ID} + cloud.auth: ${ELASTIC_CLOUD_AUTH} + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-leaderelection-daemonset-modules + namespace: kube-system + labels: + k8s-app: metricbeat +data: + system.yml: |- + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + #- core + #- diskio + #- socket + processes: ['.*'] + process.include_top_n: + by_cpu: 5 # include top 5 processes by CPU + by_memory: 5 # include top 5 processes by memory + + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)' + kubernetes.yml: |- + - module: kubernetes + metricsets: + - node + - system + - pod + - container + - volume + period: 10s + host: ${NODE_NAME} + hosts: ["https://${NODE_NAME}:10250"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.verification_mode: "none" + # If there is a CA bundle that contains the issuer of the certificate used in the Kubelet API, + # remove ssl.verification_mode entry and use the CA, for instance: + #ssl.certificate_authorities: + #- /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + # Currently `proxy` metricset is not supported on Openshift, comment out section + - module: kubernetes + metricsets: + - proxy + period: 10s + host: ${NODE_NAME} + hosts: ["localhost:10249"] +--- +# Deploy a Metricbeat instance per node for node metrics retrieval +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: metricbeat-leaderelection + namespace: kube-system + labels: + k8s-app: metricbeat +spec: + selector: + matchLabels: + k8s-app: metricbeat + template: + metadata: + labels: + k8s-app: metricbeat + spec: + serviceAccountName: metricbeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: metricbeat + image: docker.elastic.co/beats/metricbeat:8.0.0 + args: [ + "-c", "/etc/metricbeat.yml", + "-e", + "-system.hostfs=/hostfs", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: ELASTIC_CLOUD_ID + value: + - name: ELASTIC_CLOUD_AUTH + value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/metricbeat.yml + readOnly: true + subPath: metricbeat.yml + - name: data + mountPath: /usr/share/metricbeat/data + - name: modules + mountPath: /usr/share/metricbeat/modules.d + readOnly: true + - name: dockersock + mountPath: /var/run/docker.sock + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: dockersock + hostPath: + path: /var/run/docker.sock + - name: config + configMap: + defaultMode: 0640 + name: metricbeat-leaderelection-daemonset-config + - name: modules + configMap: + defaultMode: 0640 + name: metricbeat-leaderelection-daemonset-modules + - name: data + hostPath: + # When metricbeat runs as non-root user, this directory needs to be writable by group (g+w) + path: /var/lib/metricbeat-data + type: DirectoryOrCreate +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metricbeat +subjects: +- kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metricbeat + labels: + k8s-app: metricbeat +rules: +- apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + - secrets + verbs: ["get", "list", "watch"] +- apiGroups: ["extensions"] + resources: + - replicasets + verbs: ["get", "list", "watch"] +- apiGroups: ["apps"] + resources: + - statefulsets + - deployments + - replicasets + verbs: ["get", "list", "watch"] +- apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get +- nonResourceURLs: + - "/metrics" + verbs: + - get +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - '*' +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +--- diff --git a/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset-configmap.yaml new file mode 100644 index 00000000000..7c264416ddb --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset-configmap.yaml @@ -0,0 +1,126 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-leaderelection-daemonset-config + namespace: kube-system + labels: + k8s-app: metricbeat +data: + metricbeat.yml: |- + metricbeat.config.modules: + # Mounted `metricbeat-daemonset-modules` configmap: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + # To enable hints based autodiscover uncomment this: + #metricbeat.autodiscover: + # providers: + # - type: kubernetes + # node: ${NODE_NAME} + # hints.enabled: true + # Unique Autodiscover provider that handles singleton instance in the whole cluster for some unique data sources, + # like kube-state-metrics + metricbeat.autodiscover: + providers: + - type: kubernetes + scope: cluster + node: ${NODE_NAME} + unique: true + # identifier: + templates: + - config: + - module: kubernetes + hosts: ["kube-state-metrics:8080"] + period: 10s + add_metadata: true + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + - state_cronjob + - state_resourcequota + - state_statefulset + # Uncomment this to get k8s events: + #- event + #- module: kubernetes + # metricsets: + # - apiserver + # hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + # period: 30s + + processors: + - add_cloud_metadata: + + cloud.id: ${ELASTIC_CLOUD_ID} + cloud.auth: ${ELASTIC_CLOUD_AUTH} + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-leaderelecction-daemonset-modules + namespace: kube-system + labels: + k8s-app: metricbeat +data: + system.yml: |- + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + #- core + #- diskio + #- socket + processes: ['.*'] + process.include_top_n: + by_cpu: 5 # include top 5 processes by CPU + by_memory: 5 # include top 5 processes by memory + + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)' + kubernetes.yml: |- + - module: kubernetes + metricsets: + - node + - system + - pod + - container + - volume + period: 10s + host: ${NODE_NAME} + hosts: ["https://${NODE_NAME}:10250"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.verification_mode: "none" + # If there is a CA bundle that contains the issuer of the certificate used in the Kubelet API, + # remove ssl.verification_mode entry and use the CA, for instance: + #ssl.certificate_authorities: + #- /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + # Currently `proxy` metricset is not supported on Openshift, comment out section + - module: kubernetes + metricsets: + - proxy + period: 10s + host: ${NODE_NAME} + hosts: ["localhost:10249"] diff --git a/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset.yaml b/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset.yaml new file mode 100644 index 00000000000..5bae792061f --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset.yaml @@ -0,0 +1,97 @@ +# Deploy a Metricbeat instance per node for node metrics retrieval +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: metricbeat-leaderelection + namespace: kube-system + labels: + k8s-app: metricbeat +spec: + selector: + matchLabels: + k8s-app: metricbeat + template: + metadata: + labels: + k8s-app: metricbeat + spec: + serviceAccountName: metricbeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: metricbeat + image: docker.elastic.co/beats/metricbeat:%VERSION% + args: [ + "-c", "/etc/metricbeat.yml", + "-e", + "-system.hostfs=/hostfs", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: ELASTIC_CLOUD_ID + value: + - name: ELASTIC_CLOUD_AUTH + value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/metricbeat.yml + readOnly: true + subPath: metricbeat.yml + - name: data + mountPath: /usr/share/metricbeat/data + - name: modules + mountPath: /usr/share/metricbeat/modules.d + readOnly: true + - name: dockersock + mountPath: /var/run/docker.sock + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: dockersock + hostPath: + path: /var/run/docker.sock + - name: config + configMap: + defaultMode: 0640 + name: metricbeat-leaderelection-daemonset-config + - name: modules + configMap: + defaultMode: 0640 + name: metricbeat-leaderelection-daemonset-modules + - name: data + hostPath: + # When metricbeat runs as non-root user, this directory needs to be writable by group (g+w) + path: /var/lib/metricbeat-data + type: DirectoryOrCreate From 4eb95196b73f0c47389820e24ae4d683f0c49877 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 13 Aug 2020 12:51:13 +0300 Subject: [PATCH 2/6] fix config Signed-off-by: chrismark --- .../metricbeat-leaderelection-kubernetes.yaml | 14 ++++++-------- ...ricbeat-leaderelection-daemonset-configmap.yaml | 10 ++++------ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/deploy/kubernetes/metricbeat-leaderelection-kubernetes.yaml b/deploy/kubernetes/metricbeat-leaderelection-kubernetes.yaml index ac6da3d2a45..8f837af19fb 100644 --- a/deploy/kubernetes/metricbeat-leaderelection-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-leaderelection-kubernetes.yaml @@ -14,16 +14,14 @@ data: # Reload module configs as they change: reload.enabled: false - # To enable hints based autodiscover uncomment this: - #metricbeat.autodiscover: - # providers: - # - type: kubernetes - # node: ${NODE_NAME} - # hints.enabled: true - # Unique Autodiscover provider that handles singleton instance in the whole cluster for some unique data sources, - # like kube-state-metrics metricbeat.autodiscover: providers: + # To enable hints based autodiscover uncomment this: + #- type: kubernetes + # node: ${NODE_NAME} + # hints.enabled: true + # Unique Autodiscover provider that handles singleton instance in the whole cluster for some unique data sources, + # like kube-state-metrics - type: kubernetes scope: cluster node: ${NODE_NAME} diff --git a/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset-configmap.yaml index 7c264416ddb..f8fc4d11d3a 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset-configmap.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset-configmap.yaml @@ -14,16 +14,14 @@ data: # Reload module configs as they change: reload.enabled: false - # To enable hints based autodiscover uncomment this: - #metricbeat.autodiscover: - # providers: - # - type: kubernetes - # node: ${NODE_NAME} - # hints.enabled: true # Unique Autodiscover provider that handles singleton instance in the whole cluster for some unique data sources, # like kube-state-metrics metricbeat.autodiscover: providers: + # To enable hints based autodiscover uncomment this: + #- type: kubernetes + # node: ${NODE_NAME} + # hints.enabled: true - type: kubernetes scope: cluster node: ${NODE_NAME} From 9b10a6ca9dc676f85fe0f7dc11cd0e96500170c3 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 13 Aug 2020 13:08:15 +0300 Subject: [PATCH 3/6] Add docs Signed-off-by: chrismark --- .../docs/running-on-kubernetes.asciidoc | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/metricbeat/docs/running-on-kubernetes.asciidoc b/metricbeat/docs/running-on-kubernetes.asciidoc index 411a7c9ae25..4266b012bbe 100644 --- a/metricbeat/docs/running-on-kubernetes.asciidoc +++ b/metricbeat/docs/running-on-kubernetes.asciidoc @@ -197,3 +197,40 @@ metricbeat 1 1 1 1 1m ------------------------------------------------ Metrics should start flowing to Elasticsearch. + + +[float] +==== Deploying Metricbeat Daemonset with Leader Election enabled + +In some cases users may want to avoid deploying both a Deployment and a Daemonset +to collect cluser-wide metrics and node-level metrics. +In this, we provide the option to deploy only Metricbeat only as a Deamonset +and leverage the leader election feature which will allow to define configurations +that will be enabled only by the leader Pod between the Pods of the Daemonset. +Here is an example of a configuration that enables leader election: +[source,yaml] +----- +metricbeat.autodiscover: +providers: +- type: kubernetes + scope: cluster + node: ${NODE_NAME} + unique: true + identifier: leaderelectionmetricbeat + templates: + - config: + - module: kubernetes + hosts: ["kube-state-metrics:8080"] + period: 10s + add_metadata: true + metricsets: + - state_node +----- +Users can find more info about the `unique` and `identifier` options at <>. + +Users can download and deploy this Daemonset: +["source", "sh", subs="attributes"] +------------------------------------------------ +curl -L -O https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/kubernetes/metricbeat-leaderelection-kubernetes.yaml +kubectl create -f metricbeat-leaderelection-kubernetes.yaml +------------------------------------------------ From 08fb001fe9c7e506f9ef89d346f88883f1a0d849 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 13 Aug 2020 15:35:18 +0300 Subject: [PATCH 4/6] Unify manifests Signed-off-by: chrismark --- deploy/kubernetes/metricbeat-kubernetes.yaml | 49 ++- .../metricbeat-leaderelection-kubernetes.yaml | 286 ------------------ .../metricbeat-daemonset-configmap.yaml | 47 ++- .../metricbeat/metricbeat-deployment.yaml | 2 + ...at-leaderelection-daemonset-configmap.yaml | 124 -------- .../metricbeat-leaderelection-daemonset.yaml | 97 ------ 6 files changed, 86 insertions(+), 519 deletions(-) delete mode 100644 deploy/kubernetes/metricbeat-leaderelection-kubernetes.yaml delete mode 100644 deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset-configmap.yaml delete mode 100644 deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset.yaml diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index 32cd5568025..29140db504c 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -14,12 +14,47 @@ data: # Reload module configs as they change: reload.enabled: false - # To enable hints based autodiscover uncomment this: - #metricbeat.autodiscover: - # providers: - # - type: kubernetes - # node: ${NODE_NAME} - # hints.enabled: true + metricbeat.autodiscover: + providers: + # To enable hints based autodiscover uncomment this: + #- type: kubernetes + # node: ${NODE_NAME} + # hints.enabled: true + # Uncomment the following to enable leader election provider that handles + # singleton instance configuration across the Daemonset Pods of the whole cluster + # in order to monitor some unique data sources, like kube-state-metrics. + # When enabling this remember to also delete the Deployment or just set the replicas of the + # Deployment to 0. + #- type: kubernetes + # scope: cluster + # node: ${NODE_NAME} + # unique: true + # # identifier: + # templates: + # - config: + # - module: kubernetes + # hosts: ["kube-state-metrics:8080"] + # period: 10s + # add_metadata: true + # metricsets: + # - state_node + # - state_deployment + # - state_replicaset + # - state_pod + # - state_container + # - state_cronjob + # - state_resourcequota + # - state_statefulset + # # Uncomment this to get k8s events: + # #- event + # - module: kubernetes + # metricsets: + # - apiserver + # hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + # period: 30s processors: - add_cloud_metadata: @@ -258,6 +293,8 @@ metadata: labels: k8s-app: metricbeat spec: + # Set to 0 if using leader election provider with the Daemonset + replicas: 1 selector: matchLabels: k8s-app: metricbeat diff --git a/deploy/kubernetes/metricbeat-leaderelection-kubernetes.yaml b/deploy/kubernetes/metricbeat-leaderelection-kubernetes.yaml deleted file mode 100644 index 8f837af19fb..00000000000 --- a/deploy/kubernetes/metricbeat-leaderelection-kubernetes.yaml +++ /dev/null @@ -1,286 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-leaderelection-daemonset-config - namespace: kube-system - labels: - k8s-app: metricbeat -data: - metricbeat.yml: |- - metricbeat.config.modules: - # Mounted `metricbeat-daemonset-modules` configmap: - path: ${path.config}/modules.d/*.yml - # Reload module configs as they change: - reload.enabled: false - - metricbeat.autodiscover: - providers: - # To enable hints based autodiscover uncomment this: - #- type: kubernetes - # node: ${NODE_NAME} - # hints.enabled: true - # Unique Autodiscover provider that handles singleton instance in the whole cluster for some unique data sources, - # like kube-state-metrics - - type: kubernetes - scope: cluster - node: ${NODE_NAME} - unique: true - # identifier: - templates: - - config: - - module: kubernetes - hosts: ["kube-state-metrics:8080"] - period: 10s - add_metadata: true - metricsets: - - state_node - - state_deployment - - state_replicaset - - state_pod - - state_container - - state_cronjob - - state_resourcequota - - state_statefulset - # Uncomment this to get k8s events: - #- event - #- module: kubernetes - # metricsets: - # - apiserver - # hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] - # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - # ssl.certificate_authorities: - # - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - # period: 30s - - processors: - - add_cloud_metadata: - - cloud.id: ${ELASTIC_CLOUD_ID} - cloud.auth: ${ELASTIC_CLOUD_AUTH} - - output.elasticsearch: - hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] - username: ${ELASTICSEARCH_USERNAME} - password: ${ELASTICSEARCH_PASSWORD} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-leaderelection-daemonset-modules - namespace: kube-system - labels: - k8s-app: metricbeat -data: - system.yml: |- - - module: system - period: 10s - metricsets: - - cpu - - load - - memory - - network - - process - - process_summary - #- core - #- diskio - #- socket - processes: ['.*'] - process.include_top_n: - by_cpu: 5 # include top 5 processes by CPU - by_memory: 5 # include top 5 processes by memory - - - module: system - period: 1m - metricsets: - - filesystem - - fsstat - processors: - - drop_event.when.regexp: - system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)' - kubernetes.yml: |- - - module: kubernetes - metricsets: - - node - - system - - pod - - container - - volume - period: 10s - host: ${NODE_NAME} - hosts: ["https://${NODE_NAME}:10250"] - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - ssl.verification_mode: "none" - # If there is a CA bundle that contains the issuer of the certificate used in the Kubelet API, - # remove ssl.verification_mode entry and use the CA, for instance: - #ssl.certificate_authorities: - #- /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - # Currently `proxy` metricset is not supported on Openshift, comment out section - - module: kubernetes - metricsets: - - proxy - period: 10s - host: ${NODE_NAME} - hosts: ["localhost:10249"] ---- -# Deploy a Metricbeat instance per node for node metrics retrieval -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: metricbeat-leaderelection - namespace: kube-system - labels: - k8s-app: metricbeat -spec: - selector: - matchLabels: - k8s-app: metricbeat - template: - metadata: - labels: - k8s-app: metricbeat - spec: - serviceAccountName: metricbeat - terminationGracePeriodSeconds: 30 - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: metricbeat - image: docker.elastic.co/beats/metricbeat:8.0.0 - args: [ - "-c", "/etc/metricbeat.yml", - "-e", - "-system.hostfs=/hostfs", - ] - env: - - name: ELASTICSEARCH_HOST - value: elasticsearch - - name: ELASTICSEARCH_PORT - value: "9200" - - name: ELASTICSEARCH_USERNAME - value: elastic - - name: ELASTICSEARCH_PASSWORD - value: changeme - - name: ELASTIC_CLOUD_ID - value: - - name: ELASTIC_CLOUD_AUTH - value: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - runAsUser: 0 - # If using Red Hat OpenShift uncomment this: - #privileged: true - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - name: config - mountPath: /etc/metricbeat.yml - readOnly: true - subPath: metricbeat.yml - - name: data - mountPath: /usr/share/metricbeat/data - - name: modules - mountPath: /usr/share/metricbeat/modules.d - readOnly: true - - name: dockersock - mountPath: /var/run/docker.sock - - name: proc - mountPath: /hostfs/proc - readOnly: true - - name: cgroup - mountPath: /hostfs/sys/fs/cgroup - readOnly: true - volumes: - - name: proc - hostPath: - path: /proc - - name: cgroup - hostPath: - path: /sys/fs/cgroup - - name: dockersock - hostPath: - path: /var/run/docker.sock - - name: config - configMap: - defaultMode: 0640 - name: metricbeat-leaderelection-daemonset-config - - name: modules - configMap: - defaultMode: 0640 - name: metricbeat-leaderelection-daemonset-modules - - name: data - hostPath: - # When metricbeat runs as non-root user, this directory needs to be writable by group (g+w) - path: /var/lib/metricbeat-data - type: DirectoryOrCreate ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: metricbeat -subjects: -- kind: ServiceAccount - name: metricbeat - namespace: kube-system -roleRef: - kind: ClusterRole - name: metricbeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: metricbeat - labels: - k8s-app: metricbeat -rules: -- apiGroups: [""] - resources: - - nodes - - namespaces - - events - - pods - - secrets - verbs: ["get", "list", "watch"] -- apiGroups: ["extensions"] - resources: - - replicasets - verbs: ["get", "list", "watch"] -- apiGroups: ["apps"] - resources: - - statefulsets - - deployments - - replicasets - verbs: ["get", "list", "watch"] -- apiGroups: - - "" - resources: - - nodes/stats - verbs: - - get -- nonResourceURLs: - - "/metrics" - verbs: - - get -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - '*' ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: metricbeat - namespace: kube-system - labels: - k8s-app: metricbeat ---- diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml index a244dda551a..3b0da9cc503 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml @@ -14,12 +14,47 @@ data: # Reload module configs as they change: reload.enabled: false - # To enable hints based autodiscover uncomment this: - #metricbeat.autodiscover: - # providers: - # - type: kubernetes - # node: ${NODE_NAME} - # hints.enabled: true + metricbeat.autodiscover: + providers: + # To enable hints based autodiscover uncomment this: + #- type: kubernetes + # node: ${NODE_NAME} + # hints.enabled: true + # Uncomment the following to enable leader election provider that handles + # singleton instance configuration across the Daemonset Pods of the whole cluster + # in order to monitor some unique data sources, like kube-state-metrics. + # When enabling this remember to also delete the Deployment or just set the replicas of the + # Deployment to 0. + #- type: kubernetes + # scope: cluster + # node: ${NODE_NAME} + # unique: true + # # identifier: + # templates: + # - config: + # - module: kubernetes + # hosts: ["kube-state-metrics:8080"] + # period: 10s + # add_metadata: true + # metricsets: + # - state_node + # - state_deployment + # - state_replicaset + # - state_pod + # - state_container + # - state_cronjob + # - state_resourcequota + # - state_statefulset + # # Uncomment this to get k8s events: + # #- event + # - module: kubernetes + # metricsets: + # - apiserver + # hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + # period: 30s processors: - add_cloud_metadata: diff --git a/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml b/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml index 0e11187cac3..f82b204b63d 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml @@ -7,6 +7,8 @@ metadata: labels: k8s-app: metricbeat spec: + # Set to 0 if using leader election provider with the Daemonset + replicas: 1 selector: matchLabels: k8s-app: metricbeat diff --git a/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset-configmap.yaml deleted file mode 100644 index f8fc4d11d3a..00000000000 --- a/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset-configmap.yaml +++ /dev/null @@ -1,124 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-leaderelection-daemonset-config - namespace: kube-system - labels: - k8s-app: metricbeat -data: - metricbeat.yml: |- - metricbeat.config.modules: - # Mounted `metricbeat-daemonset-modules` configmap: - path: ${path.config}/modules.d/*.yml - # Reload module configs as they change: - reload.enabled: false - - # Unique Autodiscover provider that handles singleton instance in the whole cluster for some unique data sources, - # like kube-state-metrics - metricbeat.autodiscover: - providers: - # To enable hints based autodiscover uncomment this: - #- type: kubernetes - # node: ${NODE_NAME} - # hints.enabled: true - - type: kubernetes - scope: cluster - node: ${NODE_NAME} - unique: true - # identifier: - templates: - - config: - - module: kubernetes - hosts: ["kube-state-metrics:8080"] - period: 10s - add_metadata: true - metricsets: - - state_node - - state_deployment - - state_replicaset - - state_pod - - state_container - - state_cronjob - - state_resourcequota - - state_statefulset - # Uncomment this to get k8s events: - #- event - #- module: kubernetes - # metricsets: - # - apiserver - # hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] - # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - # ssl.certificate_authorities: - # - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - # period: 30s - - processors: - - add_cloud_metadata: - - cloud.id: ${ELASTIC_CLOUD_ID} - cloud.auth: ${ELASTIC_CLOUD_AUTH} - - output.elasticsearch: - hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] - username: ${ELASTICSEARCH_USERNAME} - password: ${ELASTICSEARCH_PASSWORD} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-leaderelecction-daemonset-modules - namespace: kube-system - labels: - k8s-app: metricbeat -data: - system.yml: |- - - module: system - period: 10s - metricsets: - - cpu - - load - - memory - - network - - process - - process_summary - #- core - #- diskio - #- socket - processes: ['.*'] - process.include_top_n: - by_cpu: 5 # include top 5 processes by CPU - by_memory: 5 # include top 5 processes by memory - - - module: system - period: 1m - metricsets: - - filesystem - - fsstat - processors: - - drop_event.when.regexp: - system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)' - kubernetes.yml: |- - - module: kubernetes - metricsets: - - node - - system - - pod - - container - - volume - period: 10s - host: ${NODE_NAME} - hosts: ["https://${NODE_NAME}:10250"] - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - ssl.verification_mode: "none" - # If there is a CA bundle that contains the issuer of the certificate used in the Kubelet API, - # remove ssl.verification_mode entry and use the CA, for instance: - #ssl.certificate_authorities: - #- /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - # Currently `proxy` metricset is not supported on Openshift, comment out section - - module: kubernetes - metricsets: - - proxy - period: 10s - host: ${NODE_NAME} - hosts: ["localhost:10249"] diff --git a/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset.yaml b/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset.yaml deleted file mode 100644 index 5bae792061f..00000000000 --- a/deploy/kubernetes/metricbeat/metricbeat-leaderelection-daemonset.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# Deploy a Metricbeat instance per node for node metrics retrieval -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: metricbeat-leaderelection - namespace: kube-system - labels: - k8s-app: metricbeat -spec: - selector: - matchLabels: - k8s-app: metricbeat - template: - metadata: - labels: - k8s-app: metricbeat - spec: - serviceAccountName: metricbeat - terminationGracePeriodSeconds: 30 - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: metricbeat - image: docker.elastic.co/beats/metricbeat:%VERSION% - args: [ - "-c", "/etc/metricbeat.yml", - "-e", - "-system.hostfs=/hostfs", - ] - env: - - name: ELASTICSEARCH_HOST - value: elasticsearch - - name: ELASTICSEARCH_PORT - value: "9200" - - name: ELASTICSEARCH_USERNAME - value: elastic - - name: ELASTICSEARCH_PASSWORD - value: changeme - - name: ELASTIC_CLOUD_ID - value: - - name: ELASTIC_CLOUD_AUTH - value: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - runAsUser: 0 - # If using Red Hat OpenShift uncomment this: - #privileged: true - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - name: config - mountPath: /etc/metricbeat.yml - readOnly: true - subPath: metricbeat.yml - - name: data - mountPath: /usr/share/metricbeat/data - - name: modules - mountPath: /usr/share/metricbeat/modules.d - readOnly: true - - name: dockersock - mountPath: /var/run/docker.sock - - name: proc - mountPath: /hostfs/proc - readOnly: true - - name: cgroup - mountPath: /hostfs/sys/fs/cgroup - readOnly: true - volumes: - - name: proc - hostPath: - path: /proc - - name: cgroup - hostPath: - path: /sys/fs/cgroup - - name: dockersock - hostPath: - path: /var/run/docker.sock - - name: config - configMap: - defaultMode: 0640 - name: metricbeat-leaderelection-daemonset-config - - name: modules - configMap: - defaultMode: 0640 - name: metricbeat-leaderelection-daemonset-modules - - name: data - hostPath: - # When metricbeat runs as non-root user, this directory needs to be writable by group (g+w) - path: /var/lib/metricbeat-data - type: DirectoryOrCreate From 1c1e97a48dc14d3dad578b114583e92fc219cf32 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 13 Aug 2020 15:39:57 +0300 Subject: [PATCH 5/6] Fix docs Signed-off-by: chrismark --- metricbeat/docs/running-on-kubernetes.asciidoc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/metricbeat/docs/running-on-kubernetes.asciidoc b/metricbeat/docs/running-on-kubernetes.asciidoc index 4266b012bbe..5001074df5b 100644 --- a/metricbeat/docs/running-on-kubernetes.asciidoc +++ b/metricbeat/docs/running-on-kubernetes.asciidoc @@ -228,9 +228,12 @@ providers: ----- Users can find more info about the `unique` and `identifier` options at <>. -Users can download and deploy this Daemonset: +Users can enable the respective parts the Daemonset ConfigMap and +set the `replicas` of the Deployment to `0` in order to only deploy +the Daemonset on the cluster with the leader election provider enabled +in order to collect cluster-wide metrics: ["source", "sh", subs="attributes"] ------------------------------------------------ -curl -L -O https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/kubernetes/metricbeat-leaderelection-kubernetes.yaml -kubectl create -f metricbeat-leaderelection-kubernetes.yaml +curl -L -O https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/kubernetes/metricbeat-kubernetes.yaml +kubectl create -f metricbeat-kubernetes.yaml ------------------------------------------------ From fe70e8f25fa8cc1f46a0ea3bc5afee671fa9e4c3 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 13 Aug 2020 17:09:50 +0300 Subject: [PATCH 6/6] review changes Signed-off-by: chrismark --- .../docs/running-on-kubernetes.asciidoc | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/metricbeat/docs/running-on-kubernetes.asciidoc b/metricbeat/docs/running-on-kubernetes.asciidoc index 5001074df5b..0fa34f5df95 100644 --- a/metricbeat/docs/running-on-kubernetes.asciidoc +++ b/metricbeat/docs/running-on-kubernetes.asciidoc @@ -204,27 +204,28 @@ Metrics should start flowing to Elasticsearch. In some cases users may want to avoid deploying both a Deployment and a Daemonset to collect cluser-wide metrics and node-level metrics. -In this, we provide the option to deploy only Metricbeat only as a Deamonset -and leverage the leader election feature which will allow to define configurations -that will be enabled only by the leader Pod between the Pods of the Daemonset. +For this case, we provide the option to deploy Metricbeat only as a Daemonset +and leverage the leader election feature which allows to define configurations +that are enabled only by the leader Pod. The leader Pod is automatically chosen +between the Pods of the Daemonset. Here is an example of a configuration that enables leader election: [source,yaml] ----- metricbeat.autodiscover: -providers: -- type: kubernetes - scope: cluster - node: ${NODE_NAME} - unique: true - identifier: leaderelectionmetricbeat - templates: - - config: - - module: kubernetes - hosts: ["kube-state-metrics:8080"] - period: 10s - add_metadata: true - metricsets: - - state_node + providers: + - type: kubernetes + scope: cluster + node: ${NODE_NAME} + unique: true + identifier: leaderelectionmetricbeat + templates: + - config: + - module: kubernetes + hosts: ["kube-state-metrics:8080"] + period: 10s + add_metadata: true + metricsets: + - state_node ----- Users can find more info about the `unique` and `identifier` options at <>.