diff --git a/.travis.yml b/.travis.yml index 0a4baf85bfb..710fb90856b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ env: - GOX_FLAGS="-arch amd64" - DOCKER_COMPOSE_VERSION=1.11.1 - GO_VERSION="$(cat .go-version)" + - TRAVIS_ETCD_VERSION=v3.2.8 jobs: include: @@ -95,6 +96,26 @@ jobs: go: $GO_VERSION stage: test + # Kubernetes + - os: linux + install: deploy/kubernetes/.travis/setup.sh + env: + - TARGETS="-C deploy/kubernetes test" + - TRAVIS_KUBE_VERSION=v1.6.11 + stage: test + - os: linux + install: deploy/kubernetes/.travis/setup.sh + env: + - TARGETS="-C deploy/kubernetes test" + - TRAVIS_KUBE_VERSION=v1.7.7 + stage: test + - os: linux + install: deploy/kubernetes/.travis/setup.sh + env: + - TARGETS="-C deploy/kubernetes test" + - TRAVIS_KUBE_VERSION=v1.8.0 + stage: test + addons: apt: packages: diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index ffe1445402d..0015e39bb4b 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -50,10 +50,14 @@ https://github.com/elastic/beats/compare/v6.0.0-rc2...master[Check the HEAD diff *Filebeat* +- Add Kubernetes manifests to deploy Filebeat. {pull}5349[5349] + *Heartbeat* *Metricbeat* +- Add Kubernetes manifests to deploy Metricbeat. {pull}5349[5349] + *Packetbeat* *Winlogbeat* diff --git a/Makefile b/Makefile index 38847cdffc1..96a574461dd 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,7 @@ coverage-report: .PHONY: update update: notice @$(foreach var,$(PROJECTS),$(MAKE) -C $(var) update || exit 1;) + @$(MAKE) -C deploy/kubernetes all .PHONY: clean clean: @@ -97,7 +98,7 @@ docs: @sh libbeat/scripts/build_docs.sh $(PROJECTS) .PHONY: package -package: update beats-dashboards +package: update beats-dashboards kubernetes-manifests @$(foreach var,$(BEATS),SNAPSHOT=$(SNAPSHOT) $(MAKE) -C $(var) package || exit 1;) @echo "Start building the dashboards package" @@ -105,10 +106,16 @@ package: update beats-dashboards @BUILD_DIR=$(CURDIR)/build SNAPSHOT=$(SNAPSHOT) $(MAKE) -C dev-tools/packer package-dashboards $(CURDIR)/build/upload/build_id.txt @mv build/upload build/dashboards-upload + @echo "Start building kubernetes manifests" + @mkdir -p build/upload/ + @BUILD_DIR=${BUILD_DIR} SNAPSHOT=$(SNAPSHOT) $(MAKE) -C dev-tools/packer package-kubernetes ${BUILD_DIR}/upload/build_id.txt + @mv build/upload build/kubernetes-upload + @# Copy build files over to top build directory @mkdir -p build/upload/ @$(foreach var,$(BEATS),cp -r $(var)/build/upload/ build/upload/$(var) || exit 1;) @cp -r build/dashboards-upload build/upload/dashboards + @cp -r build/kubernetes-upload build/upload/kubernetes @# Run tests on the generated packages. @go test ./dev-tools/package_test.go -files "$(CURDIR)/build/upload/*/*" @@ -139,3 +146,10 @@ notice: python-env python-env: @test -d $(PYTHON_ENV) || virtualenv $(VIRTUALENV_PARAMS) $(PYTHON_ENV) @$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8 six + +# Build kubernetes manifests +.PHONY: kubernetes-manifests +kubernetes-manifests: + @mkdir -p build/kubernetes + $(MAKE) -C deploy/kubernetes all + cp deploy/kubernetes/*.yaml build/kubernetes diff --git a/deploy/kubernetes/.travis/setup.sh b/deploy/kubernetes/.travis/setup.sh new file mode 100755 index 00000000000..7de6f3adc26 --- /dev/null +++ b/deploy/kubernetes/.travis/setup.sh @@ -0,0 +1,61 @@ +# This script assumes Docker is already installed +#!/bin/bash + +set -x + +# set docker0 to promiscuous mode +sudo ip link set docker0 promisc on + +# install etcd +wget https://github.com/coreos/etcd/releases/download/$TRAVIS_ETCD_VERSION/etcd-$TRAVIS_ETCD_VERSION-linux-amd64.tar.gz +tar xzf etcd-$TRAVIS_ETCD_VERSION-linux-amd64.tar.gz +sudo mv etcd-$TRAVIS_ETCD_VERSION-linux-amd64/etcd /usr/local/bin/etcd +rm etcd-$TRAVIS_ETCD_VERSION-linux-amd64.tar.gz +rm -rf etcd-$TRAVIS_ETCD_VERSION-linux-amd64 + +# download kubectl +wget https://storage.googleapis.com/kubernetes-release/release/$TRAVIS_KUBE_VERSION/bin/linux/amd64/kubectl +chmod +x kubectl +sudo mv kubectl /usr/local/bin/kubectl + +# download kubernetes +git clone https://github.com/kubernetes/kubernetes $HOME/kubernetes + +# install cfssl +go get -u github.com/cloudflare/cfssl/cmd/... + +pushd $HOME/kubernetes + git checkout $TRAVIS_KUBE_VERSION + kubectl config set-credentials myself --username=admin --password=admin + kubectl config set-context local --cluster=local --user=myself + kubectl config set-cluster local --server=http://localhost:8080 + kubectl config use-context local + + # start kubernetes in the background + sudo PATH=$PATH:/home/travis/.gimme/versions/go1.7.linux.amd64/bin/go \ + KUBE_ENABLE_CLUSTER_DNS=true \ + hack/local-up-cluster.sh & +popd + +# Wait until kube is up and running +TIMEOUT=0 +TIMEOUT_COUNT=800 +until $(curl --output /dev/null --silent http://localhost:8080) || [ $TIMEOUT -eq $TIMEOUT_COUNT ]; do + echo "Kube is not up yet" + let TIMEOUT=TIMEOUT+1 + sleep 1 +done + +if [ $TIMEOUT -eq $TIMEOUT_COUNT ]; then + echo "Kubernetes is not up and running" + exit 1 +fi + +echo "Kubernetes is deployed and reachable" + +# Try and sleep before issuing chown. Currently, Kubernetes is started by +# a command that is run in the background. Technically Kubernetes could be +# up and running, but those files might not exist yet as the previous command +# could create them after Kube starts successfully. +sleep 30 +sudo chown -R $USER:$USER $HOME/.kube diff --git a/deploy/kubernetes/Makefile b/deploy/kubernetes/Makefile new file mode 100644 index 00000000000..68d2ad9fd91 --- /dev/null +++ b/deploy/kubernetes/Makefile @@ -0,0 +1,21 @@ +ALL=filebeat metricbeat +BEAT_VERSION=$(shell head -n 1 ../../libbeat/docs/version.asciidoc | cut -c 17- ) + +all: ${ALL:=-kubernetes.yaml} + +test: all + for FILE in $(shell ls *-kubernetes.yaml); do \ + BEAT=$$(echo $$FILE | cut -d \- -f 1); \ + kubectl create -f $$FILE; \ + done + +clean: + @for f in $(ALL); do rm -f "$$f-kubernetes.yaml"; done + +%-kubernetes.yaml: %/*.yaml + @echo "Generating $*-kubernetes.yaml" + @rm -f $*-kubernetes.yaml + @for f in $(shell ls $*/*.yaml); do \ + sed "s/%VERSION%/${BEAT_VERSION}/g" $$f >> $*-kubernetes.yaml; \ + echo --- >> $*-kubernetes.yaml; \ + done diff --git a/deploy/kubernetes/README.md b/deploy/kubernetes/README.md new file mode 100644 index 00000000000..65275dc4e45 --- /dev/null +++ b/deploy/kubernetes/README.md @@ -0,0 +1,11 @@ +# Beats Kubernetes manifests examples + +## Getting started + +This is the list of officially supported Beats, with example manifests to run +them in Kubernetes: + +Beat | Description +---- | ---- +[filebeat](filebeat) | Tails and ships logs +[metricbeat](metricbeat) | Fetches sets of metrics from the operating system and services diff --git a/deploy/kubernetes/filebeat-kubernetes.yaml b/deploy/kubernetes/filebeat-kubernetes.yaml new file mode 100644 index 00000000000..5899b9b3831 --- /dev/null +++ b/deploy/kubernetes/filebeat-kubernetes.yaml @@ -0,0 +1,160 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: kube-system + labels: + k8s-app: filebeat + kubernetes.io/cluster-service: "true" +data: + filebeat.yml: |- + filebeat.config: + prospectors: + # Mounted `filebeat-prospectors` configmap: + path: ${path.config}/prospectors.d/*.yml + # Reload prospectors configs as they change: + reload.enabled: false + modules: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + processors: + - add_cloud_metadata: + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-prospectors + namespace: kube-system + labels: + k8s-app: filebeat + kubernetes.io/cluster-service: "true" +data: + kubernetes.yml: |- + - type: log + paths: + - /var/lib/docker/containers/*/*.log + json.message_key: log + json.keys_under_root: true + processors: + - add_kubernetes_metadata: + in_cluster: true + namespace: ${POD_NAMESPACE} +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: filebeat + kubernetes.io/cluster-service: "true" + spec: + serviceAccountName: filebeat + terminationGracePeriodSeconds: 30 + containers: + - name: filebeat + image: docker.elastic.co/beats/filebeat:6.0.0-rc1 + args: [ + "-c", "/etc/filebeat.yml", + "-e", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/filebeat.yml + readOnly: true + subPath: filebeat.yml + - name: prospectors + mountPath: /usr/share/filebeat/prospectors.d + readOnly: true + - name: data + mountPath: /usr/share/filebeat/data + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + volumes: + - name: config + configMap: + defaultMode: 0600 + name: filebeat-config + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: prospectors + configMap: + defaultMode: 0600 + name: filebeat-prospectors + - name: data + emptyDir: {} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: filebeat + labels: + k8s-app: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + verbs: + - get + - watch + - list +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +--- diff --git a/deploy/kubernetes/filebeat/README.md b/deploy/kubernetes/filebeat/README.md new file mode 100644 index 00000000000..e5e8a70bf35 --- /dev/null +++ b/deploy/kubernetes/filebeat/README.md @@ -0,0 +1,35 @@ +# Filebeat + +## Ship logs from Kubernetes to Elasticsearch + +### Kubernetes DaemonSet + +By deploying filebeat as a [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) +we ensure we get a running filebeat daemon on each node of the cluster. + +Docker logs host folder (`/var/lib/docker/containers`) is mounted on the +filebeat container. Filebeat will start a prospector for these files and start +harvesting them as they appear. + +Everything is deployed under `kube-system` namespace, you can change that by +updating YAML manifests under this folder. + +### Settings + +We use official [Beats Docker images](https://github.com/elastic/beats-docker), +as they allow external files configuration, a [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) +is used for kubernetes specific settings. Check [filebeat-configmap.yaml](filebeat-configmap.yaml) +for details. + +Also, [filebeat-daemonset.yaml](filebeat-daemonset.yaml) uses a set of environment +variables to configure Elasticsearch output: + +Variable | Default | Description +-------- | ------- | ----------- +ELASTICSEARCH_HOST | elasticsearch | Elasticsearch host +ELASTICSEARCH_PORT | 9200 | Elasticsearch port +ELASTICSEARCH_USERNAME | elastic | Elasticsearch username for HTTP auth +ELASTICSEARCH_PASSWORD | changeme | Elasticsearch password + +If there is an existing `elasticsearch` service in the kubernetes cluster these +defaults will use it. diff --git a/deploy/kubernetes/filebeat/filebeat-configmap.yaml b/deploy/kubernetes/filebeat/filebeat-configmap.yaml new file mode 100644 index 00000000000..f1fc331ac8c --- /dev/null +++ b/deploy/kubernetes/filebeat/filebeat-configmap.yaml @@ -0,0 +1,49 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: kube-system + labels: + k8s-app: filebeat + kubernetes.io/cluster-service: "true" +data: + filebeat.yml: |- + filebeat.config: + prospectors: + # Mounted `filebeat-prospectors` configmap: + path: ${path.config}/prospectors.d/*.yml + # Reload prospectors configs as they change: + reload.enabled: false + modules: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + processors: + - add_cloud_metadata: + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-prospectors + namespace: kube-system + labels: + k8s-app: filebeat + kubernetes.io/cluster-service: "true" +data: + kubernetes.yml: |- + - type: log + paths: + - /var/lib/docker/containers/*/*.log + json.message_key: log + json.keys_under_root: true + processors: + - add_kubernetes_metadata: + in_cluster: true + namespace: ${POD_NAMESPACE} diff --git a/deploy/kubernetes/filebeat/filebeat-daemonset.yaml b/deploy/kubernetes/filebeat/filebeat-daemonset.yaml new file mode 100644 index 00000000000..3ec9766d646 --- /dev/null +++ b/deploy/kubernetes/filebeat/filebeat-daemonset.yaml @@ -0,0 +1,72 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: filebeat + kubernetes.io/cluster-service: "true" + spec: + serviceAccountName: filebeat + terminationGracePeriodSeconds: 30 + containers: + - name: filebeat + image: docker.elastic.co/beats/filebeat:%VERSION% + args: [ + "-c", "/etc/filebeat.yml", + "-e", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/filebeat.yml + readOnly: true + subPath: filebeat.yml + - name: prospectors + mountPath: /usr/share/filebeat/prospectors.d + readOnly: true + - name: data + mountPath: /usr/share/filebeat/data + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + volumes: + - name: config + configMap: + defaultMode: 0600 + name: filebeat-config + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: prospectors + configMap: + defaultMode: 0600 + name: filebeat-prospectors + - name: data + emptyDir: {} diff --git a/deploy/kubernetes/filebeat/filebeat-role-binding.yaml b/deploy/kubernetes/filebeat/filebeat-role-binding.yaml new file mode 100644 index 00000000000..f24259ffa6a --- /dev/null +++ b/deploy/kubernetes/filebeat/filebeat-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/filebeat/filebeat-role.yaml b/deploy/kubernetes/filebeat/filebeat-role.yaml new file mode 100644 index 00000000000..160bb9044fa --- /dev/null +++ b/deploy/kubernetes/filebeat/filebeat-role.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: filebeat + labels: + k8s-app: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + verbs: + - get + - watch + - list diff --git a/deploy/kubernetes/filebeat/filebeat-service-account.yaml b/deploy/kubernetes/filebeat/filebeat-service-account.yaml new file mode 100644 index 00000000000..e243881a1d6 --- /dev/null +++ b/deploy/kubernetes/filebeat/filebeat-service-account.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml new file mode 100644 index 00000000000..8b183827331 --- /dev/null +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -0,0 +1,279 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-config + namespace: kube-system + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" +data: + metricbeat.yml: |- + metricbeat.config.modules: + # Mounted `metricbeat-daemonset-modules` configmap: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + processors: + - add_cloud_metadata: + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-daemonset-modules + namespace: kube-system + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" +data: + system.yml: |- + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + #- core + #- diskio + #- socket + processes: ['.*'] + process.include_top_n: + by_cpu: 5 # include top 5 processes by CPU + by_memory: 5 # include top 5 processes by memory + + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + kubernetes.yml: |- + - module: kubernetes + metricsets: + - node + - system + - pod + - container + - volume + period: 10s + hosts: ["localhost:10255"] +--- +# Deploy a Metricbeat instance per node for node metrics retrieval +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" + spec: + serviceAccountName: metricbeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + containers: + - name: metricbeat + image: docker.elastic.co/beats/metricbeat:6.0.0-rc1 + args: [ + "-c", "/etc/metricbeat.yml", + "-e", + "-system.hostfs=/hostfs", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/metricbeat.yml + readOnly: true + subPath: metricbeat.yml + - name: modules + mountPath: /usr/share/metricbeat/modules.d + readOnly: true + - name: dockersock + mountPath: /var/run/docker.sock + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: dockersock + hostPath: + path: /var/run/docker.sock + - name: config + configMap: + defaultMode: 0600 + name: metricbeat-config + - name: modules + configMap: + defaultMode: 0600 + name: metricbeat-daemonset-modules + - name: data + emptyDir: {} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-deployment-modules + namespace: kube-system + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" +data: + # This module requires `kube-state-metrics` up and running under `kube-system` namespace + kubernetes.yml: |- + - module: kubernetes + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + # Uncomment this to get k8s events: + #- events + period: 10s + hosts: ["kube-state-metrics:8080"] +--- +# Deploy singleton instance in the whole cluster for some unique data sources, like kube-state-metrics +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" + spec: + containers: + - name: metricbeat + image: docker.elastic.co/beats/metricbeat:6.0.0-rc1 + args: [ + "-c", "/etc/metricbeat.yml", + "-e", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/metricbeat.yml + readOnly: true + subPath: metricbeat.yml + - name: modules + mountPath: /usr/share/metricbeat/modules.d + readOnly: true + volumes: + - name: config + configMap: + defaultMode: 0600 + name: metricbeat-config + - name: modules + configMap: + defaultMode: 0600 + name: metricbeat-deployment-modules +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: metricbeat +subjects: +- kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: metricbeat + labels: + k8s-app: metricbeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - events + - pods + verbs: + - get + - watch + - list +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +--- diff --git a/deploy/kubernetes/metricbeat/README.md b/deploy/kubernetes/metricbeat/README.md new file mode 100644 index 00000000000..186807d5e70 --- /dev/null +++ b/deploy/kubernetes/metricbeat/README.md @@ -0,0 +1,31 @@ +# Metricbeat + +## Ship metrics from Kubernetes to Elasticsearch + +### Kubernetes DaemonSet + +By deploying metricbeat as a [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) +we ensure we get a running metricbeat daemon on each node of the cluster. + +Everything is deployed under `kube-system` namespace, you can change that by +updating YAML manifests under this folder. + +### Settings + +We use official [Beats Docker images](https://github.com/elastic/beats-docker), +as they allow external files configuration, a [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) +is used for kubernetes specific settings. Check [metricbeat-configmap.yaml](metricbeat-configmap.yaml) +for details. + +Also, [metricbeat-daemonset.yaml](metricbeat-daemonset.yaml) uses a set of environment +variables to configure Elasticsearch output: + +Variable | Default | Description +-------- | ------- | ----------- +ELASTICSEARCH_HOST | elasticsearch | Elasticsearch host +ELASTICSEARCH_PORT | 9200 | Elasticsearch port +ELASTICSEARCH_USERNAME | elastic | Elasticsearch username for HTTP auth +ELASTICSEARCH_PASSWORD | changeme | Elasticsearch password + +If there is an existing `elasticsearch` service in the kubernetes cluster these +defaults will use it. diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml new file mode 100644 index 00000000000..232f8153663 --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml @@ -0,0 +1,70 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-config + namespace: kube-system + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" +data: + metricbeat.yml: |- + metricbeat.config.modules: + # Mounted `metricbeat-daemonset-modules` configmap: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + processors: + - add_cloud_metadata: + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-daemonset-modules + namespace: kube-system + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" +data: + system.yml: |- + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + #- core + #- diskio + #- socket + processes: ['.*'] + process.include_top_n: + by_cpu: 5 # include top 5 processes by CPU + by_memory: 5 # include top 5 processes by memory + + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + kubernetes.yml: |- + - module: kubernetes + metricsets: + - node + - system + - pod + - container + - volume + period: 10s + hosts: ["localhost:10255"] diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml new file mode 100644 index 00000000000..e2afada445d --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml @@ -0,0 +1,84 @@ +# Deploy a Metricbeat instance per node for node metrics retrieval +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" + spec: + serviceAccountName: metricbeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + containers: + - name: metricbeat + image: docker.elastic.co/beats/metricbeat:%VERSION% + args: [ + "-c", "/etc/metricbeat.yml", + "-e", + "-system.hostfs=/hostfs", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/metricbeat.yml + readOnly: true + subPath: metricbeat.yml + - name: modules + mountPath: /usr/share/metricbeat/modules.d + readOnly: true + - name: dockersock + mountPath: /var/run/docker.sock + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: dockersock + hostPath: + path: /var/run/docker.sock + - name: config + configMap: + defaultMode: 0600 + name: metricbeat-config + - name: modules + configMap: + defaultMode: 0600 + name: metricbeat-daemonset-modules + - name: data + emptyDir: {} diff --git a/deploy/kubernetes/metricbeat/metricbeat-deployment-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-deployment-configmap.yaml new file mode 100644 index 00000000000..def9b995556 --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-deployment-configmap.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-deployment-modules + namespace: kube-system + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" +data: + # This module requires `kube-state-metrics` up and running under `kube-system` namespace + kubernetes.yml: |- + - module: kubernetes + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + # Uncomment this to get k8s events: + #- events + period: 10s + hosts: ["kube-state-metrics:8080"] diff --git a/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml b/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml new file mode 100644 index 00000000000..9c9063af78d --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml @@ -0,0 +1,61 @@ +# Deploy singleton instance in the whole cluster for some unique data sources, like kube-state-metrics +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" + spec: + containers: + - name: metricbeat + image: docker.elastic.co/beats/metricbeat:%VERSION% + args: [ + "-c", "/etc/metricbeat.yml", + "-e", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/metricbeat.yml + readOnly: true + subPath: metricbeat.yml + - name: modules + mountPath: /usr/share/metricbeat/modules.d + readOnly: true + volumes: + - name: config + configMap: + defaultMode: 0600 + name: metricbeat-config + - name: modules + configMap: + defaultMode: 0600 + name: metricbeat-deployment-modules diff --git a/deploy/kubernetes/metricbeat/metricbeat-role-binding.yaml b/deploy/kubernetes/metricbeat/metricbeat-role-binding.yaml new file mode 100644 index 00000000000..8a74c8f81a1 --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: metricbeat +subjects: +- kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/metricbeat/metricbeat-role.yaml b/deploy/kubernetes/metricbeat/metricbeat-role.yaml new file mode 100644 index 00000000000..ebc9039cb54 --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-role.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: metricbeat + labels: + k8s-app: metricbeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - events + - pods + verbs: + - get + - watch + - list diff --git a/deploy/kubernetes/metricbeat/metricbeat-service-account.yaml b/deploy/kubernetes/metricbeat/metricbeat-service-account.yaml new file mode 100644 index 00000000000..2b2fd46dd31 --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-service-account.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat diff --git a/dev-tools/packer/Makefile b/dev-tools/packer/Makefile index cf447425e9a..c00d4c997e4 100644 --- a/dev-tools/packer/Makefile +++ b/dev-tools/packer/Makefile @@ -39,6 +39,11 @@ package-dashboards: echo Creating the Dashboards package BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/dashboards/build.sh +.PHONY: package-kubernetes +package-kubernetes: + echo Creating the kubernetes manifests package + BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/kubernetes/build.sh + .PHONY: deps deps: go get -u github.com/tsg/gotpl diff --git a/dev-tools/packer/platforms/kubernetes/build.sh b/dev-tools/packer/platforms/kubernetes/build.sh new file mode 100755 index 00000000000..4f0d1b2f197 --- /dev/null +++ b/dev-tools/packer/platforms/kubernetes/build.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e + +BASEDIR=$(dirname "$0") +ARCHDIR=${BASEDIR}/../../ + +runid=kubernetes + +cat ${ARCHDIR}/version.yml > ${BUILD_DIR}/settings-$runid.yml +gotpl ${BASEDIR}/run.sh.j2 < ${BUILD_DIR}/settings-$runid.yml > ${BUILD_DIR}/run-$runid.sh +chmod +x ${BUILD_DIR}/run-$runid.sh + +docker run --rm -v ${BUILD_DIR}:/build \ + -e BUILDID=$BUILDID -e SNAPSHOT=$SNAPSHOT -e RUNID=$runid -e BEAT_NAME=$BEAT_NAME \ + tudorg/fpm /build/run-$runid.sh + +rm ${BUILD_DIR}/settings-$runid.yml ${BUILD_DIR}/run-$runid.sh diff --git a/dev-tools/packer/platforms/kubernetes/run.sh.j2 b/dev-tools/packer/platforms/kubernetes/run.sh.j2 new file mode 100644 index 00000000000..b781c97644d --- /dev/null +++ b/dev-tools/packer/platforms/kubernetes/run.sh.j2 @@ -0,0 +1,14 @@ +#!/bin/bash + +# this is executed in the docker fpm image +set -e +cd /build + +mkdir -p upload +cp -r /build/kubernetes/*.yaml upload/ + +cd upload +for FILE in $(ls *.yaml); do + sha1sum $FILE | awk '{print $1;}' > $FILE.sha1 + echo "Created upload/$FILE.sha1" +done diff --git a/filebeat/docs/running-on-kubernetes.asciidoc b/filebeat/docs/running-on-kubernetes.asciidoc new file mode 100644 index 00000000000..31e3ddfa672 --- /dev/null +++ b/filebeat/docs/running-on-kubernetes.asciidoc @@ -0,0 +1,75 @@ +[[running-on-kubernetes]] +=== Running Filebeat on Kubernetes + +Filebeat <> can be used on Kubernetes to +retrieve and ship container logs. + +ifeval::["{release-state}"=="unreleased"] + +However, version {stack-version} of {beatname_uc} has not yet been +released, so no Docker image is currently available for this version. + +endif::[] + + +[float] +==== Kubernetes deploy manifests + +By deploying Filebeat as a https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/[DaemonSet] +we ensure we get a running instance on each node of the cluster. + +Docker logs host folder (`/var/lib/docker/containers`) is mounted on the Filebeat +container. Filebeat will start a prospector for these files and start harvesting +them as they appear. + +Everything is deployed under `kube-system` namespace, you can change that by +updating the YAML file. + +To get the manifests just run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +curl -L -O https://raw.githubusercontent.com/elastic/beats/{doc-branch}/deploy/kubernetes/filebeat-kubernetes.yaml +------------------------------------------------ + +[float] +==== Settings + +Some parameters are exposed in the manifest to configure logs destination, by +default they will use an existing Elasticsearch deploy if it's present, but you +may want to change that behavior, so just edit the YAML file and modify them: + +["source", "yaml", subs="attributes"] +------------------------------------------------ +- name: ELASTICSEARCH_HOST + value: elasticsearch +- name: ELASTICSEARCH_PORT + value: "9200" +- name: ELASTICSEARCH_USERNAME + value: elastic +- name: ELASTICSEARCH_PASSWORD + value: changeme +------------------------------------------------ + +[float] +==== Deploy + +To deploy Filebeat to Kubernetes just run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +kubectl deploy -f kubernetes/filebeat-kubernetes.yaml +------------------------------------------------ + +Then you should be able to check the status by running: + +["source", "sh", subs="attributes"] +------------------------------------------------ +$ kubectl --namespace=kube-system get ds/filebeat + +NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE-SELECTOR AGE +filebeat 32 32 0 32 0 1m +------------------------------------------------ + +Logs should start flowing to Elasticsearch, all annotated with <> +processor. diff --git a/filebeat/docs/setting-up-running.asciidoc b/filebeat/docs/setting-up-running.asciidoc index 757d463de66..8e001b87cee 100644 --- a/filebeat/docs/setting-up-running.asciidoc +++ b/filebeat/docs/setting-up-running.asciidoc @@ -20,6 +20,8 @@ This section includes additional information on how to set up and run * <> +* <> + //MAINTAINERS: If you add a new file to this section, make sure you update the bulletedl ist ^^ too. @@ -28,3 +30,5 @@ include::../../libbeat/docs/shared-directory-layout.asciidoc[] include::../../libbeat/docs/command-reference.asciidoc[] include::./running-on-docker.asciidoc[] + +include::./running-on-kubernetes.asciidoc[] diff --git a/metricbeat/docs/running-on-kubernetes.asciidoc b/metricbeat/docs/running-on-kubernetes.asciidoc new file mode 100644 index 00000000000..3b27e0ebd7b --- /dev/null +++ b/metricbeat/docs/running-on-kubernetes.asciidoc @@ -0,0 +1,86 @@ +[[running-on-kubernetes]] +=== Running Metricbeat on Kubernetes + +Metricbeat <> can be used on Kubernetes to +retrieve cluster metrics. + +ifeval::["{release-state}"=="unreleased"] + +However, version {stack-version} of {beatname_uc} has not yet been +released, so no Docker image is currently available for this version. + +endif::[] + + +[float] +==== Kubernetes deploy manifests + +Metricbeat is deployed in two different ways at the same time: + +By deploying Metricbeat as a https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/[DaemonSet] +we ensure we get a running instance on each node of the cluster. It will be used +to retrieve most metrics from the host, like system metrics, Docker stats and +metrics from all the services running on top of Kubernetes. + +A single Metricbeat instance is also created using a https://kubernetes.io/docs/concepts/workloads/controllers/Deployment/[Deployment]. +It will retrieve metrics that are unique for the whole cluster, like +Kubernetes events or https://github.com/kubernetes/kube-state-metrics[kube-state-metrics]. + +Everything is deployed under `kube-system` namespace, you can change that by +updating the YAML file. + +To get the manifests just run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +curl -L -O https://raw.githubusercontent.com/elastic/beats/{doc-branch}/deploy/kubernetes/metricbeat-kubernetes.yaml +------------------------------------------------ + +[float] +==== Settings + +Some parameters are exposed in the manifest to configure logs destination, by +default they will use an existing Elasticsearch deploy if it's present, but you +may want to change that behavior, so just edit the YAML file and modify them: + +["source", "yaml", subs="attributes"] +------------------------------------------------ +- name: ELASTICSEARCH_HOST + value: elasticsearch +- name: ELASTICSEARCH_PORT + value: "9200" +- name: ELASTICSEARCH_USERNAME + value: elastic +- name: ELASTICSEARCH_PASSWORD + value: changeme +------------------------------------------------ + +[float] +==== Deploy + +Metricbeat gets some metrics from https://github.com/kubernetes/kube-state-metrics#usage[kube-state-metrics], +you will need to deploy it if it's not already running. + +To deploy Metricbeat to Kubernetes just run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +kubectl deploy -f kubernetes/metricbeat-kubernetes.yaml +------------------------------------------------ + +Then you should be able to check the status by running: + +["source", "sh", subs="attributes"] +------------------------------------------------ +$ kubectl --namespace=kube-system get ds/metricbeat + +NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE-SELECTOR AGE +metricbeat 32 32 0 32 0 1m + +$ kubectl --namespace=kube-system get deploy/metricbeat + +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +metricbeat 1 1 1 1 1m +------------------------------------------------ + +Metrics should start flowing to Elasticsearch. diff --git a/metricbeat/docs/setting-up-running.asciidoc b/metricbeat/docs/setting-up-running.asciidoc index 388cd8c5d88..d9526cb4ecd 100644 --- a/metricbeat/docs/setting-up-running.asciidoc +++ b/metricbeat/docs/setting-up-running.asciidoc @@ -20,6 +20,8 @@ This section includes additional information on how to set up and run * <> +* <> + //MAINTAINERS: If you add a new file to this section, make sure you update the bulletedl ist ^^ too. include::../../libbeat/docs/shared-directory-layout.asciidoc[] @@ -27,3 +29,5 @@ include::../../libbeat/docs/shared-directory-layout.asciidoc[] include::../../libbeat/docs/command-reference.asciidoc[] include::./running-on-docker.asciidoc[] + +include::./running-on-kubernetes.asciidoc[]