From d8f209297ce2ee044a88dae26286c66f0688317f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 18 Oct 2017 18:11:51 +0200 Subject: [PATCH] Add kubernetes manifests (#5349) * Add kubernetes manifests Import https://github.com/elastic/beats-kubernetes, these manifests deploy filebeat & metricbeat to Kubernetes * Add travis tests * Add packaging * Add docs * Update CHANGELOG * Fix Makefile * Check k8s health every second --- .travis.yml | 21 +++++ CHANGELOG.asciidoc | 2 + Makefile | 19 +++- deploy/kubernetes/.gitignore | 1 + deploy/kubernetes/.travis/setup.sh | 66 ++++++++++++++ deploy/kubernetes/Makefile | 22 +++++ deploy/kubernetes/README.md | 11 +++ deploy/kubernetes/filebeat/README.md | 35 ++++++++ .../filebeat/filebeat-configmap.yaml | 49 +++++++++++ .../filebeat/filebeat-daemonset.yaml | 72 ++++++++++++++++ .../filebeat/filebeat-role-binding.yaml | 12 +++ deploy/kubernetes/filebeat/filebeat-role.yaml | 15 ++++ .../filebeat/filebeat-service-account.yaml | 7 ++ deploy/kubernetes/metricbeat/README.md | 31 +++++++ .../metricbeat-daemonset-configmap.yaml | 70 +++++++++++++++ .../metricbeat/metricbeat-daemonset.yaml | 84 ++++++++++++++++++ .../metricbeat-deployment-configmap.yaml | 22 +++++ .../metricbeat/metricbeat-deployment.yaml | 61 +++++++++++++ .../metricbeat/metricbeat-role-binding.yaml | 12 +++ .../metricbeat/metricbeat-role.yaml | 16 ++++ .../metricbeat-service-account.yaml | 7 ++ dev-tools/packer/Makefile | 5 ++ .../packer/platforms/kubernetes/build.sh | 18 ++++ .../packer/platforms/kubernetes/run.sh.j2 | 14 +++ filebeat/docs/running-on-kubernetes.asciidoc | 75 ++++++++++++++++ filebeat/docs/setting-up-running.asciidoc | 4 + .../docs/running-on-kubernetes.asciidoc | 86 +++++++++++++++++++ metricbeat/docs/setting-up-running.asciidoc | 4 + 28 files changed, 838 insertions(+), 3 deletions(-) create mode 100644 deploy/kubernetes/.gitignore create mode 100755 deploy/kubernetes/.travis/setup.sh create mode 100644 deploy/kubernetes/Makefile create mode 100644 deploy/kubernetes/README.md create mode 100644 deploy/kubernetes/filebeat/README.md create mode 100644 deploy/kubernetes/filebeat/filebeat-configmap.yaml create mode 100644 deploy/kubernetes/filebeat/filebeat-daemonset.yaml create mode 100644 deploy/kubernetes/filebeat/filebeat-role-binding.yaml create mode 100644 deploy/kubernetes/filebeat/filebeat-role.yaml create mode 100644 deploy/kubernetes/filebeat/filebeat-service-account.yaml create mode 100644 deploy/kubernetes/metricbeat/README.md create mode 100644 deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml create mode 100644 deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml create mode 100644 deploy/kubernetes/metricbeat/metricbeat-deployment-configmap.yaml create mode 100644 deploy/kubernetes/metricbeat/metricbeat-deployment.yaml create mode 100644 deploy/kubernetes/metricbeat/metricbeat-role-binding.yaml create mode 100644 deploy/kubernetes/metricbeat/metricbeat-role.yaml create mode 100644 deploy/kubernetes/metricbeat/metricbeat-service-account.yaml create mode 100755 dev-tools/packer/platforms/kubernetes/build.sh create mode 100644 dev-tools/packer/platforms/kubernetes/run.sh.j2 create mode 100644 filebeat/docs/running-on-kubernetes.asciidoc create mode 100644 metricbeat/docs/running-on-kubernetes.asciidoc diff --git a/.travis.yml b/.travis.yml index 0a4baf85bfb..710fb90856b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ env: - GOX_FLAGS="-arch amd64" - DOCKER_COMPOSE_VERSION=1.11.1 - GO_VERSION="$(cat .go-version)" + - TRAVIS_ETCD_VERSION=v3.2.8 jobs: include: @@ -95,6 +96,26 @@ jobs: go: $GO_VERSION stage: test + # Kubernetes + - os: linux + install: deploy/kubernetes/.travis/setup.sh + env: + - TARGETS="-C deploy/kubernetes test" + - TRAVIS_KUBE_VERSION=v1.6.11 + stage: test + - os: linux + install: deploy/kubernetes/.travis/setup.sh + env: + - TARGETS="-C deploy/kubernetes test" + - TRAVIS_KUBE_VERSION=v1.7.7 + stage: test + - os: linux + install: deploy/kubernetes/.travis/setup.sh + env: + - TARGETS="-C deploy/kubernetes test" + - TRAVIS_KUBE_VERSION=v1.8.0 + stage: test + addons: apt: packages: diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 0cc6d472ec4..5d981054601 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -85,6 +85,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Add support for `/var/log/containers/` log path in `add_kubernetes_metadata` processor. {pull}4981[4981] - Remove error log from runnerfactory as error is returned by API. {pull}5085[5085] - Remove error log from runnerfactory as error is returned by API. {pull}5085[5085] +- Add Kubernetes manifests to deploy Filebeat. {pull}5349[5349] *Heartbeat* @@ -97,6 +98,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Add experimental `queue` metricset to RabbitMQ module. {pull}4788[4788] - Add additional php-fpm pool status kpis for Metricbeat module {pull}5287[5287] - Auto-select a hostname (based on the host on which the Beat is running) in the Host Overview dashboard. {pull}5340[5340] +- Add Kubernetes manifests to deploy Metricbeat. {pull}5349[5349] - Add etcd module. {issue}4970[4970] *Packetbeat* diff --git a/Makefile b/Makefile index 60bc0a45ff3..8b143679f5c 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ REVIEWDOG_REPO=github.com/haya14busa/reviewdog/cmd/reviewdog # Runs complete testsuites (unit, system, integration) for all beats with coverage and race detection. # Also it builds the docs and the generators -.PHONY: setup-commit-hook +.PHONY: setup-commit-hook setup-commit-hook: @cp script/pre_commit.sh .git/hooks/pre-commit @chmod 751 .git/hooks/pre-commit @@ -105,7 +105,7 @@ docs: sh ./script/build_docs.sh dev-guide github.com/elastic/beats/docs/devguide ${BUILD_DIR} .PHONY: package -package: update beats-dashboards +package: update beats-dashboards kubernetes-manifests @$(foreach var,$(BEATS),SNAPSHOT=$(SNAPSHOT) $(MAKE) -C $(var) package || exit 1;) @echo "Start building the dashboards package" @@ -113,10 +113,16 @@ package: update beats-dashboards @BUILD_DIR=${BUILD_DIR} SNAPSHOT=$(SNAPSHOT) $(MAKE) -C dev-tools/packer package-dashboards ${BUILD_DIR}/upload/build_id.txt @mv build/upload build/dashboards-upload + @echo "Start building kubernetes manifests" + @mkdir -p build/upload/ + @BUILD_DIR=${BUILD_DIR} SNAPSHOT=$(SNAPSHOT) $(MAKE) -C dev-tools/packer package-kubernetes ${BUILD_DIR}/upload/build_id.txt + @mv build/upload build/kubernetes-upload + @# Copy build files over to top build directory @mkdir -p build/upload/ @$(foreach var,$(BEATS),cp -r $(var)/build/upload/ build/upload/$(var) || exit 1;) @cp -r build/dashboards-upload build/upload/dashboards + @cp -r build/kubernetes-upload build/upload/kubernetes @# Run tests on the generated packages. @go test ./dev-tools/package_test.go -files "${BUILD_DIR}/upload/*/*" @@ -149,6 +155,13 @@ python-env: @$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8 six # Tests if apm works with the current code -.PHONY: python-env +.PHONY: test-apm test-apm: sh ./script/test_apm.sh + +# Build kubernetes manifests +.PHONY: kubernetes-manifests +kubernetes-manifests: + @mkdir -p build/kubernetes + $(MAKE) -C deploy/kubernetes all + cp deploy/kubernetes/*.yaml build/kubernetes diff --git a/deploy/kubernetes/.gitignore b/deploy/kubernetes/.gitignore new file mode 100644 index 00000000000..9ee6009332a --- /dev/null +++ b/deploy/kubernetes/.gitignore @@ -0,0 +1 @@ +*-kubernetes-*.yaml diff --git a/deploy/kubernetes/.travis/setup.sh b/deploy/kubernetes/.travis/setup.sh new file mode 100755 index 00000000000..921facc373f --- /dev/null +++ b/deploy/kubernetes/.travis/setup.sh @@ -0,0 +1,66 @@ +# This script assumes Docker is already installed +#!/bin/bash + +set -x + +# set docker0 to promiscuous mode +sudo ip link set docker0 promisc on + +# install etcd +wget https://github.com/coreos/etcd/releases/download/$TRAVIS_ETCD_VERSION/etcd-$TRAVIS_ETCD_VERSION-linux-amd64.tar.gz +tar xzf etcd-$TRAVIS_ETCD_VERSION-linux-amd64.tar.gz +sudo mv etcd-$TRAVIS_ETCD_VERSION-linux-amd64/etcd /usr/local/bin/etcd +rm etcd-$TRAVIS_ETCD_VERSION-linux-amd64.tar.gz +rm -rf etcd-$TRAVIS_ETCD_VERSION-linux-amd64 + +# download kubectl +wget https://storage.googleapis.com/kubernetes-release/release/$TRAVIS_KUBE_VERSION/bin/linux/amd64/kubectl +chmod +x kubectl +sudo mv kubectl /usr/local/bin/kubectl + +# download kubernetes +git clone https://github.com/kubernetes/kubernetes $HOME/kubernetes + +# install cfssl +go get -u github.com/cloudflare/cfssl/cmd/... + +pushd $HOME/kubernetes + git checkout $TRAVIS_KUBE_VERSION + kubectl config set-credentials myself --username=admin --password=admin + kubectl config set-context local --cluster=local --user=myself + kubectl config set-cluster local --server=http://localhost:8080 + kubectl config use-context local + + # start kubernetes in the background + sudo PATH=$PATH:/home/travis/.gimme/versions/go1.7.linux.amd64/bin/go \ + KUBE_ENABLE_CLUSTER_DNS=true \ + hack/local-up-cluster.sh & +popd + +# Wait until kube is up and running +TIMEOUT=0 +TIMEOUT_COUNT=800 +until $(curl --output /dev/null --silent http://localhost:8080) || [ $TIMEOUT -eq $TIMEOUT_COUNT ]; do + echo "Kube is not up yet" + let TIMEOUT=TIMEOUT+1 + sleep 1 +done + +if [ $TIMEOUT -eq $TIMEOUT_COUNT ]; then + echo "Kubernetes is not up and running" + exit 1 +fi + +echo "Kubernetes is deployed and reachable" + +# Try and sleep before issuing chown. Currently, Kubernetes is started by +# a command that is run in the background. Technically Kubernetes could be +# up and running, but those files might not exist yet as the previous command +# could create them after Kube starts successfully. +sleep 5 + +sudo chown -R $USER:$USER $HOME/.kube + +# Have seen issues where chown does not instantly change file permissions. +# When this happens the build.sh cript can have failures. +sleep 30 diff --git a/deploy/kubernetes/Makefile b/deploy/kubernetes/Makefile new file mode 100644 index 00000000000..e46ab1d4f01 --- /dev/null +++ b/deploy/kubernetes/Makefile @@ -0,0 +1,22 @@ +ALL=filebeat metricbeat +BEAT_VERSION=$(shell head -n 1 ../../libbeat/docs/version.asciidoc | cut -c 17- ) + +all: ${ALL:=-kubernetes.yaml} + +test: all + ls *-kubernetes-${BEAT_VERSION}.yaml + for FILE in $(shell ls *-kubernetes-${BEAT_VERSION}.yaml); do \ + BEAT=$$(echo $$FILE | cut -d \- -f 1); \ + kubectl create -f $$FILE; \ + done + +clean: + @for f in $(ALL); do rm -f "$$f-kubernetes-${BEAT_VERSION}.yaml"; done + +%-kubernetes.yaml: %/*.yaml + @echo "Generating $*-kubernetes-${BEAT_VERSION}.yaml" + @rm -f $*-kubernetes-${BEAT_VERSION}.yaml + @for f in $(shell ls $*/*.yaml); do \ + sed "s/%VERSION%/${BEAT_VERSION}/g" $$f >> $*-kubernetes-${BEAT_VERSION}.yaml; \ + echo --- >> $*-kubernetes-${BEAT_VERSION}.yaml; \ + done diff --git a/deploy/kubernetes/README.md b/deploy/kubernetes/README.md new file mode 100644 index 00000000000..65275dc4e45 --- /dev/null +++ b/deploy/kubernetes/README.md @@ -0,0 +1,11 @@ +# Beats Kubernetes manifests examples + +## Getting started + +This is the list of officially supported Beats, with example manifests to run +them in Kubernetes: + +Beat | Description +---- | ---- +[filebeat](filebeat) | Tails and ships logs +[metricbeat](metricbeat) | Fetches sets of metrics from the operating system and services diff --git a/deploy/kubernetes/filebeat/README.md b/deploy/kubernetes/filebeat/README.md new file mode 100644 index 00000000000..e5e8a70bf35 --- /dev/null +++ b/deploy/kubernetes/filebeat/README.md @@ -0,0 +1,35 @@ +# Filebeat + +## Ship logs from Kubernetes to Elasticsearch + +### Kubernetes DaemonSet + +By deploying filebeat as a [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) +we ensure we get a running filebeat daemon on each node of the cluster. + +Docker logs host folder (`/var/lib/docker/containers`) is mounted on the +filebeat container. Filebeat will start a prospector for these files and start +harvesting them as they appear. + +Everything is deployed under `kube-system` namespace, you can change that by +updating YAML manifests under this folder. + +### Settings + +We use official [Beats Docker images](https://github.com/elastic/beats-docker), +as they allow external files configuration, a [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) +is used for kubernetes specific settings. Check [filebeat-configmap.yaml](filebeat-configmap.yaml) +for details. + +Also, [filebeat-daemonset.yaml](filebeat-daemonset.yaml) uses a set of environment +variables to configure Elasticsearch output: + +Variable | Default | Description +-------- | ------- | ----------- +ELASTICSEARCH_HOST | elasticsearch | Elasticsearch host +ELASTICSEARCH_PORT | 9200 | Elasticsearch port +ELASTICSEARCH_USERNAME | elastic | Elasticsearch username for HTTP auth +ELASTICSEARCH_PASSWORD | changeme | Elasticsearch password + +If there is an existing `elasticsearch` service in the kubernetes cluster these +defaults will use it. diff --git a/deploy/kubernetes/filebeat/filebeat-configmap.yaml b/deploy/kubernetes/filebeat/filebeat-configmap.yaml new file mode 100644 index 00000000000..f1fc331ac8c --- /dev/null +++ b/deploy/kubernetes/filebeat/filebeat-configmap.yaml @@ -0,0 +1,49 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: kube-system + labels: + k8s-app: filebeat + kubernetes.io/cluster-service: "true" +data: + filebeat.yml: |- + filebeat.config: + prospectors: + # Mounted `filebeat-prospectors` configmap: + path: ${path.config}/prospectors.d/*.yml + # Reload prospectors configs as they change: + reload.enabled: false + modules: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + processors: + - add_cloud_metadata: + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-prospectors + namespace: kube-system + labels: + k8s-app: filebeat + kubernetes.io/cluster-service: "true" +data: + kubernetes.yml: |- + - type: log + paths: + - /var/lib/docker/containers/*/*.log + json.message_key: log + json.keys_under_root: true + processors: + - add_kubernetes_metadata: + in_cluster: true + namespace: ${POD_NAMESPACE} diff --git a/deploy/kubernetes/filebeat/filebeat-daemonset.yaml b/deploy/kubernetes/filebeat/filebeat-daemonset.yaml new file mode 100644 index 00000000000..3ec9766d646 --- /dev/null +++ b/deploy/kubernetes/filebeat/filebeat-daemonset.yaml @@ -0,0 +1,72 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: filebeat + kubernetes.io/cluster-service: "true" + spec: + serviceAccountName: filebeat + terminationGracePeriodSeconds: 30 + containers: + - name: filebeat + image: docker.elastic.co/beats/filebeat:%VERSION% + args: [ + "-c", "/etc/filebeat.yml", + "-e", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/filebeat.yml + readOnly: true + subPath: filebeat.yml + - name: prospectors + mountPath: /usr/share/filebeat/prospectors.d + readOnly: true + - name: data + mountPath: /usr/share/filebeat/data + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + volumes: + - name: config + configMap: + defaultMode: 0600 + name: filebeat-config + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: prospectors + configMap: + defaultMode: 0600 + name: filebeat-prospectors + - name: data + emptyDir: {} diff --git a/deploy/kubernetes/filebeat/filebeat-role-binding.yaml b/deploy/kubernetes/filebeat/filebeat-role-binding.yaml new file mode 100644 index 00000000000..f24259ffa6a --- /dev/null +++ b/deploy/kubernetes/filebeat/filebeat-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/filebeat/filebeat-role.yaml b/deploy/kubernetes/filebeat/filebeat-role.yaml new file mode 100644 index 00000000000..160bb9044fa --- /dev/null +++ b/deploy/kubernetes/filebeat/filebeat-role.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: filebeat + labels: + k8s-app: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + verbs: + - get + - watch + - list diff --git a/deploy/kubernetes/filebeat/filebeat-service-account.yaml b/deploy/kubernetes/filebeat/filebeat-service-account.yaml new file mode 100644 index 00000000000..e243881a1d6 --- /dev/null +++ b/deploy/kubernetes/filebeat/filebeat-service-account.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat diff --git a/deploy/kubernetes/metricbeat/README.md b/deploy/kubernetes/metricbeat/README.md new file mode 100644 index 00000000000..186807d5e70 --- /dev/null +++ b/deploy/kubernetes/metricbeat/README.md @@ -0,0 +1,31 @@ +# Metricbeat + +## Ship metrics from Kubernetes to Elasticsearch + +### Kubernetes DaemonSet + +By deploying metricbeat as a [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) +we ensure we get a running metricbeat daemon on each node of the cluster. + +Everything is deployed under `kube-system` namespace, you can change that by +updating YAML manifests under this folder. + +### Settings + +We use official [Beats Docker images](https://github.com/elastic/beats-docker), +as they allow external files configuration, a [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) +is used for kubernetes specific settings. Check [metricbeat-configmap.yaml](metricbeat-configmap.yaml) +for details. + +Also, [metricbeat-daemonset.yaml](metricbeat-daemonset.yaml) uses a set of environment +variables to configure Elasticsearch output: + +Variable | Default | Description +-------- | ------- | ----------- +ELASTICSEARCH_HOST | elasticsearch | Elasticsearch host +ELASTICSEARCH_PORT | 9200 | Elasticsearch port +ELASTICSEARCH_USERNAME | elastic | Elasticsearch username for HTTP auth +ELASTICSEARCH_PASSWORD | changeme | Elasticsearch password + +If there is an existing `elasticsearch` service in the kubernetes cluster these +defaults will use it. diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml new file mode 100644 index 00000000000..232f8153663 --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml @@ -0,0 +1,70 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-config + namespace: kube-system + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" +data: + metricbeat.yml: |- + metricbeat.config.modules: + # Mounted `metricbeat-daemonset-modules` configmap: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + processors: + - add_cloud_metadata: + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-daemonset-modules + namespace: kube-system + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" +data: + system.yml: |- + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + #- core + #- diskio + #- socket + processes: ['.*'] + process.include_top_n: + by_cpu: 5 # include top 5 processes by CPU + by_memory: 5 # include top 5 processes by memory + + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + kubernetes.yml: |- + - module: kubernetes + metricsets: + - node + - system + - pod + - container + - volume + period: 10s + hosts: ["localhost:10255"] diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml new file mode 100644 index 00000000000..e2afada445d --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml @@ -0,0 +1,84 @@ +# Deploy a Metricbeat instance per node for node metrics retrieval +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" + spec: + serviceAccountName: metricbeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + containers: + - name: metricbeat + image: docker.elastic.co/beats/metricbeat:%VERSION% + args: [ + "-c", "/etc/metricbeat.yml", + "-e", + "-system.hostfs=/hostfs", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/metricbeat.yml + readOnly: true + subPath: metricbeat.yml + - name: modules + mountPath: /usr/share/metricbeat/modules.d + readOnly: true + - name: dockersock + mountPath: /var/run/docker.sock + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: dockersock + hostPath: + path: /var/run/docker.sock + - name: config + configMap: + defaultMode: 0600 + name: metricbeat-config + - name: modules + configMap: + defaultMode: 0600 + name: metricbeat-daemonset-modules + - name: data + emptyDir: {} diff --git a/deploy/kubernetes/metricbeat/metricbeat-deployment-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-deployment-configmap.yaml new file mode 100644 index 00000000000..def9b995556 --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-deployment-configmap.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-deployment-modules + namespace: kube-system + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" +data: + # This module requires `kube-state-metrics` up and running under `kube-system` namespace + kubernetes.yml: |- + - module: kubernetes + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + # Uncomment this to get k8s events: + #- events + period: 10s + hosts: ["kube-state-metrics:8080"] diff --git a/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml b/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml new file mode 100644 index 00000000000..9c9063af78d --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml @@ -0,0 +1,61 @@ +# Deploy singleton instance in the whole cluster for some unique data sources, like kube-state-metrics +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: metricbeat + kubernetes.io/cluster-service: "true" + spec: + containers: + - name: metricbeat + image: docker.elastic.co/beats/metricbeat:%VERSION% + args: [ + "-c", "/etc/metricbeat.yml", + "-e", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/metricbeat.yml + readOnly: true + subPath: metricbeat.yml + - name: modules + mountPath: /usr/share/metricbeat/modules.d + readOnly: true + volumes: + - name: config + configMap: + defaultMode: 0600 + name: metricbeat-config + - name: modules + configMap: + defaultMode: 0600 + name: metricbeat-deployment-modules diff --git a/deploy/kubernetes/metricbeat/metricbeat-role-binding.yaml b/deploy/kubernetes/metricbeat/metricbeat-role-binding.yaml new file mode 100644 index 00000000000..8a74c8f81a1 --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: metricbeat +subjects: +- kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/metricbeat/metricbeat-role.yaml b/deploy/kubernetes/metricbeat/metricbeat-role.yaml new file mode 100644 index 00000000000..ebc9039cb54 --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-role.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: metricbeat + labels: + k8s-app: metricbeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - events + - pods + verbs: + - get + - watch + - list diff --git a/deploy/kubernetes/metricbeat/metricbeat-service-account.yaml b/deploy/kubernetes/metricbeat/metricbeat-service-account.yaml new file mode 100644 index 00000000000..2b2fd46dd31 --- /dev/null +++ b/deploy/kubernetes/metricbeat/metricbeat-service-account.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat diff --git a/dev-tools/packer/Makefile b/dev-tools/packer/Makefile index cf447425e9a..c00d4c997e4 100644 --- a/dev-tools/packer/Makefile +++ b/dev-tools/packer/Makefile @@ -39,6 +39,11 @@ package-dashboards: echo Creating the Dashboards package BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/dashboards/build.sh +.PHONY: package-kubernetes +package-kubernetes: + echo Creating the kubernetes manifests package + BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/kubernetes/build.sh + .PHONY: deps deps: go get -u github.com/tsg/gotpl diff --git a/dev-tools/packer/platforms/kubernetes/build.sh b/dev-tools/packer/platforms/kubernetes/build.sh new file mode 100755 index 00000000000..4f0d1b2f197 --- /dev/null +++ b/dev-tools/packer/platforms/kubernetes/build.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e + +BASEDIR=$(dirname "$0") +ARCHDIR=${BASEDIR}/../../ + +runid=kubernetes + +cat ${ARCHDIR}/version.yml > ${BUILD_DIR}/settings-$runid.yml +gotpl ${BASEDIR}/run.sh.j2 < ${BUILD_DIR}/settings-$runid.yml > ${BUILD_DIR}/run-$runid.sh +chmod +x ${BUILD_DIR}/run-$runid.sh + +docker run --rm -v ${BUILD_DIR}:/build \ + -e BUILDID=$BUILDID -e SNAPSHOT=$SNAPSHOT -e RUNID=$runid -e BEAT_NAME=$BEAT_NAME \ + tudorg/fpm /build/run-$runid.sh + +rm ${BUILD_DIR}/settings-$runid.yml ${BUILD_DIR}/run-$runid.sh diff --git a/dev-tools/packer/platforms/kubernetes/run.sh.j2 b/dev-tools/packer/platforms/kubernetes/run.sh.j2 new file mode 100644 index 00000000000..b781c97644d --- /dev/null +++ b/dev-tools/packer/platforms/kubernetes/run.sh.j2 @@ -0,0 +1,14 @@ +#!/bin/bash + +# this is executed in the docker fpm image +set -e +cd /build + +mkdir -p upload +cp -r /build/kubernetes/*.yaml upload/ + +cd upload +for FILE in $(ls *.yaml); do + sha1sum $FILE | awk '{print $1;}' > $FILE.sha1 + echo "Created upload/$FILE.sha1" +done diff --git a/filebeat/docs/running-on-kubernetes.asciidoc b/filebeat/docs/running-on-kubernetes.asciidoc new file mode 100644 index 00000000000..c6ca656f767 --- /dev/null +++ b/filebeat/docs/running-on-kubernetes.asciidoc @@ -0,0 +1,75 @@ +[[running-on-kubernetes]] +=== Running Filebeat on Kubernetes + +Filebeat <> can be used on Kubernetes to +retrieve and ship container logs. + +ifeval::["{release-state}"=="unreleased"] + +However, version {stack-version} of {beatname_uc} has not yet been +released, so no Docker image is currently available for this version. + +endif::[] + + +[float] +==== Kubernetes deploy manifests + +By deploying Filebeat as a https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/[DaemonSet] +we ensure we get a running instance on each node of the cluster. + +Docker logs host folder (`/var/lib/docker/containers`) is mounted on the Filebeat +container. Filebeat will start a prospector for these files and start harvesting +them as they appear. + +Everything is deployed under `kube-system` namespace, you can change that by +updating the YAML file. + +To get the manifests just run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/kubernetes/filebeat-kubernetes-{version}.yaml +------------------------------------------------ + +[float] +==== Settings + +Some parameters are exposed in the manifest to configure logs destination, by +default they will use an existing Elasticsearch deploy if it's present, but you +may want to change that behavior, so just edit the YAML file and modify them: + +["source", "yaml", subs="attributes"] +------------------------------------------------ +- name: ELASTICSEARCH_HOST + value: elasticsearch +- name: ELASTICSEARCH_PORT + value: "9200" +- name: ELASTICSEARCH_USERNAME + value: elastic +- name: ELASTICSEARCH_PASSWORD + value: changeme +------------------------------------------------ + +[float] +==== Deploy + +To deploy Filebeat to Kubernetes just run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +kubectl deploy -f kubernetes/filebeat-kubernetes-{version}.yaml +------------------------------------------------ + +Then you should be able to check the status by running: + +["source", "sh", subs="attributes"] +------------------------------------------------ +$ kubectl --namespace=kube-system get ds/filebeat + +NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE-SELECTOR AGE +filebeat 32 32 0 32 0 1m +------------------------------------------------ + +Logs should start flowing to Elasticsearch, all annotated with <> +processor. diff --git a/filebeat/docs/setting-up-running.asciidoc b/filebeat/docs/setting-up-running.asciidoc index 757d463de66..8e001b87cee 100644 --- a/filebeat/docs/setting-up-running.asciidoc +++ b/filebeat/docs/setting-up-running.asciidoc @@ -20,6 +20,8 @@ This section includes additional information on how to set up and run * <> +* <> + //MAINTAINERS: If you add a new file to this section, make sure you update the bulletedl ist ^^ too. @@ -28,3 +30,5 @@ include::../../libbeat/docs/shared-directory-layout.asciidoc[] include::../../libbeat/docs/command-reference.asciidoc[] include::./running-on-docker.asciidoc[] + +include::./running-on-kubernetes.asciidoc[] diff --git a/metricbeat/docs/running-on-kubernetes.asciidoc b/metricbeat/docs/running-on-kubernetes.asciidoc new file mode 100644 index 00000000000..912d3f5fef5 --- /dev/null +++ b/metricbeat/docs/running-on-kubernetes.asciidoc @@ -0,0 +1,86 @@ +[[running-on-kubernetes]] +=== Running Metricbeat on Kubernetes + +Metricbeat <> can be used on Kubernetes to +retrieve cluster metrics. + +ifeval::["{release-state}"=="unreleased"] + +However, version {stack-version} of {beatname_uc} has not yet been +released, so no Docker image is currently available for this version. + +endif::[] + + +[float] +==== Kubernetes deploy manifests + +Metricbeat is deployed in two different ways at the same time: + +By deploying Metricbeat as a https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/[DaemonSet] +we ensure we get a running instance on each node of the cluster. It will be used +to retrieve most metrics from the host, like system metrics, Docker stats and +metrics from all the services running on top of Kubernetes. + +A single Metricbeat instance is also created using a https://kubernetes.io/docs/concepts/workloads/controllers/Deployment/[Deployment]. +It will retrieve metrics that are unique for the whole cluster, like +Kubernetes events or https://github.com/kubernetes/kube-state-metrics[kube-state-metrics]. + +Everything is deployed under `kube-system` namespace, you can change that by +updating the YAML file. + +To get the manifests just run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/kubernetes/metricbeat-kubernetes-{version}.yaml +------------------------------------------------ + +[float] +==== Settings + +Some parameters are exposed in the manifest to configure logs destination, by +default they will use an existing Elasticsearch deploy if it's present, but you +may want to change that behavior, so just edit the YAML file and modify them: + +["source", "yaml", subs="attributes"] +------------------------------------------------ +- name: ELASTICSEARCH_HOST + value: elasticsearch +- name: ELASTICSEARCH_PORT + value: "9200" +- name: ELASTICSEARCH_USERNAME + value: elastic +- name: ELASTICSEARCH_PASSWORD + value: changeme +------------------------------------------------ + +[float] +==== Deploy + +Metricbeat gets some metrics from https://github.com/kubernetes/kube-state-metrics#usage[kube-state-metrics], +you will need to deploy it if it's not already running. + +To deploy Metricbeat to Kubernetes just run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +kubectl deploy -f kubernetes/metricbeat-kubernetes-{version}.yaml +------------------------------------------------ + +Then you should be able to check the status by running: + +["source", "sh", subs="attributes"] +------------------------------------------------ +$ kubectl --namespace=kube-system get ds/metricbeat + +NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE-SELECTOR AGE +metricbeat 32 32 0 32 0 1m + +$ kubectl --namespace=kube-system get deploy/metricbeat + +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +metricbeat 1 1 1 1 1m +------------------------------------------------ + +Metrics should start flowing to Elasticsearch. diff --git a/metricbeat/docs/setting-up-running.asciidoc b/metricbeat/docs/setting-up-running.asciidoc index 388cd8c5d88..d9526cb4ecd 100644 --- a/metricbeat/docs/setting-up-running.asciidoc +++ b/metricbeat/docs/setting-up-running.asciidoc @@ -20,6 +20,8 @@ This section includes additional information on how to set up and run * <> +* <> + //MAINTAINERS: If you add a new file to this section, make sure you update the bulletedl ist ^^ too. include::../../libbeat/docs/shared-directory-layout.asciidoc[] @@ -27,3 +29,5 @@ include::../../libbeat/docs/shared-directory-layout.asciidoc[] include::../../libbeat/docs/command-reference.asciidoc[] include::./running-on-docker.asciidoc[] + +include::./running-on-kubernetes.asciidoc[]