Automatic merge from master to 6.x branch

.gitignore

@@ -15,6 +15,7 @@
 *.generated.yml
 coverage.out
 .python-version
+beat.db
 
 # Editor swap files
 *.swp

.travis.yml

@@ -14,6 +14,7 @@ env:
     - GOX_FLAGS="-arch amd64"
     - DOCKER_COMPOSE_VERSION=1.11.1
     - GO_VERSION="$(cat .go-version)"
+    - TRAVIS_ETCD_VERSION=v3.2.8
 
 jobs:
   include:
@@ -95,6 +96,26 @@ jobs:
       go: $GO_VERSION
       stage: test
 
+    # Kubernetes
+    - os: linux
+      install: deploy/kubernetes/.travis/setup.sh
+      env:
+        - TARGETS="-C deploy/kubernetes test"
+        - TRAVIS_KUBE_VERSION=v1.6.11
+      stage: test
+    - os: linux
+      install: deploy/kubernetes/.travis/setup.sh
+      env:
+        - TARGETS="-C deploy/kubernetes test"
+        - TRAVIS_KUBE_VERSION=v1.7.7
+      stage: test
+    - os: linux
+      install: deploy/kubernetes/.travis/setup.sh
+      env:
+        - TARGETS="-C deploy/kubernetes test"
+        - TRAVIS_KUBE_VERSION=v1.8.0
+      stage: test
+
 addons:
   apt:
     packages:

CHANGELOG.asciidoc

@@ -14,6 +14,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 
 *Affecting all Beats*
 
+- Determine log level for kafka output. {pull}5397[5397]
+
 *Auditbeat*
 
 *Filebeat*
@@ -22,6 +24,9 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 
 *Metricbeat*
 
+- Rename `heap_init` field to `heap.init`. {pull}5320[5320]
+
+
 *Packetbeat*
 
 - Remove not-working `runoptions.uid` and `runoptions.gid` options in Packetbeat. {pull}5261[5261]
@@ -35,12 +40,15 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Fix data race accessing watched containers. {issue}5147[5147]
 - Remove ID() from Runner interface {issue}5153[5153]
 - Do not require template if index change and template disabled {pull}5319[5319]
+- Correctly send configured `Host` header to the remote server. {issue}4842[4842]
 
 *Auditbeat*
 
 *Filebeat*
 
 - Fix default paths for redis 4.0.1 logs on macOS {pull}5173[5173]
+- Fix Filebeat not starting if command line and modules configs are used together. {issue}5376[5376]
+- Add support for adding string tags {pull}5395{5395}
 
 *Heartbeat*
 
@@ -49,9 +57,12 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Change field type of http header from nested to object {pull}5258[5258]
 - Use `beat.name` instead of `beat.hostname` in the Host Overview dashboard. {pull}5340[5340]
 - Fix the loading of 5.x dashboards. {issue}5277[5277]
+- Fix the fetching of process information when some data is missing under MacOS X. {issue}5337[5337]
 
 *Packetbeat*
 
+- Fix http status phrase parsing not allow spaces. {pull}5312[5312]
+
 *Winlogbeat*
 
 ==== Added
@@ -64,16 +75,20 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Fix reloader error message to only print on actual error {pull}5066[5066]
 - Add support for enabling TLS renegotiation. {issue}4386[4386]
 - Add Azure VM support for add_cloud_metadata processor {pull}5355[5355]
+- Add `output.file.permission` config option. {pull}4638[4638]
 
 *Auditbeat*
 
+- Add support for SHA3 hash algorithms to the file integrity module. {issue}5345[5345]
+
 *Filebeat*
 
 - Add PostgreSQL module with slowlog support. {pull}4763[4763]
 - Add Kafka log module. {pull}4885[4885]
 - Add support for `/var/log/containers/` log path in `add_kubernetes_metadata` processor. {pull}4981[4981]
 - Remove error log from runnerfactory as error is returned by API. {pull}5085[5085]
 - Remove error log from runnerfactory as error is returned by API. {pull}5085[5085]
+- Add Kubernetes manifests to deploy Filebeat. {pull}5349[5349]
 
 *Heartbeat*
 
@@ -86,6 +101,9 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Add experimental `queue` metricset to RabbitMQ module. {pull}4788[4788]
 - Add additional php-fpm pool status kpis for Metricbeat module {pull}5287[5287]
 - Auto-select a hostname (based on the host on which the Beat is running) in the Host Overview dashboard. {pull}5340[5340]
+- Add Kubernetes manifests to deploy Metricbeat. {pull}5349[5349]
+- Add etcd module. {issue}4970[4970]
+- Add ip address of docker containers to event. {pull}5379[5379]
 
 *Packetbeat*
 

Makefile

@@ -16,7 +16,7 @@ REVIEWDOG_REPO=github.com/haya14busa/reviewdog/cmd/reviewdog
 # Runs complete testsuites (unit, system, integration) for all beats with coverage and race detection.
 # Also it builds the docs and the generators
 
-.PHONY: setup-commit-hook 
+.PHONY: setup-commit-hook
 setup-commit-hook:
 	@cp script/pre_commit.sh .git/hooks/pre-commit
 	@chmod 751 .git/hooks/pre-commit
@@ -50,6 +50,7 @@ coverage-report:
 .PHONY: update
 update: notice
 	@$(foreach var,$(PROJECTS),$(MAKE) -C $(var) update || exit 1;)
+	@$(MAKE) -C deploy/kubernetes all
 
 .PHONY: clean
 clean:
@@ -105,18 +106,24 @@ docs:
 	sh ./script/build_docs.sh dev-guide github.com/elastic/beats/docs/devguide ${BUILD_DIR}
 
 .PHONY: package
-package: update beats-dashboards
+package: update beats-dashboards kubernetes-manifests
 	@$(foreach var,$(BEATS),SNAPSHOT=$(SNAPSHOT) $(MAKE) -C $(var) package || exit 1;)
 
 	@echo "Start building the dashboards package"
 	@mkdir -p build/upload/
 	@BUILD_DIR=${BUILD_DIR} SNAPSHOT=$(SNAPSHOT) $(MAKE) -C dev-tools/packer package-dashboards ${BUILD_DIR}/upload/build_id.txt
 	@mv build/upload build/dashboards-upload
 
+	@echo "Start building kubernetes manifests"
+	@mkdir -p build/upload/
+	@BUILD_DIR=${BUILD_DIR} SNAPSHOT=$(SNAPSHOT) $(MAKE) -C dev-tools/packer package-kubernetes ${BUILD_DIR}/upload/build_id.txt
+	@mv build/upload build/kubernetes-upload
+
 	@# Copy build files over to top build directory
 	@mkdir -p build/upload/
 	@$(foreach var,$(BEATS),cp -r $(var)/build/upload/ build/upload/$(var)  || exit 1;)
 	@cp -r build/dashboards-upload build/upload/dashboards
+	@cp -r build/kubernetes-upload build/upload/kubernetes
 	@# Run tests on the generated packages.
 	@go test ./dev-tools/package_test.go -files "${BUILD_DIR}/upload/*/*"
 
@@ -149,6 +156,13 @@ python-env:
 	@$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8 six
 
 # Tests if apm works with the current code
-.PHONY: python-env
+.PHONY: test-apm
 test-apm:
 	sh ./script/test_apm.sh
+
+# Build kubernetes manifests
+.PHONY: kubernetes-manifests
+kubernetes-manifests:
+	@mkdir -p build/kubernetes
+	$(MAKE) -C deploy/kubernetes all
+	cp deploy/kubernetes/*.yaml build/kubernetes