Upgrade to Python3

.ci/docker/golang-mage/Dockerfile

@@ -19,8 +19,7 @@ RUN git clone https://github.com/magefile/mage \
   && go get github.com/elastic/apm-server/vendor/golang.org/x/tools/cmd/goimports
 
 RUN apt-get update -y -qq \
-  && apt-get install -y -qq  python-pip \
-  && rm -rf /var/lib/apt/lists/* \
-  && pip install virtualenv
+  && apt-get install -y -qq  python3 python3-pip python3-venv \
+  && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /go

Dockerfile

@@ -4,10 +4,13 @@ MAINTAINER Nicolas Ruflin <[email protected]>
 RUN set -x && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
-         netcat python-pip virtualenv && \
+         netcat python3 python3-pip python3-venv && \
     apt-get clean
 
-RUN pip install --upgrade setuptools
+ENV PYTHON_ENV=/tmp/python-env
+
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade setuptools
 
 # Setup work environment
 ENV APM_SERVER_PATH /go/src/github.com/elastic/apm-server

Makefile

@@ -44,7 +44,7 @@ ES_SUPERUSER_PASS?=changeme
 
 # updates beats updates the framework part and go parts of beats
 .PHONY: update-beats
-update-beats: govendor
+update-beats: python-env govendor
 	rm -rf vendor/github.com/elastic/beats
 	@govendor fetch github.com/elastic/beats/...@$(BEATS_VERSION)
 	@govendor fetch github.com/elastic/beats/libbeat/generator/fields@$(BEATS_VERSION)
@@ -140,7 +140,7 @@ staticcheck:
 check-deps: test-deps golint staticcheck
 
 .PHONY: check-full
-check-full: check-deps check
+check-full: python-env check-deps check
 	@# Validate that all updates were committed
 	@$(MAKE) update
 	@$(MAKE) check
@@ -164,7 +164,7 @@ apm-docs:  ## @build Builds the documentation for APM Server and APM Overview
 
 
 .PHONY: update-beats-docs
-update-beats-docs:
+update-beats-docs: python-env
 	@python script/copy-docs.py
 	@$(MAKE) docs
 
@@ -198,7 +198,7 @@ import-dashboards:
 	echo "APM loads dashboards via Kibana, not the APM Server"
 
 .PHONY: check-changelogs
-check-changelogs: ## @testing Checks the changelogs for certain branches.
+check-changelogs: python-env ## @testing Checks the changelogs for certain branches.
 	@python script/check_changelogs.py
 
 .PHONY: rm-empty-folders

NOTICE.txt

@@ -587,7 +587,7 @@ SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats
 Version: master
-Revision: 990a5f76a3dc435bf783c1f44d9363abac211a0e
+Revision: 02ada23de4ec9d5c9a4e977c4a772f8cb619cfaf
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/beats/LICENSE.txt:
 --------------------------------------------------------------------
@@ -698,7 +698,7 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: github.com/elastic/go-ucfg
-Revision: 093a6898c440d2e5e93abf09850068c60428b2cd
+Revision: ab69586e10820006b250d04c98cc3b848e227808
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/go-ucfg/LICENSE:
 --------------------------------------------------------------------

README.md

@@ -41,13 +41,12 @@ in the same directory with the name apm-server.
 make
 ```
 
-You also need to create all files needed by the APM Server by running the additional command below. 
+You also need to create all files needed by the APM Server by running the additional command below.
 
 ```
 make update
 ```
-Note that this requires to have `virtualenv` installed. Make sure that `virtualenv` creates Python2
-environment, otherwise specify `PYTHON_EXE` variable pointed at Python2 executable.
+Note that this requires to have `python >= 3.7` and `venv` installed.
 
 ### Run
 
@@ -97,7 +96,7 @@ See [releases](RELEASES.md) for an APM Server release checklist.
 
 ## Update Dependencies
 
-The `apm-server` has two types of dependencies, 
+The `apm-server` has two types of dependencies,
 the Golang packages managed with *Govendor* and a dependency to the *Beats Framework*.
 
 ### Govendor
@@ -127,9 +126,9 @@ BEATS_VERSION=f240148065af94d55c5149e444482b9635801f27 make update-beats
 ```
 ### Go-elasticsearch client Update
 
-It is important to keep the used [go-elasticsearch client](https://github.com/elastic/go-elasticsearch) in sync with the according major version. 
+It is important to keep the used [go-elasticsearch client](https://github.com/elastic/go-elasticsearch) in sync with the according major version.
 We also recommend to use the latest available client for minor versions.
-Since APM Server does not yet support go modules, you can update the dependency using govendor, e.g. by running: 
+Since APM Server does not yet support go modules, you can update the dependency using govendor, e.g. by running:
 ```
 git clone --branch v7.4.1 https://github.com/elastic/go-elasticsearch.git $GOPATH/src/github.com/elastic/go-elasticsearch/v7
 govendor add github.com/elastic/go-elasticsearch/v7/^

TESTING.md

@@ -22,7 +22,7 @@ While developing new tests or troubleshooting test failures, it is handy to run
 example from within an editor, while still allowing all dependencies to run in containers.  To accomplish this:
 
 * Run `make build-image start-environment` to start docker containers for the Elastic Stack.
-* Run `PYTHON_EXE=python2.7 make python-env` to build a python virtualenv
+* Run `PYTHON_EXE=python3 make python-env` to build a python virtualenv
 * Run `make apm-server.test` to (re)build the executable used to run the apm-server
 * Run tests using the `run-system-tests` target, eg:
  ```
@@ -39,9 +39,9 @@ For insights about test-coverage, run `make coverage-report`. The test coverage
 ## Snapshot-Testing
 Some tests make use of the concept of _snapshot_ or _approvals testing_. If running tests leads to changed snapshots, you can use the `approvals` tool to update the snapshots.
 Following workflow is intended:
-* Run `make update` to create the `approvals` binary that supports reviewing changes. 
+* Run `make update` to create the `approvals` binary that supports reviewing changes.
 * Run `make unit` to create a `*.received.json` file for every newly created or changed snapshot.
-* Run `./approvals` to review and interactively accept the changes. 
+* Run `./approvals` to review and interactively accept the changes.
 
 ## Benchmarking
 
@@ -55,7 +55,7 @@ A good way to present your results is by using `benchcmp`.
 With your changes in the current working tree, do:
 
 ```
-$ go get -u golang.org/x/tools/cmd/benchcmp 
+$ go get -u golang.org/x/tools/cmd/benchcmp
 $ make bench > new.txt
 $ git checkout master
 $ make bench > old.txt

_beats/.go-version

@@ -1 +1 @@
-1.13.7
+1.13.8

_beats/CHANGELOG-developer.next.asciidoc

@@ -28,6 +28,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Altered all remaining uses of mapval to use the renamed and enhanced version: https://github.com/elastic/go-lookslike[go-lookslike] instead, which is a separate project. The mapval tree is now gone. {pull}14165[14165]
 - Move light modules to OSS. {pull}14369[14369]
 - Deprecate test flags, `generate` and `update_expected`, in favor of `data`. {pull}15292[15292]
+- Python 3 is required now to run python tests and tools. {pull}14798[14798]
 
 ==== Bugfixes
 

_beats/CHANGELOG.asciidoc

@@ -128,6 +128,7 @@ processing events. (CVE-2019-17596) See https://www.elastic.co/community/securit
 - Libbeat HTTP's Server can listen to a unix socket using the `unix:///tmp/hello.sock` syntax. {pull}13655[13655]
 - Libbeat HTTP's Server can listen to a Windows named pipe using the `npipe:///hello` syntax. {pull}13655[13655]
 - Adding new `Enterprise` license type to the licenser. {issue}14246[14246]
+- Add endpoint config in AWS config to support using custom endpoint accessing AWS APIs. {issue}16245[16245] {pull}16263[16263]
 
 *Auditbeat*
 

_beats/CHANGELOG.next.asciidoc

@@ -24,10 +24,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Journalbeat*
 
+- Improve parsing of syslog.pid in journalbeat to strip the username when present {pull}16116[16116]
+
 
 *Metricbeat*
 
 - Make use of secure port when accessing Kubelet API {pull}16063[16063]
+- Add Tomcat overview dashboard {pull}14026[14026]
 
 *Packetbeat*
 
@@ -52,6 +55,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix issue where default go logger is not discarded when either * or stdout is selected. {issue}10251[10251] {pull}15708[15708]
 - Upgrade go-ucfg to latest v0.8.1. {pull}15937{15937}
 - Fix index names for indexing not always guaranteed to be lower case. {pull}16081[16081]
+- Add `ssl.ca_sha256` option to the supported TLS option, this allow to check that a specific certificate is used as part of the verified chain. {issue}15717[15717]
+- Fix loading processors from annotation hints. {pull}16348[16348]
 
 *Auditbeat*
 
@@ -83,6 +88,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change lookup_fields from metricset.host to service.address {pull}15883[15883]
 - Add dedot for cloudwatch metric name. {issue}15916[15916] {pull}15917[15917]
 - Fixed issue `logstash-xpack` module suddenly ceasing to monitor Logstash. {issue}15974[15974] {pull}16044[16044]
+- Fix skipping protocol scheme by light modules. {pull}16205[pull]
+- Made `logstash-xpack` module once again have parity with internally-collected Logstash monitoring data. {pull}16198[16198]
 
 *Packetbeat*
 
@@ -99,19 +106,27 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Affecting all Beats*
 
 - Add document_id setting to decode_json_fields processor. {pull}15859[15859]
+- Include network information by default on add_host_metadata and add_observer_metadata. {issue}15347[15347] {pull}16077[16077]
 - Add `aws_ec2` provider for autodiscover. {issue}12518[12518] {pull}14823[14823]
 
 *Auditbeat*
 
 
 *Filebeat*
+
 - Set event.outcome field based on googlecloud audit log output. {pull}15731[15731]
 - Add dashboard for AWS ELB fileset. {pull}15804[15804]
 - Add dashboard for AWS vpcflow fileset. {pull}16007[16007]
 - Add ECS tls fields to zeek:smtp,rdp,ssl and aws:s3access,elb {issue}15757[15757] {pull}15935[15936]
 - Add custom string mapping to CEF module to support Forcepoint NGFW {issue}14663[14663] {pull}15910[15910]
+- Add ingress nginx controller fileset {pull}16197[16197]
 - move create-[module,fileset,fields] to mage and enable in x-pack/filebeat {pull}15836[15836]
 - Add ECS tls and categorization fields to apache module. {issue}16032[16032] {pull}16121[16121]
+- Add MQTT input. {issue}15602[15602] {pull}16204[16204]
+- Add ECS categorization fields to activemq module. {issue}16151[16151] {pull}16201[16201]
+- Add a TLS test and more debug output to httpjson input {pull}16315[16315]
+- Add an SSL config example in config.yml for filebeat MISP module. {pull}16320[16320] 
+- Improve ECS categorization, container & process field mappings in auditd module. {issue}16153[16153] {pull}16280[16280]
 
 *Heartbeat*
 
@@ -144,6 +159,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add collecting AuroraDB metrics in rds metricset. {issue}14142[14142] {pull}16004[16004]
 - Reuse connections in SQL module. {pull}16001[16001]
 - Improve the `logstash` module (when `xpack.enabled` is set to `true`) to use the override `cluster_uuid` returned by Logstash APIs. {issue}15772[15772] {pull}15795[15795]
+- Add kubernetes storage class support via kube-state-metrics. {pull}16145[16145]
+- Add support for NATS 2.1. {pull}16317[16317]
+- Add Load Balancing metricset to GCP {pull}15559[15559]
+- Add support for Dropwizard metrics 4.1. {pull}16332[16332]
+- Improve the `haproxy` module to support metrics exposed via HTTPS. {issue}14579[14579] {pull}16333[16333]
 
 *Packetbeat*
 

_beats/dev-tools/aggregate_coverage.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """Simple script to concatenate coverage reports.
 """

_beats/dev-tools/cherrypick_pr

@@ -1,6 +1,8 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """Cherry pick and backport a PR"""
+from __future__ import print_function
 
+from builtins import input
 import sys
 import os
 import argparse
@@ -45,7 +47,7 @@ def main():
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog=usage)
     parser.add_argument("to_branch",
-                        help="To branch (e.g 5.0)")
+                        help="To branch (e.g 7.x)")
     parser.add_argument("pr_number",
                         help="The PR number being merged (e.g. 2345)")
     parser.add_argument("commit_hashes", metavar="hash", nargs="+",
@@ -75,7 +77,7 @@ def main():
     tmp_branch = "backport_{}_{}".format(args.pr_number, args.to_branch)
 
     if not vars(args)["continue"]:
-        if not args.yes and raw_input("This will destroy all local changes. " +
+        if not args.yes and input("This will destroy all local changes. " +
                                       "Continue? [y/n]: ") != "y":
             return 1
         check_call("git reset --hard", shell=True)
@@ -110,15 +112,16 @@ def main():
 
     if args.diff:
         call("git diff {}".format(args.to_branch), shell=True)
-        if raw_input("Continue? [y/n]: ") != "y":
+        if input("Continue? [y/n]: ") != "y":
             print("Aborting cherry-pick.")
             return 1
 
     print("Ready to push branch.")
 
     remote = args.remote
     if not remote:
-        remote = raw_input("To which remote should I push? (your fork): ")
+        remote = input("To which remote should I push? (your fork): ")
+
     call("git push {} :{} > /dev/null".format(remote, tmp_branch),
          shell=True)
     check_call("git push --set-upstream {} {}"
@@ -138,15 +141,15 @@ def main():
         # get the github username from the remote where we pushed
         remote_url = check_output("git remote get-url {}".format(remote),
                                   shell=True)
-        remote_user = re.search("github.com[:/](.+)/beats", remote_url).group(1)
+        remote_user = re.search("github.com[:/](.+)/beats", str(remote_url)).group(1)
 
         # create PR
         request = session.post(base + "/pulls", json=dict(
-            title="Cherry-pick #{} to {}: {}".format(args.pr_number, args.to_branch, original_pr["title"].encode('utf-8')),
+            title="Cherry-pick #{} to {}: {}".format(args.pr_number, args.to_branch, original_pr["title"]),
             head=remote_user + ":" + tmp_branch,
             base=args.to_branch,
             body="Cherry-pick of PR #{} to {} branch. Original message: \n\n{}"
-            .format(args.pr_number, args.to_branch, original_pr["body"].encode('utf-8'))
+            .format(args.pr_number, args.to_branch, original_pr["body"])
         ))
         if request.status_code > 299:
             print("Creating PR failed: {}".format(request.json()))

_beats/dev-tools/deploy

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import os
 import argparse
 from subprocess import check_call