Migrate scripts to python 3 (#14798)

Several changes in test and tools code to migrate to Python 3 and stop supporting Python 2. From now on, Python 3 is required to run tests and many make and mage targets. Switch over to `python3 -m venv` instead of `virtualenv`. It's the recommended way to create virtual Python environments in 3.4 and above. Remove any direct references to python to always allow a specific python binary or version to be used with the PYTHON_EXE environmet variable. Use python3 in the shebang for python scripts. python is going to be reserved for python2 in Ubuntu, and PEP394 mentions that python should be used in the shebang line only for scripts that are source compatible with both Python 2 and 3. Update documentation to reflect these changes and add information about the use of Python for beats development. Prepare the tests for checks based on the local ip, this IP can be [::1] if ipv4 is not available, or 127.0.1.1 in some machines, as in the latest Ubuntu images used in Travis. Change checks based on sys.platform, as the values for linux have changed in Python 3. Several assertions in tests have been adapted to its newer versions. Some flaky tests that are more flaky with Python 3 have been fixed or skipped. Replace use of nosetests with mage wrapper in script for Windows CI workers. Mage manages their own python virtual environments, this is preferred to directly call python commands. Co-authored-by: Noémi Ványi <[email protected]> Co-authored-by: Benjamin Wohlwend <[email protected]> Co-authored-by: Michael Madden <[email protected]> Co-authored-by: Andrew Kroh <[email protected]>
elastic · Feb 17, 2020 · 9375fb9 · 9375fb9
1 parent 32cc6da
commit 9375fb9
Show file tree

Hide file tree

Showing 169 changed files with 1,020 additions and 820 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -166,12 +166,6 @@ jobs:
       go: $TRAVIS_GO_VERSION
       stage: test
 
-    # Docs
-    - os: linux
-      env: TARGETS="docs"
-      go: $TRAVIS_GO_VERSION
-      stage: test
-
     # Kubernetes
     - os: linux
       install: deploy/kubernetes/.travis/setup.sh
@@ -199,7 +193,6 @@ jobs:
             retries: true
           update: true
           packages:
-            - python-virtualenv
             - libpcap-dev
             - xsltproc
             - libxml2-utils
@@ -218,16 +211,22 @@ addons:
     config:
       retries: true
     update: true
+    sources:
+      - deadsnakes
     packages:
-      - python-virtualenv
       - libpcap-dev
       - xsltproc
       - libxml2-utils
       - libsystemd-journal-dev
       - librpm-dev
+      # From deadsnakes PPA
+      - python3.6
+      - python3.6-venv
 
 before_install:
+  - if [ x$TRAVIS_DIST = xtrusty ]; then sudo ln -sf python3.6 /usr/bin/python3; fi
   - python --version
+  - python3 --version
   - umask 022
   - chmod -R go-w $GOPATH/src/github.com/elastic/beats
   # Docker-compose installation

diff --git a/Makefile b/Makefile
@@ -4,7 +4,9 @@ BEATS?=auditbeat filebeat heartbeat journalbeat metricbeat packetbeat winlogbeat
 PROJECTS=libbeat $(BEATS)
 PROJECTS_ENV=libbeat filebeat metricbeat
 PYTHON_ENV?=$(BUILD_DIR)/python-env
-VIRTUALENV_PARAMS?=
+PYTHON_EXE?=python3
+PYTHON_ENV_EXE=${PYTHON_ENV}/bin/$(notdir ${PYTHON_EXE})
+VENV_PARAMS?=
 FIND=find . -type f -not -path "*/vendor/*" -not -path "*/build/*" -not -path "*/.git/*"
 GOLINT=golint
 GOLINT_REPO=golang.org/x/lint/golint
@@ -88,8 +90,8 @@ clean-vendor:
 .PHONY: check
 check: python-env
 	@$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;)
-	@# Checks also python files which are not part of the beats
-	@$(FIND) -name *.py -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
+	@$(FIND) -name *.py -name *.py -not -path "*/build/*" -not -path "*/vendor/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
+	@$(FIND) -name *.py -not -path "*/build/*" -not -path "*/vendor/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false)
 	@# Validate that all updates were committed
 	@$(MAKE) update
 	@$(MAKE) check-headers
@@ -136,13 +138,13 @@ docs:
 .PHONY: notice
 notice: python-env
 	@echo "Generating NOTICE"
-	@$(PYTHON_ENV)/bin/python dev-tools/generate_notice.py .
+	@${PYTHON_ENV_EXE} dev-tools/generate_notice.py .
 
 # Sets up the virtual python environment
 .PHONY: python-env
 python-env:
-	@test -d $(PYTHON_ENV) || virtualenv $(VIRTUALENV_PARAMS) $(PYTHON_ENV)
-	@$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8==1.3.5 six
+	@test -d $(PYTHON_ENV) || ${PYTHON_EXE} -m venv $(VENV_PARAMS) $(PYTHON_ENV)
+	@$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8==1.3.5 pylint==2.4.4
 	@# Work around pip bug. See: https://github.com/pypa/pip/issues/4464
 	@find $(PYTHON_ENV) -type d -name dist-packages -exec sh -c "echo dist-packages > {}.pth" ';'
 

diff --git a/Vagrantfile b/Vagrantfile
@@ -24,8 +24,8 @@
 # freebsd and openbsd
 # -------------------
 #   - Use gmake instead of make.
-#   - Folder syncing doesn't work well. Consider copying the files into the box or
-#     cloning the project inside the box.
+#   - Folder syncing doesn't work well. Consider copying the files into the box
+#     or cloning the project inside the box.
 ###
 
 # Read the branch's Go version from the .go-version file.
@@ -82,27 +82,14 @@ if (-Not (Get-Command "choco" -ErrorAction SilentlyContinue)) {
 choco feature disable -n=showDownloadProgress
 
 if (-Not (Get-Command "python" -ErrorAction SilentlyContinue)) {
-    echo "Installing python2"
-    choco install python2 -y -r
+    echo "Installing python 3"
+    choco install python -y -r --version 3.8.1.20200110
     refreshenv
-    $env:PATH = "$env:PATH;C:\\Python27;C:\\Python27\\Scripts"
+    $env:PATH = "$env:PATH;C:\\Python38;C:\\Python38\\Scripts"
 }
 
-if (-Not (Get-Command "pip" -ErrorAction SilentlyContinue)) {
-    echo "Installing pip"
-    Invoke-WebRequest https://bootstrap.pypa.io/get-pip.py -OutFile get-pip.py
-    python get-pip.py -U --force-reinstall 2>&1 | %{ "$_" }
-    rm get-pip.py
-    Invoke-WebRequest
-} else {
-    echo "Updating pip"
-    python -m pip install --upgrade pip 2>&1 | %{ "$_" }
-}
-
-if (-Not (Get-Command "virtualenv" -ErrorAction SilentlyContinue)) {
-    echo "Installing virtualenv"
-    python -m pip install virtualenv 2>&1 | %{ "$_" }
-}
+echo "Updating pip"
+python -m pip install --upgrade pip 2>&1 | %{ "$_" }
 
 if (-Not (Get-Command "git" -ErrorAction SilentlyContinue)) {
     echo "Installing git"
@@ -113,6 +100,9 @@ if (-Not (Get-Command "gcc" -ErrorAction SilentlyContinue)) {
     echo "Installing mingw (gcc)"
     choco install mingw -y -r
 }
+
+echo "Setting PYTHON_ENV in VM to point to C:\\beats-python-env."
+[System.Environment]::SetEnvironmentVariable("PYTHON_ENV", "C:\\beats-python-env", [System.EnvironmentVariableTarget]::Machine)
 SCRIPT
 
 # Provisioning for Unix/Linux
@@ -129,7 +119,7 @@ def linuxGvmProvision(arch="amd64")
   return <<SCRIPT
 mkdir -p ~/bin
 if [ ! -e "~/bin/gvm" ]; then
-  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.1.0/gvm-linux-#{arch}
+  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.2.1/gvm-linux-#{arch}
   chmod +x ~/bin/gvm
   ~/bin/gvm #{GO_VERSION}
   echo 'export GOPATH=$HOME/go' >> ~/.bash_profile
@@ -145,7 +135,7 @@ def linuxDebianProvision()
 #!/usr/bin/env bash
 set -eio pipefail
 apt-get update
-apt-get install -y make gcc python-pip python-virtualenv git
+apt-get install -y make gcc python3 python3-pip python3-venv git
 SCRIPT
 end
 
@@ -245,7 +235,8 @@ Vagrant.configure(2) do |config|
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
     c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
-    c.vm.provision "shell", inline: "yum install -y make gcc python-pip python-virtualenv git rpm-devel"
+    c.vm.provision "shell", inline: "yum install -y make gcc git rpm-devel epel-release"
+    c.vm.provision "shell", inline: "yum install -y python34 python34-pip"
   end
 
   config.vm.define "centos7", primary: true do |c|
@@ -254,25 +245,16 @@ Vagrant.configure(2) do |config|
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
     c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
-    c.vm.provision "shell", inline: "yum install -y make gcc python-pip python-virtualenv git rpm-devel"
+    c.vm.provision "shell", inline: "yum install -y make gcc python3 python3-pip git rpm-devel"
   end
 
-  config.vm.define "fedora29", primary: true do |c|
-    c.vm.box = "bento/fedora-29"
+  config.vm.define "fedora31", primary: true do |c|
+    c.vm.box = "bento/fedora-31"
     c.vm.network :forwarded_port, guest: 22, host: 2231, id: "ssh", auto_correct: true
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
     c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
-    c.vm.provision "shell", inline: "dnf install -y make gcc python-pip python-virtualenv git rpm-devel"
-  end
-
-  config.vm.define "sles12", primary: true do |c|
-    c.vm.box = "elastic/sles-12-x86_64"
-    c.vm.network :forwarded_port, guest: 22, host: 2232, id: "ssh", auto_correct: true
-
-    c.vm.provision "shell", inline: $unixProvision, privileged: false
-    c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
-    c.vm.provision "shell", inline: "pip install virtualenv"
+    c.vm.provision "shell", inline: "dnf install -y make gcc python3 python3-pip git rpm-devel"
   end
 
   config.vm.define "archlinux", primary: true do |c|
@@ -281,6 +263,6 @@ Vagrant.configure(2) do |config|
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
     c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
-    c.vm.provision "shell", inline: "pacman -Sy && pacman -S --noconfirm make gcc python-pip python-virtualenv git"
+    c.vm.provision "shell", inline: "pacman -Sy && pacman -S --noconfirm make gcc python python-pip git"
   end
 end
diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile
@@ -3,11 +3,14 @@ FROM golang:1.13.8
 RUN \
     apt-get update \
       && apt-get install -y --no-install-recommends \
-         python-pip \
-         virtualenv \
+         python3 \
+         python3-pip \
+         python3-venv \
          librpm-dev \
       && rm -rf /var/lib/apt/lists/*
 
-RUN pip install --upgrade pip
-RUN pip install --upgrade setuptools
-RUN pip install --upgrade docker-compose==1.23.2
+ENV PYTHON_ENV=/tmp/python-env
+
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade setuptools
+RUN pip3 install --upgrade docker-compose==1.23.2
diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc
@@ -2949,7 +2949,7 @@ Example: `docker` and `k8s` labels.
 
 type: object
 
-example: {'application': 'foo-bar', 'env': 'production'}
+example: { "application": "foo-bar", "env": "production" }
 
 --
 

diff --git a/auditbeat/include/fields.go b/auditbeat/include/fields.go
diff --git a/auditbeat/tests/system/auditbeat.py b/auditbeat/tests/system/auditbeat.py
@@ -24,7 +24,7 @@ def setUpClass(self):
 
     def create_file(self, path, contents):
         f = open(path, 'wb')
-        f.write(contents)
+        f.write(bytes(contents, "utf-8"))
         f.close()
 
     def check_event(self, event, expected):

diff --git a/auditbeat/tests/system/test_show_command.py b/auditbeat/tests/system/test_show_command.py
@@ -73,16 +73,16 @@ def test_show_auditd_rules(self):
         os.unlink(output_file)
         assert len(lines) >= len(rules)
         # get rid of automatic rule
-        if '-F key=rule' not in lines[0]:
+        if b'-F key=rule' not in lines[0]:
             del lines[0]
 
         for i in range(len(rules)):
             expected = rules[i]
             got = lines[i].strip()
-            assert expected == got, \
+            assert expected == got.decode("utf-8"), \
                 "rule {0} doesn't match. expected='{1}' got='{2}'".format(
-                    i, expected, got
-                )
+                    i, expected, got.decode("utf-8")
+            )
 
     @unittest.skipUnless(is_root(), "Requires root")
     def test_show_auditd_status(self):
@@ -107,7 +107,7 @@ def test_show_auditd_status(self):
         self.run_beat(extra_args=['show', 'auditd-status'],
                       exit_code=0,
                       output=output_file)
-        fhandle = os.fdopen(fd, 'rb')
+        fhandle = os.fdopen(fd, 'r')
         lines = fhandle.readlines()
         fhandle.close()
         os.unlink(output_file)
@@ -122,5 +122,5 @@ def test_show_auditd_status(self):
             assert n >= 0, "Field '{0}' has negative value {1}".format(k, v)
             fields[k] = True
 
-        for (k, v) in fields.iteritems():
+        for (k, v) in fields.items():
             assert v, "Field {0} not found".format(k)
diff --git a/dev-tools/aggregate_coverage.py b/dev-tools/aggregate_coverage.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """Simple script to concatenate coverage reports.
 """

diff --git a/dev-tools/cherrypick_pr b/dev-tools/cherrypick_pr
@@ -1,6 +1,8 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """Cherry pick and backport a PR"""
+from __future__ import print_function
 
+from builtins import input
 import sys
 import os
 import argparse
@@ -45,7 +47,7 @@ def main():
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog=usage)
     parser.add_argument("to_branch",
-                        help="To branch (e.g 5.0)")
+                        help="To branch (e.g 7.x)")
     parser.add_argument("pr_number",
                         help="The PR number being merged (e.g. 2345)")
     parser.add_argument("commit_hashes", metavar="hash", nargs="+",
@@ -75,7 +77,7 @@ def main():
     tmp_branch = "backport_{}_{}".format(args.pr_number, args.to_branch)
 
     if not vars(args)["continue"]:
-        if not args.yes and raw_input("This will destroy all local changes. " +
+        if not args.yes and input("This will destroy all local changes. " +
                                       "Continue? [y/n]: ") != "y":
             return 1
         check_call("git reset --hard", shell=True)
@@ -110,15 +112,16 @@ def main():
 
     if args.diff:
         call("git diff {}".format(args.to_branch), shell=True)
-        if raw_input("Continue? [y/n]: ") != "y":
+        if input("Continue? [y/n]: ") != "y":
             print("Aborting cherry-pick.")
             return 1
 
     print("Ready to push branch.")
 
     remote = args.remote
     if not remote:
-        remote = raw_input("To which remote should I push? (your fork): ")
+        remote = input("To which remote should I push? (your fork): ")
+
     call("git push {} :{} > /dev/null".format(remote, tmp_branch),
          shell=True)
     check_call("git push --set-upstream {} {}"
@@ -138,15 +141,15 @@ def main():
         # get the github username from the remote where we pushed
         remote_url = check_output("git remote get-url {}".format(remote),
                                   shell=True)
-        remote_user = re.search("github.com[:/](.+)/beats", remote_url).group(1)
+        remote_user = re.search("github.com[:/](.+)/beats", str(remote_url)).group(1)
 
         # create PR
         request = session.post(base + "/pulls", json=dict(
-            title="Cherry-pick #{} to {}: {}".format(args.pr_number, args.to_branch, original_pr["title"].encode('utf-8')),
+            title="Cherry-pick #{} to {}: {}".format(args.pr_number, args.to_branch, original_pr["title"]),
             head=remote_user + ":" + tmp_branch,
             base=args.to_branch,
             body="Cherry-pick of PR #{} to {} branch. Original message: \n\n{}"
-            .format(args.pr_number, args.to_branch, original_pr["body"].encode('utf-8'))
+            .format(args.pr_number, args.to_branch, original_pr["body"])
         ))
         if request.status_code > 299:
             print("Creating PR failed: {}".format(request.json()))

diff --git a/dev-tools/deploy b/dev-tools/deploy
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import os
 import argparse
 from subprocess import check_call

diff --git a/dev-tools/generate_notice.py b/dev-tools/generate_notice.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import glob
 import os
 import datetime
@@ -332,7 +330,7 @@ def detect_license_summary(content):
     # replace all white spaces with a single space
     content = re.sub(r"\s+", ' ', content)
     # replace smart quotes with less intelligent ones
-    content = content.replace(b'\xe2\x80\x9c', '"').replace(b'\xe2\x80\x9d', '"')
+    content = content.replace(bytes(b'\xe2\x80\x9c').decode(), '"').replace(bytes(b'\xe2\x80\x9d').decode(), '"')
     if any(sentence in content[0:1000] for sentence in APACHE2_LICENSE_TITLES):
         return "Apache-2.0"
     if any(sentence in content[0:1000] for sentence in MIT_LICENSES):

diff --git a/dev-tools/get_version b/dev-tools/get_version
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import os
 import re
 import argparse

diff --git a/dev-tools/jenkins_ci.ps1 b/dev-tools/jenkins_ci.ps1
@@ -64,6 +64,11 @@ $packages = ($packages|group|Select -ExpandProperty Name) -join ","
 exec { go test -race -c -cover -covermode=atomic -coverpkg $packages } "go test -race -cover FAILURE"
 
 if (Test-Path "tests\system") {
-    Set-Location -Path tests\system
-    exec { nosetests --with-timer --with-xunit --xunit-file=../../build/TEST-system.xml } "System test FAILURE"
+    echo "Running python tests"
+    choco install python -y -r --no-progress --version 3.8.1.20200110
+    refreshenv
+    $env:PATH = "C:\Python38;C:\Python38\Scripts;$env:PATH"
+    $env:PYTHON_ENV = "$env:TEMP\python-env"
+    python --version
+    exec { mage pythonUnitTest } "System test FAILURE"
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -2949,7 +2949,7 @@ Example: `docker` and `k8s` labels. @@
     type: object
-    example: {'application': 'foo-bar', 'env': 'production'}
+    example: { "application": "foo-bar", "env": "production" }
     --
@@ Expand Down @@