From 9375fb93c22710a1975ce6b98f06d397af6edcce Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 17 Feb 2020 10:31:36 +0100 Subject: [PATCH] Migrate scripts to python 3 (#14798) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several changes in test and tools code to migrate to Python 3 and stop supporting Python 2. From now on, Python 3 is required to run tests and many make and mage targets. Switch over to `python3 -m venv` instead of `virtualenv`. It's the recommended way to create virtual Python environments in 3.4 and above. Remove any direct references to python to always allow a specific python binary or version to be used with the PYTHON_EXE environmet variable. Use python3 in the shebang for python scripts. python is going to be reserved for python2 in Ubuntu, and PEP394 mentions that python should be used in the shebang line only for scripts that are source compatible with both Python 2 and 3. Update documentation to reflect these changes and add information about the use of Python for beats development. Prepare the tests for checks based on the local ip, this IP can be [::1] if ipv4 is not available, or 127.0.1.1 in some machines, as in the latest Ubuntu images used in Travis. Change checks based on sys.platform, as the values for linux have changed in Python 3. Several assertions in tests have been adapted to its newer versions. Some flaky tests that are more flaky with Python 3 have been fixed or skipped. Replace use of nosetests with mage wrapper in script for Windows CI workers. Mage manages their own python virtual environments, this is preferred to directly call python commands. Co-authored-by: Noémi Ványi Co-authored-by: Benjamin Wohlwend Co-authored-by: Michael Madden Co-authored-by: Andrew Kroh --- .travis.yml | 15 ++- Makefile | 14 +-- Vagrantfile | 56 ++++------- auditbeat/Dockerfile | 13 ++- auditbeat/docs/fields.asciidoc | 2 +- auditbeat/include/fields.go | 2 +- auditbeat/tests/system/auditbeat.py | 2 +- auditbeat/tests/system/test_show_command.py | 12 +-- dev-tools/aggregate_coverage.py | 2 +- dev-tools/cherrypick_pr | 19 ++-- dev-tools/deploy | 2 +- dev-tools/generate_notice.py | 4 +- dev-tools/get_version | 2 +- dev-tools/jenkins_ci.ps1 | 9 +- dev-tools/mage/docs.go | 2 +- dev-tools/mage/integtest.go | 1 + dev-tools/mage/kibana.go | 2 +- dev-tools/mage/pytest.go | 38 ++++--- dev-tools/merge_pr | 2 +- dev-tools/open_pr | 2 +- dev-tools/promote_docs | 2 +- dev-tools/set_docs_version | 2 +- dev-tools/set_version | 2 +- docs/devguide/contributing.asciidoc | 15 ++- .../creating-beat-from-metricbeat.asciidoc | 11 ++- docs/devguide/metricset-details.asciidoc | 2 +- docs/devguide/modules-dev-guide.asciidoc | 3 +- docs/devguide/newbeat.asciidoc | 2 +- docs/devguide/python.asciidoc | 90 +++++++++++++++++ docs/devguide/testing.asciidoc | 2 +- filebeat/Dockerfile | 13 ++- filebeat/docs/fields.asciidoc | 2 +- filebeat/include/fields.go | 2 +- filebeat/tests/load/Makefile | 2 +- .../tests/open-file-handlers/log_stdout.py | 2 +- filebeat/tests/system/test_autodiscover.py | 4 +- filebeat/tests/system/test_crawler.py | 54 +++++----- filebeat/tests/system/test_harvester.py | 47 ++++----- filebeat/tests/system/test_input.py | 6 +- filebeat/tests/system/test_json.py | 2 +- filebeat/tests/system/test_keystore.py | 2 +- filebeat/tests/system/test_load.py | 2 +- filebeat/tests/system/test_modules.py | 2 +- filebeat/tests/system/test_multiline.py | 48 ++++----- filebeat/tests/system/test_processors.py | 40 ++++---- filebeat/tests/system/test_registrar.py | 15 ++- .../tests/system/test_registrar_upgrade.py | 2 +- filebeat/tests/system/test_shutdown.py | 2 +- filebeat/tests/system/test_stdin.py | 10 +- filebeat/tests/system/test_syslog.py | 6 +- filebeat/tests/system/test_tcp.py | 2 +- filebeat/tests/system/test_tcp_tls.py | 6 +- filebeat/tests/system/test_udp.py | 2 +- heartbeat/Dockerfile | 22 +++-- heartbeat/docs/fields.asciidoc | 2 +- heartbeat/include/fields.go | 2 +- heartbeat/magefile.go | 2 + heartbeat/monitors/active/tcp/tcp_test.go | 27 ++++- heartbeat/tests/system/heartbeat.py | 8 +- heartbeat/tests/system/test_autodiscovery.py | 4 +- heartbeat/tests/system/test_base.py | 2 +- heartbeat/tests/system/test_monitor.py | 2 +- heartbeat/tests/system/test_telemetry.py | 8 +- journalbeat/Dockerfile | 22 +++-- journalbeat/docs/fields.asciidoc | 2 +- journalbeat/include/fields.go | 2 +- journalbeat/magefile.go | 2 + libbeat/Dockerfile | 25 +++-- libbeat/_meta/fields.ecs.yml | 4 +- .../common/kubernetes/metadata/namespace.go | 15 +-- libbeat/docs/version.asciidoc | 2 +- libbeat/magefile.go | 2 + libbeat/outputs/logstash/ca_invalid_test.key | 98 +++++++++---------- libbeat/outputs/logstash/ca_invalid_test.pem | 58 +++++------ libbeat/outputs/logstash/ca_test.key | 98 +++++++++---------- libbeat/outputs/logstash/ca_test.pem | 54 +++++----- libbeat/outputs/logstash/logstash_test.go | 5 +- .../outputs/transport/transptest/ca_test.key | 98 +++++++++---------- .../outputs/transport/transptest/ca_test.pem | 58 +++++------ .../outputs/transport/transptest/testing.go | 21 ++-- .../transport/transptest/testing_test.go | 6 +- libbeat/scripts/Makefile | 22 +++-- libbeat/scripts/generate_makefile_doc.py | 3 +- libbeat/tests/system/beat/beat.py | 22 ++--- libbeat/tests/system/beat/compose.py | 15 ++- libbeat/tests/system/idxmgmt.py | 13 ++- libbeat/tests/system/keystore.py | 2 +- libbeat/tests/system/requirements.txt | 8 +- .../system/test_cmd_setup_index_management.py | 4 - libbeat/tests/system/test_dashboard.py | 7 +- libbeat/tests/system/test_ilm.py | 6 +- libbeat/tests/system/test_meta.py | 4 +- libbeat/tests/system/test_monitoring.py | 5 +- libbeat/tests/system/test_seccomp.py | 4 +- libbeat/tests/system/test_template.py | 2 +- libbeat/tests/system/test_umask.py | 4 +- metricbeat/Dockerfile | 13 ++- metricbeat/Makefile | 4 +- metricbeat/docs/fields.asciidoc | 2 +- metricbeat/include/fields/fields.go | 2 +- metricbeat/module/apache/test_apache.py | 16 +-- metricbeat/module/consul/test_consul.py | 4 +- .../elasticsearch/test_elasticsearch.py | 8 +- metricbeat/module/golang/test_golang.py | 2 +- metricbeat/module/haproxy/test_haproxy.py | 4 +- metricbeat/module/http/test_http.py | 4 +- metricbeat/module/kibana/test_kibana.py | 8 +- .../module/kubernetes/test_kubernetes.py | 2 +- metricbeat/module/logstash/test_logstash.py | 7 +- metricbeat/module/mongodb/test_mongodb.py | 2 +- metricbeat/module/mysql/test_mysql.py | 2 +- metricbeat/module/nats/test_nats.py | 8 +- metricbeat/module/php_fpm/test_phpfpm.py | 2 +- .../module/postgresql/test_postgresql.py | 2 +- .../module/prometheus/test_prometheus.py | 2 +- metricbeat/module/redis/test_redis.py | 22 ++--- metricbeat/module/system/test_system.py | 26 ++--- metricbeat/module/uwsgi/test_uwsgi.py | 2 +- metricbeat/module/zookeeper/test_zookeeper.py | 8 +- metricbeat/scripts/create_metricset.py | 4 +- metricbeat/tests/system/metricbeat.py | 8 +- metricbeat/tests/system/requirements.txt | 2 +- metricbeat/tests/system/test_cmd.py | 6 +- metricbeat/tests/system/test_config.py | 4 +- metricbeat/tests/system/test_processors.py | 10 +- packetbeat/Dockerfile | 13 ++- packetbeat/Makefile | 2 +- packetbeat/docs/fields.asciidoc | 2 +- packetbeat/include/fields.go | 2 +- packetbeat/scripts/create_tcp_protocol.py | 2 +- .../system/gen/memcache/tcp_counter_ops.py | 2 +- .../tests/system/gen/memcache/tcp_delete.py | 2 +- .../gen/memcache/tcp_multi_store_load.py | 2 +- .../gen/memcache/tcp_single_load_store.py | 2 +- .../tests/system/gen/memcache/tcp_stats.py | 2 +- .../system/gen/memcache/udp_counter_ops.py | 2 +- .../tests/system/gen/memcache/udp_delete.py | 2 +- .../system/gen/memcache/udp_multi_store.py | 2 +- .../system/gen/memcache/udp_single_store.py | 2 +- .../tests/system/test_0001_mysql_spaces.py | 4 +- .../test_0064_mysql_prepare_statement.py | 4 +- .../tests/system/test_0065_unmatched_http.py | 2 +- script/generate.py | 2 +- script/kibana-migration.py | 12 +-- script/renamed_fields.py | 6 +- script/update_golang_x.py | 1 - .../elastic/go-txfile/meta_sizing.py | 6 +- .../google/gopacket/layers/test_creator.py | 4 +- .../tsg/gopacket/layers/test_creator.py | 4 +- winlogbeat/docs/fields.asciidoc | 2 +- winlogbeat/include/fields.go | 2 +- winlogbeat/tests/system/test_eventlogging.py | 9 +- winlogbeat/tests/system/test_wineventlog.py | 9 +- winlogbeat/tests/system/winlogbeat.py | 32 +++--- .../auditbeat/tests/system/test_metricsets.py | 6 +- .../tests/system/test_system_socket.py | 61 ++++++------ x-pack/functionbeat/Dockerfile | 21 ++-- x-pack/functionbeat/docs/fields.asciidoc | 2 +- x-pack/functionbeat/include/fields.go | 2 +- x-pack/functionbeat/tests/system/test_base.py | 2 +- x-pack/libbeat/Dockerfile | 20 ++-- x-pack/libbeat/magefile.go | 2 + .../libbeat/tests/system/test_management.py | 2 +- .../module/activemq/test_activemq.py | 3 + .../module/appsearch/_meta/Dockerfile | 5 +- .../module/appsearch/docker-compose.yml | 3 + .../module/cockroachdb/test_cockroachdb.py | 4 +- .../metricbeat/module/coredns/test_coredns.py | 2 +- x-pack/metricbeat/module/mssql/test_mssql.py | 4 +- 169 files changed, 1020 insertions(+), 820 deletions(-) create mode 100644 docs/devguide/python.asciidoc diff --git a/.travis.yml b/.travis.yml index c56f43f6dc0..a3171ff7061 100644 --- a/.travis.yml +++ b/.travis.yml @@ -166,12 +166,6 @@ jobs: go: $TRAVIS_GO_VERSION stage: test - # Docs - - os: linux - env: TARGETS="docs" - go: $TRAVIS_GO_VERSION - stage: test - # Kubernetes - os: linux install: deploy/kubernetes/.travis/setup.sh @@ -199,7 +193,6 @@ jobs: retries: true update: true packages: - - python-virtualenv - libpcap-dev - xsltproc - libxml2-utils @@ -218,16 +211,22 @@ addons: config: retries: true update: true + sources: + - deadsnakes packages: - - python-virtualenv - libpcap-dev - xsltproc - libxml2-utils - libsystemd-journal-dev - librpm-dev + # From deadsnakes PPA + - python3.6 + - python3.6-venv before_install: + - if [ x$TRAVIS_DIST = xtrusty ]; then sudo ln -sf python3.6 /usr/bin/python3; fi - python --version + - python3 --version - umask 022 - chmod -R go-w $GOPATH/src/github.com/elastic/beats # Docker-compose installation diff --git a/Makefile b/Makefile index c8421ffc800..8567031ad1e 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,9 @@ BEATS?=auditbeat filebeat heartbeat journalbeat metricbeat packetbeat winlogbeat PROJECTS=libbeat $(BEATS) PROJECTS_ENV=libbeat filebeat metricbeat PYTHON_ENV?=$(BUILD_DIR)/python-env -VIRTUALENV_PARAMS?= +PYTHON_EXE?=python3 +PYTHON_ENV_EXE=${PYTHON_ENV}/bin/$(notdir ${PYTHON_EXE}) +VENV_PARAMS?= FIND=find . -type f -not -path "*/vendor/*" -not -path "*/build/*" -not -path "*/.git/*" GOLINT=golint GOLINT_REPO=golang.org/x/lint/golint @@ -88,8 +90,8 @@ clean-vendor: .PHONY: check check: python-env @$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;) - @# Checks also python files which are not part of the beats - @$(FIND) -name *.py -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120 {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false) + @$(FIND) -name *.py -name *.py -not -path "*/build/*" -not -path "*/vendor/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120 {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false) + @$(FIND) -name *.py -not -path "*/build/*" -not -path "*/vendor/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false) @# Validate that all updates were committed @$(MAKE) update @$(MAKE) check-headers @@ -136,13 +138,13 @@ docs: .PHONY: notice notice: python-env @echo "Generating NOTICE" - @$(PYTHON_ENV)/bin/python dev-tools/generate_notice.py . + @${PYTHON_ENV_EXE} dev-tools/generate_notice.py . # Sets up the virtual python environment .PHONY: python-env python-env: - @test -d $(PYTHON_ENV) || virtualenv $(VIRTUALENV_PARAMS) $(PYTHON_ENV) - @$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8==1.3.5 six + @test -d $(PYTHON_ENV) || ${PYTHON_EXE} -m venv $(VENV_PARAMS) $(PYTHON_ENV) + @$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8==1.3.5 pylint==2.4.4 @# Work around pip bug. See: https://github.com/pypa/pip/issues/4464 @find $(PYTHON_ENV) -type d -name dist-packages -exec sh -c "echo dist-packages > {}.pth" ';' diff --git a/Vagrantfile b/Vagrantfile index 68792c791a6..b7cc6f03523 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -24,8 +24,8 @@ # freebsd and openbsd # ------------------- # - Use gmake instead of make. -# - Folder syncing doesn't work well. Consider copying the files into the box or -# cloning the project inside the box. +# - Folder syncing doesn't work well. Consider copying the files into the box +# or cloning the project inside the box. ### # Read the branch's Go version from the .go-version file. @@ -82,27 +82,14 @@ if (-Not (Get-Command "choco" -ErrorAction SilentlyContinue)) { choco feature disable -n=showDownloadProgress if (-Not (Get-Command "python" -ErrorAction SilentlyContinue)) { - echo "Installing python2" - choco install python2 -y -r + echo "Installing python 3" + choco install python -y -r --version 3.8.1.20200110 refreshenv - $env:PATH = "$env:PATH;C:\\Python27;C:\\Python27\\Scripts" + $env:PATH = "$env:PATH;C:\\Python38;C:\\Python38\\Scripts" } -if (-Not (Get-Command "pip" -ErrorAction SilentlyContinue)) { - echo "Installing pip" - Invoke-WebRequest https://bootstrap.pypa.io/get-pip.py -OutFile get-pip.py - python get-pip.py -U --force-reinstall 2>&1 | %{ "$_" } - rm get-pip.py - Invoke-WebRequest -} else { - echo "Updating pip" - python -m pip install --upgrade pip 2>&1 | %{ "$_" } -} - -if (-Not (Get-Command "virtualenv" -ErrorAction SilentlyContinue)) { - echo "Installing virtualenv" - python -m pip install virtualenv 2>&1 | %{ "$_" } -} +echo "Updating pip" +python -m pip install --upgrade pip 2>&1 | %{ "$_" } if (-Not (Get-Command "git" -ErrorAction SilentlyContinue)) { echo "Installing git" @@ -113,6 +100,9 @@ if (-Not (Get-Command "gcc" -ErrorAction SilentlyContinue)) { echo "Installing mingw (gcc)" choco install mingw -y -r } + +echo "Setting PYTHON_ENV in VM to point to C:\\beats-python-env." +[System.Environment]::SetEnvironmentVariable("PYTHON_ENV", "C:\\beats-python-env", [System.EnvironmentVariableTarget]::Machine) SCRIPT # Provisioning for Unix/Linux @@ -129,7 +119,7 @@ def linuxGvmProvision(arch="amd64") return <