From 44a39f8fbcf1bc907f6f3fadfec66e77355ebffa Mon Sep 17 00:00:00 2001 From: simitt Date: Tue, 18 Feb 2020 19:56:16 +0100 Subject: [PATCH] Update to python3 add python-env to make targets --- .ci/docker/golang-mage/Dockerfile | 5 ++--- Dockerfile | 7 +++++-- Makefile | 8 ++++---- README.md | 11 +++++------ TESTING.md | 8 ++++---- docs/version.asciidoc | 2 +- script/are_kibana_saved_objects_updated.py | 2 +- script/check_changelogs.py | 2 +- script/copy-docs.py | 2 +- script/generate_notice_overrides.py | 2 +- script/is_beats_updated.py | 2 +- script/jenkins/check-changelogs.sh | 2 +- script/payload-compare.py | 1 + script/renamed_fields.py | 2 +- script/update_govendor_deps.py | 2 +- tests/Dockerfile | 7 +++++-- 16 files changed, 35 insertions(+), 30 deletions(-) diff --git a/.ci/docker/golang-mage/Dockerfile b/.ci/docker/golang-mage/Dockerfile index 37339059020..e7b492cb338 100644 --- a/.ci/docker/golang-mage/Dockerfile +++ b/.ci/docker/golang-mage/Dockerfile @@ -19,8 +19,7 @@ RUN git clone https://github.com/magefile/mage \ && go get github.com/elastic/apm-server/vendor/golang.org/x/tools/cmd/goimports RUN apt-get update -y -qq \ - && apt-get install -y -qq python-pip \ - && rm -rf /var/lib/apt/lists/* \ - && pip install virtualenv + && apt-get install -y -qq python3 python3-pip python3-venv \ + && rm -rf /var/lib/apt/lists/* WORKDIR /go diff --git a/Dockerfile b/Dockerfile index 2fd471451e3..2b183da4a75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,13 @@ MAINTAINER Nicolas Ruflin RUN set -x && \ apt-get update && \ apt-get install -y --no-install-recommends \ - netcat python-pip virtualenv && \ + netcat python3 python3-pip python3-venv && \ apt-get clean -RUN pip install --upgrade setuptools +ENV PYTHON_ENV=/tmp/python-env + +RUN pip3 install --upgrade pip +RUN pip3 install --upgrade setuptools # Setup work environment ENV APM_SERVER_PATH /go/src/github.com/elastic/apm-server diff --git a/Makefile b/Makefile index d5ea3f0a6a9..381963f089f 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ ES_SUPERUSER_PASS?=changeme # updates beats updates the framework part and go parts of beats .PHONY: update-beats -update-beats: govendor +update-beats: python-env govendor rm -rf vendor/github.com/elastic/beats @govendor fetch github.com/elastic/beats/...@$(BEATS_VERSION) @govendor fetch github.com/elastic/beats/libbeat/generator/fields@$(BEATS_VERSION) @@ -140,7 +140,7 @@ staticcheck: check-deps: test-deps golint staticcheck .PHONY: check-full -check-full: check-deps check +check-full: python-env check-deps check @# Validate that all updates were committed @$(MAKE) update @$(MAKE) check @@ -164,7 +164,7 @@ apm-docs: ## @build Builds the documentation for APM Server and APM Overview .PHONY: update-beats-docs -update-beats-docs: +update-beats-docs: python-env @python script/copy-docs.py @$(MAKE) docs @@ -198,7 +198,7 @@ import-dashboards: echo "APM loads dashboards via Kibana, not the APM Server" .PHONY: check-changelogs -check-changelogs: ## @testing Checks the changelogs for certain branches. +check-changelogs: python-env ## @testing Checks the changelogs for certain branches. @python script/check_changelogs.py .PHONY: rm-empty-folders diff --git a/README.md b/README.md index 8f1535a3308..6b331ede51c 100644 --- a/README.md +++ b/README.md @@ -41,13 +41,12 @@ in the same directory with the name apm-server. make ``` -You also need to create all files needed by the APM Server by running the additional command below. +You also need to create all files needed by the APM Server by running the additional command below. ``` make update ``` -Note that this requires to have `virtualenv` installed. Make sure that `virtualenv` creates Python2 -environment, otherwise specify `PYTHON_EXE` variable pointed at Python2 executable. +Note that this requires to have `python >= 3.7` and `venv` installed. ### Run @@ -97,7 +96,7 @@ See [releases](RELEASES.md) for an APM Server release checklist. ## Update Dependencies -The `apm-server` has two types of dependencies, +The `apm-server` has two types of dependencies, the Golang packages managed with *Govendor* and a dependency to the *Beats Framework*. ### Govendor @@ -127,9 +126,9 @@ BEATS_VERSION=f240148065af94d55c5149e444482b9635801f27 make update-beats ``` ### Go-elasticsearch client Update -It is important to keep the used [go-elasticsearch client](https://github.com/elastic/go-elasticsearch) in sync with the according major version. +It is important to keep the used [go-elasticsearch client](https://github.com/elastic/go-elasticsearch) in sync with the according major version. We also recommend to use the latest available client for minor versions. -Since APM Server does not yet support go modules, you can update the dependency using govendor, e.g. by running: +Since APM Server does not yet support go modules, you can update the dependency using govendor, e.g. by running: ``` git clone --branch v7.4.1 https://github.com/elastic/go-elasticsearch.git $GOPATH/src/github.com/elastic/go-elasticsearch/v7 govendor add github.com/elastic/go-elasticsearch/v7/^ diff --git a/TESTING.md b/TESTING.md index 4e0c0a04289..178e8010e3c 100644 --- a/TESTING.md +++ b/TESTING.md @@ -22,7 +22,7 @@ While developing new tests or troubleshooting test failures, it is handy to run example from within an editor, while still allowing all dependencies to run in containers. To accomplish this: * Run `make build-image start-environment` to start docker containers for the Elastic Stack. -* Run `PYTHON_EXE=python2.7 make python-env` to build a python virtualenv +* Run `PYTHON_EXE=python3 make python-env` to build a python virtualenv * Run `make apm-server.test` to (re)build the executable used to run the apm-server * Run tests using the `run-system-tests` target, eg: ``` @@ -39,9 +39,9 @@ For insights about test-coverage, run `make coverage-report`. The test coverage ## Snapshot-Testing Some tests make use of the concept of _snapshot_ or _approvals testing_. If running tests leads to changed snapshots, you can use the `approvals` tool to update the snapshots. Following workflow is intended: -* Run `make update` to create the `approvals` binary that supports reviewing changes. +* Run `make update` to create the `approvals` binary that supports reviewing changes. * Run `make unit` to create a `*.received.json` file for every newly created or changed snapshot. -* Run `./approvals` to review and interactively accept the changes. +* Run `./approvals` to review and interactively accept the changes. ## Benchmarking @@ -55,7 +55,7 @@ A good way to present your results is by using `benchcmp`. With your changes in the current working tree, do: ``` -$ go get -u golang.org/x/tools/cmd/benchcmp +$ go get -u golang.org/x/tools/cmd/benchcmp $ make bench > new.txt $ git checkout master $ make bench > old.txt diff --git a/docs/version.asciidoc b/docs/version.asciidoc index 0fd76a63e7c..feb9f825ecc 100644 --- a/docs/version.asciidoc +++ b/docs/version.asciidoc @@ -1,7 +1,7 @@ // doc-branch can be: master, 8.0, 8.1, etc. :doc-branch: master :go-version: 1.13.7 -:python: 2.7.9 +:python: 3.7 :docker: 1.12 :docker-compose: 1.11 diff --git a/script/are_kibana_saved_objects_updated.py b/script/are_kibana_saved_objects_updated.py index bf87b16c10b..33b9875d74e 100755 --- a/script/are_kibana_saved_objects_updated.py +++ b/script/are_kibana_saved_objects_updated.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function diff --git a/script/check_changelogs.py b/script/check_changelogs.py index d9d98e47a37..aed0f5fe0ff 100644 --- a/script/check_changelogs.py +++ b/script/check_changelogs.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import io import hashlib diff --git a/script/copy-docs.py b/script/copy-docs.py index 33751a2366e..cbace4272b3 100644 --- a/script/copy-docs.py +++ b/script/copy-docs.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function import os diff --git a/script/generate_notice_overrides.py b/script/generate_notice_overrides.py index b985fa91da6..824e300da59 100755 --- a/script/generate_notice_overrides.py +++ b/script/generate_notice_overrides.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Generate data used to put the correct revision information in NOTICE.TXT. """ diff --git a/script/is_beats_updated.py b/script/is_beats_updated.py index d3ebb49207d..c4bcdba25bf 100755 --- a/script/is_beats_updated.py +++ b/script/is_beats_updated.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function diff --git a/script/jenkins/check-changelogs.sh b/script/jenkins/check-changelogs.sh index 23d5e9aeaa9..806f9cf75f3 100755 --- a/script/jenkins/check-changelogs.sh +++ b/script/jenkins/check-changelogs.sh @@ -4,4 +4,4 @@ set -exuo pipefail docker run --rm \ -v "$PWD":/usr/src/myapp \ -w /usr/src/myapp python:3.6.8-alpine \ - sh -c 'pip install requests; python script/check_changelogs.py' + sh -c 'pip3 install requests; python3 script/check_changelogs.py' diff --git a/script/payload-compare.py b/script/payload-compare.py index 5daf479412b..f7fa02c6120 100644 --- a/script/payload-compare.py +++ b/script/payload-compare.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 from uuid import uuid4 from collections import defaultdict import json diff --git a/script/renamed_fields.py b/script/renamed_fields.py index 1bccf8878d1..f1352bf3fea 100644 --- a/script/renamed_fields.py +++ b/script/renamed_fields.py @@ -1,4 +1,4 @@ -# python renamed_fields.py > ../docs/field-name-changes.asciidoc +# python3 renamed_fields.py > ../docs/field-name-changes.asciidoc import yaml diff --git a/script/update_govendor_deps.py b/script/update_govendor_deps.py index a66281ff146..9e9382d644a 100755 --- a/script/update_govendor_deps.py +++ b/script/update_govendor_deps.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Update dependencies of the given govendor'd package. For https://github.com/elastic/apm-server/issues/450. diff --git a/tests/Dockerfile b/tests/Dockerfile index 3ce19fedc9e..692e7c2caff 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -4,10 +4,13 @@ MAINTAINER Nicolas Ruflin RUN set -x && \ apt-get update && \ apt-get install -y --no-install-recommends \ - netcat python-pip virtualenv && \ + netcat python3 python3-pip python3-venv && \ apt-get clean -RUN pip install --upgrade setuptools +ENV PYTHON_ENV=/tmp/python-env + +RUN pip3 install --upgrade pip +RUN pip3 install --upgrade setuptools # Setup work environment ENV APMSERVER_PATH /go/src/github.com/elastic/apm-server