From a6a05e3bfe82941f8b9bb626f7ae50c1592025a2 Mon Sep 17 00:00:00 2001 From: simitt Date: Tue, 18 Feb 2020 19:47:42 +0100 Subject: [PATCH] Update system tests to python3 * upgrade python code * upgrade windows test setup to python3 --- magefile.go | 5 ++ script/jenkins/ci.ps1 | 10 ++- script/jenkins/windows-test.ps1 | 10 ++- tests/system/apmserver.py | 15 ++-- tests/system/test_access.py | 95 ++++++++++-------------- tests/system/test_apikey.py | 2 +- tests/system/test_integration.py | 2 +- tests/system/test_integration_acm.py | 4 +- tests/system/test_integration_logging.py | 2 - tests/system/test_jaeger.py | 3 +- tests/system/test_monitoring.py | 13 ++-- tests/system/test_requests.py | 15 ++-- 12 files changed, 82 insertions(+), 94 deletions(-) diff --git a/magefile.go b/magefile.go index 9a30f9e7311..631d59ce702 100644 --- a/magefile.go +++ b/magefile.go @@ -231,6 +231,11 @@ func GoTestIntegration(ctx context.Context) error { return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) } +// PythonUnitTest executes the python system tests. +func PythonUnitTest() error { + return mage.PythonNoseTest(mage.DefaultPythonTestUnitArgs()) +} + // ----------------------------------------------------------------------------- // Customizations specific to apm-server. diff --git a/script/jenkins/ci.ps1 b/script/jenkins/ci.ps1 index 525f143180f..770d7699d6d 100755 --- a/script/jenkins/ci.ps1 +++ b/script/jenkins/ci.ps1 @@ -57,5 +57,11 @@ echo "System testing $env:beat" $packages = $(go list ./... | select-string -Pattern "/vendor/" -NotMatch | select-string -Pattern "/scripts/cmd/" -NotMatch) $packages = ($packages|group|Select -ExpandProperty Name) -join "," exec { go test -race -c -cover -covermode=atomic -coverpkg $packages } "go test FAILURE" -Set-Location -Path tests/system -exec { nosetests --with-timer --with-xunit --xunit-file=../../build/TEST-system.xml } "System test FAILURE" + +echo "Running python tests" +choco install python -y -r --no-progress --version 3.8.1.20200110 +refreshenv +$env:PATH = "C:\Python38;C:\Python38\Scripts;$env:PATH" +$env:PYTHON_ENV = "$env:TEMP\python-env" +python --version +exec { mage pythonUnitTest } "System test FAILURE" diff --git a/script/jenkins/windows-test.ps1 b/script/jenkins/windows-test.ps1 index 98d9ad96dbb..fc8c19d7426 100644 --- a/script/jenkins/windows-test.ps1 +++ b/script/jenkins/windows-test.ps1 @@ -57,5 +57,11 @@ echo "System testing $env:beat" $packages = $(go list ./... | select-string -Pattern "/vendor/" -NotMatch | select-string -Pattern "/scripts/cmd/" -NotMatch) $packages = ($packages|group|Select -ExpandProperty Name) -join "," exec { go test -race -c -cover -covermode=atomic -coverpkg $packages } "go test FAILURE" -Set-Location -Path tests/system -exec { nosetests --with-timer --with-xunit --xunit-file=../../build/TEST-system.xml } "System test FAILURE" + +echo "Running python tests" +choco install python -y -r --no-progress --version 3.8.1.20200110 +refreshenv +$env:PATH = "C:\Python38;C:\Python38\Scripts;$env:PATH" +$env:PYTHON_ENV = "$env:TEMP\python-env" +python --version +exec { mage pythonUnitTest } "System test FAILURE" diff --git a/tests/system/apmserver.py b/tests/system/apmserver.py index 2d142b7ec09..0e4bf793014 100644 --- a/tests/system/apmserver.py +++ b/tests/system/apmserver.py @@ -2,13 +2,12 @@ import json import os import re -import sets import shutil import sys import threading import time import unittest -from urlparse import urlparse +from urllib.parse import urlparse from elasticsearch import Elasticsearch, NotFoundError from nose.tools import nottest @@ -272,7 +271,7 @@ def wait_until_pipeline_logged(self): wait_until(lambda: self.log_contains(msg), name="pipelines registration") def load_docs_with_template(self, data_path, url, endpoint, expected_events_count, - query_index=None, max_timeout=10, extra_headers=None): + query_index=None, max_timeout=10, extra_headers=None, file_mode="r"): if query_index is None: query_index = apm_prefix @@ -281,7 +280,7 @@ def load_docs_with_template(self, data_path, url, endpoint, expected_events_coun if extra_headers: headers.update(extra_headers) - with open(data_path) as f: + with open(data_path, file_mode) as f: r = requests.post(url, data=f, headers=headers) assert r.status_code == 202, r.status_code @@ -388,7 +387,7 @@ def get_doc_id(doc): # as they are dependent on the environment. rec_id = get_doc_id(rec) rec_observer = rec['observer'] - self.assertEqual(sets.Set(rec_observer.keys()), sets.Set( + self.assertEqual(set(rec_observer.keys()), set( ["hostname", "version", "id", "ephemeral_id", "type", "version_major"])) assert rec_observer["version"].startswith(str(rec_observer["version_major"]) + ".") for appr in approved: @@ -415,12 +414,12 @@ def get_doc_id(doc): # Create a dynamic Exception subclass so we can fake its name to look like the original exception. class ApprovalException(Exception): def __init__(self, cause): - super(ApprovalException, self).__init__(cause.message) + super(ApprovalException, self).__init__(cause.args) def __str__(self): - return self.message + "\n\nReceived data differs from approved data. Run 'make update' and then 'approvals' to verify the diff." + return "{}\n\nReceived data differs from approved data. Run 'make update' and then 'approvals' to verify the diff.".format(self.args) ApprovalException.__name__ = type(exc).__name__ - raise ApprovalException, exc, sys.exc_info()[2] + raise ApprovalException(exc).with_traceback(sys.exc_info()[2]) class ClientSideBaseTest(ServerBaseTest): diff --git a/tests/system/test_access.py b/tests/system/test_access.py index a74369eeab1..d67352f1318 100644 --- a/tests/system/test_access.py +++ b/tests/system/test_access.py @@ -87,7 +87,7 @@ def setUp(self): "event": self.privilege_event, "sourcemap": self.privilege_sourcemap } - self.privileges_all = self.privileges.values() + self.privileges_all = list(self.privileges.values()) self.privilege_any = "*" # resources @@ -148,7 +148,8 @@ def create_api_key(self, privileges, resources, application="apm"): assert resp.status_code == 200, resp.status_code id = resp.json()["id"] wait_until(lambda: self.api_key_exists(id), name="create api key") - return base64.b64encode("{}:{}".format(id, resp.json()["api_key"])) + enc = "utf-8" + return str(base64.b64encode("{}:{}".format(id, resp.json()["api_key"]).encode(enc)), enc) def create_api_key_header(self, privileges, resources, application="apm"): return "ApiKey {}".format(self.create_api_key(privileges, resources, application=application)) @@ -265,7 +266,7 @@ def test_root(self): for token in self.unauthorized_keys: resp = requests.get(url, headers=headers(token)) assert resp.status_code == 200, "token: {}, status_code: {}".format(token, resp.status_code) - assert resp.content == '', "token: {}, response: {}".format(token, resp.content) + assert resp.text == '', "token: {}, response: {}".format(token, resp.content) keys_one_privilege = [self.api_key_privilege_config, self.api_key_privilege_sourcemap, self.api_key_privilege_event] @@ -412,20 +413,15 @@ def config(self): cfg.update(self.ssl_overrides()) return cfg - def send_http_request(self, cert=None, verify=False, protocol='https'): - # verify decides whether or not the client should verify the servers certificate - return requests.post("{}://localhost:8200/intake/v2/events".format(protocol), - headers={'content-type': 'application/x-ndjson'}, - data=self.get_event_payload(), - cert=cert, - verify=verify) - - def ssl_connect(self, protocol=ssl.PROTOCOL_TLSv1_2, ciphers=None): + def ssl_connect(self, protocol=ssl.PROTOCOL_TLSv1_2, ciphers=None, cert=None, key=None, ca_cert=None): context = ssl.SSLContext(protocol) if ciphers: context.set_ciphers(ciphers) - context.load_verify_locations(self.ca_cert) - context.load_cert_chain(certfile=self.server_cert, keyfile=self.server_key, password=self.password) + if not ca_cert: + ca_cert = self.ca_cert + context.load_verify_locations(ca_cert) + if cert and key: + context.load_cert_chain(certfile=cert, keyfile=key, password=self.password) s = context.wrap_socket(ssl.socket()) s.connect((self.host, self.port)) @@ -445,17 +441,13 @@ def ssl_overrides(self): return {"ssl_client_authentication": "none"} def test_https_no_cert_ok(self): - r = self.send_http_request(verify=self.ca_cert) - assert r.status_code == 202, r.status_code + self.ssl_connect() def test_http_fails(self): with self.assertRaises(Exception): - self.send_http_request(protocol='http') - - @raises(SSLError) - def test_https_server_validation_fails(self): - r = self.send_http_request(verify=True) - assert r.status_code == 202, r.status_code + return requests.post("http://localhost:8200/intake/v2/events", + headers={'content-type': 'application/x-ndjson'}, + data=self.get_event_payload()) @integration_test @@ -463,20 +455,16 @@ class TestSSLEnabledOptionalClientVerificationTest(TestSecureServerBaseTest): # no ssl_overrides necessary as `optional` is default def test_https_no_certificate_ok(self): - r = self.send_http_request(verify=self.ca_cert) - assert r.status_code == 202, r.status_code + self.ssl_connect() - @raises(SSLError) + @raises(ssl.SSLError) def test_https_verify_cert_if_given(self): - self.send_http_request(verify=self.ca_cert, - cert=(self.simple_cert, self.simple_key)) + self.ssl_connect(cert=self.simple_cert, key=self.simple_key) - @raises(SSLError) + @raises(ssl.SSLError) def test_https_self_signed_cert(self): # CA is not configured server side, so self signed certs are not valid - r = self.send_http_request(verify=self.ca_cert, - cert=(self.client_cert, self.client_key)) - assert r.status_code == 202, r.status_code + self.ssl_connect(cert=self.client_cert, key=self.client_key) @integration_test @@ -484,22 +472,18 @@ class TestSSLEnabledOptionalClientVerificationWithCATest(TestSecureServerBaseTes def ssl_overrides(self): return {"ssl_certificate_authorities": self.ca_cert} - @raises(SSLError) + @raises(ssl.SSLError) def test_https_no_certificate(self): # since CA is configured, client auth is required - r = self.send_http_request(verify=self.ca_cert) - assert r.status_code == 202, r.status_code + self.ssl_connect() - @raises(SSLError) + @raises(ssl.SSLError) def test_https_verify_cert_if_given(self): # invalid certificate - self.send_http_request(verify=self.ca_cert, - cert=(self.simple_cert, self.simple_key)) + self.ssl_connect(cert=self.simple_cert, key=self.simple_key) def test_https_auth_cert_ok(self): - r = self.send_http_request(verify=self.ca_cert, - cert=(self.client_cert, self.client_key)) - assert r.status_code == 202, r.status_code + self.ssl_connect(cert=self.client_cert, key=self.client_key) @integration_test @@ -508,19 +492,16 @@ def ssl_overrides(self): return {"ssl_client_authentication": "required", "ssl_certificate_authorities": self.ca_cert} - @raises(SSLError) + @raises(ssl.SSLError) def test_https_no_cert_fails(self): - self.send_http_request(verify=self.ca_cert) + self.ssl_connect() - @raises(SSLError) + @raises(ssl.SSLError) def test_https_invalid_cert_fails(self): - self.send_http_request(verify=self.ca_cert, - cert=(self.simple_cert, self.simple_key)) + self.ssl_connect(cert=self.simple_cert, key=self.simple_key) def test_https_auth_cert_ok(self): - r = self.send_http_request(verify=self.ca_cert, - cert=(self.client_cert, self.client_key)) - assert r.status_code == 202, r.status_code + self.ssl_connect(cert=self.client_cert, key=self.client_key) @integration_test @@ -530,13 +511,13 @@ def ssl_overrides(self): @raises(ssl.SSLError) def test_tls_v1_0(self): - self.ssl_connect(protocol=ssl.PROTOCOL_TLSv1) + self.ssl_connect(protocol=ssl.PROTOCOL_TLSv1, cert=self.server_cert, key=self.server_key) def test_tls_v1_1(self): - self.ssl_connect(protocol=ssl.PROTOCOL_TLSv1_1) + self.ssl_connect(protocol=ssl.PROTOCOL_TLSv1_1, cert=self.server_cert, key=self.server_key) def test_tls_v1_2(self): - self.ssl_connect() + self.ssl_connect(cert=self.server_cert, key=self.server_key) @integration_test @@ -547,10 +528,10 @@ def ssl_overrides(self): @raises(ssl.SSLError) def test_tls_v1_1(self): - self.ssl_connect(protocol=ssl.PROTOCOL_TLSv1_1) + self.ssl_connect(protocol=ssl.PROTOCOL_TLSv1_1, cert=self.server_cert, key=self.server_key) def test_tls_v1_2(self): - self.ssl_connect() + self.ssl_connect(cert=self.server_cert, key=self.server_key) @integration_test @@ -560,18 +541,18 @@ def ssl_overrides(self): "ssl_certificate_authorities": self.ca_cert} def test_https_no_cipher_set(self): - self.ssl_connect() + self.ssl_connect(cert=self.server_cert, key=self.server_key) def test_https_supports_cipher(self): # set the same cipher in the client as set in the server - self.ssl_connect(ciphers='ECDHE-RSA-AES128-GCM-SHA256') + self.ssl_connect(ciphers='ECDHE-RSA-AES128-GCM-SHA256', cert=self.server_cert, key=self.server_key) def test_https_unsupported_cipher(self): # client only offers unsupported cipher with self.assertRaisesRegexp(ssl.SSLError, 'SSLV3_ALERT_HANDSHAKE_FAILURE'): - self.ssl_connect(ciphers='ECDHE-RSA-AES256-SHA384') + self.ssl_connect(ciphers='ECDHE-RSA-AES256-SHA384', cert=self.server_cert, key=self.server_key) def test_https_no_cipher_selected(self): # client provides invalid cipher with self.assertRaisesRegexp(ssl.SSLError, 'No cipher can be selected'): - self.ssl_connect(ciphers='AES1sd28-CCM8') + self.ssl_connect(ciphers='AES1sd28-CCM8', cert=self.server_cert, key=self.server_key) diff --git a/tests/system/test_apikey.py b/tests/system/test_apikey.py index 2f9296ca84a..29fb377440b 100644 --- a/tests/system/test_apikey.py +++ b/tests/system/test_apikey.py @@ -129,7 +129,7 @@ def test_verify_all(self): for privilege in ["ingest", "sourcemap", "agent-config"]: result = self.subcommand_output("verify", "--credentials", apikey["credentials"], "--" + privilege) assert len(result) == 1, result - assert result.values()[0] is True + assert list(result.values())[0] is True def test_verify_each(self): apikey = self.create("--ingest") diff --git a/tests/system/test_integration.py b/tests/system/test_integration.py index 704ded1c053..b2c8cd18200 100644 --- a/tests/system/test_integration.py +++ b/tests/system/test_integration.py @@ -185,7 +185,7 @@ def count_library_frames(doc, lf): if "stacktrace" not in doc: return for frame in doc["stacktrace"]: - if frame.has_key("library_frame"): + if "library_frame" in frame: k = "true" if frame["library_frame"] else "false" lf[k] += 1 else: diff --git a/tests/system/test_integration_acm.py b/tests/system/test_integration_acm.py index 331a2d57d91..9d8b033c904 100644 --- a/tests/system/test_integration_acm.py +++ b/tests/system/test_integration_acm.py @@ -1,8 +1,6 @@ import time -import unittest -from urlparse import urljoin +from urllib.parse import urljoin import uuid - import requests from apmserver import ElasticTest, integration_test diff --git a/tests/system/test_integration_logging.py b/tests/system/test_integration_logging.py index 16e3948f3c0..6defdb287bb 100644 --- a/tests/system/test_integration_logging.py +++ b/tests/system/test_integration_logging.py @@ -1,6 +1,4 @@ import json -import unittest - import requests from apmserver import ElasticTest, integration_test diff --git a/tests/system/test_jaeger.py b/tests/system/test_jaeger.py index 6a901f28c01..ab6be8ab75f 100644 --- a/tests/system/test_jaeger.py +++ b/tests/system/test_jaeger.py @@ -37,7 +37,8 @@ def test_jaeger_http(self): """ jaeger_span_thrift = self.get_testdata_path('jaeger', 'span.thrift') self.load_docs_with_template(jaeger_span_thrift, self.jaeger_http_url, 'transaction', 1, - extra_headers={"content-type": "application/vnd.apache.thrift.binary"}) + extra_headers={"content-type": "application/vnd.apache.thrift.binary"}, + file_mode="rb") self.assert_no_logged_warnings() transaction_docs = self.wait_for_events('transaction', 1) diff --git a/tests/system/test_monitoring.py b/tests/system/test_monitoring.py index 569e79d0f56..138e9870278 100644 --- a/tests/system/test_monitoring.py +++ b/tests/system/test_monitoring.py @@ -1,10 +1,9 @@ -from apmserver import integration_test -from apmserver import ElasticTest - import os -import urlparse -from elasticsearch import Elasticsearch +from urllib.parse import urlparse, urlunparse +from elasticsearch import Elasticsearch +from apmserver import integration_test +from apmserver import ElasticTest from helper import wait_until @@ -17,7 +16,7 @@ def config(self): # separately from elasticsearch_host. This is necessary # because Beats will use the userinfo from the URL in # preference to specified values. - url = urlparse.urlparse(cfg["elasticsearch_host"]) + url = urlparse(cfg["elasticsearch_host"]) if url.username: cfg["elasticsearch_username"] = url.username if url.password: @@ -34,7 +33,7 @@ def config(self): admin_es.xpack.security.change_password(username="apm_system", body='{"password":"%s"}' % monitoring_password) cfg.update({ - "elasticsearch_host": urlparse.urlunparse(url), + "elasticsearch_host": urlunparse(url), "monitoring_enabled": "true", "monitoring_elasticsearch_username": "apm_system", "monitoring_elasticsearch_password": monitoring_password, diff --git a/tests/system/test_requests.py b/tests/system/test_requests.py index d037874a4d3..049b75645d2 100644 --- a/tests/system/test_requests.py +++ b/tests/system/test_requests.py @@ -4,16 +4,11 @@ import threading import time import zlib +from io import BytesIO from apmserver import ServerBaseTest, ClientSideBaseTest, CorsBaseTest -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - - class Test(ServerBaseTest): def test_ok(self): @@ -46,7 +41,7 @@ def test_validation_fail(self): data = self.get_event_payload(name="invalid-event.ndjson") r = self.request_intake(data=data) assert r.status_code == 400, r.status_code - assert "error validating JSON document against schema" in r.content, r.content + assert "error validating JSON document against schema" in r.text, r.text def test_rum_default_disabled(self): r = self.request_intake(url='http://localhost:8200/intake/v2/rum/events') @@ -58,8 +53,8 @@ def test_healthcheck(self): assert r.status_code == 200, r.status_code def test_gzip(self): - events = self.get_event_payload() - out = StringIO() + events = self.get_event_payload().encode("utf-8") + out = BytesIO() with gzip.GzipFile(fileobj=out, mode="w") as f: f.write(events) @@ -69,7 +64,7 @@ def test_gzip(self): assert r.status_code == 202, r.status_code def test_deflate(self): - events = self.get_event_payload() + events = self.get_event_payload().encode("utf-8") compressed_data = zlib.compress(events) r = requests.post(self.intake_url, data=compressed_data,