Skip to content

Commit

Permalink
test: Restructure tests to deduplicate cmake runs (#305)
Browse files Browse the repository at this point in the history
  • Loading branch information
Swatinem authored Jun 17, 2020
1 parent b972a24 commit 205e38f
Show file tree
Hide file tree
Showing 9 changed files with 166 additions and 151 deletions.
2 changes: 1 addition & 1 deletion azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ stages:
- stage: package
displayName: Create Release Archive
dependsOn: test
condition: eq(variables['System.PullRequest.IsFork'], 'False')
condition: and(succeeded('test'), eq(variables['System.PullRequest.IsFork'], 'False'))
jobs:
- job:
pool:
Expand Down
74 changes: 1 addition & 73 deletions tests/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ def make_dsn(httpserver, auth="uiaeosnrtdy", id=123456):


def run(cwd, exe, args, env=dict(os.environ), **kwargs):
__tracebackhide__ = True
if os.environ.get("ANDROID_API"):
# older android emulators do not correctly pass down the returncode
# so we basically echo the return code, and parse it manually
Expand Down Expand Up @@ -72,79 +73,6 @@ def check_output(*args, **kwargs):
return stdout


def cmake(cwd, targets, options=None):
if options is None:
options = {}
options.update(
{
"CMAKE_RUNTIME_OUTPUT_DIRECTORY": cwd,
"CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG": cwd,
"CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE": cwd,
}
)
if os.environ.get("ANDROID_API") and os.environ.get("ANDROID_NDK"):
# See: https://developer.android.com/ndk/guides/cmake
toolchain = "{}/ndk/{}/build/cmake/android.toolchain.cmake".format(
os.environ["ANDROID_HOME"], os.environ["ANDROID_NDK"]
)
options.update(
{
"CMAKE_TOOLCHAIN_FILE": toolchain,
"ANDROID_ABI": os.environ.get("ANDROID_ARCH") or "x86",
"ANDROID_NATIVE_API_LEVEL": os.environ["ANDROID_API"],
}
)
cmake = ["cmake"]
if "scan-build" in os.environ.get("RUN_ANALYZER", ""):
cmake = ["scan-build", "cmake"]
configcmd = [*cmake]
for key, value in options.items():
configcmd.append("-D{}={}".format(key, value))
if sys.platform == "win32" and os.environ.get("TEST_X86"):
configcmd.append("-AWin32")
elif sys.platform == "linux" and os.environ.get("TEST_X86"):
configcmd.append("-DSENTRY_BUILD_FORCE32=ON")
if "asan" in os.environ.get("RUN_ANALYZER", ""):
configcmd.append("-DWITH_ASAN_OPTION=ON")

cmakelists_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
configcmd.append(cmakelists_dir)

# we have to set `-Werror` for this cmake invocation only, otherwise
# completely unrelated things will break
env = dict(os.environ)
if os.environ.get("ERROR_ON_WARNINGS"):
env["CFLAGS"] = env["CXXFLAGS"] = "-Werror"
if sys.platform == "win32":
# MP = object level parallelism, WX = warnings as errors
cpus = os.cpu_count()
env["CFLAGS"] = env["CXXFLAGS"] = "/WX /MP{}".format(cpus)
if "gcc" in os.environ.get("RUN_ANALYZER", ""):
env["CFLAGS"] = env["CXXFLAGS"] = "{} -fanalyzer".format(env["CFLAGS"])

print("\n{} > {}".format(cwd, " ".join(configcmd)), flush=True)
subprocess.run(configcmd, cwd=cwd, env=env, check=True)

buildcmd = [*cmake, "--build", ".", "--parallel"]
for target in targets:
buildcmd.extend(["--target", target])
print("{} > {}".format(cwd, " ".join(buildcmd)), flush=True)
subprocess.run(buildcmd, cwd=cwd, check=True)

if os.environ.get("ANDROID_API"):
# copy the output to the android image via adb
subprocess.run(
[
"{}/platform-tools/adb".format(os.environ["ANDROID_HOME"]),
"push",
"./",
"/data/local/tmp",
],
cwd=cwd,
check=True,
)


# Adapted from: https://raw.githubusercontent.com/getsentry/sentry-python/276acae955ee13f7ac3a7728003626eff6d943a8/sentry_sdk/envelope.py


Expand Down
104 changes: 104 additions & 0 deletions tests/cmake.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
import os
import sys
import subprocess


class CMake:
def __init__(self, factory):
self.runs = dict()
self.factory = factory

def compile(self, targets, options=None):
if options is None:
options = dict()
key = (
";".join(targets),
";".join(f"{k}={v}" for k, v in options.items()),
)

if key not in self.runs:
cwd = self.factory.mktemp("cmake")
cmake(cwd, targets, options)
self.runs[key] = cwd

return self.runs[key]


def cmake(cwd, targets, options=None):
__tracebackhide__ = True
if options is None:
options = {}
options.update(
{
"CMAKE_RUNTIME_OUTPUT_DIRECTORY": cwd,
"CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG": cwd,
"CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE": cwd,
}
)
if os.environ.get("ANDROID_API") and os.environ.get("ANDROID_NDK"):
# See: https://developer.android.com/ndk/guides/cmake
toolchain = "{}/ndk/{}/build/cmake/android.toolchain.cmake".format(
os.environ["ANDROID_HOME"], os.environ["ANDROID_NDK"]
)
options.update(
{
"CMAKE_TOOLCHAIN_FILE": toolchain,
"ANDROID_ABI": os.environ.get("ANDROID_ARCH") or "x86",
"ANDROID_NATIVE_API_LEVEL": os.environ["ANDROID_API"],
}
)
cmake = ["cmake"]
if "scan-build" in os.environ.get("RUN_ANALYZER", ""):
cmake = ["scan-build", "cmake"]
configcmd = [*cmake]
for key, value in options.items():
configcmd.append("-D{}={}".format(key, value))
if sys.platform == "win32" and os.environ.get("TEST_X86"):
configcmd.append("-AWin32")
elif sys.platform == "linux" and os.environ.get("TEST_X86"):
configcmd.append("-DSENTRY_BUILD_FORCE32=ON")
if "asan" in os.environ.get("RUN_ANALYZER", ""):
configcmd.append("-DWITH_ASAN_OPTION=ON")

cmakelists_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
configcmd.append(cmakelists_dir)

# we have to set `-Werror` for this cmake invocation only, otherwise
# completely unrelated things will break
env = dict(os.environ)
if os.environ.get("ERROR_ON_WARNINGS"):
env["CFLAGS"] = env["CXXFLAGS"] = "-Werror"
if sys.platform == "win32":
# MP = object level parallelism, WX = warnings as errors
cpus = os.cpu_count()
env["CFLAGS"] = env["CXXFLAGS"] = "/WX /MP{}".format(cpus)
if "gcc" in os.environ.get("RUN_ANALYZER", ""):
env["CFLAGS"] = env["CXXFLAGS"] = "{} -fanalyzer".format(env["CFLAGS"])

print("\n{} > {}".format(cwd, " ".join(configcmd)), flush=True)
try:
subprocess.run(configcmd, cwd=cwd, env=env, check=True)
except subprocess.CalledProcessError:
pytest.fail("cmake configure failed")

buildcmd = [*cmake, "--build", ".", "--parallel"]
for target in targets:
buildcmd.extend(["--target", target])
print("{} > {}".format(cwd, " ".join(buildcmd)), flush=True)
try:
subprocess.run(buildcmd, cwd=cwd, check=True)
except subprocess.CalledProcessError:
pytest.fail("cmake build failed")

if os.environ.get("ANDROID_API"):
# copy the output to the android image via adb
subprocess.run(
[
"{}/platform-tools/adb".format(os.environ["ANDROID_HOME"]),
"push",
"./",
"/data/local/tmp",
],
cwd=cwd,
check=True,
)
26 changes: 6 additions & 20 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import os
import pytest
import re
from . import cmake, run
from . import run
from .cmake import CMake


def enumerate_unittests():
Expand All @@ -20,23 +21,8 @@ def pytest_generate_tests(metafunc):
metafunc.parametrize("unittest", enumerate_unittests())


class Unittests:
def __init__(self, dir):
# for unit tests, the backend does not matter, and we want to keep
# the compile-times down
cmake(
dir,
["sentry_test_unit"],
{"SENTRY_BACKEND": "none", "SENTRY_TRANSPORT": "none"},
)
self.dir = dir

def run(self, test):
env = dict(os.environ)
run(self.dir, "sentry_test_unit", ["--no-summary", test], check=True, env=env)


@pytest.fixture(scope="session")
def unittests(tmp_path_factory):
tmpdir = tmp_path_factory.mktemp("unittests")
return Unittests(tmpdir)
def cmake(tmp_path_factory):
cmake = CMake(tmp_path_factory)

return cmake.compile
7 changes: 2 additions & 5 deletions tests/test_integration_crashpad.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import pytest
from .conditions import has_crashpad
from . import cmake

# TODO:
# * with crashpad backend:
Expand All @@ -10,9 +9,7 @@


@pytest.mark.skipif(not has_crashpad, reason="test needs crashpad backend")
def test_crashpad_build(tmp_path):
def test_crashpad_build(cmake):
cmake(
tmp_path,
["sentry_example"],
{"SENTRY_BACKEND": "crashpad", "SENTRY_TRANSPORT": "none"},
["sentry_example"], {"SENTRY_BACKEND": "crashpad", "SENTRY_TRANSPORT": "none"},
)
33 changes: 15 additions & 18 deletions tests/test_integration_http.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import subprocess
import sys
import os
from . import cmake, make_dsn, check_output, run, Envelope
from . import make_dsn, check_output, run, Envelope
from .conditions import has_http, has_inproc, has_breakpad
from .assertions import (
assert_attachment,
Expand All @@ -23,9 +23,8 @@
)


def test_capture_http(tmp_path, httpserver):
# we want to have the default transport
cmake(tmp_path, ["sentry_example"], {"SENTRY_BACKEND": "none"})
def test_capture_http(cmake, httpserver):
tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"})

httpserver.expect_oneshot_request(
"/api/123456/envelope/", headers={"x-sentry-auth": auth_header},
Expand Down Expand Up @@ -53,9 +52,8 @@ def test_capture_http(tmp_path, httpserver):
assert_event(envelope)


def test_session_http(tmp_path, httpserver):
# we want to have the default transport
cmake(tmp_path, ["sentry_example"], {"SENTRY_BACKEND": "none"})
def test_session_http(cmake, httpserver):
tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"})

httpserver.expect_request(
"/api/123456/envelope/", headers={"x-sentry-auth": auth_header},
Expand Down Expand Up @@ -84,9 +82,8 @@ def test_session_http(tmp_path, httpserver):
assert_session(envelope, {"init": True, "status": "exited", "errors": 0})


def test_capture_and_session_http(tmp_path, httpserver):
# we want to have the default transport
cmake(tmp_path, ["sentry_example"], {"SENTRY_BACKEND": "none"})
def test_capture_and_session_http(cmake, httpserver):
tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"})

httpserver.expect_request(
"/api/123456/envelope/", headers={"x-sentry-auth": auth_header},
Expand All @@ -113,8 +110,8 @@ def test_capture_and_session_http(tmp_path, httpserver):


@pytest.mark.skipif(not has_inproc, reason="test needs inproc backend")
def test_inproc_crash_http(tmp_path, httpserver):
cmake(tmp_path, ["sentry_example"], {"SENTRY_BACKEND": "inproc"})
def test_inproc_crash_http(cmake, httpserver):
tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "inproc"})

child = run(tmp_path, "sentry_example", ["start-session", "attachment", "crash"])
assert child.returncode # well, its a crash after all
Expand Down Expand Up @@ -149,8 +146,8 @@ def test_inproc_crash_http(tmp_path, httpserver):


@pytest.mark.skipif(not has_inproc, reason="test needs inproc backend")
def test_inproc_dump_inflight(tmp_path, httpserver):
cmake(tmp_path, ["sentry_example"], {"SENTRY_BACKEND": "inproc"})
def test_inproc_dump_inflight(cmake, httpserver):
tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "inproc"})

httpserver.expect_request(
"/api/123456/envelope/", headers={"x-sentry-auth": auth_header},
Expand All @@ -167,8 +164,8 @@ def test_inproc_dump_inflight(tmp_path, httpserver):


@pytest.mark.skipif(not has_breakpad, reason="test needs breakpad backend")
def test_breakpad_crash_http(tmp_path, httpserver):
cmake(tmp_path, ["sentry_example"], {"SENTRY_BACKEND": "breakpad"})
def test_breakpad_crash_http(cmake, httpserver):
tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "breakpad"})

child = run(tmp_path, "sentry_example", ["start-session", "attachment", "crash"])
assert child.returncode # well, its a crash after all
Expand Down Expand Up @@ -204,8 +201,8 @@ def test_breakpad_crash_http(tmp_path, httpserver):


@pytest.mark.skipif(not has_breakpad, reason="test needs breakpad backend")
def test_breakpad_dump_inflight(tmp_path, httpserver):
cmake(tmp_path, ["sentry_example"], {"SENTRY_BACKEND": "breakpad"})
def test_breakpad_dump_inflight(cmake, httpserver):
tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "breakpad"})

httpserver.expect_request(
"/api/123456/envelope/", headers={"x-sentry-auth": auth_header},
Expand Down
11 changes: 5 additions & 6 deletions tests/test_integration_ratelimits.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,14 @@
import pytest
import os
from . import cmake, make_dsn, run
from . import make_dsn, run
from .conditions import has_http

if not has_http:
pytest.skip("tests need http", allow_module_level=True)


def test_retry_after(tmp_path, httpserver):
# we want to have the default transport
cmake(tmp_path, ["sentry_example"], {"SENTRY_BACKEND": "none"})
def test_retry_after(cmake, httpserver):
tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"})

env = dict(os.environ, SENTRY_DSN=make_dsn(httpserver))

Expand All @@ -27,8 +26,8 @@ def test_retry_after(tmp_path, httpserver):
assert len(httpserver.log) == 2


def test_rate_limits(tmp_path, httpserver):
cmake(tmp_path, ["sentry_example"], {"SENTRY_BACKEND": "none"})
def test_rate_limits(cmake, httpserver):
tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"})

env = dict(os.environ, SENTRY_DSN=make_dsn(httpserver))
headers = {"X-Sentry-Rate-Limits": "60::organization"}
Expand Down
Loading

0 comments on commit 205e38f

Please sign in to comment.