From f139aaed0a94db88c9a46d74f3cbc55aebfe456a Mon Sep 17 00:00:00 2001 From: Kirill Zhdanov Date: Sun, 11 Feb 2024 15:54:46 +0700 Subject: [PATCH 1/3] fix: collect artifacts on test setup/teardown (#117) --- pytest.ini | 5 ++ pytest_playwright/pytest_playwright.py | 78 +++++++++++++++++++++++++- tests/test_playwright.py | 69 +++++++++++++++++++++++ 3 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..615ecc6 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +; [pytest] +; log_cli=true +; log_cli_level=DEBUG +; log_cli_format=%(asctime)s.%(msecs)03d %(levelname)s %(name)s:%(filename)s:%(lineno)s: %(message)s +; log_cli_date_format=%Y-%m-%d %H:%M:%S diff --git a/pytest_playwright/pytest_playwright.py b/pytest_playwright/pytest_playwright.py index 7fe0fb0..1cbae79 100644 --- a/pytest_playwright/pytest_playwright.py +++ b/pytest_playwright/pytest_playwright.py @@ -18,7 +18,7 @@ import sys import warnings from typing import Any, Callable, Dict, Generator, List, Optional - +import logging import pytest from playwright.sync_api import ( Browser, @@ -33,6 +33,8 @@ import tempfile +log = logging.getLogger(__name__) + artifacts_folder = tempfile.TemporaryDirectory(prefix="playwright-pytest-") @@ -74,6 +76,34 @@ def pytest_configure(config: Any) -> None: "markers", "browser_context_args(**kwargs): provide additional arguments to browser.new_context()", ) + log.debug("pytest_configure") + class Teardown: + failed = False + setattr(config, "teardown", Teardown) + + +def pytest_runtest_teardown(item): + # import faulthandler + # faulthandler.dump_traceback(all_threads=True) + log.debug("pytest_runtest_teardown") + item.config.teardown.failed = True + + +def pytest_sessionfinish(session, exitstatus): + log.debug("pytest_runtest_teardown") + log.debug(f"{exitstatus=}") + + +def pytest_exception_interact(node, call, report): + log.debug("1pytest_exception_interact") + log.debug(f"{node.config.teardown.failed=}") + log.debug(f"{node.session.testsfailed=}") + excinfo = call.exc_info if hasattr(call, "exc_info") else None + log.debug(f"{excinfo=}") + + if call.when == "teardown": + log.debug("2pytest_exception_interact") + node.config.teardown.failed = True # Making test result information available in fixtures @@ -256,10 +286,52 @@ def context( ) yield context + # import traceback + # log.debug(f"{dir(traceback)=}") + # log.debug(f"{traceback.print_last()=}") + # stack = traceback.extract_stack() + # log.debug(f"{stack=}") + # for frame_summary in stack: + # log.debug(f"{frame_summary.filename=}") + # log.debug(f"{frame_summary.name=}") + # log.debug(f"{frame_summary.colno=}") + + + # log.debug(f"{traceback.print_stack()=}") + # log.debug(f"{traceback.print_exc()=}") + + log.debug(f"{request.session.testscollected=}") + log.debug(f"{request.session.exitstatus=}") + log.debug(f"{request.session.testsfailed=}") + + if request.session.testsfailed: + log.debug("Only print if failed") + + log.debug("context") # If request.node is missing rep_call, then some error happened during execution # that prevented teardown, but should still be counted as a failure - failed = request.node.rep_call.failed if hasattr(request.node, "rep_call") else True + failed_setup = request.node.rep_setup.failed if hasattr(request.node, "rep_setup") else False + failed_call = request.node.rep_call.failed if hasattr(request.node, "rep_call") else False + failed_teardown = request.node.rep_teardown.failed if hasattr(request.node, "rep_teardown") else False + + failed_xteardown = request.config.teardown.failed if hasattr(request.config, "teardown") else False + + + failed = failed_setup or failed_call or failed_xteardown + log.debug(f"{failed=}") + log.debug(f"{failed_setup=}") + log.debug(f"{failed_call=}") + log.debug(f"{failed_teardown=}") + log.debug(f"{failed_xteardown=}") + + + log.debug(f"{hasattr(request.node, 'rep_setup')=}") + log.debug(f"{hasattr(request.node, 'rep_call')=}") + log.debug(f"{hasattr(request.node, 'rep_teardown')=}") + + log.debug(f"{hasattr(request.config, 'teardown')=}") + if capture_trace: retain_trace = tracing_option == "on" or ( @@ -272,9 +344,11 @@ def context( context.tracing.stop() screenshot_option = pytestconfig.getoption("--screenshot") + log.debug(f"{screenshot_option=}") capture_screenshot = screenshot_option == "on" or ( failed and screenshot_option == "only-on-failure" ) + log.debug(f"{capture_screenshot=}") if capture_screenshot: for index, page in enumerate(pages): human_readable_status = "failed" if failed else "finished" diff --git a/tests/test_playwright.py b/tests/test_playwright.py index bceb8e9..45c10c7 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -703,6 +703,73 @@ def test_failing(page): _assert_folder_tree(test_results_dir, expected) +def test_artifacts_retain_on_setup_failure(testdir: pytest.Testdir) -> None: + testdir.makepyfile( + """ + import pytest + @pytest.fixture + def failed_setup_call(page): + assert 1 == page.evaluate("1 + 1") + yield page + + def test_failing(page, failed_setup_call): + assert 2 == page.evaluate("1 + 1") + """ + ) + result = testdir.runpytest( + "--screenshot", + "only-on-failure" + ) + result.assert_outcomes(errors=1) + test_results_dir = os.path.join(testdir.tmpdir, "test-results") + + expected = [ + { + "name": "test-artifacts-retain-on-setup-failure-py-test-failing-chromium", + "children": [ + { + "name": "test-failed-1.png", + } + ], + } + ] + _assert_folder_tree(test_results_dir, expected) + +# request.config.teardown.failed = False +def test_artifacts_on_teardown(testdir: pytest.Testdir) -> None: + testdir.makepyfile( + """ + import pytest + @pytest.fixture + def failed_teardown_call(page, request): + yield page + assert 1 == page.evaluate("1 + 1") + + def test_passing(page, failed_teardown_call): + assert 2 == page.evaluate("1 + 1") + """ + ) + result = testdir.runpytest( + "--screenshot", + "only-on-failure" + ) + result.assert_outcomes(passed=1, errors=1) # сделать чтобы скрин не сохранялся при успешном тесте (и все успешные тирдауны и сетапы) + test_results_dir = os.path.join(testdir.tmpdir, "test-results") + print(f"{os.listdir(test_results_dir)=}") + print(os.listdir(f"{test_results_dir}/test-artifacts-on-teardown-py-test-passing-chromium")) + expected = [ + { + "name": "test-artifacts-on-teardown-py-test-passing-chromium", + "children": [ + { + "name": "test-failed-1.png", + } + ], + } + ] + _assert_folder_tree(test_results_dir, expected) + + def test_should_work_with_test_names_which_exceeds_256_characters( testdir: pytest.Testdir, ) -> None: @@ -734,8 +801,10 @@ def _assert_folder_tree(root: str, expected_tree: List[Any]) -> None: for file in expected_tree: if isinstance(file["name"], str): if "children" in file: + print(f"{file=}") assert os.path.isdir(os.path.join(root, file["name"])) else: + print(f"{file=}") assert os.path.isfile(os.path.join(root, file["name"])) if isinstance(file["name"], re.Pattern): assert any([file["name"].match(item) for item in os.listdir(root)]) From 097bf59dd25a631beed06e29c984bb3ddef0b16f Mon Sep 17 00:00:00 2001 From: Kirill Zhdanov Date: Mon, 12 Feb 2024 15:09:30 +0700 Subject: [PATCH 2/3] fix: parse tb for teardown exception (#117) --- pytest_playwright/pytest_playwright.py | 98 +++++++------------------- 1 file changed, 25 insertions(+), 73 deletions(-) diff --git a/pytest_playwright/pytest_playwright.py b/pytest_playwright/pytest_playwright.py index 1cbae79..ecec6fe 100644 --- a/pytest_playwright/pytest_playwright.py +++ b/pytest_playwright/pytest_playwright.py @@ -16,9 +16,12 @@ import shutil import os import sys + +import traceback + import warnings from typing import Any, Callable, Dict, Generator, List, Optional -import logging + import pytest from playwright.sync_api import ( Browser, @@ -33,8 +36,6 @@ import tempfile -log = logging.getLogger(__name__) - artifacts_folder = tempfile.TemporaryDirectory(prefix="playwright-pytest-") @@ -76,34 +77,6 @@ def pytest_configure(config: Any) -> None: "markers", "browser_context_args(**kwargs): provide additional arguments to browser.new_context()", ) - log.debug("pytest_configure") - class Teardown: - failed = False - setattr(config, "teardown", Teardown) - - -def pytest_runtest_teardown(item): - # import faulthandler - # faulthandler.dump_traceback(all_threads=True) - log.debug("pytest_runtest_teardown") - item.config.teardown.failed = True - - -def pytest_sessionfinish(session, exitstatus): - log.debug("pytest_runtest_teardown") - log.debug(f"{exitstatus=}") - - -def pytest_exception_interact(node, call, report): - log.debug("1pytest_exception_interact") - log.debug(f"{node.config.teardown.failed=}") - log.debug(f"{node.session.testsfailed=}") - excinfo = call.exc_info if hasattr(call, "exc_info") else None - log.debug(f"{excinfo=}") - - if call.when == "teardown": - log.debug("2pytest_exception_interact") - node.config.teardown.failed = True # Making test result information available in fixtures @@ -286,52 +259,33 @@ def context( ) yield context - # import traceback - # log.debug(f"{dir(traceback)=}") - # log.debug(f"{traceback.print_last()=}") - # stack = traceback.extract_stack() - # log.debug(f"{stack=}") - # for frame_summary in stack: - # log.debug(f"{frame_summary.filename=}") - # log.debug(f"{frame_summary.name=}") - # log.debug(f"{frame_summary.colno=}") - - - # log.debug(f"{traceback.print_stack()=}") - - # log.debug(f"{traceback.print_exc()=}") - - log.debug(f"{request.session.testscollected=}") - log.debug(f"{request.session.exitstatus=}") - log.debug(f"{request.session.testsfailed=}") - - if request.session.testsfailed: - log.debug("Only print if failed") - - log.debug("context") # If request.node is missing rep_call, then some error happened during execution # that prevented teardown, but should still be counted as a failure - failed_setup = request.node.rep_setup.failed if hasattr(request.node, "rep_setup") else False - failed_call = request.node.rep_call.failed if hasattr(request.node, "rep_call") else False - failed_teardown = request.node.rep_teardown.failed if hasattr(request.node, "rep_teardown") else False - - failed_xteardown = request.config.teardown.failed if hasattr(request.config, "teardown") else False - - - failed = failed_setup or failed_call or failed_xteardown - log.debug(f"{failed=}") - log.debug(f"{failed_setup=}") - log.debug(f"{failed_call=}") - log.debug(f"{failed_teardown=}") - log.debug(f"{failed_xteardown=}") + failed_setup = ( + request.node.rep_setup.failed if hasattr(request.node, "rep_setup") else False + ) + failed_call = ( + request.node.rep_call.failed if hasattr(request.node, "rep_call") else False + ) + passed_setup = ( + request.node.rep_setup.passed if hasattr(request.node, "rep_setup") else False + ) + passed_call = ( + request.node.rep_call.passed if hasattr(request.node, "rep_call") else False + ) - log.debug(f"{hasattr(request.node, 'rep_setup')=}") - log.debug(f"{hasattr(request.node, 'rep_call')=}") - log.debug(f"{hasattr(request.node, 'rep_teardown')=}") + failed_xteardown = False - log.debug(f"{hasattr(request.config, 'teardown')=}") + if (passed_setup or passed_call) and not (failed_setup or failed_call): + # check tb under stack if any other teardown was failed, False by default + # looks like workaround for https://github.com/pytest-dev/pytest/issues/9909 + for trace, _ in traceback.walk_stack(None): + if trace.f_locals.get("these_exceptions"): + failed_xteardown = True + break + failed = failed_setup or failed_call or failed_xteardown if capture_trace: retain_trace = tracing_option == "on" or ( @@ -344,11 +298,9 @@ def context( context.tracing.stop() screenshot_option = pytestconfig.getoption("--screenshot") - log.debug(f"{screenshot_option=}") capture_screenshot = screenshot_option == "on" or ( failed and screenshot_option == "only-on-failure" ) - log.debug(f"{capture_screenshot=}") if capture_screenshot: for index, page in enumerate(pages): human_readable_status = "failed" if failed else "finished" From 50f951c8c5b47459fb10f4f478cea636582ab931 Mon Sep 17 00:00:00 2001 From: Kirill Zhdanov Date: Mon, 12 Feb 2024 15:10:14 +0700 Subject: [PATCH 3/3] test: cover teardown edge cases; remove logs (#117) --- pytest.ini | 5 --- tests/test_playwright.py | 82 ++++++++++++++++++++++++++++++++++------ 2 files changed, 71 insertions(+), 16 deletions(-) delete mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 615ecc6..0000000 --- a/pytest.ini +++ /dev/null @@ -1,5 +0,0 @@ -; [pytest] -; log_cli=true -; log_cli_level=DEBUG -; log_cli_format=%(asctime)s.%(msecs)03d %(levelname)s %(name)s:%(filename)s:%(lineno)s: %(message)s -; log_cli_date_format=%Y-%m-%d %H:%M:%S diff --git a/tests/test_playwright.py b/tests/test_playwright.py index 45c10c7..e0e5c6c 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -718,25 +718,34 @@ def test_failing(page, failed_setup_call): ) result = testdir.runpytest( "--screenshot", - "only-on-failure" + "only-on-failure", + "--video", + "retain-on-failure", + "--tracing", + "retain-on-failure", ) result.assert_outcomes(errors=1) test_results_dir = os.path.join(testdir.tmpdir, "test-results") - expected = [ { "name": "test-artifacts-retain-on-setup-failure-py-test-failing-chromium", "children": [ + { + "name": re.compile(r".*webm"), + }, { "name": "test-failed-1.png", - } + }, + { + "name": "trace.zip", + }, ], } ] _assert_folder_tree(test_results_dir, expected) -# request.config.teardown.failed = False -def test_artifacts_on_teardown(testdir: pytest.Testdir) -> None: + +def test_artifacts_retain_on_teardown_failure(testdir: pytest.Testdir) -> None: testdir.makepyfile( """ import pytest @@ -751,25 +760,76 @@ def test_passing(page, failed_teardown_call): ) result = testdir.runpytest( "--screenshot", - "only-on-failure" + "only-on-failure", + "--video", + "retain-on-failure", + "--tracing", + "retain-on-failure", ) - result.assert_outcomes(passed=1, errors=1) # сделать чтобы скрин не сохранялся при успешном тесте (и все успешные тирдауны и сетапы) + result.assert_outcomes(passed=1, errors=1) test_results_dir = os.path.join(testdir.tmpdir, "test-results") - print(f"{os.listdir(test_results_dir)=}") - print(os.listdir(f"{test_results_dir}/test-artifacts-on-teardown-py-test-passing-chromium")) expected = [ { - "name": "test-artifacts-on-teardown-py-test-passing-chromium", + "name": "test-artifacts-retain-on-teardown-failure-py-test-passing-chromium", "children": [ + { + "name": re.compile(r".*webm"), + }, { "name": "test-failed-1.png", - } + }, + { + "name": "trace.zip", + }, ], } ] _assert_folder_tree(test_results_dir, expected) +def test_empty_artifacts_on_teardown(testdir: pytest.Testdir) -> None: + testdir.makepyfile( + """ + import pytest + @pytest.fixture + def failed_teardown_call(page, request): + yield page + assert 2 == page.evaluate("1 + 1") + + def test_passing(page, failed_teardown_call): + assert 2 == page.evaluate("1 + 1") + """ + ) + result = testdir.runpytest( + "--screenshot", + "only-on-failure", + "--video", + "retain-on-failure", + "--tracing", + "retain-on-failure", + ) + result.assert_outcomes(passed=1) + test_results_dir = os.path.join(testdir.tmpdir, "test-results") + expected = [ + { + "name": "test-empty-artifacts-on-teardown-py-test-passing-chromium", + "children": [ + { + "name": re.compile(r".*webm"), + }, + { + "name": "test-failed-1.png", + }, + { + "name": "trace.zip", + }, + ], + } + ] + with pytest.raises(FileNotFoundError): + _assert_folder_tree(test_results_dir, expected) + + def test_should_work_with_test_names_which_exceeds_256_characters( testdir: pytest.Testdir, ) -> None: