Skip to content

Commit

Permalink
error report refactoring (fixes #205 fixes #201 fixes #169 via #208)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sseliverstov
sseliverstov authored Feb 13, 2018
1 parent 07438c1 commit 76c1bf8
Show file tree
Hide file tree
Showing 23 changed files with 717 additions and 208 deletions.
2 changes: 1 addition & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
.DS_Store
.idea
.tox
.cache
.pytest_cache
.python-version

*.pyc
Expand Down
4 changes: 2 additions & 2 deletions allure-behave/features/steps/report_steps.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from allure_commons_test.result import has_attachment
from allure_commons_test.result import has_parameter
from allure_commons_test.result import has_status_details
from allure_commons_test.result import with_status_message
from allure_commons_test.result import with_message_contains
from allure_commons_test.container import has_container
from allure_commons_test.container import has_before, has_after
from allure_commons_test.label import has_severity
Expand Down Expand Up @@ -86,7 +86,7 @@ def step_status(context, item, status):
@then(u'this {item} has status details with message "{message}"')
def step_status(context, item, message):
context_matcher = getattr(context, item)
matcher = partial(context_matcher, has_status_details, with_status_message, message)
matcher = partial(context_matcher, has_status_details, with_message_contains, message)
assert_that(context.allure_report, matcher())


Expand Down
48 changes: 21 additions & 27 deletions allure-pytest/src/listener.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from allure_pytest.utils import allure_full_name, allure_package, allure_name
from allure_pytest.utils import get_status, get_status_details
from allure_pytest.utils import get_outcome_status, get_outcome_status_details
from allure_pytest.utils import get_pytest_report_status


class AllureListener(object):
Expand Down Expand Up @@ -142,45 +143,38 @@ def pytest_fixture_post_finalizer(self, fixturedef):
@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(self, item, call):
uuid = self._cache.set(item.nodeid)

report = (yield).get_result()
allure_item = self.allure_logger.get_item(uuid)
status = allure_item.status or None

test_result = self.allure_logger.get_test(uuid)
status = get_pytest_report_status(report)
status_details = None

if call.excinfo and hasattr(call.excinfo.value, 'msg'):
status_details = StatusDetails(message=call.excinfo.value.msg)
elif hasattr(report, 'wasxfail'):
status_details = StatusDetails(message=report.wasxfail)
elif report.failed:
if call.excinfo:
status_details = StatusDetails(message=call.excinfo.exconly(), trace=report.longreprtext)
if (status != Status.SKIPPED
and not (call.excinfo.errisinstance(AssertionError)
or call.excinfo.errisinstance(pytest.fail.Exception))):
status = Status.BROKEN

if status == Status.PASSED and hasattr(report, 'wasxfail'):
reason = report.wasxfail
message = 'XPASS {reason}'.format(reason=reason) if reason else 'XPASS'
status_details = StatusDetails(message=message)

if report.when == 'setup':
if report.passed:
status = Status.PASSED
if report.failed:
status = Status.BROKEN
if report.skipped:
status = Status.SKIPPED
test_result.status = status
test_result.statusDetails = status_details

if report.when == 'call':
if report.passed and status == Status.PASSED:
pass
if report.failed:
status = Status.FAILED
if report.skipped:
status = Status.SKIPPED
if test_result.status == Status.PASSED:
test_result.status = status
test_result.statusDetails = status_details

if report.when == 'teardown':
if report.failed and status == Status.PASSED:
status = Status.BROKEN

test_result = self.allure_logger.get_test(uuid)
if test_result:
if status_details:
if status in (Status.FAILED, Status.BROKEN) and test_result.status == Status.PASSED:
test_result.status = status
test_result.statusDetails = status_details
else:
test_result.status = status

@allure_commons.hookimpl
def attach_data(self, body, name, attachment_type, extension):
Expand Down
22 changes: 15 additions & 7 deletions allure-pytest/src/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,16 +120,24 @@ def get_outcome_status_details(outcome):

def get_status(exception):
if exception:
if isinstance(exception, AssertionError):
if isinstance(exception, AssertionError) or isinstance(exception, pytest.fail.Exception):
return Status.FAILED
elif isinstance(exception, pytest.skip.Exception):
return Status.SKIPPED
return Status.PASSED
return Status.BROKEN
else:
return Status.PASSED


def get_status_details(exception_type, exception, exception_traceback):
if isinstance(exception, pytest.skip.Exception):
return StatusDetails(message=exception.msg)
elif exception:
return StatusDetails(message=format_exception(exception_type, exception),
trace=format_traceback(exception_traceback))
message = format_exception(exception_type, exception)
trace = format_traceback(exception_traceback)
return StatusDetails(message=message, trace=trace) if message or trace else None


def get_pytest_report_status(pytest_report):
pytest_statuses = ('failed', 'passed', 'skipped')
statuses = (Status.FAILED, Status.PASSED, Status.SKIPPED)
for pytest_status, status in zip(pytest_statuses, statuses):
if getattr(pytest_report, pytest_status):
return status
2 changes: 1 addition & 1 deletion allure-pytest/test/fixtures/function_scope/finalizers_simple_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def test_two_fixures_with_finalizer(fixture_with_passed_finalizer, fixture_faile
... finalizer='failed_finalizer'),
... with_status('failed'),
... has_status_details(
... with_status_message('AssertionError')
... with_message_contains('AssertionError')
... )
... )
... )
Expand Down
2 changes: 1 addition & 1 deletion allure-pytest/test/fixtures/function_scope/fixtures_parametrization_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def test_function_scope_parametrized_fixture(parametrized_fixture):
... has_before('parametrized_fixture',
... with_status('passed' if passed else 'failed'),
... has_status_details(
... with_status_message('AssertionError')
... with_message_contains('AssertionError')
... ) if not passed else anything()
... )
... )
Expand Down
74 changes: 74 additions & 0 deletions allure-pytest/test/status/base_call_status_test.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import pytest


def test_passed():
"""
>>> allure_report = getfixture('allure_report')
>>> assert_that(allure_report,
... has_test_case('test_passed',
... with_status('passed')
... )
... )
"""
pass


def test_failed():
"""
>>> allure_report = getfixture('allure_report')
>>> assert_that(allure_report,
... has_test_case('test_failed',
... with_status('failed'),
... has_status_details(with_message_contains("AssertionError"),
... with_trace_contains("def test_failed():")
... )
... )
... )
"""
assert False


def test_broken():
"""
>>> allure_report = getfixture('allure_report')
>>> assert_that(allure_report,
... has_test_case('test_broken',
... with_status('broken'),
... has_status_details(with_message_contains("IndentationError"),
... with_trace_contains("def test_broken():")
... )
... )
... )
"""
raise IndentationError()


def test_call_pytest_fail():
"""
>>> allure_report = getfixture('allure_report')
>>> assert_that(allure_report,
... has_test_case('test_call_pytest_fail',
... with_status('failed'),
... has_status_details(with_message_contains("Failed: <Failed instance>"),
... with_trace_contains("def test_call_pytest_fail():")
... )
... )
... )
"""
pytest.fail()


def test_call_pytest_fail_with_reason():
"""
>>> allure_report = getfixture('allure_report')
>>> assert_that(allure_report,
... has_test_case('test_call_pytest_fail_with_reason',
... with_status('failed'),
... has_status_details(with_message_contains("Fail message"),
... with_trace_contains("def test_call_pytest_fail():")
... )
... )
... )
"""
pytest.fail("Fail message")

140 changes: 140 additions & 0 deletions allure-pytest/test/status/base_setup_status_test.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
import pytest


@pytest.fixture
def failed_fixture():
assert False


def test_failed_fixture(failed_fixture):
"""
>>> allure_report = getfixture('allure_report')
>>> assert_that(allure_report,
... has_test_case('test_failed_fixture',
... with_status('failed'),
... has_status_details(with_message_contains("AssertionError"),
... with_trace_contains("def failed_fixture():")
... ),
... has_container(allure_report,
... has_before('failed_fixture',
... with_status('failed'),
... has_status_details(with_message_contains("AssertionError"),
... with_trace_contains("failed_fixture")
... ),
... ),
... )
... )
... )
"""
pass


@pytest.fixture
def broken_fixture():
raise IndexError


def test_broken_fixture(broken_fixture):
"""
>>> allure_report = getfixture('allure_report')
>>> assert_that(allure_report,
... has_test_case('test_broken_fixture',
... with_status('broken'),
... has_status_details(with_message_contains("IndexError"),
... with_trace_contains("def broken_fixture():")
... ),
... has_container(allure_report,
... has_before('broken_fixture',
... with_status('broken'),
... has_status_details(with_message_contains("IndexError"),
... with_trace_contains("broken_fixture")
... ),
... ),
... )
... )
... )
"""
pass


@pytest.fixture
def skip_fixture():
pytest.skip()


def test_skip_fixture(skip_fixture):
"""
>>> allure_report = getfixture('allure_report')
>>> assert_that(allure_report,
... has_test_case('test_skip_fixture',
... with_status('skipped'),
... has_status_details(with_message_contains("Skipped: <Skipped instance>")),
... has_container(allure_report,
... has_before('skip_fixture',
... with_status('skipped'),
... has_status_details(with_message_contains("Skipped: <Skipped instance>"),
... with_trace_contains("skip_fixture")
... ),
... ),
... )
... )
... )
"""


@pytest.fixture
def pytest_fail_fixture():
pytest.fail()


def test_pytest_fail_fixture(pytest_fail_fixture):
"""
>>> allure_report = getfixture('allure_report')
>>> assert_that(allure_report,
... has_test_case('test_pytest_fail_fixture',
... with_status('failed'),
... has_status_details(with_message_contains("Failed: <Failed instance>"),
... with_trace_contains("def pytest_fail_fixture():")
... ),
... has_container(allure_report,
... has_before('pytest_fail_fixture',
... with_status('failed'),
... has_status_details(with_message_contains("Failed: <Failed instance>"),
... with_trace_contains("pytest_fail_fixture")
... ),
... ),
... )
... )
... )
"""
pass


@pytest.fixture
def pytest_fail_with_reason_fixture():
pytest.fail("Fail message")


def test_pytest_fail_with_reason_fixture(pytest_fail_with_reason_fixture):
"""
>>> allure_report = getfixture('allure_report')
>>> assert_that(allure_report,
... has_test_case('test_pytest_fail_with_reason_fixture',
... with_status('failed'),
... has_status_details(with_message_contains("Fail message"),
... with_trace_contains("def pytest_fail_with_reason_fixture():")
... ),
... has_container(allure_report,
... has_before('pytest_fail_with_reason_fixture',
... with_status('failed'),
... has_status_details(with_message_contains("Fail message"),
... with_trace_contains("pytest_fail_with_reason_fixture")
... ),
... ),
... )
... )
... )
"""
pass


Loading

0 comments on commit 76c1bf8

Please sign in to comment.