From 3dcd24b118aa9813db84c5956b938016a8a4f936 Mon Sep 17 00:00:00 2001 From: Jeffrey Meadows Date: Sun, 8 Nov 2015 12:29:20 -0800 Subject: [PATCH] Refactor pytest plugin. This commit partially rewrites the pytest plugin so that retrying a test includes setup and teardown of test fixtures. This is a potentially **breaking change**, but it actually matches up with how the nose plugin works and is probably what most users would want. Fixes #53. --- .pylintrc | 4 +- flaky/_flaky_plugin.py | 127 +++--- flaky/flaky_pytest_plugin.py | 368 ++++++++---------- test/pytest_generate_example/conftest.py | 2 +- .../test_pytest_generate_example.py | 8 +- test/test_flaky_pytest_plugin.py | 30 +- test/test_pytest_example.py | 52 ++- test/test_pytest_options_example.py | 14 +- tox.ini | 6 +- 9 files changed, 293 insertions(+), 318 deletions(-) diff --git a/.pylintrc b/.pylintrc index dfb8e86..ee56d9e 100644 --- a/.pylintrc +++ b/.pylintrc @@ -94,7 +94,7 @@ const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ class-rgx=[A-Z_][a-zA-Z0-9]+$ # Regular expression which should only match correct function names -function-rgx=[a-z_][a-z0-9_]{2,50}$ +function-rgx=[a-z_][a-z0-9_]{2,60}$ # Regular expression which should only match correct method names method-rgx=[a-z_][a-z0-9_]{2,60}$ @@ -134,7 +134,7 @@ docstring-min-length=-1 [FORMAT] # Maximum number of characters on a single line. -max-line-length=80 +max-line-length=120 # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ diff --git a/flaky/_flaky_plugin.py b/flaky/_flaky_plugin.py index 69045df..26af52a 100644 --- a/flaky/_flaky_plugin.py +++ b/flaky/_flaky_plugin.py @@ -100,6 +100,16 @@ def _log_intermediate_failure(self, err, flaky, name): ) self._log_test_failure(name, err, message) + def _should_handle_test_error_or_failure(self, test, name, err): + if not self._has_flaky_attributes(test): + return False, False + flaky = self._get_flaky_attributes(test) + flaky[FlakyNames.CURRENT_RUNS] += 1 + has_failed = self._has_flaky_test_failed(flaky) + if not has_failed: + return True, self._should_rerun_test(test, name, err) + return False, False + def _handle_test_error_or_failure(self, test, err): """ Handle a flaky test error or failure. @@ -127,33 +137,31 @@ def _handle_test_error_or_failure(self, test, err): _, _, name = self._get_test_declaration_callable_and_name(test) except AttributeError: return False - current_runs = self._get_flaky_attribute( - test, - FlakyNames.CURRENT_RUNS, - ) - if current_runs is None: - return False - current_runs += 1 - self._set_flaky_attribute( + + need_reruns, should_rerun = self._should_handle_test_error_or_failure( test, - FlakyNames.CURRENT_RUNS, - current_runs, + name, + err, ) - self._add_flaky_test_failure(test, err) - flaky = self._get_flaky_attributes(test) - if not self._has_flaky_test_failed(flaky): - if self._should_rerun_test(test, name, err): - self._log_intermediate_failure(err, flaky, name) - self._rerun_test(test) - return True + if self._has_flaky_attributes(test): + self._add_flaky_test_failure(test, err) + flaky = self._get_flaky_attributes(test) + runs = flaky[FlakyNames.CURRENT_RUNS] + 1 + self._set_flaky_attribute(test, FlakyNames.CURRENT_RUNS, runs) + if need_reruns: + flaky = self._get_flaky_attributes(test) + if should_rerun: + self._log_intermediate_failure(err, flaky, name) + self._rerun_test(test) + return True + else: + message = self._not_rerun_message + self._log_test_failure(name, err, message) + return False else: - message = self._not_rerun_message - self._log_test_failure(name, err, message) - return False - else: - self._report_final_failure(err, flaky, name) - return False + self._report_final_failure(err, flaky, name) + return False def _should_rerun_test(self, test, name, err): """ @@ -194,6 +202,14 @@ def _rerun_test(self, test): """ raise NotImplementedError # pragma: no cover + def _should_handle_test_success(self, test): + if not self._has_flaky_attributes(test): + return False + flaky = self._get_flaky_attributes(test) + flaky[FlakyNames.CURRENT_PASSES] += 1 + flaky[FlakyNames.CURRENT_RUNS] += 1 + return not self._has_flaky_test_succeeded(flaky) + def _handle_test_success(self, test): """ Handle a flaky test success. @@ -212,48 +228,37 @@ def _handle_test_success(self, test): :rtype: `bool` """ - _, _, name = self._get_test_declaration_callable_and_name(test) - current_runs = self._get_flaky_attribute( - test, - FlakyNames.CURRENT_RUNS - ) - if current_runs is None: + try: + _, _, name = self._get_test_declaration_callable_and_name(test) + except AttributeError: return False - current_runs += 1 - current_passes = self._get_flaky_attribute( - test, - FlakyNames.CURRENT_PASSES - ) - current_passes += 1 - self._set_flaky_attribute( - test, - FlakyNames.CURRENT_RUNS, - current_runs - ) - self._set_flaky_attribute( - test, - FlakyNames.CURRENT_PASSES, - current_passes - ) - flaky = self._get_flaky_attributes(test) - need_reruns = not self._has_flaky_test_succeeded(flaky) - if self._flaky_success_report: + need_reruns = self._should_handle_test_success(test) + + if self._has_flaky_attributes(test): + flaky = self._get_flaky_attributes(test) min_passes = flaky[FlakyNames.MIN_PASSES] - self._stream.writelines([ - ensure_unicode_string(name), - ' passed {0} out of the required {1} times. '.format( - current_passes, - min_passes, - ), - ]) - if need_reruns: - self._stream.write( - 'Running test again until it passes {0} times.\n'.format( + passes = flaky[FlakyNames.CURRENT_PASSES] + 1 + runs = flaky[FlakyNames.CURRENT_RUNS] + 1 + self._set_flaky_attribute(test, FlakyNames.CURRENT_PASSES, passes) + self._set_flaky_attribute(test, FlakyNames.CURRENT_RUNS, runs) + + if self._flaky_success_report: + self._stream.writelines([ + ensure_unicode_string(name), + ' passed {0} out of the required {1} times. '.format( + passes, min_passes, + ), + ]) + if need_reruns: + self._stream.write( + 'Running test again until it passes {0} times.\n'.format( + min_passes, + ) ) - ) - else: - self._stream.write('Success!\n') + else: + self._stream.write('Success!\n') + if need_reruns: self._rerun_test(test) return need_reruns diff --git a/flaky/flaky_pytest_plugin.py b/flaky/flaky_pytest_plugin.py index af0d802..9193e31 100644 --- a/flaky/flaky_pytest_plugin.py +++ b/flaky/flaky_pytest_plugin.py @@ -1,81 +1,29 @@ # coding: utf-8 from __future__ import unicode_literals -import py +import pytest # pylint:disable=import-error -from _pytest.runner import CallInfo, Skipped +from _pytest.runner import CallInfo # pylint:enable=import-error from flaky._flaky_plugin import _FlakyPlugin -def pytest_runtest_protocol(item, nextitem): - """ - Pytest hook to override how tests are run. - """ - PLUGIN.run_test(item, nextitem) - return True - - -def pytest_terminal_summary(terminalreporter): - """ - Pytest hook to write details about flaky tests to the test report. - :param terminalreporter: - Terminal reporter object. Supports stream writing operations. - :type terminalreporter: - :class: `TerminalReporter` - """ - PLUGIN.terminal_summary(terminalreporter) - - -def pytest_addoption(parser): - """ - Pytest hook to add an option to the argument parser. - :param parser: - Parser for command line arguments and ini-file values. - :type parser: - :class:`Parser` - """ - PLUGIN.add_report_option(parser.addoption) - - group = parser.getgroup( - "Force flaky", "Force all tests to be flaky.") - PLUGIN.add_force_flaky_options(group.addoption) - - class FlakyXdist(object): + def __init__(self, plugin): + super(FlakyXdist, self).__init__() + self._plugin = plugin + def pytest_testnodedown(self, node, error): + """ + Pytest hook for responding to a test node shutting down. + Copy slave flaky report output so it's available on the master flaky report. + """ # pylint: disable=unused-argument, no-self-use if hasattr(node, 'slaveoutput') and 'flaky_report' in node.slaveoutput: - PLUGIN.stream.write(node.slaveoutput['flaky_report']) - - -def pytest_configure(config): - """ - Pytest hook to get information about how the test run has been configured. - :param config: - The pytest configuration object for this test run. - :type config: - :class:`Configuration` - """ - PLUGIN.flaky_report = config.option.flaky_report - PLUGIN.flaky_success_report = config.option.flaky_success_report - PLUGIN.force_flaky = config.option.force_flaky - PLUGIN.max_runs = config.option.max_runs - PLUGIN.min_passes = config.option.min_passes - PLUGIN.runner = config.pluginmanager.getplugin("runner") - if config.pluginmanager.hasplugin('xdist'): - config.pluginmanager.register(FlakyXdist()) - PLUGIN.config = config - if hasattr(config, 'slaveoutput'): - config.slaveoutput['flaky_report'] = '' - - -def pytest_sessionfinish(): - if hasattr(PLUGIN.config, 'slaveoutput'): - PLUGIN.config.slaveoutput['flaky_report'] += PLUGIN.stream.getvalue() + self._plugin.stream.write(node.slaveoutput['flaky_report']) class FlakyPlugin(_FlakyPlugin): @@ -84,12 +32,152 @@ class FlakyPlugin(_FlakyPlugin): """ runner = None - _info = None flaky_report = True force_flaky = False max_runs = None min_passes = None config = None + _call_infos = {} + + def pytest_runtest_protocol(self, item, nextitem): + """ + Pytest hook to override how tests are run. + + Runs a test collected by py.test. First, monkey patches the builtin + runner module to call back to FlakyPlugin.call_runtest_hook rather + than its own. Then defer to the builtin runner module to run the test. + :param item: + py.test wrapper for the test function to be run + :type item: + :class:`Function` + :param nextitem: + py.test wrapper for the next test function to be run + :type nextitem: + :class:`Function` + """ + test_instance = self._get_test_instance(item) + self._copy_flaky_attributes(item, test_instance) + if self.force_flaky and not self._has_flaky_attributes(item): + self._make_test_flaky( + item, + self.max_runs, + self.min_passes, + ) + patched_call_runtest_hook = self.runner.call_runtest_hook + self._call_infos = {} + should_rerun = True + try: + self.runner.call_runtest_hook = self.call_runtest_hook + while should_rerun: + self.runner.pytest_runtest_protocol(item, nextitem) + call_info = self._call_infos.get('call', None) + if call_info is None: + return + run = self._call_infos['call'] + passed = run.excinfo is None + if passed: + should_rerun = self.add_success(item) + else: + should_rerun = self.add_failure(item, run.excinfo) + if not should_rerun: + item.excinfo = run.excinfo + finally: + self.runner.call_runtest_hook = patched_call_runtest_hook + del self._call_infos + return True + + @pytest.hookimpl(tryfirst=True, hookwrapper=True) + def pytest_runtest_makereport(self, item, call): + """ + Pytest hook to intercept the report for reruns. + + Change the report's outcome to 'passed' if flaky is going to handle the test run. + That way, pytest will not mark the run as failed. + """ + outcome = yield + if call.when == 'call': + report = outcome.get_result() + report.item = item + report.original_outcome = report.outcome + if report.failed: + if self._should_handle_test_error_or_failure(item, None, None)[1]: + report.outcome = 'passed' + + @pytest.hookimpl(tryfirst=True, hookwrapper=True) + def pytest_report_teststatus(self, report): + """ + Pytest hook to only add final runs to the report. + + Given a test report, get the correpsonding test status. + For tests that flaky is handling, return the empty status + so it isn't reported; otherwise, don't change the status. + """ + outcome = yield + if report.when == 'call': + item = report.item + if report.original_outcome == 'passed': + if self._should_handle_test_success(item): + outcome.force_result(('', '', '')) + elif report.original_outcome == 'failed': + if self._should_handle_test_error_or_failure(item, None, None)[1]: + outcome.force_result(('', '', '')) + delattr(report, 'item') + + def pytest_terminal_summary(self, terminalreporter): + """ + Pytest hook to write details about flaky tests to the test report. + + Write details about flaky tests to the test report. + + :param terminalreporter: + Terminal reporter object. Supports stream writing operations. + :type terminalreporter: + :class: `TerminalReporter` + """ + if self.flaky_report: + self._add_flaky_report(terminalreporter) + + def pytest_addoption(self, parser): + """ + Pytest hook to add an option to the argument parser. + :param parser: + Parser for command line arguments and ini-file values. + :type parser: + :class:`Parser` + """ + self.add_report_option(parser.addoption) + + group = parser.getgroup( + "Force flaky", "Force all tests to be flaky.") + self.add_force_flaky_options(group.addoption) + + def pytest_configure(self, config): + """ + Pytest hook to get information about how the test run has been configured. + :param config: + The pytest configuration object for this test run. + :type config: + :class:`Configuration` + """ + self.flaky_report = config.option.flaky_report + self.flaky_success_report = config.option.flaky_success_report + self.force_flaky = config.option.force_flaky + self.max_runs = config.option.max_runs + self.min_passes = config.option.min_passes + self.runner = config.pluginmanager.getplugin("runner") + if config.pluginmanager.hasplugin('xdist'): + config.pluginmanager.register(FlakyXdist(self)) + self.config = config + if hasattr(config, 'slaveoutput'): + config.slaveoutput['flaky_report'] = '' + + def pytest_sessionfinish(self): + """ + Pytest hook to take a final action after the session is complete. + Copy flaky report contents so that the master process can read it. + """ + if hasattr(self.config, 'slaveoutput'): + self.config.slaveoutput['flaky_report'] += self.stream.getvalue() @property def stream(self): @@ -133,35 +221,6 @@ def _get_test_instance(item): test_instance = item.parent.obj return test_instance - def run_test(self, item, nextitem): - """ - Runs a test collected by py.test. First, monkey patches the builtin - runner module to call back to FlakyPlugin.call_runtest_hook rather - than its own. Then defer to the builtin runner module to run the test. - :param item: - py.test wrapper for the test function to be run - :type item: - :class:`Function` - :param nextitem: - py.test wrapper for the next test function to be run - :type nextitem: - :class:`Function` - """ - test_instance = self._get_test_instance(item) - self._copy_flaky_attributes(item, test_instance) - if self.force_flaky and not self._has_flaky_attributes(item): - self._make_test_flaky( - item, - self.max_runs, - self.min_passes, - ) - patched_call_runtest_hook = self.runner.call_runtest_hook - try: - self.runner.call_runtest_hook = self.call_runtest_hook - self.runner.pytest_runtest_protocol(item, nextitem) - finally: - self.runner.call_runtest_hook = patched_call_runtest_hook - def call_runtest_hook(self, item, when, **kwds): """ Monkey patched from the runner plugin. Responsible for running @@ -174,41 +233,32 @@ def call_runtest_hook(self, item, when, **kwds): """ hookname = "pytest_runtest_" + when ihook = getattr(item.ihook, hookname) - return FlakyCallInfo( - self, - item, + call_info = CallInfo( lambda: ihook(item=item, **kwds), - when=when + when=when, ) + self._call_infos[when] = call_info + return call_info - def add_success(self, info, item): + def add_success(self, item): """ Called when a test succeeds. Count remaining retries and compare with number of required successes that have not yet been achieved; retry if necessary. - :param info: - Information about the test call. - :type info: - :class: `FlakyCallInfo` :param item: py.test wrapper for the test function that has succeeded :type item: :class:`Function` """ - self._info = info return self._handle_test_success(item) - def add_failure(self, info, item, err): + def add_failure(self, item, err): """ Called when a test fails. Count remaining retries and compare with number of required successes that have not yet been achieved; retry if necessary. - :param info: - Information about the test call. - :type info: - :class: `FlakyCallInfo` :param item: py.test wrapper for the test function that has succeeded :type item: @@ -218,24 +268,12 @@ def add_failure(self, info, item, err): :type err: :class: `ExceptionInfo` """ - self._info = info if err is not None: error = (err.type, err.value, err.traceback) else: error = (None, None, None) return self._handle_test_error_or_failure(item, error) - def terminal_summary(self, stream): - """ - Write details about flaky tests to the test report. - :param stream: - The test stream to which the report can be written. - :type stream: - :class: `TerminalReporter` - """ - if self.flaky_report: - self._add_flaky_report(stream) - @staticmethod def _get_test_callable_name(test): """ @@ -295,93 +333,9 @@ def _get_test_declaration_callable_and_name(cls, test): def _rerun_test(self, test): """Base class override. Rerun a flaky test.""" - self._info.call(test.runtest, self) - - -class FlakyCallInfo(CallInfo): - """ - Subclass of pytest default runner's CallInfo. - This subclass has an extracted call method to support - calling the test function again in the case of a rerun. - """ - excinfo = None - result = None - - def __init__(self, plugin, item, func, when): - # pylint:disable=super-init-not-called - #: context of invocation: one of "setup", "call", - #: "teardown", "memocollect" - self._item = item - self._want_rerun = [] - self.excinfo = None - from functools import partial - CallInfo.__init__(self, partial(self.call, func, plugin), when) - - def _handle_error(self, plugin): - """ - Handle an error that occurs during test execution. - If the test is marked flaky and there are reruns remaining, - don't report the test as failed. - """ - # pylint:disable=no-member - err = self.excinfo or py.code.ExceptionInfo() - # pylint:enable=no-member - self.excinfo = None - self._want_rerun.append(plugin.add_failure( - self, - self._item, - err, - )) - self.excinfo = None if self._want_rerun[0] else err - - def call(self, func, plugin): - """ - Call the test function, handling success or failure. - :param func: - The test function to run. - :type func: - `callable` - :param plugin: - Plugin class for flaky that can handle test success or failure. - :type plugin: - :class: `FlakyPlugin` - """ - is_call = self.when == 'call' - try: - self.result = func() - # pytest's unittest plugin for some reason doesn't actually raise - # errors. It just adds them to the unittest result. In order to - # determine whether or not the test needs to be rerun, this - # code looks for the _excinfo attribute set by the plugin. - excinfo = getattr(self._item, '_excinfo', None) - if isinstance(excinfo, list) and len(excinfo) > 0: - self.excinfo = excinfo.pop(0) - except KeyboardInterrupt: - raise - except Skipped: - # pylint:disable=no-member - err = py.code.ExceptionInfo() - # pylint:enable=no-member - self.excinfo = err - return - # pylint:disable=bare-except - except: - if is_call: - self._handle_error(plugin) - else: - raise - else: - if is_call: - if self.excinfo is not None: - if self.excinfo.typename != 'Skipped': - self._handle_error(plugin) - else: - handled_success = plugin.add_success( - self, - self._item, - ) - if not handled_success: - self.excinfo = None PLUGIN = FlakyPlugin() +for _pytest_hook in dir(PLUGIN): + if _pytest_hook.startswith('pytest_'): + globals()[_pytest_hook] = getattr(PLUGIN, _pytest_hook) diff --git a/test/pytest_generate_example/conftest.py b/test/pytest_generate_example/conftest.py index 10d3aec..a09d326 100644 --- a/test/pytest_generate_example/conftest.py +++ b/test/pytest_generate_example/conftest.py @@ -5,4 +5,4 @@ def pytest_generate_tests(metafunc): if 'dummy_list' in metafunc.fixturenames: - metafunc.parametrize("dummy_list", [[]]) + metafunc.parametrize("dummy_list", [['foo']]) diff --git a/test/pytest_generate_example/test_pytest_generate_example.py b/test/pytest_generate_example/test_pytest_generate_example.py index 8a5f75b..4212e4b 100644 --- a/test/pytest_generate_example/test_pytest_generate_example.py +++ b/test/pytest_generate_example/test_pytest_generate_example.py @@ -14,7 +14,9 @@ def test_something_flaky(dummy_list): class TestExample(object): _threshold = -1 + @staticmethod @flaky - def test_flaky_thing_that_fails_then_succeeds(self, dummy_list): - self._threshold += 1 - assert self._threshold >= 1 + def test_flaky_thing_that_fails_then_succeeds(dummy_list): + # pylint:disable=unused-argument + TestExample._threshold += 1 + assert TestExample._threshold >= 1 diff --git a/test/test_flaky_pytest_plugin.py b/test/test_flaky_pytest_plugin.py index b0c887d..7bf58a6 100644 --- a/test/test_flaky_pytest_plugin.py +++ b/test/test_flaky_pytest_plugin.py @@ -9,19 +9,15 @@ from flaky import flaky from flaky import _flaky_plugin from flaky.flaky_pytest_plugin import ( + CallInfo, FlakyPlugin, - FlakyCallInfo, FlakyXdist, PLUGIN, - pytest_sessionfinish, ) from flaky.names import FlakyNames from flaky.utils import unicode_type -# pylint:disable=redefined-outer-name - - @pytest.fixture def mock_io(monkeypatch): mock_string_io = StringIO() @@ -107,7 +103,7 @@ def runtest(self): pass -class MockFlakyCallInfo(FlakyCallInfo): +class MockFlakyCallInfo(CallInfo): def __init__(self, item, when): # pylint:disable=super-init-not-called # super init not called because it has unwanted side effects @@ -123,7 +119,7 @@ def test_flaky_plugin_report(flaky_plugin, mock_io, string_io): expected_string_io.write(flaky_report) expected_string_io.write('\n===End Flaky Test Report===\n') mock_io.write(flaky_report) - flaky_plugin.terminal_summary(string_io) + flaky_plugin.pytest_terminal_summary(string_io) assert string_io.getvalue() == expected_string_io.getvalue() @@ -147,7 +143,7 @@ def test_flaky_xdist_nodedown( assign_slaveoutput, mock_xdist_error ): - flaky_xdist = FlakyXdist() + flaky_xdist = FlakyXdist(PLUGIN) node = Mock() if assign_slaveoutput: node.slaveoutput = mock_xdist_node_slaveoutput @@ -185,7 +181,7 @@ def test_flaky_session_finish_copies_flaky_report( PLUGIN.stream.write(stream_report) PLUGIN.config = Mock() PLUGIN.config.slaveoutput = {'flaky_report': initial_report} - pytest_sessionfinish() + PLUGIN.pytest_sessionfinish() assert PLUGIN.config.slaveoutput['flaky_report'] == expected_report @@ -201,10 +197,7 @@ def test_flaky_plugin_can_suppress_success_report( flaky_plugin._flaky_success_report = False # pylint:enable=protected-access call_info.when = 'call' - actual_plugin_handles_success = flaky_plugin.add_success( - call_info, - flaky_test, - ) + actual_plugin_handles_success = flaky_plugin.add_success(flaky_test) assert actual_plugin_handles_success is False assert string_io.getvalue() == mock_io.getvalue() @@ -283,7 +276,7 @@ def test_flaky_plugin_ignores_success_for_non_flaky_test( string_io, mock_io, ): - flaky_plugin.add_success(call_info, flaky_test) + flaky_plugin.add_success(flaky_test) self._assert_test_ignored(mock_io, string_io, call_info) def test_flaky_plugin_ignores_failure_for_non_flaky_test( @@ -294,7 +287,7 @@ def test_flaky_plugin_ignores_failure_for_non_flaky_test( string_io, mock_io, ): - flaky_plugin.add_failure(call_info, flaky_test, None) + flaky_plugin.add_failure(flaky_test, None) self._assert_test_ignored(mock_io, string_io, call_info) def test_flaky_plugin_handles_failure( @@ -391,7 +384,6 @@ def rerun_filter(err, name, test, plugin): call_info.when = 'call' actual_plugin_handles_failure = flaky_plugin.add_failure( - call_info, flaky_test, mock_error, ) @@ -445,10 +437,7 @@ def _test_flaky_plugin_handles_success( expected_plugin_handles_success = too_few_passes and retries_remaining info.when = 'call' - actual_plugin_handles_success = plugin.add_success( - info, - test, - ) + actual_plugin_handles_success = plugin.add_success(test) assert expected_plugin_handles_success == actual_plugin_handles_success self._assert_flaky_attributes_contains( @@ -516,7 +505,6 @@ def _test_flaky_plugin_handles_failure( info.when = 'call' actual_plugin_handles_failure = plugin.add_failure( - info, test, mock_error, ) diff --git a/test/test_pytest_example.py b/test/test_pytest_example.py index f25e730..a52ba22 100644 --- a/test/test_pytest_example.py +++ b/test/test_pytest_example.py @@ -19,9 +19,17 @@ def test_something_flaky(dummy_list=[]): assert len(dummy_list) > 1 -class TestExample(object): - _threshold = -1 +@pytest.fixture(scope='class') +def threshold_provider(): + return {'threshold': -1} + + +@pytest.fixture(scope='class') +def threshold_provider_2(): + return {'threshold': -1} + +class TestExample(object): def test_non_flaky_thing(self): """Flaky will not interact with this test""" pass @@ -31,23 +39,25 @@ def test_non_flaky_failing_thing(self): """Flaky will also not interact with this test""" assert self == 1 + @staticmethod @flaky(3, 2) - def test_flaky_thing_that_fails_then_succeeds(self): + def test_flaky_thing_that_fails_then_succeeds(threshold_provider): """ Flaky will run this test 3 times. It will fail once and then succeed twice. """ - self._threshold += 1 - assert self._threshold >= 1 + threshold_provider['threshold'] += 1 + assert threshold_provider['threshold'] >= 1 + @staticmethod @flaky(3, 2) - def test_flaky_thing_that_succeeds_then_fails_then_succeeds(self): + def test_flaky_thing_that_succeeds_then_fails_then_succeeds(threshold_provider_2): """ Flaky will run this test 3 times. It will succeed once, fail once, and then succeed one more time. """ - self._threshold += 1 - assert self._threshold != 1 + threshold_provider_2['threshold'] += 1 + assert threshold_provider_2['threshold'] != 1 @flaky(2, 2) def test_flaky_thing_that_always_passes(self): @@ -68,26 +78,28 @@ def test_flaky_thing_that_always_fails(self): class TestExampleFlakyTests(object): _threshold = -1 - def test_flaky_thing_that_fails_then_succeeds(self): + @staticmethod + def test_flaky_thing_that_fails_then_succeeds(): """ Flaky will run this test twice. It will fail once and then succeed. """ - self._threshold += 1 - assert self._threshold >= 1 + TestExampleFlakyTests._threshold += 1 + assert TestExampleFlakyTests._threshold >= 1 @flaky class TestExampleFlakyTestCase(TestCase): _threshold = -1 - def test_flaky_thing_that_fails_then_succeeds(self): + @staticmethod + def test_flaky_thing_that_fails_then_succeeds(): """ Flaky will run this test twice. It will fail once and then succeed. """ - self._threshold += 1 - assert self._threshold >= 1 + TestExampleFlakyTestCase._threshold += 1 + assert TestExampleFlakyTestCase._threshold >= 1 class TestFlakySubclass(TestExampleFlakyTestCase): @@ -101,3 +113,15 @@ def _test_flaky_doctest(): True """ return True + + +@pytest.fixture +def my_fixture(): + return 42 + + +@flaky +def test_requiring_my_fixture(my_fixture, dummy_list=[]): + # pylint:disable=dangerous-default-value,unused-argument + dummy_list.append(0) + assert len(dummy_list) > 1 diff --git a/test/test_pytest_options_example.py b/test/test_pytest_options_example.py index 34314bf..509ad9a 100644 --- a/test/test_pytest_options_example.py +++ b/test/test_pytest_options_example.py @@ -18,28 +18,30 @@ def test_something_flaky(dummy_list=[]): class TestExample(object): _threshold = -2 + @staticmethod @flaky(3, 1) - def test_flaky_thing_that_fails_then_succeeds(self): + def test_flaky_thing_that_fails_then_succeeds(): """ Flaky will run this test 3 times. It will fail twice and then succeed once. This ensures that the flaky decorator overrides any command-line options we specify. """ - self._threshold += 1 - assert self._threshold >= 1 + TestExample._threshold += 1 + assert TestExample._threshold >= 1 @flaky(3, 1) class TestExampleFlakyTests(object): _threshold = -2 - def test_flaky_thing_that_fails_then_succeeds(self): + @staticmethod + def test_flaky_thing_that_fails_then_succeeds(): """ Flaky will run this test 3 times. It will fail twice and then succeed once. This ensures that the flaky decorator on a test suite overrides any command-line options we specify. """ - self._threshold += 1 - assert self._threshold >= 1 + TestExampleFlakyTests._threshold += 1 + assert TestExampleFlakyTests._threshold >= 1 diff --git a/tox.ini b/tox.ini index c000883..eed5445 100644 --- a/tox.ini +++ b/tox.ini @@ -23,13 +23,13 @@ commands = [testenv:pep8] commands = - pep8 flaky - pep8 test + pep8 --ignore=E501 flaky + pep8 --ignore=E501 test [testenv:pylint] commands = pylint --rcfile=.pylintrc flaky - pylint --rcfile=.pylintrc test -d C0330 + pylint --rcfile=.pylintrc test -d C0330,W0621 [testenv:coverage] commands =