Introduce no_fnmatch_line/no_re_match_line in pytester

changelog/5914.feature.rst

@@ -0,0 +1,19 @@
+``pytester`` learned two new functions, `no_fnmatch_line <https://docs.pytest.org/en/latest/reference.html#_pytest.pytester.LineMatcher.no_fnmatch_line>`_ and
+`no_re_match_line <https://docs.pytest.org/en/latest/reference.html#_pytest.pytester.LineMatcher.no_re_match_line>`_.
+
+The functions are used to ensure the captured text *does not* match the given
+pattern.
+
+The previous idiom was to use ``re.match``:
+
+.. code-block:: python
+
+    assert re.match(pat, result.stdout.str()) is None
+
+Or the ``in`` operator:
+
+.. code-block:: python
+
+    assert text in result.stdout.str()
+
+But the new functions produce best output on failure.

src/_pytest/pytester.py

@@ -1318,8 +1318,7 @@ def fnmatch_lines(self, lines2):
 
         The argument is a list of lines which have to match and can use glob
         wildcards.  If they do not match a pytest.fail() is called.  The
-        matches and non-matches are also printed on stdout.
-
+        matches and non-matches are also shown as part of the error message.
         """
         __tracebackhide__ = True
         self._match_lines(lines2, fnmatch, "fnmatch")
@@ -1330,8 +1329,7 @@ def re_match_lines(self, lines2):
         The argument is a list of lines which have to match using ``re.match``.
         If they do not match a pytest.fail() is called.
 
-        The matches and non-matches are also printed on stdout.
-
+        The matches and non-matches are also shown as part of the error message.
         """
         __tracebackhide__ = True
         self._match_lines(lines2, lambda name, pat: re.match(pat, name), "re.match")
@@ -1374,3 +1372,40 @@ def _match_lines(self, lines2, match_func, match_nickname):
             else:
                 self._log("remains unmatched: {!r}".format(line))
                 pytest.fail(self._log_text)
+
+    def no_fnmatch_line(self, pat):
+        """Ensure captured lines do not match the given pattern, using ``fnmatch.fnmatch``.
+
+        :param str pat: the pattern to match lines.
+        """
+        __tracebackhide__ = True
+        self._no_match_line(pat, fnmatch, "fnmatch")
+
+    def no_re_match_line(self, pat):
+        """Ensure captured lines do not match the given pattern, using ``re.match``.
+
+        :param str pat: the regular expression to match lines.
+        """
+        __tracebackhide__ = True
+        self._no_match_line(pat, lambda name, pat: re.match(pat, name), "re.match")
+
+    def _no_match_line(self, pat, match_func, match_nickname):
+        """Ensure captured lines does not have a the given pattern, using ``fnmatch.fnmatch``
+
+        :param str pat: the pattern to match lines
+        """
+        __tracebackhide__ = True
+        nomatch_printed = False
+        try:
+            for line in self.lines:
+                if match_func(line, pat):
+                    self._log("%s:" % match_nickname, repr(pat))
+                    self._log("   with:", repr(line))
+                    pytest.fail(self._log_text)
+                else:
+                    if not nomatch_printed:
+                        self._log("nomatch:", repr(pat))
+                        nomatch_printed = True
+                    self._log("    and:", repr(line))
+        finally:
+            self._log_output = []

testing/acceptance_test.py

@@ -246,7 +246,7 @@ def test_issue93_initialnode_importing_capturing(self, testdir):
         )
         result = testdir.runpytest()
         assert result.ret == ExitCode.NO_TESTS_COLLECTED
-        assert "should not be seen" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*should not be seen*")
         assert "stderr42" not in result.stderr.str()
 
     def test_conftest_printing_shows_if_error(self, testdir):
@@ -954,7 +954,7 @@ def test_with_failing_collection(self, testdir):
         result.stdout.fnmatch_lines(["*Interrupted: 1 errors during collection*"])
         # Collection errors abort test execution, therefore no duration is
         # output
-        assert "duration" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*duration*")
 
     def test_with_not(self, testdir):
         testdir.makepyfile(self.source)
@@ -1008,7 +1008,7 @@ def main():
     result = testdir.runpython(target)
     assert result.ret == 0
     result.stderr.fnmatch_lines(["*not found*foo*"])
-    assert "INTERNALERROR>" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*INTERNALERROR>*")
 
 
 def test_import_plugin_unicode_name(testdir):

testing/code/test_excinfo.py

@@ -399,7 +399,7 @@ def test_division_zero():
     result = testdir.runpytest()
     assert result.ret != 0
     result.stdout.fnmatch_lines(["*AssertionError*Pattern*[123]*not found*"])
-    assert "__tracebackhide__ = True" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*__tracebackhide__ = True*")
 
     result = testdir.runpytest("--fulltrace")
     assert result.ret != 0
@@ -1343,7 +1343,8 @@ def test(tmpdir):
     )
     result = testdir.runpytest()
     result.stdout.fnmatch_lines(["* 1 failed in *"])
-    assert "INTERNALERROR" not in result.stdout.str() + result.stderr.str()
+    result.stdout.no_fnmatch_line("*INTERNALERROR*")
+    result.stderr.no_fnmatch_line("*INTERNALERROR*")
 
 
 @pytest.mark.usefixtures("limited_recursion_depth")

testing/logging/test_fixture.py

@@ -46,7 +46,7 @@ def test2(caplog):
     )
     result = testdir.runpytest()
     result.stdout.fnmatch_lines(["*log from test1*", "*2 failed in *"])
-    assert "log from test2" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*log from test2*")
 
 
 def test_with_statement(caplog):

testing/logging/test_reporting.py

@@ -109,7 +109,7 @@ def test_foo():
             "=* 1 failed in *=",
         ]
     )
-    assert "DEBUG" not in result.stdout.str()
+    result.stdout.no_re_match_line("DEBUG")
 
 
 def test_setup_logging(testdir):
@@ -282,7 +282,7 @@ def test_log_cli(request):
             "WARNING*test_log_cli_default_level.py* message will be shown*",
         ]
     )
-    assert "INFO message won't be shown" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*INFO message won't be shown*")
     # make sure that that we get a '0' exit code for the testsuite
     assert result.ret == 0
 
@@ -566,7 +566,7 @@ def test_log_cli(request):
             "PASSED",  # 'PASSED' on its own line because the log message prints a new line
         ]
     )
-    assert "This log message won't be shown" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*This log message won't be shown*")
 
     # make sure that that we get a '0' exit code for the testsuite
     assert result.ret == 0
@@ -580,7 +580,7 @@ def test_log_cli(request):
             "PASSED",  # 'PASSED' on its own line because the log message prints a new line
         ]
     )
-    assert "This log message won't be shown" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*This log message won't be shown*")
 
     # make sure that that we get a '0' exit code for the testsuite
     assert result.ret == 0
@@ -616,7 +616,7 @@ def test_log_cli(request):
             "PASSED",  # 'PASSED' on its own line because the log message prints a new line
         ]
     )
-    assert "This log message won't be shown" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*This log message won't be shown*")
 
     # make sure that that we get a '0' exit code for the testsuite
     assert result.ret == 0
@@ -942,15 +942,15 @@ def test_simple():
             ]
         )
     elif verbose == "-q":
-        assert "collected 1 item*" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*collected 1 item**")
         expected_lines.extend(
             [
                 "*test_collection_collect_only_live_logging.py::test_simple*",
                 "no tests ran in 0.[0-9][0-9]s",
             ]
         )
     elif verbose == "-qq":
-        assert "collected 1 item*" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*collected 1 item**")
         expected_lines.extend(["*test_collection_collect_only_live_logging.py: 1*"])
 
     result.stdout.fnmatch_lines(expected_lines)
@@ -983,7 +983,7 @@ def test_simple():
 
     result = testdir.runpytest()
 
-    assert "--- live log collection ---" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*--- live log collection ---*")
 
     assert result.ret == 0
     assert os.path.isfile(log_file)

testing/python/collect.py

@@ -1139,7 +1139,7 @@ class Test(object):
     """
     )
     result = testdir.runpytest()
-    assert "TypeError" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*TypeError*")
     assert result.ret == ExitCode.NO_TESTS_COLLECTED
 
 

testing/python/fixtures.py

@@ -455,7 +455,7 @@ def test_lookup_error(unknown):
                 "*1 error*",
             ]
         )
-        assert "INTERNAL" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*INTERNAL*")
 
     def test_fixture_excinfo_leak(self, testdir):
         # on python2 sys.excinfo would leak into fixture executions
@@ -2647,7 +2647,7 @@ def test_finish():
             *3 passed*
         """
         )
-        assert "error" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*error*")
 
     def test_fixture_finalizer(self, testdir):
         testdir.makeconftest(
@@ -3151,7 +3151,7 @@ def arg1():
             *hello world*
         """
         )
-        assert "arg0" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*arg0*")
 
     @pytest.mark.parametrize("testmod", [True, False])
     def test_show_fixtures_conftest(self, testdir, testmod):

testing/python/setup_only.py

@@ -27,7 +27,7 @@ def test_arg1(arg1):
     result.stdout.fnmatch_lines(
         ["*SETUP    F arg1*", "*test_arg1 (fixtures used: arg1)*", "*TEARDOWN F arg1*"]
     )
-    assert "_arg0" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*_arg0*")
 
 
 def test_show_different_scopes(testdir, mode):

testing/python/show_fixtures_per_test.py

@@ -1,6 +1,6 @@
 def test_no_items_should_not_show_output(testdir):
     result = testdir.runpytest("--fixtures-per-test")
-    assert "fixtures used by" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*fixtures used by*")
     assert result.ret == 0
 
 
@@ -30,7 +30,7 @@ def test_arg1(arg1):
             "    arg1 docstring",
         ]
     )
-    assert "_arg0" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*_arg0*")
 
 
 def test_fixtures_in_conftest(testdir):

testing/test_assertion.py

@@ -1034,7 +1034,7 @@ def test_hello():
     result = testdir.runpytest()
     assert "3 == 4" in result.stdout.str()
     result = testdir.runpytest_subprocess("--assert=plain")
-    assert "3 == 4" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*3 == 4*")
 
 
 def test_triple_quoted_string_issue113(testdir):
@@ -1046,7 +1046,7 @@ def test_hello():
     )
     result = testdir.runpytest("--fulltrace")
     result.stdout.fnmatch_lines(["*1 failed*"])
-    assert "SyntaxError" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*SyntaxError*")
 
 
 def test_traceback_failure(testdir):

testing/test_assertrewrite.py

@@ -914,7 +914,7 @@ def test_rewrite_warning_using_pytest_plugins(self, testdir):
         testdir.chdir()
         result = testdir.runpytest_subprocess()
         result.stdout.fnmatch_lines(["*= 1 passed in *=*"])
-        assert "pytest-warning summary" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*pytest-warning summary*")
 
     def test_rewrite_warning_using_pytest_plugins_env_var(self, testdir, monkeypatch):
         monkeypatch.setenv("PYTEST_PLUGINS", "plugin")
@@ -932,7 +932,7 @@ def test():
         testdir.chdir()
         result = testdir.runpytest_subprocess()
         result.stdout.fnmatch_lines(["*= 1 passed in *=*"])
-        assert "pytest-warning summary" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*pytest-warning summary*")
 
 
 class TestAssertionRewriteHookDetails:
@@ -1124,7 +1124,7 @@ def test_long_repr():
     """
     )
     result = testdir.runpytest()
-    assert "unbalanced braces" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*unbalanced braces*")
 
 
 class TestIssue925:

testing/test_cacheprovider.py

@@ -327,7 +327,7 @@ def test_always_fails():
         result = testdir.runpytest("--lf", "--ff")
         # Test order will be failing tests firs
         result.stdout.fnmatch_lines(["test_b.py*"])
-        assert "test_a.py" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*test_a.py*")
 
     def test_lastfailed_difference_invocations(self, testdir, monkeypatch):
         monkeypatch.setenv("PYTHONDONTWRITEBYTECODE", "1")
@@ -660,11 +660,11 @@ def test_lf_and_ff_prints_no_needless_message(self, quiet, opt, testdir):
         if quiet:
             args.append("-q")
         result = testdir.runpytest(*args)
-        assert "run all" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*run all*")
 
         result = testdir.runpytest(*args)
         if quiet:
-            assert "run all" not in result.stdout.str()
+            result.stdout.no_fnmatch_line("*run all*")
         else:
             assert "rerun previous" in result.stdout.str()
 

testing/test_capture.py

@@ -609,12 +609,12 @@ def test_normal():
             *while capture is disabled*
         """
         )
-        assert "captured before" not in result.stdout.str()
-        assert "captured after" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*captured before*")
+        result.stdout.no_fnmatch_line("*captured after*")
         if no_capture:
             assert "test_normal executed" in result.stdout.str()
         else:
-            assert "test_normal executed" not in result.stdout.str()
+            result.stdout.no_fnmatch_line("*test_normal executed*")
 
     @pytest.mark.parametrize("fixture", ["capsys", "capfd"])
     def test_fixture_use_by_other_fixtures(self, testdir, fixture):
@@ -650,8 +650,8 @@ def test_captured_print(captured_print):
         )
         result = testdir.runpytest_subprocess()
         result.stdout.fnmatch_lines(["*1 passed*"])
-        assert "stdout contents begin" not in result.stdout.str()
-        assert "stderr contents begin" not in result.stdout.str()
+        result.stdout.no_fnmatch_line("*stdout contents begin*")
+        result.stdout.no_fnmatch_line("*stderr contents begin*")
 
     @pytest.mark.parametrize("cap", ["capsys", "capfd"])
     def test_fixture_use_by_other_fixtures_teardown(self, testdir, cap):
@@ -721,7 +721,7 @@ def pytest_runtest_setup():
     testdir.makepyfile("def test_func(): pass")
     result = testdir.runpytest()
     assert result.ret == 0
-    assert "hello19" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*hello19*")
 
 
 def test_capture_badoutput_issue412(testdir):
@@ -1388,7 +1388,7 @@ def test_spam_in_thread():
     result = testdir.runpytest_subprocess(str(p))
     assert result.ret == 0
     assert result.stderr.str() == ""
-    assert "IOError" not in result.stdout.str()
+    result.stdout.no_fnmatch_line("*IOError*")
 
 
 def test_pickling_and_unpickling_encoded_file():