pytester: Hookrecorder: improve assertoutcome

Before:

        def assertoutcome(self, passed: int = 0, skipped: int = 0, failed: int = 0) -> None:
            realpassed, realskipped, realfailed = self.listoutcomes()
            assert passed == len(realpassed)
    >       assert skipped == len(realskipped)
    E       assert 1 == 0
    E        +  where 0 = len([])

After:

    >       reprec = testdir.inline_run(testpath, "-s")
    E       AssertionError: ([], [], [<TestReport 'nodeid' when='call' outcome='failed'>])
    E       assert {'failed': 1, 'passed': 0, 'skipped': 0} == {'failed': 0, 'passed': 0, 'skipped': 1}

Author: blueyed

changelog/6176.improvement.rst

@@ -0,0 +1 @@
+Improved failure reporting with pytester's ``Hookrecorder.assertoutcome``.

src/_pytest/pytester.py

@@ -332,10 +332,17 @@ def countoutcomes(self) -> List[int]:
         return [len(x) for x in self.listoutcomes()]
 
     def assertoutcome(self, passed: int = 0, skipped: int = 0, failed: int = 0) -> None:
-        realpassed, realskipped, realfailed = self.listoutcomes()
-        assert passed == len(realpassed)
-        assert skipped == len(realskipped)
-        assert failed == len(realfailed)
+        __tracebackhide__ = True
+
+        outcomes = self.listoutcomes()
+        realpassed, realskipped, realfailed = outcomes
+        obtained = {
+            "passed": len(realpassed),
+            "skipped": len(realskipped),
+            "failed": len(realfailed),
+        }
+        expected = {"passed": passed, "skipped": skipped, "failed": failed}
+        assert obtained == expected, outcomes
 
     def clear(self) -> None:
         self.calls[:] = []

testing/test_assertion.py

@@ -70,7 +70,14 @@ def test_dummy_failure(testdir):  # how meta!
         """
         )
         result = testdir.runpytest_subprocess()
-        result.stdout.fnmatch_lines(["*assert 1 == 0*"])
+        result.stdout.fnmatch_lines(
+            [
+                "E * AssertionError: ([[][]], [[][]], [[]<TestReport *>[]])*",
+                "E * assert"
+                " {'failed': 1, 'passed': 0, 'skipped': 0} =="
+                " {'failed': 0, 'passed': 1, 'skipped': 0}",
+            ]
+        )
 
     @pytest.mark.parametrize("mode", ["plain", "rewrite"])
     def test_pytest_plugins_rewrite(self, testdir, mode):