diff --git a/README.rst b/README.rst index 747072e..399c5b5 100644 --- a/README.rst +++ b/README.rst @@ -352,6 +352,10 @@ be marked as failed. Compatibility ------------- +* When using ``--junitxml``, failed attempts that are re-run are recorded as + ```` (or ```` for setup/teardown failures) + elements in the test's final ````, following the convention used + by Maven Surefire. * This plugin is *not* compatible with pytest-xdist's --looponfail flag. * This plugin is *not* compatible with the core --pdb flag. * This plugin is *not* compatible with the plugin diff --git a/changes/170.feature.rst b/changes/170.feature.rst new file mode 100644 index 0000000..16ec8d4 --- /dev/null +++ b/changes/170.feature.rst @@ -0,0 +1,3 @@ +Record failed attempts that are re-run as ```` (and +```` for setup/teardown failures) elements in the test's final +```` in JUnit XML output, instead of dropping them. diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 616e658..9acada3 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -14,6 +14,7 @@ from contextlib import suppress from itertools import chain from typing import Any +from xml.etree import ElementTree import pytest from _pytest.outcomes import Exit, fail @@ -46,6 +47,11 @@ failed_subtests_key = _failed_subtests_key SubtestReport = _SubtestReport +try: + from _pytest.junitxml import bin_xml_escape +except ImportError: + bin_xml_escape = None + try: from xdist.newhooks import pytest_handlecrashitem @@ -729,6 +735,81 @@ def pytest_configure(config): else: config.failures_db = StatusDB() # no-op db + config.pluginmanager.register(JunitXmlRerunReporter(config)) + + +def _get_junitxml_plugin(config): + """Return the registered LogXML plugin, or None if --junitxml is off.""" + for plugin in config.pluginmanager.get_plugins(): + if ( + type(plugin).__module__ == "_pytest.junitxml" + and type(plugin).__name__ == "LogXML" + ): + return plugin + return None + + +class JunitXmlRerunReporter: + """Record failed attempts as ``flakyFailure`` elements in JUnit XML. + + pytest's junitxml plugin ignores reports with outcome "rerun" and + finalizes the ```` element after every attempt's teardown + report, so the failure information of retried attempts is lost. Stash the + rerun reports and append ```` elements (the convention used + by Maven Surefire and the ``flaky`` plugin; ``flakyError`` for setup and + teardown failures) when the final ``call`` report arrives, so they land + in the last attempt's ````. + """ + + def __init__(self, config): + self.config = config + self.pending = {} + + @staticmethod + def _key(report): + return ( + report.nodeid, + getattr(report, "worker_id", None), + getattr(report, "item_index", None), + getattr(report, "node", None), + ) + + @pytest.hookimpl(tryfirst=True) + def pytest_runtest_logreport(self, report): + if bin_xml_escape is None: + return + key = self._key(report) + if report.outcome == "rerun": + self.pending.setdefault(key, []).append(report) + return + if report.when != "call" and not report.failed: + return + # a failed setup or teardown report that is not a rerun means the + # attempts are exhausted in that phase and no call report follows + reruns = self.pending.pop(key, None) + if not reruns: + return + xml = _get_junitxml_plugin(self.config) + if xml is None: + return + reporter = xml.node_reporter(report) + reporter.record_testreport(report) + for rerun_report in reruns: + if rerun_report.longrepr is None: + continue + tag = "flakyFailure" if rerun_report.when == "call" else "flakyError" + reprcrash = getattr(rerun_report.longrepr, "reprcrash", None) + message = ( + reprcrash.message + if reprcrash is not None + else str(rerun_report.longrepr) + ) + node = ElementTree.Element( + tag, type="failure", message=bin_xml_escape(message) + ) + node.text = bin_xml_escape(str(rerun_report.longrepr)) + reporter.append(node) + class XDistHooks: def pytest_sessionstart(self, session): diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index a4208c5..ffc8490 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -548,6 +548,60 @@ def test_pass(): assert_outcomes(result, passed=1, rerun=2) +def _junitxml_testcases(testdir, filename="result.xml"): + import xml.etree.ElementTree as ET + + tree = ET.parse(str(testdir.tmpdir.join(filename))) # noqa: S314 + return tree.getroot().findall(".//testcase") + + +def test_junitxml_records_rerun_failures_as_flaky_failure(testdir): + testdir.makepyfile( + f""" + def test_pass(): + {temporary_failure(2)}""" + ) + result = testdir.runpytest("--reruns", "2", "--junitxml=result.xml") + assert_outcomes(result, passed=1, rerun=2) + + testcases = _junitxml_testcases(testdir) + flaky_failures = testcases[-1].findall("flakyFailure") + assert len(flaky_failures) == 2 + assert testcases[-1].findall("failure") == [] + assert "Failure: 1" in flaky_failures[0].get("message") + assert "Failure: 1" in flaky_failures[0].text + + +def test_junitxml_rerun_failures_before_final_failure(testdir): + testdir.makepyfile("def test_fail(): assert False") + result = testdir.runpytest("--reruns", "1", "--junitxml=result.xml") + assert_outcomes(result, passed=0, failed=1, rerun=1) + + testcases = _junitxml_testcases(testdir) + assert len(testcases[-1].findall("flakyFailure")) == 1 + assert len(testcases[-1].findall("failure")) == 1 + + +def test_junitxml_rerun_errors_before_final_setup_failure(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.fixture + def broken_fixture(): + raise ValueError("setup always fails") + + def test_fail(broken_fixture): pass + """ + ) + result = testdir.runpytest("--reruns", "1", "--junitxml=result.xml") + assert_outcomes(result, passed=0, failed=0, error=1, rerun=1) + + testcases = _junitxml_testcases(testdir) + assert len(testcases[-1].findall("flakyError")) == 1 + assert len(testcases[-1].findall("error")) == 1 + + def test_no_extra_test_summary_for_reruns_by_default(testdir): testdir.makepyfile( f"""