Adds --fail-on-flaky option. (#276)

Fails the test run with custom exit code when test passed on rerun.

Co-authored-by: Michael Howitz <[email protected]>

Author: filiplajszczak

CHANGES.rst

@@ -4,10 +4,16 @@ Changelog
 14.1 (unreleased)
 -----------------
 
+Features
+++++++++
+
 - Fix compatibility with pytest 8.2.
   (`#267 <https://github.com/pytest-dev/pytest-rerunfailures/issues/267>`_)
 
 
+- Add ``--fail-on-flaky`` option to fail the test run with custom exit code
+  when test passed on rerun.
+
 14.0 (2024-03-13)
 -----------------
 

src/pytest_rerunfailures.py

@@ -85,6 +85,13 @@ def pytest_addoption(parser):
         "regex provided. Pass this flag multiple times to accumulate a list "
         "of regexes to match",
     )
+    group._addoption(
+        "--fail-on-flaky",
+        action="store_true",
+        dest="fail_on_flaky",
+        help="Fail the test run with exit code 7 if a flaky test passes on a rerun.",
+    )
+
     arg_type = "string"
     parser.addini("reruns", RERUNS_DESC, type=arg_type)
     parser.addini("reruns_delay", RERUNS_DELAY_DESC, type=arg_type)
@@ -614,3 +621,13 @@ def show_rerun(terminalreporter, lines):
         for rep in rerun:
             pos = rep.nodeid
             lines.append(f"RERUN {pos}")
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_sessionfinish(session, exitstatus):
+    if exitstatus != 0:
+        return
+
+    if session.config.option.fail_on_flaky:
+        if session.config.getvalue("reruns") > 0:
+            session.exitstatus = 7

tests/test_pytest_rerunfailures.py

@@ -177,6 +177,28 @@ def test_pass():
     assert_outcomes(result, passed=1, rerun=1)
 
 
+def test_run_with_fail_on_flaky_fails_with_custom_error_code_after_pass_on_rerun(
+    testdir,
+):
+    testdir.makepyfile(
+        f"""
+        def test_pass():
+            {temporary_failure()}"""
+    )
+    result = testdir.runpytest("--reruns", "1", "--fail-on-flaky")
+    assert_outcomes(result, passed=1, rerun=1)
+    assert result.ret == 7
+
+
+def test_run_fails_with_code_1_after_consistent_test_failure_even_with_fail_on_flaky(
+    testdir,
+):
+    testdir.makepyfile("def test_fail(): assert False")
+    result = testdir.runpytest("--reruns", "1", "--fail-on-flaky")
+    assert_outcomes(result, passed=0, failed=1, rerun=1)
+    assert result.ret == 1
+
+
 @pytest.mark.skipif(not has_xdist, reason="requires xdist with crashitem")
 def test_rerun_passes_after_temporary_test_crash(testdir):
     # note: we need two tests because there is a bug where xdist