Merge pull request #4384 from tybug/realize-for-failure

Add `for_failure: bool = False` to `provider.realize`

Author: Zac-HD

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+Add a ``for_failure: bool = False`` parameter to ``provider.realize`` in :ref:`alternative backends <alternative-backends>`, so that symbolic-based backends can increase their timeouts when realizing failures, which are more important than regular examples.

hypothesis-python/src/hypothesis/internal/conjecture/data.py

@@ -850,10 +850,9 @@ def draw_float(
         *,
         allow_nan: bool = True,
         smallest_nonzero_magnitude: float = SMALLEST_SUBNORMAL,
-        # TODO: consider supporting these float widths at the IR level in the
-        # future.
+        # TODO: consider supporting these float widths at the choice sequence
+        # level in the future.
         # width: Literal[16, 32, 64] = 64,
-        # exclude_min and exclude_max handled higher up,
         forced: Optional[float] = None,
         observe: bool = True,
     ) -> float:
@@ -1006,12 +1005,12 @@ def _pop_choice(
             bytes: "bytes",
         }[type(choice)]
         # If we're trying to:
-        # * draw a different ir type at the same location
-        # * draw the same ir type with a different constraints, which does not permit
+        # * draw a different choice type at the same location
+        # * draw the same choice type with a different constraints, which does not permit
         #   the current value
         #
         # then we call this a misalignment, because the choice sequence has
-        # slipped from what we expected at some point. An easy misalignment is
+        # changed from what we expected at some point. An easy misalignment is
         #
         #   one_of(integers(0, 100), integers(101, 200))
         #

hypothesis-python/src/hypothesis/internal/conjecture/engine.py

@@ -9,6 +9,7 @@
 # obtain one at https://mozilla.org/MPL/2.0/.
 
 import importlib
+import inspect
 import math
 import textwrap
 import time
@@ -23,7 +24,7 @@
 import attr
 
 from hypothesis import HealthCheck, Phase, Verbosity, settings as Settings
-from hypothesis._settings import local_settings
+from hypothesis._settings import local_settings, note_deprecation
 from hypothesis.database import ExampleDatabase, choices_from_bytes, choices_to_bytes
 from hypothesis.errors import (
     BackendCannotProceed,
@@ -213,9 +214,25 @@ def kill_branch(self) -> NoReturn:
         raise ContainsDiscard
 
 
-def realize_choices(data: ConjectureData) -> None:
+def realize_choices(data: ConjectureData, *, for_failure: bool) -> None:
+    # backwards-compatibility with backends without for_failure, can remove
+    # in a few months
+    kwargs = {}
+    if for_failure:
+        if "for_failure" in inspect.signature(data.provider.realize).parameters:
+            kwargs["for_failure"] = True
+        else:
+            note_deprecation(
+                f"{type(data.provider).__qualname__}.realize does not have the "
+                "for_failure parameter. This will be an error in future versions "
+                "of Hypothesis. (If you installed this backend from a separate "
+                "package, upgrading that package may help).",
+                has_codemod=False,
+                since="RELEASEDAY",
+            )
+
     for node in data.nodes:
-        value = data.provider.realize(node.value)
+        value = data.provider.realize(node.value, **kwargs)
         expected_type = {
             "string": str,
             "float": float,
@@ -231,7 +248,10 @@ def realize_choices(data: ConjectureData) -> None:
 
         constraints = cast(
             ChoiceConstraintsT,
-            {k: data.provider.realize(v) for k, v in node.constraints.items()},
+            {
+                k: data.provider.realize(v, **kwargs)
+                for k, v in node.constraints.items()
+            },
         )
         node.value = value
         node.constraints = constraints
@@ -497,7 +517,7 @@ def test_function(self, data: ConjectureData) -> None:
         except BaseException:
             data.freeze()
             if self.settings.backend != "hypothesis":
-                realize_choices(data)
+                realize_choices(data, for_failure=True)
             self.save_choices(data.choices)
             raise
         finally:
@@ -517,7 +537,7 @@ def test_function(self, data: ConjectureData) -> None:
                 }
                 self.stats_per_test_case.append(call_stats)
                 if self.settings.backend != "hypothesis":
-                    realize_choices(data)
+                    realize_choices(data, for_failure=data.status is Status.INTERESTING)
 
                 self._cache(data)
                 if data.misaligned_at is not None:  # pragma: no branch # coverage bug?

hypothesis-python/src/hypothesis/internal/conjecture/providers.py

@@ -317,15 +317,22 @@ def __init__(self, conjecturedata: Optional["ConjectureData"], /) -> None:
     def per_test_case_context_manager(self):
         return contextlib.nullcontext()
 
-    def realize(self, value: T) -> T:
+    def realize(self, value: T, *, for_failure: bool = False) -> T:
         """
         Called whenever hypothesis requires a concrete (non-symbolic) value from
         a potentially symbolic value. Hypothesis will not check that `value` is
         symbolic before calling `realize`, so you should handle the case where
         `value` is non-symbolic.
 
         The returned value should be non-symbolic.  If you cannot provide a value,
-        raise hypothesis.errors.BackendCannotProceed("discard_test_case")
+        raise hypothesis.errors.BackendCannotProceed("discard_test_case").
+
+        If for_failure is True, the value is associated with a failing example.
+        In this case, the backend should spend substantially more effort when
+        attempting to realize the value, since it is important to avoid discarding
+        failing  examples. Backends may still raise BackendCannotProceed when
+        for_failure is True, if realization is truly impossible or if realization
+        takes significantly longer than expected (say, 5 minutes).
         """
         return value
 

hypothesis-python/tests/conjecture/test_alt_backend.py

@@ -47,7 +47,11 @@
 from hypothesis.internal.intervalsets import IntervalSet
 
 from tests.common.debug import minimal
-from tests.common.utils import capture_observations, capture_out
+from tests.common.utils import (
+    capture_observations,
+    capture_out,
+    checks_deprecated_behaviour,
+)
 from tests.conjecture.common import nodes
 
 
@@ -377,7 +381,7 @@ def test_function(n):
 
 
 class BadRealizeProvider(TrivialProvider):
-    def realize(self, value):
+    def realize(self, value, *, for_failure=False):
         return None
 
 
@@ -399,7 +403,7 @@ def test_function(n):
 class RealizeProvider(TrivialProvider):
     avoid_realization = True
 
-    def realize(self, value):
+    def realize(self, value, *, for_failure=False):
         if isinstance(value, int):
             return 42
         return value
@@ -473,7 +477,7 @@ def observe_information_messages(self, *, lifetime):
             yield {"type": "alert", "title": "Trivial alert", "content": "message here"}
             yield {"type": "info", "title": "trivial-data", "content": {"k2": "v2"}}
 
-    def realize(self, value):
+    def realize(self, value, *, for_failure=False):
         # Get coverage of the can't-realize path for observability outputs
         raise BackendCannotProceed
 
@@ -665,3 +669,21 @@ def f(n):
         # full message as of writing: "backend='soundness_test' claimed to
         # verify this test passes - please send them a bug report!"
         assert all("backend" not in note for note in e.value.__notes__)
+
+
+class NoForFailureProvider(TrivialProvider):
+    def realize(self, value):
+        return value
+
+
+@checks_deprecated_behaviour
+def test_realize_without_for_failure():
+    with temp_register_backend("no_for_failure", NoForFailureProvider):
+
+        @given(st.integers())
+        @settings(backend="no_for_failure", database=None)
+        def f(n):
+            assert n != 1
+
+        with pytest.raises(AssertionError):
+            f()

pytest.ini

@@ -24,3 +24,4 @@ filterwarnings =
     default:`np\.object` is a deprecated alias for the builtin `object`:DeprecationWarning
     # pytest-cov can't see into subprocesses; we'll see <100% covered if this is an issue
     ignore:Module hypothesis.* was previously imported, but not measured
+    ignore:CrosshairPrimitiveProvider.realize does not have the for_failure parameter