Add shrink pass for reordering examples

hypothesis-python/RELEASE.rst

@@ -1,23 +1,21 @@
 RELEASE_TYPE: patch
 
-This release improves the shrinker's ability to handle situations where there
-is an additive constraint between two values.
+This release improves the shrinker's ability to reorder examples.
 
 For example, consider the following test:
 
-
 .. code-block:: python
 
     import hypothesis.strategies as st
     from hypothesis import given
 
-    @given(st.integers(), st.integers())
-    def test_does_not_exceed_100(m, n):
-        assert m + n < 100
+    @given(st.text(), st.text())
+    def test_does_not_exceed_100(x, y):
+        assert x != y
 
-Previously this could have failed with almost any pair ``(m, n)`` with
-``0 <= m <= n`` and ``m + n == 100``. Now it should almost always fail with
-``m=0, n=100``.
+Previously this could have failed with either of ``x="", y="0"`` or
+``x="0", y=""``. Now it should always fail with ``x="", y="0"``.
 
-This is a relatively niche specialisation, but can be useful in situations
-where e.g. a bug is triggered by an integer overflow.
+This will allow the shrinker to produce more consistent results, especially in
+cases where test cases contain some ordered collection whose actual order does
+not matter.

hypothesis-python/src/hypothesis/internal/conjecture/engine.py

@@ -1335,6 +1335,7 @@ def greedy_shrink(self):
             if not run_expensive_shrinks:
                 continue
 
+            self.reorder_examples()
             self.shrink_offset_pairs()
             self.interval_deletion_with_block_lowering()
             self.pass_to_interval()
@@ -2340,3 +2341,44 @@ def trial(x, y):
                             )
                     j += 1
             i += 1
+
+    def reorder_examples(self):
+        """This pass allows us to reorder pairs of examples which come from the
+        same strategy (or strategies that happen to pun to the same label by
+        accident, but that shouldn't happen often).
+
+        For example, consider the following:
+
+        .. code-block:: python
+
+            import hypothesis.strategies as st
+            from hypothesis import given
+
+            @given(st.text(), st.text())
+            def test_does_not_exceed_100(x, y):
+                assert x != y
+
+        Without the ability to reorder x and y this could fail either with
+        ``x="", ``y="0"``, or the other way around. With reordering it will
+        reliably fail with ``x=""``, ``y="0"``.
+        """
+        i = 0
+        while i < len(self.shrink_target.examples):
+            j = i + 1
+            while j < len(self.shrink_target.examples):
+                ex1 = self.shrink_target.examples[i]
+                ex2 = self.shrink_target.examples[j]
+                if ex1.label == ex2.label and ex2.start >= ex1.end:
+                    buf = self.shrink_target.buffer
+                    attempt = (
+                        buf[:ex1.start] +
+                        buf[ex2.start:ex2.end] +
+                        buf[ex1.end:ex2.start] +
+                        buf[ex1.start:ex1.end] +
+                        buf[ex2.end:]
+                    )
+                    assert len(attempt) == len(buf)
+                    if attempt < buf:
+                        self.incorporate_new_buffer(attempt)
+                j += 1
+            i += 1

hypothesis-python/tests/cover/test_conjecture_engine.py

@@ -1372,3 +1372,27 @@ def x(data):
                 if m + n == 200:
                     data.mark_interesting()
     assert len(x) == 1
+
+
+def test_can_reorder_examples(monkeypatch):
+    monkeypatch.setattr(
+        ConjectureRunner, 'generate_new_examples',
+        lambda runner: runner.test_function(
+            ConjectureData.for_buffer([1, 0, 1, 1, 0, 1, 0, 0, 0])))
+
+    monkeypatch.setattr(
+        Shrinker, 'shrink', Shrinker.reorder_examples,
+    )
+
+    @run_to_buffer
+    def x(data):
+        total = 0
+        for _ in range(5):
+            data.start_example(0)
+            if data.draw_bits(8):
+                total += data.draw_bits(9)
+            data.stop_example(0)
+        if total == 2:
+            data.mark_interesting()
+
+    assert list(x) == [0, 0, 0, 1, 0, 1, 1, 0, 1]