Add shrink pass for reordering examples

hypothesis-python/RELEASE.rst

@@ -0,0 +1,21 @@
+RELEASE_TYPE: patch
+
+This release improves the shrinker's ability to reorder examples.
+
+For example, consider the following test:
+
+.. code-block:: python
+
+    import hypothesis.strategies as st
+    from hypothesis import given
+
+    @given(st.text(), st.text())
+    def test_does_not_exceed_100(x, y):
+        assert x != y
+
+Previously this could have failed with either of ``x="", y="0"`` or
+``x="0", y=""``. Now it should always fail with ``x="", y="0"``.
+
+This will allow the shrinker to produce more consistent results, especially in
+cases where test cases contain some ordered collection whose actual order does
+not matter.

hypothesis-python/src/hypothesis/internal/conjecture/engine.py

@@ -1335,6 +1335,7 @@ def greedy_shrink(self):
             if not run_expensive_shrinks:
                 continue
 
+            self.reorder_examples()
             self.shrink_offset_pairs()
             self.interval_deletion_with_block_lowering()
             self.pass_to_interval()
@@ -2279,3 +2280,44 @@ def attempt(new_ordering):
                     break
             else:
                 i += 1
+
+    def reorder_examples(self):
+        """This pass allows us to reorder pairs of examples which come from the
+        same strategy (or strategies that happen to pun to the same label by
+        accident, but that shouldn't happen often).
+
+        For example, consider the following:
+
+        .. code-block:: python
+
+            import hypothesis.strategies as st
+            from hypothesis import given
+
+            @given(st.text(), st.text())
+            def test_does_not_exceed_100(x, y):
+                assert x != y
+
+        Without the ability to reorder x and y this could fail either with
+        ``x="", ``y="0"``, or the other way around. With reordering it will
+        reliably fail with ``x=""``, ``y="0"``.
+        """
+        i = 0
+        while i < len(self.shrink_target.examples):
+            j = i + 1
+            while j < len(self.shrink_target.examples):
+                ex1 = self.shrink_target.examples[i]
+                ex2 = self.shrink_target.examples[j]
+                if ex1.label == ex2.label and ex2.start >= ex1.end:
+                    buf = self.shrink_target.buffer
+                    attempt = (
+                        buf[:ex1.start] +
+                        buf[ex2.start:ex2.end] +
+                        buf[ex1.end:ex2.start] +
+                        buf[ex1.start:ex1.end] +
+                        buf[ex2.end:]
+                    )
+                    assert len(attempt) == len(buf)
+                    if attempt < buf:
+                        self.incorporate_new_buffer(attempt)
+                j += 1
+            i += 1

hypothesis-python/tests/cover/test_conjecture_engine.py

@@ -1330,3 +1330,27 @@ def f(data):
         runner.run()
 
     assert runner.exit_reason == ExitReason.finished
+
+
+def test_can_reorder_examples(monkeypatch):
+    monkeypatch.setattr(
+        ConjectureRunner, 'generate_new_examples',
+        lambda runner: runner.test_function(
+            ConjectureData.for_buffer([1, 0, 1, 1, 0, 1, 0, 0, 0])))
+
+    monkeypatch.setattr(
+        Shrinker, 'shrink', Shrinker.reorder_examples,
+    )
+
+    @run_to_buffer
+    def x(data):
+        total = 0
+        for _ in range(5):
+            data.start_example(0)
+            if data.draw_bits(8):
+                total += data.draw_bits(9)
+            data.stop_example(0)
+        if total == 2:
+            data.mark_interesting()
+
+    assert list(x) == [0, 0, 0, 1, 0, 1, 1, 0, 1]