diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 00000000000..145133ade6b --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,21 @@ +RELEASE_TYPE: patch + +This release improves the shrinker's ability to reorder examples. + +For example, consider the following test: + +.. code-block:: python + + import hypothesis.strategies as st + from hypothesis import given + + @given(st.text(), st.text()) + def test_does_not_exceed_100(x, y): + assert x != y + +Previously this could have failed with either of ``x="", y="0"`` or +``x="0", y=""``. Now it should always fail with ``x="", y="0"``. + +This will allow the shrinker to produce more consistent results, especially in +cases where test cases contain some ordered collection whose actual order does +not matter. diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py index 1af09df342b..381ef3dca36 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py @@ -1335,6 +1335,7 @@ def greedy_shrink(self): if not run_expensive_shrinks: continue + self.reorder_examples() self.shrink_offset_pairs() self.interval_deletion_with_block_lowering() self.pass_to_interval() @@ -2279,3 +2280,44 @@ def attempt(new_ordering): break else: i += 1 + + def reorder_examples(self): + """This pass allows us to reorder pairs of examples which come from the + same strategy (or strategies that happen to pun to the same label by + accident, but that shouldn't happen often). + + For example, consider the following: + + .. code-block:: python + + import hypothesis.strategies as st + from hypothesis import given + + @given(st.text(), st.text()) + def test_does_not_exceed_100(x, y): + assert x != y + + Without the ability to reorder x and y this could fail either with + ``x="", ``y="0"``, or the other way around. With reordering it will + reliably fail with ``x=""``, ``y="0"``. + """ + i = 0 + while i < len(self.shrink_target.examples): + j = i + 1 + while j < len(self.shrink_target.examples): + ex1 = self.shrink_target.examples[i] + ex2 = self.shrink_target.examples[j] + if ex1.label == ex2.label and ex2.start >= ex1.end: + buf = self.shrink_target.buffer + attempt = ( + buf[:ex1.start] + + buf[ex2.start:ex2.end] + + buf[ex1.end:ex2.start] + + buf[ex1.start:ex1.end] + + buf[ex2.end:] + ) + assert len(attempt) == len(buf) + if attempt < buf: + self.incorporate_new_buffer(attempt) + j += 1 + i += 1 diff --git a/hypothesis-python/tests/cover/test_conjecture_engine.py b/hypothesis-python/tests/cover/test_conjecture_engine.py index e9d6f1d9bd7..d8e8d93d929 100644 --- a/hypothesis-python/tests/cover/test_conjecture_engine.py +++ b/hypothesis-python/tests/cover/test_conjecture_engine.py @@ -1330,3 +1330,27 @@ def f(data): runner.run() assert runner.exit_reason == ExitReason.finished + + +def test_can_reorder_examples(monkeypatch): + monkeypatch.setattr( + ConjectureRunner, 'generate_new_examples', + lambda runner: runner.test_function( + ConjectureData.for_buffer([1, 0, 1, 1, 0, 1, 0, 0, 0]))) + + monkeypatch.setattr( + Shrinker, 'shrink', Shrinker.reorder_examples, + ) + + @run_to_buffer + def x(data): + total = 0 + for _ in range(5): + data.start_example(0) + if data.draw_bits(8): + total += data.draw_bits(9) + data.stop_example(0) + if total == 2: + data.mark_interesting() + + assert list(x) == [0, 0, 0, 1, 0, 1, 1, 0, 1]