Add size to fscheck3 replay (#515)

* Fix replay for fscheck3

(cherry picked from commit f7d1690)

* Update docs

(cherry picked from commit e5943f6)

* Fix to show the final seed instead of the original seed for fscheck3

Using the original seed has the same problem

(cherry picked from commit e69fb64)

* Add back original seed to fscheck3 output

(cherry picked from commit 701746a)

* Add note about fscheck2 replay size migration to readme

* Switch to use `int option` for replay size

To allow more flexibility and match fscheck's type

Expecto.FsCheck/FsCheck.fs

@@ -71,7 +71,7 @@ module ExpectoFsCheck =
                           (String.concat " " data.Labels)
 
               let focus =
-                sprintf "Focus on error:\n\t%s (%A, %A) \"%s\"" methodName (uint64 std) (uint64 gen) name
+                sprintf "Focus on error:\n\t%s (%A, %A, None) \"%s\"" methodName (uint64 std) (uint64 gen) name
 
               sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s"
                       (numTests data.NumberOfTests) parameters shrunk
@@ -92,7 +92,7 @@ module ExpectoFsCheck =
           MaxFail = 1000
           // We're converting uint64s to a smaller type, but it shouldn't be an issue because users are only using the
           // values given in the test output, which are only ints when running FsCheck 2
-          Replay = Option.map Random.StdGen (config.replay |> Option.map (fun (seed, gamma) -> int seed, int gamma))
+          Replay = Option.map Random.StdGen (config.replay |> Option.map (fun (seed, gamma, _) -> int seed, int gamma))
           Name = name
           StartSize = config.startSize
           EndSize = config.endSize

Expecto.FsCheck3/FsCheck3.fs

@@ -47,7 +47,7 @@ module ExpectoFsCheck =
             | TestResult.Failed (_,_,_, Outcome.Failed (:? IgnoreException as e),_,_,_) ->
               raise e
 
-            | TestResult.Failed (data, original, shrunk, outcome,originalSeed,_finalSeed,size) ->
+            | TestResult.Failed (data, original, shrunk, outcome,originalSeed,finalSeed,size) ->
               let parameters =
                 original
                 |> List.map (sprintf "%A")
@@ -70,12 +70,14 @@ module ExpectoFsCheck =
                 | _ -> sprintf "Labels of failing property (one or more is failing): %s\n"
                           (String.concat " " data.Labels)
 
+              let original =
+                sprintf "Original seed: (%A, %A)" originalSeed.Seed originalSeed.Gamma
               let focus =
-                sprintf "Focus on error:\n\t%s (%A, %A) \"%s\"" methodName originalSeed.Seed originalSeed.Gamma name
+                sprintf "Focus on error:\n\t%s (%A, %A, Some %A) \"%s\"" methodName finalSeed.Seed finalSeed.Gamma size name
 
-              sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s"
+              sprintf "Failed after %s. %s%s\nResult:\n\t%A\n%s%s%s\n%s"
                       (numTests data.NumberOfTests) parameters shrunk
-                      outcome labels (stampsToString data.Stamps) focus
+                      outcome labels (stampsToString data.Stamps) original focus
               |> FailedException
               |> raise
 
@@ -91,7 +93,7 @@ module ExpectoFsCheck =
       let config =
         Config.Default
             .WithMaxTest(config.maxTest)
-            .WithReplay(Option.map (fun (seed,gamma) -> {Rnd = Rnd(seed, gamma); Size = None}) config.replay)
+            .WithReplay(Option.map (fun (seed,gamma,size) -> {Rnd = Rnd(seed, gamma); Size = size}) config.replay)
             .WithName(name)
             .WithStartSize(config.startSize)
             .WithEndSize(config.endSize)

Expecto.Tests.FsCheck3/FsCheck3Tests.fs

@@ -60,7 +60,7 @@ let focused =
   testList "FsCheck focused" [
     testCase "ignore me" <| ignore
 
-    etestProperty (1UL,3UL) "Deliberately failing test" <|
+    etestProperty (1UL,3UL,Some 50) "Deliberately failing test" <|
       fun a b c ->
         // wrong on purpose to test failures
         a * (b + c) = a * a + a * c
@@ -87,14 +87,15 @@ let runFsCheckFocusedTests =
     match (getResult ["FsCheck focused";"Deliberately failing test"]).result with
     | TestResult.Failed actual ->
       let expected = "
-Failed after 3 tests. Parameters:
-	-1 1 2
-Shrunk 2 times to:
-	-1 0 0
+Failed after 1 test. Parameters:
+	47 43 -38
+Shrunk 9 times to:
+	1 0 0
 Result:
 	Failed System.Exception: Expected true, got false.
+Original seed: (1UL, 3UL)
 Focus on error:
-	etestProperty (1UL, 3UL) \"Deliberately failing test\""
+	etestProperty (1UL, 3UL, Some 50) \"Deliberately failing test\""
       Expect.equal actual expected "It should fail with the right message"
     | x ->
       failtestf "Expected Failed, actual was: %A" x
@@ -104,7 +105,7 @@ let config =
   testList "FsCheck config" [
     testCase "ignore me" ignore
 
-    etestPropertyWithConfig (1UL,3UL) FsCheckConfig.defaultConfig
+    etestPropertyWithConfig (1UL,3UL,Some 50) FsCheckConfig.defaultConfig
       "Deliberately failing test" <|
       fun a b c ->
         // wrong on purpose to test failures
@@ -132,14 +133,15 @@ let runFsCheckConfigTests =
     match (getResult ["FsCheck config";"Deliberately failing test"]).result with
     | TestResult.Failed actual ->
       let expected = "
-Failed after 3 tests. Parameters:
-	-1 1 2
-Shrunk 2 times to:
-	-1 0 0
+Failed after 1 test. Parameters:
+	47 43 -38
+Shrunk 9 times to:
+	1 0 0
 Result:
 	Failed System.Exception: Expected true, got false.
+Original seed: (1UL, 3UL)
 Focus on error:
-	etestPropertyWithConfig (1UL, 3UL) \"Deliberately failing test\""
+	etestPropertyWithConfig (1UL, 3UL, Some 50) \"Deliberately failing test\""
       Expect.equal actual expected "It should fail with the right message."
 
     | x ->

Expecto.Tests/FsCheckTests.fs

@@ -60,7 +60,7 @@ let focused =
   testList "FsCheck focused" [
     testCase "ignore me" <| ignore
 
-    etestProperty (1UL,2UL) "Deliberately failing test" <|
+    etestProperty (1UL,2UL,None) "Deliberately failing test" <|
       fun a b c ->
         // wrong on purpose to test failures
         a * (b + c) = a * a + a * c
@@ -94,7 +94,7 @@ Shrunk 4 times to:
 Result:
 	False
 Focus on error:
-	etestProperty (1UL, 2UL) \"Deliberately failing test\""
+	etestProperty (1UL, 2UL, None) \"Deliberately failing test\""
       Expect.equal actual expected "It should fail with the right message"
     | x ->
       failtestf "Expected Failed, actual was: %A" x
@@ -104,7 +104,7 @@ let config =
   testList "FsCheck config" [
     testCase "ignore me" ignore
 
-    etestPropertyWithConfig (1UL,2UL) FsCheckConfig.defaultConfig
+    etestPropertyWithConfig (1UL,2UL,None) FsCheckConfig.defaultConfig
       "Deliberately failing test" <|
       fun a b c ->
         // wrong on purpose to test failures
@@ -139,7 +139,7 @@ Shrunk 4 times to:
 Result:
 	False
 Focus on error:
-	etestPropertyWithConfig (1UL, 2UL) \"Deliberately failing test\""
+	etestPropertyWithConfig (1UL, 2UL, None) \"Deliberately failing test\""
       Expect.equal actual expected "It should fail with the right message."
 
     | x ->

Expecto/Model.fs

@@ -15,8 +15,9 @@ type FsCheckConfig =
     startSize: int
     /// The size to use for the last test, when all the tests are passing. The size increases linearly between Start- and EndSize.
     endSize: int
-    /// If set, the seed to use to start testing. Allows reproduction of previous runs.
-    replay: (uint64 * uint64) option
+    /// (seed * gamma * size) corresponding to the FsCheck3 replay config. Allows reproduction of previous runs. The
+    /// size is ignored with FsCheck2.
+    replay: (uint64 * uint64 * int option) option
     /// The Arbitrary instances on this class will be merged in back to front order, i.e. instances for the same generated type at the front
     /// of the list will override those at the back. The instances on Arb.Default are always known, and are at the back (so they can always be
     /// overridden)

README.md

@@ -1372,5 +1372,5 @@ This might be due to how terminals/the locking thereof work: try running your te
 ## Migration notes
 
 ### 11.0.0
-- Any usages of the `replay` (a.k.a `stdGen` with `etestProperty*` functions) config with FsCheck tests will need to be updated to use `uint64` by appending `UL` to the literals, e.g. from `(1865288075, 296281834)` to `(1865288075UL, 296281834UL)`.
+- Any usages of the `replay` (a.k.a `stdGen` with `etestProperty*` functions) config with FsCheck tests will need to be updated to use `uint64` by appending `UL` to the literals. They will also now require a third item indicating the size. E.g. from `(1865288075, 296281834)` to `(1865288075UL, 296281834UL, Some 3)`. With FsCheck 2 the size is ignored and `None` can be used.
 - FsCheck 2 is no longer supported, so we're switching Expecto.FsCheck to use FsCheck 3 by default, even though FsCheck 3 is still in release candidate state. If you still want FsCheck2, we will continue to release FsCheck2 support for the time being using a version suffix, e.g. [11.0.0-fscheck2](https://www.nuget.org/packages/Expecto.FsCheck/11.0.0-alpha1-fscheck2)  

RELEASE_NOTES.md

@@ -1,3 +1,9 @@
+### 11.0.0-alpha4 - 2025-01-06
+* Breaking change: Add third item to `FsCheckConfig.replay` indicating the size
+    * Fixes issue where the replay seed without the size was playing all tests leading up to the failure, making debugging
+      more difficult
+    * Existing FsCheck 2 users can enter `None` for the size, because it is ignored.
+
 ### 11.0.0-alpha3 - 2024-10-13
 * Add testParamAsync and testParamTask (#512), thanks @1eyewonder