Implement labelledExamples

core/shared/src/main/scala/hedgehog/core/Coverage.scala

@@ -89,6 +89,17 @@ object Coverage {
   def empty[A]: Coverage[A] =
     Coverage(Map.empty[LabelName, Label[A]])
 
+  def labels(cv: Coverage[_]): List[LabelName] =
+    cv.labels.keysIterator.toList
+
+  def covers(cv: Coverage[Cover], name: LabelName): Boolean =
+    cv.labels.get(name).exists(_.annotation match {
+      case Cover.Cover =>
+        true
+      case Cover.NoCover =>
+        false
+    })
+
   def count(cv: Coverage[Cover]): Coverage[CoverCount] =
     cv.copy(labels = cv.labels.map { case (k, l) =>
       k -> l.copy(annotation = CoverCount.fromCover(l.annotation))
@@ -105,3 +116,16 @@ object Coverage {
   }
 }
 
+case class Examples(examples: Map[LabelName, List[Log]])
+
+object Examples {
+
+  def empty[A]: Examples =
+    Examples(Map.empty[LabelName, List[Log]])
+
+  def addTo(examples: Examples, labels: List[LabelName])(seek: LabelName => List[Log]): Examples = {
+    Examples(labels.foldLeft(examples.examples) { (m, name) =>
+      m.updated(name, m.get(name).filter(_.nonEmpty).getOrElse(seek(name)))
+    })
+  }
+}

core/shared/src/main/scala/hedgehog/core/PropertyT.scala

@@ -46,12 +46,13 @@ case class PropertyConfig(
     testLimit: SuccessCount
   , discardLimit: DiscardCount
   , shrinkLimit: ShrinkLimit
+  , withExamples: WithExamples
   )
 
 object PropertyConfig {
 
   def default: PropertyConfig =
-    PropertyConfig(SuccessCount(100), DiscardCount(100), ShrinkLimit(1000))
+    PropertyConfig(SuccessCount(100), DiscardCount(100), ShrinkLimit(1000), WithExamples.NoExamples)
 }
 
 case class PropertyT[A](
@@ -132,38 +133,51 @@ object PropertyT {
 trait PropertyTReporting {
 
   @annotation.tailrec
-  final def takeSmallest(n: ShrinkCount, slimit: ShrinkLimit, t: Tree[Option[(List[Log], Option[Result])]]): Status =
-    t.value match {
+  final def takeSmallestG[A, B](n: ShrinkCount, slimit: ShrinkLimit, t: Tree[A])(p: A => Boolean)(e: (ShrinkCount, A) => B): B = {
+    if (n.value < slimit.value && p(t.value)) {
+      findMap(t.children.value)(m => if (p(m.value)) some(m) else Option.empty) match {
+        case None =>
+          e(n, t.value)
+
+        case Some(m) =>
+          takeSmallestG(n.inc, slimit, m)(p)(e)
+      }
+    } else {
+      e(n, t.value)
+    }
+  }
+
+  def takeSmallest(n: ShrinkCount, slimit: ShrinkLimit, t: Tree[Option[(Journal, Option[Result])]]): Status =
+    takeSmallestG(n, slimit, t) {
       case None =>
+        false
+
+      case Some((_, r)) =>
+        r.forall(!_.success)
+    } {
+      case (_, None) =>
         Status.gaveUp
 
-      case Some((w, r)) =>
-        if (r.forall(!_.success)) {
-          if (n.value >= slimit.value) {
-            Status.failed(n, w ++ r.map(_.logs).getOrElse(Nil))
-          } else {
-            findMap(t.children.value)(m =>
-              m.value match {
-                case Some((_, None)) =>
-                  some(m)
-                case Some((_, Some(r2))) =>
-                  if (!r2.success)
-                    some(m)
-                  else
-                    Option.empty
-                case None =>
-                  Option.empty
-              }
-            ) match {
-              case Some(m) =>
-                takeSmallest(n.inc, slimit, m)
-              case None =>
-                Status.failed(n, w ++ r.map(_.logs).getOrElse(Nil))
-            }
-          }
-        } else {
+      case (n, Some((j, r))) =>
+        if (r.forall(!_.success))
+          Status.failed(n, j.logs ++ r.map(_.logs).getOrElse(Nil))
+        else
           Status.ok
-        }
+    }
+
+  def takeSmallestExample(n: ShrinkCount, slimit: ShrinkLimit, name: LabelName, t: Tree[Option[(Journal, Option[Result])]]): List[Log] =
+    takeSmallestG(n, slimit, t) {
+      case None =>
+        false
+
+      case Some((j, r)) =>
+        r.exists(_.success) && Coverage.covers(j.coverage, name)
+    } {
+      case (_, None) =>
+        Nil
+
+      case (_, Some((j, _))) =>
+        j.logs
     }
 
   def report(config: PropertyConfig, size0: Option[Size], seed0: Seed, p: PropertyT[Result]): Report = {
@@ -172,17 +186,25 @@ trait PropertyTReporting {
     // Start the size at whatever remainder we have to ensure we run with "max" at least once
     val sizeInit = Size((Size.max % Math.min(config.testLimit.value, Size.max)) + sizeInc.value)
     @annotation.tailrec
-    def loop(successes: SuccessCount, discards: DiscardCount, size: Size, seed: Seed, coverage: Coverage[CoverCount]): Report =
+    def loop(successes: SuccessCount, discards: DiscardCount, size: Size, seed: Seed, coverage: Coverage[CoverCount], examples: Examples): Report =
       if (successes.value >= config.testLimit.value)
         // we've hit the test limit
         Coverage.split(coverage, successes) match {
           case (_, Nil) =>
-            Report(successes, discards, coverage, OK)
+            config.withExamples match {
+              case WithExamples.WithExamples =>
+                if (examples.examples.exists(_._2.isEmpty))
+                  Report(successes, discards, coverage, examples, Status.failed(ShrinkCount(0), List("Insufficient examples.")))
+                else
+                  Report(successes, discards, coverage, examples, OK)
+              case WithExamples.NoExamples =>
+                Report(successes, discards, coverage, examples, OK)
+            }
           case _ =>
-            Report(successes, discards, coverage, Status.failed(ShrinkCount(0), List("Insufficient coverage.")))
+            Report(successes, discards, coverage, examples, Status.failed(ShrinkCount(0), List("Insufficient coverage.")))
         }
       else if (discards.value >= config.discardLimit.value)
-        Report(successes, discards, coverage, GaveUp)
+        Report(successes, discards, coverage, examples, GaveUp)
       else {
         val x =
           try {
@@ -191,23 +213,33 @@ trait PropertyTReporting {
             case e: Exception =>
               Property.error(e).run.run(size, seed)
           }
-        val t = x.map(_._2.map { case (l, r) => (l.logs, r) })
+        val t = x.map(_._2)
         x.value._2 match {
           case None =>
-            loop(successes, discards.inc, size.incBy(sizeInc), x.value._1, coverage)
-
-          case Some((_, None)) =>
-            Report(successes, discards, coverage, takeSmallest(ShrinkCount(0), config.shrinkLimit, t))
-
-          case Some((j, Some(r))) =>
-            if (!r.success){
-              Report(successes, discards, coverage, takeSmallest(ShrinkCount(0), config.shrinkLimit, t))
-            } else
-              loop(successes.inc, discards, size.incBy(sizeInc), x.value._1,
-                Coverage.union(Coverage.count(j.coverage), coverage)(_ + _))
+            loop(successes, discards.inc, size.incBy(sizeInc), x.value._1, coverage, examples)
+
+          case Some((j, r)) =>
+            if (r.forall(!_.success)) {
+              Report(successes, discards, coverage, examples, takeSmallest(ShrinkCount(0), config.shrinkLimit, t))
+            } else {
+              val coverage2 = Coverage.union(Coverage.count(j.coverage), coverage)(_ + _)
+              val examples2 =
+                config.withExamples match {
+                  case WithExamples.WithExamples =>
+                    Examples.addTo(examples, Coverage.labels(j.coverage)) { name =>
+                      if (Coverage.covers(j.coverage, name))
+                        takeSmallestExample(ShrinkCount(0), config.shrinkLimit, name, t)
+                      else
+                        Nil
+                    }
+                  case WithExamples.NoExamples =>
+                    examples
+                }
+              loop(successes.inc, discards, size.incBy(sizeInc), x.value._1, coverage2, examples2)
+            }
         }
       }
-    loop(SuccessCount(0), DiscardCount(0), size0.getOrElse(sizeInit), seed0, Coverage.empty)
+    loop(SuccessCount(0), DiscardCount(0), size0.getOrElse(sizeInit), seed0, Coverage.empty, Examples.empty)
   }
 
   def recheck(config: PropertyConfig, size: Size, seed: Seed)(p: PropertyT[Result]): Report =
@@ -247,6 +279,15 @@ case class DiscardCount(value: Int) {
     DiscardCount(value + 1)
 }
 
+/** Whether the report should include an example for each label. */
+sealed trait WithExamples
+
+object WithExamples {
+
+  case object WithExamples extends WithExamples
+  case object NoExamples extends WithExamples
+}
+
 /**
  * The status of a property test run.
  *
@@ -270,4 +311,4 @@ object Status {
     OK
 }
 
-case class Report(tests: SuccessCount, discards: DiscardCount, coverage: Coverage[CoverCount], status: Status)
+case class Report(tests: SuccessCount, discards: DiscardCount, coverage: Coverage[CoverCount], examples: Examples, status: Status)

test/shared/src/test/scala/hedgehog/GenTest.scala

@@ -70,12 +70,12 @@ object GenTest extends Properties {
 
   def testFromSomeSome: Result = {
     val r = Property.checkRandom(PropertyConfig.default, Gen.fromSome(Gen.constant(Result.success).option).forAll)
-    r ==== Report(SuccessCount(100), DiscardCount(0), Coverage.empty, OK)
+    r ==== Report(SuccessCount(100), DiscardCount(0), Coverage.empty, Examples.empty, OK)
   }
 
   def testFromSomeNone: Result = {
     val r = Property.checkRandom(PropertyConfig.default, Gen.fromSome(Gen.constant(Option.empty[Result])).forAll)
-    r ==== Report(SuccessCount(0), DiscardCount(100), Coverage.empty, GaveUp)
+    r ==== Report(SuccessCount(0), DiscardCount(100), Coverage.empty, Examples.empty, GaveUp)
   }
 
   def testApplicative: Result = {

test/shared/src/test/scala/hedgehog/LabelledExamplesTest.scala

@@ -0,0 +1,33 @@
+package hedgehog
+
+import hedgehog.core._
+import hedgehog.runner._
+
+object LabelledExamplesTest extends Properties {
+
+  def tests: List[Test] =
+    List(
+      property("testLabelledExamples", testLabelledExamples)
+    )
+
+  def prop: Property =
+    for {
+      _ <- Gen.int(Range.linear(0, 10)).list(Range.linear(0, 10)).forAll
+        .classify("empty", _.isEmpty)
+        .classify("nonempty", _.nonEmpty)
+    } yield Result.success
+
+  def testLabelledExamples: Property = {
+    for {
+      examples <- Gen.generate { (size, seed) =>
+        val config = PropertyConfig.default.copy(withExamples = WithExamples.WithExamples)
+        val labelledExamples = Property.report(config, Some(size), seed, prop)
+        Seed(seed.seed.next) -> labelledExamples.examples
+      }.forAll
+    } yield
+      examples ==== Examples(Map(
+        LabelName("empty") -> List(Info("List()"))
+      , LabelName("nonempty") -> List(Info("List(0)"))
+      ))
+  }
+}

test/shared/src/test/scala/hedgehog/PropertyTest.scala

@@ -23,7 +23,7 @@ object PropertyTest extends Properties {
       y <- int(Range.linear(0, 50)).log("y")
       _ <- if (y % 2 == 0) Property.discard else Property.point(())
     } yield Result.assert(y < 87 && x <= 'r'), seed)
-    r ==== Report(SuccessCount(2), DiscardCount(4), Coverage.empty, Failed(ShrinkCount(2), List(
+    r ==== Report(SuccessCount(2), DiscardCount(4), Coverage.empty, Examples.empty, Failed(ShrinkCount(2), List(
         ForAll("x", "s")
       , ForAll("y", "1"))
       ))
@@ -39,7 +39,7 @@ object PropertyTest extends Properties {
       (if (y % 2 == 0) Property.discard else Property.point(())).map(_ =>
       Result.assert(y < 87 && x <= 'r')
     )}, seed)
-    r ==== Report(SuccessCount(2), DiscardCount(4), Coverage.empty, Failed(ShrinkCount(2), List(
+    r ==== Report(SuccessCount(2), DiscardCount(4), Coverage.empty, Examples.empty, Failed(ShrinkCount(2), List(
         ForAll("x", "s")
       , ForAll("y", "1"))
       ))
@@ -107,7 +107,7 @@ object PropertyTest extends Properties {
       y <- order(expensive).log("expensive")
     } yield Result.assert(merge(x, y).total.value == x.total.value + y.total.value)
       , seed)
-    r ==== Report(SuccessCount(1), DiscardCount(0), Coverage.empty, Failed(ShrinkCount(4), List(
+    r ==== Report(SuccessCount(1), DiscardCount(0), Coverage.empty, Examples.empty, Failed(ShrinkCount(4), List(
         ForAll("cheap", "Order(List())")
       , ForAll("expensive", "Order(List(Item(oculus,USD(1000))))"
       ))))