Add support for verbose mode

README.md

@@ -309,15 +309,19 @@ It can be parametrized using its second argument.
 
 ```typescript
 export interface Parameters {
-    seed?: number;     // optional, initial seed of the generator: Date.now() by default
-    numRuns?: number; // optional, number of runs before success: 100 by default 
-    timeout?: number;  // optional, only taken into account for asynchronous runs (asyncProperty)
-                       // specify a timeout in milliseconds, maximum time for the predicate to return its result
-                       // only works for async code, will not interrupt a synchronous code: disabled by default
-    path?: string;     // optional, way to replay a failing property directly with the counterexample
-                       // it can be fed with the counterexamplePath returned by the failing test (requires seed too)
+    seed?: number;      // optional, initial seed of the generator: Date.now() by default
+    numRuns?: number;   // optional, number of runs before success: 100 by default 
+    timeout?: number;   // optional, only taken into account for asynchronous runs (asyncProperty)
+                        // specify a timeout in milliseconds, maximum time for the predicate to return its result
+                        // only works for async code, will not interrupt a synchronous code: disabled by default
+    path?: string;      // optional, way to replay a failing property directly with the counterexample
+                        // it can be fed with the counterexamplePath returned by the failing test (requires seed too)
     logger?: (v: string) => void; // optional, log output: console.log by default
     unbiased?: boolean; // optional, force the use of unbiased arbitraries: biased by default
+    verbose?: boolean;  // optional, enable verbose mode: false by default
+                        // when enabling verbose mode,
+                        // you will be provided the list of all failing entries encountered
+                        // whenever a property fails - useful to detect patterns
 }
 ```
 

src/check/runner/Runner.ts

@@ -13,8 +13,12 @@ import { pathWalk } from './utils/PathWalker';
 import { throwIfFailed } from './utils/utils';
 
 /** @hidden */
-function runIt<Ts>(property: IProperty<Ts>, initialValues: IterableIterator<Shrinkable<Ts>>): RunExecution<Ts> {
-  const runExecution = new RunExecution<Ts>(false);
+function runIt<Ts>(
+  property: IProperty<Ts>,
+  initialValues: IterableIterator<Shrinkable<Ts>>,
+  verbose: boolean
+): RunExecution<Ts> {
+  const runExecution = new RunExecution<Ts>(verbose);
   let done = false;
   let values: IterableIterator<Shrinkable<Ts>> = initialValues;
   while (!done) {
@@ -37,9 +41,10 @@ function runIt<Ts>(property: IProperty<Ts>, initialValues: IterableIterator<Shri
 /** @hidden */
 async function asyncRunIt<Ts>(
   property: IProperty<Ts>,
-  initialValues: IterableIterator<Shrinkable<Ts>>
+  initialValues: IterableIterator<Shrinkable<Ts>>,
+  verbose: boolean
 ): Promise<RunExecution<Ts>> {
-  const runExecution = new RunExecution<Ts>(false);
+  const runExecution = new RunExecution<Ts>(verbose);
   let done = false;
   let values: IterableIterator<Shrinkable<Ts>> = initialValues;
   while (!done) {
@@ -101,8 +106,10 @@ function check<Ts>(rawProperty: IProperty<Ts>, params?: Parameters) {
   }
   const initialValues = qParams.path.length === 0 ? g() : pathWalk(qParams.path, g());
   return property.isAsync()
-    ? asyncRunIt(property, initialValues).then(e => e.toRunDetails(qParams.seed, qParams.path, qParams.numRuns))
-    : runIt(property, initialValues).toRunDetails(qParams.seed, qParams.path, qParams.numRuns);
+    ? asyncRunIt(property, initialValues, qParams.verbose).then(e =>
+        e.toRunDetails(qParams.seed, qParams.path, qParams.numRuns)
+      )
+    : runIt(property, initialValues, qParams.verbose).toRunDetails(qParams.seed, qParams.path, qParams.numRuns);
 }
 
 /**

src/check/runner/configuration/Parameters.ts

@@ -31,4 +31,13 @@ export interface Parameters {
    * Force the use of unbiased arbitraries: biased by default
    */
   unbiased?: boolean;
+  /**
+   * Enable verbose mode: false by default
+   *
+   * When enabling verbose mode
+   * you will be provided the list of all failing entries encountered whenever a property fails
+   *
+   * It can prove very useful to detect pattern in the inputs causing the problem to occur
+   */
+  verbose?: boolean;
 }

src/check/runner/configuration/QualifiedParameters.ts

@@ -14,6 +14,7 @@ export class QualifiedParameters {
   path: string;
   logger: (v: string) => void;
   unbiased: boolean;
+  verbose: boolean;
 
   private static readSeed = (p?: Parameters): number => (p != null && p.seed != null ? p.seed : Date.now());
   private static readNumRuns = (p?: Parameters): number => {
@@ -26,6 +27,7 @@ export class QualifiedParameters {
   private static readTimeout = (p?: Parameters): number | null => (p != null && p.timeout != null ? p.timeout : null);
   private static readPath = (p?: Parameters): string => (p != null && p.path != null ? p.path : '');
   private static readUnbiased = (p?: Parameters): boolean => p != null && p.unbiased === true;
+  private static readVerbose = (p?: Parameters): boolean => p != null && p.verbose === true;
   private static readLogger = (p?: Parameters): ((v: string) => void) => {
     if (p != null && p.logger != null) return p.logger;
     return (v: string) => {
@@ -45,7 +47,8 @@ export class QualifiedParameters {
       timeout: QualifiedParameters.readTimeout(p),
       logger: QualifiedParameters.readLogger(p),
       path: QualifiedParameters.readPath(p),
-      unbiased: QualifiedParameters.readUnbiased(p)
+      unbiased: QualifiedParameters.readUnbiased(p),
+      verbose: QualifiedParameters.readVerbose(p)
     };
   }
 

src/check/runner/utils/utils.ts

@@ -28,7 +28,13 @@ function throwIfFailed<Ts>(out: RunDetails<Ts>) {
         out.counterexample
       )}
 Shrunk ${out.numShrinks} time(s)
-Got error: ${out.error}`
+Got error: ${out.error}
+
+${
+        out.failures.length === 0
+          ? 'Hint: Enable verbose mode in order to have the list of all failing values encountered during the run'
+          : `Encountered failures were:\n- ${out.failures.map(pretty).join('\n- ')}`
+      }`
     );
   }
 }

test/unit/check/runner/Runner.spec.ts

@@ -136,6 +136,29 @@ describe('Runner', () => {
           return true;
         })
       ));
+    it('Should not provide list of failures by default (no verbose)', () => {
+      const p: IProperty<[number]> = {
+        isAsync: () => false,
+        generate: () => new Shrinkable([42]) as Shrinkable<[number]>,
+        run: () => 'failure'
+      };
+      const out = check(p) as RunDetails<[number]>;
+      assert.ok(out.failures.length === 0);
+    });
+    it('Should provide the list of failures in verbose mode', () => {
+      const g = function*() {
+        yield new Shrinkable([48]) as Shrinkable<[number]>;
+        yield new Shrinkable([12]) as Shrinkable<[number]>;
+      };
+      const p: IProperty<[number]> = {
+        isAsync: () => false,
+        generate: () => new Shrinkable([42], () => stream(g())) as Shrinkable<[number]>,
+        run: () => 'failure'
+      };
+      const out = check(p, { verbose: true }) as RunDetails<[number]>;
+      assert.ok(out.failures.length !== 0);
+      assert.deepStrictEqual(out.failures, [[42], [48]]);
+    });
     it('Should build the right counterexamplePath', () =>
       fc.assert(
         fc.property(fc.integer(), fc.array(fc.nat(99), 1, 100), (seed, failurePoints) => {
@@ -369,5 +392,33 @@ describe('Runner', () => {
       }
       assert.ok(false, 'Expected an exception, got success');
     });
+    it('Should not provide list of failures by default (no verbose)', () => {
+      const p: IProperty<[number]> = {
+        isAsync: () => false,
+        generate: () => new Shrinkable([42]) as Shrinkable<[number]>,
+        run: () => 'failure'
+      };
+      try {
+        rAssert(p);
+      } catch (err) {
+        assert.ok(err.message.indexOf('Encountered failures were:') === -1);
+      }
+    });
+    it('Should provide the list of failures in verbose mode', () => {
+      const g = function*() {
+        yield new Shrinkable([48]) as Shrinkable<[number]>;
+        yield new Shrinkable([12]) as Shrinkable<[number]>;
+      };
+      const p: IProperty<[number]> = {
+        isAsync: () => false,
+        generate: () => new Shrinkable([42], () => stream(g())) as Shrinkable<[number]>,
+        run: () => 'failure'
+      };
+      try {
+        rAssert(p, { verbose: true });
+      } catch (err) {
+        assert.ok(err.message.indexOf('Encountered failures were:\n- [42]\n- [48]') !== -1);
+      }
+    });
   });
 });

test/unit/check/runner/configuration/QualifiedParameters.spec.ts

@@ -19,7 +19,8 @@ const parametersArbitrary = fc.record<any>(
     numRuns: fc.nat(),
     timeout: fc.nat(),
     path: fc.array(fc.nat()).map(arr => arr.join(':')),
-    unbiased: fc.boolean()
+    unbiased: fc.boolean(),
+    verbose: fc.boolean()
   },
   { withDeletedKeys: true }
 ) as fc.Arbitrary<Parameters>;
@@ -31,7 +32,13 @@ describe('QualifiedParameters', () => {
         fc.property(parametersArbitrary, params => {
           const qualifiedParams = QualifiedParameters.read(params);
           for (const key of Object.keys(params)) {
-            assert.strictEqual((qualifiedParams as any)[key], (params as any)[key]);
+            assert.strictEqual(
+              (qualifiedParams as any)[key],
+              (params as any)[key],
+              `Unexpected value encountered in qualified - ${
+                (qualifiedParams as any)[key]
+              } - for key ${key}, expected: ${(params as any)[key]}`
+            );
           }
         })
       ));