✨ Add the ability to provide a custom reporter

src/check/runner/Runner.ts

@@ -13,7 +13,7 @@ import { RunnerIterator } from './RunnerIterator';
 import { SourceValuesIterator } from './SourceValuesIterator';
 import { toss } from './Tosser';
 import { pathWalk } from './utils/PathWalker';
-import { throwIfFailed } from './utils/RunDetailsFormatter';
+import { reportRunDetails } from './utils/RunDetailsFormatter';
 import { IAsyncProperty } from '../property/AsyncProperty';
 import { IProperty } from '../property/Property';
 
@@ -99,7 +99,14 @@ function check<Ts>(rawProperty: IRawProperty<Ts>, params?: Parameters<Ts>) {
     throw new Error('Invalid property encountered, please use a valid property');
   if (rawProperty.run == null)
     throw new Error('Invalid property encountered, please use a valid property not an arbitrary');
-  const qParams = QualifiedParameters.read({ ...readConfigureGlobal(), ...params });
+  const qParams: QualifiedParameters<Ts> = QualifiedParameters.read<Ts>({
+    ...readConfigureGlobal(),
+    ...params,
+  });
+  if (qParams.reporter !== null && qParams.asyncReporter !== null)
+    throw new Error('Invalid parameters encountered, reporter and asyncReporter cannot be specified together');
+  if (qParams.asyncReporter !== null && rawProperty.isAsync())
+    throw new Error('Invalid parameters encountered, only asyncProperty can be used when asyncReporter specified');
   const property = decorateProperty(rawProperty, qParams);
   const generator = toss(property, qParams.seed, qParams.randomType, qParams.examples);
 
@@ -109,13 +116,14 @@ function check<Ts>(rawProperty: IRawProperty<Ts>, params?: Parameters<Ts>) {
   const sourceValues = new SourceValuesIterator(initialValues, maxInitialIterations, maxSkips);
   return property.isAsync()
     ? asyncRunIt(property, sourceValues, qParams.verbose, qParams.markInterruptAsFailure).then((e) =>
-        e.toRunDetails(qParams.seed, qParams.path, qParams.numRuns, maxSkips)
+        e.toRunDetails(qParams.seed, qParams.path, qParams.numRuns, maxSkips, qParams)
       )
     : runIt(property, sourceValues, qParams.verbose, qParams.markInterruptAsFailure).toRunDetails(
         qParams.seed,
         qParams.path,
         qParams.numRuns,
-        maxSkips
+        maxSkips,
+        qParams
       );
 }
 
@@ -144,8 +152,8 @@ function assert<Ts>(property: IProperty<Ts>, params?: Parameters<Ts>): void;
 function assert<Ts>(property: IRawProperty<Ts>, params?: Parameters<Ts>): Promise<void> | void;
 function assert<Ts>(property: IRawProperty<Ts>, params?: Parameters<Ts>) {
   const out = check(property, params);
-  if (property.isAsync()) return (out as Promise<RunDetails<Ts>>).then(throwIfFailed);
-  else throwIfFailed(out as RunDetails<Ts>);
+  if (property.isAsync()) return (out as Promise<RunDetails<Ts>>).then(reportRunDetails);
+  else reportRunDetails(out as RunDetails<Ts>);
 }
 
 export { check, assert };

src/check/runner/Sampler.ts

@@ -31,7 +31,7 @@ function streamSample<Ts>(
     typeof params === 'number'
       ? { ...readConfigureGlobal(), numRuns: params }
       : { ...readConfigureGlobal(), ...params };
-  const qParams: QualifiedParameters<Ts> = QualifiedParameters.read(extendedParams);
+  const qParams: QualifiedParameters<Ts> = QualifiedParameters.read<Ts>(extendedParams);
   const tossedValues: Stream<() => Shrinkable<Ts>> = stream(
     toss(toProperty(generator, qParams), qParams.seed, qParams.randomType, qParams.examples)
   );
@@ -94,7 +94,7 @@ function statistics<Ts>(
     typeof params === 'number'
       ? { ...readConfigureGlobal(), numRuns: params }
       : { ...readConfigureGlobal(), ...params };
-  const qParams = QualifiedParameters.read(extendedParams);
+  const qParams: QualifiedParameters<Ts> = QualifiedParameters.read<Ts>(extendedParams);
   const recorded: { [key: string]: number } = {};
   for (const g of streamSample(generator, params)) {
     const out = classify(g);

src/check/runner/configuration/GlobalParameters.ts

@@ -3,7 +3,7 @@ import { Parameters } from './Parameters';
 
 const globalParametersSymbol = Symbol.for('fast-check/GlobalParameters');
 
-export type GlobalParameters = Pick<Parameters, Exclude<keyof Parameters, 'path' | 'examples'>>;
+export type GlobalParameters = Pick<Parameters<unknown>, Exclude<keyof Parameters<unknown>, 'path' | 'examples'>>;
 
 /**
  * Define global parameters that will be used by all the runners

src/check/runner/configuration/Parameters.ts

@@ -1,6 +1,7 @@
 import { RandomGenerator } from 'pure-rand';
 import { RandomType } from './RandomType';
 import { VerbosityLevel } from './VerbosityLevel';
+import { RunDetails } from '../reporter/RunDetails';
 
 /**
  * Customization of the parameters used to run the properties
@@ -112,4 +113,11 @@ export interface Parameters<T = void> {
    * it replays only the minimal counterexample.
    */
   endOnFailure?: boolean;
+
+  // TODO Default typings when T=void should be unknown
+  // Add note explaining that only one of them can be specified
+  // Add note explaining that specifying asyncReporter require users to use asyncProperty everywhere
+  // TODO ensure reporter is not asynchronous in typings
+  reporter?: (runDetails: RunDetails<T>) => void;
+  asyncReporter?: (runDetails: RunDetails<T>) => Promise<void>;
 }

src/check/runner/configuration/QualifiedParameters.ts

@@ -1,6 +1,7 @@
 import prand, { RandomGenerator } from 'pure-rand';
 import { Parameters } from './Parameters';
 import { VerbosityLevel } from './VerbosityLevel';
+import { RunDetails } from '../reporter/RunDetails';
 
 /**
  * @hidden
@@ -24,6 +25,8 @@ export class QualifiedParameters<T> {
   skipAllAfterTimeLimit: number | null;
   interruptAfterTimeLimit: number | null;
   markInterruptAsFailure: boolean;
+  reporter: ((runDetails: RunDetails<T>) => void) | null;
+  asyncReporter: ((runDetails: RunDetails<T>) => Promise<void>) | null;
 
   constructor(op?: Parameters<T>) {
     const p = op || {};
@@ -44,6 +47,43 @@ export class QualifiedParameters<T> {
     this.unbiased = QualifiedParameters.readBoolean(p, 'unbiased');
     this.examples = QualifiedParameters.readOrDefault(p, 'examples', []);
     this.endOnFailure = QualifiedParameters.readBoolean(p, 'endOnFailure');
+    this.reporter = QualifiedParameters.readOrDefault(p, 'reporter', null);
+    this.asyncReporter = QualifiedParameters.readOrDefault(p, 'asyncReporter', null);
+  }
+
+  toParameters(): Parameters<T> {
+    const orUndefined = <V>(value: V | null) => (value !== null ? value : undefined);
+    const parameters = {
+      seed: this.seed,
+      randomType: this.randomType,
+      numRuns: this.numRuns,
+      maxSkipsPerRun: this.maxSkipsPerRun,
+      timeout: orUndefined(this.timeout),
+      skipAllAfterTimeLimit: orUndefined(this.skipAllAfterTimeLimit),
+      interruptAfterTimeLimit: orUndefined(this.interruptAfterTimeLimit),
+      markInterruptAsFailure: this.markInterruptAsFailure,
+      path: this.path,
+      logger: this.logger,
+      unbiased: this.unbiased,
+      verbose: this.verbose,
+      examples: this.examples,
+      endOnFailure: this.endOnFailure,
+      reporter: orUndefined(this.reporter),
+      asyncReporter: orUndefined(this.asyncReporter),
+    };
+
+    // As we do not want to miss any of the parameters,
+    // we want the compilation to fail in case we missed one when building `parameters`
+    // in the code above. `failIfMissing` is ensuring that for us.
+
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-expect-error
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const _failIfMissing: keyof Parameters<T> extends keyof typeof parameters
+      ? true
+      : 'Some properties of Parameters<T> have not been specified' = true;
+
+    return parameters;
   }
 
   private static readSeed = <T>(p: Parameters<T>): number => {

src/check/runner/reporter/RunDetails.ts

@@ -1,5 +1,6 @@
 import { VerbosityLevel } from '../configuration/VerbosityLevel';
 import { ExecutionTree } from './ExecutionTree';
+import { Parameters } from '../configuration/Parameters';
 
 /**
  * Post-run details produced by {@link check}
@@ -129,4 +130,11 @@ interface RunDetailsWithDoc<Ts> {
    * Verbosity level required by the user
    */
   verbose: VerbosityLevel;
+  /**
+   * Configuration of the run
+   *
+   * It includes both local parameters set on `fc.assert` or `fc.check`
+   * and global ones specified using `fc.configureGlobal`
+   */
+  runConfiguration: Parameters<Ts>;
 }

src/check/runner/reporter/RunExecution.ts

@@ -2,6 +2,7 @@ import { VerbosityLevel } from '../configuration/VerbosityLevel';
 import { ExecutionStatus } from './ExecutionStatus';
 import { ExecutionTree } from './ExecutionTree';
 import { RunDetails } from './RunDetails';
+import { QualifiedParameters } from '../configuration/QualifiedParameters';
 
 /**
  * @hidden
@@ -91,7 +92,13 @@ export class RunExecution<Ts> {
     return [...offsetItems.slice(0, offsetItems.length - 1), `${middle}`, ...remainingItems.slice(1)].join(':');
   };
 
-  toRunDetails(seed: number, basePath: string, numRuns: number, maxSkips: number): RunDetails<Ts> {
+  toRunDetails(
+    seed: number,
+    basePath: string,
+    numRuns: number,
+    maxSkips: number,
+    qParams: QualifiedParameters<Ts>
+  ): RunDetails<Ts> {
     if (!this.isSuccess()) {
       // encountered a property failure
       return {
@@ -114,6 +121,7 @@ export class RunExecution<Ts> {
         failures: this.extractFailures(),
         executionSummary: this.rootExecutionTrees,
         verbose: this.verbosity,
+        runConfiguration: qParams.toParameters(),
       };
     }
 
@@ -144,6 +152,7 @@ export class RunExecution<Ts> {
       failures: [],
       executionSummary: this.rootExecutionTrees,
       verbose: this.verbosity,
+      runConfiguration: qParams.toParameters(),
     };
   }
 }

src/check/runner/utils/RunDetailsFormatter.ts

@@ -153,4 +153,16 @@ function throwIfFailed<Ts>(out: RunDetails<Ts>): void {
   throw new Error(defaultReportMessage(out));
 }
 
-export { defaultReportMessage, throwIfFailed };
+/**
+ * @hidden
+ * In case this code has to be executed synchronously the caller
+ * has to make sure that no asyncReporter has been defined
+ * otherwise it might trigger an unchecked promise
+ */
+function reportRunDetails<Ts>(out: RunDetails<Ts>): Promise<void> | void {
+  if (out.runConfiguration.asyncReporter) return out.runConfiguration.asyncReporter(out);
+  else if (out.runConfiguration.reporter) return out.runConfiguration.reporter(out);
+  else return throwIfFailed(out);
+}
+
+export { defaultReportMessage, reportRunDetails };

test/type/index.test-d.ts

@@ -5,6 +5,28 @@ import * as fc from 'fast-check';
 expectType<void>(fc.assert(fc.property(fc.nat(), () => {})));
 expectType<Promise<void>>(fc.assert(fc.asyncProperty(fc.nat(), async () => {})));
 
+// assert (beforeEach, afterEach)
+expectError(fc.assert(fc.property(fc.nat(), () => {}).beforeEach(async () => {})));
+expectError(fc.assert(fc.property(fc.nat(), () => {}).afterEach(async () => {})));
+
+// assert (reporter)
+expectType(
+  fc.assert(
+    fc.property(fc.nat(), fc.string(), () => {}),
+    {
+      reporter: (out: fc.RunDetails<[number, string]>) => {},
+    }
+  )
+);
+expectError(
+  fc.assert(
+    fc.property(fc.nat(), () => {}),
+    {
+      reporter: (out: fc.RunDetails<[string, string]>) => {},
+    }
+  )
+);
+
 // property
 expectType(fc.property(fc.nat(), (a) => {}) as fc.IProperty<[number]>);
 expectType(fc.property(fc.nat(), fc.string(), (a, b) => {}) as fc.IProperty<[number, string]>);
@@ -17,8 +39,6 @@ expectType(
   )
 );
 expectError(fc.property(fc.nat(), fc.string(), (a: number, b: number) => {}));
-expectError(fc.assert(fc.property(fc.nat(), () => {}).beforeEach(async () => {})));
-expectError(fc.assert(fc.property(fc.nat(), () => {}).afterEach(async () => {})));
 
 // asyncProperty
 expectType(fc.asyncProperty(fc.nat(), async (a) => {}) as fc.IAsyncProperty<[number]>);
@@ -110,3 +130,7 @@ expectType<fc.Arbitrary<number[]>>(fc.dedup(fc.nat(), 5)); // TODO Typings shoul
 // func arbitrary
 expectType<fc.Arbitrary<() => number>>(fc.func(fc.nat()));
 expectError(fc.func(1));
+
+// configureGlobal
+expectType(fc.configureGlobal({ reporter: (out: fc.RunDetails<unknown>) => {} }));
+expectError(fc.configureGlobal({ reporter: (out: fc.RunDetails<[number]>) => {} }));