✨ Implement interruptAfterTimeLimit (#428)

documentation/1-Guides/Runners.md

@@ -107,6 +107,9 @@ export interface Parameters<T = void> {
                         // it replays only the minimal counterexample
     skipAllAfterTimeLimit?: number; // optional, skip all runs after a given time limit
                         // in milliseconds (relies on Date.now): disabled by default
+    interruptAfterTimeLimit?: number; // optional, interrupt test execution after a given time limit
+                        // in milliseconds (relies on Date.now): disabled by default
+    markInterruptAsFailure?: boolean; // optional, mark interrupted runs as failure: disabled by default
 }
 ```
 

src/check/precondition/PreconditionFailure.ts

@@ -2,7 +2,7 @@
 export class PreconditionFailure extends Error {
   private static readonly SharedFootPrint: symbol = Symbol.for('fast-check/PreconditionFailure');
   private readonly footprint: symbol;
-  constructor() {
+  constructor(readonly interruptExecution: boolean = false) {
     super();
     this.footprint = PreconditionFailure.SharedFootPrint;
   }

src/check/property/SkipAfterProperty.ts

@@ -5,14 +5,19 @@ import { IProperty } from './IProperty';
 /** @hidden */
 export class SkipAfterProperty<Ts> implements IProperty<Ts> {
   private skipAfterTime: number;
-  constructor(readonly property: IProperty<Ts>, readonly getTime: () => number, timeLimit: number) {
+  constructor(
+    readonly property: IProperty<Ts>,
+    readonly getTime: () => number,
+    timeLimit: number,
+    readonly interruptExecution: boolean
+  ) {
     this.skipAfterTime = this.getTime() + timeLimit;
   }
   isAsync = () => this.property.isAsync();
   generate = (mrng: Random, runId?: number) => this.property.generate(mrng, runId);
   run = (v: Ts) => {
     if (this.getTime() >= this.skipAfterTime) {
-      return new PreconditionFailure();
+      return new PreconditionFailure(this.interruptExecution);
     }
     return this.property.run(v);
   };

src/check/runner/DecorateProperty.ts

@@ -5,14 +5,19 @@ import { UnbiasedProperty } from '../property/UnbiasedProperty';
 import { QualifiedParameters } from './configuration/QualifiedParameters';
 
 /** @hidden */
-type MinimalQualifiedParameters<Ts> = Pick<QualifiedParameters<Ts>, 'unbiased' | 'timeout' | 'skipAllAfterTimeLimit'>;
+type MinimalQualifiedParameters<Ts> = Pick<
+  QualifiedParameters<Ts>,
+  'unbiased' | 'timeout' | 'skipAllAfterTimeLimit' | 'interruptAfterTimeLimit'
+>;
 
 /** @hidden */
 export function decorateProperty<Ts>(rawProperty: IProperty<Ts>, qParams: MinimalQualifiedParameters<Ts>) {
   let prop = rawProperty;
   if (rawProperty.isAsync() && qParams.timeout != null) prop = new TimeoutProperty(prop, qParams.timeout);
   if (qParams.unbiased === true) prop = new UnbiasedProperty(prop);
   if (qParams.skipAllAfterTimeLimit != null)
-    prop = new SkipAfterProperty(prop, Date.now, qParams.skipAllAfterTimeLimit);
+    prop = new SkipAfterProperty(prop, Date.now, qParams.skipAllAfterTimeLimit, false);
+  if (qParams.interruptAfterTimeLimit != null)
+    prop = new SkipAfterProperty(prop, Date.now, qParams.interruptAfterTimeLimit, true);
   return prop;
 }

src/check/runner/Runner.ts

@@ -21,9 +21,10 @@ import { throwIfFailed } from './utils/RunDetailsFormatter';
 function runIt<Ts>(
   property: IProperty<Ts>,
   sourceValues: SourceValuesIterator<Shrinkable<Ts>>,
-  verbose: VerbosityLevel
+  verbose: VerbosityLevel,
+  interruptedAsFailure: boolean
 ): RunExecution<Ts> {
-  const runner = new RunnerIterator(sourceValues, verbose);
+  const runner = new RunnerIterator(sourceValues, verbose, interruptedAsFailure);
   for (const v of runner) {
     const out = property.run(v) as PreconditionFailure | string | null;
     runner.handleResult(out);
@@ -35,9 +36,10 @@ function runIt<Ts>(
 async function asyncRunIt<Ts>(
   property: IProperty<Ts>,
   sourceValues: SourceValuesIterator<Shrinkable<Ts>>,
-  verbose: VerbosityLevel
+  verbose: VerbosityLevel,
+  interruptedAsFailure: boolean
 ): Promise<RunExecution<Ts>> {
-  const runner = new RunnerIterator(sourceValues, verbose);
+  const runner = new RunnerIterator(sourceValues, verbose, interruptedAsFailure);
   for (const v of runner) {
     const out = await property.run(v);
     runner.handleResult(out);
@@ -106,10 +108,10 @@ function check<Ts>(rawProperty: IProperty<Ts>, params?: Parameters<Ts>) {
   const initialValues = buildInitialValues(generator, qParams);
   const sourceValues = new SourceValuesIterator(initialValues, maxInitialIterations, maxSkips);
   return property.isAsync()
-    ? asyncRunIt(property, sourceValues, qParams.verbose).then(e =>
+    ? asyncRunIt(property, sourceValues, qParams.verbose, qParams.markInterruptAsFailure).then(e =>
         e.toRunDetails(qParams.seed, qParams.path, qParams.numRuns, maxSkips)
       )
-    : runIt(property, sourceValues, qParams.verbose).toRunDetails(
+    : runIt(property, sourceValues, qParams.verbose, qParams.markInterruptAsFailure).toRunDetails(
         qParams.seed,
         qParams.path,
         qParams.numRuns,

src/check/runner/RunnerIterator.ts

@@ -18,8 +18,12 @@ export class RunnerIterator<Ts> implements IterableIterator<Ts> {
   private currentIdx: number;
   private currentShrinkable: Shrinkable<Ts>;
   private nextValues: IterableIterator<Shrinkable<Ts>>;
-  constructor(readonly sourceValues: SourceValuesIterator<Shrinkable<Ts>>, verbose: VerbosityLevel) {
-    this.runExecution = new RunExecution<Ts>(verbose);
+  constructor(
+    readonly sourceValues: SourceValuesIterator<Shrinkable<Ts>>,
+    verbose: VerbosityLevel,
+    interruptedAsFailure: boolean
+  ) {
+    this.runExecution = new RunExecution<Ts>(verbose, interruptedAsFailure);
     this.currentIdx = -1;
     this.nextValues = sourceValues;
   }
@@ -28,7 +32,7 @@ export class RunnerIterator<Ts> implements IterableIterator<Ts> {
   }
   next(value?: any): IteratorResult<Ts> {
     const nextValue = this.nextValues.next();
-    if (nextValue.done) {
+    if (nextValue.done || this.runExecution.interrupted) {
       return { done: true, value };
     }
     this.currentShrinkable = nextValue.value;
@@ -42,9 +46,14 @@ export class RunnerIterator<Ts> implements IterableIterator<Ts> {
       this.currentIdx = -1;
       this.nextValues = this.currentShrinkable.shrink();
     } else if (result != null) {
-      // skipped run
-      this.runExecution.skip(this.currentShrinkable.value_);
-      this.sourceValues.skippedOne();
+      if (!result.interruptExecution) {
+        // skipped run
+        this.runExecution.skip(this.currentShrinkable.value_);
+        this.sourceValues.skippedOne();
+      } else {
+        // interrupt signal
+        this.runExecution.interrupt();
+      }
     } else {
       // successful run
       this.runExecution.success(this.currentShrinkable.value_);

src/check/runner/configuration/Parameters.ts

@@ -56,6 +56,25 @@ export interface Parameters<T = void> {
    * Indeed, it might not reached the requested number of successful runs.
    */
   skipAllAfterTimeLimit?: number;
+  /**
+   * Interrupt test execution after a given time limit: disabled by default
+   *
+   * NOTE: Relies on `Date.now()`.
+   *
+   * NOTE:
+   * Useful to avoid having too long running processes in your CI.
+   * Replay capability (see {@link seed}, {@link path}) can still be used if needed.
+   *
+   * WARNING:
+   * If the test got interrupted before any failure occured
+   * and before it reached the requested number of runs specified by {@link numRuns}
+   * it will be marked as success. Except if {@link markInterruptAsFailure} as been set to `true`
+   */
+  interruptAfterTimeLimit?: number;
+  /**
+   * Mark interrupted runs as failed runs: disabled by default
+   */
+  markInterruptAsFailure?: boolean;
   /**
    * Way to replay a failing property directly with the counterexample.
    * It can be fed with the counterexamplePath returned by the failing test (requires `seed` too).

src/check/runner/configuration/QualifiedParameters.ts

@@ -22,6 +22,8 @@ export class QualifiedParameters<T> {
   examples: T[];
   endOnFailure: boolean;
   skipAllAfterTimeLimit: number | null;
+  interruptAfterTimeLimit: number | null;
+  markInterruptAsFailure: boolean;
 
   private static readSeed = <T>(p: Parameters<T>): number => {
     // No seed specified
@@ -93,6 +95,8 @@ export class QualifiedParameters<T> {
       maxSkipsPerRun: QualifiedParameters.readOrDefault(p, 'maxSkipsPerRun', 100),
       timeout: QualifiedParameters.readOrDefault(p, 'timeout', null),
       skipAllAfterTimeLimit: QualifiedParameters.readOrDefault(p, 'skipAllAfterTimeLimit', null),
+      interruptAfterTimeLimit: QualifiedParameters.readOrDefault(p, 'interruptAfterTimeLimit', null),
+      markInterruptAsFailure: QualifiedParameters.readBoolean(p, 'markInterruptAsFailure'),
       logger: QualifiedParameters.readOrDefault(p, 'logger', (v: string) => {
         // tslint:disable-next-line:no-console
         console.log(v);

src/check/runner/reporter/RunDetails.ts

@@ -11,6 +11,10 @@ export interface RunDetails<Ts> {
    * Does the property failed during the execution of {@link check}?
    */
   failed: boolean;
+  /**
+   * Was the execution interrupted?
+   */
+  interrupted: boolean;
   /**
    * Number of runs
    *

src/check/runner/reporter/RunExecution.ts

@@ -18,12 +18,14 @@ export class RunExecution<Ts> {
   failure: string;
   numSkips: number;
   numSuccesses: number;
+  interrupted: boolean;
 
-  constructor(readonly verbosity: VerbosityLevel) {
+  constructor(readonly verbosity: VerbosityLevel, readonly interruptedAsFailure: boolean) {
     this.rootExecutionTrees = [];
     this.currentLevelExecutionTrees = this.rootExecutionTrees;
     this.numSkips = 0;
     this.numSuccesses = 0;
+    this.interrupted = false;
   }
 
   private appendExecutionTree(status: ExecutionStatus, value: Ts) {
@@ -58,6 +60,9 @@ export class RunExecution<Ts> {
       ++this.numSuccesses;
     }
   }
+  interrupt() {
+    this.interrupted = true;
+  }
 
   private isSuccess = (): boolean => this.pathToFailure == null;
   private firstFailure = (): number => (this.pathToFailure ? +this.pathToFailure.split(':')[0] : -1);
@@ -90,6 +95,7 @@ export class RunExecution<Ts> {
       // encountered a property failure
       return {
         failed: true,
+        interrupted: this.interrupted,
         numRuns: this.firstFailure() + 1 - this.numSkips,
         numSkips: this.numSkips,
         numShrinks: this.numShrinks(),
@@ -106,6 +112,7 @@ export class RunExecution<Ts> {
       // too many skips
       return {
         failed: true,
+        interrupted: this.interrupted,
         numRuns: this.numSuccesses,
         numSkips: this.numSkips,
         numShrinks: 0,
@@ -119,8 +126,9 @@ export class RunExecution<Ts> {
       };
     }
     return {
-      failed: false,
-      numRuns,
+      failed: this.interrupted ? this.interruptedAsFailure : false,
+      interrupted: this.interrupted,
+      numRuns: this.numSuccesses,
       numSkips: this.numSkips,
       numShrinks: 0,
       seed,

src/check/runner/utils/RunDetailsFormatter.ts

@@ -49,7 +49,7 @@ function formatExecutionSummary<Ts>(executionTrees: ExecutionTree<Ts>[]): string
 
 /** @hidden */
 function preFormatTooManySkipped<Ts>(out: RunDetails<Ts>) {
-  const message = `Failed to run property, too many pre-condition failures encountered\n\nRan ${
+  const message = `Failed to run property, too many pre-condition failures encountered\n{ seed: ${out.seed} }\n\nRan ${
     out.numRuns
   } time(s)\nSkipped ${out.numSkips} time(s)`;
   let details: string | null = null;
@@ -90,11 +90,33 @@ function preFormatFailure<Ts>(out: RunDetails<Ts>) {
   return { message, details, hints };
 }
 
+/** @hidden */
+function preFormatEarlyInterrupted<Ts>(out: RunDetails<Ts>) {
+  const message = `Property interrupted after ${out.numRuns} tests\n{ seed: ${out.seed} }`;
+  let details: string | null = null;
+  const hints = [];
+
+  if (out.verbose >= VerbosityLevel.VeryVerbose) {
+    details = formatExecutionSummary(out.executionSummary);
+  } else {
+    hints.push(
+      'Enable verbose mode at level VeryVerbose in order to check all generated values and their associated status'
+    );
+  }
+
+  return { message, details, hints };
+}
+
 /** @hidden */
 function throwIfFailed<Ts>(out: RunDetails<Ts>) {
   if (!out.failed) return;
 
-  const { message, details, hints } = out.counterexample == null ? preFormatTooManySkipped(out) : preFormatFailure(out);
+  const { message, details, hints } =
+    out.counterexample == null
+      ? out.interrupted
+        ? preFormatEarlyInterrupted(out)
+        : preFormatTooManySkipped(out)
+      : preFormatFailure(out);
 
   let errorMessage = message;
   if (details != null) errorMessage += `\n\n${details}`;

test/e2e/SkipAllAfterTime.spec.ts

@@ -12,9 +12,58 @@ describe(`SkipAllAfterTime (seed: ${seed})`, () => {
       { skipAllAfterTimeLimit: 0 }
     );
     expect(out.failed).toBe(true); // Not enough tests have been executed
+    expect(out.interrupted).toBe(false);
     expect(out.numRuns).toBe(0);
     expect(out.numShrinks).toBe(0);
     expect(out.numSkips).toBe(10001); // maxSkipsPerRun(100) * numRuns(100) +1
     expect(numRuns).toBe(0); // Expired immediately (timeout = 0)
   });
+  it('should interrupt as soon as delay expires and mark run as success (no failure before)', () => {
+    let numRuns = 0;
+    const out = fc.check(
+      fc.property(fc.integer(), x => {
+        ++numRuns;
+        return true;
+      }),
+      { interruptAfterTimeLimit: 0 }
+    );
+    expect(out.failed).toBe(false); // No failure received before interrupt signal
+    expect(out.interrupted).toBe(true);
+    expect(out.numRuns).toBe(0);
+    expect(out.numShrinks).toBe(0);
+    expect(out.numSkips).toBe(0);
+    expect(numRuns).toBe(0); // Expired immediately (timeout = 0)
+  });
+  it('should interrupt as soon as delay expires and mark run as failure if asked to', () => {
+    let numRuns = 0;
+    const out = fc.check(
+      fc.property(fc.integer(), x => {
+        ++numRuns;
+        return true;
+      }),
+      { interruptAfterTimeLimit: 0, markInterruptAsFailure: true }
+    );
+    expect(out.failed).toBe(true); // No failure received before interrupt signal
+    expect(out.interrupted).toBe(true);
+    expect(out.numRuns).toBe(0);
+    expect(out.numShrinks).toBe(0);
+    expect(out.numSkips).toBe(0);
+    expect(numRuns).toBe(0); // Expired immediately (timeout = 0)
+  });
+  it('should consider interrupt with higer priority than skip', () => {
+    let numRuns = 0;
+    const out = fc.check(
+      fc.property(fc.integer(), x => {
+        ++numRuns;
+        return true;
+      }),
+      { interruptAfterTimeLimit: 0, skipAllAfterTimeLimit: 0 }
+    );
+    expect(out.failed).toBe(false); // No failure received before interrupt signal
+    expect(out.interrupted).toBe(true);
+    expect(out.numRuns).toBe(0);
+    expect(out.numShrinks).toBe(0);
+    expect(out.numSkips).toBe(0);
+    expect(numRuns).toBe(0); // Expired immediately (timeout = 0)
+  });
 });