diff --git a/src/check/runner/Runner.ts b/src/check/runner/Runner.ts index b851b4013c9..2f480606b8b 100644 --- a/src/check/runner/Runner.ts +++ b/src/check/runner/Runner.ts @@ -1,9 +1,11 @@ +import { Stream, stream } from '../../fast-check'; import Shrinkable from '../arbitrary/definition/Shrinkable'; import { AsyncProperty } from '../property/AsyncProperty'; import IProperty from '../property/IProperty'; import { Property } from '../property/Property'; import { TimeoutProperty } from '../property/TimeoutProperty'; -import toss from './Tosser'; +import { pathWalk } from './utils/PathWalker'; +import { toss } from './Tosser'; import { Parameters, QualifiedParameters, RunDetails, RunExecution, throwIfFailed } from './utils/utils'; function runIt(property: IProperty, initialValues: IterableIterator>): RunExecution { @@ -62,9 +64,10 @@ function check(rawProperty: IProperty, params?: Parameters) { function* g() { for (let idx = 0; idx < qParams.num_runs; ++idx) yield generator.next().value(); } + const initialValues = pathWalk(qParams.path, g()); return property.isAsync() - ? asyncRunIt(property, g()).then(e => e.toRunDetails(qParams)) - : runIt(property, g()).toRunDetails(qParams); + ? asyncRunIt(property, initialValues).then(e => e.toRunDetails(qParams)) + : runIt(property, initialValues).toRunDetails(qParams); } function assert(property: AsyncProperty, params?: Parameters): Promise; diff --git a/src/check/runner/Sampler.ts b/src/check/runner/Sampler.ts index 9ca93a89c0f..4200b3f9e37 100644 --- a/src/check/runner/Sampler.ts +++ b/src/check/runner/Sampler.ts @@ -1,16 +1,24 @@ import { stream } from '../../stream/Stream'; import Arbitrary from '../arbitrary/definition/Arbitrary'; +import Shrinkable from '../arbitrary/definition/Shrinkable'; import IProperty from '../property/IProperty'; import toss from './Tosser'; +import { pathWalk } from './utils/PathWalker'; import { Parameters, QualifiedParameters } from './utils/utils'; +function streamSample( + generator: IProperty | Arbitrary, + params?: Parameters | number +): IterableIterator { + const qParams: QualifiedParameters = QualifiedParameters.read_or_num_runs(params); + const tossedValues: IterableIterator> = stream(toss(generator, qParams.seed)).map(s => s()); + return stream(pathWalk(qParams.path, tossedValues)) + .take(qParams.num_runs) + .map(s => s.value); +} + function sample(generator: IProperty | Arbitrary, params?: Parameters | number): Ts[] { - const qParams = QualifiedParameters.read_or_num_runs(params); - return [ - ...stream(toss(generator, qParams.seed)) - .take(qParams.num_runs) - .map(s => s().value) - ]; + return [...streamSample(generator, params)]; } interface Dictionary { @@ -52,9 +60,7 @@ function statistics( ): void { const qParams = QualifiedParameters.read_or_num_runs(params); const recorded: Dictionary = {}; - for (const g of stream(toss(generator, qParams.seed)) - .take(qParams.num_runs) - .map(s => s().value)) { + for (const g of streamSample(generator, params)) { const out = classify(g); const categories: string[] = Array.isArray(out) ? out : [out]; for (const c of categories) { diff --git a/src/check/runner/utils/PathWalker.ts b/src/check/runner/utils/PathWalker.ts new file mode 100644 index 00000000000..57a5436d09b --- /dev/null +++ b/src/check/runner/utils/PathWalker.ts @@ -0,0 +1,20 @@ +import { Stream, stream } from '../../../stream/Stream'; +import Shrinkable from '../../arbitrary/definition/Shrinkable'; + +export function pathWalk( + path: string, + initialValues: IterableIterator> +): IterableIterator> { + let values: Stream> = stream(initialValues); + const segments: number[] = path.split(':').map((text: string) => +text); + if (segments.length === 0) return values; + values = values.drop(segments[0]); + for (const s of segments.slice(1)) { + // tslint:disable-next-line:no-non-null-assertion + values = values + .getNthOrLast(0)! + .shrink() + .drop(s); + } + return values; +} diff --git a/src/check/runner/utils/utils.ts b/src/check/runner/utils/utils.ts index b9e16a8317a..09964644a99 100644 --- a/src/check/runner/utils/utils.ts +++ b/src/check/runner/utils/utils.ts @@ -2,17 +2,20 @@ interface Parameters { seed?: number; num_runs?: number; timeout?: number; + path?: string; logger?(v: string): void; } class QualifiedParameters { seed: number; num_runs: number; timeout: number | null; + path: string; logger: (v: string) => void; private static read_seed = (p?: Parameters): number => (p != null && p.seed != null ? p.seed : Date.now()); private static read_num_runs = (p?: Parameters): number => (p != null && p.num_runs != null ? p.num_runs : 100); private static read_timeout = (p?: Parameters): number | null => (p != null && p.timeout != null ? p.timeout : null); + private static read_path = (p?: Parameters): string => (p != null && p.path != null ? p.path : ''); private static read_logger = (p?: Parameters): ((v: string) => void) => p != null && p.logger != null ? p.logger : (v: string) => console.log(v); @@ -21,7 +24,8 @@ class QualifiedParameters { seed: QualifiedParameters.read_seed(p), num_runs: QualifiedParameters.read_num_runs(p), timeout: QualifiedParameters.read_timeout(p), - logger: QualifiedParameters.read_logger(p) + logger: QualifiedParameters.read_logger(p), + path: QualifiedParameters.read_path(p) }; } static read_or_num_runs(p?: Parameters | number): QualifiedParameters { @@ -88,6 +92,13 @@ class RunExecution { private numShrinks = (): number => (this.pathToFailure ? this.pathToFailure.split(':').length - 1 : 0); toRunDetails(qParams: QualifiedParameters): RunDetails { + const mergePaths = (offsetPath, path) => { + if (offsetPath.length === 0) return path; + const offsetItems = offsetPath.split(':'); + const remainingItems = path.split(':'); + const middle = +offsetItems[offsetItems.length - 1] + +remainingItems[0]; + return [...offsetItems.slice(0, offsetItems.length - 1), `${middle}`, ...remainingItems.slice(1)].join(':'); + }; return this.isSuccess() ? successFor(qParams) : failureFor( @@ -95,7 +106,7 @@ class RunExecution { this.firstFailure() + 1, this.numShrinks(), this.value!, - this.pathToFailure!, + mergePaths(qParams.path, this.pathToFailure!), this.failure ); } diff --git a/test/e2e/ReplayFailures.spec.ts b/test/e2e/ReplayFailures.spec.ts new file mode 100644 index 00000000000..5be6b0288d4 --- /dev/null +++ b/test/e2e/ReplayFailures.spec.ts @@ -0,0 +1,106 @@ +import * as assert from 'power-assert'; +import fc from '../../src/fast-check'; + +const seed = Date.now(); +describe(`ReplayFailures (seed: ${seed})`, () => { + const propArbitrary = fc.set(fc.hexaString()); + const propCheck = data => { + // element at should not contain the first character of the element just before + // 01, 12, 20 - is correct + // 01, 12, 21 - is not + if (data.length === 0) return true; + for (let idx = 1; idx < data.length; ++idx) { + if (data[idx].indexOf(data[idx - 1][0]) !== -1) return false; + } + return true; + }; + const prop = fc.property(propArbitrary, propCheck); + + describe('fc.sample', () => { + it('Should rebuild counterexample using sample and (path, seed)', () => { + const out = fc.check(prop, { seed: seed }); + assert.ok(out.failed, 'Should have failed'); + assert.deepStrictEqual( + fc.sample(propArbitrary, { seed: seed, path: out.counterexample_path, num_runs: 1 })[0], + out.counterexample[0] + ); + }); + it('Should rebuild the whole shrink path using sample', () => { + let failuresRecorded = []; + const out = fc.check( + fc.property(propArbitrary, data => { + if (propCheck(data)) return true; + failuresRecorded.push(data); + return false; + }), + { seed: seed } + ); + assert.ok(out.failed, 'Should have failed'); + + let replayedFailures = []; + const segments = out.counterexample_path.split(':'); + for (let idx = 1; idx !== segments.length + 1; ++idx) { + const p = segments.slice(0, idx).join(':'); + const g = fc.sample(propArbitrary, { seed: seed, path: p, num_runs: 1 }); + replayedFailures.push(g[0]); + } + assert.deepStrictEqual(replayedFailures, failuresRecorded); + }); + }); + describe('fc.assert', () => { + it('Should start from the minimal counterexample given its path', () => { + const out = fc.check(prop, { seed: seed }); + assert.ok(out.failed, 'Should have failed'); + + let numCalls = 0; + let numValidCalls = 0; + let validCallIndex = -1; + const out2 = fc.check( + fc.property(propArbitrary, data => { + try { + assert.deepStrictEqual(data, out.counterexample[0]); + validCallIndex = numCalls; + ++numValidCalls; + } catch (err) {} + ++numCalls; + return propCheck(data); + }), + { seed: seed, path: out.counterexample_path } + ); + assert.equal(numValidCalls, 1); + assert.equal(validCallIndex, 0); + assert.equal(out2.num_runs, 1); + assert.equal(out2.num_shrinks, 0); + assert.equal(out2.counterexample_path, out.counterexample_path); + assert.deepStrictEqual(out2.counterexample, out.counterexample); + }); + it('Should start from any position in the path', () => { + const out = fc.check(prop, { seed: seed }); + assert.ok(out.failed, 'Should have failed'); + + const segments = out.counterexample_path.split(':'); + for (let idx = 1; idx !== segments.length + 1; ++idx) { + const p = segments.slice(0, idx).join(':'); + const outMiddlePath = fc.check(prop, { seed: seed, path: p }); + assert.equal(outMiddlePath.num_runs, 1); + assert.equal(outMiddlePath.num_shrinks, out.num_shrinks - idx + 1); + assert.equal(outMiddlePath.counterexample_path, out.counterexample_path); + assert.deepStrictEqual(outMiddlePath.counterexample, out.counterexample); + } + }); + it('Should take initial path into account when computing path', () => { + const out = fc.check(prop, { seed: seed }); + assert.ok(out.failed, 'Should have failed'); + + const segments = out.counterexample_path.split(':'); + const playOnIndex = seed % segments.length; + + for (let offset = 0; offset !== +segments[playOnIndex]; ++offset) { + const p = [...segments.slice(0, playOnIndex), offset].join(':'); + const outMiddlePath = fc.check(prop, { seed: seed, path: p }); + assert.equal(outMiddlePath.counterexample_path, out.counterexample_path); + assert.deepStrictEqual(outMiddlePath.counterexample, out.counterexample); + } + }); + }); +});