feat(hit limit): infinite loop prevention in karma-runner

e2e/test/hit-limit/karma.conf.js

@@ -0,0 +1,16 @@
+module.exports = function(config) {
+  config.set({
+    frameworks: ['jasmine'],
+    files: [
+      'src/*.js',
+      'test/*.karma.spec.js'
+    ],
+    reporters: ['progress'],
+    colors: true,
+    logLevel: config.LOG_INFO,
+    autoWatch: false,
+    browsers: ['ChromeHeadless'],
+    singleRun: true,
+    concurrency: Infinity
+  })
+}

e2e/test/hit-limit/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "hit-limit",
+  "version": "1.0.0",
+  "private": true,
+  "description": "A e2e test project to verify the infinite loop prevention behavior in different test runner (hit limit)",
+  "main": "index.js",
+  "scripts": {
+    "test:karma": "karma start",
+    "test": "mocha --no-package --timeout 60000 --require \"../../tasks/ts-node-register.js\" verify/verify.ts"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@stryker-mutator/karma-runner": "^5.2.2"
+  }
+}

e2e/test/hit-limit/src/loop.js

@@ -0,0 +1,17 @@
+/**
+ * This loop will result in an infinite loop when Stryker mutates it.
+ */
+function loop(n, action) {
+  let goOn = true;
+  while(goOn) {
+    action(n);
+    n--;
+    goOn = n > 0;
+  }
+}
+
+try{
+  if(module?.exports){
+    module.exports.loop = loop;
+  }
+} catch (err) { }

e2e/test/hit-limit/stryker.conf.json

@@ -0,0 +1,13 @@
+{
+  "$schema": "../../node_modules/@stryker-mutator/core/schema/stryker-schema.json",
+  "packageManager": "npm",
+  "reporters": ["json"],
+  "testRunner_comment": "Test runner is set in verify.ts",
+  "coverageAnalysis": "perTest",
+  "karma": {
+    "projectType": "custom",
+    "configFile": "karma.conf.js",
+    "config": {}
+  },
+  "concurrency": 1
+}

e2e/test/hit-limit/test/loop.karma.spec.js

@@ -0,0 +1,7 @@
+describe('loop', () => {
+  it('should result in 15 for n=5 and a sum function', () => {
+    let result = 0;
+    loop(5, n => result += n);
+    expect(result).toBe(15);
+  });
+});

e2e/test/hit-limit/verify/verify.ts

@@ -0,0 +1,14 @@
+import { expect } from 'chai';
+import { Stryker } from '@stryker-mutator/core';
+import { MutantStatus } from 'mutation-testing-report-schema/api';
+
+describe('Limit counter', () => {
+
+  it('should limit infinite loops in the karma-runner', async () => {
+    const stryker = new Stryker({ testRunner: 'karma' });
+    const results = await stryker.runMutationTest();
+    const timeoutResults = results.filter(res => res.status === MutantStatus.Timeout);
+    expect(timeoutResults).lengthOf(3);
+    timeoutResults.forEach((result) => expect(result.statusReason).eq('Hit limit reached (501/500)'));
+  });
+});

packages/api/src/core/instrument.ts

@@ -8,13 +8,17 @@ export const INSTRUMENTER_CONSTANTS = Object.freeze({
   MUTATION_COVERAGE_OBJECT: identity('mutantCoverage'),
   ACTIVE_MUTANT: identity('activeMutant'),
   CURRENT_TEST_ID: identity('currentTestId'),
+  HIT_COUNT: identity('hitCount'),
+  HIT_LIMIT: identity('hitLimit'),
   ACTIVE_MUTANT_ENV_VARIABLE: '__STRYKER_ACTIVE_MUTANT__',
 } as const);
 
 export interface InstrumenterContext {
   activeMutant?: string;
   currentTestId?: string;
   mutantCoverage?: MutantCoverage;
+  hitCount?: number;
+  hitLimit?: number;
 }
 
 function identity<T extends keyof InstrumenterContext>(key: T): T {

packages/api/src/core/mutant.ts

@@ -28,6 +28,7 @@ export interface Mutant extends Pick<schema.MutantResult, 'id' | 'location' | 'm
 export type MutantTestCoverage = Mutant &
   Pick<schema.MutantResult, 'coveredBy' | 'static'> & {
     estimatedNetTime: number;
+    hitCount?: number;
   };
 
 /**

packages/api/src/test-runner/dry-run-result.ts

@@ -23,6 +23,10 @@ export interface TimeoutDryRunResult {
    * The status of the run
    */
   status: DryRunStatus.Timeout;
+  /**
+   * An optional reason for the timeout
+   */
+  reason?: string;
 }
 
 export interface ErrorDryRunResult {

packages/api/src/test-runner/mutant-run-result.ts

@@ -9,6 +9,10 @@ export type MutantRunResult = ErrorMutantRunResult | KilledMutantRunResult | Sur
 
 export interface TimeoutMutantRunResult {
   status: MutantRunStatus.Timeout;
+  /**
+   * An optional reason for the timeout
+   */
+  reason?: string;
 }
 
 export interface KilledMutantRunResult {

packages/api/src/test-runner/run-options.ts

@@ -16,6 +16,7 @@ export interface DryRunOptions extends RunOptions {
 
 export interface MutantRunOptions extends RunOptions {
   testFilter?: string[];
+  hitLimit?: number;
   activeMutant: Mutant;
   sandboxFileName: string;
 }

packages/api/src/test-runner/run-result-helpers.ts

@@ -1,9 +1,16 @@
 import { TestStatus } from './test-status';
-import { DryRunResult } from './dry-run-result';
+import { DryRunResult, TimeoutDryRunResult } from './dry-run-result';
 import { MutantRunResult, MutantRunStatus } from './mutant-run-result';
 import { DryRunStatus } from './dry-run-status';
 import { FailedTestResult } from './test-result';
 
+export function determineHitLimitReached(hitCount: number | undefined, hitLimit: number | undefined): TimeoutDryRunResult | undefined {
+  if (hitCount !== undefined && hitLimit !== undefined && hitCount > hitLimit) {
+    return { status: DryRunStatus.Timeout, reason: `Hit limit reached (${hitCount}/${hitLimit})` };
+  }
+  return;
+}
+
 export function toMutantRunResult(dryRunResult: DryRunResult): MutantRunResult {
   switch (dryRunResult.status) {
     case DryRunStatus.Complete: {
@@ -31,6 +38,7 @@ export function toMutantRunResult(dryRunResult: DryRunResult): MutantRunResult {
     case DryRunStatus.Timeout:
       return {
         status: MutantRunStatus.Timeout,
+        reason: dryRunResult.reason,
       };
   }
 }

packages/api/test/unit/test_runner/run-result-helpers.spec.ts

@@ -1,55 +1,74 @@
 import { expect } from 'chai';
 
-import { TestStatus, toMutantRunResult, DryRunStatus, MutantRunResult, MutantRunStatus } from '../../../src/test-runner';
+import {
+  TestStatus,
+  toMutantRunResult,
+  DryRunStatus,
+  MutantRunResult,
+  MutantRunStatus,
+  determineHitLimitReached,
+  TimeoutDryRunResult,
+} from '../../../src/test-runner';
 
 describe('runResultHelpers', () => {
+  describe(determineHitLimitReached.name, () => {
+    it('should determine a timeout result when hit count is higher than limit', () => {
+      const expected: TimeoutDryRunResult = { status: DryRunStatus.Timeout, reason: 'Hit limit reached (10/9)' };
+      const actual = determineHitLimitReached(10, 9);
+      expect(actual).deep.eq(expected);
+    });
+    it('should not determine a timeout result when hit count is less than or equal to limit', () => {
+      const actual = determineHitLimitReached(10, 10);
+      expect(actual).undefined;
+    });
+  });
+
   describe(toMutantRunResult.name, () => {
     it('should convert "timeout" to "timeout"', () => {
-      const expected: MutantRunResult = { status: MutantRunStatus.Timeout };
-      expect(toMutantRunResult({ status: DryRunStatus.Timeout })).deep.eq(expected);
+      const expected: MutantRunResult = { status: MutantRunStatus.Timeout, reason: 'timeout reason' };
+      const actual = toMutantRunResult({ status: DryRunStatus.Timeout, reason: 'timeout reason' });
+      expect(actual).deep.eq(expected);
     });
 
     it('should convert "error" to "error"', () => {
       const expected: MutantRunResult = { status: MutantRunStatus.Error, errorMessage: 'some error' };
-      expect(toMutantRunResult({ status: DryRunStatus.Error, errorMessage: 'some error' })).deep.eq(expected);
+      const actual = toMutantRunResult({ status: DryRunStatus.Error, errorMessage: 'some error' });
+      expect(actual).deep.eq(expected);
     });
 
     it('should report a failed test as "killed"', () => {
       const expected: MutantRunResult = { status: MutantRunStatus.Killed, failureMessage: 'expected foo to be bar', killedBy: '42', nrOfTests: 3 };
-      expect(
-        toMutantRunResult({
-          status: DryRunStatus.Complete,
-          tests: [
-            { status: TestStatus.Success, id: 'success1', name: 'success1', timeSpentMs: 42 },
-            { status: TestStatus.Failed, id: '42', name: 'error', timeSpentMs: 42, failureMessage: 'expected foo to be bar' },
-            { status: TestStatus.Success, id: 'success2', name: 'success2', timeSpentMs: 42 },
-          ],
-        })
-      ).deep.eq(expected);
+      const actual = toMutantRunResult({
+        status: DryRunStatus.Complete,
+        tests: [
+          { status: TestStatus.Success, id: 'success1', name: 'success1', timeSpentMs: 42 },
+          { status: TestStatus.Failed, id: '42', name: 'error', timeSpentMs: 42, failureMessage: 'expected foo to be bar' },
+          { status: TestStatus.Success, id: 'success2', name: 'success2', timeSpentMs: 42 },
+        ],
+      });
+      expect(actual).deep.eq(expected);
     });
 
     it('should report only succeeded tests as "survived"', () => {
       const expected: MutantRunResult = { status: MutantRunStatus.Survived, nrOfTests: 3 };
-      expect(
-        toMutantRunResult({
-          status: DryRunStatus.Complete,
-          tests: [
-            { status: TestStatus.Success, id: 'success1', name: 'success1', timeSpentMs: 42 },
-            { status: TestStatus.Success, id: '42', name: 'error', timeSpentMs: 42 },
-            { status: TestStatus.Success, id: 'success2', name: 'success2', timeSpentMs: 42 },
-          ],
-        })
-      ).deep.eq(expected);
+      const actual = toMutantRunResult({
+        status: DryRunStatus.Complete,
+        tests: [
+          { status: TestStatus.Success, id: 'success1', name: 'success1', timeSpentMs: 42 },
+          { status: TestStatus.Success, id: '42', name: 'error', timeSpentMs: 42 },
+          { status: TestStatus.Success, id: 'success2', name: 'success2', timeSpentMs: 42 },
+        ],
+      });
+      expect(actual).deep.eq(expected);
     });
 
     it('should report an empty suite as "survived"', () => {
       const expected: MutantRunResult = { status: MutantRunStatus.Survived, nrOfTests: 0 };
-      expect(
-        toMutantRunResult({
-          status: DryRunStatus.Complete,
-          tests: [],
-        })
-      ).deep.eq(expected);
+      const actual = toMutantRunResult({
+        status: DryRunStatus.Complete,
+        tests: [],
+      });
+      expect(actual).deep.eq(expected);
     });
 
     it("should set nrOfTests with the amount of tests that weren't `skipped`", () => {