feat(incremental): add incremental mode (#3609)

Add `--incremental` mode. With incremental mode active, Stryker will use the results from the previous run to determine which mutants need to be retested, yielding much faster results. You can enable incremental mode with `--incremental` (or using `"incremental": true` in the config file),

These changes also introduce `--incrementalFile` and `--force` to help with several incremental use cases. 

See https://stryker-mutator.io/docs/stryker-js/incremental for more details

.github/workflows/ci.yml

@@ -23,9 +23,26 @@ jobs:
         cache: 'npm'
         cache-dependency-path: '**/package-lock.json'
     - name: Install dependencies
-      run: npm install || npm install # retry once, install on windows is flaky...
+      run: npm ci || npm ci # retry once, install on windows is flaky...
     - name: Build & lint & test
       run: npm run all
+
+  incremental_mutation_test:
+      runs-on: 'ubuntu-latest'
+      steps:
+      - uses: actions/checkout@v3
+      - name: Install dependencies
+        run: npm ci
+      - name: Build
+        run: npm run build
+      - name: Download incremental reports
+        run: npm run download-incremental-reports
+        env:
+          BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+      - name: Run stryker run --incremental
+        run: npm run test:mutation:incremental
+        env:
+          STRYKER_DASHBOARD_API_KEY: ${{ secrets.STRYKER_DASHBOARD_API_KEY }}
 
   e2e:
     runs-on: ${{ matrix.os }}
@@ -46,7 +63,7 @@ jobs:
         cache: 'npm'
         cache-dependency-path: '**/package-lock.json'
     - name: Install dependencies
-      run: npm install || npm install # retry once, install on windows is flaky...
+      run: npm ci || npm ci # retry once, install on windows is flaky...
     - name: Build packages
       run: npm run build
     - name: Run e2e tests

.vscode/launch.json

@@ -4,6 +4,16 @@
   // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
   "configurations": [
+    {
+      "name": "🚀 Run current file",
+      "program": "${file}",
+      "request": "launch",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "cwd": "${fileDirname}",
+      "type": "node"
+    },
     {
       "type": "node",
       "request": "attach",

CHANGELOG.md

@@ -3,6 +3,17 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [6.2.0-beta.0](https://github.com/stryker-mutator/stryker-js/compare/v6.1.2...v6.2.0-beta.0) (2022-06-28)
+
+
+### Features
+
+* **incremental:** add incremental mode ([04cf8a2](https://github.com/stryker-mutator/stryker-js/commit/04cf8a2f87fea5ebe941a1357636389193d7dc13))
+
+
+
+
+
 ## [6.1.2](https://github.com/stryker-mutator/stryker-js/compare/v6.1.1...v6.1.2) (2022-06-28)
 
 **Note:** Version bump only for package stryker-parent

docs/configuration.md

@@ -169,6 +169,15 @@ Config file: `"files": ["src/**/*.js", "!src/**/index.js", "test/**/*.js"]`
 
 **DEPRECATED**. Please use [`ignorePatterns`](#ignorepatterns-string) instead, or use [mutate](#mutate-string) to select which files to mutate. 
 
+### `force` [`boolean`]
+
+Default: `false`<br />
+Command line: `--force`<br />
+Config file: `"force": true`<br />
+
+Run all mutants, even if [`incremental`](#incremental-boolean) is provided and an incremental file exists. Can be used to force a rebuild of the incremental file.
+See [incremental](./incremental.md#forcing-reruns)
+
 ### `ignorePatterns` [`string[]`]
 
 Default: `[]`<br />
@@ -205,6 +214,24 @@ In this example, `'👋'` on line 1 would be mutated to an empty string by the S
 
 _Note:_ Enabling `--ignoreStatic` requires `"coverageAnalysis": "perTest"`, because detecting which mutant is static is done during the initial test run and needs per test coverage analysis.
 
+### `incremental` [`boolean`]
+
+Default: `false`<br />
+Command line: `--incremental`<br />
+Config file: `"incremental": true`<br />
+
+Enable 'incremental mode'. Stryker will store results in a file and use that file to speed up the next `--incremental` run.
+See [incremental](./incremental.md) for more details.
+
+### `incrementalFile` [`string`]
+
+Default: `"reports/stryker-incremental.json"`<br />
+Command line: `--incrementalFile reports/stryker-incremental-alternative.json`<br />
+Config file: `"incrementalFile": "reports/stryker-incremental-alternative.json"`<br />
+
+Specify the file to use for incremental mode.
+See [incremental](./incremental.md) for more details.
+
 ### `inPlace` [`boolean`]
 
 Default: `false`<br />

docs/incremental.md

@@ -0,0 +1,92 @@
+---
+title: Incremental
+custom_edit_url: https://github.com/stryker-mutator/stryker-js/edit/master/docs/incremental.md
+---
+
+StrykerJS is fast! It uses advanced techniques to speed up mutation testing, like coverage analysis, hit limit counters, and hot-reload. However, mutation testing still takes time. You might want to speed things up further for larger code bases, fast feedback, or Continuous Integration (CI) scenarios.
+
+## Incremental mode
+
+_Available since Stryker 6.2_
+
+StrykerJS supports incremental mutation testing to speed things up further. When running in [`--incremental`](./configuration.md#incremental-boolean) mode, StrykerJS will track the changes you make to your code and tests and only runs mutation testing on the changed code. But, of course, it will still provide you with the full mutation report at the end!
+
+You enable incremental mode with the `--incremental` flag:
+
+```
+npx stryker run --incremental
+```
+
+_Setting `"incremental": true` in your stryker.conf.json file is also supported_
+
+StrykerJS stores the previous result in a "reports/stryker-incremental.json" file (determined by the [--incrementalFile](./configuration.md#incrementalfile-string) option). The next time StrykerJS runs, it will read this JSON file and try to reuse as much of it as possible. 
+
+Reuse is possible when:
+- A mutant was "Killed"; the culprit test still exists, and it didn't change.
+- A mutant was not "Killed"; no new test covers it, and no tests changed.
+
+StrykerJS will do a git-like diff of your code and test files to the previous version it finds in the incremental report file in order to match the mutants and tests to the current version of the code.
+
+You can see the statistics of the incremental analysis right after the dry run is performed. It looks like this:
+
+```
+        Mutants:        1 files changed (+2 -2)
+        Tests:          2 files changed (+22 -21)
+        Result:         3731 of 3965 mutant result(s).
+```
+
+Here you can see that:
+- One file with mutants changed (2 mutants added, 2 mutants removed)
+- Two test files changed (22 tests added and 21 tests removed)
+- In total, Stryker will reuse 3731 mutant results, and only 234 mutants need to run.
+
+**Note**: The dry run remains required; as it discovers tests, mutation coverage per test, and ensures Stryker runs successfully when no mutant is active.
+
+## Limitations
+
+Running in incremental mode, Stryker will do its best to produce an accurate mutation testing report. However, there are some limitations here:
+- Stryker will not detect any changes you've made in files other than mutated files and test files.
+- Detecting test file changes is only supported if the test runner plugin supports reporting the test files. (see support table below)
+- Detecting test changes is only supported if the test runner plugin supports reporting test locations. (see support table below)
+- Any other changes to your environment are not detected, such as updates to other files, updated (dev) dependencies, changes to environment variables, changes to `.snap` files, readme files, etc.
+- [Static mutants](../../mutation-testing-elements/static-mutants/) don't have test coverage; thus, Stryker won't detect test changes for them.
+
+| Test runner plugin | Test reporting                  |
+| ------------------ | ------------------------------- |
+| 🃏 Jest            | ✅ Full                            |
+| ☕ Mocha           | ⚠ Tests per file without location |
+| 🟣 Jasmine         | ⚠ Test names only                 |
+| 🔵 Karma           | ⚠ Test names only                 |
+| 🥒 CucumberJS      | ✅ Full                            |
+| ▶ Command          | ❌ Nothing                         |
+
+You can use this table to understand why StrykerJS decides not to rerun a specific mutant even though you've changed tests covering that mutant.
+
+- **Full**  
+  Tests are reported together with their exact locations. Stryker will do a detailed diff to see which specific tests changed.
+- **Tests per file without location**  
+  Stryker knows from which files the tests originated, but not their exact locations. Therefore, Stryker assumes all tests inside a file changed when that file changed.
+- **Test names only**  
+  Stryker can't determine where the tests are located and thus cannot detect when a test changed. As a result, Stryker will only see test changes for tests that are added or removed.
+- **Nothing**  
+  All test details are unknown incremental mode will only detect changes in mutants, not their tests.
+
+## Forcing reruns
+
+With these limitations in mind, you can probably imagine a scenario where you want to force specific mutants to run while using incremental mode. You can do this with `--force`. If you run `--force`, you tell StrykerJS to rerun all mutants in scope, regardless of the incremental file.
+
+Using `--force` is especially beneficial when combined with a custom `--mutate` pattern. I.e., if you only want to rerun the mutants in `src/app.js`, you use:
+
+```
+npx stryker run --incremental --force --mutate src/app.js
+```
+
+You can even specify individual lines to mutate:
+
+```
+npx stryker run --incremental --force --mutate src/app.js:5-7
+```
+
+In this example, you tell Stryker to only run mutation testing for lines 5 through 7 in the `src/app.js` file and update the incremental report.
+
+Using the combination of `--incremental` with a custom `--mutate` pattern, StrykerJS will not remove mutants that are not in scope and still report a full mutation report.

docs/sidebar.json

@@ -5,6 +5,7 @@
   "stryker-js/config-file",
   "stryker-js/configuration",
   "stryker-js/disable-mutants",
+  "stryker-js/incremental",
   "stryker-js/plugins",
   "stryker-js/cucumber-runner",
   "stryker-js/jasmine-runner",

e2e/test/incremental/.mocharc.json

@@ -0,0 +1,4 @@
+{
+  "require": ["spec/chai-register.js", "spec/chai-setup.js"],
+  "spec": ["spec/*.spec.js"]
+}

e2e/test/incremental/cucumber.js

@@ -0,0 +1,4 @@
+export default { 
+  publishQuiet: true,
+  import: ['features/**/*.js']
+}

e2e/test/incremental/features/concat.feature

@@ -0,0 +1,10 @@
+Feature: concat
+
+    Scenario: concat a and b
+    Given input 'foo'
+    When concat with 'bar'
+    Then the result should be 'foobar'
+
+    Scenario: greet me
+    When I greet 'me'
+    Then the result should be '👋 me'

e2e/test/incremental/features/concat.feature.changed

@@ -0,0 +1,10 @@
+Feature: concat
+
+    Scenario: concat a and b
+    Given input 'foo'
+    When concat with 'bar'
+    Then the result should be 'foobar'
+
+    Scenario: greet me
+    When I greet 'me'
+    Then the result should be '👋 me 🙋‍♀️'

e2e/test/incremental/features/concat.feature.original

@@ -0,0 +1,10 @@
+Feature: concat
+
+    Scenario: concat a and b
+    Given input 'foo'
+    When concat with 'bar'
+    Then the result should be 'foobar'
+
+    Scenario: greet me
+    When I greet 'me'
+    Then the result should be '👋 me'

e2e/test/incremental/features/log.feature

@@ -0,0 +1,7 @@
+Feature: Log
+
+    Scenario: log
+    Given input 1
+    When I log
+    Then there should be no Error
+

e2e/test/incremental/features/math.feature

@@ -0,0 +1,13 @@
+Feature: Math
+
+    Scenario: add
+    Given input 1
+    When add with 0
+    Then the result should be 1
+
+    Scenario: multiply
+    Given input 2
+    When multiplied with 0
+    Then the result should be 0
+
+    # Missing feature for `addOne` -> surviving / noCoverage mutant

e2e/test/incremental/features/math.feature.changed

@@ -0,0 +1,13 @@
+Feature: Math
+
+    Scenario: add
+    Given input 1
+    When add with 0
+    Then the result should be 1
+
+    Scenario: multiply
+    Given input 2
+    When multiplied with 3
+    Then the result should be 6
+
+    # Missing feature for `addOne` -> surviving / noCoverage mutant

e2e/test/incremental/features/math.feature.original

@@ -0,0 +1,13 @@
+Feature: Math
+
+    Scenario: add
+    Given input 1
+    When add with 0
+    Then the result should be 1
+
+    Scenario: multiply
+    Given input 2
+    When multiplied with 0
+    Then the result should be 0
+
+    # Missing feature for `addOne` -> surviving / noCoverage mutant

e2e/test/incremental/features/step-definitions/general-steps.js

@@ -0,0 +1,46 @@
+import { Given, When, Then } from '@cucumber/cucumber';
+import { expect } from 'chai';
+import { concat, greet } from '../../src/concat.js';
+import { log } from '../../src/log.js';
+import { add, multiply } from '../../src/math.js';
+
+
+Given('input {string}', function (input) {
+  this.input = input;
+});
+
+Given('input {int}', function (input) {
+  this.input = input;
+});
+
+When('concat with {string}', function (other) {
+  this.result = concat(this.input, other);
+});
+
+When('I greet {string}', function (subject) {
+  this.result = greet(subject);
+});
+
+When('add with {int}', function (other) {
+  this.result = add(this.input, other);
+});
+
+When('multiplied with {int}', function (other) {
+  this.result = multiply(this.input, other);
+});
+
+Then('the result should be {string}', function (expected) {
+  expect(this.result).eq(expected);
+});
+Then('the result should be {int}', function (expected) {
+  expect(this.result).eq(expected);
+});
+
+When('I log', function () {
+  // Write code here that turns the phrase above into concrete actions
+  this.result = log(this.input);
+});
+
+Then('there should be no Error', function () {
+});
+

e2e/test/incremental/jasmine.json

@@ -0,0 +1,7 @@
+{
+  "spec_files": [
+    "*.spec.js"
+  ],
+  "spec_dir": "spec",
+  "jsLoader": "import"
+}

e2e/test/incremental/jest.config.json

@@ -0,0 +1,5 @@
+{
+  "testEnvironment": "node",
+  "testMatch": ["<rootDir>/spec/*.spec.js"],
+  "transform": {}
+}

e2e/test/incremental/karma.conf.cjs

@@ -0,0 +1,22 @@
+// Karma configuration
+// Generated on Tue Nov 30 2021 09:57:14 GMT+0100 (Central European Standard Time)
+
+module.exports = function (config) {
+  config.set({
+    basePath: '',
+    frameworks: ['chai', 'jasmine'],
+    files: [
+      { pattern: 'src/**/*.js', type: 'module' },
+      { pattern: 'spec/chai-setup.js', type: 'module' },
+      { pattern: 'spec/**/*.spec.js', type: 'module' },
+    ],
+    reporters: ['progress'],
+    port: 9876,
+    colors: true,
+    logLevel: config.LOG_INFO,
+    autoWatch: false,
+    browsers: ['ChromeHeadless'],
+    singleRun: true,
+    concurrency: Infinity,
+  });
+};