feat!: move test runner into a separate package (#2721)

Closes vitest-dev/vitest#1470

.vitepress/config.ts

@@ -85,6 +85,7 @@ export default withPwa(defineConfig({
       { text: 'Guide', link: '/guide/' },
       { text: 'API', link: '/api/' },
       { text: 'Config', link: '/config/' },
+      { text: 'Advanced', link: '/advanced/api' },
       {
         text: `v${version}`,
         items: [
@@ -102,6 +103,21 @@ export default withPwa(defineConfig({
 
     sidebar: {
       // TODO: bring sidebar of apis and config back
+      '/advanced': [
+        {
+          text: 'Advanced',
+          items: [
+            {
+              text: 'Vitest Node API',
+              link: '/advanced/api',
+            },
+            {
+              text: 'Runner API',
+              link: '/advanced/runner',
+            },
+          ],
+        },
+      ],
       '/': [
         {
           text: 'Guide',

advanced/api.md

@@ -0,0 +1,67 @@
+# Node API
+
+::: warning
+Vitest exposes experimental private API. Breaking changes might not follow semver, please pin Vitest's version when using it.
+:::
+
+## startVitest
+
+You can start running Vitest tests using its Node API:
+
+```js
+import { startVitest } from 'vitest/node'
+
+const vitest = await startVitest('test', ['tests/run-only.test.ts'])
+
+await vitest?.close()
+```
+
+`startVitest` function returns `Vitest` instance if tests can be started. It returns `undefined`, if one of the following occurs:
+
+- Vitest didn't find "vite" package (usually installed with Vitest)
+- If coverage is enabled and run mode is "test", but the coverage package is not installed (`@vitest/coverage-c8` or `@vitest/coverage-istanbul`)
+- If the environment package is not installed (`jsdom`/`happy-dom`/`@edge-runtime/vm`)
+
+If `undefined` is returned or tests failed during the run, Vitest sets `process.exitCode` to `1`.
+
+If watch mode is not enabled, Vitest will call `close` method.
+
+If watch mode is enabled and the terminal supports TTY, Vitest will register console shortcuts.
+
+## createVitest
+
+You can create Vitest instance yourself using `createVitest` function. It returns the same `Vitest` instance as `startVitest`, but it doesn't start tests and doesn't validate installed packages.
+
+```js
+import { createVitest } from 'vitest/node'
+
+const vitest = await createVitest('test', {
+  watch: false,
+})
+```
+
+## Vitest
+
+Vitest instance requires the current test mode. I can be either:
+
+- `test` when running runtime tests
+- `benchmark` when running benchmarks
+- `typecheck` when running type tests
+
+### mode
+
+#### test
+
+Test mode will only call functions inside `test` or `it`, and throws an error when `bench` is encountered. This mode uses `include` and `exclude` options in the config to find test files.
+
+#### benchmark
+
+Benchmark mode calls `bench` functions and throws an error, when it encounters `test` or `it`. This mode uses `benchmark.include` and `benchmark.exclude` options in the config to find benchmark files.
+
+#### typecheck
+
+Typecheck mode doesn't _run_ tests. It only analyses types and gives a summary. This mode uses `typecheck.include` and `typecheck.exclude` options in the config to find files to analyze.
+
+### start
+
+You can start running tests or benchmarks with `start` method. You can pass an array of strings to filter test files.

advanced/runner.md

@@ -0,0 +1,148 @@
+# Test Runner
+
+::: warning
+This is advanced API. If you are just running tests, you probably don't need this. It is primarily used by library authors.
+:::
+
+You can specify a path to your test runner with the `runner` option in your configuration file. This file should have a default export with a class implementing these methods:
+
+```ts
+export interface VitestRunner {
+  /**
+   * First thing that's getting called before actually collecting and running tests.
+   */
+  onBeforeCollect?(paths: string[]): unknown
+  /**
+   * Called after collecting tests and before "onBeforeRun".
+   */
+  onCollected?(files: File[]): unknown
+
+  /**
+   * Called before running a single test. Doesn't have "result" yet.
+   */
+  onBeforeRunTest?(test: Test): unknown
+  /**
+   * Called before actually running the test function. Already has "result" with "state" and "startTime".
+   */
+  onBeforeTryTest?(test: Test, retryCount: number): unknown
+  /**
+   * Called after result and state are set.
+   */
+  onAfterRunTest?(test: Test): unknown
+  /**
+   * Called right after running the test function. Doesn't have new state yet. Will not be called, if the test function throws.
+   */
+  onAfterTryTest?(test: Test, retryCount: number): unknown
+
+  /**
+   * Called before running a single suite. Doesn't have "result" yet.
+   */
+  onBeforeRunSuite?(suite: Suite): unknown
+  /**
+   * Called after running a single suite. Has state and result.
+   */
+  onAfterRunSuite?(suite: Suite): unknown
+
+  /**
+   * If defined, will be called instead of usual Vitest suite partition and handling.
+   * "before" and "after" hooks will not be ignored.
+   */
+  runSuite?(suite: Suite): Promise<void>
+  /**
+   * If defined, will be called instead of usual Vitest handling. Useful, if you have your custom test function.
+   * "before" and "after" hooks will not be ignored.
+   */
+  runTest?(test: Test): Promise<void>
+
+  /**
+   * Called, when a task is updated. The same as "onTaskUpdate" in a reporter, but this is running in the same thread as tests.
+   */
+  onTaskUpdate?(task: [string, TaskResult | undefined][]): Promise<void>
+
+  /**
+   * Called before running all tests in collected paths.
+   */
+  onBeforeRun?(files: File[]): unknown
+  /**
+   * Called right after running all tests in collected paths.
+   */
+  onAfterRun?(files: File[]): unknown
+  /**
+   * Called when new context for a test is defined. Useful, if you want to add custom properties to the context.
+   * If you only want to define custom context with a runner, consider using "beforeAll" in "setupFiles" instead.
+   */
+  extendTestContext?(context: TestContext): TestContext
+  /**
+   * Called, when files are imported. Can be called in two situations: when collecting tests and when importing setup files.
+   */
+  importFile(filepath: string, source: VitestRunnerImportSource): unknown
+  /**
+   * Publically available configuration.
+   */
+  config: VitestRunnerConfig
+}
+```
+
+When initiating this class, Vitest passes down Vitest config, - you should expose it as a `config` property.
+
+::: warning
+`importFile` method in your custom runner must be inlined in `deps.inline` config option, if you call Node `import` inside.
+:::
+
+::: tip
+Snapshot support and some other features depend on the runner. If you don't want to lose it, you can extend your runner from `VitestTestRunner` imported from `vitest/runners`. It also exposes `BenchmarkNodeRunner`, if you want to extend benchmark functionality.
+:::
+
+## Your task function
+
+You can extend Vitest task system with your tasks. A task is an object that is part of a suite. It is automatically added to the current suite with a `suite.custom` method:
+
+```js
+// ./utils/custom.js
+import { getCurrentSuite, setFn } from 'vitest/suite'
+export { describe, beforeAll, afterAll } from 'vitest'
+
+// this function will be called, when Vitest collects tasks
+export const myCustomTask = function (name, fn) {
+  const task = getCurrentSuite().custom(name)
+  task.meta = {
+    customPropertyToDifferentiateTask: true
+  }
+  setFn(task, fn || (() => {}))
+}
+```
+
+```js
+// ./garden/tasks.test.js
+import { afterAll, beforeAll, describe, myCustomTask } from '../utils/custom.js'
+import { gardener } from './gardener.js'
+
+deccribe('take care of the garden', () => {
+  beforeAll(() => {
+    gardener.putWorkingClothes()
+  })
+
+  myCustomTask('weed the grass', () => {
+    gardener.weedTheGrass()
+  })
+  myCustomTask('water flowers', () => {
+    gardener.waterFlowers()
+  })
+
+  afterAll(() => {
+    gardener.goHome()
+  })
+})
+```
+
+```bash
+vitest ./garder/tasks.test.js
+```
+
+::: warning
+If you don't have a custom runner or didn't define `runTest` method, Vitest will try to retrieve a task automatically. If you didn't add a function with `setFn`, it will fail.
+:::
+
+::: tip
+Custom task system supports hooks and contexts. If you want to support property chaining (like, `only`, `skip`, and your custom ones), you can import `createChainable` from `vitest/suite` and wrap your function with it. You will need to call `custom` as `custom.call(this)`, if you decide to do this.
+:::

config/index.md

@@ -148,6 +148,13 @@ TypeError: default is not a function
 
 By default, Vitest assumes you are using a bundler to bypass this and will not fail, but you can disable this behaviour manually, if you code is not processed.
 
+### runner
+
+- **Type**: `VitestRunnerConstructor`
+- **Default**: `node`, when running tests, or `benchmark`, when running benchmarks
+
+Path to a custom test runner. This is an advanced feature and should be used with custom library runners. You can read more about it in [the documentation](/advanced/runner).
+
 ### benchmark
 
 - **Type:** `{ include?, exclude?, ... }`

guide/ui.md

@@ -22,7 +22,7 @@ Then you can visit the Vitest UI at <a href="http://localhost:51204/__vitest__/"
 <img alt="Vitest UI" img-light src="https://user-images.githubusercontent.com/11247099/171992267-5cae2fa0-b927-400a-8eb1-da776974cb61.png">
 <img alt="Vitest UI" img-dark src="https://user-images.githubusercontent.com/11247099/171992272-7c6057e2-80c3-4b17-a7b6-0ac28e5a5e0b.png">
 
-Since Vitest 0.26.0, UI can also be used as a reporter. Use `'html'` reporter in your Vitest configuration to generate HTML output and preview results of your tests:
+Since Vitest 0.26.0, UI can also be used as a reporter. Use `'html'` reporter in your Vitest configuration to generate HTML output and preview the results of your tests:
 
 ```ts
 // vitest.config.ts