chore(test-harness): generate tests out of scenarios (#2213)

.circleci/config.yml

@@ -23,8 +23,17 @@ defaults:
     - "12.22"
     - "14.18"
     - "16.13"
+  pkg-cache-path: &pkg-cache-path /tmp/pkg-node-binaries
 
 commands:
+  compute-hashes:
+    steps:
+      - run:
+          name: Compute commit SHAs to be used for caching purposes
+          command: |
+            git rev-parse HEAD~1 > /tmp/previous-commit-sha
+            git rev-parse HEAD > /tmp/current-commit-sha
+
   cached-dependencies:
     steps:
       - restore_cache:
@@ -38,7 +47,8 @@ commands:
           name: Save Yarn Package Cache
           key: *yarn-cache-key
           paths:
-            - ~/.yarn
+            - ~/.yarn # Unix
+            - ~/AppData/Local/Yarn/Berry/cache # Windows
 
   build:
     description: >-
@@ -55,6 +65,8 @@ commands:
     steps:
       - run:
           name: Create the Spectral Binary
+          environment:
+            PKG_CACHE_PATH: *pkg-cache-path
           command: yarn workspace @stoplight/spectral-cli pkg . --public --targets << parameters.targets >> --output binaries/spectral
 
   install-and-build:
@@ -101,12 +113,30 @@ commands:
           - linux
           - windows
         default: linux
+      node-version:
+        type: string
+      max-workers:
+        type: integer
+        default: 3
     steps:
+      - compute-hashes
+      - restore_cache:
+          name: Restore test-harness artifacts
+          keys:
+            - test-harness-{{ checksum "/tmp/previous-commit-sha" }}-<< parameters.os >>-<< parameters.node-version >>
       - build-binary:
           targets: << parameters.os >>
       - run:
           name: Run harness tests
-          command: yarn test.harness
+          command: yarn test.harness --maxWorkers=<< parameters.max-workers >>
+      - save_cache:
+          name: Retain test-harness artifacts
+          key: test-harness-{{ checksum "/tmp/current-commit-sha" }}-<< parameters.os >>-<< parameters.node-version >>
+          paths:
+            - ./.cache/spectral-test-harness
+            - *pkg-cache-path
+            - ./test-harness/tests/
+          when: always
 
   prepare-release:
     steps:
@@ -142,6 +172,8 @@ jobs:
       - test-node
       - test-harness:
           os: linux
+          node-version: << parameters.node-version >>
+          max-workers: 3
 
   test-browser:
     docker:
@@ -165,6 +197,8 @@ jobs:
           max-workers: 3
       - test-harness:
           os: windows
+          node-version: current
+          max-workers: 3
 
   build-nix-binaries:
     docker:

.eslintignore

@@ -1,4 +1,5 @@
 **/__fixtures__/**
 /test-harness/**/*.yaml
+/test-harness/tests/
 /packages/*/dist
 /packages/*/CHANGELOG.md

.gitignore

@@ -12,6 +12,7 @@ node_modules
 
 packages/cli/binaries
 /test-harness/tmp/
+/test-harness/tests/
 packages/*/dist
 
 # testing
@@ -31,7 +32,6 @@ packages/*/dist
 .vscode
 .idea
 .nyc_output
-.DS_Store
 .awcache
 .cache-loader
 .cache

.swcrc

@@ -0,0 +1,7 @@
+{
+  "jsc": {
+    "parser": {
+      "syntax": "typescript"
+    }
+  }
+}

package.json

@@ -33,7 +33,8 @@
     "lint.prettier": "prettier --ignore-path .eslintignore --ignore-unknown --check packages/core/src/ruleset/meta/*.json packages/rulesets/src/{asyncapi,oas}/schemas/*.json docs/**/*.md README.md",
     "pretest": "yarn workspace @stoplight/spectral-ruleset-migrator pretest",
     "test": "yarn pretest && yarn test.karma && yarn test.jest",
-    "test.harness": "jest -c ./test-harness/jest.config.js",
+    "pretest.harness": "ts-node -T test-harness/scripts/generate-tests.ts",
+    "test.harness": "yarn pretest.harness && jest -c test-harness/jest.config.mjs",
     "test.jest": "jest --silent --cacheDirectory=.cache/.jest",
     "test.karma": "karma start",
     "prepare": "husky install",
@@ -68,6 +69,9 @@
     "@semantic-release/changelog": "^6.0.1",
     "@semantic-release/git": "^10.0.1",
     "@semantic-release/github": "^8.0.2",
+    "@swc/core": "^1.2.210",
+    "@swc/jest": "^0.2.21",
+    "@types/file-entry-cache": "^5.0.2",
     "@types/jest": "^27.0.3",
     "@types/jest-when": "^2.7.3",
     "@types/karma": "^6.3.1",
@@ -76,7 +80,6 @@
     "@types/node-fetch": "^2.5.12",
     "@types/node-powershell": "^3.1.1",
     "@types/text-table": "^0.2.2",
-    "@types/tmp": "^0.2.2",
     "@typescript-eslint/eslint-plugin": "^4.33.0",
     "@typescript-eslint/parser": "^4.33.0",
     "eslint": "^7.32.0",
@@ -85,6 +88,7 @@
     "eslint-plugin-prettier": "^4.0.0",
     "fast-glob": "^3.2.7",
     "fetch-mock": "^9.11.0",
+    "file-entry-cache": "^6.0.1",
     "husky": "^7.0.4",
     "jest": "^27.4.3",
     "jest-mock": "^27.4.2",

packages/ruleset-migrator/package.json

@@ -45,8 +45,8 @@
     "prettier": "^2.4.1"
   },
   "scripts": {
-    "pretest": "ts-node ./scripts/generate-test-fixtures.ts && yarn prebuild",
-    "prebuild": "ts-node ./scripts/compile-schemas.ts",
+    "pretest": "ts-node -T ./scripts/generate-test-fixtures.ts && yarn prebuild",
+    "prebuild": "ts-node -T ./scripts/compile-schemas.ts",
     "release": "semantic-release -e semantic-release-monorepo"
   }
 }