From b3127ae3abe408678db141849613aabea09b4514 Mon Sep 17 00:00:00 2001 From: Seth Silesky <5115498+silesky@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:08:42 -0500 Subject: [PATCH] Improve CI speed, add nx (#1651) --- .github/workflows/ci.yml | 184 ++++++++++++------ .nxignore | 1 + README.md | 11 +- docs/testing.md | 2 +- nx.json | 37 ++++ package.json | 8 +- packages/actions-shared/package.json | 5 + .../browser-destination-runtime/package.json | 5 + .../package.json | 2 +- packages/browser-destinations/package.json | 8 +- packages/browser-destinations/tsconfig.json | 1 + packages/cli-internal/package.json | 5 + packages/cli/src/__tests__/init.test.ts | 2 + packages/core/package.json | 5 + packages/destination-actions/package.json | 7 +- .../destination-subscriptions/package.json | 5 + scripts/assert-lockfile-updated.sh | 11 ++ scripts/assert-types-updated.sh | 12 ++ yarn.lock | 8 +- 19 files changed, 241 insertions(+), 78 deletions(-) create mode 100644 .nxignore create mode 100644 nx.json create mode 100644 scripts/assert-lockfile-updated.sh create mode 100644 scripts/assert-types-updated.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30292a181a..1bb23da289 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,116 +7,175 @@ on: pull_request: jobs: - test-and-build: + test: + name: Unit tests runs-on: ubuntu-20.04 - timeout-minutes: 30 - strategy: matrix: node-version: [18.x] steps: - - uses: actions/checkout@v2 + # See nx recipe: https://nx.dev/recipes/ci/monorepo-ci-github-actions + - uses: actions/checkout@v3 with: persist-credentials: false + fetch-depth: 0 # nx recipe - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: ${{ matrix.node-version }} registry-url: 'https://registry.npmjs.org' + cache: yarn - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" + - name: Use Github Personal Access Token + run: git config --global url."https://${{ secrets.GH_PAT }}@github.com/".insteadOf ssh://git@github.com/ - - uses: actions/cache@v2 - id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) + - uses: nrwl/nx-set-shas@v3 # nx recipe + + - name: Install Dependencies + run: yarn install --frozen-lockfile + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + - name: Build (Affected) + run: NODE_ENV=production yarn nx affected -t build --parallel=3 # nx recipe + + - name: Test (Affected) + run: yarn nx affected -t test --parallel=3 # nx recipe + + lint: + name: Lint + runs-on: ubuntu-20.04 + timeout-minutes: 20 + strategy: + matrix: + node-version: [18.x] + + steps: + - uses: actions/checkout@v3 with: - path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- + persist-credentials: false + fetch-depth: 0 # nx recipe + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + registry-url: 'https://registry.npmjs.org' + cache: yarn - name: Use Github Personal Access Token run: git config --global url."https://${{ secrets.GH_PAT }}@github.com/".insteadOf ssh://git@github.com/ + - uses: nrwl/nx-set-shas@v3 # nx recipe + - name: Install Dependencies run: yarn install --frozen-lockfile env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - name: Build - run: NODE_ENV=production yarn build + - name: Build # TODO: This monorepo should be refactored so packages can be linted invidually. "affected" will not work ATM. + run: NODE_ENV=production yarn build # nx recipe - name: Lint env: NODE_OPTIONS: '--max-old-space-size=4096' run: yarn lint - - name: Validate - run: yarn validate + validate: + name: Validate + runs-on: ubuntu-20.04 + timeout-minutes: 20 + strategy: + matrix: + node-version: [18.x] - - name: Test - run: yarn test + steps: + - uses: actions/checkout@v3 + with: + persist-credentials: false + fetch-depth: 0 # nx recipe + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + registry-url: 'https://registry.npmjs.org' + cache: yarn + + - name: Use Github Personal Access Token + run: git config --global url."https://${{ secrets.GH_PAT }}@github.com/".insteadOf ssh://git@github.com/ + + - uses: nrwl/nx-set-shas@v3 # nx recipe + + - name: Install Dependencies + run: yarn install --frozen-lockfile + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + - name: Assert yarn.lock is up-to-date + run: bash scripts/assert-lockfile-updated.sh + + - name: Build # TODO: This monorepo should be refactored so packages can be linted invidually. "affected" will not work ATM. + run: NODE_ENV=production yarn build # nx recipe + + - name: Validate Definitions + run: yarn validate - - name: destination-subscriptions size + - name: Destination Subscription Size run: | if $(lerna changed | grep -q destination-subscriptions); then yarn subscriptions size fi - # browser-tests-destination: - # # env: # Disable saucelabs - we blew through our quota. - # # SAUCE_USERNAME: ${{secrets.SAUCE_USERNAME}} - # # SAUCE_ACCESS_KEY: ${{secrets.SAUCE_ACCESS_KEY}} + - name: Assert generated types are up-to-date + run: bash scripts/assert-types-updated.sh - # runs-on: ubuntu-20.04 - - # timeout-minutes: 20 - - # strategy: - # matrix: - # node-version: [18.x] - - # steps: - # - uses: actions/checkout@master + browser-destination-bundle-qa: + name: Browser Destination Bundle QA + # env: # Disable saucelabs - we blew through our quota. + # SAUCE_USERNAME: ${{secrets.SAUCE_USERNAME}} + # SAUCE_ACCESS_KEY: ${{secrets.SAUCE_ACCESS_KEY}} + runs-on: ubuntu-20.04 + timeout-minutes: 20 + strategy: + matrix: + node-version: [18.x] - # - name: Use Node.js ${{ matrix.node-version }} - # uses: actions/setup-node@v2 - # with: - # node-version: ${{ matrix.node-version }} - # registry-url: 'https://registry.npmjs.org' + steps: + - uses: actions/checkout@v3 + with: + persist-credentials: false - # - name: Get yarn cache directory path - # id: yarn-cache-dir-path - # run: echo "::set-output name=dir::$(yarn cache dir)" + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + registry-url: 'https://registry.npmjs.org' + cache: yarn - # - uses: actions/cache@v2 - # id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) - # with: - # path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - # key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - # restore-keys: | - # ${{ runner.os }}-yarn- + - name: Use Github Personal Access Token + run: git config --global url."https://${{ secrets.GH_PAT }}@github.com/".insteadOf ssh://git@github.com/ - # - name: Install Dependencies - # run: yarn install --frozen-lockfile - # env: - # NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + - name: Install Dependencies + run: yarn install --frozen-lockfile + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - # - name: Build - # run: NODE_ENV=production yarn build:browser-destinations && yarn browser build-web + - name: Build + run: NODE_ENV=production yarn build:browser-bundles - # - name: Run Saucelabs Tests - # working-directory: packages/browser-destinations-integration-tests - # shell: bash - # run: | - # yarn start-destination-server & - # yarn test:sauce + # - name: Run Saucelabs Tests + # working-directory: packages/browser-destinations-integration-tests + # shell: bash + # run: | + # yarn start-destination-server & + # yarn test:sauce browser-tests-core: + name: 'Browser tests: actions-core' runs-on: ubuntu-20.04 timeout-minutes: 10 @@ -164,6 +223,7 @@ jobs: run: yarn test-browser snyk: + name: Snyk runs-on: ubuntu-20.04 timeout-minutes: 5 diff --git a/.nxignore b/.nxignore new file mode 100644 index 0000000000..ea31b7529b --- /dev/null +++ b/.nxignore @@ -0,0 +1 @@ +packages/cli-internal diff --git a/README.md b/README.md index f666d4255b..e6d094bca5 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,13 @@ For more detailed instruction, see the following READMEs: ### Local development -This is a monorepo with multiple packages leveraging [`lerna`](https://github.com/lerna/lerna) with [Yarn Workspaces](https://classic.yarnpkg.com/en/docs/workspaces): +This is a monorepo with multiple packages leveraging: + +- [`lerna`](https://github.com/lerna/lerna) for publishing +- [`nx`](https://nx.dev) for dependency-tree aware building, linting, testing, and caching (migration away from `lerna` in progress!). +- [Yarn Workspaces](https://classic.yarnpkg.com/en/docs/workspaces) for package symlinking and hoisting. + +Structure: - `packages/ajv-human-errors` - a wrapper around [AJV](https://ajv.js.org/) errors to produce more friendly validation messages - `packages/browser-destinations` - destination definitions that run on device via Analytics 2.0 @@ -66,9 +72,8 @@ yarn login # Requires node 18.12.1, optionally: nvm use 18.12.1 yarn --ignore-optional -yarn bootstrap -yarn build yarn install +yarn build # Run unit tests to ensure things are working! For partners who don't have access to internal packages, you can run: yarn test-partners diff --git a/docs/testing.md b/docs/testing.md index fe47fd8b3e..26c8a64d3d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -325,7 +325,7 @@ export NODE_ENV=test ## Code Coverage -Code coverage is automatically collected upon completion of `yarn test`. Results may be inspected by examining the HTML report found at `coverage/lcov-report/index.html`, or directly in your IDE if _lcov_ is supported. +Code coverage is collected upon completion of `yarn test --coverage`. Results may be inspected by examining the HTML report found at `coverage/lcov-report/index.html`, or directly in your IDE if _lcov_ is supported. ## Post Deployment Change Testing diff --git a/nx.json b/nx.json new file mode 100644 index 0000000000..3fb6d6e687 --- /dev/null +++ b/nx.json @@ -0,0 +1,37 @@ +{ + "$schema": "./node_modules/nx/schemas/nx-schema.json", + "namedInputs": { + "sharedGlobals": ["{workspaceRoot}/nx.json", "{workspaceRoot}/tsconfig.json"], + "default": ["{projectRoot}/**/*", "sharedGlobals"], + "production": [ + "default", + "{projectRoot}/tsconfig.json", + "{projectRoot}/tsconfig.build.json", + "{projectRoot}/webpack.config*", + "{projectRoot}/babel.config*", + "!{projectRoot}/**/?(*.)+(spec|test).[jt]s?(x)?(.snap)", + "!{projectRoot}/**/test/**/*" + ] + }, + "tasksRunnerOptions": { + "default": { + "runner": "nx/tasks-runners/default", + "options": { + "cacheableOperations": ["build", "test"] + } + } + }, + "targetDefaults": { + "build": { + "inputs": ["production", "^production"], + "dependsOn": ["^build"] + }, + "test": { + "inputs": ["default", "^production"], + "dependsOn": ["build"] + } + }, + "affected": { + "defaultBase": "main" + } +} diff --git a/package.json b/package.json index 3642661243..359dc42ea7 100644 --- a/package.json +++ b/package.json @@ -18,13 +18,13 @@ "cli-internal": "yarn workspace @segment/actions-cli-internal", "core": "yarn workspace @segment/actions-core", "bootstrap": "lerna bootstrap", - "build": "./bin/run generate:types && lerna run build --stream --ignore @segment/actions-cli-internal && yarn browser build-web", - "build:browser-destinations": "yarn lerna run build --scope=@segment/destinations-manifest --include-dependencies --stream && yarn browser build-web", + "build": "nx run-many -t build", + "build:browser-bundles": "nx build @segment/destinations-manifest && nx build-web @segment/browser-destinations", "types": "./bin/run generate:types", "validate": "./bin/run validate", "lint": "ls -d ./packages/* | xargs -I {} eslint '{}/**/*.ts' --cache", "subscriptions": "yarn workspace @segment/destination-subscriptions", - "test": "lerna run test --stream", + "test": "nx run-many -t test", "test-partners": "lerna run test --stream --ignore @segment/actions-core --ignore @segment/actions-cli --ignore @segment/ajv-human-errors", "test-browser": "bash scripts/test-browser.sh", "typecheck": "lerna run typecheck --stream", @@ -68,7 +68,7 @@ "prettier": "^2.4.1", "process": "^0.11.10", "timers-browserify": "^2.0.12", - "ts-jest": "^27.0.0", + "ts-jest": "^27.0.7", "ts-node": "^9.1.1", "typescript": "4.3.5", "ws": "^8.5.0" diff --git a/packages/actions-shared/package.json b/packages/actions-shared/package.json index 76da8f403b..c440564e16 100644 --- a/packages/actions-shared/package.json +++ b/packages/actions-shared/package.json @@ -46,6 +46,11 @@ }, "jest": { "preset": "ts-jest", + "globals": { + "ts-jest": { + "isolatedModules": true + } + }, "testEnvironment": "node", "modulePathIgnorePatterns": [ "/dist/" diff --git a/packages/browser-destination-runtime/package.json b/packages/browser-destination-runtime/package.json index d3ee67b499..d6265944d8 100644 --- a/packages/browser-destination-runtime/package.json +++ b/packages/browser-destination-runtime/package.json @@ -72,6 +72,11 @@ }, "jest": { "preset": "ts-jest", + "globals": { + "ts-jest": { + "isolatedModules": true + } + }, "testEnvironment": "node", "modulePathIgnorePatterns": [ "/dist/" diff --git a/packages/browser-destinations-integration-tests/package.json b/packages/browser-destinations-integration-tests/package.json index 3843b78a24..17f02fab8a 100644 --- a/packages/browser-destinations-integration-tests/package.json +++ b/packages/browser-destinations-integration-tests/package.json @@ -12,7 +12,7 @@ "test:sauce": "wdio wdio.conf.sauce.ts", "test:local": "wdio wdio.conf.local.ts", "start-destination-server": "yarn ts-node src/server/start-destination-server.ts", - "browser-destinations:build": "NODE_ENV=production yarn lerna run build --scope=@segment/browser-destinations --include-dependencies --stream" + "browser-destinations:build": "NODE_ENV=production yarn nx build-web @segment/browser-destinations" }, "devDependencies": { "@wdio/cli": "^7.26.0", diff --git a/packages/browser-destinations/package.json b/packages/browser-destinations/package.json index 938160f6c1..c235dc2188 100644 --- a/packages/browser-destinations/package.json +++ b/packages/browser-destinations/package.json @@ -34,7 +34,6 @@ "@babel/preset-typescript": "^7.13.0", "@types/gtag.js": "^0.0.13", "@types/jest": "^27.0.0", - "babel-jest": "^27.3.1", "compression-webpack-plugin": "^7.1.2", "concurrently": "^6.3.0", "globby": "^11.0.2", @@ -63,8 +62,13 @@ "@segment/actions-shared": "/../actions-shared/src", "@segment/browser-destination-runtime/(.*)": "/../browser-destination-runtime/src/$1" }, + "globals": { + "ts-jest": { + "isolatedModules": true + } + }, "transform": { - "^.+\\.[t|j]sx?$": "babel-jest" + "^.+\\.[t|j]sx?$": "ts-jest" }, "transformIgnorePatterns": [ "/node_modules/(?!(@segment/analytics-next|@braze/web-sdk/)).+\\.js$" diff --git a/packages/browser-destinations/tsconfig.json b/packages/browser-destinations/tsconfig.json index af9bff8d07..2d4ba95c55 100644 --- a/packages/browser-destinations/tsconfig.json +++ b/packages/browser-destinations/tsconfig.json @@ -1,6 +1,7 @@ { "extends": "../../tsconfig.json", "compilerOptions": { + "allowJs": true, // remove ts-jest warning "module": "esnext", "removeComments": false, "baseUrl": ".", diff --git a/packages/cli-internal/package.json b/packages/cli-internal/package.json index 4721224e68..64d730633a 100644 --- a/packages/cli-internal/package.json +++ b/packages/cli-internal/package.json @@ -90,6 +90,11 @@ }, "jest": { "preset": "ts-jest", + "globals": { + "ts-jest": { + "isolatedModules": true + } + }, "testRegex": "((\\.|/)(test))\\.(tsx?|json)$", "modulePathIgnorePatterns": [ "/dist/" diff --git a/packages/cli/src/__tests__/init.test.ts b/packages/cli/src/__tests__/init.test.ts index 2a47b109ce..05d21100b4 100644 --- a/packages/cli/src/__tests__/init.test.ts +++ b/packages/cli/src/__tests__/init.test.ts @@ -14,6 +14,8 @@ import * as path from 'path' import * as prompt from '../lib/prompt' import * as rimraf from 'rimraf' +jest.setTimeout(10000) + describe('cli init command', () => { const testDir = path.join('.', 'testResults') beforeAll(() => { diff --git a/packages/core/package.json b/packages/core/package.json index 94d9013a46..e049cfc5ff 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -95,6 +95,11 @@ }, "jest": { "preset": "ts-jest", + "globals": { + "ts-jest": { + "isolatedModules": true + } + }, "testEnvironment": "node", "modulePathIgnorePatterns": [ "/dist/" diff --git a/packages/destination-actions/package.json b/packages/destination-actions/package.json index 41bdb57fbf..8457d09b1e 100644 --- a/packages/destination-actions/package.json +++ b/packages/destination-actions/package.json @@ -27,7 +27,7 @@ "clean": "tsc -b tsconfig.build.json --clean", "postclean": "rm -rf dist", "prepublishOnly": "yarn build", - "test": "jest --coverage", + "test": "jest", "typecheck": "tsc -p tsconfig.build.json --noEmit" }, "devDependencies": { @@ -55,6 +55,11 @@ }, "jest": { "preset": "ts-jest", + "globals": { + "ts-jest": { + "isolatedModules": true + } + }, "testEnvironment": "node", "modulePathIgnorePatterns": [ "/dist/" diff --git a/packages/destination-subscriptions/package.json b/packages/destination-subscriptions/package.json index 6ad3a829e1..5b68b50ba4 100644 --- a/packages/destination-subscriptions/package.json +++ b/packages/destination-subscriptions/package.json @@ -38,6 +38,11 @@ }, "jest": { "preset": "ts-jest", + "globals": { + "ts-jest": { + "isolatedModules": true + } + }, "testEnvironment": "node", "modulePathIgnorePatterns": [ "/dist/" diff --git a/scripts/assert-lockfile-updated.sh b/scripts/assert-lockfile-updated.sh new file mode 100644 index 0000000000..43c4689bcf --- /dev/null +++ b/scripts/assert-lockfile-updated.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# asserts lockfile is up-to-date (https://github.com/yarnpkg/yarn/issues/5840#top). +# This can be removed when yarn is updated to a version that contains --immutable + +yarn install + +git diff yarn.lock +if ! git diff --exit-code yarn.lock; then + echo "Changes were detected in yarn.lock file after running 'yarn install', which is not expected. Please run 'yarn install' locally and commit the changes." + exit 1 +fi diff --git a/scripts/assert-types-updated.sh b/scripts/assert-types-updated.sh new file mode 100644 index 0000000000..ba916fcef9 --- /dev/null +++ b/scripts/assert-types-updated.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +echo "Checking if generated types are up-to-date" + +yarn types + +if [ -n "$(git status --porcelain | grep generated-types.ts)" ]; then + echo "Please run 'yarn types' and commit the result!" + exit 1 +fi + +echo "Generated types are up-to-date" diff --git a/yarn.lock b/yarn.lock index 7dc4740707..130d9e157c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16225,10 +16225,10 @@ ts-custom-error@^3.2.0: resolved "https://registry.yarnpkg.com/ts-custom-error/-/ts-custom-error-3.2.0.tgz#ff8f80a3812bab9dc448536312da52dce1b720fb" integrity sha512-cBvC2QjtvJ9JfWLvstVnI45Y46Y5dMxIaG1TDMGAD/R87hpvqFL+7LhvUDhnRCfOnx/xitollFWWvUKKKhbN0A== -ts-jest@^27.0.0: - version "27.0.7" - resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-27.0.7.tgz#fb7c8c8cb5526ab371bc1b23d06e745652cca2d0" - integrity sha512-O41shibMqzdafpuP+CkrOL7ykbmLh+FqQrXEmV9CydQ5JBk0Sj0uAEF5TNNe94fZWKm3yYvWa/IbyV4Yg1zK2Q== +ts-jest@^27.0.7: + version "27.1.5" + resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-27.1.5.tgz#0ddf1b163fbaae3d5b7504a1e65c914a95cff297" + integrity sha512-Xv6jBQPoBEvBq/5i2TeSG9tt/nqkbpcurrEG1b+2yfBrcJelOZF9Ml6dmyMh7bcW9JyFbRYpR5rxROSlBLTZHA== dependencies: bs-logger "0.x" fast-json-stable-stringify "2.x"