From 0b8c2cc2749a299190b57e7f9159e1b9a3998f41 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 1 Dec 2022 21:33:07 -0800 Subject: [PATCH] Apply publish step optimizations (#43620) Follow-up to https://github.com/vercel/next.js/pull/32337 this removes the un-necessary step where we fetch all of the tags which requires pulling a lot of un-necessary git history inflating cache size and publish times. The only reason these tags were needing to be fetched is due to an issue in how the `actions/checkout` step works (https://github.com/actions/checkout/issues/882). This reduces the publish times by at least 4 minutes by removing the tags fetching step https://github.com/vercel/next.js/actions/runs/3598569786/jobs/6061449995#step:16:14 As a further optimization this adds concurrency to the `npm publish` calls themselves to hopefully reduce time spent there as well. --- .github/workflows/build_test_deploy.yml | 5 +- .../{fetch-tags.mjs => check-is-release.js} | 28 +-- scripts/publish-native.js | 180 ++++++++++-------- scripts/publish-release.js | 50 +++-- scripts/release-stats.sh | 14 +- 5 files changed, 150 insertions(+), 127 deletions(-) rename scripts/{fetch-tags.mjs => check-is-release.js} (50%) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 3ad8cc36f014ac..a359ba20d137f9 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -84,11 +84,10 @@ jobs: - run: pnpm install - run: pnpm run build - run: node run-tests.js --timings --write-timings -g 1/1 - - run: node ./scripts/fetch-tags.mjs ${{ github.sha }} - id: check-release run: | - if [[ $(git describe --exact-match 2> /dev/null || :) = v* ]]; + if [[ $(node ./scripts/check-is-release.js 2> /dev/null || :) = v* ]]; then echo "::set-output name=IS_RELEASE::true" else @@ -893,7 +892,7 @@ jobs: - run: npm i -g pnpm@${PNPM_VERSION} - run: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc - - run: ./scripts/publish-native.js $GITHUB_REF + - run: ./scripts/publish-native.js - run: ./scripts/publish-release.js testDeployE2E: diff --git a/scripts/fetch-tags.mjs b/scripts/check-is-release.js similarity index 50% rename from scripts/fetch-tags.mjs rename to scripts/check-is-release.js index e37dab2d509306..143124bd3c8e88 100755 --- a/scripts/fetch-tags.mjs +++ b/scripts/check-is-release.js @@ -1,6 +1,6 @@ -import { execSync } from 'child_process' -import execa from 'execa' -;(async () => { +const { execSync } = require('child_process') + +const checkIsRelease = async () => { let commitId = process.argv[2] || '' // parse only the last string which should be version if @@ -14,21 +14,13 @@ import execa from 'execa' const versionString = commitMsg.split(' ').pop().trim() const publishMsgRegex = /^v\d{1,}\.\d{1,}\.\d{1,}(-\w{1,}\.\d{1,})?$/ - console.log({ commitId, commitMsg, versionString }) - if (publishMsgRegex.test(versionString)) { - console.log('publish commit, fetching tags') - - const result = await execa( - 'git', - ['fetch', '--depth=1', 'origin', '+refs/tags/*:refs/tags/*'], - { - stdio: ['ignore', 'inherit', 'inherit'], - } - ) - - process.exit(result.exitCode) + console.log(versionString) + process.exit(0) } else { - console.log('not publish commit') + console.log('not publish commit', { commitId, commitMsg, versionString }) + process.exit(1) } -})() +} + +checkIsRelease() diff --git a/scripts/publish-native.js b/scripts/publish-native.js index f8ad52eba7a624..457ee6d67cf188 100755 --- a/scripts/publish-native.js +++ b/scripts/publish-native.js @@ -1,18 +1,20 @@ #!/usr/bin/env node const path = require('path') -const { readFile, readdir, writeFile } = require('fs/promises') +const execa = require('execa') const { copy } = require('fs-extra') -const { execSync } = require('child_process') +const { Sema } = require('async-sema') +const { readFile, readdir, writeFile } = require('fs/promises') const cwd = process.cwd() ;(async function () { try { + const publishSema = new Sema(2) + let version = JSON.parse( await readFile(path.join(cwd, 'lerna.json')) ).version - let gitref = process.argv.slice(2)[0] // Copy binaries to package folders, update version, and publish let nativePackagesDir = path.join(cwd, 'packages/next-swc/crates/napi/npm') @@ -20,93 +22,111 @@ const cwd = process.cwd() (name) => !name.startsWith('.') ) - for (let platform of platforms) { - try { - let binaryName = `next-swc.${platform}.node` - await copy( - path.join(cwd, 'packages/next-swc/native', binaryName), - path.join(nativePackagesDir, platform, binaryName) - ) - let pkg = JSON.parse( - await readFile(path.join(nativePackagesDir, platform, 'package.json')) - ) - pkg.version = version - await writeFile( - path.join(nativePackagesDir, platform, 'package.json'), - JSON.stringify(pkg, null, 2) - ) - execSync( - `npm publish ${path.join( - nativePackagesDir, - platform - )} --access public ${ - gitref.includes('canary') ? ' --tag canary' : '' - }` - ) - } catch (err) { - // don't block publishing other versions on single platform error - console.error(`Failed to publish`, platform) + await Promise.all( + platforms.map(async (platform) => { + await publishSema.acquire() - if ( - err.message && - err.message.includes( - 'You cannot publish over the previously published versions' + try { + let binaryName = `next-swc.${platform}.node` + await copy( + path.join(cwd, 'packages/next-swc/native', binaryName), + path.join(nativePackagesDir, platform, binaryName) + ) + let pkg = JSON.parse( + await readFile( + path.join(nativePackagesDir, platform, 'package.json') + ) + ) + pkg.version = version + await writeFile( + path.join(nativePackagesDir, platform, 'package.json'), + JSON.stringify(pkg, null, 2) ) - ) { - console.error('Ignoring already published error', platform) - } else { - // throw err + await execa( + `npm publish ${path.join( + nativePackagesDir, + platform + )} --access public ${ + version.includes('canary') ? ' --tag canary' : '' + }`, + { stdio: 'inherit' } + ) + } catch (err) { + // don't block publishing other versions on single platform error + console.error(`Failed to publish`, platform, err) + + if ( + err.message && + err.message.includes( + 'You cannot publish over the previously published versions' + ) + ) { + console.error('Ignoring already published error', platform, err) + } else { + // throw err + } + } finally { + publishSema.release() } - } - // lerna publish in next step sill fail if git status is not clean - execSync( - `git update-index --skip-worktree ${path.join( - nativePackagesDir, - platform, - 'package.json' - )}` - ) - } + // lerna publish in next step sill fail if git status is not clean + await execa( + `git update-index --skip-worktree ${path.join( + nativePackagesDir, + platform, + 'package.json' + )}`, + { stdio: 'inherit' } + ) + }) + ) // Update name/version of wasm packages and publish let wasmDir = path.join(cwd, 'packages/next-swc/crates/wasm') - for (let wasmTarget of ['web', 'nodejs']) { - let wasmPkg = JSON.parse( - await readFile(path.join(wasmDir, `pkg-${wasmTarget}/package.json`)) - ) - wasmPkg.name = `@next/swc-wasm-${wasmTarget}` - wasmPkg.version = version - await writeFile( - path.join(wasmDir, `pkg-${wasmTarget}/package.json`), - JSON.stringify(wasmPkg, null, 2) - ) + await Promise.all( + ['web', 'nodejs'].map(async (wasmTarget) => { + await publishSema.acquire() - try { - execSync( - `npm publish ${path.join( - wasmDir, - `pkg-${wasmTarget}` - )} --access public ${ - gitref.includes('canary') ? ' --tag canary' : '' - }` + let wasmPkg = JSON.parse( + await readFile(path.join(wasmDir, `pkg-${wasmTarget}/package.json`)) ) - } catch (err) { - // don't block publishing other versions on single platform error - console.error(`Failed to publish`, wasmTarget) + wasmPkg.name = `@next/swc-wasm-${wasmTarget}` + wasmPkg.version = version - if ( - err.message && - err.message.includes( - 'You cannot publish over the previously published versions' + await writeFile( + path.join(wasmDir, `pkg-${wasmTarget}/package.json`), + JSON.stringify(wasmPkg, null, 2) + ) + + try { + await execa( + `npm publish ${path.join( + wasmDir, + `pkg-${wasmTarget}` + )} --access public ${ + version.includes('canary') ? ' --tag canary' : '' + }`, + { stdio: 'inherit' } ) - ) { - console.error('Ignoring already published error', wasmTarget) - } else { - // throw err + } catch (err) { + // don't block publishing other versions on single platform error + console.error(`Failed to publish`, wasmTarget, err) + + if ( + err.message && + err.message.includes( + 'You cannot publish over the previously published versions' + ) + ) { + console.error('Ignoring already published error', wasmTarget) + } else { + // throw err + } + } finally { + publishSema.release() } - } - } + }) + ) // Update optional dependencies versions let nextPkg = JSON.parse( @@ -122,7 +142,9 @@ const cwd = process.cwd() JSON.stringify(nextPkg, null, 2) ) // lerna publish in next step will fail if git status is not clean - execSync('git update-index --skip-worktree packages/next/package.json') + await execa('git update-index --skip-worktree packages/next/package.json', { + stdio: 'inherit', + }) } catch (err) { console.error(err) process.exit(1) diff --git a/scripts/publish-release.js b/scripts/publish-release.js index 22a02bbcfeb4a1..92c4cab4aaeafc 100755 --- a/scripts/publish-release.js +++ b/scripts/publish-release.js @@ -2,22 +2,20 @@ // @ts-check const path = require('path') -const { readdir } = require('fs/promises') +const execa = require('execa') +const { Sema } = require('async-sema') const { execSync } = require('child_process') -const { readJson } = require('fs-extra') +const { readJson, readdir } = require('fs-extra') const cwd = process.cwd() ;(async function () { let isCanary = true - if (!process.env.NPM_TOKEN) { - console.log('No NPM_TOKEN, exiting...') - return - } - try { - const tagOutput = execSync('git describe --exact-match').toString() + const tagOutput = execSync( + `node ${path.join(__dirname, 'check-is-release.js')}` + ).toString() console.log(tagOutput) if (tagOutput.trim().startsWith('v')) { @@ -34,15 +32,23 @@ const cwd = process.cwd() } console.log(`Publishing ${isCanary ? 'canary' : 'stable'}`) + if (!process.env.NPM_TOKEN) { + console.log('No NPM_TOKEN, exiting...') + return + } + const packagesDir = path.join(cwd, 'packages') const packageDirs = await readdir(packagesDir) + const publishSema = new Sema(2) const publish = async (pkg, retry = 0) => { try { - execSync( + await publishSema.acquire() + await execa( `npm publish ${path.join(packagesDir, pkg)} --access public${ isCanary ? ' --tag canary' : '' - }` + }`, + { stdio: 'inherit' } ) } catch (err) { console.error(`Failed to publish ${pkg}`, err) @@ -66,18 +72,22 @@ const cwd = process.cwd() await publish(pkg, retry + 1) } throw err + } finally { + publishSema.release() } } - for (const packageDir of packageDirs) { - const pkgJson = await readJson( - path.join(packagesDir, packageDir, 'package.json') - ) + await Promise.all( + packageDirs.map(async (packageDir) => { + const pkgJson = await readJson( + path.join(packagesDir, packageDir, 'package.json') + ) - if (pkgJson.private) { - console.log(`Skipping private package ${packageDir}`) - continue - } - await publish(packageDir) - } + if (pkgJson.private) { + console.log(`Skipping private package ${packageDir}`) + return + } + await publish(packageDir) + }) + ) })() diff --git a/scripts/release-stats.sh b/scripts/release-stats.sh index 975aaf96ebb564..a926b4eed23e33 100755 --- a/scripts/release-stats.sh +++ b/scripts/release-stats.sh @@ -1,11 +1,12 @@ #!/bin/bash -git describe --exact-match - -if [[ ! $? -eq 0 ]];then - echo "Nothing to publish, exiting.." - touch .github/actions/next-stats-action/SKIP_NEXT_STATS.txt - exit 0; +if [[ $(node ./scripts/check-is-release.js 2> /dev/null || :) = v* ]]; + then + echo "Publish occurred, running release stats..." + else + echo "Not publish commit, exiting..." + touch .github/actions/next-stats-action/SKIP_NEXT_STATS.txt + exit 0; fi if [[ -z "$NPM_TOKEN" ]];then @@ -13,6 +14,5 @@ if [[ -z "$NPM_TOKEN" ]];then exit 0; fi -echo "Publish occurred, running release stats..." echo "Waiting 30 seconds to allow publish to finalize" sleep 30