From 7b66fb5dc44b92fb4ecbd6a7d52b67a3e15339eb Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner <paulgschwendtner@gmail.com> Date: Sat, 16 Oct 2021 14:46:11 +0200 Subject: [PATCH] fix(ng-dev/release): ensure installed node modules do not break within Bazel (#271) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit fixes an issue where old modules from other version branches are preserved in the `node_modules` of a newly checked-out version branch. Old/extraneous modules can cause the Bazel NodeJS rules to throw with an error when preparing the `@npm//` workspace. Yarn v1 has a bug where nested node modules are not cleaned properly. Yarn v2+ supposedly fixes that, but for now we are still on v1.x and in general it seems more safe to clean the node modules. Here is the related Yarn bug: https://github.com/yarnpkg/yarn/issues/8146. Here is an example Bazel NodeJS error: ``` ✓ Created pull request #23758 in angular/components. ✓ Release staging pull request has been created. Please ask team members to review: https://github.com/angular/components/pull/23758. ⠋ Waiting for pull request #23758 to be merged. ✓ Pull request #23758 has been merged. [1/5] Validating package.json... [2/5] Resolving packages... warning Resolution field "https-proxy-agent@5.0.0" is incompatible with requested version "https-proxy-agent@^2.2.1" warning Resolution field "https-proxy-agent@5.0.0" is incompatible with requested version "https-proxy-agent@^2.2.1" warning Resolution field "typescript@4.3.2" is incompatible with requested version "typescript@^3.2.2" warning Resolution field "https-proxy-agent@5.0.0" is incompatible with requested version "https-proxy-agent@^2.2.1" warning Resolution field "https-proxy-agent@5.0.0" is incompatible with requested version "https-proxy-agent@^4.0.0" warning Resolution field "https-proxy-agent@5.0.0" is incompatible with requested version "https-proxy-agent@^4.0.0" [3/5] Fetching packages... [4/5] Linking dependencies... warning "@angular/bazel > tsickle@0.38.1" has incorrect peer dependency "typescript@~3.8.2". warning " > @bazel/concatjs@4.0.0-beta.0" has unmet peer dependency "karma-junit-reporter@>=2.0.0". warning " > tsickle@0.39.1" has incorrect peer dependency "typescript@~3.9.5". [5/5] Building fresh packages... $ node tools/postinstall/apply-patches.js && ngcc --properties module main --create-ivy-entry-points && node tools/postinstall/update-ngcc-main-fields.js patching file node_modules/@angular/compiler/bundles/compiler.umd.js Hunk #1 succeeded at 26295 (offset 5289 lines). Hunk #2 succeeded at 26319 (offset 5289 lines). Hunk #3 succeeded at 26330 (offset 5289 lines). patching file node_modules/@bazel/typescript/internal/build_defs.bzl Hunk #1 succeeded at 123 with fuzz 1 (offset 21 lines). Hunk #2 succeeded at 141 with fuzz 1 (offset 21 lines). Hunk #3 succeeded at 178 (offset 20 lines). Hunk #4 succeeded at 201 (offset 20 lines). Hunk #5 succeeded at 217 (offset 21 lines). patching file node_modules/@bazel/typescript/internal/tsc_wrapped/tsc_wrapped.js Hunk #1 succeeded at 396 (offset 15 lines). Patching file /Users/andrewjs/git/components/node_modules/@angular/bazel/src/ng_package/packager.js with 2 edits.. Patching file /Users/andrewjs/git/components/node_modules/@angular/compiler-cli/src/metadata/bundle_index_host.js with 1 edits.. Patching file /Users/andrewjs/git/components/node_modules/@angular/compiler-cli/src/ngtsc/entry_point/src/logic.js with 1 edits.. Patching file /Users/andrewjs/git/components/node_modules/@angular/compiler-cli/src/transformers/compiler_host.js with 1 edits.. Patching file /Users/andrewjs/git/components/node_modules/@angular/bazel/src/ngc-wrapped/index.js with 1 edits.. Patching file /Users/andrewjs/git/components/node_modules/@angular/bazel/src/ng_module.bzl with 4 edits.. Compiling @angular/core : module as esm2015 Compiling @angular/animations : module as esm2015 Compiling @angular/compiler/testing : module as esm2015 Compiling @angular/animations : main as umd Compiling @angular/compiler/testing : main as umd Compiling @angular/core/testing : module as esm2015 Compiling @angular/animations/browser : module as esm2015 Compiling @angular/common : module as esm2015 Compiling @angular/core : main as umd Compiling @angular/platform-browser : module as esm2015 Compiling @angular/common/http : module as esm2015 Compiling @angular/common/testing : module as esm2015 Compiling @angular/platform-browser-dynamic : module as esm2015 Compiling @angular/platform-browser/testing : module as esm2015 Compiling @angular/platform-browser/animations : module as esm2015 Compiling @angular/router : module as esm2015 Compiling @angular/platform-browser-dynamic/testing : module as esm2015 Compiling @angular/platform-server : module as esm2015 Compiling @angular/animations/browser/testing : module as esm2015 Compiling @angular/common/http/testing : module as esm2015 Compiling @angular/forms : module as esm2015 Compiling @angular/platform-server/init : module as esm2015 Compiling @angular/platform-server/testing : module as esm2015 Compiling @angular/router/testing : module as esm2015 Compiling @angular/animations/browser/testing : main as umd Compiling @angular/animations/browser : main as umd Compiling @angular/common/http : main as umd Compiling @angular/common : main as umd Compiling @angular/common/http/testing : main as umd Compiling @angular/forms : main as umd Compiling @angular/platform-browser : main as umd Compiling @angular/platform-server/init : main as umd Compiling @angular/core/testing : main as umd Compiling @angular/platform-browser-dynamic : main as umd Compiling @angular/platform-browser/testing : main as umd Compiling @angular/platform-browser-dynamic/testing : main as umd Compiling @angular/platform-browser/animations : main as umd Compiling @angular/platform-server : main as umd Compiling @angular/platform-server/testing : main as umd Compiling @angular/common/testing : main as umd Compiling @angular/router/testing : main as umd Compiling @angular/router : main as umd $ husky install husky - Git hooks installed ✓ Installed project dependencies. ⠋ Building release output. Building release packages... Compiling with Ivy: false Starting local Bazel server and connecting to it... Loading: 0 packages loaded Loading: 0 packages loaded Loading: 0 packages loaded Loading: 0 packages loaded Loading: 0 packages loaded Loading: 0 packages loaded Loading: 0 packages loaded Loading: 0 packages loaded Loading: 0 packages loaded [1/5] Validating package.json... [2/5] Resolving packages... warning Resolution field "https-proxy-agent@5.0.0" is incompatible with requested version "https-proxy-agent@^2.2.1" warning Resolution field "https-proxy-agent@5.0.0" is incompatible with requested version "https-proxy-agent@^2.2.1" warning Resolution field "typescript@4.3.2" is incompatible with requested version "typescript@^3.2.2" warning Resolution field "https-proxy-agent@5.0.0" is incompatible with requested version "https-proxy-agent@^2.2.1" warning Resolution field "https-proxy-agent@5.0.0" is incompatible with requested version "https-proxy-agent@^4.0.0" warning Resolution field "https-proxy-agent@5.0.0" is incompatible with requested version "https-proxy-agent@^4.0.0" Loading: 0 packages loaded success Already up-to-date. $ node tools/postinstall/apply-patches.js && ngcc --properties module main --create-ivy-entry-points && node tools/postinstall/update-ngcc-main-fields.js Patch: /Users/andrewjs/git/components/tools/postinstall/flat_module_factory_resolution.patch has been applied already. Skipping.. Patch: /Users/andrewjs/git/components/tools/postinstall/manifest_externs_hermeticity.patch has been applied already. Skipping.. File /Users/andrewjs/git/components/node_modules/@angular/bazel/src/ng_package/packager.js is already patched. Skipping.. File /Users/andrewjs/git/components/node_modules/@angular/compiler-cli/src/metadata/bundle_index_host.js is already patched. Skipping.. File /Users/andrewjs/git/components/node_modules/@angular/compiler-cli/src/ngtsc/entry_point/src/logic.js is already patched. Skipping.. File /Users/andrewjs/git/components/node_modules/@angular/compiler-cli/src/transformers/compiler_host.js is already patched. Skipping.. File /Users/andrewjs/git/components/node_modules/@angular/bazel/src/ngc-wrapped/index.js is already patched. Skipping.. File /Users/andrewjs/git/components/node_modules/@angular/bazel/src/ng_module.bzl is already patched. Skipping.. $ husky install husky - Git hooks installed Loading: 0 packages loaded could not find dependency 'google-protobuf' of '@angular/dev-infra-private/node_modules/@bazel/worker' INFO: Repository npm instantiated at: /Users/andrewjs/git/components/WORKSPACE:51:13: in <toplevel> /private/var/tmp/_bazel_andrewjs/784c5492abc453cc085a1c5145e774ee/external/build_bazel_rules_nodejs/index.bzl:83:18: in yarn_install Repository rule yarn_install defined at: /private/var/tmp/_bazel_andrewjs/784c5492abc453cc085a1c5145e774ee/external/build_bazel_rules_nodejs/internal/npm_install/npm_install.bzl:795:31: in <toplevel> Loading: 0 packages loaded ERROR: An error occurred during the fetch of repository 'npm': Traceback (most recent call last): File "/private/var/tmp/_bazel_andrewjs/784c5492abc453cc085a1c5145e774ee/external/build_bazel_rules_nodejs/internal/npm_install/npm_install.bzl", line 793, column 24, in _yarn_install_impl _create_build_files(repository_ctx, "yarn_install", node, repository_ctx.attr.yarn_lock, repository_ctx.attr.generate_local_modules_build_files) File "/private/var/tmp/_bazel_andrewjs/784c5492abc453cc085a1c5145e774ee/external/build_bazel_rules_nodejs/internal/npm_install/npm_install.bzl", line 425, column 13, in _create_build_files fail("generate_build_file.ts failed: \nSTDOUT:\n%s\nSTDERR:\n%s" % (result.stdout, result.stderr)) Error in fail: generate_build_file.ts failed: STDOUT: STDERR: could not find dependency 'google-protobuf' of '@angular/dev-infra-private/node_modules/@bazel/worker' ERROR: Error fetching repository: Traceback (most recent call last): File "/private/var/tmp/_bazel_andrewjs/784c5492abc453cc085a1c5145e774ee/external/build_bazel_rules_nodejs/internal/npm_install/npm_install.bzl", line 793, column 24, in _yarn_install_impl _create_build_files(repository_ctx, "yarn_install", node, repository_ctx.attr.yarn_lock, repository_ctx.attr.generate_local_modules_build_files) File "/private/var/tmp/_bazel_andrewjs/784c5492abc453cc085a1c5145e774ee/external/build_bazel_rules_nodejs/internal/npm_install/npm_install.bzl", line 425, column 13, in _create_build_files fail("generate_build_file.ts failed: \nSTDOUT:\n%s\nSTDERR:\n%s" % (result.stdout, result.stderr)) Error in fail: generate_build_file.ts failed: STDOUT: STDERR: could not find dependency 'google-protobuf' of '@angular/dev-infra-private/node_modules/@bazel/worker' ERROR: no such package '@npm//@bazel/protractor': generate_build_file.ts failed: STDOUT: STDERR: could not find dependency 'google-protobuf' of '@angular/dev-infra-private/node_modules/@bazel/worker' Loading: 0 packages loaded Loading: 0 packages loaded (node:14775) UnhandledPromiseRejectionWarning: Error: Command failed: bazel query --output=label "attr('tags', '\[.*release-package.*\]', //src/...) intersect kind('.*_package', //src/...)" at checkExecSyncError (child_process.js:760:11) at Object.execSync (child_process.js:833:15) at exec (/Users/andrewjs/git/components/scripts/build-packages-dist.ts:139:18) at buildReleasePackages (/Users/andrewjs/git/components/scripts/build-packages-dist.ts:72:19) at performNpmReleaseBuild (/Users/andrewjs/git/components/scripts/build-packages-dist.ts:48:10) at /Users/andrewjs/git/components/.ng-dev/release.ts:96:12 at Generator.next (<anonymous>) at fulfilled (/Users/andrewjs/git/components/.ng-dev/release.ts:5:58) (Use `node --trace-warnings ...` to show where the warning was created) (node:14775) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). To terminate the node process on unhandled promise rejection, use the CLI flag `--unhandled-rejections=strict` (see https://nodejs.org/api/cli.html#cli_unhandled_rejections_mode). (rejection id: 1) (node:14775) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code. ✘ Could not build release output. Please check output above. ``` PR Close #271 --- ng-dev/release/publish/actions.ts | 18 +++++++++++++++--- ng-dev/release/publish/actions/cut-stable.ts | 3 ++- .../actions/tag-recent-major-as-latest.ts | 2 +- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/ng-dev/release/publish/actions.ts b/ng-dev/release/publish/actions.ts index 564d1d3d2..1c040014a 100644 --- a/ng-dev/release/publish/actions.ts +++ b/ng-dev/release/publish/actions.ts @@ -377,6 +377,18 @@ export abstract class ReleaseAction { this.git.run(['checkout', '-q', 'FETCH_HEAD', '--detach']); } + /** Installs all Yarn dependencies in the current branch. */ + protected async installDependenciesForCurrentBranch() { + const nodeModulesDir = join(this.projectDir, 'node_modules'); + // Note: We delete all contents of the `node_modules` first. This is necessary + // because Yarn could preserve extraneous/outdated nested modules that will cause + // unexpected build failures with the NodeJS Bazel `@npm` workspace generation. + // This is a workaround for: https://github.com/yarnpkg/yarn/issues/8146. Even though + // we might be able to fix this with Yarn 2+, it is reasonable ensuring clean node modules. + await fs.rm(nodeModulesDir, {force: true, recursive: true, maxRetries: 3}); + await invokeYarnInstallCommand(this.projectDir); + } + /** * Creates a commit for the specified files with the given message. * @param message Message for the created commit @@ -585,14 +597,14 @@ export abstract class ReleaseAction { // Checkout the publish branch and build the release packages. await this.checkoutUpstreamBranch(publishBranch); + // Install the project dependencies for the publish branch. + await this.installDependenciesForCurrentBranch(); - // Install the project dependencies for the publish branch, and then build the release - // packages. Note that we do not directly call the build packages function from the release + // Note that we do not directly call the build packages function from the release // config. We only want to build and publish packages that have been configured in the given // publish branch. e.g. consider we publish patch version and a new package has been // created in the `next` branch. The new package would not be part of the patch branch, // so we cannot build and publish it. - await invokeYarnInstallCommand(this.projectDir); const builtPackages = await invokeReleaseBuildCommand(); // Verify the packages built are the correct version. diff --git a/ng-dev/release/publish/actions/cut-stable.ts b/ng-dev/release/publish/actions/cut-stable.ts index 0beed8245..befd70bf5 100644 --- a/ng-dev/release/publish/actions/cut-stable.ts +++ b/ng-dev/release/publish/actions/cut-stable.ts @@ -67,7 +67,8 @@ export class CutStableAction extends ReleaseAction { // NPM dist tag for new packages part of the released major, nor would it be acceptable // to skip the LTS tag for packages which are no longer part of the new major. await this.checkoutUpstreamBranch(previousPatch.branchName); - await invokeYarnInstallCommand(this.projectDir); + await this.installDependenciesForCurrentBranch(); + await invokeSetNpmDistCommand(ltsTagForPatch, previousPatch.version); } diff --git a/ng-dev/release/publish/actions/tag-recent-major-as-latest.ts b/ng-dev/release/publish/actions/tag-recent-major-as-latest.ts index ddefb90d4..3e45ef39d 100644 --- a/ng-dev/release/publish/actions/tag-recent-major-as-latest.ts +++ b/ng-dev/release/publish/actions/tag-recent-major-as-latest.ts @@ -34,7 +34,7 @@ export class TagRecentMajorAsLatest extends ReleaseAction { override async perform() { await this.updateGithubReleaseEntryToStable(this.active.latest.version); await this.checkoutUpstreamBranch(this.active.latest.branchName); - await invokeYarnInstallCommand(this.projectDir); + await this.installDependenciesForCurrentBranch(); await invokeSetNpmDistCommand('latest', this.active.latest.version); }