fix(ng-dev/release): ensure installed node modules do not break withi…

…n Bazel (#271)

This commit fixes an issue where old modules from other version branches
are preserved in the `node_modules` of a newly checked-out version
branch. Old/extraneous modules can cause the Bazel NodeJS rules to throw
with an error when preparing the `@npm//` workspace.

Yarn v1 has a bug where nested node modules are not cleaned properly.
Yarn v2+ supposedly fixes that, but for now we are still on v1.x and
in general it seems more safe to clean the node modules.

Here is the related Yarn bug: yarnpkg/yarn#8146.
Here is an example Bazel NodeJS error:

```
  ✓   Created pull request #23758 in angular/components.
  ✓   Release staging pull request has been created.
      Please ask team members to review: angular/components#23758.
 ⠋ Waiting for pull request #23758 to be merged.
  ✓   Pull request #23758 has been merged.
[1/5] Validating package.json...
[2/5] Resolving packages...
warning Resolution field "[email protected]" is incompatible with requested version "https-proxy-agent@^2.2.1"
warning Resolution field "[email protected]" is incompatible with requested version "https-proxy-agent@^2.2.1"
warning Resolution field "[email protected]" is incompatible with requested version "typescript@^3.2.2"
warning Resolution field "[email protected]" is incompatible with requested version "https-proxy-agent@^2.2.1"
warning Resolution field "[email protected]" is incompatible with requested version "https-proxy-agent@^4.0.0"
warning Resolution field "[email protected]" is incompatible with requested version "https-proxy-agent@^4.0.0"
[3/5] Fetching packages...
[4/5] Linking dependencies...
warning "@angular/bazel > [email protected]" has incorrect peer dependency "typescript@~3.8.2".
warning " > @bazel/[email protected]" has unmet peer dependency "karma-junit-reporter@>=2.0.0".
warning " > [email protected]" has incorrect peer dependency "typescript@~3.9.5".
[5/5] Building fresh packages...
$ node tools/postinstall/apply-patches.js && ngcc --properties module main --create-ivy-entry-points && node tools/postinstall/update-ngcc-main-fields.js
patching file node_modules/@angular/compiler/bundles/compiler.umd.js
Hunk #1 succeeded at 26295 (offset 5289 lines).
Hunk #2 succeeded at 26319 (offset 5289 lines).
Hunk #3 succeeded at 26330 (offset 5289 lines).
patching file node_modules/@bazel/typescript/internal/build_defs.bzl
Hunk #1 succeeded at 123 with fuzz 1 (offset 21 lines).
Hunk #2 succeeded at 141 with fuzz 1 (offset 21 lines).
Hunk #3 succeeded at 178 (offset 20 lines).
Hunk #4 succeeded at 201 (offset 20 lines).
Hunk #5 succeeded at 217 (offset 21 lines).
patching file node_modules/@bazel/typescript/internal/tsc_wrapped/tsc_wrapped.js
Hunk #1 succeeded at 396 (offset 15 lines).
Patching file /Users/andrewjs/git/components/node_modules/@angular/bazel/src/ng_package/packager.js with 2 edits..
Patching file /Users/andrewjs/git/components/node_modules/@angular/compiler-cli/src/metadata/bundle_index_host.js with 1 edits..
Patching file /Users/andrewjs/git/components/node_modules/@angular/compiler-cli/src/ngtsc/entry_point/src/logic.js with 1 edits..
Patching file /Users/andrewjs/git/components/node_modules/@angular/compiler-cli/src/transformers/compiler_host.js with 1 edits..
Patching file /Users/andrewjs/git/components/node_modules/@angular/bazel/src/ngc-wrapped/index.js with 1 edits..
Patching file /Users/andrewjs/git/components/node_modules/@angular/bazel/src/ng_module.bzl with 4 edits..
Compiling @angular/core : module as esm2015
Compiling @angular/animations : module as esm2015
Compiling @angular/compiler/testing : module as esm2015
Compiling @angular/animations : main as umd
Compiling @angular/compiler/testing : main as umd
Compiling @angular/core/testing : module as esm2015
Compiling @angular/animations/browser : module as esm2015
Compiling @angular/common : module as esm2015
Compiling @angular/core : main as umd
Compiling @angular/platform-browser : module as esm2015
Compiling @angular/common/http : module as esm2015
Compiling @angular/common/testing : module as esm2015
Compiling @angular/platform-browser-dynamic : module as esm2015
Compiling @angular/platform-browser/testing : module as esm2015
Compiling @angular/platform-browser/animations : module as esm2015
Compiling @angular/router : module as esm2015
Compiling @angular/platform-browser-dynamic/testing : module as esm2015
Compiling @angular/platform-server : module as esm2015
Compiling @angular/animations/browser/testing : module as esm2015
Compiling @angular/common/http/testing : module as esm2015
Compiling @angular/forms : module as esm2015
Compiling @angular/platform-server/init : module as esm2015
Compiling @angular/platform-server/testing : module as esm2015
Compiling @angular/router/testing : module as esm2015
Compiling @angular/animations/browser/testing : main as umd
Compiling @angular/animations/browser : main as umd
Compiling @angular/common/http : main as umd
Compiling @angular/common : main as umd
Compiling @angular/common/http/testing : main as umd
Compiling @angular/forms : main as umd
Compiling @angular/platform-browser : main as umd
Compiling @angular/platform-server/init : main as umd
Compiling @angular/core/testing : main as umd
Compiling @angular/platform-browser-dynamic : main as umd
Compiling @angular/platform-browser/testing : main as umd
Compiling @angular/platform-browser-dynamic/testing : main as umd
Compiling @angular/platform-browser/animations : main as umd
Compiling @angular/platform-server : main as umd
Compiling @angular/platform-server/testing : main as umd
Compiling @angular/common/testing : main as umd
Compiling @angular/router/testing : main as umd
Compiling @angular/router : main as umd
$ husky install
husky - Git hooks installed
  ✓   Installed project dependencies.
 ⠋ Building release output.
  Building release packages...
  Compiling with Ivy: false
Starting local Bazel server and connecting to it...
Loading: 0 packages loaded
Loading: 0 packages loaded
Loading: 0 packages loaded
Loading: 0 packages loaded
Loading: 0 packages loaded
Loading: 0 packages loaded
Loading: 0 packages loaded
Loading: 0 packages loaded
Loading: 0 packages loaded
[1/5] Validating package.json...
[2/5] Resolving packages...
warning Resolution field "[email protected]" is incompatible with requested version "https-proxy-agent@^2.2.1"
warning Resolution field "[email protected]" is incompatible with requested version "https-proxy-agent@^2.2.1"
warning Resolution field "[email protected]" is incompatible with requested version "typescript@^3.2.2"
warning Resolution field "[email protected]" is incompatible with requested version "https-proxy-agent@^2.2.1"
warning Resolution field "[email protected]" is incompatible with requested version "https-proxy-agent@^4.0.0"
warning Resolution field "[email protected]" is incompatible with requested version "https-proxy-agent@^4.0.0"
Loading: 0 packages loaded
success Already up-to-date.
$ node tools/postinstall/apply-patches.js && ngcc --properties module main --create-ivy-entry-points && node tools/postinstall/update-ngcc-main-fields.js
Patch: /Users/andrewjs/git/components/tools/postinstall/flat_module_factory_resolution.patch has been applied already. Skipping..
Patch: /Users/andrewjs/git/components/tools/postinstall/manifest_externs_hermeticity.patch has been applied already. Skipping..
File /Users/andrewjs/git/components/node_modules/@angular/bazel/src/ng_package/packager.js is already patched. Skipping..
File /Users/andrewjs/git/components/node_modules/@angular/compiler-cli/src/metadata/bundle_index_host.js is already patched. Skipping..
File /Users/andrewjs/git/components/node_modules/@angular/compiler-cli/src/ngtsc/entry_point/src/logic.js is already patched. Skipping..
File /Users/andrewjs/git/components/node_modules/@angular/compiler-cli/src/transformers/compiler_host.js is already patched. Skipping..
File /Users/andrewjs/git/components/node_modules/@angular/bazel/src/ngc-wrapped/index.js is already patched. Skipping..
File /Users/andrewjs/git/components/node_modules/@angular/bazel/src/ng_module.bzl is already patched. Skipping..
$ husky install
husky - Git hooks installed
Loading: 0 packages loaded
could not find dependency 'google-protobuf' of '@angular/dev-infra-private/node_modules/@bazel/worker'
INFO: Repository npm instantiated at:
  /Users/andrewjs/git/components/WORKSPACE:51:13: in <toplevel>
  /private/var/tmp/_bazel_andrewjs/784c5492abc453cc085a1c5145e774ee/external/build_bazel_rules_nodejs/index.bzl:83:18: in yarn_install
Repository rule yarn_install defined at:
  /private/var/tmp/_bazel_andrewjs/784c5492abc453cc085a1c5145e774ee/external/build_bazel_rules_nodejs/internal/npm_install/npm_install.bzl:795:31: in <toplevel>
Loading: 0 packages loaded
ERROR: An error occurred during the fetch of repository 'npm':
   Traceback (most recent call last):
	File "/private/var/tmp/_bazel_andrewjs/784c5492abc453cc085a1c5145e774ee/external/build_bazel_rules_nodejs/internal/npm_install/npm_install.bzl", line 793, column 24, in _yarn_install_impl
		_create_build_files(repository_ctx, "yarn_install", node, repository_ctx.attr.yarn_lock, repository_ctx.attr.generate_local_modules_build_files)
	File "/private/var/tmp/_bazel_andrewjs/784c5492abc453cc085a1c5145e774ee/external/build_bazel_rules_nodejs/internal/npm_install/npm_install.bzl", line 425, column 13, in _create_build_files
		fail("generate_build_file.ts failed: \nSTDOUT:\n%s\nSTDERR:\n%s" % (result.stdout, result.stderr))
Error in fail: generate_build_file.ts failed:
STDOUT:

STDERR:
could not find dependency 'google-protobuf' of '@angular/dev-infra-private/node_modules/@bazel/worker'
ERROR: Error fetching repository: Traceback (most recent call last):
	File "/private/var/tmp/_bazel_andrewjs/784c5492abc453cc085a1c5145e774ee/external/build_bazel_rules_nodejs/internal/npm_install/npm_install.bzl", line 793, column 24, in _yarn_install_impl
		_create_build_files(repository_ctx, "yarn_install", node, repository_ctx.attr.yarn_lock, repository_ctx.attr.generate_local_modules_build_files)
	File "/private/var/tmp/_bazel_andrewjs/784c5492abc453cc085a1c5145e774ee/external/build_bazel_rules_nodejs/internal/npm_install/npm_install.bzl", line 425, column 13, in _create_build_files
		fail("generate_build_file.ts failed: \nSTDOUT:\n%s\nSTDERR:\n%s" % (result.stdout, result.stderr))
Error in fail: generate_build_file.ts failed:
STDOUT:

STDERR:
could not find dependency 'google-protobuf' of '@angular/dev-infra-private/node_modules/@bazel/worker'
ERROR: no such package '@npm//@bazel/protractor': generate_build_file.ts failed:
STDOUT:

STDERR:
could not find dependency 'google-protobuf' of '@angular/dev-infra-private/node_modules/@bazel/worker'
Loading: 0 packages loaded
Loading: 0 packages loaded
(node:14775) UnhandledPromiseRejectionWarning: Error: Command failed: bazel query --output=label "attr('tags', '\[.*release-package.*\]', //src/...) intersect kind('.*_package', //src/...)"
    at checkExecSyncError (child_process.js:760:11)
    at Object.execSync (child_process.js:833:15)
    at exec (/Users/andrewjs/git/components/scripts/build-packages-dist.ts:139:18)
    at buildReleasePackages (/Users/andrewjs/git/components/scripts/build-packages-dist.ts:72:19)
    at performNpmReleaseBuild (/Users/andrewjs/git/components/scripts/build-packages-dist.ts:48:10)
    at /Users/andrewjs/git/components/.ng-dev/release.ts:96:12
    at Generator.next (<anonymous>)
    at fulfilled (/Users/andrewjs/git/components/.ng-dev/release.ts:5:58)
(Use `node --trace-warnings ...` to show where the warning was created)
(node:14775) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). To terminate the node process on unhandled promise rejection, use the CLI flag `--unhandled-rejections=strict` (see https://nodejs.org/api/cli.html#cli_unhandled_rejections_mode). (rejection id: 1)
(node:14775) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.
  ✘   Could not build release output. Please check output above.
```

PR Close #271

ng-dev/release/publish/actions.ts

@@ -377,6 +377,18 @@ export abstract class ReleaseAction {
     this.git.run(['checkout', '-q', 'FETCH_HEAD', '--detach']);
   }
 
+  /** Installs all Yarn dependencies in the current branch. */
+  protected async installDependenciesForCurrentBranch() {
+    const nodeModulesDir = join(this.projectDir, 'node_modules');
+    // Note: We delete all contents of the `node_modules` first. This is necessary
+    // because Yarn could preserve extraneous/outdated nested modules that will cause
+    // unexpected build failures with the NodeJS Bazel `@npm` workspace generation.
+    // This is a workaround for: https://github.com/yarnpkg/yarn/issues/8146. Even though
+    // we might be able to fix this with Yarn 2+, it is reasonable ensuring clean node modules.
+    await fs.rm(nodeModulesDir, {force: true, recursive: true, maxRetries: 3});
+    await invokeYarnInstallCommand(this.projectDir);
+  }
+
   /**
    * Creates a commit for the specified files with the given message.
    * @param message Message for the created commit
@@ -585,14 +597,14 @@ export abstract class ReleaseAction {
 
     // Checkout the publish branch and build the release packages.
     await this.checkoutUpstreamBranch(publishBranch);
+    // Install the project dependencies for the publish branch.
+    await this.installDependenciesForCurrentBranch();
 
-    // Install the project dependencies for the publish branch, and then build the release
-    // packages. Note that we do not directly call the build packages function from the release
+    // Note that we do not directly call the build packages function from the release
     // config. We only want to build and publish packages that have been configured in the given
     // publish branch. e.g. consider we publish patch version and a new package has been
     // created in the `next` branch. The new package would not be part of the patch branch,
     // so we cannot build and publish it.
-    await invokeYarnInstallCommand(this.projectDir);
     const builtPackages = await invokeReleaseBuildCommand();
 
     // Verify the packages built are the correct version.

ng-dev/release/publish/actions/cut-stable.ts

@@ -67,7 +67,8 @@ export class CutStableAction extends ReleaseAction {
       // NPM dist tag for new packages part of the released major, nor would it be acceptable
       // to skip the LTS tag for packages which are no longer part of the new major.
       await this.checkoutUpstreamBranch(previousPatch.branchName);
-      await invokeYarnInstallCommand(this.projectDir);
+      await this.installDependenciesForCurrentBranch();
+
       await invokeSetNpmDistCommand(ltsTagForPatch, previousPatch.version);
     }
 

ng-dev/release/publish/actions/tag-recent-major-as-latest.ts

@@ -34,7 +34,7 @@ export class TagRecentMajorAsLatest extends ReleaseAction {
   override async perform() {
     await this.updateGithubReleaseEntryToStable(this.active.latest.version);
     await this.checkoutUpstreamBranch(this.active.latest.branchName);
-    await invokeYarnInstallCommand(this.projectDir);
+    await this.installDependenciesForCurrentBranch();
     await invokeSetNpmDistCommand('latest', this.active.latest.version);
   }