feat(ng-dev/ci): create a common tool for gathering test results from…

… bazel (#239)

Create a common tool for gathering test results from bazel into a single directory for CircleCI to
ingest for test result tracking on CI.

PR Close #239

ng-dev/BUILD.bazel

@@ -19,6 +19,7 @@ ts_library(
     ],
     deps = [
         "//ng-dev/caretaker",
+        "//ng-dev/ci",
         "//ng-dev/commit-message",
         "//ng-dev/format",
         "//ng-dev/misc",

ng-dev/ci/BUILD.bazel

@@ -0,0 +1,13 @@
+load("//tools:defaults.bzl", "ts_library")
+
+ts_library(
+    name = "ci",
+    srcs = [
+        "cli.ts",
+    ],
+    visibility = ["//ng-dev:__subpackages__"],
+    deps = [
+        "//ng-dev/ci/gather-test-results",
+        "@npm//@types/yargs",
+    ],
+)

ng-dev/ci/cli.ts

@@ -0,0 +1,14 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {Argv} from 'yargs';
+import {GatherTestResultsModule} from './gather-test-results/cli';
+
+/** Build the parser for the ci commands. */
+export function buildCiParser(yargs: Argv) {
+  return yargs.help().strict().command(GatherTestResultsModule);
+}

ng-dev/ci/gather-test-results/BUILD.bazel

@@ -0,0 +1,16 @@
+load("//tools:defaults.bzl", "ts_library")
+
+ts_library(
+    name = "gather-test-results",
+    srcs = [
+        "cli.ts",
+        "index.ts",
+    ],
+    visibility = ["//ng-dev:__subpackages__"],
+    deps = [
+        "//bazel/protos:test_status",
+        "//ng-dev/utils",
+        "@npm//@types/node",
+        "@npm//@types/yargs",
+    ],
+)

ng-dev/ci/gather-test-results/cli.ts

@@ -0,0 +1,41 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {Arguments, Argv, CommandModule} from 'yargs';
+import {copyTestResultFiles} from '.';
+import {error, red} from '../../utils/console';
+/** Command line options. */
+export interface Options {
+  force: boolean;
+}
+
+/** Yargs command builder for the command. */
+function builder(argv: Argv): Argv<Options> {
+  return argv.option('force', {
+    type: 'boolean',
+    default: false,
+    description: 'Whether to force the command to run, ignoring the CI environment check',
+  });
+}
+
+/** Yargs command handler for the command. */
+async function handler({force}: Arguments<Options>) {
+  if (force === false && process.env['CI'] === undefined) {
+    error(red('Aborting, `gather-test-results` is only meant to be run on CI.'));
+    process.exit(1);
+  }
+  copyTestResultFiles();
+}
+
+/** CLI command module. */
+export const GatherTestResultsModule: CommandModule<{}, Options> = {
+  builder,
+  handler,
+  command: 'gather-test-results',
+  describe: 'Gather test result files into single directory for consumption by CircleCI',
+};

ng-dev/ci/gather-test-results/index.ts

@@ -0,0 +1,116 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {blaze} from '../../../bazel/protos/test_status_pb';
+import {spawnSync} from '../../utils/child-process';
+import {join, extname} from 'path';
+import {
+  mkdirSync,
+  rmSync,
+  readFileSync,
+  statSync,
+  readdirSync,
+  copyFileSync,
+  writeFileSync,
+} from 'fs';
+import {debug, info} from '../../utils/console';
+import {GitClient} from '../../utils/git/git-client';
+
+/** Bazel's TestResultData proto Message. */
+const TestResultData = blaze.TestResultData;
+
+type TestResultFiles = [xmlFile: string, cacheProtoFile: string];
+
+/**
+ * A JUnit test report to always include signaling to CircleCI that tests were requested.
+ *
+ * `testsuite` and `testcase` elements are required for CircleCI to properly parse the report.
+ */
+const baseTestReport = `
+ <?xml version="1.0" encoding="UTF-8" ?>
+ <testsuites disabled="0" errors="0" failures="0" tests="0" time="0">
+   <testsuite name="">
+     <testcase name=""/>
+   </testsuite>
+ </testsuites>
+ `.trim();
+
+function getTestLogsDirectoryPath() {
+  const {stdout, status} = spawnSync('yarn', ['-s', 'bazel', 'info', 'bazel-testlogs']);
+
+  if (status === 0) {
+    return stdout.trim();
+  }
+  throw Error(`Unable to determine the path to the directory containing Bazel's testlog.`);
+}
+
+/**
+ * Discover all test results, which @bazel/jasmine stores as `test.xml` files, in the directory and
+ * return back the list of absolute file paths.
+ */
+function findAllTestResultFiles(dirPath: string, files: TestResultFiles[]) {
+  for (const file of readdirSync(dirPath)) {
+    const filePath = join(dirPath, file);
+    if (statSync(filePath).isDirectory()) {
+      files = findAllTestResultFiles(filePath, files);
+    } else {
+      // Only the test result files, which are XML with the .xml extension, should be discovered.
+      if (extname(file) === '.xml') {
+        files.push([filePath, join(dirPath, 'test.cache_status')]);
+      }
+    }
+  }
+  return files;
+}
+
+export function copyTestResultFiles() {
+  /** Total number of files copied, also used as a index to number copied files. */
+  let copiedFileCount = 0;
+  /** The absolute path to the directory containing test logs from bazel tests. */
+  const testLogsDir = getTestLogsDirectoryPath();
+  /** List of test result files. */
+  const testResultPaths = findAllTestResultFiles(testLogsDir, []);
+  /** The full path to the root of the repository base. */
+  const projectBaseDir = GitClient.get().baseDir;
+  /**
+   * Absolute path to a directory to contain the JUnit test result files.
+   *
+   * Note: The directory created needs to contain a subdirectory which contains the test results in
+   * order for CircleCI to properly discover the test results.
+   */
+  const destDirPath = join(projectBaseDir, 'test-results/_');
+
+  // Ensure that an empty directory exists to contain the test results reports for upload.
+  rmSync(destDirPath, {recursive: true, force: true});
+  mkdirSync(destDirPath, {recursive: true});
+
+  // By always uploading at least one result file, CircleCI will understand that a tests actions were
+  // called for in the bazel test run, even if not tests were actually executed due to cache hits. By
+  // always making sure to upload at least one test result report, CircleCI always include the
+  // workflow in its aggregated data and provide better metrics about the number of executed tests per
+  // run.
+  writeFileSync(join(destDirPath, `results.xml`), baseTestReport);
+  debug('Added base test report to test-results directory.');
+
+  // Copy each of the test result files to the central test result directory which CircleCI discovers
+  // test results in.
+  testResultPaths.forEach(([xmlFilePath, cacheStatusFilePath]) => {
+    const shortFilePath = xmlFilePath.substr(testLogsDir.length + 1);
+    const testResultData = TestResultData.decode(readFileSync(cacheStatusFilePath));
+
+    if (testResultData.remotelyCached && testResultData.testPassed) {
+      debug(`Skipping copy of ${shortFilePath} as it was a passing remote cache hit`);
+    } else {
+      const destFilePath = join(destDirPath, `results-${copiedFileCount++}.xml`);
+      copyFileSync(xmlFilePath, destFilePath);
+      debug(`Copying ${shortFilePath}`);
+    }
+  });
+
+  info(`Copied ${copiedFileCount} test result file(s) for upload.`);
+}

ng-dev/cli.ts

@@ -18,6 +18,7 @@ import {buildReleaseParser} from './release/cli';
 import {tsCircularDependenciesBuilder} from './ts-circular-dependencies/index';
 import {captureLogOutputForCommand} from './utils/console';
 import {buildMiscParser} from './misc/cli';
+import {buildCiParser} from './ci/cli';
 
 yargs
   .scriptName('ng-dev')
@@ -33,6 +34,7 @@ yargs
   .command('caretaker <command>', '', buildCaretakerParser)
   .command('misc <command>', '', buildMiscParser)
   .command('ngbot <command>', false, buildNgbotParser)
+  .command('ci <command>', false, buildCiParser)
   .wrap(120)
   .strict()
   .parse();