[FTR] Add test suite metrics tracking/output

packages/kbn-test/src/functional_test_runner/functional_test_runner.ts

@@ -30,6 +30,7 @@ import {
   setupMocha,
   runTests,
   Config,
+  SuiteTracker,
 } from './lib';
 
 export class FunctionalTestRunner {
@@ -52,6 +53,8 @@ export class FunctionalTestRunner {
 
   async run() {
     return await this._run(async (config, coreProviders) => {
+      SuiteTracker.startTracking(this.lifecycle, this.configFile);
+
       const providers = new ProviderCollection(this.log, [
         ...coreProviders,
         ...readProviderSpec('Service', config.get('services')),

packages/kbn-test/src/functional_test_runner/lib/index.ts

@@ -23,3 +23,4 @@ export { readConfigFile, Config } from './config';
 export { readProviderSpec, ProviderCollection, Provider } from './providers';
 export { runTests, setupMocha } from './mocha';
 export { FailureMetadata } from './failure_metadata';
+export { SuiteTracker } from './suite_tracker';

packages/kbn-test/src/functional_test_runner/lib/suite_tracker.test.ts

@@ -0,0 +1,197 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import fs from 'fs';
+import { join, resolve } from 'path';
+
+jest.mock('fs');
+jest.mock('@kbn/dev-utils', () => {
+  return { REPO_ROOT: '/dev/null/root' };
+});
+
+import { REPO_ROOT } from '@kbn/dev-utils';
+import { Lifecycle } from './lifecycle';
+import { SuiteTracker } from './suite_tracker';
+
+const DEFAULT_TEST_METADATA_PATH = join(REPO_ROOT, 'target', 'test_metadata.json');
+const MOCK_CONFIG_PATH = join('test', 'config.js');
+const MOCK_TEST_PATH = join('test', 'apps', 'test.js');
+const ENVS_TO_RESET = ['TEST_METADATA_PATH'];
+
+describe('SuiteTracker', () => {
+  const originalEnvs: Record<string, string> = {};
+
+  beforeEach(() => {
+    for (const env of ENVS_TO_RESET) {
+      if (env in process.env) {
+        originalEnvs[env] = process.env[env] || '';
+        delete process.env[env];
+      }
+    }
+  });
+
+  afterEach(() => {
+    for (const env of ENVS_TO_RESET) {
+      delete process.env[env];
+    }
+
+    for (const env of Object.keys(originalEnvs)) {
+      process.env[env] = originalEnvs[env];
+    }
+
+    jest.resetAllMocks();
+  });
+
+  let MOCKS: Record<string, object>;
+
+  const createMock = (overrides = {}) => {
+    return {
+      file: resolve(REPO_ROOT, MOCK_TEST_PATH),
+      title: 'A Test',
+      suiteTag: MOCK_TEST_PATH,
+      ...overrides,
+    };
+  };
+
+  const runLifecycleWithMocks = async (mocks: object[], fn: (objs: any) => any = () => {}) => {
+    const lifecycle = new Lifecycle();
+    const suiteTracker = SuiteTracker.startTracking(
+      lifecycle,
+      resolve(REPO_ROOT, MOCK_CONFIG_PATH)
+    );
+
+    const ret = { lifecycle, suiteTracker };
+
+    for (const mock of mocks) {
+      await lifecycle.beforeTestSuite.trigger(mock);
+    }
+
+    if (fn) {
+      fn(ret);
+    }
+
+    for (const mock of mocks.reverse()) {
+      await lifecycle.afterTestSuite.trigger(mock);
+    }
+
+    return ret;
+  };
+
+  beforeEach(() => {
+    MOCKS = {
+      WITH_TESTS: createMock({ tests: [{}] }), // i.e. a describe with tests in it
+      WITHOUT_TESTS: createMock(), // i.e. a describe with only other describes in it
+    };
+  });
+
+  it('collects metadata for a single suite with multiple describe()s', async () => {
+    const { suiteTracker } = await runLifecycleWithMocks([MOCKS.WITHOUT_TESTS, MOCKS.WITH_TESTS]);
+
+    const suites = suiteTracker.getAllFinishedSuites();
+    expect(suites.length).toBe(1);
+    const suite = suites[0];
+
+    expect(suite).toMatchObject({
+      config: MOCK_CONFIG_PATH,
+      file: MOCK_TEST_PATH,
+      tag: MOCK_TEST_PATH,
+      hasTests: true,
+      success: true,
+    });
+  });
+
+  it('writes metadata to a file when cleanup is triggered', async () => {
+    const { lifecycle, suiteTracker } = await runLifecycleWithMocks([MOCKS.WITH_TESTS]);
+    await lifecycle.cleanup.trigger();
+
+    const suites = suiteTracker.getAllFinishedSuites();
+
+    const call = (fs.writeFileSync as jest.Mock).mock.calls[0];
+    expect(call[0]).toEqual(DEFAULT_TEST_METADATA_PATH);
+    expect(call[1]).toEqual(JSON.stringify(suites, null, 2));
+  });
+
+  it('respects TEST_METADATA_PATH env var for metadata target override', async () => {
+    process.env.TEST_METADATA_PATH = resolve(REPO_ROOT, '../fake-test-path');
+    const { lifecycle } = await runLifecycleWithMocks([MOCKS.WITH_TESTS]);
+    await lifecycle.cleanup.trigger();
+
+    expect((fs.writeFileSync as jest.Mock).mock.calls[0][0]).toEqual(
+      process.env.TEST_METADATA_PATH
+    );
+  });
+
+  it('identifies suites with tests as leaf suites', async () => {
+    const root = createMock({ title: 'root', file: join(REPO_ROOT, 'root.js') });
+    const parent = createMock({ parent: root });
+    const withTests = createMock({ parent, tests: [{}] });
+
+    const { suiteTracker } = await runLifecycleWithMocks([root, parent, withTests]);
+    const suites = suiteTracker.getAllFinishedSuites();
+
+    const finishedRoot = suites.find(s => s.title === 'root');
+    const finishedWithTests = suites.find(s => s.title !== 'root');
+
+    expect(finishedRoot).toBeTruthy();
+    expect(finishedRoot?.hasTests).toBeFalsy();
+    expect(finishedWithTests?.hasTests).toBe(true);
+  });
+
+  describe('with a failing suite', () => {
+    let root: any;
+    let parent: any;
+    let failed: any;
+
+    beforeEach(() => {
+      root = createMock({ file: join(REPO_ROOT, 'root.js') });
+      parent = createMock({ parent: root });
+      failed = createMock({ parent, tests: [{}] });
+    });
+
+    it('marks parent suites as not successful when a test fails', async () => {
+      const { suiteTracker } = await runLifecycleWithMocks(
+        [root, parent, failed],
+        async ({ lifecycle }) => {
+          await lifecycle.testFailure.trigger(Error('test'), { parent: failed });
+        }
+      );
+
+      const suites = suiteTracker.getAllFinishedSuites();
+      expect(suites.length).toBe(2);
+      for (const suite of suites) {
+        expect(suite.success).toBeFalsy();
+      }
+    });
+
+    it('marks parent suites as not successful when a test hook fails', async () => {
+      const { suiteTracker } = await runLifecycleWithMocks(
+        [root, parent, failed],
+        async ({ lifecycle }) => {
+          await lifecycle.testHookFailure.trigger(Error('test'), { parent: failed });
+        }
+      );
+
+      const suites = suiteTracker.getAllFinishedSuites();
+      expect(suites.length).toBe(2);
+      for (const suite of suites) {
+        expect(suite.success).toBeFalsy();
+      }
+    });
+  });
+});

packages/kbn-test/src/functional_test_runner/lib/suite_tracker.ts

@@ -0,0 +1,147 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import fs from 'fs';
+import { dirname, relative, resolve } from 'path';
+
+import { REPO_ROOT } from '@kbn/dev-utils';
+
+import { Lifecycle } from './lifecycle';
+
+export interface SuiteInProgress {
+  startTime?: Date;
+  endTime?: Date;
+  success?: boolean;
+}
+
+export interface SuiteWithMetadata {
+  config: string;
+  file: string;
+  tag: string;
+  title: string;
+  startTime: Date;
+  endTime: Date;
+  duration: number;
+  success: boolean;
+  hasTests: boolean;
+}
+
+const getTestMetadataPath = () => {
+  return process.env.TEST_METADATA_PATH || resolve(REPO_ROOT, 'target', 'test_metadata.json');
+};
+
+export class SuiteTracker {
+  finishedSuitesByConfig: Record<string, Record<string, SuiteWithMetadata>> = {};
+  inProgressSuites: Map<object, SuiteInProgress> = new Map<object, SuiteInProgress>();
+
+  static startTracking(lifecycle: Lifecycle, configPath: string): SuiteTracker {
+    const suiteTracker = new SuiteTracker(lifecycle, configPath);
+    return suiteTracker;
+  }
+
+  getTracked(suite: object): SuiteInProgress {
+    if (!this.inProgressSuites.has(suite)) {
+      this.inProgressSuites.set(suite, { success: undefined } as SuiteInProgress);
+    }
+    return this.inProgressSuites.get(suite)!;
+  }
+
+  constructor(lifecycle: Lifecycle, configPathAbsolute: string) {
+    if (fs.existsSync(getTestMetadataPath())) {
+      fs.unlinkSync(getTestMetadataPath());
+    } else {
+      fs.mkdirSync(dirname(getTestMetadataPath()), { recursive: true });
+    }
+
+    const config = relative(REPO_ROOT, configPathAbsolute);
+
+    lifecycle.beforeTestSuite.add(suite => {
+      const tracked = this.getTracked(suite);
+      tracked.startTime = new Date();
+    });
+
+    // If a test fails, we want to make sure all of the ancestors, all the way up to the root, get marked as failed
+    // This information is not available on the mocha objects without traversing all descendants of a given node
+    const handleFailure = (_: any, test: any) => {
+      let parent = test.parent;
+
+      // Infinite loop protection, just in case
+      for (let i = 0; i < 500 && parent; i++) {
+        if (this.inProgressSuites.has(parent)) {
+          this.getTracked(parent).success = false;
+        }
+        parent = parent.parent;
+      }
+    };
+
+    lifecycle.testFailure.add(handleFailure);
+    lifecycle.testHookFailure.add(handleFailure);
+
+    lifecycle.afterTestSuite.add(suite => {
+      const tracked = this.getTracked(suite);
+      tracked.endTime = new Date();
+
+      // The suite ended without any children failing, so we can mark it as successful
+      if (typeof tracked.success === 'undefined') {
+        tracked.success = true;
+      }
+
+      let duration = tracked.endTime.getTime() - (tracked.startTime || new Date()).getTime();
+      duration = Math.floor(duration / 1000);
+
+      const file = relative(REPO_ROOT, suite.file);
+
+      this.finishedSuitesByConfig[config] = this.finishedSuitesByConfig[config] || {};
+
+      // This will get called multiple times for a test file that has multiple describes in it or similar
+      // This is okay, because the last one that fires is always the root of the file, which is is the one we ultimately want
+      this.finishedSuitesByConfig[config][file] = {
+        ...tracked,
+        duration,
+        config,
+        file,
+        tag: suite.suiteTag,
+        title: suite.title,
+        hasTests: !!(
+          (suite.tests && suite.tests.length) ||
+          // The below statement is so that `hasTests` will bubble up nested describes in the same file
+          (this.finishedSuitesByConfig[config][file] &&
+            this.finishedSuitesByConfig[config][file].hasTests)
+        ),
+      } as SuiteWithMetadata;
+    });
+
+    lifecycle.cleanup.add(() => {
+      const suites = this.getAllFinishedSuites();
+
+      fs.writeFileSync(getTestMetadataPath(), JSON.stringify(suites, null, 2));
+    });
+  }
+
+  getAllFinishedSuites() {
+    const flattened: SuiteWithMetadata[] = [];
+    for (const byFile of Object.values(this.finishedSuitesByConfig)) {
+      for (const suite of Object.values(byFile)) {
+        flattened.push(suite);
+      }
+    }
+
+    flattened.sort((a, b) => b.duration - a.duration);
+    return flattened;
+  }
+}