CI Reporter for saved objects field count (#70580)

* CI Reporter for saved objects field count

* Metrics needs to be an array

* Fix type failures

* Link to field count issue

* Revert "Link to field count issue"

This reverts commit 8c0126b.

* Break down field count per type

* Don't log total metric as metrics report already calculates this

* Add saved objects field count ci metrics test to codeowners

* Address review comments

* Add field count CI metrics for disabled plugins

Co-authored-by: Elastic Machine <[email protected]>

.github/CODEOWNERS

@@ -144,6 +144,7 @@
 /x-pack/plugins/licensing/  @elastic/kibana-platform
 /x-pack/plugins/global_search/  @elastic/kibana-platform
 /x-pack/plugins/cloud/ @elastic/kibana-platform
+/x-pack/test/saved_objects_field_count/ @elastic/kibana-platform
 /packages/kbn-config-schema/ @elastic/kibana-platform
 /src/legacy/server/config/ @elastic/kibana-platform
 /src/legacy/server/http/ @elastic/kibana-platform

Jenkinsfile

@@ -41,6 +41,7 @@ kibanaPipeline(timeoutMinutes: 155, checkPrChanges: true, setCommitStatus: true)
             'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
             'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
             'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'),
+            'xpack-savedObjectsFieldMetrics': kibanaPipeline.functionalTestProcess('xpack-savedObjectsFieldMetrics', './test/scripts/jenkins_xpack_saved_objects_field_metrics.sh'),
             // 'xpack-pageLoadMetrics': kibanaPipeline.functionalTestProcess('xpack-pageLoadMetrics', './test/scripts/jenkins_xpack_page_load_metrics.sh'),
             'xpack-securitySolutionCypress': { processNumber ->
               whenChanged(['x-pack/plugins/security_solution/', 'x-pack/test/security_solution_cypress/']) {

test/scripts/jenkins_xpack_saved_objects_field_metrics.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+source test/scripts/jenkins_test_setup_xpack.sh
+
+checks-reporter-with-killswitch "Capture Kibana Saved Objects field count metrics" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$installDir" \
+    --config test/saved_objects_field_count/config.ts;

x-pack/test/saved_objects_field_count/config.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
+import { testRunner } from './runner';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const kibanaCommonTestsConfig = await readConfigFile(
+    require.resolve('../../../test/common/config.js')
+  );
+
+  return {
+    ...kibanaCommonTestsConfig.getAll(),
+
+    testRunner,
+
+    esTestCluster: {
+      license: 'trial',
+      from: 'snapshot',
+      serverArgs: ['path.repo=/tmp/'],
+    },
+
+    kbnTestServer: {
+      ...kibanaCommonTestsConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...kibanaCommonTestsConfig.get('kbnTestServer.serverArgs'),
+        // Enable plugins that are disabled by default to include their metrics
+        // TODO: Find a way to automatically enable all discovered plugins
+        '--xpack.ingestManager.enabled=true',
+        '--xpack.lists.enabled=true',
+        '--xpack.securitySolution.enabled=true',
+      ],
+    },
+  };
+}

x-pack/test/saved_objects_field_count/runner.ts

@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { CiStatsReporter } from '@kbn/dev-utils';
+import { FtrProviderContext } from './../functional/ftr_provider_context';
+
+const IGNORED_FIELDS = [
+  // The following fields are returned by the _field_caps API but aren't counted
+  // towards the index field limit.
+  '_seq_no',
+  '_id',
+  '_version',
+  '_field_names',
+  '_ignored',
+  '_feature',
+  '_index',
+  '_routing',
+  '_source',
+  '_type',
+  '_nested_path',
+  '_timestamp',
+  // migrationVersion is dynamic so will be anywhere between 1..type count
+  // depending on which objects are present in the index when querying the
+  // field caps API. See https://github.com/elastic/kibana/issues/70815
+  'migrationVersion',
+];
+
+export async function testRunner({ getService }: FtrProviderContext) {
+  const log = getService('log');
+  const es = getService('es');
+
+  const reporter = CiStatsReporter.fromEnv(log);
+
+  log.debug('Saved Objects field count metrics starting');
+
+  const {
+    body: { fields },
+  } = await es.fieldCaps({
+    index: '.kibana',
+    fields: '*',
+  });
+
+  const fieldCountPerTypeMap: Map<string, number> = Object.keys(fields)
+    .map((f) => f.split('.')[0])
+    .filter((f) => !IGNORED_FIELDS.includes(f))
+    .reduce((accumulator, f) => {
+      accumulator.set(f, accumulator.get(f) + 1 || 1);
+      return accumulator;
+    }, new Map());
+
+  const metrics = Array.from(fieldCountPerTypeMap.entries())
+    .sort((a, b) => a[0].localeCompare(b[0]))
+    .map(([fieldType, count]) => ({
+      group: 'Saved Objects .kibana field count',
+      id: fieldType,
+      value: count,
+    }));
+
+  log.debug(
+    'Saved Objects field count metrics:\n',
+    metrics.map(({ id, value }) => `${id}:${value}`).join('\n')
+  );
+  await reporter.metrics(metrics);
+  log.debug('Saved Objects field count metrics done');
+}