[7.x] Index pattern scripted field / runtime field usage collection (#95366)

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.md

@@ -14,6 +14,6 @@ export declare class IndexPatternsServiceProvider implements Plugin<void, IndexP
 
 |  Method | Modifiers | Description |
 |  --- | --- | --- |
-|  [setup(core, { logger, expressions })](./kibana-plugin-plugins-data-server.indexpatternsserviceprovider.setup.md) |  |  |
+|  [setup(core, { logger, expressions, usageCollection })](./kibana-plugin-plugins-data-server.indexpatternsserviceprovider.setup.md) |  |  |
 |  [start(core, { fieldFormats, logger })](./kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md) |  |  |
 

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.setup.md

@@ -7,15 +7,15 @@
 <b>Signature:</b>
 
 ```typescript
-setup(core: CoreSetup<DataPluginStartDependencies, DataPluginStart>, { logger, expressions }: IndexPatternsServiceSetupDeps): void;
+setup(core: CoreSetup<IndexPatternsServiceStartDeps, DataPluginStart>, { logger, expressions, usageCollection }: IndexPatternsServiceSetupDeps): void;
 ```
 
 ## Parameters
 
 |  Parameter | Type | Description |
 |  --- | --- | --- |
-|  core | <code>CoreSetup&lt;DataPluginStartDependencies, DataPluginStart&gt;</code> |  |
-|  { logger, expressions } | <code>IndexPatternsServiceSetupDeps</code> |  |
+|  core | <code>CoreSetup&lt;IndexPatternsServiceStartDeps, DataPluginStart&gt;</code> |  |
+|  { logger, expressions, usageCollection } | <code>IndexPatternsServiceSetupDeps</code> |  |
 
 <b>Returns:</b>
 

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md

@@ -8,7 +8,7 @@
 
 ```typescript
 start(core: CoreStart, { fieldFormats, logger }: IndexPatternsServiceStartDeps): {
-        indexPatternsServiceFactory: (savedObjectsClient: SavedObjectsClientContract, elasticsearchClient: ElasticsearchClient) => Promise<IndexPatternsCommonService>;
+        indexPatternsServiceFactory: (savedObjectsClient: Pick<import("../../../../core/server").SavedObjectsClient, "get" | "delete" | "create" | "bulkCreate" | "checkConflicts" | "find" | "bulkGet" | "resolve" | "update" | "addToNamespaces" | "deleteFromNamespaces" | "bulkUpdate" | "removeReferencesTo" | "openPointInTimeForType" | "closePointInTime" | "createPointInTimeFinder" | "errors">, elasticsearchClient: ElasticsearchClient) => Promise<IndexPatternsCommonService>;
     };
 ```
 
@@ -22,6 +22,6 @@ start(core: CoreStart, { fieldFormats, logger }: IndexPatternsServiceStartDeps):
 <b>Returns:</b>
 
 `{
-        indexPatternsServiceFactory: (savedObjectsClient: SavedObjectsClientContract, elasticsearchClient: ElasticsearchClient) => Promise<IndexPatternsCommonService>;
+        indexPatternsServiceFactory: (savedObjectsClient: Pick<import("../../../../core/server").SavedObjectsClient, "get" | "delete" | "create" | "bulkCreate" | "checkConflicts" | "find" | "bulkGet" | "resolve" | "update" | "addToNamespaces" | "deleteFromNamespaces" | "bulkUpdate" | "removeReferencesTo" | "openPointInTimeForType" | "closePointInTime" | "createPointInTimeFinder" | "errors">, elasticsearchClient: ElasticsearchClient) => Promise<IndexPatternsCommonService>;
     }`
 

src/plugins/data/server/index_patterns/index_patterns_service.ts

@@ -13,9 +13,11 @@ import {
   Logger,
   SavedObjectsClientContract,
   ElasticsearchClient,
+  UiSettingsServiceStart,
 } from 'kibana/server';
 import { ExpressionsServerSetup } from 'src/plugins/expressions/server';
-import { DataPluginStartDependencies, DataPluginStart } from '../plugin';
+import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
+import { DataPluginStart } from '../plugin';
 import { registerRoutes } from './routes';
 import { indexPatternSavedObjectType } from '../saved_objects';
 import { capabilitiesProvider } from './capabilities_provider';
@@ -26,6 +28,7 @@ import { UiSettingsServerToCommon } from './ui_settings_wrapper';
 import { IndexPatternsApiServer } from './index_patterns_api_client';
 import { SavedObjectsClientServerToCommon } from './saved_objects_client_wrapper';
 import { DataRequestHandlerContext } from '../types';
+import { registerIndexPatternsUsageCollector } from './register_index_pattern_usage_collection';
 
 export interface IndexPatternsServiceStart {
   indexPatternsServiceFactory: (
@@ -37,17 +40,52 @@ export interface IndexPatternsServiceStart {
 export interface IndexPatternsServiceSetupDeps {
   expressions: ExpressionsServerSetup;
   logger: Logger;
+  usageCollection?: UsageCollectionSetup;
 }
 
 export interface IndexPatternsServiceStartDeps {
   fieldFormats: FieldFormatsStart;
   logger: Logger;
 }
 
+export const indexPatternsServiceFactory = ({
+  logger,
+  uiSettings,
+  fieldFormats,
+}: {
+  logger: Logger;
+  uiSettings: UiSettingsServiceStart;
+  fieldFormats: FieldFormatsStart;
+}) => async (
+  savedObjectsClient: SavedObjectsClientContract,
+  elasticsearchClient: ElasticsearchClient
+) => {
+  const uiSettingsClient = uiSettings.asScopedToClient(savedObjectsClient);
+  const formats = await fieldFormats.fieldFormatServiceFactory(uiSettingsClient);
+
+  return new IndexPatternsCommonService({
+    uiSettings: new UiSettingsServerToCommon(uiSettingsClient),
+    savedObjectsClient: new SavedObjectsClientServerToCommon(savedObjectsClient),
+    apiClient: new IndexPatternsApiServer(elasticsearchClient),
+    fieldFormats: formats,
+    onError: (error) => {
+      logger.error(error);
+    },
+    onNotification: ({ title, text }) => {
+      logger.warn(`${title} : ${text}`);
+    },
+    onUnsupportedTimePattern: ({ index, title }) => {
+      logger.warn(
+        `Currently querying all indices matching ${index}. ${title} should be migrated to a wildcard-based index pattern.`
+      );
+    },
+  });
+};
+
 export class IndexPatternsServiceProvider implements Plugin<void, IndexPatternsServiceStart> {
   public setup(
-    core: CoreSetup<DataPluginStartDependencies, DataPluginStart>,
-    { logger, expressions }: IndexPatternsServiceSetupDeps
+    core: CoreSetup<IndexPatternsServiceStartDeps, DataPluginStart>,
+    { logger, expressions, usageCollection }: IndexPatternsServiceSetupDeps
   ) {
     core.savedObjects.registerType(indexPatternSavedObjectType);
     core.capabilities.registerProvider(capabilitiesProvider);
@@ -71,37 +109,18 @@ export class IndexPatternsServiceProvider implements Plugin<void, IndexPatternsS
     registerRoutes(core.http, core.getStartServices);
 
     expressions.registerFunction(getIndexPatternLoad({ getStartServices: core.getStartServices }));
+    registerIndexPatternsUsageCollector(core.getStartServices, usageCollection);
   }
 
   public start(core: CoreStart, { fieldFormats, logger }: IndexPatternsServiceStartDeps) {
     const { uiSettings } = core;
 
     return {
-      indexPatternsServiceFactory: async (
-        savedObjectsClient: SavedObjectsClientContract,
-        elasticsearchClient: ElasticsearchClient
-      ) => {
-        const uiSettingsClient = uiSettings.asScopedToClient(savedObjectsClient);
-        const formats = await fieldFormats.fieldFormatServiceFactory(uiSettingsClient);
-
-        return new IndexPatternsCommonService({
-          uiSettings: new UiSettingsServerToCommon(uiSettingsClient),
-          savedObjectsClient: new SavedObjectsClientServerToCommon(savedObjectsClient),
-          apiClient: new IndexPatternsApiServer(elasticsearchClient),
-          fieldFormats: formats,
-          onError: (error) => {
-            logger.error(error);
-          },
-          onNotification: ({ title, text }) => {
-            logger.warn(`${title} : ${text}`);
-          },
-          onUnsupportedTimePattern: ({ index, title }) => {
-            logger.warn(
-              `Currently querying all indices matching ${index}. ${title} should be migrated to a wildcard-based index pattern.`
-            );
-          },
-        });
-      },
+      indexPatternsServiceFactory: indexPatternsServiceFactory({
+        logger,
+        uiSettings,
+        fieldFormats,
+      }),
     };
   }
 }

src/plugins/data/server/index_patterns/register_index_pattern_usage_collection.test.ts

@@ -0,0 +1,140 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import {
+  minMaxAvgLoC,
+  updateMin,
+  updateMax,
+  getIndexPatternTelemetry,
+} from './register_index_pattern_usage_collection';
+import { IndexPatternsCommonService } from '..';
+
+const scriptA = 'emit(0);';
+const scriptB = 'emit(1);\nemit(2);';
+const scriptC = 'emit(3);\nemit(4)\nemit(5)';
+
+const scriptedFieldA = { script: scriptA };
+const scriptedFieldB = { script: scriptB };
+const scriptedFieldC = { script: scriptC };
+
+const runtimeFieldA = { runtimeField: { script: { source: scriptA } } };
+const runtimeFieldB = { runtimeField: { script: { source: scriptB } } };
+const runtimeFieldC = { runtimeField: { script: { source: scriptC } } };
+
+const indexPatterns = ({
+  getIds: async () => [1, 2, 3],
+  get: jest.fn().mockResolvedValue({
+    getScriptedFields: () => [],
+    fields: [],
+  }),
+} as any) as IndexPatternsCommonService;
+
+describe('index pattern usage collection', () => {
+  it('minMaxAvgLoC calculates min, max, and average ', () => {
+    const scripts = [scriptA, scriptB, scriptC];
+    expect(minMaxAvgLoC(scripts)).toEqual({ min: 1, max: 3, avg: 2 });
+    expect(minMaxAvgLoC([undefined, undefined, undefined])).toEqual({ min: 0, max: 0, avg: 0 });
+  });
+
+  it('updateMin returns minimum value', () => {
+    expect(updateMin(undefined, 1)).toEqual(1);
+    expect(updateMin(1, 0)).toEqual(0);
+  });
+
+  it('updateMax returns maximum value', () => {
+    expect(updateMax(undefined, 1)).toEqual(1);
+    expect(updateMax(1, 0)).toEqual(1);
+  });
+
+  describe('calculates index pattern usage', () => {
+    const countSummaryDefault = {
+      min: undefined,
+      max: undefined,
+      avg: undefined,
+    };
+
+    it('when there are no runtime fields or scripted fields', async () => {
+      expect(await getIndexPatternTelemetry(indexPatterns)).toEqual({
+        indexPatternsCount: 3,
+        indexPatternsWithScriptedFieldCount: 0,
+        indexPatternsWithRuntimeFieldCount: 0,
+        scriptedFieldCount: 0,
+        runtimeFieldCount: 0,
+        perIndexPattern: {
+          scriptedFieldCount: countSummaryDefault,
+          runtimeFieldCount: countSummaryDefault,
+          scriptedFieldLineCount: countSummaryDefault,
+          runtimeFieldLineCount: countSummaryDefault,
+        },
+      });
+    });
+
+    it('when there are both runtime fields or scripted fields', async () => {
+      indexPatterns.get = jest.fn().mockResolvedValue({
+        getScriptedFields: () => [scriptedFieldA, scriptedFieldB, scriptedFieldC],
+        fields: [runtimeFieldA, runtimeFieldB, runtimeFieldC],
+      });
+
+      expect(await getIndexPatternTelemetry(indexPatterns)).toEqual({
+        indexPatternsCount: 3,
+        indexPatternsWithScriptedFieldCount: 3,
+        indexPatternsWithRuntimeFieldCount: 3,
+        scriptedFieldCount: 9,
+        runtimeFieldCount: 9,
+        perIndexPattern: {
+          scriptedFieldCount: { min: 3, max: 3, avg: 3 },
+          runtimeFieldCount: { min: 3, max: 3, avg: 3 },
+          scriptedFieldLineCount: { min: 1, max: 3, avg: 2 },
+          runtimeFieldLineCount: { min: 1, max: 3, avg: 2 },
+        },
+      });
+    });
+
+    it('when there are only runtime fields', async () => {
+      indexPatterns.get = jest.fn().mockResolvedValue({
+        getScriptedFields: () => [],
+        fields: [runtimeFieldA, runtimeFieldB, runtimeFieldC],
+      });
+
+      expect(await getIndexPatternTelemetry(indexPatterns)).toEqual({
+        indexPatternsCount: 3,
+        indexPatternsWithScriptedFieldCount: 0,
+        indexPatternsWithRuntimeFieldCount: 3,
+        scriptedFieldCount: 0,
+        runtimeFieldCount: 9,
+        perIndexPattern: {
+          scriptedFieldCount: countSummaryDefault,
+          runtimeFieldCount: { min: 3, max: 3, avg: 3 },
+          scriptedFieldLineCount: countSummaryDefault,
+          runtimeFieldLineCount: { min: 1, max: 3, avg: 2 },
+        },
+      });
+    });
+
+    it('when there are only scripted fields', async () => {
+      indexPatterns.get = jest.fn().mockResolvedValue({
+        getScriptedFields: () => [scriptedFieldA, scriptedFieldB, scriptedFieldC],
+        fields: [],
+      });
+
+      expect(await getIndexPatternTelemetry(indexPatterns)).toEqual({
+        indexPatternsCount: 3,
+        indexPatternsWithScriptedFieldCount: 3,
+        indexPatternsWithRuntimeFieldCount: 0,
+        scriptedFieldCount: 9,
+        runtimeFieldCount: 0,
+        perIndexPattern: {
+          scriptedFieldCount: { min: 3, max: 3, avg: 3 },
+          runtimeFieldCount: countSummaryDefault,
+          scriptedFieldLineCount: { min: 1, max: 3, avg: 2 },
+          runtimeFieldLineCount: countSummaryDefault,
+        },
+      });
+    });
+  });
+});