[7.5] [Telemetry] change of optin status telemetry (#50158) (#50608)

* initial push

* self code review

* ignore node-fetch type

* usageFetcher api

* user agent metric

* telemetry plugin collector

* remove extra unused method

* remove unused import

* type check

* fix collections tests

* pass kfetch as dep

* add ui metrics integration test for user agent

* dont start ui metrics when not authenticated

* user agent count always 1

* fix broken ui-metric integration tests

* try using config.get

* avoid fetching configs if sending

* type unknown -> string

* check if fetcher is causing the issue

* disable ui_metric from functional tests

* enable ui_metric back again

* ignore keyword above 256

* check requesting app first

* clean up after all the debugging :)

* fix tests

* always return 200 for ui metric reporting

* remove boom import

* logout after removing role/user

* undo some changes in tests

* inside try catch

* prevent potential race conditions in priorities with =

* use snake_case for telemetry plugin collection

* refactors needed to extract cluster uuid based on collection

* refactoring uuid getters

* revert unneeded changes from merge

* finish server/browser fetching

* skip a test  :(

* skip handle_old

* merge master

* fix failing tests

src/legacy/core_plugins/telemetry/index.ts

@@ -53,7 +53,6 @@ const telemetry = (kibana: any) => {
           then: Joi.valid(true).default(true),
           otherwise: Joi.boolean().default(true),
         }),
-
         // `config` is used internally and not intended to be set
         config: Joi.string().default(Joi.ref('$defaultConfigPath')),
         banner: Joi.boolean().default(true),
@@ -66,6 +65,15 @@ const telemetry = (kibana: any) => {
             `https://telemetry.elastic.co/xpack/${ENDPOINT_VERSION}/send`
           ),
         }),
+        optInStatusUrl: Joi.when('$dev', {
+          is: true,
+          then: Joi.string().default(
+            `https://telemetry-staging.elastic.co/opt_in_status/${ENDPOINT_VERSION}/send`
+          ),
+          otherwise: Joi.string().default(
+            `https://telemetry.elastic.co/opt_in_status/${ENDPOINT_VERSION}/send`
+          ),
+        }),
         sendUsageFrom: Joi.string()
           .allow(['server', 'browser'])
           .default('browser'),
@@ -101,6 +109,7 @@ const telemetry = (kibana: any) => {
             config.get('telemetry.allowChangingOptInStatus') !== false &&
             getXpackConfigWithDeprecated(config, 'telemetry.banner'),
           telemetryOptedIn: config.get('telemetry.optIn'),
+          telemetryOptInStatusUrl: config.get('telemetry.optInStatusUrl'),
           allowChangingOptInStatus: config.get('telemetry.allowChangingOptInStatus'),
           telemetrySendUsageFrom: config.get('telemetry.sendUsageFrom'),
         };
@@ -140,7 +149,6 @@ const telemetry = (kibana: any) => {
       } as any) as CoreSetup;
 
       telemetryPlugin(initializerContext).setup(coreSetup);
-
       // register collectors
       server.usage.collectorSet.register(createTelemetryPluginUsageCollector(server));
       server.usage.collectorSet.register(createLocalizationUsageCollector(server));

src/legacy/core_plugins/telemetry/public/hacks/welcome_banner/click_banner.test.js

@@ -54,7 +54,7 @@ const getTelemetryOptInProvider = ({ simulateFailure = false, simulateError = fa
     addBasePath: (url) => url
   };
 
-  const provider = new TelemetryOptInProvider(injector, chrome);
+  const provider = new TelemetryOptInProvider(injector, chrome, false);
 
   if (simulateError) {
     provider.setOptIn = () => Promise.reject('unhandled error');

src/legacy/core_plugins/telemetry/public/hacks/welcome_banner/handle_old_settings.test.js

@@ -49,7 +49,7 @@ const getTelemetryOptInProvider = (enabled, { simulateFailure = false } = {}) =>
     }
   };
 
-  return new TelemetryOptInProvider($injector, chrome);
+  return new TelemetryOptInProvider($injector, chrome, false);
 };
 
 describe('handle_old_settings', () => {

src/legacy/core_plugins/telemetry/public/services/telemetry_opt_in.test.js

@@ -48,7 +48,7 @@ describe('TelemetryOptInProvider', () => {
       }
     };
 
-    const provider = new TelemetryOptInProvider(mockInjector, mockChrome);
+    const provider = new TelemetryOptInProvider(mockInjector, mockChrome, false);
     return {
       provider,
       mockHttp,

src/legacy/core_plugins/telemetry/public/services/telemetry_opt_in.ts

@@ -26,7 +26,36 @@ import { i18n } from '@kbn/i18n';
 let bannerId: string | null = null;
 let currentOptInStatus = false;
 
-export function TelemetryOptInProvider($injector: any, chrome: any) {
+async function sendOptInStatus($injector: any, chrome: any, enabled: boolean) {
+  const telemetryOptInStatusUrl = npStart.core.injectedMetadata.getInjectedVar(
+    'telemetryOptInStatusUrl'
+  ) as string;
+  const $http = $injector.get('$http');
+
+  try {
+    const optInStatus = await $http.post(
+      chrome.addBasePath('/api/telemetry/v2/clusters/_opt_in_stats'),
+      {
+        enabled,
+        unencrypted: false,
+      }
+    );
+
+    if (optInStatus.data && optInStatus.data.length) {
+      return await fetch(telemetryOptInStatusUrl, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(optInStatus.data),
+      });
+    }
+  } catch (err) {
+    // Sending the ping is best-effort. Telemetry tries to send the ping once and discards it immediately if sending fails.
+    // swallow any errors
+  }
+}
+export function TelemetryOptInProvider($injector: any, chrome: any, sendOptInStatusChange = true) {
   currentOptInStatus = npStart.core.injectedMetadata.getInjectedVar('telemetryOptedIn') as boolean;
   const allowChangingOptInStatus = npStart.core.injectedMetadata.getInjectedVar(
     'allowChangingOptInStatus'
@@ -49,6 +78,9 @@ export function TelemetryOptInProvider($injector: any, chrome: any) {
 
       try {
         await $http.post(chrome.addBasePath('/api/telemetry/v2/optIn'), { enabled });
+        if (sendOptInStatusChange) {
+          await sendOptInStatus($injector, chrome, enabled);
+        }
         currentOptInStatus = enabled;
       } catch (error) {
         toastNotifications.addError(error, {

src/legacy/core_plugins/telemetry/server/collection_manager.ts

@@ -18,83 +18,186 @@
  */
 
 import { encryptTelemetry } from './collectors';
+import { CallCluster } from '../../elasticsearch';
 
 export type EncryptedStatsGetterConfig = { unencrypted: false } & {
   server: any;
-  start: any;
-  end: any;
-  isDev: boolean;
+  start: string;
+  end: string;
 };
 
 export type UnencryptedStatsGetterConfig = { unencrypted: true } & {
   req: any;
-  start: any;
-  end: any;
-  isDev: boolean;
+  start: string;
+  end: string;
 };
 
+export interface ClusterDetails {
+  clusterUuid: string;
+}
+
 export interface StatsCollectionConfig {
-  callCluster: any;
+  callCluster: CallCluster;
   server: any;
-  start: any;
-  end: any;
+  start: string;
+  end: string;
 }
 
 export type StatsGetterConfig = UnencryptedStatsGetterConfig | EncryptedStatsGetterConfig;
+export type ClusterDetailsGetter = (config: StatsCollectionConfig) => Promise<ClusterDetails[]>;
+export type StatsGetter = (
+  clustersDetails: ClusterDetails[],
+  config: StatsCollectionConfig
+) => Promise<any[]>;
 
-export type StatsGetter = (config: StatsGetterConfig) => Promise<any[]>;
-
-export const getStatsCollectionConfig = (
-  config: StatsGetterConfig,
-  esClustser: string
-): StatsCollectionConfig => {
-  const { start, end } = config;
-  const server = config.unencrypted ? config.req.server : config.server;
-  const { callWithRequest, callWithInternalUser } = server.plugins.elasticsearch.getCluster(
-    esClustser
-  );
-  const callCluster = config.unencrypted
-    ? (...args: any[]) => callWithRequest(config.req, ...args)
-    : callWithInternalUser;
-
-  return { server, callCluster, start, end };
-};
+interface CollectionConfig {
+  title: string;
+  priority: number;
+  esCluster: string;
+  statsGetter: StatsGetter;
+  clusterDetailsGetter: ClusterDetailsGetter;
+}
+interface Collection {
+  statsGetter: StatsGetter;
+  clusterDetailsGetter: ClusterDetailsGetter;
+  esCluster: string;
+  title: string;
+}
 
 export class TelemetryCollectionManager {
-  private getterMethod?: StatsGetter;
-  private collectionTitle?: string;
-  private getterMethodPriority = -1;
-
-  public setStatsGetter = (statsGetter: StatsGetter, title: string, priority = 0) => {
-    if (priority > this.getterMethodPriority) {
-      this.getterMethod = statsGetter;
-      this.collectionTitle = title;
-      this.getterMethodPriority = priority;
+  private usageGetterMethodPriority = -1;
+  private collections: Collection[] = [];
+
+  public setCollection = (collectionConfig: CollectionConfig) => {
+    const { title, priority, esCluster, statsGetter, clusterDetailsGetter } = collectionConfig;
+
+    if (typeof priority !== 'number') {
+      throw new Error('priority must be set.');
+    }
+    if (priority === this.usageGetterMethodPriority) {
+      throw new Error(`A Usage Getter with the same priority is already set.`);
     }
-  };
 
-  private getStats = async (config: StatsGetterConfig) => {
-    if (!this.getterMethod) {
-      throw Error('Stats getter method not set.');
+    if (priority > this.usageGetterMethodPriority) {
+      if (!statsGetter) {
+        throw Error('Stats getter method not set.');
+      }
+      if (!esCluster) {
+        throw Error('esCluster name must be set for the getCluster method.');
+      }
+      if (!clusterDetailsGetter) {
+        throw Error('Cluser UUIds method is not set.');
+      }
+
+      this.collections.unshift({
+        statsGetter,
+        clusterDetailsGetter,
+        esCluster,
+        title,
+      });
+      this.usageGetterMethodPriority = priority;
     }
-    const usageData = await this.getterMethod(config);
+  };
 
-    if (config.unencrypted) return usageData;
-    return encryptTelemetry(usageData, config.isDev);
+  private getStatsCollectionConfig = async (
+    collection: Collection,
+    config: StatsGetterConfig
+  ): Promise<StatsCollectionConfig> => {
+    const { start, end } = config;
+    const server = config.unencrypted ? config.req.server : config.server;
+    const { callWithRequest, callWithInternalUser } = server.plugins.elasticsearch.getCluster(
+      collection.esCluster
+    );
+    const callCluster = config.unencrypted
+      ? (...args: any[]) => callWithRequest(config.req, ...args)
+      : callWithInternalUser;
+
+    return { server, callCluster, start, end };
   };
-  public getCollectionTitle = () => {
-    return this.collectionTitle;
+
+  private getOptInStatsForCollection = async (
+    collection: Collection,
+    optInStatus: boolean,
+    statsCollectionConfig: StatsCollectionConfig
+  ) => {
+    const clustersDetails = await collection.clusterDetailsGetter(statsCollectionConfig);
+    return clustersDetails.map(({ clusterUuid }) => ({
+      cluster_uuid: clusterUuid,
+      opt_in_status: optInStatus,
+    }));
   };
 
-  public getStatsGetter = () => {
-    if (!this.getterMethod) {
-      throw Error('Stats getter method not set.');
+  private getUsageForCollection = async (
+    collection: Collection,
+    statsCollectionConfig: StatsCollectionConfig
+  ) => {
+    const clustersDetails = await collection.clusterDetailsGetter(statsCollectionConfig);
+
+    if (clustersDetails.length === 0) {
+      // don't bother doing a further lookup, try next collection.
+      return;
     }
-    return {
-      getStats: this.getStats,
-      priority: this.getterMethodPriority,
-      title: this.collectionTitle,
-    };
+
+    return await collection.statsGetter(clustersDetails, statsCollectionConfig);
+  };
+
+  public getOptInStats = async (optInStatus: boolean, config: StatsGetterConfig) => {
+    for (const collection of this.collections) {
+      const statsCollectionConfig = await this.getStatsCollectionConfig(collection, config);
+      try {
+        const optInStats = await this.getOptInStatsForCollection(
+          collection,
+          optInStatus,
+          statsCollectionConfig
+        );
+        if (optInStats && optInStats.length) {
+          statsCollectionConfig.server.log(
+            ['debug', 'telemetry', 'collection'],
+            `Got Opt In stats using ${collection.title} collection.`
+          );
+          if (config.unencrypted) {
+            return optInStats;
+          }
+          const isDev = statsCollectionConfig.server.config().get('env.dev');
+          return encryptTelemetry(optInStats, isDev);
+        }
+      } catch (err) {
+        statsCollectionConfig.server.log(
+          ['debu', 'telemetry', 'collection'],
+          `Failed to collect any opt in stats with registered collections.`
+        );
+        // swallow error to try next collection;
+      }
+    }
+
+    return [];
+  };
+  public getStats = async (config: StatsGetterConfig) => {
+    for (const collection of this.collections) {
+      const statsCollectionConfig = await this.getStatsCollectionConfig(collection, config);
+      try {
+        const usageData = await this.getUsageForCollection(collection, statsCollectionConfig);
+        if (usageData && usageData.length) {
+          statsCollectionConfig.server.log(
+            ['debug', 'telemetry', 'collection'],
+            `Got Usage using ${collection.title} collection.`
+          );
+          if (config.unencrypted) {
+            return usageData;
+          }
+          const isDev = statsCollectionConfig.server.config().get('env.dev');
+          return encryptTelemetry(usageData, isDev);
+        }
+      } catch (err) {
+        statsCollectionConfig.server.log(
+          ['debu', 'telemetry', 'collection'],
+          `Failed to collect any usage with registered collections.`
+        );
+        // swallow error to try next collection;
+      }
+    }
+
+    return [];
   };
 }
 

src/legacy/core_plugins/telemetry/server/collectors/telemetry_plugin/telemetry_plugin_collector.ts

@@ -19,7 +19,7 @@
 
 import { TELEMETRY_STATS_TYPE } from '../../../common/constants';
 import { getTelemetrySavedObject, TelemetrySavedObject } from '../../telemetry_repository';
-import { getTelemetryOptIn, getTelemetryUsageFetcher } from '../../telemetry_config';
+import { getTelemetryOptIn, getTelemetrySendUsageFrom } from '../../telemetry_config';
 export interface TelemetryUsageStats {
   opt_in_status?: boolean | null;
   usage_fetcher?: 'browser' | 'server';
@@ -53,7 +53,7 @@ export function createCollectorFetch(server: any) {
         configTelemetryOptIn,
       }),
       last_reported: telemetrySavedObject ? telemetrySavedObject.lastReported : undefined,
-      usage_fetcher: getTelemetryUsageFetcher({
+      usage_fetcher: getTelemetrySendUsageFrom({
         telemetrySavedObject,
         configTelemetrySendUsageFrom,
       }),