[APM] Improvements to data telemetry (#70524)

Make some changes to how we deal with data telemetry in APM and reduce the number of fields we're storing in Saved Objects in the .kibana index.

Add a telemetry doc in dev_docs explaining how telemetry is collected and how to make updates. (In this PR the docs only cover data telemetry, but there's a space for the behavioral telemetry docs.)

Stop storing the mapping for the data telemetry in the Saved Object but instead use `{ dynamic: false }`.

This reduces the number of fields used by APM in the .kibana index (as requested in #43673.)

Before:

```bash
> curl -s -X GET "admin:changeme@localhost:9200/.kibana/_field_caps?fields=*&pretty=true" |  jq '.fields|length'
653
```

After:

```bash
> curl -s -X GET "admin:changeme@localhost:9200/.kibana/_field_caps?fields=*&pretty=true" |  jq '.fields|length'
415
```

We don't need the mapping anymore for storing the saved object, but we still do need to update the telemetry repository when the mapping changes, and the `upload-telemetry-data` script uses that mapping when generating data.

For these purposes the mapping in now defined in TypeScript in a function in common/apm_telemetry.ts.

It's broken down into some variables that and put together as the same mapping object that was there before, but having it in this form should make it easier to update.

A new script, `merge-telemetry-mapping`, takes the telemetry repository's xpack-phone-home.json mapping, merges in the result of our mapping and replaces the file. The result can be committed to the telemetry repo, making it easier to make changes to the mapping.

References #61583
Fixes #67032

x-pack/plugins/apm/common/agent_name.ts

@@ -15,15 +15,14 @@ import { AgentName } from '../typings/es_schemas/ui/fields/agent';
  */
 
 export const AGENT_NAMES: AgentName[] = [
-  'java',
-  'js-base',
-  'rum-js',
   'dotnet',
   'go',
   'java',
+  'js-base',
   'nodejs',
   'python',
   'ruby',
+  'rum-js',
 ];
 
 export function isAgentName(agentName: string): agentName is AgentName {

x-pack/plugins/apm/common/apm_telemetry.test.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  getApmTelemetryMapping,
+  mergeApmTelemetryMapping,
+} from './apm_telemetry';
+
+describe('APM telemetry helpers', () => {
+  describe('getApmTelemetry', () => {
+    it('generates a JSON object with the telemetry mapping', () => {
+      expect(getApmTelemetryMapping()).toMatchSnapshot();
+    });
+  });
+
+  describe('mergeApmTelemetryMapping', () => {
+    describe('with an invalid mapping', () => {
+      it('throws an error', () => {
+        expect(() => mergeApmTelemetryMapping({})).toThrowError();
+      });
+    });
+
+    describe('with a valid mapping', () => {
+      it('merges the mapping', () => {
+        // This is "valid" in the sense that it has all of the deep fields
+        // needed to merge. It's not a valid mapping opbject.
+        const validTelemetryMapping = {
+          mappings: {
+            properties: {
+              stack_stats: {
+                properties: {
+                  kibana: {
+                    properties: { plugins: { properties: { apm: {} } } },
+                  },
+                },
+              },
+            },
+          },
+        };
+
+        expect(
+          mergeApmTelemetryMapping(validTelemetryMapping)?.mappings.properties
+            .stack_stats.properties.kibana.properties.plugins.properties.apm
+        ).toEqual(getApmTelemetryMapping());
+      });
+    });
+  });
+});

x-pack/plugins/apm/common/apm_telemetry.ts

@@ -0,0 +1,229 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { produce } from 'immer';
+import { AGENT_NAMES } from './agent_name';
+
+/**
+ * Generate an object containing the mapping used for APM telemetry. Can be used
+ * with the `upload-telemetry-data` script or to update the mapping in the
+ * telemetry repository.
+ *
+ * This function breaks things up to make the mapping easier to understand.
+ */
+export function getApmTelemetryMapping() {
+  const keyword = {
+    type: 'keyword',
+    ignore_above: 1024,
+  };
+
+  const long = {
+    type: 'long',
+  };
+
+  const allProperties = {
+    properties: {
+      all: long,
+    },
+  };
+
+  const oneDayProperties = {
+    properties: {
+      '1d': long,
+    },
+  };
+
+  const oneDayAllProperties = {
+    properties: {
+      '1d': long,
+      all: long,
+    },
+  };
+
+  const msProperties = {
+    properties: {
+      ms: long,
+    },
+  };
+
+  const tookProperties = {
+    properties: {
+      took: msProperties,
+    },
+  };
+
+  const compositeNameVersionProperties = {
+    properties: {
+      composite: keyword,
+      name: keyword,
+      version: keyword,
+    },
+  };
+
+  const agentProperties = {
+    properties: { version: keyword },
+  };
+
+  const serviceProperties = {
+    properties: {
+      framework: compositeNameVersionProperties,
+      language: compositeNameVersionProperties,
+      runtime: compositeNameVersionProperties,
+    },
+  };
+
+  return {
+    properties: {
+      agents: {
+        properties: AGENT_NAMES.reduce<Record<string, any>>(
+          (previousValue, currentValue) => {
+            previousValue[currentValue] = {
+              properties: {
+                agent: agentProperties,
+                service: serviceProperties,
+              },
+            };
+
+            return previousValue;
+          },
+          {}
+        ),
+      },
+      counts: {
+        properties: {
+          agent_configuration: allProperties,
+          error: oneDayAllProperties,
+          max_error_groups_per_service: oneDayProperties,
+          max_transaction_groups_per_service: oneDayProperties,
+          metric: oneDayAllProperties,
+          onboarding: oneDayAllProperties,
+          services: oneDayProperties,
+          sourcemap: oneDayAllProperties,
+          span: oneDayAllProperties,
+          traces: oneDayProperties,
+          transaction: oneDayAllProperties,
+        },
+      },
+      cardinality: {
+        properties: {
+          user_agent: {
+            properties: {
+              original: {
+                properties: {
+                  all_agents: oneDayProperties,
+                  rum: oneDayProperties,
+                },
+              },
+            },
+          },
+          transaction: {
+            properties: {
+              name: {
+                properties: {
+                  all_agents: oneDayProperties,
+                  rum: oneDayProperties,
+                },
+              },
+            },
+          },
+        },
+      },
+      has_any_services: {
+        type: 'boolean',
+      },
+      indices: {
+        properties: {
+          all: {
+            properties: {
+              total: {
+                properties: {
+                  docs: {
+                    properties: {
+                      count: long,
+                    },
+                  },
+                  store: {
+                    properties: {
+                      size_in_bytes: long,
+                    },
+                  },
+                },
+              },
+            },
+          },
+          shards: {
+            properties: {
+              total: long,
+            },
+          },
+        },
+      },
+      integrations: {
+        properties: {
+          ml: {
+            properties: {
+              all_jobs_count: long,
+            },
+          },
+        },
+      },
+      retainment: {
+        properties: {
+          error: msProperties,
+          metric: msProperties,
+          onboarding: msProperties,
+          span: msProperties,
+          transaction: msProperties,
+        },
+      },
+      services_per_agent: {
+        properties: AGENT_NAMES.reduce<Record<string, any>>(
+          (previousValue, currentValue) => {
+            previousValue[currentValue] = { ...long, null_value: 0 };
+            return previousValue;
+          },
+          {}
+        ),
+      },
+      tasks: {
+        properties: {
+          agent_configuration: tookProperties,
+          agents: tookProperties,
+          cardinality: tookProperties,
+          groupings: tookProperties,
+          indices_stats: tookProperties,
+          integrations: tookProperties,
+          processor_events: tookProperties,
+          services: tookProperties,
+          versions: tookProperties,
+        },
+      },
+      version: {
+        properties: {
+          apm_server: {
+            properties: {
+              major: long,
+              minor: long,
+              patch: long,
+            },
+          },
+        },
+      },
+    },
+  };
+}
+
+/**
+ * Merge a telemetry mapping object (from https://github.com/elastic/telemetry/blob/master/config/templates/xpack-phone-home.json)
+ * with the output from `getApmTelemetryMapping`.
+ */
+export function mergeApmTelemetryMapping(
+  xpackPhoneHomeMapping: Record<string, any>
+) {
+  return produce(xpackPhoneHomeMapping, (draft: Record<string, any>) => {
+    draft.mappings.properties.stack_stats.properties.kibana.properties.plugins.properties.apm = getApmTelemetryMapping();
+    return draft;
+  });
+}

x-pack/plugins/apm/dev_docs/telemetry.md

@@ -0,0 +1,69 @@
+# APM Telemetry
+
+In order to learn about our customers' usage and experience of APM, we collect
+two types of telemetry, which we'll refer to here as "Data Telemetry" and
+"Behavioral Telemetry."
+
+This document will explain how they are collected and how to make changes to
+them.
+
+[The telemetry repository has information about accessing the clusters](https://github.com/elastic/telemetry#kibana-access).
+Telemetry data is uploaded to the "xpack-phone-home" indices.
+
+## Data Telemetry
+
+Information that can be derived from a cluster's APM indices is queried and sent
+to the telemetry cluster using the
+[Usage Collection plugin](../../../../src/plugins/usage_collection/README.md).
+
+During the APM server-side plugin's setup phase a
+[Saved Object](https://www.elastic.co/guide/en/kibana/master/managing-saved-objects.html)
+for APM telemetry is registered and a
+[task manager](../../task_manager/server/README.md) task is registered and started.
+The task periodically queries the APM indices and saves the results in the Saved
+Object, and the usage collector periodically gets the data from the saved object
+and uploads it to the telemetry cluster.
+
+Once uploaded to the telemetry cluster, the data telemetry is stored in
+`stack_stats.kibana.plugins.apm` in the xpack-phone-home index.
+
+### Generating sample data
+
+The script in `scripts/upload-telemetry-data` can generate sample telemetry data and upload it to a cluster of your choosing.
+
+You'll need to set the `GITHUB_TOKEN` environment variable to a token that has `repo` scope so it can read from the
+[elastic/telemetry](https://github.com/elastic/telemetry) repository. (You probably have a token that works for this in
+~/.backport/config.json.)
+
+The script will run as the `elastic` user using the elasticsearch hosts and password settings from the config/kibana.yml
+and/or config/kibana.dev.yml files.
+
+Running the script with `--clear` will delete the index first.
+
+After running the script you should see sample telemetry data in the "xpack-phone-home" index.
+
+### Updating Data Telemetry Mappings
+
+In order for fields to be searchable on the telemetry cluster, they need to be
+added to the cluster's mapping. The mapping is defined in
+[the telemetry repository's xpack-phone-home template](https://github.com/elastic/telemetry/blob/master/config/templates/xpack-phone-home.json).
+
+The mapping for the telemetry data is here under `stack_stats.kibana.plugins.apm`.
+
+The mapping used there can be generated with the output of the [`getTelemetryMapping`](../common/apm_telemetry.ts) function.
+
+To make a change to the mapping, edit this function, run the tests to update the snapshots, then use the `merge_telemetry_mapping` script to merge the data into the telemetry repository.
+
+If the [telemetry repository](https://github.com/elastic/telemetry) is cloned as a sibling to the kibana directory, you can run the following from x-pack/plugins/apm:
+
+```bash
+node ./scripts/merge-telemetry-mapping.js ../../../../telemetry/config/templates/xpack-phone-home.json
+```
+
+this will replace the contents of the mapping in the repository checkout with the updated mapping. You can then [follow the telemetry team's instructions](https://github.com/elastic/telemetry#mappings) for opening a pull request with the mapping changes.
+
+## Behavioral Telemetry
+
+Behavioral telemetry is recorded with the ui_metrics and application_usage methods from the Usage Collection plugin.
+
+Please fill this in with more details.

x-pack/plugins/apm/public/plugin.ts

@@ -66,8 +66,9 @@ export interface ApmPluginStartDeps {
 }
 
 export class ApmPlugin implements Plugin<ApmPluginSetup, ApmPluginStart> {
-  private readonly initializerContext: PluginInitializerContext<ConfigSchema>;
-  constructor(initializerContext: PluginInitializerContext<ConfigSchema>) {
+  constructor(
+    private readonly initializerContext: PluginInitializerContext<ConfigSchema>
+  ) {
     this.initializerContext = initializerContext;
   }
   public setup(core: CoreSetup, plugins: ApmPluginSetupDeps) {

x-pack/plugins/apm/readme.md

@@ -162,3 +162,4 @@ You can access the development environment at http://localhost:9001.
 - [Cypress integration tests](./e2e/README.md)
 - [VSCode setup instructions](./dev_docs/vscode_setup.md)
 - [Github PR commands](./dev_docs/github_commands.md)
+- [Telemetry](./dev_docs/telemetry.md)