Merge branch '7.12' into backport/7.12/pr-91171

docs/discover/context.asciidoc

@@ -0,0 +1,60 @@
+[[discover-document-context]]
+== View surrounding documents
+
+Once you've narrowed your search to a specific event in *Discover*,
+you can inspect the documents that occurred
+immediately before and after the event.
+To view the surrounding documents, your index pattern must contain time-based events.
+
+. In the document table, click the expand icon (>).
+. Click *View surrounding documents.*
++
+In the context view, documents are sorted by the time field specified in the index pattern
+and displayed using the same set of columns as the *Discover* view from which
+the context was opened. The anchor document is highlighted in blue.
++
+[role="screenshot"]
+image::images/discover-context.png[Image showing context view feature, with anchor documents highlighted in blue]
++
+The filters you applied in *Discover* are carried over to the context view. Pinned 
+filters remain active, while normal filters are copied in a disabled state.
+
++
+[role="screenshot"]
+image::images/discover-context-filters-inactive.png[Filter in context view]
+
+. To find the documents of interest, add filters.
+
+. To increase the number of documents that surround the anchor document, click *Load*.
+By default, five documents are added with each click.
++
+[role="screenshot"]
+image::images/discover-context-load-newer-documents.png[Load button and the number of documents to load]
+
+
+[float]
+[[configure-context-ContextView]]
+=== Configure the context view
+
+Configure the appearance and behavior in *Advanced Settings*.
+
+. Open the main menu, then click *Stack Management > Advanced Settings*.
+. Search for `context`, then edit the settings.
++
+[horizontal]
+`context:defaultSize`:: The number of documents to display by default.
+`context:step`:: The default number of documents to load with each button click. The default is 5.
+`context:tieBreakerFields`:: The field to use for tiebreaking in case of equal time field values.
+The default is the `_doc` field.
++
+You can enter a comma-separated list of field
+names, which is checked in sequence for suitability when a context is
+displayed. The first suitable field is used as the tiebreaking
+field. A field is suitable if the field exists and is sortable in the index
+pattern the context is based on.
++
+Although not required, it is recommended to only
+use fields that have {ref}/doc-values.html[doc values] enabled to achieve
+good performance and avoid unnecessary {ref}/modules-fielddata.html[field
+data] usage. Common examples for suitable fields include log line numbers,
+monotonically increasing counters and high-precision timestamps.

docs/user/discover.asciidoc

@@ -161,7 +161,8 @@ image:images/document-table-expanded.png[Table view with document expanded]
 . Scan through the fields and their values. If you find a field of interest,
 hover of its name for filters and other controls.
 
-. To view documents that occurred before or after the event you are looking at, click **View surrounding documents**.
+. To view documents that occurred before or after the event you are looking at, click
+<<discover-document-context, View surrounding documents>>.
 
 . For direct access to a particular document, click **View single document**.
 +
@@ -218,4 +219,6 @@ include::{kib-repo-dir}/discover/set-time-filter.asciidoc[]
 
 include::{kib-repo-dir}/discover/search.asciidoc[]
 
+include::{kib-repo-dir}/discover/context.asciidoc[]
+
 include::{kib-repo-dir}/discover/search-for-relevance.asciidoc[]

x-pack/plugins/apm/common/utils/as_mutable_array.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// Sometimes we use `as const` to have a more specific type,
+// because TypeScript by default will widen the value type of an
+// array literal. Consider the following example:
+//
+// const filter = [
+//   { term: { 'agent.name': 'nodejs' } },
+//   { range: { '@timestamp': { gte: 'now-15m ' }}
+// ];
+
+// The result value type will be:
+
+// const filter: ({
+//   term: {
+//     'agent.name'?: string
+//   };
+//   range?: undefined
+// } | {
+//   term?: undefined;
+//   range: {
+//     '@timestamp': {
+//       gte: string
+//     }
+//   }
+// })[];
+
+// This can sometimes leads to issues. In those cases, we can
+// use `as const`. However, the Readonly<any> type is not compatible
+// with Array<any>. This function returns a mutable version of a type.
+
+export function asMutableArray<T extends Readonly<any>>(
+  arr: T
+): T extends Readonly<[...infer U]> ? U : unknown[] {
+  return arr as any;
+}

x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts

@@ -35,14 +35,14 @@ interface AggregationParams {
   environment?: string;
   setup: ServicesItemsSetup;
   searchAggregatedTransactions: boolean;
+  maxNumServices: number;
 }
 
-const MAX_NUMBER_OF_SERVICES = 500;
-
 export async function getServiceTransactionStats({
   environment,
   setup,
   searchAggregatedTransactions,
+  maxNumServices,
 }: AggregationParams) {
   return withApmSpan('get_service_transaction_stats', async () => {
     const { apmEventClient, start, end, esFilter } = setup;
@@ -86,7 +86,7 @@ export async function getServiceTransactionStats({
           services: {
             terms: {
               field: SERVICE_NAME,
-              size: MAX_NUMBER_OF_SERVICES,
+              size: maxNumServices,
             },
             aggs: {
               transactionType: {
@@ -98,7 +98,6 @@ export async function getServiceTransactionStats({
                   environments: {
                     terms: {
                       field: SERVICE_ENVIRONMENT,
-                      missing: '',
                     },
                   },
                   sample: {
@@ -141,9 +140,9 @@ export async function getServiceTransactionStats({
         return {
           serviceName: bucket.key as string,
           transactionType: topTransactionTypeBucket.key as string,
-          environments: topTransactionTypeBucket.environments.buckets
-            .map((environmentBucket) => environmentBucket.key as string)
-            .filter(Boolean),
+          environments: topTransactionTypeBucket.environments.buckets.map(
+            (environmentBucket) => environmentBucket.key as string
+          ),
           agentName: topTransactionTypeBucket.sample.top[0].metrics[
             AGENT_NAME
           ] as AgentName,

x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
+import {
+  AGENT_NAME,
+  SERVICE_ENVIRONMENT,
+  SERVICE_NAME,
+} from '../../../../common/elasticsearch_fieldnames';
+import { environmentQuery, rangeQuery } from '../../../../common/utils/queries';
+import { ProcessorEvent } from '../../../../common/processor_event';
+import { Setup, SetupTimeRange } from '../../helpers/setup_request';
+import { withApmSpan } from '../../../utils/with_apm_span';
+
+export function getServicesFromMetricDocuments({
+  environment,
+  setup,
+  maxNumServices,
+  kuery,
+}: {
+  setup: Setup & SetupTimeRange;
+  environment?: string;
+  maxNumServices: number;
+  kuery?: string;
+}) {
+  return withApmSpan('get_services_from_metric_documents', async () => {
+    const { apmEventClient, start, end, esFilter } = setup;
+
+    const response = await apmEventClient.search({
+      apm: {
+        events: [ProcessorEvent.metric],
+      },
+      body: {
+        size: 0,
+        query: {
+          bool: {
+            filter: [
+              ...rangeQuery(start, end),
+              ...environmentQuery(environment),
+              ...esFilter,
+            ],
+          },
+        },
+        aggs: {
+          services: {
+            terms: {
+              field: SERVICE_NAME,
+              size: maxNumServices,
+            },
+            aggs: {
+              environments: {
+                terms: {
+                  field: SERVICE_ENVIRONMENT,
+                },
+              },
+              latest: {
+                top_metrics: {
+                  metrics: { field: AGENT_NAME } as const,
+                  sort: { '@timestamp': 'desc' },
+                },
+              },
+            },
+          },
+        },
+      },
+    });
+
+    return (
+      response.aggregations?.services.buckets.map((bucket) => {
+        return {
+          serviceName: bucket.key as string,
+          environments: bucket.environments.buckets.map(
+            (envBucket) => envBucket.key as string
+          ),
+          agentName: bucket.latest.top[0].metrics[AGENT_NAME] as AgentName,
+        };
+      }) ?? []
+    );
+  });
+}

x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts

@@ -6,15 +6,19 @@
  */
 
 import { Logger } from '@kbn/logging';
+import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import { joinByKey } from '../../../../common/utils/join_by_key';
-import { getServicesProjection } from '../../../projections/services';
 import { withApmSpan } from '../../../utils/with_apm_span';
 import { Setup, SetupTimeRange } from '../../helpers/setup_request';
 import { getHealthStatuses } from './get_health_statuses';
+import { getServicesFromMetricDocuments } from './get_services_from_metric_documents';
 import { getServiceTransactionStats } from './get_service_transaction_stats';
+import { getServicesProjection } from '../../../projections/services';
 
 export type ServicesItemsSetup = Setup & SetupTimeRange;
 
+const MAX_NUMBER_OF_SERVICES = 500;
+
 export async function getServicesItems({
   environment,
   setup,
@@ -35,26 +39,47 @@ export async function getServicesItems({
       }),
       setup,
       searchAggregatedTransactions,
+      maxNumServices: MAX_NUMBER_OF_SERVICES,
     };
 
-    const [transactionStats, healthStatuses] = await Promise.all([
+    const [
+      transactionStats,
+      servicesFromMetricDocuments,
+      healthStatuses,
+    ] = await Promise.all([
       getServiceTransactionStats(params),
+      getServicesFromMetricDocuments(params),
       getHealthStatuses(params).catch((err) => {
         logger.error(err);
         return [];
       }),
     ]);
 
-    const apmServices = transactionStats.map(({ serviceName }) => serviceName);
+    const foundServiceNames = transactionStats.map(
+      ({ serviceName }) => serviceName
+    );
+
+    const servicesWithOnlyMetricDocuments = servicesFromMetricDocuments.filter(
+      ({ serviceName }) => !foundServiceNames.includes(serviceName)
+    );
+
+    const allServiceNames = foundServiceNames.concat(
+      servicesWithOnlyMetricDocuments.map(({ serviceName }) => serviceName)
+    );
 
     // make sure to exclude health statuses from services
     // that are not found in APM data
     const matchedHealthStatuses = healthStatuses.filter(({ serviceName }) =>
-      apmServices.includes(serviceName)
+      allServiceNames.includes(serviceName)
     );
 
-    const allMetrics = [...transactionStats, ...matchedHealthStatuses];
-
-    return joinByKey(allMetrics, 'serviceName');
+    return joinByKey(
+      asMutableArray([
+        ...transactionStats,
+        ...servicesWithOnlyMetricDocuments,
+        ...matchedHealthStatuses,
+      ] as const),
+      'serviceName'
+    );
   });
 }