diff --git a/docs/discover/context.asciidoc b/docs/discover/context.asciidoc new file mode 100644 index 000000000000..9131c81781fc --- /dev/null +++ b/docs/discover/context.asciidoc @@ -0,0 +1,60 @@ +[[discover-document-context]] +== View surrounding documents + +Once you've narrowed your search to a specific event in *Discover*, +you can inspect the documents that occurred +immediately before and after the event. +To view the surrounding documents, your index pattern must contain time-based events. + +. In the document table, click the expand icon (>). +. Click *View surrounding documents.* ++ +In the context view, documents are sorted by the time field specified in the index pattern +and displayed using the same set of columns as the *Discover* view from which +the context was opened. The anchor document is highlighted in blue. ++ +[role="screenshot"] +image::images/discover-context.png[Image showing context view feature, with anchor documents highlighted in blue] ++ +The filters you applied in *Discover* are carried over to the context view. Pinned +filters remain active, while normal filters are copied in a disabled state. + ++ +[role="screenshot"] +image::images/discover-context-filters-inactive.png[Filter in context view] + +. To find the documents of interest, add filters. + +. To increase the number of documents that surround the anchor document, click *Load*. +By default, five documents are added with each click. ++ +[role="screenshot"] +image::images/discover-context-load-newer-documents.png[Load button and the number of documents to load] + + +[float] +[[configure-context-ContextView]] +=== Configure the context view + +Configure the appearance and behavior in *Advanced Settings*. + +. Open the main menu, then click *Stack Management > Advanced Settings*. +. Search for `context`, then edit the settings. ++ +[horizontal] +`context:defaultSize`:: The number of documents to display by default. +`context:step`:: The default number of documents to load with each button click. The default is 5. +`context:tieBreakerFields`:: The field to use for tiebreaking in case of equal time field values. +The default is the `_doc` field. ++ +You can enter a comma-separated list of field +names, which is checked in sequence for suitability when a context is +displayed. The first suitable field is used as the tiebreaking +field. A field is suitable if the field exists and is sortable in the index +pattern the context is based on. ++ +Although not required, it is recommended to only +use fields that have {ref}/doc-values.html[doc values] enabled to achieve +good performance and avoid unnecessary {ref}/modules-fielddata.html[field +data] usage. Common examples for suitable fields include log line numbers, +monotonically increasing counters and high-precision timestamps. \ No newline at end of file diff --git a/docs/discover/images/discover-context-filters-active.png b/docs/discover/images/discover-context-filters-active.png new file mode 100644 index 000000000000..9aa70ab138b0 Binary files /dev/null and b/docs/discover/images/discover-context-filters-active.png differ diff --git a/docs/discover/images/discover-context-filters-inactive.png b/docs/discover/images/discover-context-filters-inactive.png new file mode 100644 index 000000000000..c53a8cedff33 Binary files /dev/null and b/docs/discover/images/discover-context-filters-inactive.png differ diff --git a/docs/discover/images/discover-context-load-newer-documents.png b/docs/discover/images/discover-context-load-newer-documents.png new file mode 100644 index 000000000000..9c4a74d39b3c Binary files /dev/null and b/docs/discover/images/discover-context-load-newer-documents.png differ diff --git a/docs/discover/images/discover-context.png b/docs/discover/images/discover-context.png new file mode 100644 index 000000000000..8ce68047e0d7 Binary files /dev/null and b/docs/discover/images/discover-context.png differ diff --git a/docs/images/Discover-ContextView.png b/docs/images/Discover-ContextView.png deleted file mode 100644 index b9682764f457..000000000000 Binary files a/docs/images/Discover-ContextView.png and /dev/null differ diff --git a/docs/user/discover.asciidoc b/docs/user/discover.asciidoc index 42ac1e22ce16..11abe975374e 100644 --- a/docs/user/discover.asciidoc +++ b/docs/user/discover.asciidoc @@ -161,7 +161,8 @@ image:images/document-table-expanded.png[Table view with document expanded] . Scan through the fields and their values. If you find a field of interest, hover of its name for filters and other controls. -. To view documents that occurred before or after the event you are looking at, click **View surrounding documents**. +. To view documents that occurred before or after the event you are looking at, click +<>. . For direct access to a particular document, click **View single document**. + @@ -218,4 +219,6 @@ include::{kib-repo-dir}/discover/set-time-filter.asciidoc[] include::{kib-repo-dir}/discover/search.asciidoc[] +include::{kib-repo-dir}/discover/context.asciidoc[] + include::{kib-repo-dir}/discover/search-for-relevance.asciidoc[] diff --git a/x-pack/plugins/apm/common/utils/as_mutable_array.ts b/x-pack/plugins/apm/common/utils/as_mutable_array.ts new file mode 100644 index 000000000000..ce1d7e607ec4 --- /dev/null +++ b/x-pack/plugins/apm/common/utils/as_mutable_array.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// Sometimes we use `as const` to have a more specific type, +// because TypeScript by default will widen the value type of an +// array literal. Consider the following example: +// +// const filter = [ +// { term: { 'agent.name': 'nodejs' } }, +// { range: { '@timestamp': { gte: 'now-15m ' }} +// ]; + +// The result value type will be: + +// const filter: ({ +// term: { +// 'agent.name'?: string +// }; +// range?: undefined +// } | { +// term?: undefined; +// range: { +// '@timestamp': { +// gte: string +// } +// } +// })[]; + +// This can sometimes leads to issues. In those cases, we can +// use `as const`. However, the Readonly type is not compatible +// with Array. This function returns a mutable version of a type. + +export function asMutableArray>( + arr: T +): T extends Readonly<[...infer U]> ? U : unknown[] { + return arr as any; +} diff --git a/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap b/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap index 3e68831ee7cb..2819a62c3037 100644 --- a/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap @@ -109,7 +109,6 @@ Array [ "environments": Object { "terms": Object { "field": "service.environment", - "missing": "", }, }, "outcomes": Object { @@ -193,6 +192,61 @@ Array [ "size": 0, }, }, + Object { + "apm": Object { + "events": Array [ + "metric", + ], + }, + "body": Object { + "aggs": Object { + "services": Object { + "aggs": Object { + "environments": Object { + "terms": Object { + "field": "service.environment", + }, + }, + "latest": Object { + "top_metrics": Object { + "metrics": Object { + "field": "agent.name", + }, + "sort": Object { + "@timestamp": "desc", + }, + }, + }, + }, + "terms": Object { + "field": "service.name", + "size": 500, + }, + }, + }, + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "range": Object { + "@timestamp": Object { + "format": "epoch_millis", + "gte": 1528113600000, + "lte": 1528977600000, + }, + }, + }, + Object { + "term": Object { + "service.environment": "test", + }, + }, + ], + }, + }, + "size": 0, + }, + }, ] `; diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts index e1f8bca83829..a03bbc337625 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts @@ -35,14 +35,14 @@ interface AggregationParams { environment?: string; setup: ServicesItemsSetup; searchAggregatedTransactions: boolean; + maxNumServices: number; } -const MAX_NUMBER_OF_SERVICES = 500; - export async function getServiceTransactionStats({ environment, setup, searchAggregatedTransactions, + maxNumServices, }: AggregationParams) { return withApmSpan('get_service_transaction_stats', async () => { const { apmEventClient, start, end, esFilter } = setup; @@ -86,7 +86,7 @@ export async function getServiceTransactionStats({ services: { terms: { field: SERVICE_NAME, - size: MAX_NUMBER_OF_SERVICES, + size: maxNumServices, }, aggs: { transactionType: { @@ -98,7 +98,6 @@ export async function getServiceTransactionStats({ environments: { terms: { field: SERVICE_ENVIRONMENT, - missing: '', }, }, sample: { @@ -141,9 +140,9 @@ export async function getServiceTransactionStats({ return { serviceName: bucket.key as string, transactionType: topTransactionTypeBucket.key as string, - environments: topTransactionTypeBucket.environments.buckets - .map((environmentBucket) => environmentBucket.key as string) - .filter(Boolean), + environments: topTransactionTypeBucket.environments.buckets.map( + (environmentBucket) => environmentBucket.key as string + ), agentName: topTransactionTypeBucket.sample.top[0].metrics[ AGENT_NAME ] as AgentName, diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts new file mode 100644 index 000000000000..7149098a29bb --- /dev/null +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; +import { + AGENT_NAME, + SERVICE_ENVIRONMENT, + SERVICE_NAME, +} from '../../../../common/elasticsearch_fieldnames'; +import { environmentQuery, rangeQuery } from '../../../../common/utils/queries'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { Setup, SetupTimeRange } from '../../helpers/setup_request'; +import { withApmSpan } from '../../../utils/with_apm_span'; + +export function getServicesFromMetricDocuments({ + environment, + setup, + maxNumServices, + kuery, +}: { + setup: Setup & SetupTimeRange; + environment?: string; + maxNumServices: number; + kuery?: string; +}) { + return withApmSpan('get_services_from_metric_documents', async () => { + const { apmEventClient, start, end, esFilter } = setup; + + const response = await apmEventClient.search({ + apm: { + events: [ProcessorEvent.metric], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...esFilter, + ], + }, + }, + aggs: { + services: { + terms: { + field: SERVICE_NAME, + size: maxNumServices, + }, + aggs: { + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + }, + }, + latest: { + top_metrics: { + metrics: { field: AGENT_NAME } as const, + sort: { '@timestamp': 'desc' }, + }, + }, + }, + }, + }, + }, + }); + + return ( + response.aggregations?.services.buckets.map((bucket) => { + return { + serviceName: bucket.key as string, + environments: bucket.environments.buckets.map( + (envBucket) => envBucket.key as string + ), + agentName: bucket.latest.top[0].metrics[AGENT_NAME] as AgentName, + }; + }) ?? [] + ); + }); +} diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts index c2677af03848..c6a0949325f1 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts @@ -6,15 +6,19 @@ */ import { Logger } from '@kbn/logging'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { joinByKey } from '../../../../common/utils/join_by_key'; -import { getServicesProjection } from '../../../projections/services'; import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; import { getHealthStatuses } from './get_health_statuses'; +import { getServicesFromMetricDocuments } from './get_services_from_metric_documents'; import { getServiceTransactionStats } from './get_service_transaction_stats'; +import { getServicesProjection } from '../../../projections/services'; export type ServicesItemsSetup = Setup & SetupTimeRange; +const MAX_NUMBER_OF_SERVICES = 500; + export async function getServicesItems({ environment, setup, @@ -35,26 +39,47 @@ export async function getServicesItems({ }), setup, searchAggregatedTransactions, + maxNumServices: MAX_NUMBER_OF_SERVICES, }; - const [transactionStats, healthStatuses] = await Promise.all([ + const [ + transactionStats, + servicesFromMetricDocuments, + healthStatuses, + ] = await Promise.all([ getServiceTransactionStats(params), + getServicesFromMetricDocuments(params), getHealthStatuses(params).catch((err) => { logger.error(err); return []; }), ]); - const apmServices = transactionStats.map(({ serviceName }) => serviceName); + const foundServiceNames = transactionStats.map( + ({ serviceName }) => serviceName + ); + + const servicesWithOnlyMetricDocuments = servicesFromMetricDocuments.filter( + ({ serviceName }) => !foundServiceNames.includes(serviceName) + ); + + const allServiceNames = foundServiceNames.concat( + servicesWithOnlyMetricDocuments.map(({ serviceName }) => serviceName) + ); // make sure to exclude health statuses from services // that are not found in APM data const matchedHealthStatuses = healthStatuses.filter(({ serviceName }) => - apmServices.includes(serviceName) + allServiceNames.includes(serviceName) ); - const allMetrics = [...transactionStats, ...matchedHealthStatuses]; - - return joinByKey(allMetrics, 'serviceName'); + return joinByKey( + asMutableArray([ + ...transactionStats, + ...servicesWithOnlyMetricDocuments, + ...matchedHealthStatuses, + ] as const), + 'serviceName' + ); }); } diff --git a/x-pack/test/apm_api_integration/tests/services/top_services.ts b/x-pack/test/apm_api_integration/tests/services/top_services.ts index 3afaec653fcb..a8aa673a285c 100644 --- a/x-pack/test/apm_api_integration/tests/services/top_services.ts +++ b/x-pack/test/apm_api_integration/tests/services/top_services.ts @@ -261,6 +261,51 @@ export default function ApiTest({ getService }: FtrProviderContext) { } ); + registry.when( + 'APM Services Overview with a basic license when data is loaded excluding transaction events', + { config: 'basic', archives: [archiveName] }, + () => { + it('includes services that only report metric data', async () => { + interface Response { + status: number; + body: APIReturnType<'GET /api/apm/services'>; + } + + const [unfilteredResponse, filteredResponse] = await Promise.all([ + supertest.get( + `/api/apm/services?start=${start}&end=${end}&uiFilters={}` + ) as Promise, + supertest.get( + `/api/apm/services?start=${start}&end=${end}&uiFilters=${encodeURIComponent( + '{ "kuery": "not (processor.event:transaction)" }' + )}` + ) as Promise, + ]); + + expect(unfilteredResponse.body.items.length).to.be.greaterThan(0); + + const unfilteredServiceNames = unfilteredResponse.body.items + .map((item) => item.serviceName) + .sort(); + + const filteredServiceNames = filteredResponse.body.items + .map((item) => item.serviceName) + .sort(); + + expect(unfilteredServiceNames).to.eql(filteredServiceNames); + + expect( + filteredResponse.body.items.every((item) => { + // make sure it did not query transaction data + return isEmpty(item.avgResponseTime); + }) + ).to.be(true); + + expect(filteredResponse.body.items.every((item) => !!item.agentName)).to.be(true); + }); + } + ); + registry.when( 'APM Services overview with a trial license when data is loaded', { config: 'trial', archives: [archiveName] },