[ProfilingxAPM] Link APM from Profiling UI (#180677)

closes #178719

A new ES API has been created to support linking APM from the Profiling
UI. It's called `topN/functions`. The new API allows grouping fields. So
we first fetch functions grouping by `service.name` and when the user
opens the APM Transactions we make another request grouping by
`transaction.name`.

A new Advanced setting was created to toggle the old API on (fetch
functions from Stacktraces API): It's turned off by default.
<img width="1235" alt="Screenshot 2024-04-12 at 10 39 36"
src="https://github.com/elastic/kibana/assets/55978943/ee6e7731-2f44-43ca-9793-23ba87e22e6e">

When there are services on the selected function:
*If we cannot find the transaction, we show `N/A`.
<img width="933" alt="Screenshot 2024-04-12 at 10 16 34"
src="https://github.com/elastic/kibana/assets/55978943/2c5dbf60-3a47-4f4c-a46d-8a0984e0e482">

When there are **no** services on the selected function:
*hide the APM transactions section
<img width="921" alt="Screenshot 2024-04-12 at 10 59 14"
src="https://github.com/elastic/kibana/assets/55978943/3fc4c5b1-da62-47c8-97a8-8bcbd1ae1b75">

--
Performance boost:
The new API is faster than the Stacktraces API, especially because
there's no logic on the Kibana side.
Stacktraces API:
<img width="1210" alt="Screenshot 2024-04-12 at 10 50 26"
src="https://github.com/elastic/kibana/assets/55978943/158d73d1-ed91-4652-97c1-c7c3328d5e3d">

TopN/Functions API:
<img width="1195" alt="Screenshot 2024-04-12 at 10 51 20"
src="https://github.com/elastic/kibana/assets/55978943/2de4ef46-eb8a-4557-b7b8-a1c2fed6fd8a">

---------

Co-authored-by: kibanamachine <[email protected]>

docs/management/advanced-options.asciidoc

@@ -486,6 +486,9 @@ If you're enrolled in the AWS Enterprise Discount Program (EDP), enter your disc
 [[observability-profiling-azure-cost-discount-rate]]`observability:profilingAzureCostDiscountRate`::
 If you have an Azure Enterprise Agreement with Microsoft, enter your discount rate to update the profiling cost calculation.
 
+[[observability-profiling-use-topNFunctions-from-stacktraces]]`observability:profilingFetchTopNFunctionsFromStacktraces`::
+Switch to fetch the TopN Functions from the Stacktraces API.
+
 [[observability-profiling-cost-per-vcpu-per-hour]]`observability:profilingCostPervCPUPerHour`::
 Default Hourly Cost per CPU Core for machines not on AWS or Azure.
 

packages/kbn-profiling-utils/common/es_functions.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export interface Frame {
+  frame_type: number;
+  inline: boolean;
+  address_or_line: number;
+  function_name: string;
+  file_name: string;
+  line_number: number;
+  executable_file_name: string;
+}
+
+interface TopNFunction {
+  id: string;
+  rank: number;
+  frame: Frame;
+  sub_groups: Record<string, number>;
+  self_count: number;
+  total_count: number;
+  self_annual_co2_tons: number;
+  total_annual_co2_tons: number;
+  self_annual_costs_usd: number;
+  total_annual_costs_usd: number;
+}
+
+export interface ESTopNFunctions {
+  self_count: number;
+  total_count: number;
+  self_annual_co2_tons: number;
+  self_annual_cost_usd: number;
+  topn: TopNFunction[];
+}
+
+export type AggregationField = 'service.name' | 'transaction.name';

packages/kbn-profiling-utils/common/functions.ts

@@ -49,6 +49,7 @@ type TopNFunction = Pick<
 > & {
   Id: string;
   Rank: number;
+  subGroups: Record<string, number>;
 };
 
 export interface TopNFunctions {
@@ -207,6 +208,7 @@ export function createTopNFunctions({
         selfAnnualCostUSD: frameAndCount.selfAnnualCostUSD,
         totalAnnualCO2kgs: frameAndCount.totalAnnualCO2kgs,
         totalAnnualCostUSD: frameAndCount.totalAnnualCostUSD,
+        subGroups: {},
       };
     });
 

packages/kbn-profiling-utils/index.ts

@@ -53,3 +53,4 @@ export type {
 } from './common/profiling';
 export type { ProfilingStatus } from './common/profiling_status';
 export type { TopNFunctions } from './common/functions';
+export type { AggregationField, ESTopNFunctions } from './common/es_functions';

src/plugins/kibana_usage_collection/server/collectors/management/schema.ts

@@ -671,4 +671,8 @@ export const stackManagementSchema: MakeSchemaFrom<UsageStats> = {
     type: 'keyword',
     _meta: { description: 'Non-default value of setting.' },
   },
+  'observability:profilingFetchTopNFunctionsFromStacktraces': {
+    type: 'boolean',
+    _meta: { description: 'Non-default value of setting.' },
+  },
 };

src/plugins/kibana_usage_collection/server/collectors/management/types.ts

@@ -176,4 +176,5 @@ export interface UsageStats {
   'observability:apmEnableTransactionProfiling': boolean;
   'devTools:enablePersistentConsole': boolean;
   'aiAssistant:preferredAIAssistantType': string;
+  'observability:profilingFetchTopNFunctionsFromStacktraces': boolean;
 }

src/plugins/telemetry/schema/oss_plugins.json

@@ -10451,6 +10451,12 @@
           "_meta": {
             "description": "Non-default value of setting."
           }
+        },
+        "observability:profilingFetchTopNFunctionsFromStacktraces": {
+          "type": "boolean",
+          "_meta": {
+            "description": "Non-default value of setting."
+          }
         }
       }
     },
@@ -11887,4 +11893,4 @@
       }
     }
   }
-}
+}

x-pack/plugins/observability_solution/apm/public/components/app/transaction_details_link/index.tsx

@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EuiEmptyPrompt } from '@elastic/eui';
+import React from 'react';
+import { i18n } from '@kbn/i18n';
+import { Redirect } from 'react-router-dom';
+import { css } from '@emotion/react';
+import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher';
+import { getRedirectToTransactionDetailPageUrl } from '../trace_link/get_redirect_to_transaction_detail_page_url';
+import { useApmParams } from '../../../hooks/use_apm_params';
+import { useTimeRange } from '../../../hooks/use_time_range';
+
+export function TransactionDetailsByNameLink() {
+  const {
+    query: {
+      rangeFrom = 'now-15m',
+      rangeTo = 'now',
+      transactionName,
+      serviceName,
+    },
+  } = useApmParams('/link-to/transaction');
+
+  const { start, end } = useTimeRange({ rangeFrom, rangeTo });
+
+  const { data = { transaction: null }, status } = useFetcher(
+    (callApmApi) => {
+      return callApmApi('GET /internal/apm/transactions', {
+        params: {
+          query: {
+            start,
+            end,
+            transactionName,
+            serviceName,
+          },
+        },
+      });
+    },
+    [start, end, transactionName, serviceName]
+  );
+
+  if (status === FETCH_STATUS.SUCCESS) {
+    if (data.transaction) {
+      return (
+        <Redirect
+          to={getRedirectToTransactionDetailPageUrl({
+            transaction: data.transaction,
+            rangeFrom,
+            rangeTo,
+          })}
+        />
+      );
+    }
+
+    return (
+      <div
+        css={css`
+          height: 100%;
+          display: flex;
+        `}
+      >
+        <EuiEmptyPrompt
+          iconType="apmTrace"
+          title={
+            <h2>
+              {i18n.translate(
+                'xpack.apm.transactionDetailsLink.h2.transactionNotFound',
+                { defaultMessage: 'No transaction found' }
+              )}
+            </h2>
+          }
+        />
+      </div>
+    );
+  }
+
+  return (
+    <div
+      css={css`
+        height: 100%;
+        display: flex;
+      `}
+    >
+      <EuiEmptyPrompt
+        iconType="apmTrace"
+        title={
+          <h2>
+            {i18n.translate(
+              'xpack.apm.transactionDetailsLink.h2.fetchingTransactionLabel',
+              { defaultMessage: 'Fetching transaction...' }
+            )}
+          </h2>
+        }
+      />
+    </div>
+  );
+}

x-pack/plugins/observability_solution/apm/public/components/routing/apm_route_config.tsx

@@ -23,6 +23,7 @@ import { ApmMainTemplate } from './templates/apm_main_template';
 import { ServiceGroupsList } from '../app/service_groups';
 import { offsetRt } from '../../../common/comparison_rt';
 import { diagnosticsRoute } from '../app/diagnostics';
+import { TransactionDetailsByNameLink } from '../app/transaction_details_link';
 
 const ServiceGroupsTitle = i18n.translate(
   'xpack.apm.views.serviceGroups.title',
@@ -34,6 +35,18 @@ const ServiceGroupsTitle = i18n.translate(
  * creates the routes.
  */
 const apmRoutes = {
+  '/link-to/transaction': {
+    element: <TransactionDetailsByNameLink />,
+    params: t.type({
+      query: t.intersection([
+        t.type({ transactionName: t.string, serviceName: t.string }),
+        t.partial({
+          rangeFrom: t.string,
+          rangeTo: t.string,
+        }),
+      ]),
+    }),
+  },
   '/link-to/transaction/{transactionId}': {
     element: <TransactionLink />,
     params: t.intersection([
@@ -69,7 +82,12 @@ const apmRoutes = {
   },
   '/': {
     element: (
-      <Breadcrumb title="APM" href="/">
+      <Breadcrumb
+        title={i18n.translate('xpack.apm..breadcrumb.apmLabel', {
+          defaultMessage: 'APM',
+        })}
+        href="/"
+      >
         <Outlet />
       </Breadcrumb>
     ),

x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts

@@ -36,6 +36,7 @@ import {
 import { getSpan } from '../transactions/get_span';
 import { Transaction } from '../../../typings/es_schemas/ui/transaction';
 import { Span } from '../../../typings/es_schemas/ui/span';
+import { getTransactionByName } from '../transactions/get_transaction_by_name';
 
 const tracesRoute = createApmServerRoute({
   endpoint: 'GET /internal/apm/traces',
@@ -186,6 +187,42 @@ const transactionByIdRoute = createApmServerRoute({
   },
 });
 
+const transactionByNameRoute = createApmServerRoute({
+  endpoint: 'GET /internal/apm/transactions',
+  params: t.type({
+    query: t.intersection([
+      rangeRt,
+      t.type({
+        transactionName: t.string,
+        serviceName: t.string,
+      }),
+    ]),
+  }),
+  options: { tags: ['access:apm'] },
+  handler: async (
+    resources
+  ): Promise<{
+    transaction: Transaction;
+  }> => {
+    const {
+      params: {
+        query: { start, end, transactionName, serviceName },
+      },
+    } = resources;
+
+    const apmEventClient = await getApmEventClient(resources);
+    return {
+      transaction: await getTransactionByName({
+        transactionName,
+        apmEventClient,
+        start,
+        end,
+        serviceName,
+      }),
+    };
+  },
+});
+
 const findTracesRoute = createApmServerRoute({
   endpoint: 'GET /internal/apm/traces/find',
   params: t.type({
@@ -338,4 +375,5 @@ export const traceRouteRepository = {
   ...aggregatedCriticalPathRoute,
   ...transactionFromTraceByIdRoute,
   ...spanFromTraceByIdRoute,
+  ...transactionByNameRoute,
 };

x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { rangeQuery } from '@kbn/observability-plugin/server';
+import { ApmDocumentType } from '../../../../common/document_type';
+import {
+  SERVICE_NAME,
+  TRANSACTION_NAME,
+} from '../../../../common/es_fields/apm';
+import { RollupInterval } from '../../../../common/rollup';
+import { asMutableArray } from '../../../../common/utils/as_mutable_array';
+import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
+
+export async function getTransactionByName({
+  transactionName,
+  serviceName,
+  apmEventClient,
+  start,
+  end,
+}: {
+  transactionName: string;
+  serviceName: string;
+  apmEventClient: APMEventClient;
+  start: number;
+  end: number;
+}) {
+  const resp = await apmEventClient.search('get_transaction', {
+    apm: {
+      sources: [
+        {
+          documentType: ApmDocumentType.TransactionEvent,
+          rollupInterval: RollupInterval.None,
+        },
+      ],
+    },
+    body: {
+      track_total_hits: false,
+      size: 1,
+      terminate_after: 1,
+      query: {
+        bool: {
+          filter: asMutableArray([
+            { term: { [TRANSACTION_NAME]: transactionName } },
+            { term: { [SERVICE_NAME]: serviceName } },
+            ...rangeQuery(start, end),
+          ]),
+        },
+      },
+    },
+  });
+
+  return resp.hits.hits[0]?._source;
+}

x-pack/plugins/observability_solution/observability/common/index.ts

@@ -54,6 +54,7 @@ export {
   profilingAzureCostDiscountRate,
   apmEnableTransactionProfiling,
   apmEnableServiceInventoryTableSearchBar,
+  profilingFetchTopNFunctionsFromStacktraces,
 } from './ui_settings_keys';
 
 export {

x-pack/plugins/observability_solution/observability/common/ui_settings_keys.ts

@@ -43,3 +43,5 @@ export const profilingAWSCostDiscountRate = 'observability:profilingAWSCostDisco
 export const profilingCostPervCPUPerHour = 'observability:profilingCostPervCPUPerHour';
 export const profilingAzureCostDiscountRate = 'observability:profilingAzureCostDiscountRate';
 export const apmEnableTransactionProfiling = 'observability:apmEnableTransactionProfiling';
+export const profilingFetchTopNFunctionsFromStacktraces =
+  'observability:profilingFetchTopNFunctionsFromStacktraces';