[APM] Remove rate aggregations (#114187)

* fixing throughput chart api * change backends * adding intervalString to the observability callback functions * fixing transaction group detailed stats * fixing tests * fixing test * fixing obs tests * fixing tests * adding tests * fixing ci * using data generator * changing name * fixing i18n * updating opbs test to use data generator * fixing api tests * fixing tests * using data generator to run the tests * fixing tests * fixing test
elastic · Oct 14, 2021 · d7e7dbf · d7e7dbf
1 parent 7fb99fb
commit d7e7dbf
Show file tree

Hide file tree

Showing 53 changed files with 2,675 additions and 2,342 deletions.
diff --git a/x-pack/plugins/apm/common/utils/formatters/duration.ts b/x-pack/plugins/apm/common/utils/formatters/duration.ts
@@ -13,8 +13,6 @@ import { asDecimalOrInteger, asInteger, asDecimal } from './formatters';
 import { TimeUnit } from './datetime';
 import { Maybe } from '../../../typings/common';
 import { isFiniteNumber } from '../is_finite_number';
-// eslint-disable-next-line @kbn/eslint/no-restricted-paths
-import type { ThroughputUnit } from '../../../server/lib/helpers/calculate_throughput';
 
 interface FormatterOptions {
   defaultValue?: string;
@@ -182,13 +180,12 @@ export function asTransactionRate(value: Maybe<number>) {
   });
 }
 
-export function asExactTransactionRate(value: number, unit: ThroughputUnit) {
+export function asExactTransactionRate(value: number) {
   return i18n.translate('xpack.apm.exactTransactionRateLabel', {
-    defaultMessage: `{value} { unit, select, minute {tpm} other {tps} }`,
-    values: { value: asDecimalOrInteger(value), unit },
+    defaultMessage: `{value} tpm`,
+    values: { value: asDecimalOrInteger(value) },
   });
 }
-
 /**
  * Converts value and returns it formatted - 00 unit
  */

diff --git a/.../plugins/apm/public/components/app/service_overview/service_overview_throughput_chart.tsx b/.../plugins/apm/public/components/app/service_overview/service_overview_throughput_chart.tsx
@@ -135,29 +135,16 @@ export function ServiceOverviewThroughputChart({
                 'xpack.apm.serviceOverview.throughtputChartTitle',
                 { defaultMessage: 'Throughput' }
               )}
-              {data.throughputUnit === 'second'
-                ? i18n.translate(
-                    'xpack.apm.serviceOverview.throughtputPerSecondChartTitle',
-                    { defaultMessage: ' (per second)' }
-                  )
-                : ''}
             </h2>
           </EuiTitle>
         </EuiFlexItem>
 
         <EuiFlexItem grow={false}>
           <EuiIconTip
-            content={
-              data.throughputUnit === 'minute'
-                ? i18n.translate('xpack.apm.serviceOverview.tpmHelp', {
-                    defaultMessage:
-                      'Throughput is measured in transactions per minute (tpm)',
-                  })
-                : i18n.translate('xpack.apm.serviceOverview.tpsHelp', {
-                    defaultMessage:
-                      'Throughput is measured in transactions per second (tps)',
-                  })
-            }
+            content={i18n.translate('xpack.apm.serviceOverview.tpmHelp', {
+              defaultMessage:
+                'Throughput is measured in transactions per minute (tpm)',
+            })}
             position="right"
           />
         </EuiFlexItem>
@@ -169,7 +156,7 @@ export function ServiceOverviewThroughputChart({
         showAnnotations={false}
         fetchStatus={status}
         timeseries={timeseries}
-        yLabelFormat={(y) => asExactTransactionRate(y, data.throughputUnit)}
+        yLabelFormat={asExactTransactionRate}
         customTheme={comparisonChartTheme}
       />
     </EuiPanel>

diff --git a/x-pack/plugins/apm/public/services/rest/apm_observability_overview_fetchers.test.ts b/x-pack/plugins/apm/public/services/rest/apm_observability_overview_fetchers.test.ts
@@ -23,7 +23,8 @@ describe('Observability dashboard data', () => {
       start: 'now-15m',
       end: 'now',
     },
-    bucketSize: '600s',
+    intervalString: '600s',
+    bucketSize: 600,
   };
   afterEach(() => {
     callApmApiMock.mockClear();

diff --git a/x-pack/plugins/apm/public/services/rest/apm_observability_overview_fetchers.ts b/x-pack/plugins/apm/public/services/rest/apm_observability_overview_fetchers.ts
@@ -15,6 +15,7 @@ export const fetchObservabilityOverviewPageData = async ({
   absoluteTime,
   relativeTime,
   bucketSize,
+  intervalString,
 }: FetchDataParams): Promise<ApmFetchDataResponse> => {
   const data = await callApmApi({
     endpoint: 'GET /internal/apm/observability_overview',
@@ -24,6 +25,7 @@ export const fetchObservabilityOverviewPageData = async ({
         start: new Date(absoluteTime.start).toISOString(),
         end: new Date(absoluteTime.end).toISOString(),
         bucketSize,
+        intervalString,
       },
     },
   });

diff --git a/x-pack/plugins/apm/server/index.ts b/x-pack/plugins/apm/server/index.ts
@@ -38,7 +38,7 @@ const configSchema = schema.object({
       schema.literal(SearchAggregatedTransactionSetting.always),
       schema.literal(SearchAggregatedTransactionSetting.never),
     ],
-    { defaultValue: SearchAggregatedTransactionSetting.auto }
+    { defaultValue: SearchAggregatedTransactionSetting.never }
   ),
   telemetryCollectionEnabled: schema.boolean({ defaultValue: true }),
   metricsInterval: schema.number({ defaultValue: 30 }),

diff --git a/x-pack/plugins/apm/server/lib/backends/get_throughput_charts_for_backend.ts b/x-pack/plugins/apm/server/lib/backends/get_throughput_charts_for_backend.ts
@@ -13,8 +13,9 @@ import { environmentQuery } from '../../../common/utils/environment_query';
 import { kqlQuery, rangeQuery } from '../../../../observability/server';
 import { ProcessorEvent } from '../../../common/processor_event';
 import { Setup } from '../helpers/setup_request';
-import { getMetricsDateHistogramParams } from '../helpers/metrics';
 import { getOffsetInMs } from '../../../common/utils/get_offset_in_ms';
+import { getBucketSize } from '../helpers/get_bucket_size';
+import { calculateThroughputWithInterval } from '../helpers/calculate_throughput';
 
 export async function getThroughputChartsForBackend({
   backendName,
@@ -41,6 +42,12 @@ export async function getThroughputChartsForBackend({
     offset,
   });
 
+  const { intervalString, bucketSize } = getBucketSize({
+    start: startWithOffset,
+    end: endWithOffset,
+    minBucketSize: 60,
+  });
+
   const response = await apmEventClient.search('get_throughput_for_backend', {
     apm: {
       events: [ProcessorEvent.metric],
@@ -59,16 +66,16 @@ export async function getThroughputChartsForBackend({
       },
       aggs: {
         timeseries: {
-          date_histogram: getMetricsDateHistogramParams({
-            start: startWithOffset,
-            end: endWithOffset,
-            metricsInterval: 60,
-          }),
+          date_histogram: {
+            field: '@timestamp',
+            fixed_interval: intervalString,
+            min_doc_count: 0,
+            extended_bounds: { min: startWithOffset, max: endWithOffset },
+          },
           aggs: {
-            throughput: {
-              rate: {
+            spanDestinationLatencySum: {
+              sum: {
                 field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT,
-                unit: 'minute',
               },
             },
           },
@@ -81,7 +88,10 @@ export async function getThroughputChartsForBackend({
     response.aggregations?.timeseries.buckets.map((bucket) => {
       return {
         x: bucket.key + offsetInMs,
-        y: bucket.throughput.value,
+        y: calculateThroughputWithInterval({
+          bucketSize,
+          value: bucket.spanDestinationLatencySum.value || 0,
+        }),
       };
     }) ?? []
   );

diff --git a/x-pack/plugins/apm/server/lib/helpers/calculate_throughput.ts b/x-pack/plugins/apm/server/lib/helpers/calculate_throughput.ts
@@ -5,6 +5,9 @@
  * 2.0.
  */
 
+/**
+ * @deprecated use calculateThroughputWithRange instead
+ */
 export function calculateThroughput({
   start,
   end,
@@ -18,6 +21,30 @@ export function calculateThroughput({
   return value / durationAsMinutes;
 }
 
+export function calculateThroughputWithRange({
+  start,
+  end,
+  value,
+}: {
+  start: number;
+  end: number;
+  value: number;
+}) {
+  const durationAsMinutes = (end - start) / 1000 / 60;
+  return value / durationAsMinutes;
+}
+
+export function calculateThroughputWithInterval({
+  bucketSize,
+  value,
+}: {
+  bucketSize: number;
+  value: number;
+}) {
+  const durationAsMinutes = bucketSize / 60;
+  return value / durationAsMinutes;
+}
+
 export type ThroughputUnit = 'minute' | 'second';
 export function getThroughputUnit(bucketSize: number): ThroughputUnit {
   return bucketSize >= 60 ? 'minute' : 'second';

diff --git a/x-pack/plugins/apm/server/lib/observability_overview/get_transactions_per_minute.ts b/x-pack/plugins/apm/server/lib/observability_overview/get_transactions_per_minute.ts
@@ -16,17 +16,22 @@ import {
   getDocumentTypeFilterForAggregatedTransactions,
   getProcessorEventForAggregatedTransactions,
 } from '../helpers/aggregated_transactions';
-import { calculateThroughput } from '../helpers/calculate_throughput';
+import {
+  calculateThroughputWithInterval,
+  calculateThroughputWithRange,
+} from '../helpers/calculate_throughput';
 
 export async function getTransactionsPerMinute({
   setup,
   bucketSize,
   searchAggregatedTransactions,
   start,
   end,
+  intervalString,
 }: {
   setup: Setup;
-  bucketSize: string;
+  bucketSize: number;
+  intervalString: string;
   searchAggregatedTransactions: boolean;
   start: number;
   end: number;
@@ -64,12 +69,9 @@ export async function getTransactionsPerMinute({
               timeseries: {
                 date_histogram: {
                   field: '@timestamp',
-                  fixed_interval: bucketSize,
+                  fixed_interval: intervalString,
                   min_doc_count: 0,
                 },
-                aggs: {
-                  throughput: { rate: { unit: 'minute' as const } },
-                },
               },
             },
           },
@@ -90,15 +92,18 @@ export async function getTransactionsPerMinute({
     ) || aggregations.transactionType.buckets[0];
 
   return {
-    value: calculateThroughput({
+    value: calculateThroughputWithRange({
       start,
       end,
       value: topTransactionTypeBucket?.doc_count || 0,
     }),
     timeseries:
       topTransactionTypeBucket?.timeseries.buckets.map((bucket) => ({
         x: bucket.key,
-        y: bucket.throughput.value,
+        y: calculateThroughputWithInterval({
+          bucketSize,
+          value: bucket.doc_count,
+        }),
       })) || [],
   };
 }
diff --git a/x-pack/plugins/apm/server/lib/services/get_service_transaction_group_detailed_statistics.ts b/x-pack/plugins/apm/server/lib/services/get_service_transaction_group_detailed_statistics.ts
@@ -125,11 +125,6 @@ export async function getServiceTransactionGroupDetailedStatistics({
                   },
                 },
                 aggs: {
-                  throughput_rate: {
-                    rate: {
-                      unit: 'minute',
-                    },
-                  },
                   ...getLatencyAggregation(latencyAggregationType, field),
                   [EVENT_OUTCOME]: {
                     terms: {
@@ -160,7 +155,7 @@ export async function getServiceTransactionGroupDetailedStatistics({
     }));
     const throughput = bucket.timeseries.buckets.map((timeseriesBucket) => ({
       x: timeseriesBucket.key,
-      y: timeseriesBucket.throughput_rate.value,
+      y: timeseriesBucket.doc_count, // sparklines only shows trend (no axis)
     }));
     const errorRate = bucket.timeseries.buckets.map((timeseriesBucket) => ({
       x: timeseriesBucket.key,

diff --git a/x-pack/plugins/apm/server/lib/services/get_throughput.ts b/x-pack/plugins/apm/server/lib/services/get_throughput.ts
@@ -18,6 +18,7 @@ import {
   getProcessorEventForAggregatedTransactions,
 } from '../helpers/aggregated_transactions';
 import { Setup } from '../helpers/setup_request';
+import { calculateThroughputWithInterval } from '../helpers/calculate_throughput';
 
 interface Options {
   environment: string;
@@ -30,7 +31,7 @@ interface Options {
   start: number;
   end: number;
   intervalString: string;
-  throughputUnit: 'minute' | 'second';
+  bucketSize: number;
 }
 
 export async function getThroughput({
@@ -44,7 +45,7 @@ export async function getThroughput({
   start,
   end,
   intervalString,
-  throughputUnit,
+  bucketSize,
 }: Options) {
   const { apmEventClient } = setup;
 
@@ -86,13 +87,6 @@ export async function getThroughput({
             min_doc_count: 0,
             extended_bounds: { min: start, max: end },
           },
-          aggs: {
-            throughput: {
-              rate: {
-                unit: throughputUnit,
-              },
-            },
-          },
         },
       },
     },
@@ -107,7 +101,10 @@ export async function getThroughput({
     response.aggregations?.timeseries.buckets.map((bucket) => {
       return {
         x: bucket.key,
-        y: bucket.throughput.value,
+        y: calculateThroughputWithInterval({
+          bucketSize,
+          value: bucket.doc_count,
+        }),
       };
     }) ?? []
   );

diff --git a/x-pack/plugins/apm/server/routes/observability_overview.ts b/x-pack/plugins/apm/server/routes/observability_overview.ts
@@ -6,6 +6,7 @@
  */
 
 import * as t from 'io-ts';
+import { toNumberRt } from '@kbn/io-ts-utils';
 import { setupRequest } from '../lib/helpers/setup_request';
 import { getServiceCount } from '../lib/observability_overview/get_service_count';
 import { getTransactionsPerMinute } from '../lib/observability_overview/get_transactions_per_minute';
@@ -28,12 +29,15 @@ const observabilityOverviewHasDataRoute = createApmServerRoute({
 const observabilityOverviewRoute = createApmServerRoute({
   endpoint: 'GET /internal/apm/observability_overview',
   params: t.type({
-    query: t.intersection([rangeRt, t.type({ bucketSize: t.string })]),
+    query: t.intersection([
+      rangeRt,
+      t.type({ bucketSize: toNumberRt, intervalString: t.string }),
+    ]),
   }),
   options: { tags: ['access:apm'] },
   handler: async (resources) => {
     const setup = await setupRequest(resources);
-    const { bucketSize, start, end } = resources.params.query;
+    const { bucketSize, intervalString, start, end } = resources.params.query;
 
     const searchAggregatedTransactions = await getSearchAggregatedTransactions({
       apmEventClient: setup.apmEventClient,
@@ -57,6 +61,7 @@ const observabilityOverviewRoute = createApmServerRoute({
           searchAggregatedTransactions,
           start,
           end,
+          intervalString,
         }),
       ]);
       return { serviceCount, transactionPerMinute };