[Metrics UI] Optimizations for Snapshot and Inventory Metadata (elast…

…ic#83596) * [Metrics UI] Add time range to inventory metadata request * Adding optimizations for snapshot request * Adding sorting to dataset request * Only query inventory metadata for AWS * moving check inside getCloudMetadata * removing unused deps
chrisronline · Nov 19, 2020 · d6818e6 · d6818e6
1 parent 6a0f5c1
commit d6818e6
Show file tree

Hide file tree

Showing 11 changed files with 69 additions and 20 deletions.
diff --git a/x-pack/plugins/infra/common/http_api/inventory_meta_api.ts b/x-pack/plugins/infra/common/http_api/inventory_meta_api.ts
@@ -21,6 +21,7 @@ export const InventoryMetaResponseRT = rt.type({
 export const InventoryMetaRequestRT = rt.type({
   sourceId: rt.string,
   nodeType: ItemTypeRT,
+  currentTime: rt.number,
 });
 
 export type InventoryMetaRequest = rt.TypeOf<typeof InventoryMetaRequestRT>;

diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout.tsx
@@ -124,7 +124,7 @@ export const Layout = () => {
               <>
                 <TopActionContainer ref={topActionMeasureRef}>
                   <EuiFlexGroup justifyContent="spaceBetween" alignItems="center" gutterSize="m">
-                    <Toolbar nodeType={nodeType} />
+                    <Toolbar nodeType={nodeType} currentTime={currentTime} />
                     <EuiFlexItem grow={false}>
                       <IntervalLabel intervalAsString={intervalAsString} />
                     </EuiFlexItem>

diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/toolbars/toolbar.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/toolbars/toolbar.tsx
@@ -54,11 +54,12 @@ const wrapToolbarItems = (
 
 interface Props {
   nodeType: InventoryItemType;
+  currentTime: number;
 }
 
-export const Toolbar = ({ nodeType }: Props) => {
+export const Toolbar = ({ nodeType, currentTime }: Props) => {
   const { sourceId } = useSourceContext();
-  const { accounts, regions } = useInventoryMeta(sourceId, nodeType);
+  const { accounts, regions } = useInventoryMeta(sourceId, nodeType, currentTime);
   const ToolbarItems = findToolbar(nodeType);
   return wrapToolbarItems(ToolbarItems, accounts, regions);
 };
diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_inventory_meta.ts b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_inventory_meta.ts
@@ -15,7 +15,11 @@ import {
 } from '../../../../../common/http_api/inventory_meta_api';
 import { InventoryItemType } from '../../../../../common/inventory_models/types';
 
-export function useInventoryMeta(sourceId: string, nodeType: InventoryItemType) {
+export function useInventoryMeta(
+  sourceId: string,
+  nodeType: InventoryItemType,
+  currentTime: number
+) {
   const decodeResponse = (response: any) => {
     return pipe(
       InventoryMetaResponseRT.decode(response),
@@ -29,6 +33,7 @@ export function useInventoryMeta(sourceId: string, nodeType: InventoryItemType)
     JSON.stringify({
       sourceId,
       nodeType,
+      currentTime,
     }),
     decodeResponse
   );

diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_snaphot.ts b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_snaphot.ts
@@ -44,7 +44,7 @@ export function useSnapshot(
     interval: '1m',
     to: currentTime,
     from: currentTime - 1200 * 1000,
-    lookbackSize: 20,
+    lookbackSize: 5,
   };
 
   const { error, loading, response, makeRequest } = useHTTPRequest<SnapshotNodeResponse>(

diff --git a/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts b/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts
@@ -33,7 +33,7 @@ export const initInventoryMetaRoute = (libs: InfraBackendLibs) => {
     },
     async (requestContext, request, response) => {
       try {
-        const { sourceId, nodeType } = pipe(
+        const { sourceId, nodeType, currentTime } = pipe(
           InventoryMetaRequestRT.decode(request.body),
           fold(throwErrors(Boom.badRequest), identity)
         );
@@ -42,11 +42,13 @@ export const initInventoryMetaRoute = (libs: InfraBackendLibs) => {
           requestContext.core.savedObjects.client,
           sourceId
         );
+
         const awsMetadata = await getCloudMetadata(
           framework,
           requestContext,
           configuration,
-          nodeType
+          nodeType,
+          currentTime
         );
 
         return response.ok({

diff --git a/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts b/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts
@@ -25,9 +25,18 @@ export const getCloudMetadata = async (
   framework: KibanaFramework,
   req: RequestHandlerContext,
   sourceConfiguration: InfraSourceConfiguration,
-  nodeType: InventoryItemType
+  nodeType: InventoryItemType,
+  currentTime: number
 ): Promise<CloudMetaData> => {
   const model = findInventoryModel(nodeType);
+  // Only run this for AWS modules, eventually we might have more.
+  if (model.requiredModule !== 'aws') {
+    return {
+      accounts: [],
+      projects: [],
+      regions: [],
+    };
+  }
 
   const metricQuery = {
     allowNoIndices: true,
@@ -36,7 +45,18 @@ export const getCloudMetadata = async (
     body: {
       query: {
         bool: {
-          must: [{ match: { 'event.module': model.requiredModule } }],
+          must: [
+            {
+              range: {
+                [sourceConfiguration.fields.timestamp]: {
+                  gte: currentTime - 86400000, // 24 hours ago
+                  lte: currentTime,
+                  format: 'epoch_millis',
+                },
+              },
+            },
+            { match: { 'event.module': model.requiredModule } },
+          ],
         },
       },
       size: 0,

diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/find_interval_for_metrics.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/find_interval_for_metrics.ts
@@ -34,7 +34,8 @@ export const findIntervalForMetrics = async (
 
   const modules = await Promise.all(
     fields.map(
-      async (field) => await getDatasetForField(client, field as string, options.indexPattern)
+      async (field) =>
+        await getDatasetForField(client, field as string, options.indexPattern, options.timerange)
     )
   );
 

diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts
@@ -17,17 +17,34 @@ interface EventDatasetHit {
 export const getDatasetForField = async (
   client: ESSearchClient,
   field: string,
-  indexPattern: string
+  indexPattern: string,
+  timerange: { field: string; to: number; from: number }
 ) => {
   const params = {
     allowNoIndices: true,
     ignoreUnavailable: true,
     terminateAfter: 1,
     index: indexPattern,
     body: {
-      query: { exists: { field } },
+      query: {
+        bool: {
+          filter: [
+            { exists: { field } },
+            {
+              range: {
+                [timerange.field]: {
+                  gte: timerange.from,
+                  lte: timerange.to,
+                  format: 'epoch_millis',
+                },
+              },
+            },
+          ],
+        },
+      },
       size: 1,
       _source: ['event.dataset'],
+      sort: [{ [timerange.field]: { order: 'desc' } }],
     },
   };
 

diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/create_timerange_with_interval.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/create_timerange_with_interval.ts
@@ -75,7 +75,10 @@ const aggregationsToModules = async (
   const fields = await Promise.all(
     uniqueFields.map(
       async (field) =>
-        await getDatasetForField(client, field as string, options.sourceConfiguration.metricAlias)
+        await getDatasetForField(client, field as string, options.sourceConfiguration.metricAlias, {
+          ...options.timerange,
+          field: options.sourceConfiguration.fields.timestamp,
+        })
     )
   );
   return fields.filter((f) => f) as string[];

diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/get_nodes.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/get_nodes.ts
@@ -23,12 +23,11 @@ export const getNodes = async (
     snapshotRequest
   );
   const metricsApiResponse = await queryAllData(client, metricsApiRequest);
-  return copyMissingMetrics(
-    transformMetricsApiResponseToSnapshotResponse(
-      metricsApiRequest,
-      snapshotRequest,
-      source,
-      metricsApiResponse
-    )
+  const snapshotResponse = transformMetricsApiResponseToSnapshotResponse(
+    metricsApiRequest,
+    snapshotRequest,
+    source,
+    metricsApiResponse
   );
+  return copyMissingMetrics(snapshotResponse);
 };