From 0b5404c06fe1cdaa603d7d85e74fcb772849a98f Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Wed, 25 Nov 2020 10:26:04 -0700 Subject: [PATCH] [Metrics UI] Optimizations for Snapshot and Inventory Metadata (#83596) (#83751) * [Metrics UI] Add time range to inventory metadata request * Adding optimizations for snapshot request * Adding sorting to dataset request * Only query inventory metadata for AWS * moving check inside getCloudMetadata * removing unused deps Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../common/http_api/inventory_meta_api.ts | 1 + .../inventory_view/components/layout.tsx | 2 +- .../components/toolbars/toolbar.tsx | 5 ++-- .../hooks/use_inventory_meta.ts | 7 +++++- .../inventory_view/hooks/use_snaphot.ts | 2 +- .../server/routes/inventory_metadata/index.ts | 6 +++-- .../lib/get_cloud_metadata.ts | 24 +++++++++++++++++-- .../lib/find_interval_for_metrics.ts | 3 ++- .../lib/get_dataset_for_field.ts | 21 ++++++++++++++-- .../lib/create_timerange_with_interval.ts | 5 +++- .../server/routes/snapshot/lib/get_nodes.ts | 13 +++++----- 11 files changed, 69 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/infra/common/http_api/inventory_meta_api.ts b/x-pack/plugins/infra/common/http_api/inventory_meta_api.ts index 77de515c9cc46..43f3b2037e381 100644 --- a/x-pack/plugins/infra/common/http_api/inventory_meta_api.ts +++ b/x-pack/plugins/infra/common/http_api/inventory_meta_api.ts @@ -21,6 +21,7 @@ export const InventoryMetaResponseRT = rt.type({ export const InventoryMetaRequestRT = rt.type({ sourceId: rt.string, nodeType: ItemTypeRT, + currentTime: rt.number, }); export type InventoryMetaRequest = rt.TypeOf; diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout.tsx index 92aa015113b2a..2e5ddab77d374 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout.tsx @@ -124,7 +124,7 @@ export const Layout = () => { <> - + diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/toolbars/toolbar.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/toolbars/toolbar.tsx index e9ffc56d8c47f..7bcb1270c30a5 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/toolbars/toolbar.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/toolbars/toolbar.tsx @@ -54,11 +54,12 @@ const wrapToolbarItems = ( interface Props { nodeType: InventoryItemType; + currentTime: number; } -export const Toolbar = ({ nodeType }: Props) => { +export const Toolbar = ({ nodeType, currentTime }: Props) => { const { sourceId } = useSourceContext(); - const { accounts, regions } = useInventoryMeta(sourceId, nodeType); + const { accounts, regions } = useInventoryMeta(sourceId, nodeType, currentTime); const ToolbarItems = findToolbar(nodeType); return wrapToolbarItems(ToolbarItems, accounts, regions); }; diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_inventory_meta.ts b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_inventory_meta.ts index b038491690a13..01811eb21a110 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_inventory_meta.ts +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_inventory_meta.ts @@ -15,7 +15,11 @@ import { } from '../../../../../common/http_api/inventory_meta_api'; import { InventoryItemType } from '../../../../../common/inventory_models/types'; -export function useInventoryMeta(sourceId: string, nodeType: InventoryItemType) { +export function useInventoryMeta( + sourceId: string, + nodeType: InventoryItemType, + currentTime: number +) { const decodeResponse = (response: any) => { return pipe( InventoryMetaResponseRT.decode(response), @@ -29,6 +33,7 @@ export function useInventoryMeta(sourceId: string, nodeType: InventoryItemType) JSON.stringify({ sourceId, nodeType, + currentTime, }), decodeResponse ); diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_snaphot.ts b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_snaphot.ts index 0412b1b0defbf..4cfa8871b0dcc 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_snaphot.ts +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_snaphot.ts @@ -45,7 +45,7 @@ export function useSnapshot( interval: '1m', to: currentTime, from: currentTime - 1200 * 1000, - lookbackSize: 20, + lookbackSize: 5, }; const { error, loading, response, makeRequest } = useHTTPRequest( diff --git a/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts b/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts index 8b5271cb960c1..c784aa0f7d20b 100644 --- a/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts +++ b/x-pack/plugins/infra/server/routes/inventory_metadata/index.ts @@ -33,7 +33,7 @@ export const initInventoryMetaRoute = (libs: InfraBackendLibs) => { }, async (requestContext, request, response) => { try { - const { sourceId, nodeType } = pipe( + const { sourceId, nodeType, currentTime } = pipe( InventoryMetaRequestRT.decode(request.body), fold(throwErrors(Boom.badRequest), identity) ); @@ -42,11 +42,13 @@ export const initInventoryMetaRoute = (libs: InfraBackendLibs) => { requestContext.core.savedObjects.client, sourceId ); + const awsMetadata = await getCloudMetadata( framework, requestContext, configuration, - nodeType + nodeType, + currentTime ); return response.ok({ diff --git a/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts b/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts index b4288dae0c221..af9e9c5f57c5b 100644 --- a/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts +++ b/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts @@ -25,9 +25,18 @@ export const getCloudMetadata = async ( framework: KibanaFramework, req: RequestHandlerContext, sourceConfiguration: InfraSourceConfiguration, - nodeType: InventoryItemType + nodeType: InventoryItemType, + currentTime: number ): Promise => { const model = findInventoryModel(nodeType); + // Only run this for AWS modules, eventually we might have more. + if (model.requiredModule !== 'aws') { + return { + accounts: [], + projects: [], + regions: [], + }; + } const metricQuery = { allowNoIndices: true, @@ -36,7 +45,18 @@ export const getCloudMetadata = async ( body: { query: { bool: { - must: [{ match: { 'event.module': model.requiredModule } }], + must: [ + { + range: { + [sourceConfiguration.fields.timestamp]: { + gte: currentTime - 86400000, // 24 hours ago + lte: currentTime, + format: 'epoch_millis', + }, + }, + }, + { match: { 'event.module': model.requiredModule } }, + ], }, }, size: 0, diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/find_interval_for_metrics.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/find_interval_for_metrics.ts index 8ab0f4a44c85d..b3d960e30404f 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/find_interval_for_metrics.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/find_interval_for_metrics.ts @@ -34,7 +34,8 @@ export const findIntervalForMetrics = async ( const modules = await Promise.all( fields.map( - async (field) => await getDatasetForField(client, field as string, options.indexPattern) + async (field) => + await getDatasetForField(client, field as string, options.indexPattern, options.timerange) ) ); diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts index 85bb5b106c87c..15e6f7ba86d01 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts @@ -17,7 +17,8 @@ interface EventDatasetHit { export const getDatasetForField = async ( client: ESSearchClient, field: string, - indexPattern: string + indexPattern: string, + timerange: { field: string; to: number; from: number } ) => { const params = { allowNoIndices: true, @@ -25,9 +26,25 @@ export const getDatasetForField = async ( terminateAfter: 1, index: indexPattern, body: { - query: { exists: { field } }, + query: { + bool: { + filter: [ + { exists: { field } }, + { + range: { + [timerange.field]: { + gte: timerange.from, + lte: timerange.to, + format: 'epoch_millis', + }, + }, + }, + ], + }, + }, size: 1, _source: ['event.dataset'], + sort: [{ [timerange.field]: { order: 'desc' } }], }, }; diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/create_timerange_with_interval.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/create_timerange_with_interval.ts index 827e0901c1c01..833b5349f4b56 100644 --- a/x-pack/plugins/infra/server/routes/snapshot/lib/create_timerange_with_interval.ts +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/create_timerange_with_interval.ts @@ -75,7 +75,10 @@ const aggregationsToModules = async ( const fields = await Promise.all( uniqueFields.map( async (field) => - await getDatasetForField(client, field as string, options.sourceConfiguration.metricAlias) + await getDatasetForField(client, field as string, options.sourceConfiguration.metricAlias, { + ...options.timerange, + field: options.sourceConfiguration.fields.timestamp, + }) ) ); return fields.filter((f) => f) as string[]; diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/get_nodes.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/get_nodes.ts index 9332d5aee1f52..7a2985188dccf 100644 --- a/x-pack/plugins/infra/server/routes/snapshot/lib/get_nodes.ts +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/get_nodes.ts @@ -23,12 +23,11 @@ export const getNodes = async ( snapshotRequest ); const metricsApiResponse = await queryAllData(client, metricsApiRequest); - return copyMissingMetrics( - transformMetricsApiResponseToSnapshotResponse( - metricsApiRequest, - snapshotRequest, - source, - metricsApiResponse - ) + const snapshotResponse = transformMetricsApiResponseToSnapshotResponse( + metricsApiRequest, + snapshotRequest, + source, + metricsApiResponse ); + return copyMissingMetrics(snapshotResponse); };