From 5e0539feb4a060abcacd3e844df50923cd8d350b Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Thu, 18 Apr 2019 17:37:31 +0200 Subject: [PATCH] [Infra UI] Add new graphql endpoint for snapshot data (#34264) * Add new graphql endpoint for snapshot data * Polishing. * Keep type generic that is used outside snapshots * Keep one more generic type generic * Use camelCase for consistency. * Refine type names * Add return types. * Use idiomatic javascript. * Factor out getAllCompositeAggregationData() * Refine naming. * More idiomatic JavaScript, more types. --- x-pack/plugins/infra/common/graphql/types.ts | 65 +++- .../infra/public/graphql/introspection.json | 294 ++++++++++++++++++ x-pack/plugins/infra/public/graphql/types.ts | 65 +++- x-pack/plugins/infra/server/graphql/index.ts | 2 + .../infra/server/graphql/nodes/schema.gql.ts | 9 - .../infra/server/graphql/snapshot/index.ts | 8 + .../server/graphql/snapshot/resolvers.ts | 72 +++++ .../server/graphql/snapshot/schema.gql.ts | 71 +++++ x-pack/plugins/infra/server/graphql/types.ts | 169 +++++++++- x-pack/plugins/infra/server/infra_server.ts | 2 + .../infra/server/lib/compose/kibana.ts | 3 + .../plugins/infra/server/lib/infra_types.ts | 2 + .../infra/server/lib/snapshot/constants.ts | 9 + .../infra/server/lib/snapshot/index.ts | 7 + .../metric_aggregation_creators/count.ts | 20 ++ .../metric_aggregation_creators/cpu.ts | 57 ++++ .../metric_aggregation_creators/index.ts | 24 ++ .../metric_aggregation_creators/load.ts | 20 ++ .../metric_aggregation_creators/log_rate.ts | 32 ++ .../metric_aggregation_creators/memory.ts | 17 + .../metric_aggregation_creators/rate.ts | 41 +++ .../metric_aggregation_creators/rx.ts | 16 + .../metric_aggregation_creators/tx.ts | 16 + .../server/lib/snapshot/query_helpers.ts | 26 ++ .../server/lib/snapshot/response_helpers.ts | 128 ++++++++ .../infra/server/lib/snapshot/snapshot.ts | 230 ++++++++++++++ .../server/utils/get_interval_in_seconds.ts | 31 ++ 27 files changed, 1424 insertions(+), 12 deletions(-) create mode 100644 x-pack/plugins/infra/server/graphql/snapshot/index.ts create mode 100644 x-pack/plugins/infra/server/graphql/snapshot/resolvers.ts create mode 100644 x-pack/plugins/infra/server/graphql/snapshot/schema.gql.ts create mode 100644 x-pack/plugins/infra/server/lib/snapshot/constants.ts create mode 100644 x-pack/plugins/infra/server/lib/snapshot/index.ts create mode 100644 x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/count.ts create mode 100644 x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/cpu.ts create mode 100644 x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/index.ts create mode 100644 x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/load.ts create mode 100644 x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/log_rate.ts create mode 100644 x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/memory.ts create mode 100644 x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/rate.ts create mode 100644 x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/rx.ts create mode 100644 x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/tx.ts create mode 100644 x-pack/plugins/infra/server/lib/snapshot/query_helpers.ts create mode 100644 x-pack/plugins/infra/server/lib/snapshot/response_helpers.ts create mode 100644 x-pack/plugins/infra/server/lib/snapshot/snapshot.ts create mode 100644 x-pack/plugins/infra/server/utils/get_interval_in_seconds.ts diff --git a/x-pack/plugins/infra/common/graphql/types.ts b/x-pack/plugins/infra/common/graphql/types.ts index fdd283125a007..11ef85c3607d1 100644 --- a/x-pack/plugins/infra/common/graphql/types.ts +++ b/x-pack/plugins/infra/common/graphql/types.ts @@ -1,4 +1,4 @@ -/* eslint-disable */ +/* tslint:disable */ // ==================================================== // START: Typescript template @@ -38,6 +38,8 @@ export interface InfraSource { logItem: InfraLogItem; /** A hierarchy of hosts, pods, containers, services or arbitrary groups */ map?: InfraResponse | null; + /** A snapshot of nodes */ + snapshot?: InfraSnapshotResponse | null; metrics: InfraMetricData[]; } @@ -223,6 +225,33 @@ export interface InfraNodeMetric { max: number; } +export interface InfraSnapshotResponse { + /** Nodes of type host, container or pod grouped by 0, 1 or 2 terms */ + nodes: InfraSnapshotNode[]; +} + +export interface InfraSnapshotNode { + path: InfraSnapshotNodePath[]; + + metric: InfraSnapshotNodeMetric; +} + +export interface InfraSnapshotNodePath { + value: string; + + label: string; +} + +export interface InfraSnapshotNodeMetric { + name: InfraSnapshotMetricType; + + value?: number | null; + + avg?: number | null; + + max?: number | null; +} + export interface InfraMetricData { id?: InfraMetric | null; @@ -306,6 +335,18 @@ export interface InfraMetricInput { /** The type of metric */ type: InfraMetricType; } + +export interface InfraSnapshotGroupbyInput { + /** The label to use in the results for the group by for the terms group by */ + label?: string | null; + /** The field to group by from a terms aggregation, this is ignored by the filter type */ + field?: string | null; +} + +export interface InfraSnapshotMetricInput { + /** The type of metric */ + type: InfraSnapshotMetricType; +} /** The source to be created */ export interface CreateSourceInput { /** The name of the data source */ @@ -427,6 +468,11 @@ export interface MapInfraSourceArgs { filterQuery?: string | null; } +export interface SnapshotInfraSourceArgs { + timerange: InfraTimerangeInput; + + filterQuery?: string | null; +} export interface MetricsInfraSourceArgs { nodeId: string; @@ -444,6 +490,13 @@ export interface NodesInfraResponseArgs { metric: InfraMetricInput; } +export interface NodesInfraSnapshotResponseArgs { + type: InfraNodeType; + + groupBy: InfraSnapshotGroupbyInput[]; + + metric: InfraSnapshotMetricInput; +} export interface CreateSourceMutationArgs { /** The id of the source */ id: string; @@ -496,6 +549,16 @@ export enum InfraMetricType { logRate = 'logRate', } +export enum InfraSnapshotMetricType { + count = 'count', + cpu = 'cpu', + load = 'load', + memory = 'memory', + tx = 'tx', + rx = 'rx', + logRate = 'logRate', +} + export enum InfraMetric { hostSystemOverview = 'hostSystemOverview', hostCpuUsage = 'hostCpuUsage', diff --git a/x-pack/plugins/infra/public/graphql/introspection.json b/x-pack/plugins/infra/public/graphql/introspection.json index 5389ce1954102..d53ed883d2351 100644 --- a/x-pack/plugins/infra/public/graphql/introspection.json +++ b/x-pack/plugins/infra/public/graphql/introspection.json @@ -351,6 +351,35 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "snapshot", + "description": "A snapshot of nodes", + "args": [ + { + "name": "timerange", + "description": "", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "InfraTimerangeInput", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "filterQuery", + "description": "", + "type": { "kind": "SCALAR", "name": "String", "ofType": null }, + "defaultValue": null + } + ], + "type": { "kind": "OBJECT", "name": "InfraSnapshotResponse", "ofType": null }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "metrics", "description": "", @@ -1825,6 +1854,271 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "OBJECT", + "name": "InfraSnapshotResponse", + "description": "", + "fields": [ + { + "name": "nodes", + "description": "Nodes of type host, container or pod grouped by 0, 1 or 2 terms", + "args": [ + { + "name": "type", + "description": "", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "ENUM", "name": "InfraNodeType", "ofType": null } + }, + "defaultValue": null + }, + { + "name": "groupBy", + "description": "", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "InfraSnapshotGroupbyInput", + "ofType": null + } + } + } + }, + "defaultValue": null + }, + { + "name": "metric", + "description": "", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "InfraSnapshotMetricInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "OBJECT", "name": "InfraSnapshotNode", "ofType": null } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "InfraSnapshotGroupbyInput", + "description": "", + "fields": null, + "inputFields": [ + { + "name": "label", + "description": "The label to use in the results for the group by for the terms group by", + "type": { "kind": "SCALAR", "name": "String", "ofType": null }, + "defaultValue": null + }, + { + "name": "field", + "description": "The field to group by from a terms aggregation, this is ignored by the filter type", + "type": { "kind": "SCALAR", "name": "String", "ofType": null }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "InfraSnapshotMetricInput", + "description": "", + "fields": null, + "inputFields": [ + { + "name": "type", + "description": "The type of metric", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "ENUM", "name": "InfraSnapshotMetricType", "ofType": null } + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "InfraSnapshotMetricType", + "description": "", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { "name": "count", "description": "", "isDeprecated": false, "deprecationReason": null }, + { "name": "cpu", "description": "", "isDeprecated": false, "deprecationReason": null }, + { "name": "load", "description": "", "isDeprecated": false, "deprecationReason": null }, + { "name": "memory", "description": "", "isDeprecated": false, "deprecationReason": null }, + { "name": "tx", "description": "", "isDeprecated": false, "deprecationReason": null }, + { "name": "rx", "description": "", "isDeprecated": false, "deprecationReason": null }, + { "name": "logRate", "description": "", "isDeprecated": false, "deprecationReason": null } + ], + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "InfraSnapshotNode", + "description": "", + "fields": [ + { + "name": "path", + "description": "", + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "OBJECT", "name": "InfraSnapshotNodePath", "ofType": null } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "metric", + "description": "", + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "OBJECT", "name": "InfraSnapshotNodeMetric", "ofType": null } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "InfraSnapshotNodePath", + "description": "", + "fields": [ + { + "name": "value", + "description": "", + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "label", + "description": "", + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "InfraSnapshotNodeMetric", + "description": "", + "fields": [ + { + "name": "name", + "description": "", + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "ENUM", "name": "InfraSnapshotMetricType", "ofType": null } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "value", + "description": "", + "args": [], + "type": { "kind": "SCALAR", "name": "Float", "ofType": null }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "avg", + "description": "", + "args": [], + "type": { "kind": "SCALAR", "name": "Float", "ofType": null }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "max", + "description": "", + "args": [], + "type": { "kind": "SCALAR", "name": "Float", "ofType": null }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [], + "enumValues": null, + "possibleTypes": null + }, { "kind": "ENUM", "name": "InfraMetric", diff --git a/x-pack/plugins/infra/public/graphql/types.ts b/x-pack/plugins/infra/public/graphql/types.ts index fdd283125a007..11ef85c3607d1 100644 --- a/x-pack/plugins/infra/public/graphql/types.ts +++ b/x-pack/plugins/infra/public/graphql/types.ts @@ -1,4 +1,4 @@ -/* eslint-disable */ +/* tslint:disable */ // ==================================================== // START: Typescript template @@ -38,6 +38,8 @@ export interface InfraSource { logItem: InfraLogItem; /** A hierarchy of hosts, pods, containers, services or arbitrary groups */ map?: InfraResponse | null; + /** A snapshot of nodes */ + snapshot?: InfraSnapshotResponse | null; metrics: InfraMetricData[]; } @@ -223,6 +225,33 @@ export interface InfraNodeMetric { max: number; } +export interface InfraSnapshotResponse { + /** Nodes of type host, container or pod grouped by 0, 1 or 2 terms */ + nodes: InfraSnapshotNode[]; +} + +export interface InfraSnapshotNode { + path: InfraSnapshotNodePath[]; + + metric: InfraSnapshotNodeMetric; +} + +export interface InfraSnapshotNodePath { + value: string; + + label: string; +} + +export interface InfraSnapshotNodeMetric { + name: InfraSnapshotMetricType; + + value?: number | null; + + avg?: number | null; + + max?: number | null; +} + export interface InfraMetricData { id?: InfraMetric | null; @@ -306,6 +335,18 @@ export interface InfraMetricInput { /** The type of metric */ type: InfraMetricType; } + +export interface InfraSnapshotGroupbyInput { + /** The label to use in the results for the group by for the terms group by */ + label?: string | null; + /** The field to group by from a terms aggregation, this is ignored by the filter type */ + field?: string | null; +} + +export interface InfraSnapshotMetricInput { + /** The type of metric */ + type: InfraSnapshotMetricType; +} /** The source to be created */ export interface CreateSourceInput { /** The name of the data source */ @@ -427,6 +468,11 @@ export interface MapInfraSourceArgs { filterQuery?: string | null; } +export interface SnapshotInfraSourceArgs { + timerange: InfraTimerangeInput; + + filterQuery?: string | null; +} export interface MetricsInfraSourceArgs { nodeId: string; @@ -444,6 +490,13 @@ export interface NodesInfraResponseArgs { metric: InfraMetricInput; } +export interface NodesInfraSnapshotResponseArgs { + type: InfraNodeType; + + groupBy: InfraSnapshotGroupbyInput[]; + + metric: InfraSnapshotMetricInput; +} export interface CreateSourceMutationArgs { /** The id of the source */ id: string; @@ -496,6 +549,16 @@ export enum InfraMetricType { logRate = 'logRate', } +export enum InfraSnapshotMetricType { + count = 'count', + cpu = 'cpu', + load = 'load', + memory = 'memory', + tx = 'tx', + rx = 'rx', + logRate = 'logRate', +} + export enum InfraMetric { hostSystemOverview = 'hostSystemOverview', hostCpuUsage = 'hostCpuUsage', diff --git a/x-pack/plugins/infra/server/graphql/index.ts b/x-pack/plugins/infra/server/graphql/index.ts index 20aa0b75097d9..eed695d5420a3 100644 --- a/x-pack/plugins/infra/server/graphql/index.ts +++ b/x-pack/plugins/infra/server/graphql/index.ts @@ -10,6 +10,7 @@ import { logEntriesSchema } from './log_entries/schema.gql'; import { metadataSchema } from './metadata/schema.gql'; import { metricsSchema } from './metrics/schema.gql'; import { nodesSchema } from './nodes/schema.gql'; +import { snapshotSchema } from './snapshot/schema.gql'; import { sourceStatusSchema } from './source_status/schema.gql'; import { sourcesSchema } from './sources/schema.gql'; @@ -19,6 +20,7 @@ export const schemas = [ metadataSchema, logEntriesSchema, nodesSchema, + snapshotSchema, sourcesSchema, sourceStatusSchema, metricsSchema, diff --git a/x-pack/plugins/infra/server/graphql/nodes/schema.gql.ts b/x-pack/plugins/infra/server/graphql/nodes/schema.gql.ts index b1b9b57392054..3cf9ebc7cacaa 100644 --- a/x-pack/plugins/infra/server/graphql/nodes/schema.gql.ts +++ b/x-pack/plugins/infra/server/graphql/nodes/schema.gql.ts @@ -24,15 +24,6 @@ export const nodesSchema: any = gql` metric: InfraNodeMetric! } - input InfraTimerangeInput { - "The interval string to use for last bucket. The format is '{value}{unit}'. For example '5m' would return the metrics for the last 5 minutes of the timespan." - interval: String! - "The end of the timerange" - to: Float! - "The beginning of the timerange" - from: Float! - } - enum InfraOperator { gt gte diff --git a/x-pack/plugins/infra/server/graphql/snapshot/index.ts b/x-pack/plugins/infra/server/graphql/snapshot/index.ts new file mode 100644 index 0000000000000..89f6ebc13c754 --- /dev/null +++ b/x-pack/plugins/infra/server/graphql/snapshot/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { createSnapshotResolvers } from './resolvers'; +export { snapshotSchema } from './schema.gql'; diff --git a/x-pack/plugins/infra/server/graphql/snapshot/resolvers.ts b/x-pack/plugins/infra/server/graphql/snapshot/resolvers.ts new file mode 100644 index 0000000000000..13d16b1a04324 --- /dev/null +++ b/x-pack/plugins/infra/server/graphql/snapshot/resolvers.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { InfraSnapshotResponseResolvers, InfraSourceResolvers } from '../../graphql/types'; +import { InfraSnapshotRequestOptions } from '../../lib/snapshot'; +import { InfraSnapshot } from '../../lib/snapshot'; +// TODO: enable after UI has been changed to use this resolver +// import { UsageCollector } from '../../usage/usage_collector'; +import { parseFilterQuery } from '../../utils/serialized_query'; +import { ChildResolverOf, InfraResolverOf, ResultOf } from '../../utils/typed_resolvers'; +import { QuerySourceResolver } from '../sources/resolvers'; + +type InfraSourceSnapshotResolver = ChildResolverOf< + InfraResolverOf< + InfraSourceResolvers.SnapshotResolver< + { + source: ResultOf; + } & InfraSourceResolvers.SnapshotArgs + > + >, + QuerySourceResolver +>; + +type InfraNodesResolver = ChildResolverOf< + InfraResolverOf, + InfraSourceSnapshotResolver +>; + +interface SnapshotResolversDeps { + snapshot: InfraSnapshot; +} + +export const createSnapshotResolvers = ( + libs: SnapshotResolversDeps +): { + InfraSource: { + snapshot: InfraSourceSnapshotResolver; + }; + InfraSnapshotResponse: { + nodes: InfraNodesResolver; + }; +} => ({ + InfraSource: { + async snapshot(source, args) { + return { + source, + timerange: args.timerange, + filterQuery: args.filterQuery, + }; + }, + }, + InfraSnapshotResponse: { + async nodes(snapshotResponse, args, { req }) { + const { source, timerange, filterQuery } = snapshotResponse; + // TODO: see above + // UsageCollector.countNode(args.type); + const options: InfraSnapshotRequestOptions = { + filterQuery: parseFilterQuery(filterQuery), + nodeType: args.type, + groupBy: args.groupBy, + sourceConfiguration: source.configuration, + metric: args.metric, + timerange, + }; + + return await libs.snapshot.getNodes(req, options); + }, + }, +}); diff --git a/x-pack/plugins/infra/server/graphql/snapshot/schema.gql.ts b/x-pack/plugins/infra/server/graphql/snapshot/schema.gql.ts new file mode 100644 index 0000000000000..9c9ee6a664372 --- /dev/null +++ b/x-pack/plugins/infra/server/graphql/snapshot/schema.gql.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import gql from 'graphql-tag'; + +export const snapshotSchema: any = gql` + type InfraSnapshotNodeMetric { + name: InfraSnapshotMetricType! + value: Float + avg: Float + max: Float + } + + type InfraSnapshotNodePath { + value: String! + label: String! + } + + type InfraSnapshotNode { + path: [InfraSnapshotNodePath!]! + metric: InfraSnapshotNodeMetric! + } + + input InfraTimerangeInput { + "The interval string to use for last bucket. The format is '{value}{unit}'. For example '5m' would return the metrics for the last 5 minutes of the timespan." + interval: String! + "The end of the timerange" + to: Float! + "The beginning of the timerange" + from: Float! + } + + enum InfraSnapshotMetricType { + count + cpu + load + memory + tx + rx + logRate + } + + input InfraSnapshotMetricInput { + "The type of metric" + type: InfraSnapshotMetricType! + } + + input InfraSnapshotGroupbyInput { + "The label to use in the results for the group by for the terms group by" + label: String + "The field to group by from a terms aggregation, this is ignored by the filter type" + field: String + } + + type InfraSnapshotResponse { + "Nodes of type host, container or pod grouped by 0, 1 or 2 terms" + nodes( + type: InfraNodeType! + groupBy: [InfraSnapshotGroupbyInput!]! + metric: InfraSnapshotMetricInput! + ): [InfraSnapshotNode!]! + } + + extend type InfraSource { + "A snapshot of nodes" + snapshot(timerange: InfraTimerangeInput!, filterQuery: String): InfraSnapshotResponse + } +`; diff --git a/x-pack/plugins/infra/server/graphql/types.ts b/x-pack/plugins/infra/server/graphql/types.ts index 5ca3bdf6e6504..ecc8215ff0eba 100644 --- a/x-pack/plugins/infra/server/graphql/types.ts +++ b/x-pack/plugins/infra/server/graphql/types.ts @@ -1,4 +1,4 @@ -/* eslint-disable */ +/* tslint:disable */ import { InfraContext } from '../lib/infra_types'; import { GraphQLResolveInfo } from 'graphql'; @@ -66,6 +66,8 @@ export interface InfraSource { logItem: InfraLogItem; /** A hierarchy of hosts, pods, containers, services or arbitrary groups */ map?: InfraResponse | null; + /** A snapshot of nodes */ + snapshot?: InfraSnapshotResponse | null; metrics: InfraMetricData[]; } @@ -251,6 +253,33 @@ export interface InfraNodeMetric { max: number; } +export interface InfraSnapshotResponse { + /** Nodes of type host, container or pod grouped by 0, 1 or 2 terms */ + nodes: InfraSnapshotNode[]; +} + +export interface InfraSnapshotNode { + path: InfraSnapshotNodePath[]; + + metric: InfraSnapshotNodeMetric; +} + +export interface InfraSnapshotNodePath { + value: string; + + label: string; +} + +export interface InfraSnapshotNodeMetric { + name: InfraSnapshotMetricType; + + value?: number | null; + + avg?: number | null; + + max?: number | null; +} + export interface InfraMetricData { id?: InfraMetric | null; @@ -334,6 +363,18 @@ export interface InfraMetricInput { /** The type of metric */ type: InfraMetricType; } + +export interface InfraSnapshotGroupbyInput { + /** The label to use in the results for the group by for the terms group by */ + label?: string | null; + /** The field to group by from a terms aggregation, this is ignored by the filter type */ + field?: string | null; +} + +export interface InfraSnapshotMetricInput { + /** The type of metric */ + type: InfraSnapshotMetricType; +} /** The source to be created */ export interface CreateSourceInput { /** The name of the data source */ @@ -455,6 +496,11 @@ export interface MapInfraSourceArgs { filterQuery?: string | null; } +export interface SnapshotInfraSourceArgs { + timerange: InfraTimerangeInput; + + filterQuery?: string | null; +} export interface MetricsInfraSourceArgs { nodeId: string; @@ -472,6 +518,13 @@ export interface NodesInfraResponseArgs { metric: InfraMetricInput; } +export interface NodesInfraSnapshotResponseArgs { + type: InfraNodeType; + + groupBy: InfraSnapshotGroupbyInput[]; + + metric: InfraSnapshotMetricInput; +} export interface CreateSourceMutationArgs { /** The id of the source */ id: string; @@ -524,6 +577,16 @@ export enum InfraMetricType { logRate = 'logRate', } +export enum InfraSnapshotMetricType { + count = 'count', + cpu = 'cpu', + load = 'load', + memory = 'memory', + tx = 'tx', + rx = 'rx', + logRate = 'logRate', +} + export enum InfraMetric { hostSystemOverview = 'hostSystemOverview', hostCpuUsage = 'hostCpuUsage', @@ -627,6 +690,8 @@ export namespace InfraSourceResolvers { logItem?: LogItemResolver; /** A hierarchy of hosts, pods, containers, services or arbitrary groups */ map?: MapResolver; + /** A snapshot of nodes */ + snapshot?: SnapshotResolver; metrics?: MetricsResolver; } @@ -737,6 +802,17 @@ export namespace InfraSourceResolvers { filterQuery?: string | null; } + export type SnapshotResolver< + R = InfraSnapshotResponse | null, + Parent = InfraSource, + Context = InfraContext + > = Resolver; + export interface SnapshotArgs { + timerange: InfraTimerangeInput; + + filterQuery?: string | null; + } + export type MetricsResolver< R = InfraMetricData[], Parent = InfraSource, @@ -1324,6 +1400,97 @@ export namespace InfraNodeMetricResolvers { >; } +export namespace InfraSnapshotResponseResolvers { + export interface Resolvers { + /** Nodes of type host, container or pod grouped by 0, 1 or 2 terms */ + nodes?: NodesResolver; + } + + export type NodesResolver< + R = InfraSnapshotNode[], + Parent = InfraSnapshotResponse, + Context = InfraContext + > = Resolver; + export interface NodesArgs { + type: InfraNodeType; + + groupBy: InfraSnapshotGroupbyInput[]; + + metric: InfraSnapshotMetricInput; + } +} + +export namespace InfraSnapshotNodeResolvers { + export interface Resolvers { + path?: PathResolver; + + metric?: MetricResolver; + } + + export type PathResolver< + R = InfraSnapshotNodePath[], + Parent = InfraSnapshotNode, + Context = InfraContext + > = Resolver; + export type MetricResolver< + R = InfraSnapshotNodeMetric, + Parent = InfraSnapshotNode, + Context = InfraContext + > = Resolver; +} + +export namespace InfraSnapshotNodePathResolvers { + export interface Resolvers { + value?: ValueResolver; + + label?: LabelResolver; + } + + export type ValueResolver< + R = string, + Parent = InfraSnapshotNodePath, + Context = InfraContext + > = Resolver; + export type LabelResolver< + R = string, + Parent = InfraSnapshotNodePath, + Context = InfraContext + > = Resolver; +} + +export namespace InfraSnapshotNodeMetricResolvers { + export interface Resolvers { + name?: NameResolver; + + value?: ValueResolver; + + avg?: AvgResolver; + + max?: MaxResolver; + } + + export type NameResolver< + R = InfraSnapshotMetricType, + Parent = InfraSnapshotNodeMetric, + Context = InfraContext + > = Resolver; + export type ValueResolver< + R = number | null, + Parent = InfraSnapshotNodeMetric, + Context = InfraContext + > = Resolver; + export type AvgResolver< + R = number | null, + Parent = InfraSnapshotNodeMetric, + Context = InfraContext + > = Resolver; + export type MaxResolver< + R = number | null, + Parent = InfraSnapshotNodeMetric, + Context = InfraContext + > = Resolver; +} + export namespace InfraMetricDataResolvers { export interface Resolvers { id?: IdResolver; diff --git a/x-pack/plugins/infra/server/infra_server.ts b/x-pack/plugins/infra/server/infra_server.ts index a89d3cb6e8170..6057d26e917ca 100644 --- a/x-pack/plugins/infra/server/infra_server.ts +++ b/x-pack/plugins/infra/server/infra_server.ts @@ -10,6 +10,7 @@ import { createLogEntriesResolvers } from './graphql/log_entries'; import { createMetadataResolvers } from './graphql/metadata'; import { createMetricResolvers } from './graphql/metrics/resolvers'; import { createNodeResolvers } from './graphql/nodes'; +import { createSnapshotResolvers } from './graphql/snapshot'; import { createSourceStatusResolvers } from './graphql/source_status'; import { createSourcesResolvers } from './graphql/sources'; import { InfraBackendLibs } from './lib/infra_types'; @@ -21,6 +22,7 @@ export const initInfraServer = (libs: InfraBackendLibs) => { createMetadataResolvers(libs) as IResolvers, createLogEntriesResolvers(libs) as IResolvers, createNodeResolvers(libs) as IResolvers, + createSnapshotResolvers(libs) as IResolvers, createSourcesResolvers(libs) as IResolvers, createSourceStatusResolvers(libs) as IResolvers, createMetricResolvers(libs) as IResolvers, diff --git a/x-pack/plugins/infra/server/lib/compose/kibana.ts b/x-pack/plugins/infra/server/lib/compose/kibana.ts index 2c853ba5105ae..19fcd83912c83 100644 --- a/x-pack/plugins/infra/server/lib/compose/kibana.ts +++ b/x-pack/plugins/infra/server/lib/compose/kibana.ts @@ -20,6 +20,7 @@ import { InfraMetadataDomain } from '../domains/metadata_domain'; import { InfraMetricsDomain } from '../domains/metrics_domain'; import { InfraNodesDomain } from '../domains/nodes_domain'; import { InfraBackendLibs, InfraDomainLibs } from '../infra_types'; +import { InfraSnapshot } from '../snapshot'; import { InfraSourceStatus } from '../source_status'; import { InfraSources } from '../sources'; @@ -33,6 +34,7 @@ export function compose(server: Server): InfraBackendLibs { const sourceStatus = new InfraSourceStatus(new InfraElasticsearchSourceStatusAdapter(framework), { sources, }); + const snapshot = new InfraSnapshot({ sources, framework }); const domainLibs: InfraDomainLibs = { metadata: new InfraMetadataDomain(new ElasticsearchMetadataAdapter(framework), { @@ -51,6 +53,7 @@ export function compose(server: Server): InfraBackendLibs { const libs: InfraBackendLibs = { configuration, framework, + snapshot, sources, sourceStatus, ...domainLibs, diff --git a/x-pack/plugins/infra/server/lib/infra_types.ts b/x-pack/plugins/infra/server/lib/infra_types.ts index 94fe8f94f9e4f..c7ae5a77e1ea8 100644 --- a/x-pack/plugins/infra/server/lib/infra_types.ts +++ b/x-pack/plugins/infra/server/lib/infra_types.ts @@ -12,6 +12,7 @@ import { InfraLogEntriesDomain } from './domains/log_entries_domain'; import { InfraMetadataDomain } from './domains/metadata_domain'; import { InfraMetricsDomain } from './domains/metrics_domain'; import { InfraNodesDomain } from './domains/nodes_domain'; +import { InfraSnapshot } from './snapshot'; import { InfraSourceStatus } from './source_status'; import { InfraSources } from './sources'; @@ -26,6 +27,7 @@ export interface InfraDomainLibs { export interface InfraBackendLibs extends InfraDomainLibs { configuration: InfraConfigurationAdapter; framework: InfraBackendFrameworkAdapter; + snapshot: InfraSnapshot; sources: InfraSources; sourceStatus: InfraSourceStatus; } diff --git a/x-pack/plugins/infra/server/lib/snapshot/constants.ts b/x-pack/plugins/infra/server/lib/snapshot/constants.ts new file mode 100644 index 0000000000000..0420878dbcf50 --- /dev/null +++ b/x-pack/plugins/infra/server/lib/snapshot/constants.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// TODO: Make SNAPSHOT_COMPOSITE_REQUEST_SIZE configurable from kibana.yml + +export const SNAPSHOT_COMPOSITE_REQUEST_SIZE = 75; diff --git a/x-pack/plugins/infra/server/lib/snapshot/index.ts b/x-pack/plugins/infra/server/lib/snapshot/index.ts new file mode 100644 index 0000000000000..8db54da803648 --- /dev/null +++ b/x-pack/plugins/infra/server/lib/snapshot/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './snapshot'; diff --git a/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/count.ts b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/count.ts new file mode 100644 index 0000000000000..a81e7ff505cfd --- /dev/null +++ b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/count.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const count = () => { + return { + count: { + bucket_script: { + buckets_path: { count: '_count' }, + script: { + source: 'count * 1', + lang: 'expression', + }, + gap_policy: 'skip', + }, + }, + }; +}; diff --git a/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/cpu.ts b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/cpu.ts new file mode 100644 index 0000000000000..e0a44ef9036e1 --- /dev/null +++ b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/cpu.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { InfraNodeType } from '../../../graphql/types'; + +const FIELDS = { + [InfraNodeType.host]: 'system.cpu.user.pct', + [InfraNodeType.pod]: 'kubernetes.pod.cpu.usage.node.pct', + [InfraNodeType.container]: 'docker.cpu.total.pct', +}; + +export const cpu = (nodeType: InfraNodeType) => { + if (nodeType === InfraNodeType.host) { + return { + cpu_user: { + avg: { + field: 'system.cpu.user.pct', + }, + }, + cpu_system: { + avg: { + field: 'system.cpu.system.pct', + }, + }, + cpu_cores: { + max: { + field: 'system.cpu.cores', + }, + }, + cpu: { + bucket_script: { + buckets_path: { + user: 'cpu_user', + system: 'cpu_system', + cores: 'cpu_cores', + }, + script: { + source: '(params.user + params.system) / params.cores', + lang: 'painless', + }, + gap_policy: 'skip', + }, + }, + }; + } else { + return { + cpu: { + avg: { + field: FIELDS[nodeType], + }, + }, + }; + } +}; diff --git a/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/index.ts b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/index.ts new file mode 100644 index 0000000000000..e4c321324a9a9 --- /dev/null +++ b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { InfraSnapshotMetricType } from '../../../graphql/types'; +import { count } from './count'; +import { cpu } from './cpu'; +import { load } from './load'; +import { logRate } from './log_rate'; +import { memory } from './memory'; +import { rx } from './rx'; +import { tx } from './tx'; + +export const metricAggregationCreators = { + [InfraSnapshotMetricType.count]: count, + [InfraSnapshotMetricType.cpu]: cpu, + [InfraSnapshotMetricType.memory]: memory, + [InfraSnapshotMetricType.rx]: rx, + [InfraSnapshotMetricType.tx]: tx, + [InfraSnapshotMetricType.load]: load, + [InfraSnapshotMetricType.logRate]: logRate, +}; diff --git a/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/load.ts b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/load.ts new file mode 100644 index 0000000000000..3a5b97a36e7cd --- /dev/null +++ b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/load.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { InfraNodeType } from '../../../graphql/types'; + +const FIELDS = { + [InfraNodeType.host]: 'system.load.5', + [InfraNodeType.pod]: '', + [InfraNodeType.container]: '', +}; + +export const load = (nodeType: InfraNodeType) => { + const field = FIELDS[nodeType]; + if (field) { + return { load: { avg: { field } } }; + } +}; diff --git a/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/log_rate.ts b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/log_rate.ts new file mode 100644 index 0000000000000..f7f2ac26e3b96 --- /dev/null +++ b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/log_rate.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const logRate = () => { + return { + count: { + bucket_script: { + buckets_path: { count: '_count' }, + script: { + source: 'count * 1', + lang: 'expression', + }, + gap_policy: 'skip', + }, + }, + cumsum: { + cumulative_sum: { + buckets_path: 'count', + }, + }, + logRate: { + derivative: { + buckets_path: 'cumsum', + gap_policy: 'skip', + unit: '1s', + }, + }, + }; +}; diff --git a/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/memory.ts b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/memory.ts new file mode 100644 index 0000000000000..fec0c0ede3438 --- /dev/null +++ b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/memory.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { InfraNodeType } from '../../../graphql/types'; + +const FIELDS = { + [InfraNodeType.host]: 'system.memory.actual.used.pct', + [InfraNodeType.pod]: 'kubernetes.pod.memory.usage.node.pct', + [InfraNodeType.container]: 'docker.memory.usage.pct', +}; + +export const memory = (nodeType: InfraNodeType) => { + return { memory: { avg: { field: FIELDS[nodeType] } } }; +}; diff --git a/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/rate.ts b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/rate.ts new file mode 100644 index 0000000000000..4024fa7563bef --- /dev/null +++ b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/rate.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { InfraNodeType } from '../../../graphql/types'; + +interface Fields { + [InfraNodeType.container]: string; + [InfraNodeType.pod]: string; + [InfraNodeType.host]: string; +} + +export const rate = (id: string, fields: Fields) => { + return (nodeType: InfraNodeType) => { + const field = fields[nodeType]; + if (field) { + return { + [`${id}_max`]: { max: { field } }, + [`${id}_deriv`]: { + derivative: { + buckets_path: `${id}_max`, + gap_policy: 'skip', + unit: '1s', + }, + }, + [id]: { + bucket_script: { + buckets_path: { value: `${id}_deriv[normalized_value]` }, + script: { + source: 'params.value > 0.0 ? params.value : 0.0', + lang: 'painless', + }, + gap_policy: 'skip', + }, + }, + }; + } + }; +}; diff --git a/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/rx.ts b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/rx.ts new file mode 100644 index 0000000000000..fc7565adda863 --- /dev/null +++ b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/rx.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { InfraNodeType } from '../../../graphql/types'; +import { rate } from './rate'; + +const FIELDS = { + [InfraNodeType.host]: 'system.network.in.bytes', + [InfraNodeType.pod]: 'kubernetes.pod.network.rx.bytes', + [InfraNodeType.container]: 'docker.network.in.bytes', +}; + +export const rx = rate('rx', FIELDS); diff --git a/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/tx.ts b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/tx.ts new file mode 100644 index 0000000000000..6bdbb27177a90 --- /dev/null +++ b/x-pack/plugins/infra/server/lib/snapshot/metric_aggregation_creators/tx.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { InfraNodeType } from '../../../graphql/types'; +import { rate } from './rate'; + +const FIELDS = { + [InfraNodeType.host]: 'system.network.out.bytes', + [InfraNodeType.pod]: 'kubernetes.pod.network.tx.bytes', + [InfraNodeType.container]: 'docker.network.out.bytes', +}; + +export const tx = rate('tx', FIELDS); diff --git a/x-pack/plugins/infra/server/lib/snapshot/query_helpers.ts b/x-pack/plugins/infra/server/lib/snapshot/query_helpers.ts new file mode 100644 index 0000000000000..2d62432acf7ec --- /dev/null +++ b/x-pack/plugins/infra/server/lib/snapshot/query_helpers.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { metricAggregationCreators } from './metric_aggregation_creators'; +import { InfraSnapshotRequestOptions } from './snapshot'; + +export const getGroupedNodesSources = (options: InfraSnapshotRequestOptions) => { + const sources = options.groupBy.map(gb => { + return { [`${gb.field}`]: { terms: { field: gb.field } } }; + }); + sources.push({ + node: { terms: { field: options.sourceConfiguration.fields[options.nodeType] } }, + }); + return sources; +}; + +export const getMetricsSources = (options: InfraSnapshotRequestOptions) => { + return [{ node: { terms: { field: options.sourceConfiguration.fields[options.nodeType] } } }]; +}; + +export const getMetricsAggregations = (options: InfraSnapshotRequestOptions) => { + return metricAggregationCreators[options.metric.type](options.nodeType); +}; diff --git a/x-pack/plugins/infra/server/lib/snapshot/response_helpers.ts b/x-pack/plugins/infra/server/lib/snapshot/response_helpers.ts new file mode 100644 index 0000000000000..d92aaeb886e32 --- /dev/null +++ b/x-pack/plugins/infra/server/lib/snapshot/response_helpers.ts @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { isNumber, last, max, sum } from 'lodash'; +import moment from 'moment'; + +import { + InfraSnapshotMetricType, + InfraSnapshotNodePath, + InfraSnapshotNodeMetric, +} from '../../graphql/types'; +import { getIntervalInSeconds } from '../../utils/get_interval_in_seconds'; +import { InfraSnapshotRequestOptions } from './snapshot'; + +export interface InfraSnapshotNodeMetricsBucket { + key: { node: string }; + histogram: { + buckets: InfraSnapshotMetricsBucket[]; + }; +} + +// Jumping through TypeScript hoops here: +// We need an interface that has the known members 'key' and 'doc_count' and also +// an unknown number of members with unknown names but known format, containing the +// metrics. +// This union type is the only way I found to express this that TypeScript accepts. +export interface InfraSnapshotBucketWithKey { + key: string | number; + doc_count: number; +} + +export interface InfraSnapshotBucketWithValues { + [name: string]: { value: number; normalized_value?: number }; +} + +export type InfraSnapshotMetricsBucket = InfraSnapshotBucketWithKey & InfraSnapshotBucketWithValues; + +export interface InfraSnapshotNodeGroupByBucket { + key: { + node: string; + [groupByField: string]: string; + }; +} + +export const getNodePath = ( + groupBucket: InfraSnapshotNodeGroupByBucket, + options: InfraSnapshotRequestOptions +): InfraSnapshotNodePath[] => { + const node = groupBucket.key; + const path = options.groupBy.map(gb => { + return { value: node[`${gb.field}`], label: node[`${gb.field}`] }; + }); + path.push({ value: node.node, label: node.node }); + return path; +}; + +interface NodeMetricsForLookup { + [node: string]: InfraSnapshotMetricsBucket[]; +} + +export const getNodeMetricsForLookup = ( + metrics: InfraSnapshotNodeMetricsBucket[] +): NodeMetricsForLookup => { + return metrics.reduce((acc: NodeMetricsForLookup, metric) => { + acc[`${metric.key.node}`] = metric.histogram.buckets; + return acc; + }, {}); +}; + +// In the returned object, +// value contains the value from the last bucket spanning a full interval +// max and avg are calculated from all buckets returned for the timerange +export const getNodeMetrics = ( + nodeBuckets: InfraSnapshotMetricsBucket[], + options: InfraSnapshotRequestOptions +): InfraSnapshotNodeMetric => { + if (!nodeBuckets) { + return { + name: options.metric.type, + value: null, + max: null, + avg: null, + }; + } + const lastBucket = findLastFullBucket(nodeBuckets, options); + const result = { + name: options.metric.type, + value: getMetricValueFromBucket(options.metric.type, lastBucket), + max: calculateMax(nodeBuckets, options.metric.type), + avg: calculateAvg(nodeBuckets, options.metric.type), + }; + return result; +}; + +const findLastFullBucket = ( + buckets: InfraSnapshotMetricsBucket[], + options: InfraSnapshotRequestOptions +) => { + const to = moment.utc(options.timerange.to); + const bucketSize = getIntervalInSeconds(options.timerange.interval); + return buckets.reduce((current, item) => { + const itemKey = isNumber(item.key) ? item.key : parseInt(item.key, 10); + const date = moment.utc(itemKey + bucketSize * 1000); + if (!date.isAfter(to) && item.doc_count > 0) { + return item; + } + return current; + }, last(buckets)); +}; + +const getMetricValueFromBucket = ( + type: InfraSnapshotMetricType, + bucket: InfraSnapshotMetricsBucket +) => { + const metric = bucket[type]; + return (metric && (metric.normalized_value || metric.value)) || 0; +}; + +function calculateMax(buckets: InfraSnapshotMetricsBucket[], type: InfraSnapshotMetricType) { + return max(buckets.map(bucket => getMetricValueFromBucket(type, bucket))) || 0; +} + +function calculateAvg(buckets: InfraSnapshotMetricsBucket[], type: InfraSnapshotMetricType) { + return sum(buckets.map(bucket => getMetricValueFromBucket(type, bucket))) / buckets.length || 0; +} diff --git a/x-pack/plugins/infra/server/lib/snapshot/snapshot.ts b/x-pack/plugins/infra/server/lib/snapshot/snapshot.ts new file mode 100644 index 0000000000000..018b20ae2e650 --- /dev/null +++ b/x-pack/plugins/infra/server/lib/snapshot/snapshot.ts @@ -0,0 +1,230 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + InfraSnapshotGroupbyInput, + InfraSnapshotMetricInput, + InfraSnapshotNode, + InfraTimerangeInput, + InfraNodeType, + InfraSourceConfiguration, +} from '../../graphql/types'; +import { InfraBackendFrameworkAdapter, InfraFrameworkRequest } from '../adapters/framework'; +import { InfraSources } from '../sources'; + +import { JsonObject } from '../../../common/typed_json'; +import { SNAPSHOT_COMPOSITE_REQUEST_SIZE } from './constants'; +import { getGroupedNodesSources, getMetricsAggregations, getMetricsSources } from './query_helpers'; +import { + getNodeMetrics, + getNodeMetricsForLookup, + getNodePath, + InfraSnapshotNodeGroupByBucket, + InfraSnapshotNodeMetricsBucket, +} from './response_helpers'; + +export interface InfraSnapshotRequestOptions { + nodeType: InfraNodeType; + sourceConfiguration: InfraSourceConfiguration; + timerange: InfraTimerangeInput; + groupBy: InfraSnapshotGroupbyInput[]; + metric: InfraSnapshotMetricInput; + filterQuery: JsonObject | undefined; +} + +export class InfraSnapshot { + constructor( + private readonly libs: { sources: InfraSources; framework: InfraBackendFrameworkAdapter } + ) {} + + public async getNodes( + request: InfraFrameworkRequest, + options: InfraSnapshotRequestOptions + ): Promise { + // Both requestGroupedNodes and requestNodeMetrics may send several requests to elasticsearch + // in order to page through the results of their respective composite aggregations. + // Both chains of requests are supposed to run in parallel, and their results be merged + // when they have both been completed. + const groupedNodesPromise = requestGroupedNodes(request, options, this.libs.framework); + const nodeMetricsPromise = requestNodeMetrics(request, options, this.libs.framework); + + const groupedNodeBuckets = await groupedNodesPromise; + const nodeMetricBuckets = await nodeMetricsPromise; + + return mergeNodeBuckets(groupedNodeBuckets, nodeMetricBuckets, options); + } +} + +const requestGroupedNodes = async ( + request: InfraFrameworkRequest, + options: InfraSnapshotRequestOptions, + framework: InfraBackendFrameworkAdapter +): Promise => { + const query = { + allowNoIndices: true, + index: `${options.sourceConfiguration.logAlias},${options.sourceConfiguration.metricAlias}`, + ignoreUnavailable: true, + body: { + query: { + bool: { + filter: [ + ...createQueryFilterClauses(options.filterQuery), + { + range: { + [options.sourceConfiguration.fields.timestamp]: { + gte: options.timerange.from, + lte: options.timerange.to, + format: 'epoch_millis', + }, + }, + }, + ], + }, + }, + size: 0, + aggregations: { + nodes: { + composite: { + size: SNAPSHOT_COMPOSITE_REQUEST_SIZE, + sources: getGroupedNodesSources(options), + }, + }, + }, + }, + }; + + return await getAllCompositeAggregationData( + framework, + request, + query + ); +}; + +const requestNodeMetrics = async ( + request: InfraFrameworkRequest, + options: InfraSnapshotRequestOptions, + framework: InfraBackendFrameworkAdapter +): Promise => { + const index = + options.metric.type === 'logRate' + ? `${options.sourceConfiguration.logAlias}` + : `${options.sourceConfiguration.metricAlias}`; + + const query = { + allowNoIndices: true, + index, + ignoreUnavailable: true, + body: { + query: { + bool: { + filter: [ + ...createQueryFilterClauses(options.filterQuery), + { + range: { + [options.sourceConfiguration.fields.timestamp]: { + gte: options.timerange.from, + lte: options.timerange.to, + format: 'epoch_millis', + }, + }, + }, + ], + }, + }, + size: 0, + aggregations: { + nodes: { + composite: { + size: SNAPSHOT_COMPOSITE_REQUEST_SIZE, + sources: getMetricsSources(options), + }, + aggregations: { + histogram: { + date_histogram: { + field: options.sourceConfiguration.fields.timestamp, + interval: options.timerange.interval || '1m', + }, + aggregations: getMetricsAggregations(options), + }, + }, + }, + }, + }, + }; + + return await getAllCompositeAggregationData( + framework, + request, + query + ); +}; + +// buckets can be InfraSnapshotNodeGroupByBucket[] or InfraSnapshotNodeMetricsBucket[] +// but typing this in a way that makes TypeScript happy is unreadable (if possible at all) +interface InfraSnapshotAggregationResponse { + nodes: { + buckets: any[]; + after_key: { [id: string]: string }; + }; +} + +const getAllCompositeAggregationData = async ( + framework: InfraBackendFrameworkAdapter, + request: InfraFrameworkRequest, + query: any, + previousBuckets: BucketType[] = [] +): Promise => { + const response = await framework.callWithRequest<{}, InfraSnapshotAggregationResponse>( + request, + 'search', + query + ); + + // Nothing available, return the previous buckets. + if (response.hits.total.value === 0) { + return previousBuckets; + } + + // if ES doesn't return an aggregations key, something went seriously wrong. + if (!response.aggregations) { + throw new Error('Whoops!, `aggregations` key must always be returned.'); + } + + const currentBuckets = response.aggregations.nodes.buckets; + + // if there are no currentBuckets then we are finished paginating through the results + if (currentBuckets.length === 0) { + return previousBuckets; + } + + // There is possibly more data, concat previous and current buckets and call ourselves recursively. + const newQuery = { ...query }; + newQuery.body.aggregations.nodes.composite.after = response.aggregations.nodes.after_key; + return getAllCompositeAggregationData( + framework, + request, + query, + previousBuckets.concat(currentBuckets) + ); +}; + +const mergeNodeBuckets = ( + nodeGroupByBuckets: InfraSnapshotNodeGroupByBucket[], + nodeMetricsBuckets: InfraSnapshotNodeMetricsBucket[], + options: InfraSnapshotRequestOptions +): InfraSnapshotNode[] => { + const nodeMetricsForLookup = getNodeMetricsForLookup(nodeMetricsBuckets); + + return nodeGroupByBuckets.map(node => { + return { + path: getNodePath(node, options), + metric: getNodeMetrics(nodeMetricsForLookup[node.key.node], options), + }; + }); +}; + +const createQueryFilterClauses = (filterQuery: JsonObject | undefined) => + filterQuery ? [filterQuery] : []; diff --git a/x-pack/plugins/infra/server/utils/get_interval_in_seconds.ts b/x-pack/plugins/infra/server/utils/get_interval_in_seconds.ts new file mode 100644 index 0000000000000..297e5828956af --- /dev/null +++ b/x-pack/plugins/infra/server/utils/get_interval_in_seconds.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +const intervalUnits = ['y', 'M', 'w', 'd', 'h', 'm', 's', 'ms']; +const INTERVAL_STRING_RE = new RegExp('^([0-9\\.]*)\\s*(' + intervalUnits.join('|') + ')$'); + +interface UnitsToSeconds { + [unit: string]: number; +} + +const units: UnitsToSeconds = { + ms: 0.001, + s: 1, + m: 60, + h: 3600, + d: 86400, + w: 86400 * 7, + M: 86400 * 30, + y: 86400 * 356, +}; + +export const getIntervalInSeconds = (interval: string): number => { + const matches = interval.match(INTERVAL_STRING_RE); + if (matches) { + return parseFloat(matches[1]) * units[matches[2]]; + } + throw new Error('Invalid interval string format.'); +};