From ee237adf4ad82840e517df9fd1c040df43386cef Mon Sep 17 00:00:00 2001 From: Mingjie Shao Date: Wed, 20 Apr 2022 22:06:24 +0100 Subject: [PATCH] feat(logs): add QueryDefinition L2 Construct (#18655) This PR implemented the [[feature request] (aws-cloudwatch): Saved queries](https://github.com/aws/aws-cdk/issues/16395). It will let users be able to create CloudWatch Logs Insights QueryDefinition by using L2 construct. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/aws-logs/README.md | 17 ++ packages/@aws-cdk/aws-logs/lib/index.ts | 1 + .../@aws-cdk/aws-logs/lib/query-definition.ts | 165 ++++++++++++++++++ ...teg.save-logs-insights-query-definition.ts | 29 +++ .../aws-logs/test/query-definition.test.ts | 78 +++++++++ ...k-logs-querydefinition-integ.template.json | 24 +++ .../cdk.out | 1 + .../manifest.json | 34 ++++ .../tree.json | 85 +++++++++ 9 files changed, 434 insertions(+) create mode 100644 packages/@aws-cdk/aws-logs/lib/query-definition.ts create mode 100644 packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.ts create mode 100644 packages/@aws-cdk/aws-logs/test/query-definition.test.ts create mode 100644 packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/aws-cdk-logs-querydefinition-integ.template.json create mode 100644 packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/cdk.out create mode 100644 packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/manifest.json create mode 100644 packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/tree.json diff --git a/packages/@aws-cdk/aws-logs/README.md b/packages/@aws-cdk/aws-logs/README.md index 4e04059638138..80049bfc114d4 100644 --- a/packages/@aws-cdk/aws-logs/README.md +++ b/packages/@aws-cdk/aws-logs/README.md @@ -306,6 +306,23 @@ const pattern = logs.FilterPattern.spaceDelimited('time', 'component', '...', 'r .whereNumber('result_code', '!=', 200); ``` +## Logs Insights Query Definition + +Creates a query definition for CloudWatch Logs Insights. + +Example: + +```ts +new logs.QueryDefinition(this, 'QueryDefinition', { + queryDefinitionName: 'MyQuery', + queryString: new logs.QueryString({ + fields: ['@timestamp', '@message'], + sort: '@timestamp desc', + limit: 20, + }), +}); +``` + ## Notes Be aware that Log Group ARNs will always have the string `:*` appended to diff --git a/packages/@aws-cdk/aws-logs/lib/index.ts b/packages/@aws-cdk/aws-logs/lib/index.ts index 416a9c9a9b257..d8df4e7b189e5 100644 --- a/packages/@aws-cdk/aws-logs/lib/index.ts +++ b/packages/@aws-cdk/aws-logs/lib/index.ts @@ -6,6 +6,7 @@ export * from './pattern'; export * from './subscription-filter'; export * from './log-retention'; export * from './policy'; +export * from './query-definition'; // AWS::Logs CloudFormation Resources: export * from './logs.generated'; diff --git a/packages/@aws-cdk/aws-logs/lib/query-definition.ts b/packages/@aws-cdk/aws-logs/lib/query-definition.ts new file mode 100644 index 0000000000000..aea611e79c63d --- /dev/null +++ b/packages/@aws-cdk/aws-logs/lib/query-definition.ts @@ -0,0 +1,165 @@ +import { Resource } from '@aws-cdk/core'; +import { Construct } from 'constructs'; +import { CfnQueryDefinition } from '.'; +import { ILogGroup } from './log-group'; + + +/** + * Properties for a QueryString + */ +export interface QueryStringProps { + /** + * Retrieves the specified fields from log events for display. + * + * @default - no fields in QueryString + */ + readonly fields?: string[]; + + /** + * Extracts data from a log field and creates one or more ephemeral fields that you can process further in the query. + * + * @default - no parse in QueryString + */ + readonly parse?: string; + + /** + * Filters the results of a query that's based on one or more conditions. + * + * @default - no filter in QueryString + */ + readonly filter?: string; + + /** + * Uses log field values to calculate aggregate statistics. + * + * @default - no stats in QueryString + */ + readonly stats?: string; + + /** + * Sorts the retrieved log events. + * + * @default - no sort in QueryString + */ + readonly sort?: string; + + /** + * Specifies the number of log events returned by the query. + * + * @default - no limit in QueryString + */ + readonly limit?: Number; + + /** + * Specifies which fields to display in the query results. + * + * @default - no display in QueryString + */ + readonly display?: string; +} + +interface QueryStringMap { + readonly fields?: string, + readonly parse?: string, + readonly filter?: string, + readonly stats?: string, + readonly sort?: string, + readonly limit?: Number, + readonly display?: string, +} + +/** + * Define a QueryString + */ +export class QueryString { + private readonly fields?: string[]; + private readonly parse?: string; + private readonly filter?: string; + private readonly stats?: string; + private readonly sort?: string; + private readonly limit?: Number; + private readonly display?: string; + + constructor(props: QueryStringProps = {}) { + this.fields = props.fields; + this.parse = props.parse; + this.filter = props.filter; + this.stats = props.stats; + this.sort = props.sort; + this.limit = props.limit; + this.display = props.display; + } + + /** + * String representation of this QueryString. + */ + public toString(): string { + return noUndef({ + fields: this.fields !== undefined ? this.fields.join(', ') : this.fields, + parse: this.parse, + filter: this.filter, + stats: this.stats, + sort: this.sort, + limit: this.limit, + display: this.display, + }).join(' | '); + } +} + +function noUndef(x: QueryStringMap): string[] { + const ret: string[] = []; + for (const [key, value] of Object.entries(x)) { + if (value !== undefined) { + ret.push(`${key} ${value}`); + } + } + return ret; +} + +/** + * Properties for a QueryDefinition + */ +export interface QueryDefinitionProps { + /** + * Name of the query definition. + */ + readonly queryDefinitionName: string; + + /** + * The query string to use for this query definition. + */ + readonly queryString: QueryString; + + /** + * Specify certain log groups for the query definition. + * + * @default - no specified log groups + */ + readonly logGroups?: ILogGroup[]; +} + +/** + * Define a query definition for CloudWatch Logs Insights + */ +export class QueryDefinition extends Resource { + /** + * The ID of the query definition. + * + * @attribute + */ + public readonly queryDefinitionId: string; + + constructor(scope: Construct, id: string, props: QueryDefinitionProps) { + super(scope, id, { + physicalName: props.queryDefinitionName, + }); + + const queryDefinition = new CfnQueryDefinition(this, 'Resource', { + name: props.queryDefinitionName, + queryString: props.queryString.toString(), + logGroupNames: typeof props.logGroups === 'undefined' ? [] : props.logGroups.flatMap(logGroup => logGroup.logGroupName), + }); + + this.queryDefinitionId = queryDefinition.attrQueryDefinitionId; + } +} diff --git a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.ts b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.ts new file mode 100644 index 0000000000000..df930f8ece79b --- /dev/null +++ b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.ts @@ -0,0 +1,29 @@ +import { App, RemovalPolicy, Stack, StackProps } from '@aws-cdk/core'; +import { LogGroup, QueryDefinition, QueryString } from '../lib'; + +class LogsInsightsQueryDefinitionIntegStack extends Stack { + constructor(scope: App, id: string, props?: StackProps) { + super(scope, id, props); + + const logGroup = new LogGroup(this, 'LogGroup', { + removalPolicy: RemovalPolicy.DESTROY, + }); + + new QueryDefinition(this, 'QueryDefinition', { + queryDefinitionName: 'QueryDefinition', + queryString: new QueryString({ + fields: ['@timestamp', '@message'], + parse: '@message "[*] *" as loggingType, loggingMessage', + filter: 'loggingType = "ERROR"', + sort: '@timestamp desc', + limit: 20, + display: 'loggingMessage', + }), + logGroups: [logGroup], + }); + } +} + +const app = new App(); +new LogsInsightsQueryDefinitionIntegStack(app, 'aws-cdk-logs-querydefinition-integ'); +app.synth(); diff --git a/packages/@aws-cdk/aws-logs/test/query-definition.test.ts b/packages/@aws-cdk/aws-logs/test/query-definition.test.ts new file mode 100644 index 0000000000000..7a1fd123387f2 --- /dev/null +++ b/packages/@aws-cdk/aws-logs/test/query-definition.test.ts @@ -0,0 +1,78 @@ +import { Template } from '@aws-cdk/assertions'; +import { Stack } from '@aws-cdk/core'; +import { LogGroup, QueryDefinition, QueryString } from '../lib'; + +describe('query definition', () => { + test('create a query definition', () => { + // GIVEN + const stack = new Stack(); + + // WHEN + new QueryDefinition(stack, 'QueryDefinition', { + queryDefinitionName: 'MyQuery', + queryString: new QueryString({ + fields: ['@timestamp', '@message'], + sort: '@timestamp desc', + limit: 20, + }), + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Logs::QueryDefinition', { + Name: 'MyQuery', + QueryString: 'fields @timestamp, @message | sort @timestamp desc | limit 20', + }); + }); + + test('create a query definition against certain log groups', () => { + // GIVEN + const stack = new Stack(); + + // WHEN + const logGroup = new LogGroup(stack, 'MyLogGroup'); + + new QueryDefinition(stack, 'QueryDefinition', { + queryDefinitionName: 'MyQuery', + queryString: new QueryString({ + fields: ['@timestamp', '@message'], + sort: '@timestamp desc', + limit: 20, + }), + logGroups: [logGroup], + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Logs::QueryDefinition', { + Name: 'MyQuery', + QueryString: 'fields @timestamp, @message | sort @timestamp desc | limit 20', + LogGroupNames: [{ Ref: 'MyLogGroup5C0DAD85' }], + }); + }); + + test('create a query definition with all commands', () => { + // GIVEN + const stack = new Stack(); + + // WHEN + const logGroup = new LogGroup(stack, 'MyLogGroup'); + + new QueryDefinition(stack, 'QueryDefinition', { + queryDefinitionName: 'MyQuery', + queryString: new QueryString({ + fields: ['@timestamp', '@message'], + parse: '@message "[*] *" as loggingType, loggingMessage', + filter: 'loggingType = "ERROR"', + sort: '@timestamp desc', + limit: 20, + display: 'loggingMessage', + }), + logGroups: [logGroup], + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Logs::QueryDefinition', { + Name: 'MyQuery', + QueryString: 'fields @timestamp, @message | parse @message "[*] *" as loggingType, loggingMessage | filter loggingType = "ERROR" | sort @timestamp desc | limit 20 | display loggingMessage', + }); + }); +}); diff --git a/packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/aws-cdk-logs-querydefinition-integ.template.json b/packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/aws-cdk-logs-querydefinition-integ.template.json new file mode 100644 index 0000000000000..3429e2b6319f6 --- /dev/null +++ b/packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/aws-cdk-logs-querydefinition-integ.template.json @@ -0,0 +1,24 @@ +{ + "Resources": { + "LogGroupF5B46931": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "RetentionInDays": 731 + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "QueryDefinition4190BC36": { + "Type": "AWS::Logs::QueryDefinition", + "Properties": { + "Name": "QueryDefinition", + "QueryString": "fields @timestamp, @message | parse @message \"[*] *\" as loggingType, loggingMessage | filter loggingType = \"ERROR\" | sort @timestamp desc | limit 20 | display loggingMessage", + "LogGroupNames": [ + { + "Ref": "LogGroupF5B46931" + } + ] + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/cdk.out b/packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/cdk.out new file mode 100644 index 0000000000000..90bef2e09ad39 --- /dev/null +++ b/packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/cdk.out @@ -0,0 +1 @@ +{"version":"17.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/manifest.json b/packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/manifest.json new file mode 100644 index 0000000000000..45f591b3bd19e --- /dev/null +++ b/packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/manifest.json @@ -0,0 +1,34 @@ +{ + "version": "17.0.0", + "artifacts": { + "Tree": { + "type": "cdk:tree", + "properties": { + "file": "tree.json" + } + }, + "aws-cdk-logs-querydefinition-integ": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "aws-cdk-logs-querydefinition-integ.template.json", + "validateOnSynth": false + }, + "metadata": { + "/aws-cdk-logs-querydefinition-integ/LogGroup/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "LogGroupF5B46931" + } + ], + "/aws-cdk-logs-querydefinition-integ/QueryDefinition/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "QueryDefinition4190BC36" + } + ] + }, + "displayName": "aws-cdk-logs-querydefinition-integ" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/tree.json b/packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/tree.json new file mode 100644 index 0000000000000..ac464cc6aeb70 --- /dev/null +++ b/packages/@aws-cdk/aws-logs/test/save-logs-insights-query-definition.integ.snapshot/tree.json @@ -0,0 +1,85 @@ +{ + "version": "tree-0.1", + "tree": { + "id": "App", + "path": "", + "children": { + "Tree": { + "id": "Tree", + "path": "Tree", + "constructInfo": { + "fqn": "@aws-cdk/core.Construct", + "version": "0.0.0" + } + }, + "aws-cdk-logs-querydefinition-integ": { + "id": "aws-cdk-logs-querydefinition-integ", + "path": "aws-cdk-logs-querydefinition-integ", + "children": { + "LogGroup": { + "id": "LogGroup", + "path": "aws-cdk-logs-querydefinition-integ/LogGroup", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-logs-querydefinition-integ/LogGroup/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Logs::LogGroup", + "aws:cdk:cloudformation:props": { + "retentionInDays": 731 + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-logs.CfnLogGroup", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-logs.LogGroup", + "version": "0.0.0" + } + }, + "QueryDefinition": { + "id": "QueryDefinition", + "path": "aws-cdk-logs-querydefinition-integ/QueryDefinition", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-logs-querydefinition-integ/QueryDefinition/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Logs::QueryDefinition", + "aws:cdk:cloudformation:props": { + "name": "QueryDefinition", + "queryString": "fields @timestamp, @message | parse @message \"[*] *\" as loggingType, loggingMessage | filter loggingType = \"ERROR\" | sort @timestamp desc | limit 20 | display loggingMessage", + "logGroupNames": [ + { + "Ref": "LogGroupF5B46931" + } + ] + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-logs.CfnQueryDefinition", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-logs.QueryDefinition", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/core.Stack", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/core.App", + "version": "0.0.0" + } + } +} \ No newline at end of file