feat(logs): Add support for multiple parse and filter statements in Q…

…ueryString (#24022)

Currently, `QueryString` is limited to only allow a single line/statement to be provided for each query command. For some commands this makes sense (e.g. `limit`), but for `parse` and `filter` this can be limiting. Adding multiple lines for these commands is possible in the AWS console, so it makes sense for it to be supported in CDK too.

In this PR, I'm adding support for `filter` and `parse` to be provided as `string` or `string[]`, and adding/modifying various utility methods to handle this ambiguity.

I left the existing tests the same to verify no breaking changes, and added a new test for the newly enabled behavior.

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

packages/@aws-cdk/aws-logs/README.md

@@ -330,6 +330,14 @@ new logs.QueryDefinition(this, 'QueryDefinition', {
   queryDefinitionName: 'MyQuery',
   queryString: new logs.QueryString({
     fields: ['@timestamp', '@message'],
+    parseStatements: [
+      '@message "[*] *" as loggingType, loggingMessage',
+      '@message "<*>: *" as differentLoggingType, differentLoggingMessage',
+    ],
+    filterStatements: [
+      'loggingType = "ERROR"',
+      'loggingMessage = "A very strange error occurred!"',
+    ],
     sort: '@timestamp desc',
     limit: 20,
   }),

packages/@aws-cdk/aws-logs/lib/query-definition.ts

@@ -16,19 +16,43 @@ export interface QueryStringProps {
   readonly fields?: string[];
 
   /**
-  * Extracts data from a log field and creates one or more ephemeral fields that you can process further in the query.
+  * A single statement for parsing data from a log field and creating ephemeral fields that can
+  * be processed further in the query.
   *
+  * @deprecated Use `parseStatements` instead
   * @default - no parse in QueryString
   */
   readonly parse?: string;
 
   /**
-  * Filters the results of a query that's based on one or more conditions.
+  * An array of one or more statements for parsing data from a log field and creating ephemeral
+  * fields that can be processed further in the query. Each provided statement generates a separate
+  * parse line in the query string.
   *
+  * Note: If provided, this property overrides any value provided for the `parse` property.
+  *
+  * @default - no parse in QueryString
+  */
+  readonly parseStatements?: string[];
+
+  /**
+  * A single statement for filtering the results of a query based on a boolean expression.
+  *
+  * @deprecated Use `filterStatements` instead
   * @default - no filter in QueryString
   */
   readonly filter?: string;
 
+  /**
+  * An array of one or more statements for filtering the results of a query based on a boolean
+  * expression. Each provided statement generates a separate filter line in the query string.
+  *
+  * Note: If provided, this property overrides any value provided for the `filter` property.
+  *
+  * @default - no filter in QueryString
+  */
+  readonly filterStatements?: string[];
+
   /**
   * Uses log field values to calculate aggregate statistics.
   *
@@ -58,62 +82,88 @@ export interface QueryStringProps {
   readonly display?: string;
 }
 
-interface QueryStringMap {
-  readonly fields?: string,
-  readonly parse?: string,
-  readonly filter?: string,
-  readonly stats?: string,
-  readonly sort?: string,
-  readonly limit?: Number,
-  readonly display?: string,
-}
-
 /**
  * Define a QueryString
  */
 export class QueryString {
   private readonly fields?: string[];
-  private readonly parse?: string;
-  private readonly filter?: string;
+  private readonly parse: string[];
+  private readonly filter: string[];
   private readonly stats?: string;
   private readonly sort?: string;
   private readonly limit?: Number;
   private readonly display?: string;
 
   constructor(props: QueryStringProps = {}) {
     this.fields = props.fields;
-    this.parse = props.parse;
-    this.filter = props.filter;
     this.stats = props.stats;
     this.sort = props.sort;
     this.limit = props.limit;
     this.display = props.display;
+
+    // Determine parsing by either the parseStatements or parse properties, or default to empty array
+    if (props.parseStatements) {
+      this.parse = props.parseStatements;
+    } else if (props.parse) {
+      this.parse = [props.parse];
+    } else {
+      this.parse = [];
+    }
+
+    // Determine filtering by either the filterStatements or filter properties, or default to empty array
+    if (props.filterStatements) {
+      this.filter = props.filterStatements;
+    } else if (props.filter) {
+      this.filter = [props.filter];
+    } else {
+      this.filter = [];
+    }
   }
 
   /**
   * String representation of this QueryString.
   */
   public toString(): string {
-    return noUndef({
-      fields: this.fields !== undefined ? this.fields.join(', ') : this.fields,
-      parse: this.parse,
-      filter: this.filter,
-      stats: this.stats,
-      sort: this.sort,
-      limit: this.limit,
-      display: this.display,
-    }).join('\n| ');
+    return [
+      this.buildQueryLine('fields', this.fields?.join(', ')),
+      ...this.buildQueryLines('parse', this.parse),
+      ...this.buildQueryLines('filter', this.filter),
+      this.buildQueryLine('stats', this.stats),
+      this.buildQueryLine('sort', this.sort),
+      this.buildQueryLine('limit', this.limit?.toString()),
+      this.buildQueryLine('display', this.display),
+    ].filter(
+      (queryLine) => queryLine !== undefined && queryLine.length > 0,
+    ).join('\n| ');
   }
-}
 
-function noUndef(x: QueryStringMap): string[] {
-  const ret: string[] = [];
-  for (const [key, value] of Object.entries(x)) {
-    if (value !== undefined) {
-      ret.push(`${key} ${value}`);
+  /**
+   * Build an array of query lines given a command and statement(s).
+   *
+   * @param command a query command
+   * @param statements one or more query statements for the specified command, or undefined
+   * @returns an array of the query string lines generated from the provided command and statements
+   */
+  private buildQueryLines(command: string, statements?: string[]): string[] {
+    if (statements === undefined) {
+      return [];
     }
+
+    return statements.map(
+      (statement: string): string => this.buildQueryLine(command, statement),
+    );
+  }
+
+  /**
+   * Build a single query line given a command and statement.
+   *
+   * @param command a query command
+   * @param statement a single query statement
+   * @returns a single query string line generated from the provided command and statement
+   */
+  private buildQueryLine(command: string, statement?: string): string {
+    return statement ? `${command} ${statement}` : '';
   }
-  return ret;
 }
 
 /**

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/LogsInsightsQueryDefinitionIntegTestDefaultTestDeployAssert902BAAD5.assets.json

@@ -1,5 +1,5 @@
 {
-  "version": "21.0.0",
+  "version": "29.0.0",
   "files": {
     "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": {
       "source": {

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.assets.json

@@ -1,15 +1,15 @@
 {
-  "version": "21.0.0",
+  "version": "29.0.0",
   "files": {
-    "e6a0a51961925fbc37d9b81431449c256ed453f98089eb70f83850f237b4d722": {
+    "3546c78647ea20567832041c960a034787f9bfc0128226ea8fbc0894366a4dd0": {
       "source": {
         "path": "aws-cdk-logs-insights-querydefinition-integ.template.json",
         "packaging": "file"
       },
       "destinations": {
         "current_account-current_region": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "e6a0a51961925fbc37d9b81431449c256ed453f98089eb70f83850f237b4d722.json",
+          "objectKey": "3546c78647ea20567832041c960a034787f9bfc0128226ea8fbc0894366a4dd0.json",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.template.json

@@ -19,6 +19,18 @@
      }
     ]
    }
+  },
+  "QueryDefinitionWithMultipleStatements58A3EF74": {
+   "Type": "AWS::Logs::QueryDefinition",
+   "Properties": {
+    "Name": "QueryDefinitionWithMultipleStatements",
+    "QueryString": "fields @timestamp, @message\n| parse @message \"[*] *\" as loggingType, loggingMessage\n| parse @message \"<*>: *\" as differentLoggingType, differentLoggingMessage\n| filter loggingType = \"ERROR\"\n| filter loggingMessage = \"A very strange error occurred!\"\n| sort @timestamp desc\n| limit 20\n| display loggingMessage",
+    "LogGroupNames": [
+     {
+      "Ref": "LogGroupF5B46931"
+     }
+    ]
+   }
   }
  },
  "Parameters": {

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/cdk.out

@@ -1 +1 @@
-{"version":"21.0.0"}
+{"version":"29.0.0"}

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/integ.json

@@ -1,5 +1,5 @@
 {
-  "version": "21.0.0",
+  "version": "29.0.0",
   "testCases": {
     "LogsInsightsQueryDefinitionIntegTest/DefaultTest": {
       "stacks": [

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/manifest.json

@@ -1,5 +1,5 @@
 {
-  "version": "21.0.0",
+  "version": "29.0.0",
   "artifacts": {
     "aws-cdk-logs-insights-querydefinition-integ.assets": {
       "type": "cdk:asset-manifest",
@@ -17,7 +17,7 @@
         "validateOnSynth": false,
         "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}",
         "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}",
-        "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/e6a0a51961925fbc37d9b81431449c256ed453f98089eb70f83850f237b4d722.json",
+        "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/3546c78647ea20567832041c960a034787f9bfc0128226ea8fbc0894366a4dd0.json",
         "requiresBootstrapStackVersion": 6,
         "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version",
         "additionalDependencies": [
@@ -45,6 +45,12 @@
             "data": "QueryDefinition4190BC36"
           }
         ],
+        "/aws-cdk-logs-insights-querydefinition-integ/QueryDefinitionWithMultipleStatements/Resource": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "QueryDefinitionWithMultipleStatements58A3EF74"
+          }
+        ],
         "/aws-cdk-logs-insights-querydefinition-integ/BootstrapVersion": [
           {
             "type": "aws:cdk:logicalId",

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/tree.json

@@ -62,6 +62,36 @@
               "version": "0.0.0"
             }
           },
+          "QueryDefinitionWithMultipleStatements": {
+            "id": "QueryDefinitionWithMultipleStatements",
+            "path": "aws-cdk-logs-insights-querydefinition-integ/QueryDefinitionWithMultipleStatements",
+            "children": {
+              "Resource": {
+                "id": "Resource",
+                "path": "aws-cdk-logs-insights-querydefinition-integ/QueryDefinitionWithMultipleStatements/Resource",
+                "attributes": {
+                  "aws:cdk:cloudformation:type": "AWS::Logs::QueryDefinition",
+                  "aws:cdk:cloudformation:props": {
+                    "name": "QueryDefinitionWithMultipleStatements",
+                    "queryString": "fields @timestamp, @message\n| parse @message \"[*] *\" as loggingType, loggingMessage\n| parse @message \"<*>: *\" as differentLoggingType, differentLoggingMessage\n| filter loggingType = \"ERROR\"\n| filter loggingMessage = \"A very strange error occurred!\"\n| sort @timestamp desc\n| limit 20\n| display loggingMessage",
+                    "logGroupNames": [
+                      {
+                        "Ref": "LogGroupF5B46931"
+                      }
+                    ]
+                  }
+                },
+                "constructInfo": {
+                  "fqn": "@aws-cdk/aws-logs.CfnQueryDefinition",
+                  "version": "0.0.0"
+                }
+              }
+            },
+            "constructInfo": {
+              "fqn": "@aws-cdk/aws-logs.QueryDefinition",
+              "version": "0.0.0"
+            }
+          },
           "BootstrapVersion": {
             "id": "BootstrapVersion",
             "path": "aws-cdk-logs-insights-querydefinition-integ/BootstrapVersion",
@@ -97,7 +127,7 @@
                 "path": "LogsInsightsQueryDefinitionIntegTest/DefaultTest/Default",
                 "constructInfo": {
                   "fqn": "constructs.Construct",
-                  "version": "10.1.161"
+                  "version": "10.1.237"
                 }
               },
               "DeployAssert": {
@@ -143,7 +173,7 @@
         "path": "Tree",
         "constructInfo": {
           "fqn": "constructs.Construct",
-          "version": "10.1.161"
+          "version": "10.1.237"
         }
       }
     },

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.ts

@@ -11,6 +11,7 @@ class LogsInsightsQueryDefinitionIntegStack extends Stack {
       removalPolicy: RemovalPolicy.DESTROY,
     });
 
+    // Test query creation with single parse and filter statements
     new QueryDefinition(this, 'QueryDefinition', {
       queryDefinitionName: 'QueryDefinition',
       queryString: new QueryString({
@@ -23,6 +24,26 @@ class LogsInsightsQueryDefinitionIntegStack extends Stack {
       }),
       logGroups: [logGroup],
     });
+
+    // Test query creation with multiple parse and filter statements
+    new QueryDefinition(this, 'QueryDefinitionWithMultipleStatements', {
+      queryDefinitionName: 'QueryDefinitionWithMultipleStatements',
+      queryString: new QueryString({
+        fields: ['@timestamp', '@message'],
+        parseStatements: [
+          '@message "[*] *" as loggingType, loggingMessage',
+          '@message "<*>: *" as differentLoggingType, differentLoggingMessage',
+        ],
+        filterStatements: [
+          'loggingType = "ERROR"',
+          'loggingMessage = "A very strange error occurred!"',
+        ],
+        sort: '@timestamp desc',
+        limit: 20,
+        display: 'loggingMessage',
+      }),
+      logGroups: [logGroup],
+    });
   }
 }