feat(logs): Add support for multiple parse and filter statements in QueryString

packages/@aws-cdk/aws-logs/README.md

@@ -330,6 +330,14 @@ new logs.QueryDefinition(this, 'QueryDefinition', {
  queryDefinitionName: 'MyQuery',
  queryString: new logs.QueryString({
  fields: ['@timestamp', '@message'],
+ parseStatements: [
+ '@message "[*] *" as loggingType, loggingMessage',
+ '@message "<*>: *" as differentLoggingType, differentLoggingMessage',
+ ],
+ filterStatements: [
+ 'loggingType = "ERROR"',
+ 'loggingMessage = "A very strange error occurred!"',
+ ],
  sort: '@timestamp desc',
  limit: 20,
  }),

packages/@aws-cdk/aws-logs/lib/query-definition.ts

@@ -16,19 +16,43 @@ export interface QueryStringProps {
  readonly fields?: string[];
 
  /**
- * Extracts data from a log field and creates one or more ephemeral fields that you can process further in the query.
+ * A single statement for parsing data from a log field and creating ephemeral fields that can
+ * be processed further in the query.
  *
+ * @deprecated Use `parseStatements` instead
  * @default - no parse in QueryString
  */
  readonly parse?: string;
 
  /**
- * Filters the results of a query that's based on one or more conditions.
+ * An array of one or more statements for parsing data from a log field and creating ephemeral
+ * fields that can be processed further in the query. Each provided statement generates a separate
+ * parse line in the query string.
  *
+ * Note: If provided, this property overrides any value provided for the `parse` property.
+ *
+ * @default - no parse in QueryString
+ */
+ readonly parseStatements?: string[];
+
+ /**
+ * A single statement for filtering the results of a query based on a boolean expression.
+ *
+ * @deprecated Use `filterStatements` instead
  * @default - no filter in QueryString
  */
  readonly filter?: string;
 
+ /**
+ * An array of one or more statements for filtering the results of a query based on a boolean
+ * expression. Each provided statement generates a separate filter line in the query string.
+ *
+ * Note: If provided, this property overrides any value provided for the `filter` property.
+ *
+ * @default - no filter in QueryString
+ */
+ readonly filterStatements?: string[];
+
  /**
  * Uses log field values to calculate aggregate statistics.
  *
@@ -58,62 +82,88 @@ export interface QueryStringProps {
  readonly display?: string;
 }
 
-interface QueryStringMap {
- readonly fields?: string,
- readonly parse?: string,
- readonly filter?: string,
- readonly stats?: string,
- readonly sort?: string,
- readonly limit?: Number,
- readonly display?: string,
-}
-
 /**
  * Define a QueryString
  */
 export class QueryString {
  private readonly fields?: string[];
- private readonly parse?: string;
- private readonly filter?: string;
+ private readonly parse: string[];
+ private readonly filter: string[];
  private readonly stats?: string;
  private readonly sort?: string;
  private readonly limit?: Number;
  private readonly display?: string;
 
  constructor(props: QueryStringProps = {}) {
  this.fields = props.fields;
- this.parse = props.parse;
- this.filter = props.filter;
  this.stats = props.stats;
  this.sort = props.sort;
  this.limit = props.limit;
  this.display = props.display;
+
+ // Determine parsing by either the parseStatements or parse properties, or default to empty array
+ if (props.parseStatements) {
+ this.parse = props.parseStatements;
+ } else if (props.parse) {
+ this.parse = [props.parse];
+ } else {
+ this.parse = [];
+ }
+
+ // Determine filtering by either the filterStatements or filter properties, or default to empty array
+ if (props.filterStatements) {
+ this.filter = props.filterStatements;
+ } else if (props.filter) {
+ this.filter = [props.filter];
+ } else {
+ this.filter = [];
+ }
  }
 
  /**
  * String representation of this QueryString.
  */
  public toString(): string {
- return noUndef({
- fields: this.fields !== undefined ? this.fields.join(', ') : this.fields,
- parse: this.parse,
- filter: this.filter,
- stats: this.stats,
- sort: this.sort,
- limit: this.limit,
- display: this.display,
- }).join('\n| ');
+ return [
+ this.buildQueryLine('fields', this.fields?.join(', ')),
+ ...this.buildQueryLines('parse', this.parse),
+ ...this.buildQueryLines('filter', this.filter),
+ this.buildQueryLine('stats', this.stats),
+ this.buildQueryLine('sort', this.sort),
+ this.buildQueryLine('limit', this.limit?.toString()),
+ this.buildQueryLine('display', this.display),
+ ].filter(
+ (queryLine) => queryLine !== undefined && queryLine.length > 0,
+ ).join('\n| ');
  }
-}
 
-function noUndef(x: QueryStringMap): string[] {
- const ret: string[] = [];
- for (const [key, value] of Object.entries(x)) {
- if (value !== undefined) {
- ret.push(`${key} ${value}`);
+ /**
+ * Build an array of query lines given a command and statement(s).
+ *
+ * @param command a query command
+ * @param statements one or more query statements for the specified command, or undefined
+ * @returns an array of the query string lines generated from the provided command and statements
+ */
+ private buildQueryLines(command: string, statements?: string[]): string[] {
+ if (statements === undefined) {
+ return [];
  }
+
+ return statements.map(
+ (statement: string): string => this.buildQueryLine(command, statement),
+ );
+ }
+
+ /**
+ * Build a single query line given a command and statement.
+ *
+ * @param command a query command
+ * @param statement a single query statement
+ * @returns a single query string line generated from the provided command and statement
+ */
+ private buildQueryLine(command: string, statement?: string): string {
+ return statement ? `${command} ${statement}` : '';
  }
- return ret;
 }
 
 /**

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/LogsInsightsQueryDefinitionIntegTestDefaultTestDeployAssert902BAAD5.assets.json

@@ -1,5 +1,5 @@
 {
- "version": "21.0.0",
+ "version": "29.0.0",
  "files": {
  "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": {
  "source": {

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.assets.json

@@ -1,15 +1,15 @@
 {
- "version": "21.0.0",
+ "version": "29.0.0",
  "files": {
- "e6a0a51961925fbc37d9b81431449c256ed453f98089eb70f83850f237b4d722": {
+ "3546c78647ea20567832041c960a034787f9bfc0128226ea8fbc0894366a4dd0": {
  "source": {
  "path": "aws-cdk-logs-insights-querydefinition-integ.template.json",
  "packaging": "file"
  },
  "destinations": {
  "current_account-current_region": {
  "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
- "objectKey": "e6a0a51961925fbc37d9b81431449c256ed453f98089eb70f83850f237b4d722.json",
+ "objectKey": "3546c78647ea20567832041c960a034787f9bfc0128226ea8fbc0894366a4dd0.json",
  "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
  }
  }

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.template.json

@@ -19,6 +19,18 @@
  }
  ]
  }
+ },
+ "QueryDefinitionWithMultipleStatements58A3EF74": {
+ "Type": "AWS::Logs::QueryDefinition",
+ "Properties": {
+ "Name": "QueryDefinitionWithMultipleStatements",
+ "QueryString": "fields @timestamp, @message\n| parse @message \"[*] *\" as loggingType, loggingMessage\n| parse @message \"<*>: *\" as differentLoggingType, differentLoggingMessage\n| filter loggingType = \"ERROR\"\n| filter loggingMessage = \"A very strange error occurred!\"\n| sort @timestamp desc\n| limit 20\n| display loggingMessage",
+ "LogGroupNames": [
+ {
+ "Ref": "LogGroupF5B46931"
+ }
+ ]
+ }
  }
  },
  "Parameters": {

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/cdk.out

@@ -1 +1 @@
-{"version":"21.0.0"}
+{"version":"29.0.0"}

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/integ.json

@@ -1,5 +1,5 @@
 {
- "version": "21.0.0",
+ "version": "29.0.0",
  "testCases": {
  "LogsInsightsQueryDefinitionIntegTest/DefaultTest": {
  "stacks": [

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/manifest.json

@@ -1,5 +1,5 @@
 {
- "version": "21.0.0",
+ "version": "29.0.0",
  "artifacts": {
  "aws-cdk-logs-insights-querydefinition-integ.assets": {
  "type": "cdk:asset-manifest",
@@ -17,7 +17,7 @@
  "validateOnSynth": false,
  "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}",
  "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}",
- "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/e6a0a51961925fbc37d9b81431449c256ed453f98089eb70f83850f237b4d722.json",
+ "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/3546c78647ea20567832041c960a034787f9bfc0128226ea8fbc0894366a4dd0.json",
  "requiresBootstrapStackVersion": 6,
  "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version",
  "additionalDependencies": [
@@ -45,6 +45,12 @@
  "data": "QueryDefinition4190BC36"
  }
  ],
+ "/aws-cdk-logs-insights-querydefinition-integ/QueryDefinitionWithMultipleStatements/Resource": [
+ {
+ "type": "aws:cdk:logicalId",
+ "data": "QueryDefinitionWithMultipleStatements58A3EF74"
+ }
+ ],
  "/aws-cdk-logs-insights-querydefinition-integ/BootstrapVersion": [
  {
  "type": "aws:cdk:logicalId",

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/tree.json

@@ -62,6 +62,36 @@
  "version": "0.0.0"
  }
  },
+ "QueryDefinitionWithMultipleStatements": {
+ "id": "QueryDefinitionWithMultipleStatements",
+ "path": "aws-cdk-logs-insights-querydefinition-integ/QueryDefinitionWithMultipleStatements",
+ "children": {
+ "Resource": {
+ "id": "Resource",
+ "path": "aws-cdk-logs-insights-querydefinition-integ/QueryDefinitionWithMultipleStatements/Resource",
+ "attributes": {
+ "aws:cdk:cloudformation:type": "AWS::Logs::QueryDefinition",
+ "aws:cdk:cloudformation:props": {
+ "name": "QueryDefinitionWithMultipleStatements",
+ "queryString": "fields @timestamp, @message\n| parse @message \"[*] *\" as loggingType, loggingMessage\n| parse @message \"<*>: *\" as differentLoggingType, differentLoggingMessage\n| filter loggingType = \"ERROR\"\n| filter loggingMessage = \"A very strange error occurred!\"\n| sort @timestamp desc\n| limit 20\n| display loggingMessage",
+ "logGroupNames": [
+ {
+ "Ref": "LogGroupF5B46931"
+ }
+ ]
+ }
+ },
+ "constructInfo": {
+ "fqn": "@aws-cdk/aws-logs.CfnQueryDefinition",
+ "version": "0.0.0"
+ }
+ }
+ },
+ "constructInfo": {
+ "fqn": "@aws-cdk/aws-logs.QueryDefinition",
+ "version": "0.0.0"
+ }
+ },
  "BootstrapVersion": {
  "id": "BootstrapVersion",
  "path": "aws-cdk-logs-insights-querydefinition-integ/BootstrapVersion",
@@ -97,7 +127,7 @@
  "path": "LogsInsightsQueryDefinitionIntegTest/DefaultTest/Default",
  "constructInfo": {
  "fqn": "constructs.Construct",
- "version": "10.1.161"
+ "version": "10.1.237"
  }
  },
  "DeployAssert": {
@@ -143,7 +173,7 @@
  "path": "Tree",
  "constructInfo": {
  "fqn": "constructs.Construct",
- "version": "10.1.161"
+ "version": "10.1.237"
  }
  }
  },

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.ts

@@ -11,6 +11,7 @@ class LogsInsightsQueryDefinitionIntegStack extends Stack {
  removalPolicy: RemovalPolicy.DESTROY,
  });
 
+ // Test query creation with single parse and filter statements
  new QueryDefinition(this, 'QueryDefinition', {
  queryDefinitionName: 'QueryDefinition',
  queryString: new QueryString({
@@ -23,6 +24,26 @@ class LogsInsightsQueryDefinitionIntegStack extends Stack {
  }),
  logGroups: [logGroup],
  });
+
+ // Test query creation with multiple parse and filter statements
+ new QueryDefinition(this, 'QueryDefinitionWithMultipleStatements', {
+ queryDefinitionName: 'QueryDefinitionWithMultipleStatements',
+ queryString: new QueryString({
+ fields: ['@timestamp', '@message'],
+ parseStatements: [
+ '@message "[*] *" as loggingType, loggingMessage',
+ '@message "<*>: *" as differentLoggingType, differentLoggingMessage',
+ ],
+ filterStatements: [
+ 'loggingType = "ERROR"',
+ 'loggingMessage = "A very strange error occurred!"',
+ ],
+ sort: '@timestamp desc',
+ limit: 20,
+ display: 'loggingMessage',
+ }),
+ logGroups: [logGroup],
+ });
  }
 }