From 11da280fb88f0e8b67fcbecfc248566e4fb1de66 Mon Sep 17 00:00:00 2001 From: Jim Wild <james.wild6@nhs.net> Date: Wed, 16 Oct 2024 16:03:24 +0000 Subject: [PATCH 1/6] Add an alarm on average writes/sec --- SAMtemplates/alarms/main.yaml | 32 ++++++++++++++++++++++++++++++++ package-lock.json | 23 +++++------------------ 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/SAMtemplates/alarms/main.yaml b/SAMtemplates/alarms/main.yaml index e2a60ff75..8b5e9f5b8 100644 --- a/SAMtemplates/alarms/main.yaml +++ b/SAMtemplates/alarms/main.yaml @@ -220,6 +220,38 @@ Resources: OKActions: - !ImportValue lambda-resources:SlackAlertsSnsTopicArn + DynamoDBWriteConsumptionAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub "${AWS::StackName}_DynamoDB_ConsumedWriteCapacity" + ActionsEnabled: !Ref EnableAlerts + AlarmActions: + - !GetAtt SlackAlertsSnsTopic.TopicArn + InsufficientDataActions: + - !GetAtt SlackAlertsSnsTopic.TopicArn + OKActions: + - !GetAtt SlackAlertsSnsTopic.TopicArn + EvaluationPeriods: 1 + Threshold: !Ref WriteCapacityAlertThreshold + ComparisonOperator: GreaterThanOrEqualToThreshold + TreatMissingData: notBreaching + Metrics: + - Id: e1 + Label: Consumed + ReturnData: true + Expression: m1/PERIOD(m1) + - Id: m1 + ReturnData: false + MetricStat: + Metric: + Namespace: AWS/DynamoDB + MetricName: ConsumedWriteCapacityUnits + Dimensions: + - Name: TableName + Value: psu-PrescriptionStatusUpdates + Period: 60 + Stat: Sum + CPSUFhirConverterErrorLogsMetricFilter: Type: AWS::Logs::MetricFilter Properties: diff --git a/package-lock.json b/package-lock.json index 892a50952..6987679fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4448,6 +4448,7 @@ "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -6849,7 +6850,8 @@ "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true }, "node_modules/fast-levenshtein": { "version": "2.0.6", @@ -8990,7 +8992,8 @@ "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -13167,22 +13170,6 @@ "node": ">= 0.8.0" } }, - "node_modules/prettier": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz", - "integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==", - "dev": true, - "peer": true, - "bin": { - "prettier": "bin/prettier.cjs" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/prettier/prettier?sponsor=1" - } - }, "node_modules/prettier-linter-helpers": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", From 24785d0b691e6286ceabbfd7a583f88b292fada6 Mon Sep 17 00:00:00 2001 From: Jim Wild <james.wild6@nhs.net> Date: Wed, 16 Oct 2024 16:07:19 +0000 Subject: [PATCH 2/6] Parameterise threshold --- SAMtemplates/alarms/main.yaml | 5 +++++ SAMtemplates/main_template.yaml | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/SAMtemplates/alarms/main.yaml b/SAMtemplates/alarms/main.yaml index 8b5e9f5b8..96b2c30b7 100644 --- a/SAMtemplates/alarms/main.yaml +++ b/SAMtemplates/alarms/main.yaml @@ -22,6 +22,11 @@ Parameters: EnableAlerts: Type: String + WriteCapacityAlertThreshold: + Type: Number + Description: Threshold for the average consumed write capacity units per minute to trigger an alarm + Default: 500 + Resources: GetStatusUpdatesErrorsLogsMetricFilter: Type: AWS::Logs::MetricFilter diff --git a/SAMtemplates/main_template.yaml b/SAMtemplates/main_template.yaml index aa71b0cc8..14f12ddde 100644 --- a/SAMtemplates/main_template.yaml +++ b/SAMtemplates/main_template.yaml @@ -81,6 +81,12 @@ Parameters: - true - false + WriteCapacityAlertThreshold: + Type: Number + Description: Threshold for the average consumed write capacity units per minute to trigger an alarm + Default: 500 + + Resources: Tables: Type: AWS::Serverless::Application @@ -150,4 +156,5 @@ Resources: UpdatePrescriptionStatusFunctionName: !GetAtt Functions.Outputs.UpdatePrescriptionStatusFunctionName PrescriptionStatusUpdatesTableName: !GetAtt Tables.Outputs.PrescriptionStatusUpdatesTableName ConvertRequestToFhirFormatFunctionName: !GetAtt Functions.Outputs.ConvertRequestToFhirFormatFunctionName + WriteCapacityAlertThreshold: !Ref WriteCapacityAlertThreshold EnableAlerts: !Ref EnableAlerts From a8882ab631d13ef41560f84f7261576e123ade0b Mon Sep 17 00:00:00 2001 From: Jim Wild <james.wild6@nhs.net> Date: Thu, 17 Oct 2024 08:08:57 +0000 Subject: [PATCH 3/6] Fix reference --- SAMtemplates/alarms/main.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SAMtemplates/alarms/main.yaml b/SAMtemplates/alarms/main.yaml index 96b2c30b7..1ccda7761 100644 --- a/SAMtemplates/alarms/main.yaml +++ b/SAMtemplates/alarms/main.yaml @@ -231,11 +231,11 @@ Resources: AlarmName: !Sub "${AWS::StackName}_DynamoDB_ConsumedWriteCapacity" ActionsEnabled: !Ref EnableAlerts AlarmActions: - - !GetAtt SlackAlertsSnsTopic.TopicArn + - !GetAtt lambda-resources:SlackAlertsSnsTopicArn InsufficientDataActions: - - !GetAtt SlackAlertsSnsTopic.TopicArn + - !GetAtt lambda-resources:SlackAlertsSnsTopicArn OKActions: - - !GetAtt SlackAlertsSnsTopic.TopicArn + - !GetAtt lambda-resources:SlackAlertsSnsTopicArn EvaluationPeriods: 1 Threshold: !Ref WriteCapacityAlertThreshold ComparisonOperator: GreaterThanOrEqualToThreshold From 1092bc651e46ca670cc42a8c2dbb8388c3dbfc0d Mon Sep 17 00:00:00 2001 From: Jim Wild <james.wild6@nhs.net> Date: Thu, 17 Oct 2024 08:26:19 +0000 Subject: [PATCH 4/6] ref is not importvalue --- SAMtemplates/alarms/main.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SAMtemplates/alarms/main.yaml b/SAMtemplates/alarms/main.yaml index 1ccda7761..ce14c32c5 100644 --- a/SAMtemplates/alarms/main.yaml +++ b/SAMtemplates/alarms/main.yaml @@ -231,11 +231,11 @@ Resources: AlarmName: !Sub "${AWS::StackName}_DynamoDB_ConsumedWriteCapacity" ActionsEnabled: !Ref EnableAlerts AlarmActions: - - !GetAtt lambda-resources:SlackAlertsSnsTopicArn + - !ImportValue lambda-resources:SlackAlertsSnsTopicArn InsufficientDataActions: - - !GetAtt lambda-resources:SlackAlertsSnsTopicArn + - !ImportValue lambda-resources:SlackAlertsSnsTopicArn OKActions: - - !GetAtt lambda-resources:SlackAlertsSnsTopicArn + - !ImportValue lambda-resources:SlackAlertsSnsTopicArn EvaluationPeriods: 1 Threshold: !Ref WriteCapacityAlertThreshold ComparisonOperator: GreaterThanOrEqualToThreshold From a9608e2425041167fab457362422ca0c836aaa06 Mon Sep 17 00:00:00 2001 From: Jim Wild <james.wild6@nhs.net> Date: Thu, 17 Oct 2024 08:59:00 +0000 Subject: [PATCH 5/6] use ref to table name --- SAMtemplates/alarms/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SAMtemplates/alarms/main.yaml b/SAMtemplates/alarms/main.yaml index ce14c32c5..30cdd86e1 100644 --- a/SAMtemplates/alarms/main.yaml +++ b/SAMtemplates/alarms/main.yaml @@ -253,7 +253,7 @@ Resources: MetricName: ConsumedWriteCapacityUnits Dimensions: - Name: TableName - Value: psu-PrescriptionStatusUpdates + Value: !Ref PrescriptionStatusUpdatesTableName Period: 60 Stat: Sum From 6b8caa297ca5d420d281077a0bb95c03a4063454 Mon Sep 17 00:00:00 2001 From: Jim Wild <james.wild6@nhs.net> Date: Thu, 17 Oct 2024 09:49:25 +0000 Subject: [PATCH 6/6] Alter the trigger metric --- SAMtemplates/alarms/main.yaml | 23 +++++++++++++++++------ SAMtemplates/main_template.yaml | 8 ++++---- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/SAMtemplates/alarms/main.yaml b/SAMtemplates/alarms/main.yaml index 30cdd86e1..f24363045 100644 --- a/SAMtemplates/alarms/main.yaml +++ b/SAMtemplates/alarms/main.yaml @@ -22,10 +22,10 @@ Parameters: EnableAlerts: Type: String - WriteCapacityAlertThreshold: + DynamoDBUtilizationPercentageThreshold: Type: Number - Description: Threshold for the average consumed write capacity units per minute to trigger an alarm - Default: 500 + Description: When the database usage crosses this percentage of provisioned capacity, trigger an alarm + Default: 90 Resources: GetStatusUpdatesErrorsLogsMetricFilter: @@ -237,14 +237,14 @@ Resources: OKActions: - !ImportValue lambda-resources:SlackAlertsSnsTopicArn EvaluationPeriods: 1 - Threshold: !Ref WriteCapacityAlertThreshold + Threshold: !Ref DynamoDBUtilizationPercentageThreshold ComparisonOperator: GreaterThanOrEqualToThreshold TreatMissingData: notBreaching Metrics: - Id: e1 - Label: Consumed + Label: UtilizationPercentage ReturnData: true - Expression: m1/PERIOD(m1) + Expression: "(m1 / (m2 * 60)) * 100" - Id: m1 ReturnData: false MetricStat: @@ -256,6 +256,17 @@ Resources: Value: !Ref PrescriptionStatusUpdatesTableName Period: 60 Stat: Sum + - Id: m2 + ReturnData: false + MetricStat: + Metric: + Namespace: AWS/DynamoDB + MetricName: ProvisionedWriteCapacityUnits + Dimensions: + - Name: TableName + Value: !Ref PrescriptionStatusUpdatesTableName + Period: 60 + Stat: Maximum CPSUFhirConverterErrorLogsMetricFilter: Type: AWS::Logs::MetricFilter diff --git a/SAMtemplates/main_template.yaml b/SAMtemplates/main_template.yaml index 14f12ddde..cc133abba 100644 --- a/SAMtemplates/main_template.yaml +++ b/SAMtemplates/main_template.yaml @@ -81,10 +81,10 @@ Parameters: - true - false - WriteCapacityAlertThreshold: + DynamoDBUtilizationPercentageThreshold: Type: Number - Description: Threshold for the average consumed write capacity units per minute to trigger an alarm - Default: 500 + Description: When the database usage crosses this percentage of provisioned capacity, trigger an alarm + Default: 90 Resources: @@ -156,5 +156,5 @@ Resources: UpdatePrescriptionStatusFunctionName: !GetAtt Functions.Outputs.UpdatePrescriptionStatusFunctionName PrescriptionStatusUpdatesTableName: !GetAtt Tables.Outputs.PrescriptionStatusUpdatesTableName ConvertRequestToFhirFormatFunctionName: !GetAtt Functions.Outputs.ConvertRequestToFhirFormatFunctionName - WriteCapacityAlertThreshold: !Ref WriteCapacityAlertThreshold + DynamoDBUtilizationPercentageThreshold: !Ref DynamoDBUtilizationPercentageThreshold EnableAlerts: !Ref EnableAlerts