Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Scaling policy #9401

Closed
wants to merge 10 commits into from
Closed

Scaling policy #9401

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
179fdbe
Enable latency scaling alarm in all environments
georgeblahblah Nov 3, 2023
ea41a25
stash
georgeblahblah Nov 3, 2023
e6d386f
ignore ts error
ioannakok Dec 20, 2023
aa8d793
Use `asg.scaleOnMetric` method
ioannakok Dec 20, 2023
e6b4a2d
Add more intervals in step scaling policy
ioannakok Dec 21, 2023
127fb4a
Add template to define a step scaling policy based on latency
ioannakok Dec 22, 2023
6800a2b
Update snapshot
ioannakok Jan 2, 2024
189c6bc
Modify thresholds for step scaling alarm
ioannakok Jan 2, 2024
a8ba01f
refactor: Add alarm for step scaling policy
ioannakok Jan 2, 2024
43d7a91
Change metric period to be 10 seconds
ioannakok Jan 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 53 additions & 23 deletions dotcom-rendering/cdk/lib/__snapshots__/dotcom-rendering.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,40 @@ exports[`The DotcomRendering stack matches the snapshot 1`] = `
},
"Type": "AWS::IAM::Policy",
},
"HighLatencyAlarm61213652": {
"Properties": {
"ActionsEnabled": true,
"AlarmActions": [
{
"Ref": "ScaleUpAF7DDD3C",
},
],
"AlarmDescription": "Latency alarm for autoscaling",
"ComparisonOperator": "GreaterThanThreshold",
"DatapointsToAlarm": 2,
"Dimensions": [
{
"Name": "AutoScalingGroupName",
"Value": {
"Ref": "AutoscalingGroup",
},
},
],
"EvaluationPeriods": 3,
"MetricName": "Latency",
"Namespace": "AWS/ELB",
"OKActions": [
{
"Ref": "ScaleDown2D6349BF",
},
],
"Period": 10,
"Statistic": "Average",
"Threshold": 0.2,
"TreatMissingData": "missing",
},
"Type": "AWS::CloudWatch::Alarm",
},
"InstanceRole": {
"Properties": {
"AssumeRolePolicyDocument": {
Expand Down Expand Up @@ -755,15 +789,7 @@ exports[`The DotcomRendering stack matches the snapshot 1`] = `
},
"LatencyScalingAlarm": {
"Properties": {
"ActionsEnabled": false,
"AlarmActions": [
{
"Fn::GetAtt": [
"ScaleUpPolicy",
"Arn",
],
},
],
"ActionsEnabled": true,
"AlarmDescription": "Scale up if latency is GreaterThanOrEqualToThreshold of 0.2 over 1 period(s) of 60 seconds",
"ComparisonOperator": "GreaterThanOrEqualToThreshold",
"Dimensions": [
Expand All @@ -777,14 +803,6 @@ exports[`The DotcomRendering stack matches the snapshot 1`] = `
"EvaluationPeriods": 1,
"MetricName": "Latency",
"Namespace": "AWS/ELB",
"OKActions": [
{
"Fn::GetAtt": [
"ScaleDownPolicy",
"Arn",
],
},
],
"Period": 60,
"Statistic": "Average",
"Threshold": 0.2,
Expand Down Expand Up @@ -850,25 +868,37 @@ exports[`The DotcomRendering stack matches the snapshot 1`] = `
},
"Type": "AWS::IAM::Policy",
},
"ScaleDownPolicy": {
"ScaleDown2D6349BF": {
"Properties": {
"AdjustmentType": "ChangeInCapacity",
"AutoScalingGroupName": {
"Ref": "AutoscalingGroup",
},
"Cooldown": "120",
"ScalingAdjustment": -1,
"Cooldown": "15",
"PolicyType": "StepScaling",
"StepAdjustments": [
{
"MetricIntervalLowerBound": 0,
"ScalingAdjustment": -1,
},
],
},
"Type": "AWS::AutoScaling::ScalingPolicy",
},
"ScaleUpPolicy": {
"ScaleUpAF7DDD3C": {
"Properties": {
"AdjustmentType": "PercentChangeInCapacity",
"AutoScalingGroupName": {
"Ref": "AutoscalingGroup",
},
"Cooldown": "600",
"ScalingAdjustment": 100,
"Cooldown": "30",
"PolicyType": "StepScaling",
"StepAdjustments": [
{
"MetricIntervalLowerBound": 0,
"ScalingAdjustment": 100,
},
],
},
"Type": "AWS::AutoScaling::ScalingPolicy",
},
Expand Down
80 changes: 67 additions & 13 deletions dotcom-rendering/cdk/lib/dotcom-rendering.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,17 @@ import type { App } from 'aws-cdk-lib';
import { CfnOutput, Duration, Tags } from 'aws-cdk-lib';
import {
AdjustmentType,
CfnScalingPolicy,
HealthCheck,
StepScalingAction,
} from 'aws-cdk-lib/aws-autoscaling';
import { CfnAlarm } from 'aws-cdk-lib/aws-cloudwatch';
import {
Alarm,
CfnAlarm,
ComparisonOperator,
Metric,
TreatMissingData,
} from 'aws-cdk-lib/aws-cloudwatch';
import { AutoScalingAction } from 'aws-cdk-lib/aws-cloudwatch-actions';
import { InstanceType, Peer } from 'aws-cdk-lib/aws-ec2';
import { LoadBalancingProtocol } from 'aws-cdk-lib/aws-elasticloadbalancing';
import { StringParameter } from 'aws-cdk-lib/aws-ssm';
Expand Down Expand Up @@ -270,18 +277,65 @@ export class DotcomRendering extends GuStack {
/** TODO - migrate these simple scaling policies
* @see https://github.com/guardian/dotcom-rendering/issues/8345#issuecomment-1647502598
*/
const scaleUpPolicy = new CfnScalingPolicy(this, 'ScaleUpPolicy', {
// const scaleUpPolicy = new CfnScalingPolicy(this, 'ScaleUpPolicy', {
// adjustmentType: AdjustmentType.PERCENT_CHANGE_IN_CAPACITY,
// autoScalingGroupName: asg.autoScalingGroupName,
// cooldown: '600',
// scalingAdjustment: 100,
// });

// const scaleDownPolicy = new CfnScalingPolicy(this, 'ScaleDownPolicy', {
// adjustmentType: AdjustmentType.CHANGE_IN_CAPACITY,
// autoScalingGroupName: asg.autoScalingGroupName,
// cooldown: '120',
// scalingAdjustment: -1,
// });

const latencyHighAlarm = new Alarm(this, 'HighLatencyAlarm', {
actionsEnabled: true,
alarmDescription: 'Latency alarm for autoscaling',
threshold: 0.2,
comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD,
datapointsToAlarm: 2,
evaluationPeriods: 3,
metric: new Metric({
dimensionsMap: {
AutoScalingGroupName: asg.autoScalingGroupName,
},
metricName: 'Latency',
namespace: 'AWS/ELB',
period: Duration.seconds(10),
statistic: 'Average',
}),

treatMissingData: TreatMissingData.MISSING,
});

const scaleUpStep = new StepScalingAction(this, 'ScaleUp', {
adjustmentType: AdjustmentType.PERCENT_CHANGE_IN_CAPACITY,
autoScalingGroupName: asg.autoScalingGroupName,
cooldown: '600',
scalingAdjustment: 100,
autoScalingGroup: asg,
// 10 minutes for prod. This is only for testing
cooldown: Duration.seconds(30),
});

scaleUpStep.addAdjustment({
lowerBound: 0,
adjustment: 100,
});
const scaleDownPolicy = new CfnScalingPolicy(this, 'ScaleDownPolicy', {
latencyHighAlarm.addAlarmAction(new AutoScalingAction(scaleUpStep));

const scaleDownStep = new StepScalingAction(this, 'ScaleDown', {
adjustmentType: AdjustmentType.CHANGE_IN_CAPACITY,
autoScalingGroupName: asg.autoScalingGroupName,
cooldown: '120',
scalingAdjustment: -1,
autoScalingGroup: asg,

// Every 2 minutes take out one instance for prod. This is for testing purposes
cooldown: Duration.seconds(15),
});
scaleDownStep.addAdjustment({
lowerBound: 0,
adjustment: -1,
});
latencyHighAlarm.addOkAction(new AutoScalingAction(scaleDownStep));

/** Returns an appropriate alarm description given the appropriate configuration object */
const getAlarmDescription = ({
Expand All @@ -301,7 +355,7 @@ export class DotcomRendering extends GuStack {
comparisonOperator: 'GreaterThanOrEqualToThreshold',
};
new CfnAlarm(this, 'LatencyScalingAlarm', {
actionsEnabled: stage === 'PROD',
actionsEnabled: true,
alarmDescription: getAlarmDescription({
title: 'Scale up if latency',
...latencyScalingAlarmConfig,
Expand All @@ -319,8 +373,8 @@ export class DotcomRendering extends GuStack {
statistic: 'Average',
threshold: latencyScalingAlarmConfig.threshold,
comparisonOperator: latencyScalingAlarmConfig.comparisonOperator,
okActions: [scaleDownPolicy.attrArn],
alarmActions: [scaleUpPolicy.attrArn],
// okActions: [scaleDownPolicy.attrArn],
// alarmActions: [scaleUpPolicy.attrArn],
});

// Backend 5XX alarm
Expand Down
Loading