From 2c9e22ce9f972799138faf0b72e9c1ea44d6a6dc Mon Sep 17 00:00:00 2001 From: Amir Szekely Date: Mon, 29 Jan 2024 18:28:49 -0500 Subject: [PATCH 01/11] WIP --- src/providers/ec2fleet.ts | 565 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 565 insertions(+) create mode 100644 src/providers/ec2fleet.ts diff --git a/src/providers/ec2fleet.ts b/src/providers/ec2fleet.ts new file mode 100644 index 00000000..6b829b7c --- /dev/null +++ b/src/providers/ec2fleet.ts @@ -0,0 +1,565 @@ +import * as cdk from 'aws-cdk-lib'; +import { + aws_ec2 as ec2, + aws_iam as iam, + aws_logs as logs, + aws_stepfunctions as stepfunctions, + aws_stepfunctions_tasks as stepfunctions_tasks, + Duration, + RemovalPolicy, + Stack, +} from 'aws-cdk-lib'; +import { RetentionDays } from 'aws-cdk-lib/aws-logs'; +import { IntegrationPattern } from 'aws-cdk-lib/aws-stepfunctions'; +import { Construct } from 'constructs'; +import { + amiRootDevice, + Architecture, + BaseProvider, + IRunnerProvider, + IRunnerProviderStatus, + Os, + RunnerAmi, + RunnerProviderProps, + RunnerRuntimeParameters, + RunnerVersion, +} from './common'; +import { IRunnerImageBuilder, RunnerImageBuilder, RunnerImageBuilderProps, RunnerImageBuilderType, RunnerImageComponent } from '../image-builders'; +import { MINIMAL_EC2_SSM_SESSION_MANAGER_POLICY_STATEMENT } from '../utils'; + +// this script is specifically made so `poweroff` is absolutely always called +// each `{}` is a variable coming from `params` below +const linuxUserDataTemplate = `#!/bin/bash -x +IMDS_TOKEN=\`curl -sX PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60"\` +TASK_TOKEN=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:taskToken\` +logGroupName=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:logGroupName\` +runnerNamePath=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:runnerNamePath\` +githubDomainPath=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:githubDomainPath\` +ownerPath=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:ownerPath\` +repoPath=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:repoPath\` +runnerTokenPath=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:runnerTokenPath\` +labels=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:labels\` +registrationURL=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:registrationURL\` +unset IMDS_TOKEN + +heartbeat () { + while true; do + aws stepfunctions send-task-heartbeat --task-token "$TASK_TOKEN" + sleep 60 + done +} +setup_logs () { + cat < /tmp/log.conf || exit 1 + { + "logs": { + "log_stream_name": "unknown", + "logs_collected": { + "files": { + "collect_list": [ + { + "file_path": "/var/log/runner.log", + "log_group_name": "$logGroupName", + "log_stream_name": "$runnerNamePath", + "timezone": "UTC" + } + ] + } + } + } + } +EOF + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/tmp/log.conf || exit 2 +} +action () { + # Determine the value of RUNNER_FLAGS + if [ "$(< RUNNER_VERSION)" = "latest" ]; then + RUNNER_FLAGS="" + else + RUNNER_FLAGS="--disableupdate" + fi + + labelsTemplate="$labels,cdkghr:started:$(date +%s)" + + # Execute the configuration command for runner registration + sudo -Hu runner /home/runner/config.sh --unattended --url "$registrationURL" --token "$runnerTokenPath" --ephemeral --work _work --labels "$labelsTemplate" $RUNNER_FLAGS --name "$runnerNamePath" || exit 1 + + # Execute the run command + sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2 + + # Retrieve the status + STATUS=$(grep -Phors "finish job request for job [0-9a-f\\-]+ with result: \K.*" /home/runner/_diag/ | tail -n1) + + # Check and print the job status + [ -n "$STATUS" ] && echo CDKGHA JOB DONE "$labels" "$STATUS" +} +heartbeat & +if setup_logs && action | tee /var/log/runner.log 2>&1; then + aws stepfunctions send-task-success --task-token "$TASK_TOKEN" --task-output '{"ok": true}' +else + aws stepfunctions send-task-failure --task-token "$TASK_TOKEN" +fi +sleep 10 # give cloudwatch agent its default 5 seconds buffer duration to upload logs +poweroff +`.replace(/{/g, '\\{').replace(/}/g, '\\}').replace(/\\{\\}/g, '{}'); + +// this script is specifically made so `poweroff` is absolutely always called +// each `{}` is a variable coming from `params` below and their order should match the linux script +const windowsUserDataTemplate = ` +$TASK_TOKEN = "{}" +$logGroupName="{}" +$runnerNamePath="{}" +$githubDomainPath="{}" +$ownerPath="{}" +$repoPath="{}" +$runnerTokenPath="{}" +$labels="{}" +$registrationURL="{}" + +Start-Job -ScriptBlock { + while (1) { + aws stepfunctions send-task-heartbeat --task-token "$using:TASK_TOKEN" + sleep 60 + } +} +function setup_logs () { + echo '{ + "logs": { + "log_stream_name": "unknown", + "logs_collected": { + "files": { + "collect_list": [ + { + "file_path": "/actions/runner.log", + "log_group_name": "$logGroupName", + "log_stream_name": "$runnerNamePath", + "timezone": "UTC" + } + ] + } + } + } + }' | Out-File -Encoding ASCII $Env:TEMP/log.conf + & "C:/Program Files/Amazon/AmazonCloudWatchAgent/amazon-cloudwatch-agent-ctl.ps1" -a fetch-config -m ec2 -s -c file:$Env:TEMP/log.conf +} +function action () { + cd /actions + $RunnerVersion = Get-Content RUNNER_VERSION -Raw + if ($RunnerVersion -eq "latest") { $RunnerFlags = "" } else { $RunnerFlags = "--disableupdate" } + ./config.cmd --unattended --url "\${registrationUrl}" --token "\${runnerTokenPath}" --ephemeral --work _work --labels "\${labels},cdkghr:started:$(Get-Date -UFormat +%s)" $RunnerFlags --name "\${runnerNamePath}" 2>&1 | Out-File -Encoding ASCII -Append /actions/runner.log + + if ($LASTEXITCODE -ne 0) { return 1 } + ./run.cmd 2>&1 | Out-File -Encoding ASCII -Append /actions/runner.log + if ($LASTEXITCODE -ne 0) { return 2 } + + $STATUS = Select-String -Path './_diag/*.log' -Pattern 'finish job request for job [0-9a-f\\-]+ with result: (.*)' | %{$_.Matches.Groups[1].Value} | Select-Object -Last 1 + + if ($STATUS) { + echo "CDKGHA JOB DONE \${labels} $STATUS" | Out-File -Encoding ASCII -Append /actions/runner.log + } + + return 0 + +} +setup_logs +$r = action +if ($r -eq 0) { + aws stepfunctions send-task-success --task-token "$TASK_TOKEN" --task-output '{ }' +} else { + aws stepfunctions send-task-failure --task-token "$TASK_TOKEN" +} +Start-Sleep -Seconds 10 # give cloudwatch agent its default 5 seconds buffer duration to upload logs +Stop-Computer -ComputerName localhost -Force + +`.replace(/{/g, '\\{').replace(/}/g, '\\}').replace(/\\{\\}/g, '{}'); + + +/** + * Properties for {@link Ec2RunnerProvider} construct. + */ +export interface Ec2RunnerProviderProps extends RunnerProviderProps { + /** + * Runner image builder used to build AMI containing GitHub Runner and all requirements. + * + * The image builder determines the OS and architecture of the runner. + * + * @default Ec2RunnerProvider.imageBuilder() + */ + readonly imageBuilder?: IRunnerImageBuilder; + + /** + * @deprecated use imageBuilder + */ + readonly amiBuilder?: IRunnerImageBuilder; + + /** + * GitHub Actions labels used for this provider. + * + * These labels are used to identify which provider should spawn a new on-demand runner. Every job sends a webhook with the labels it's looking for + * based on runs-on. We match the labels from the webhook with the labels specified here. If all the labels specified here are present in the + * job's labels, this provider will be chosen and spawn a new runner. + * + * @default ['ec2'] + */ + readonly labels?: string[]; + + /** + * Instance type for launched runner instances. + * + * @default m5.large + */ + readonly instanceType?: ec2.InstanceType; + + /** + * Size of volume available for launched runner instances. This modifies the boot volume size and doesn't add any additional volumes. + * + * @default 30GB + */ + readonly storageSize?: cdk.Size; + + /** + * Security Group to assign to launched runner instances. + * + * @default a new security group + * + * @deprecated use {@link securityGroups} + */ + readonly securityGroup?: ec2.ISecurityGroup; + + /** + * Security groups to assign to launched runner instances. + * + * @default a new security group + */ + readonly securityGroups?: ec2.ISecurityGroup[]; + + /** + * Subnet where the runner instances will be launched. + * + * @default default subnet of account's default VPC + * + * @deprecated use {@link vpc} and {@link subnetSelection} + */ + readonly subnet?: ec2.ISubnet; + + /** + * VPC where runner instances will be launched. + * + * @default default account VPC + */ + readonly vpc?: ec2.IVpc; + + /** + * Where to place the network interfaces within the VPC. Only the first matched subnet will be used. + * + * @default default VPC subnet + */ + readonly subnetSelection?: ec2.SubnetSelection; + + /** + * Use spot instances to save money. Spot instances are cheaper but not always available and can be stopped prematurely. + * + * @default false + */ + readonly spot?: boolean; + + /** + * Set a maximum price for spot instances. + * + * @default no max price (you will pay current spot price) + */ + readonly spotMaxPrice?: string; +} + +/** + * GitHub Actions runner provider using EC2 to execute jobs. + * + * This construct is not meant to be used by itself. It should be passed in the providers property for GitHubRunners. + */ +export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { + /** + * Create new image builder that builds EC2 specific runner images. + * + * You can customize the OS, architecture, VPC, subnet, security groups, etc. by passing in props. + * + * You can add components to the image builder by calling `imageBuilder.addComponent()`. + * + * The default OS is Ubuntu running on x64 architecture. + * + * Included components: + * * `RunnerImageComponent.requiredPackages()` + * * `RunnerImageComponent.runnerUser()` + * * `RunnerImageComponent.git()` + * * `RunnerImageComponent.githubCli()` + * * `RunnerImageComponent.awsCli()` + * * `RunnerImageComponent.docker()` + * * `RunnerImageComponent.githubRunner()` + */ + public static imageBuilder(scope: Construct, id: string, props?: RunnerImageBuilderProps) { + return RunnerImageBuilder.new(scope, id, { + os: Os.LINUX_UBUNTU, + architecture: Architecture.X86_64, + builderType: RunnerImageBuilderType.AWS_IMAGE_BUILDER, + components: [ + RunnerImageComponent.requiredPackages(), + RunnerImageComponent.runnerUser(), + RunnerImageComponent.git(), + RunnerImageComponent.githubCli(), + RunnerImageComponent.awsCli(), + RunnerImageComponent.docker(), + RunnerImageComponent.githubRunner(props?.runnerVersion ?? RunnerVersion.latest()), + ], + ...props, + }); + } + + /** + * Labels associated with this provider. + */ + readonly labels: string[]; + + /** + * Grant principal used to add permissions to the runner role. + */ + readonly grantPrincipal: iam.IPrincipal; + + /** + * Log group where provided runners will save their logs. + * + * Note that this is not the job log, but the runner itself. It will not contain output from the GitHub Action but only metadata on its execution. + */ + readonly logGroup: logs.ILogGroup; + + readonly retryableErrors = [ + 'Ec2.Ec2Exception', + 'States.Timeout', + ]; + + private readonly amiBuilder: IRunnerImageBuilder; + private readonly ami: RunnerAmi; + private readonly role: iam.Role; + private readonly instanceType: ec2.InstanceType; + private readonly storageSize: cdk.Size; + private readonly spot: boolean; + private readonly spotMaxPrice: string | undefined; + private readonly vpc: ec2.IVpc; + private readonly subnets: ec2.ISubnet[]; + private readonly securityGroups: ec2.ISecurityGroup[]; + + constructor(scope: Construct, id: string, props?: Ec2RunnerProviderProps) { + super(scope, id, props); + + this.labels = props?.labels ?? ['ec2']; + this.vpc = props?.vpc ?? ec2.Vpc.fromLookup(this, 'Default VPC', { isDefault: true }); + this.securityGroups = props?.securityGroup ? [props.securityGroup] : (props?.securityGroups ?? [new ec2.SecurityGroup(this, 'SG', { vpc: this.vpc })]); + this.subnets = props?.subnet ? [props.subnet] : this.vpc.selectSubnets(props?.subnetSelection).subnets; + this.instanceType = props?.instanceType ?? ec2.InstanceType.of(ec2.InstanceClass.M5, ec2.InstanceSize.LARGE); + this.storageSize = props?.storageSize ?? cdk.Size.gibibytes(30); // 30 is the minimum for Windows + this.spot = props?.spot ?? false; + this.spotMaxPrice = props?.spotMaxPrice; + + this.amiBuilder = props?.imageBuilder ?? props?.amiBuilder ?? Ec2RunnerProvider.imageBuilder(this, 'Ami Builder', { + vpc: props?.vpc, + subnetSelection: props?.subnetSelection, + securityGroups: this.securityGroups, + }); + this.ami = this.amiBuilder.bindAmi(); + + if (!this.ami.architecture.instanceTypeMatch(this.instanceType)) { + throw new Error(`AMI architecture (${this.ami.architecture.name}) doesn't match runner instance type (${this.instanceType} / ${this.instanceType.architecture})`); + } + + this.grantPrincipal = this.role = new iam.Role(this, 'Role', { + assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'), + }); + this.grantPrincipal.addToPrincipalPolicy(new iam.PolicyStatement({ + actions: ['states:SendTaskFailure', 'states:SendTaskSuccess', 'states:SendTaskHeartbeat'], + resources: ['*'], // no support for stateMachine.stateMachineArn :( + conditions: { + StringEquals: { + 'aws:ResourceTag/aws:cloudformation:stack-id': cdk.Stack.of(this).stackId, + }, + }, + })); + this.grantPrincipal.addToPrincipalPolicy(MINIMAL_EC2_SSM_SESSION_MANAGER_POLICY_STATEMENT); + + this.logGroup = new logs.LogGroup( + this, + 'Logs', + { + retention: props?.logRetention ?? RetentionDays.ONE_MONTH, + removalPolicy: RemovalPolicy.DESTROY, + }, + ); + this.logGroup.grantWrite(this); + } + + /** + * Generate step function task(s) to start a new runner. + * + * Called by GithubRunners and shouldn't be called manually. + * + * @param parameters workflow job details + */ + getStepFunctionTask(parameters: RunnerRuntimeParameters): stepfunctions.IChainable { + // we need to build user data in two steps because passing the template as the first parameter to stepfunctions.JsonPath.format fails on syntax + + const params = [ + stepfunctions.JsonPath.taskToken, + this.logGroup.logGroupName, + parameters.runnerNamePath, + parameters.githubDomainPath, + parameters.ownerPath, + parameters.repoPath, + parameters.runnerTokenPath, + this.labels.join(','), + parameters.registrationUrl, + ]; + + const passUserData = new stepfunctions.Pass(this, `${this.labels.join(', ')} data`, { + parameters: { + userdataTemplate: this.ami.os.is(Os.WINDOWS) ? windowsUserDataTemplate : linuxUserDataTemplate, + }, + resultPath: stepfunctions.JsonPath.stringAt('$.ec2'), + }); + + // we use ec2:RunInstances because we must + // we can't use fleets because they don't let us override user data, security groups or even disk size + // we can't use requestSpotInstances because it doesn't support launch templates, and it's deprecated + // ec2:RunInstances also seemed like the only one to immediately return an error when spot capacity is not available + + // we build a complicated chain of states here because ec2:RunInstances can only try one subnet at a time + // if someone can figure out a good way to use Map for this, please open a PR + + // build a state for each subnet we want to try + const instanceProfile = new iam.CfnInstanceProfile(this, 'Instance Profile', { + roles: [this.role.roleName], + }); + const rootDeviceResource = amiRootDevice(this, this.ami.launchTemplate.launchTemplateId); + rootDeviceResource.node.addDependency(this.amiBuilder); + const subnetRunners = this.subnets.map((subnet, index) => { + return new stepfunctions_tasks.CallAwsService(this, `${this.labels.join(', ')} subnet${index+1}`, { + comment: subnet.subnetId, + integrationPattern: IntegrationPattern.WAIT_FOR_TASK_TOKEN, + service: 'ec2', + action: 'runInstances', + heartbeatTimeout: stepfunctions.Timeout.duration(Duration.minutes(10)), + // TODO somehow create launch template with security group, profile, user data, tags in imds, etc. + parameters: { + LaunchTemplate: { + // TODO update builder to update our launch templates? + LaunchTemplateId: this.ami.launchTemplate.launchTemplateId, + }, + MinCount: 1, + MaxCount: 1, + InstanceType: this.instanceType.toString(), + UserData: stepfunctions.JsonPath.base64Encode( + stepfunctions.JsonPath.format( + stepfunctions.JsonPath.stringAt('$.ec2.userdataTemplate'), + ...params, + ), + ), + InstanceInitiatedShutdownBehavior: ec2.InstanceInitiatedShutdownBehavior.TERMINATE, + IamInstanceProfile: { + Arn: instanceProfile.attrArn, + }, + MetadataOptions: { + HttpTokens: 'required', + // TODO InstanceMetadataTags: 'enabled' + }, + SecurityGroupIds: this.securityGroups.map(sg => sg.securityGroupId), + SubnetId: subnet.subnetId, + BlockDeviceMappings: [{ + DeviceName: rootDeviceResource.ref, + Ebs: { + DeleteOnTermination: true, + VolumeSize: this.storageSize.toGibibytes(), + }, + }], + InstanceMarketOptions: this.spot ? { + MarketType: 'spot', + SpotOptions: { + MaxPrice: this.spotMaxPrice, + SpotInstanceType: 'one-time', + }, + } : undefined, + }, + iamResources: ['*'], + }); + }); + + // start with the first subnet + passUserData.next(subnetRunners[0]); + + // chain up the rest of the subnets + for (let i = 1; i < subnetRunners.length; i++) { + subnetRunners[i-1].addCatch(subnetRunners[i], { + errors: ['Ec2.Ec2Exception', 'States.Timeout'], + resultPath: stepfunctions.JsonPath.stringAt('$.lastSubnetError'), + }); + } + + return passUserData; + } + + grantStateMachine(stateMachineRole: iam.IGrantable) { + stateMachineRole.grantPrincipal.addToPrincipalPolicy(new iam.PolicyStatement({ + actions: ['iam:PassRole'], + resources: [this.role.roleArn], + conditions: { + StringEquals: { + 'iam:PassedToService': 'ec2.amazonaws.com', + }, + }, + })); + + stateMachineRole.grantPrincipal.addToPrincipalPolicy(new iam.PolicyStatement({ + actions: ['ec2:createTags'], + resources: [Stack.of(this).formatArn({ + service: 'ec2', + resource: '*', + })], + })); + + stateMachineRole.grantPrincipal.addToPrincipalPolicy(new iam.PolicyStatement({ + actions: ['iam:CreateServiceLinkedRole'], + resources: ['*'], + conditions: { + StringEquals: { + 'iam:AWSServiceName': 'spot.amazonaws.com', + }, + }, + })); + } + + status(statusFunctionRole: iam.IGrantable): IRunnerProviderStatus { + statusFunctionRole.grantPrincipal.addToPrincipalPolicy(new iam.PolicyStatement({ + actions: ['ec2:DescribeLaunchTemplateVersions'], + resources: ['*'], + })); + + return { + type: this.constructor.name, + labels: this.labels, + securityGroups: this.securityGroups.map(sg => sg.securityGroupId), + roleArn: this.role.roleArn, + logGroup: this.logGroup.logGroupName, + ami: { + launchTemplate: this.ami.launchTemplate.launchTemplateId || 'unknown', + amiBuilderLogGroup: this.ami.logGroup?.logGroupName, + }, + }; + } + + /** + * The network connections associated with this resource. + */ + public get connections(): ec2.Connections { + return new ec2.Connections({ securityGroups: this.securityGroups }); + } +} + +/** + * @deprecated use {@link Ec2RunnerProvider} + */ +export class Ec2Runner extends Ec2RunnerProvider { +} From a6a7c7c5ac64c0f7a68522eece1a06f7b61feac4 Mon Sep 17 00:00:00 2001 From: Amir Szekely Date: Tue, 13 Feb 2024 10:01:29 -0500 Subject: [PATCH 02/11] Custom launch template for AMI builder --- .../aws-image-builder/builder.ts | 28 +++++++---- .../aws-image-builder/deprecated/ami.ts | 28 +++++++---- .../aws-image-builder/deprecated/common.ts | 4 +- .../aws-image-builder/deprecated/container.ts | 4 +- src/image-builders/codebuild-deprecated.ts | 4 +- src/image-builders/common.ts | 16 ++++++- test/imagebuilder.test.ts | 47 ++++++++++++++++++- 7 files changed, 102 insertions(+), 29 deletions(-) diff --git a/src/image-builders/aws-image-builder/builder.ts b/src/image-builders/aws-image-builder/builder.ts index 49f628f3..ca7dd754 100644 --- a/src/image-builders/aws-image-builder/builder.ts +++ b/src/image-builders/aws-image-builder/builder.ts @@ -23,7 +23,7 @@ import { ImageBuilderObjectBase } from './common'; import { ContainerRecipe, defaultBaseDockerImage } from './container'; import { DeleteAmiFunction } from './delete-ami-function'; import { FilterFailedBuildsFunction } from './filter-failed-builds-function'; -import { Architecture, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../providers'; +import { Architecture, BindAmiProps, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../providers'; import { singletonLambda } from '../../utils'; import { BuildImageFunction } from '../build-image-function'; import { RunnerImageBuilderBase, RunnerImageBuilderProps, uniqueImageBuilderName } from '../common'; @@ -252,6 +252,8 @@ export class ImageBuilderComponent extends ImageBuilderObjectBase { export class AwsImageBuilderRunnerImageBuilder extends RunnerImageBuilderBase { private boundDockerImage?: RunnerImage; private boundAmi?: RunnerAmi; + private boundDistribution?: imagebuilder.CfnDistributionConfiguration; + private boundDistributionLaunchTemplateCount = 0; private readonly os: Os; private readonly architecture: Architecture; private readonly baseImage: string; @@ -578,8 +580,9 @@ export class AwsImageBuilderRunnerImageBuilder extends RunnerImageBuilderBase { return this.role; } - bindAmi(): RunnerAmi { + bindAmi(props?: BindAmiProps): RunnerAmi { if (this.boundAmi) { + this.addLaunchTemplateToDistribution(props?.launchTemplate); return this.boundAmi; } @@ -590,7 +593,7 @@ export class AwsImageBuilderRunnerImageBuilder extends RunnerImageBuilderBase { const stackName = cdk.Stack.of(this).stackName; const builderName = this.node.path; - const dist = new imagebuilder.CfnDistributionConfiguration(this, 'AMI Distribution', { + this.boundDistribution = new imagebuilder.CfnDistributionConfiguration(this, 'AMI Distribution', { name: uniqueImageBuilderName(this), // description: this.description, distributions: [ @@ -608,14 +611,12 @@ export class AwsImageBuilderRunnerImageBuilder extends RunnerImageBuilderBase { 'GitHubRunners:Builder': builderName, }, }, - launchTemplateConfigurations: [ - { - launchTemplateId: launchTemplate.launchTemplateId, - }, - ], + launchTemplateConfigurations: [], }, ], }); + this.addLaunchTemplateToDistribution(launchTemplate); + this.addLaunchTemplateToDistribution(props?.launchTemplate); const recipe = new AmiRecipe(this, 'Ami Recipe', { platform: this.platform(), @@ -629,8 +630,8 @@ export class AwsImageBuilderRunnerImageBuilder extends RunnerImageBuilderBase { iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'), iam.ManagedPolicy.fromAwsManagedPolicyName('EC2InstanceProfileForImageBuilder'), ]); - this.createImage(infra, dist, log, recipe.arn, undefined); - this.createPipeline(infra, dist, log, recipe.arn, undefined); + this.createImage(infra, this.boundDistribution, log, recipe.arn, undefined); + this.createPipeline(infra, this.boundDistribution, log, recipe.arn, undefined); this.boundAmi = { launchTemplate: launchTemplate, @@ -745,6 +746,13 @@ export class AwsImageBuilderRunnerImageBuilder extends RunnerImageBuilderBase { return this.boundComponents; } + + private addLaunchTemplateToDistribution(launchTemplate?: ec2.ILaunchTemplate) { + if (launchTemplate && this.boundDistribution) { + this.boundDistribution.addPropertyOverride(`Distributions.0.LaunchTemplateConfigurations.${this.boundDistributionLaunchTemplateCount}.LaunchTemplateId`, launchTemplate.launchTemplateId); + this.boundDistributionLaunchTemplateCount++; + } + } } /** diff --git a/src/image-builders/aws-image-builder/deprecated/ami.ts b/src/image-builders/aws-image-builder/deprecated/ami.ts index 7a892a5a..06315db1 100644 --- a/src/image-builders/aws-image-builder/deprecated/ami.ts +++ b/src/image-builders/aws-image-builder/deprecated/ami.ts @@ -15,7 +15,7 @@ import { Construct } from 'constructs'; import { ImageBuilderBase } from './common'; import { LinuxUbuntuComponents } from './linux-components'; import { WindowsComponents } from './windows-components'; -import { Architecture, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../../providers'; +import { Architecture, BindAmiProps, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../../providers'; import { singletonLambda } from '../../../utils'; import { uniqueImageBuilderName } from '../../common'; import { AmiRecipe, defaultBaseAmi } from '../ami'; @@ -151,6 +151,8 @@ export interface AmiBuilderProps { */ export class AmiBuilder extends ImageBuilderBase { private boundAmi?: RunnerAmi; + private boundDistribution?: imagebuilder.CfnDistributionConfiguration; + private boundDistributionLaunchTemplateCount = 0; constructor(scope: Construct, id: string, props?: AmiBuilderProps) { super(scope, id, { @@ -248,8 +250,9 @@ export class AmiBuilder extends ImageBuilderBase { /** * Called by IRunnerProvider to finalize settings and create the AMI builder. */ - bindAmi(): RunnerAmi { + bindAmi(props?: BindAmiProps): RunnerAmi { if (this.boundAmi) { + this.addLaunchTemplateToDistribution(props?.launchTemplate); return this.boundAmi; } @@ -260,7 +263,7 @@ export class AmiBuilder extends ImageBuilderBase { const stackName = cdk.Stack.of(this).stackName; const builderName = this.node.path; - const dist = new imagebuilder.CfnDistributionConfiguration(this, 'Distribution', { + this.boundDistribution = new imagebuilder.CfnDistributionConfiguration(this, 'Distribution', { name: uniqueImageBuilderName(this), description: this.description, distributions: [ @@ -278,14 +281,12 @@ export class AmiBuilder extends ImageBuilderBase { 'GitHubRunners:Builder': builderName, }, }, - launchTemplateConfigurations: [ - { - launchTemplateId: launchTemplate.launchTemplateId, - }, - ], + launchTemplateConfigurations: [], }, ], }); + this.addLaunchTemplateToDistribution(launchTemplate); + this.addLaunchTemplateToDistribution(props?.launchTemplate); const recipe = new AmiRecipe(this, 'Ami Recipe', { platform: this.platform, @@ -299,8 +300,8 @@ export class AmiBuilder extends ImageBuilderBase { iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'), iam.ManagedPolicy.fromAwsManagedPolicyName('EC2InstanceProfileForImageBuilder'), ]); - this.createImage(infra, dist, log, recipe.arn, undefined); - this.createPipeline(infra, dist, log, recipe.arn, undefined); + this.createImage(infra, this.boundDistribution, log, recipe.arn, undefined); + this.createPipeline(infra, this.boundDistribution, log, recipe.arn, undefined); this.boundAmi = { launchTemplate: launchTemplate, @@ -356,4 +357,11 @@ export class AmiBuilder extends ImageBuilderBase { bindDockerImage(): RunnerImage { throw new Error('AmiBuilder cannot be used to build Docker images'); } + + private addLaunchTemplateToDistribution(launchTemplate?: ec2.ILaunchTemplate) { + if (launchTemplate && this.boundDistribution) { + this.boundDistribution.addPropertyOverride(`Distributions.0.LaunchTemplateConfigurations.${this.boundDistributionLaunchTemplateCount}.LaunchTemplateId`, launchTemplate.launchTemplateId); + this.boundDistributionLaunchTemplateCount++; + } + } } diff --git a/src/image-builders/aws-image-builder/deprecated/common.ts b/src/image-builders/aws-image-builder/deprecated/common.ts index 9c370b88..0fe9d8bf 100644 --- a/src/image-builders/aws-image-builder/deprecated/common.ts +++ b/src/image-builders/aws-image-builder/deprecated/common.ts @@ -1,7 +1,7 @@ import * as cdk from 'aws-cdk-lib'; import { aws_ec2 as ec2, aws_events as events, aws_iam as iam, aws_imagebuilder as imagebuilder, aws_logs as logs, RemovalPolicy } from 'aws-cdk-lib'; import { Construct } from 'constructs'; -import { Architecture, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../../providers'; +import { Architecture, BindAmiProps, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../../providers'; import { ImageBuilderBaseProps, IRunnerImageBuilder, uniqueImageBuilderName } from '../../common'; import { ImageBuilderComponent } from '../builder'; @@ -176,5 +176,5 @@ export abstract class ImageBuilderBase extends Construct implements IRunnerImage abstract bindDockerImage(): RunnerImage; - abstract bindAmi(): RunnerAmi; + abstract bindAmi(props?: BindAmiProps): RunnerAmi; } diff --git a/src/image-builders/aws-image-builder/deprecated/container.ts b/src/image-builders/aws-image-builder/deprecated/container.ts index 8b445031..2e16c7d4 100644 --- a/src/image-builders/aws-image-builder/deprecated/container.ts +++ b/src/image-builders/aws-image-builder/deprecated/container.ts @@ -15,7 +15,7 @@ import { Construct } from 'constructs'; import { ImageBuilderBase } from './common'; import { LinuxUbuntuComponents } from './linux-components'; import { WindowsComponents } from './windows-components'; -import { Architecture, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../../providers'; +import { Architecture, BindAmiProps, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../../providers'; import { singletonLambda } from '../../../utils'; import { BuildImageFunction } from '../../build-image-function'; import { uniqueImageBuilderName } from '../../common'; @@ -338,7 +338,7 @@ export class ContainerImageBuilder extends ImageBuilderBase { return cr; } - bindAmi(): RunnerAmi { + bindAmi(_?: BindAmiProps): RunnerAmi { throw new Error('ContainerImageBuilder cannot be used to build AMIs'); } } diff --git a/src/image-builders/codebuild-deprecated.ts b/src/image-builders/codebuild-deprecated.ts index 367b5f3f..d5872065 100644 --- a/src/image-builders/codebuild-deprecated.ts +++ b/src/image-builders/codebuild-deprecated.ts @@ -19,7 +19,7 @@ import { TagMutability, TagStatus } from 'aws-cdk-lib/aws-ecr'; import { RetentionDays } from 'aws-cdk-lib/aws-logs'; import { Construct } from 'constructs'; import { BuildImageFunction } from './build-image-function'; -import { IRunnerImageBuilder } from './common'; +import { BindAmiProps, IRunnerImageBuilder } from './common'; import { Architecture, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../providers'; import { singletonLambda } from '../utils'; @@ -527,7 +527,7 @@ export class CodeBuildImageBuilder extends Construct implements IRunnerImageBuil }); } - bindAmi(): RunnerAmi { + bindAmi(_?: BindAmiProps): RunnerAmi { throw new Error('CodeBuildImageBuilder does not support building AMIs'); } } diff --git a/src/image-builders/common.ts b/src/image-builders/common.ts index 3bfcd1bd..ff9817db 100644 --- a/src/image-builders/common.ts +++ b/src/image-builders/common.ts @@ -251,6 +251,18 @@ export enum RunnerImageBuilderType { AWS_IMAGE_BUILDER = 'AwsImageBuilder', } +/** + * Properties for RunnerImageBuilder.bindAmi(). + */ +export interface BindAmiProps { + /** + * Launch template to update with new generated AMI when available. + * + * @note Do not pass the same launch template to multiple builders. They will overwrite each other's AMIs. + */ + readonly launchTemplate?: ec2.ILaunchTemplate; +} + /** * Interface for constructs that build an image that can be used in {@link IRunnerProvider}. * @@ -275,7 +287,7 @@ export interface IRunnerImageBuilder { * * The AMI can be further updated over time manually or using a schedule as long as it is always written to the same launch template. */ - bindAmi(): RunnerAmi; + bindAmi(props?: BindAmiProps): RunnerAmi; } /** @@ -315,7 +327,7 @@ export abstract class RunnerImageBuilderBase extends Construct implements IConfi abstract bindDockerImage(): RunnerImage; - abstract bindAmi(): RunnerAmi; + abstract bindAmi(props?: BindAmiProps): RunnerAmi; abstract get connections(): ec2.Connections; abstract get grantPrincipal(): iam.IPrincipal; diff --git a/test/imagebuilder.test.ts b/test/imagebuilder.test.ts index 53c417f5..f0ef3f07 100644 --- a/test/imagebuilder.test.ts +++ b/test/imagebuilder.test.ts @@ -1,5 +1,5 @@ import * as cdk from 'aws-cdk-lib'; -import { aws_ec2 as ec2 } from 'aws-cdk-lib'; +import { aws_ec2 as ec2, CfnElement } from 'aws-cdk-lib'; import { Match, Template } from 'aws-cdk-lib/assertions'; import { AmiBuilder, @@ -338,3 +338,48 @@ test('Unused builder doesn\'t throw exceptions', () => { app.synth(); }); + +test('Adding more launch templates to the same builder', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'test'); + + const vpc = new ec2.Vpc(stack, 'vpc'); + + const builder = Ec2RunnerProvider.imageBuilder(stack, 'builder', { vpc }); + + const boundAmi = builder.bindAmi(); + const lt1 = new ec2.LaunchTemplate(stack, 'lt1'); + const lt2 = new ec2.LaunchTemplate(stack, 'lt2'); + + builder.bindAmi(); + builder.bindAmi({ launchTemplate: lt1 }); + builder.bindAmi({ launchTemplate: lt2 }); + + const builder2 = Ec2RunnerProvider.imageBuilder(stack, 'builder2', { vpc }); + const boundAmi2 = builder2.bindAmi({ launchTemplate: lt1 }); + + const template = Template.fromStack(stack); + + template.hasResourceProperties('AWS::ImageBuilder::DistributionConfiguration', { + Distributions: [ + { + LaunchTemplateConfigurations: [ + { LaunchTemplateId: { Ref: stack.getLogicalId(boundAmi.launchTemplate.node.defaultChild as CfnElement) } }, + { LaunchTemplateId: { Ref: stack.getLogicalId(lt1.node.defaultChild as CfnElement) } }, + { LaunchTemplateId: { Ref: stack.getLogicalId(lt2.node.defaultChild as CfnElement) } }, + ], + }, + ], + }); + + template.hasResourceProperties('AWS::ImageBuilder::DistributionConfiguration', { + Distributions: [ + { + LaunchTemplateConfigurations: [ + { LaunchTemplateId: { Ref: stack.getLogicalId(boundAmi2.launchTemplate.node.defaultChild as CfnElement) } }, + { LaunchTemplateId: { Ref: stack.getLogicalId(lt1.node.defaultChild as CfnElement) } }, + ], + }, + ], + }); +}); From f22d76a0a2e98a188facd31b444dc9edde3f8299 Mon Sep 17 00:00:00 2001 From: Amir Szekely Date: Tue, 13 Feb 2024 10:01:49 -0500 Subject: [PATCH 03/11] Custom launch template WIP --- src/providers/ec2fleet.ts | 100 +++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 49 deletions(-) diff --git a/src/providers/ec2fleet.ts b/src/providers/ec2fleet.ts index 6b829b7c..ae3b7029 100644 --- a/src/providers/ec2fleet.ts +++ b/src/providers/ec2fleet.ts @@ -344,6 +344,7 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { private readonly vpc: ec2.IVpc; private readonly subnets: ec2.ISubnet[]; private readonly securityGroups: ec2.ISecurityGroup[]; + private readonly launchTemplate: ec2.LaunchTemplate; constructor(scope: Construct, id: string, props?: Ec2RunnerProviderProps) { super(scope, id, props); @@ -357,17 +358,6 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { this.spot = props?.spot ?? false; this.spotMaxPrice = props?.spotMaxPrice; - this.amiBuilder = props?.imageBuilder ?? props?.amiBuilder ?? Ec2RunnerProvider.imageBuilder(this, 'Ami Builder', { - vpc: props?.vpc, - subnetSelection: props?.subnetSelection, - securityGroups: this.securityGroups, - }); - this.ami = this.amiBuilder.bindAmi(); - - if (!this.ami.architecture.instanceTypeMatch(this.instanceType)) { - throw new Error(`AMI architecture (${this.ami.architecture.name}) doesn't match runner instance type (${this.instanceType} / ${this.instanceType.architecture})`); - } - this.grantPrincipal = this.role = new iam.Role(this, 'Role', { assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'), }); @@ -391,6 +381,43 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { }, ); this.logGroup.grantWrite(this); + + // TODO FML + const rootDeviceResource = amiRootDevice(this, this.ami.launchTemplate.launchTemplateId); + rootDeviceResource.node.addDependency(this.amiBuilder); + + this.launchTemplate = new ec2.LaunchTemplate(this, 'Launch Template', { + instanceType: props?.instanceType ?? ec2.InstanceType.of(ec2.InstanceClass.M5, ec2.InstanceSize.LARGE), + role: this.role, + userData: ec2.UserData.custom(''), // TODO update user data to use tags + instanceInitiatedShutdownBehavior: ec2.InstanceInitiatedShutdownBehavior.TERMINATE, + requireImdsv2: true, + instanceMetadataTags: true, + blockDevices: [{ + deviceName: rootDeviceResource.ref, + volume: ec2.BlockDeviceVolume.ebs(this.storageSize.toGibibytes()), + }], + spotOptions: this.spot ? { + requestType: ec2.SpotRequestType.ONE_TIME, + maxPrice: this.spotMaxPrice ? Number(this.spotMaxPrice) : undefined, // TODO prop should be number + } : undefined, + }); + + + // TODO override network interfaces with subnet, security group, etc. + // this.launchTemplate. + //this.securityGroups.map(sg => this.launchTemplate.addSecurityGroup(sg)); + + this.amiBuilder = props?.imageBuilder ?? props?.amiBuilder ?? Ec2RunnerProvider.imageBuilder(this, 'Ami Builder', { + vpc: props?.vpc, + subnetSelection: props?.subnetSelection, + securityGroups: this.securityGroups, + }); + this.ami = this.amiBuilder.bindAmi({ launchTemplate: this.launchTemplate }); + + if (!this.ami.architecture.instanceTypeMatch(this.instanceType)) { + throw new Error(`AMI architecture (${this.ami.architecture.name}) doesn't match runner instance type (${this.instanceType} / ${this.instanceType.architecture})`); + } } /** @@ -431,57 +458,32 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { // if someone can figure out a good way to use Map for this, please open a PR // build a state for each subnet we want to try - const instanceProfile = new iam.CfnInstanceProfile(this, 'Instance Profile', { - roles: [this.role.roleName], - }); - const rootDeviceResource = amiRootDevice(this, this.ami.launchTemplate.launchTemplateId); - rootDeviceResource.node.addDependency(this.amiBuilder); const subnetRunners = this.subnets.map((subnet, index) => { return new stepfunctions_tasks.CallAwsService(this, `${this.labels.join(', ')} subnet${index+1}`, { comment: subnet.subnetId, integrationPattern: IntegrationPattern.WAIT_FOR_TASK_TOKEN, service: 'ec2', - action: 'runInstances', + action: 'createFleet', heartbeatTimeout: stepfunctions.Timeout.duration(Duration.minutes(10)), // TODO somehow create launch template with security group, profile, user data, tags in imds, etc. parameters: { LaunchTemplate: { - // TODO update builder to update our launch templates? - LaunchTemplateId: this.ami.launchTemplate.launchTemplateId, + LaunchTemplateId: this.launchTemplate.launchTemplateId, }, MinCount: 1, MaxCount: 1, - InstanceType: this.instanceType.toString(), - UserData: stepfunctions.JsonPath.base64Encode( - stepfunctions.JsonPath.format( - stepfunctions.JsonPath.stringAt('$.ec2.userdataTemplate'), - ...params, - ), - ), - InstanceInitiatedShutdownBehavior: ec2.InstanceInitiatedShutdownBehavior.TERMINATE, - IamInstanceProfile: { - Arn: instanceProfile.attrArn, - }, - MetadataOptions: { - HttpTokens: 'required', - // TODO InstanceMetadataTags: 'enabled' - }, - SecurityGroupIds: this.securityGroups.map(sg => sg.securityGroupId), - SubnetId: subnet.subnetId, - BlockDeviceMappings: [{ - DeviceName: rootDeviceResource.ref, - Ebs: { - DeleteOnTermination: true, - VolumeSize: this.storageSize.toGibibytes(), - }, - }], - InstanceMarketOptions: this.spot ? { - MarketType: 'spot', - SpotOptions: { - MaxPrice: this.spotMaxPrice, - SpotInstanceType: 'one-time', + Type: 'instant', + TagSpecification: [ + { + ResourceType: 'instance', + Tags: [ + { + Key: '', + Value: '', + }, + ], }, - } : undefined, + ], }, iamResources: ['*'], }); From cd503b4d1d9c1caf922fd55835f0f05a597bb18f Mon Sep 17 00:00:00 2001 From: Amir Szekely Date: Sun, 7 Apr 2024 14:22:52 -0400 Subject: [PATCH 04/11] WIP --- src/providers/ec2fleet.ts | 114 +++++++++++++++++++++++++------------- src/providers/index.ts | 1 + 2 files changed, 75 insertions(+), 40 deletions(-) diff --git a/src/providers/ec2fleet.ts b/src/providers/ec2fleet.ts index ae3b7029..697d88fd 100644 --- a/src/providers/ec2fleet.ts +++ b/src/providers/ec2fleet.ts @@ -1,3 +1,6 @@ +// TODO do we really need a separate ec2 and fleet provider? + +import { CreateFleetCommandInput } from '@aws-sdk/client-ec2'; import * as cdk from 'aws-cdk-lib'; import { aws_ec2 as ec2, @@ -13,7 +16,6 @@ import { RetentionDays } from 'aws-cdk-lib/aws-logs'; import { IntegrationPattern } from 'aws-cdk-lib/aws-stepfunctions'; import { Construct } from 'constructs'; import { - amiRootDevice, Architecture, BaseProvider, IRunnerProvider, @@ -28,7 +30,6 @@ import { IRunnerImageBuilder, RunnerImageBuilder, RunnerImageBuilderProps, Runne import { MINIMAL_EC2_SSM_SESSION_MANAGER_POLICY_STATEMENT } from '../utils'; // this script is specifically made so `poweroff` is absolutely always called -// each `{}` is a variable coming from `params` below const linuxUserDataTemplate = `#!/bin/bash -x IMDS_TOKEN=\`curl -sX PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60"\` TASK_TOKEN=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:taskToken\` @@ -99,8 +100,7 @@ else aws stepfunctions send-task-failure --task-token "$TASK_TOKEN" fi sleep 10 # give cloudwatch agent its default 5 seconds buffer duration to upload logs -poweroff -`.replace(/{/g, '\\{').replace(/}/g, '\\}').replace(/\\{\\}/g, '{}'); +poweroff`; // this script is specifically made so `poweroff` is absolutely always called // each `{}` is a variable coming from `params` below and their order should match the linux script @@ -176,7 +176,7 @@ Stop-Computer -ComputerName localhost -Force /** * Properties for {@link Ec2RunnerProvider} construct. */ -export interface Ec2RunnerProviderProps extends RunnerProviderProps { +export interface Ec2FleetRunnerProviderProps extends RunnerProviderProps { /** * Runner image builder used to build AMI containing GitHub Runner and all requirements. * @@ -275,7 +275,7 @@ export interface Ec2RunnerProviderProps extends RunnerProviderProps { * * This construct is not meant to be used by itself. It should be passed in the providers property for GitHubRunners. */ -export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { +export class Ec2FleetRunnerProvider extends BaseProvider implements IRunnerProvider { /** * Create new image builder that builds EC2 specific runner images. * @@ -346,7 +346,7 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { private readonly securityGroups: ec2.ISecurityGroup[]; private readonly launchTemplate: ec2.LaunchTemplate; - constructor(scope: Construct, id: string, props?: Ec2RunnerProviderProps) { + constructor(scope: Construct, id: string, props?: Ec2FleetRunnerProviderProps) { super(scope, id, props); this.labels = props?.labels ?? ['ec2']; @@ -383,18 +383,18 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { this.logGroup.grantWrite(this); // TODO FML - const rootDeviceResource = amiRootDevice(this, this.ami.launchTemplate.launchTemplateId); - rootDeviceResource.node.addDependency(this.amiBuilder); + // const rootDeviceResource = amiRootDevice(this, this.ami.launchTemplate.launchTemplateId); + // rootDeviceResource.node.addDependency(this.amiBuilder); this.launchTemplate = new ec2.LaunchTemplate(this, 'Launch Template', { instanceType: props?.instanceType ?? ec2.InstanceType.of(ec2.InstanceClass.M5, ec2.InstanceSize.LARGE), role: this.role, - userData: ec2.UserData.custom(''), // TODO update user data to use tags + userData: ec2.UserData.custom(linuxUserDataTemplate), instanceInitiatedShutdownBehavior: ec2.InstanceInitiatedShutdownBehavior.TERMINATE, requireImdsv2: true, instanceMetadataTags: true, blockDevices: [{ - deviceName: rootDeviceResource.ref, + deviceName: '/dev/sda1', // TODO rootDeviceResource.ref, volume: ec2.BlockDeviceVolume.ebs(this.storageSize.toGibibytes()), }], spotOptions: this.spot ? { @@ -403,18 +403,17 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { } : undefined, }); + this.securityGroups.map(sg => this.launchTemplate.addSecurityGroup(sg)); - // TODO override network interfaces with subnet, security group, etc. - // this.launchTemplate. - //this.securityGroups.map(sg => this.launchTemplate.addSecurityGroup(sg)); - - this.amiBuilder = props?.imageBuilder ?? props?.amiBuilder ?? Ec2RunnerProvider.imageBuilder(this, 'Ami Builder', { + this.amiBuilder = props?.imageBuilder ?? props?.amiBuilder ?? Ec2FleetRunnerProvider.imageBuilder(this, 'Ami Builder', { vpc: props?.vpc, subnetSelection: props?.subnetSelection, securityGroups: this.securityGroups, }); this.ami = this.amiBuilder.bindAmi({ launchTemplate: this.launchTemplate }); + // TODO using an existing image builder will not update the launch template on deploy -- make AWS::ImageBuilder::Image get new id on adding launch templates (or changing them?) + if (!this.ami.architecture.instanceTypeMatch(this.instanceType)) { throw new Error(`AMI architecture (${this.ami.architecture.name}) doesn't match runner instance type (${this.instanceType} / ${this.instanceType.architecture})`); } @@ -430,17 +429,6 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { getStepFunctionTask(parameters: RunnerRuntimeParameters): stepfunctions.IChainable { // we need to build user data in two steps because passing the template as the first parameter to stepfunctions.JsonPath.format fails on syntax - const params = [ - stepfunctions.JsonPath.taskToken, - this.logGroup.logGroupName, - parameters.runnerNamePath, - parameters.githubDomainPath, - parameters.ownerPath, - parameters.repoPath, - parameters.runnerTokenPath, - this.labels.join(','), - parameters.registrationUrl, - ]; const passUserData = new stepfunctions.Pass(this, `${this.labels.join(', ')} data`, { parameters: { @@ -466,20 +454,71 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { action: 'createFleet', heartbeatTimeout: stepfunctions.Timeout.duration(Duration.minutes(10)), // TODO somehow create launch template with security group, profile, user data, tags in imds, etc. - parameters: { - LaunchTemplate: { - LaunchTemplateId: this.launchTemplate.launchTemplateId, + parameters: { + TargetCapacitySpecification: { + TotalTargetCapacity: 1, + // TargetCapacityUnitType: 'units', + DefaultTargetCapacityType: this.spot ? 'spot' : 'on-demand', }, - MinCount: 1, - MaxCount: 1, + LaunchTemplateConfigs: [ + // TODO let user specify these + { + LaunchTemplateSpecification: { + LaunchTemplateId: this.launchTemplate.launchTemplateId, + Version: '$Latest', // TODO latest or default? -- latest for last created by cdk, default for last created by image builder + }, + Overrides: [ + { + SubnetId: subnet.subnetId, + // TODO AMI from image builder launch template ImageId: this.ami.launchTemplate.imageId, + ImageId: 'ami-049d60fa777700d2b', + }, + ], + // TODO Overrides + }, + ], Type: 'instant', - TagSpecification: [ + TagSpecifications: [ { ResourceType: 'instance', Tags: [ { - Key: '', - Value: '', + Key: 'GitHubRunners:args:taskToken', + // TODO Tag value exceeds the maximum length of 256 characters (Service: Ec2, Status Code: 400, Request ID: a60d7844-a5ae-49d2-aeeb-d36df39cf0fc) + // Value: stepfunctions.JsonPath.taskToken, + Value: 'nope', + }, + { + Key: 'GitHubRunners:args:logGroupName', + Value: this.logGroup.logGroupName, + }, + { + Key: 'GitHubRunners:args:runnerNamePath', + Value: parameters.runnerNamePath, + }, + { + Key: 'GitHubRunners:args:githubDomainPath', + Value: parameters.githubDomainPath, + }, + { + Key: 'GitHubRunners:args:ownerPath', + Value: parameters.ownerPath, + }, + { + Key: 'GitHubRunners:args:repoPath', + Value: parameters.repoPath, + }, + { + Key: 'GitHubRunners:args:runnerTokenPath', + Value: parameters.runnerTokenPath, + }, + { + Key: 'GitHubRunners:args:labels', + Value: this.labels.join(','), + }, + { + Key: 'GitHubRunners:args:registrationURL', + Value: parameters.registrationUrl, }, ], }, @@ -560,8 +599,3 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { } } -/** - * @deprecated use {@link Ec2RunnerProvider} - */ -export class Ec2Runner extends Ec2RunnerProvider { -} diff --git a/src/providers/index.ts b/src/providers/index.ts index 5bf48d59..0d5e8efc 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -1,4 +1,5 @@ export * from './ec2'; +export * from './ec2fleet'; export * from './ecs'; export * from './codebuild'; export * from './lambda'; From 695a3e1a82ec123cf025b2da8b04398f472681eb Mon Sep 17 00:00:00 2001 From: Amir Szekely Date: Sun, 21 Apr 2024 16:45:29 -0400 Subject: [PATCH 05/11] use ssm automation to start runner tags couldn't fit all the data --- .../aws-image-builder/builder.ts | 22 +- .../aws-image-builder/deprecated/ami.ts | 28 +- .../aws-image-builder/deprecated/common.ts | 4 +- .../aws-image-builder/deprecated/container.ts | 4 +- src/image-builders/codebuild-deprecated.ts | 4 +- src/image-builders/common.ts | 16 +- src/providers/ec2.ts | 9 - src/providers/ec2fleet.ts | 633 ++++++++++-------- test/default.integ.ts | 6 + 9 files changed, 372 insertions(+), 354 deletions(-) diff --git a/src/image-builders/aws-image-builder/builder.ts b/src/image-builders/aws-image-builder/builder.ts index 40059d1e..c1c69b66 100644 --- a/src/image-builders/aws-image-builder/builder.ts +++ b/src/image-builders/aws-image-builder/builder.ts @@ -23,7 +23,7 @@ import { ImageBuilderObjectBase } from './common'; import { ContainerRecipe, defaultBaseDockerImage } from './container'; import { DeleteAmiFunction } from './delete-ami-function'; import { FilterFailedBuildsFunction } from './filter-failed-builds-function'; -import { Architecture, BindAmiProps, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../providers'; +import { Architecture, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../providers'; import { singletonLambda } from '../../utils'; import { BuildImageFunction } from '../build-image-function'; import { RunnerImageBuilderBase, RunnerImageBuilderProps, uniqueImageBuilderName } from '../common'; @@ -294,8 +294,6 @@ export class ImageBuilderComponent extends ImageBuilderObjectBase { export class AwsImageBuilderRunnerImageBuilder extends RunnerImageBuilderBase { private boundDockerImage?: RunnerImage; private boundAmi?: RunnerAmi; - private boundDistribution?: imagebuilder.CfnDistributionConfiguration; - private boundDistributionLaunchTemplateCount = 0; private readonly os: Os; private readonly architecture: Architecture; private readonly baseImage: string; @@ -624,9 +622,8 @@ export class AwsImageBuilderRunnerImageBuilder extends RunnerImageBuilderBase { return this.role; } - bindAmi(props?: BindAmiProps): RunnerAmi { + bindAmi(): RunnerAmi { if (this.boundAmi) { - this.addLaunchTemplateToDistribution(props?.launchTemplate); return this.boundAmi; } @@ -706,7 +703,7 @@ export class AwsImageBuilderRunnerImageBuilder extends RunnerImageBuilderBase { const stackName = cdk.Stack.of(this).stackName; const builderName = this.node.path; - this.boundDistribution = new imagebuilder.CfnDistributionConfiguration(this, 'AMI Distribution', { + const dist = new imagebuilder.CfnDistributionConfiguration(this, 'AMI Distribution', { name: uniqueImageBuilderName(this), // description: this.description, distributions: [ @@ -729,8 +726,6 @@ export class AwsImageBuilderRunnerImageBuilder extends RunnerImageBuilderBase { }, ], }); - this.addLaunchTemplateToDistribution(launchTemplate); - this.addLaunchTemplateToDistribution(props?.launchTemplate); const recipe = new AmiRecipe(this, 'Ami Recipe', { platform: this.platform(), @@ -744,8 +739,8 @@ export class AwsImageBuilderRunnerImageBuilder extends RunnerImageBuilderBase { iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'), iam.ManagedPolicy.fromAwsManagedPolicyName('EC2InstanceProfileForImageBuilder'), ]); - this.createImage(infra, this.boundDistribution, log, recipe.arn, undefined); - this.createPipeline(infra, this.boundDistribution, log, recipe.arn, undefined); + this.createImage(infra, dist, log, recipe.arn, undefined); + this.createPipeline(infra, dist, log, recipe.arn, undefined); this.boundAmi = { launchTemplate: launchTemplate, @@ -859,13 +854,6 @@ export class AwsImageBuilderRunnerImageBuilder extends RunnerImageBuilderBase { return this.boundComponents; } - - private addLaunchTemplateToDistribution(launchTemplate?: ec2.ILaunchTemplate) { - if (launchTemplate && this.boundDistribution) { - this.boundDistribution.addPropertyOverride(`Distributions.0.LaunchTemplateConfigurations.${this.boundDistributionLaunchTemplateCount}.LaunchTemplateId`, launchTemplate.launchTemplateId); - this.boundDistributionLaunchTemplateCount++; - } - } } /** diff --git a/src/image-builders/aws-image-builder/deprecated/ami.ts b/src/image-builders/aws-image-builder/deprecated/ami.ts index 06315db1..7a892a5a 100644 --- a/src/image-builders/aws-image-builder/deprecated/ami.ts +++ b/src/image-builders/aws-image-builder/deprecated/ami.ts @@ -15,7 +15,7 @@ import { Construct } from 'constructs'; import { ImageBuilderBase } from './common'; import { LinuxUbuntuComponents } from './linux-components'; import { WindowsComponents } from './windows-components'; -import { Architecture, BindAmiProps, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../../providers'; +import { Architecture, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../../providers'; import { singletonLambda } from '../../../utils'; import { uniqueImageBuilderName } from '../../common'; import { AmiRecipe, defaultBaseAmi } from '../ami'; @@ -151,8 +151,6 @@ export interface AmiBuilderProps { */ export class AmiBuilder extends ImageBuilderBase { private boundAmi?: RunnerAmi; - private boundDistribution?: imagebuilder.CfnDistributionConfiguration; - private boundDistributionLaunchTemplateCount = 0; constructor(scope: Construct, id: string, props?: AmiBuilderProps) { super(scope, id, { @@ -250,9 +248,8 @@ export class AmiBuilder extends ImageBuilderBase { /** * Called by IRunnerProvider to finalize settings and create the AMI builder. */ - bindAmi(props?: BindAmiProps): RunnerAmi { + bindAmi(): RunnerAmi { if (this.boundAmi) { - this.addLaunchTemplateToDistribution(props?.launchTemplate); return this.boundAmi; } @@ -263,7 +260,7 @@ export class AmiBuilder extends ImageBuilderBase { const stackName = cdk.Stack.of(this).stackName; const builderName = this.node.path; - this.boundDistribution = new imagebuilder.CfnDistributionConfiguration(this, 'Distribution', { + const dist = new imagebuilder.CfnDistributionConfiguration(this, 'Distribution', { name: uniqueImageBuilderName(this), description: this.description, distributions: [ @@ -281,12 +278,14 @@ export class AmiBuilder extends ImageBuilderBase { 'GitHubRunners:Builder': builderName, }, }, - launchTemplateConfigurations: [], + launchTemplateConfigurations: [ + { + launchTemplateId: launchTemplate.launchTemplateId, + }, + ], }, ], }); - this.addLaunchTemplateToDistribution(launchTemplate); - this.addLaunchTemplateToDistribution(props?.launchTemplate); const recipe = new AmiRecipe(this, 'Ami Recipe', { platform: this.platform, @@ -300,8 +299,8 @@ export class AmiBuilder extends ImageBuilderBase { iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'), iam.ManagedPolicy.fromAwsManagedPolicyName('EC2InstanceProfileForImageBuilder'), ]); - this.createImage(infra, this.boundDistribution, log, recipe.arn, undefined); - this.createPipeline(infra, this.boundDistribution, log, recipe.arn, undefined); + this.createImage(infra, dist, log, recipe.arn, undefined); + this.createPipeline(infra, dist, log, recipe.arn, undefined); this.boundAmi = { launchTemplate: launchTemplate, @@ -357,11 +356,4 @@ export class AmiBuilder extends ImageBuilderBase { bindDockerImage(): RunnerImage { throw new Error('AmiBuilder cannot be used to build Docker images'); } - - private addLaunchTemplateToDistribution(launchTemplate?: ec2.ILaunchTemplate) { - if (launchTemplate && this.boundDistribution) { - this.boundDistribution.addPropertyOverride(`Distributions.0.LaunchTemplateConfigurations.${this.boundDistributionLaunchTemplateCount}.LaunchTemplateId`, launchTemplate.launchTemplateId); - this.boundDistributionLaunchTemplateCount++; - } - } } diff --git a/src/image-builders/aws-image-builder/deprecated/common.ts b/src/image-builders/aws-image-builder/deprecated/common.ts index c1fe4508..3b210786 100644 --- a/src/image-builders/aws-image-builder/deprecated/common.ts +++ b/src/image-builders/aws-image-builder/deprecated/common.ts @@ -1,7 +1,7 @@ import * as cdk from 'aws-cdk-lib'; import { aws_ec2 as ec2, aws_events as events, aws_iam as iam, aws_imagebuilder as imagebuilder, aws_logs as logs, RemovalPolicy } from 'aws-cdk-lib'; import { Construct } from 'constructs'; -import { Architecture, BindAmiProps, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../../providers'; +import { Architecture, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../../providers'; import { ImageBuilderBaseProps, IRunnerImageBuilder, uniqueImageBuilderName } from '../../common'; import { ImageBuilderComponent } from '../builder'; @@ -176,5 +176,5 @@ export abstract class ImageBuilderBase extends Construct implements IRunnerImage abstract bindDockerImage(): RunnerImage; - abstract bindAmi(props?: BindAmiProps): RunnerAmi; + abstract bindAmi(): RunnerAmi; } diff --git a/src/image-builders/aws-image-builder/deprecated/container.ts b/src/image-builders/aws-image-builder/deprecated/container.ts index 2e16c7d4..8b445031 100644 --- a/src/image-builders/aws-image-builder/deprecated/container.ts +++ b/src/image-builders/aws-image-builder/deprecated/container.ts @@ -15,7 +15,7 @@ import { Construct } from 'constructs'; import { ImageBuilderBase } from './common'; import { LinuxUbuntuComponents } from './linux-components'; import { WindowsComponents } from './windows-components'; -import { Architecture, BindAmiProps, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../../providers'; +import { Architecture, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../../../providers'; import { singletonLambda } from '../../../utils'; import { BuildImageFunction } from '../../build-image-function'; import { uniqueImageBuilderName } from '../../common'; @@ -338,7 +338,7 @@ export class ContainerImageBuilder extends ImageBuilderBase { return cr; } - bindAmi(_?: BindAmiProps): RunnerAmi { + bindAmi(): RunnerAmi { throw new Error('ContainerImageBuilder cannot be used to build AMIs'); } } diff --git a/src/image-builders/codebuild-deprecated.ts b/src/image-builders/codebuild-deprecated.ts index d5872065..367b5f3f 100644 --- a/src/image-builders/codebuild-deprecated.ts +++ b/src/image-builders/codebuild-deprecated.ts @@ -19,7 +19,7 @@ import { TagMutability, TagStatus } from 'aws-cdk-lib/aws-ecr'; import { RetentionDays } from 'aws-cdk-lib/aws-logs'; import { Construct } from 'constructs'; import { BuildImageFunction } from './build-image-function'; -import { BindAmiProps, IRunnerImageBuilder } from './common'; +import { IRunnerImageBuilder } from './common'; import { Architecture, Os, RunnerAmi, RunnerImage, RunnerVersion } from '../providers'; import { singletonLambda } from '../utils'; @@ -527,7 +527,7 @@ export class CodeBuildImageBuilder extends Construct implements IRunnerImageBuil }); } - bindAmi(_?: BindAmiProps): RunnerAmi { + bindAmi(): RunnerAmi { throw new Error('CodeBuildImageBuilder does not support building AMIs'); } } diff --git a/src/image-builders/common.ts b/src/image-builders/common.ts index ff9817db..3bfcd1bd 100644 --- a/src/image-builders/common.ts +++ b/src/image-builders/common.ts @@ -251,18 +251,6 @@ export enum RunnerImageBuilderType { AWS_IMAGE_BUILDER = 'AwsImageBuilder', } -/** - * Properties for RunnerImageBuilder.bindAmi(). - */ -export interface BindAmiProps { - /** - * Launch template to update with new generated AMI when available. - * - * @note Do not pass the same launch template to multiple builders. They will overwrite each other's AMIs. - */ - readonly launchTemplate?: ec2.ILaunchTemplate; -} - /** * Interface for constructs that build an image that can be used in {@link IRunnerProvider}. * @@ -287,7 +275,7 @@ export interface IRunnerImageBuilder { * * The AMI can be further updated over time manually or using a schedule as long as it is always written to the same launch template. */ - bindAmi(props?: BindAmiProps): RunnerAmi; + bindAmi(): RunnerAmi; } /** @@ -327,7 +315,7 @@ export abstract class RunnerImageBuilderBase extends Construct implements IConfi abstract bindDockerImage(): RunnerImage; - abstract bindAmi(props?: BindAmiProps): RunnerAmi; + abstract bindAmi(): RunnerAmi; abstract get connections(): ec2.Connections; abstract get grantPrincipal(): iam.IPrincipal; diff --git a/src/providers/ec2.ts b/src/providers/ec2.ts index dc4eadac..574a8d39 100644 --- a/src/providers/ec2.ts +++ b/src/providers/ec2.ts @@ -33,9 +33,6 @@ const linuxUserDataTemplate = `#!/bin/bash -x TASK_TOKEN="{}" logGroupName="{}" runnerNamePath="{}" -githubDomainPath="{}" -ownerPath="{}" -repoPath="{}" runnerTokenPath="{}" labels="{}" registrationURL="{}" @@ -106,9 +103,6 @@ const windowsUserDataTemplate = ` $TASK_TOKEN = "{}" $logGroupName="{}" $runnerNamePath="{}" -$githubDomainPath="{}" -$ownerPath="{}" -$repoPath="{}" $runnerTokenPath="{}" $labels="{}" $registrationURL="{}" @@ -405,9 +399,6 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { stepfunctions.JsonPath.taskToken, this.logGroup.logGroupName, parameters.runnerNamePath, - parameters.githubDomainPath, - parameters.ownerPath, - parameters.repoPath, parameters.runnerTokenPath, this.labels.join(','), parameters.registrationUrl, diff --git a/src/providers/ec2fleet.ts b/src/providers/ec2fleet.ts index 697d88fd..5955b8be 100644 --- a/src/providers/ec2fleet.ts +++ b/src/providers/ec2fleet.ts @@ -1,21 +1,20 @@ // TODO do we really need a separate ec2 and fleet provider? +// TODO let user specify fleet launch templates? something useful -import { CreateFleetCommandInput } from '@aws-sdk/client-ec2'; import * as cdk from 'aws-cdk-lib'; import { aws_ec2 as ec2, aws_iam as iam, aws_logs as logs, + aws_ssm as ssm, aws_stepfunctions as stepfunctions, - aws_stepfunctions_tasks as stepfunctions_tasks, - Duration, + aws_stepfunctions_tasks as stepfunctions_tasks, Duration, RemovalPolicy, Stack, } from 'aws-cdk-lib'; -import { RetentionDays } from 'aws-cdk-lib/aws-logs'; -import { IntegrationPattern } from 'aws-cdk-lib/aws-stepfunctions'; import { Construct } from 'constructs'; import { + amiRootDevice, Architecture, BaseProvider, IRunnerProvider, @@ -29,149 +28,6 @@ import { import { IRunnerImageBuilder, RunnerImageBuilder, RunnerImageBuilderProps, RunnerImageBuilderType, RunnerImageComponent } from '../image-builders'; import { MINIMAL_EC2_SSM_SESSION_MANAGER_POLICY_STATEMENT } from '../utils'; -// this script is specifically made so `poweroff` is absolutely always called -const linuxUserDataTemplate = `#!/bin/bash -x -IMDS_TOKEN=\`curl -sX PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60"\` -TASK_TOKEN=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:taskToken\` -logGroupName=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:logGroupName\` -runnerNamePath=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:runnerNamePath\` -githubDomainPath=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:githubDomainPath\` -ownerPath=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:ownerPath\` -repoPath=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:repoPath\` -runnerTokenPath=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:runnerTokenPath\` -labels=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:labels\` -registrationURL=\`curl -sH "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/GitHubRunners:args:registrationURL\` -unset IMDS_TOKEN - -heartbeat () { - while true; do - aws stepfunctions send-task-heartbeat --task-token "$TASK_TOKEN" - sleep 60 - done -} -setup_logs () { - cat < /tmp/log.conf || exit 1 - { - "logs": { - "log_stream_name": "unknown", - "logs_collected": { - "files": { - "collect_list": [ - { - "file_path": "/var/log/runner.log", - "log_group_name": "$logGroupName", - "log_stream_name": "$runnerNamePath", - "timezone": "UTC" - } - ] - } - } - } - } -EOF - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/tmp/log.conf || exit 2 -} -action () { - # Determine the value of RUNNER_FLAGS - if [ "$(< RUNNER_VERSION)" = "latest" ]; then - RUNNER_FLAGS="" - else - RUNNER_FLAGS="--disableupdate" - fi - - labelsTemplate="$labels,cdkghr:started:$(date +%s)" - - # Execute the configuration command for runner registration - sudo -Hu runner /home/runner/config.sh --unattended --url "$registrationURL" --token "$runnerTokenPath" --ephemeral --work _work --labels "$labelsTemplate" $RUNNER_FLAGS --name "$runnerNamePath" || exit 1 - - # Execute the run command - sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2 - - # Retrieve the status - STATUS=$(grep -Phors "finish job request for job [0-9a-f\\-]+ with result: \K.*" /home/runner/_diag/ | tail -n1) - - # Check and print the job status - [ -n "$STATUS" ] && echo CDKGHA JOB DONE "$labels" "$STATUS" -} -heartbeat & -if setup_logs && action | tee /var/log/runner.log 2>&1; then - aws stepfunctions send-task-success --task-token "$TASK_TOKEN" --task-output '{"ok": true}' -else - aws stepfunctions send-task-failure --task-token "$TASK_TOKEN" -fi -sleep 10 # give cloudwatch agent its default 5 seconds buffer duration to upload logs -poweroff`; - -// this script is specifically made so `poweroff` is absolutely always called -// each `{}` is a variable coming from `params` below and their order should match the linux script -const windowsUserDataTemplate = ` -$TASK_TOKEN = "{}" -$logGroupName="{}" -$runnerNamePath="{}" -$githubDomainPath="{}" -$ownerPath="{}" -$repoPath="{}" -$runnerTokenPath="{}" -$labels="{}" -$registrationURL="{}" - -Start-Job -ScriptBlock { - while (1) { - aws stepfunctions send-task-heartbeat --task-token "$using:TASK_TOKEN" - sleep 60 - } -} -function setup_logs () { - echo '{ - "logs": { - "log_stream_name": "unknown", - "logs_collected": { - "files": { - "collect_list": [ - { - "file_path": "/actions/runner.log", - "log_group_name": "$logGroupName", - "log_stream_name": "$runnerNamePath", - "timezone": "UTC" - } - ] - } - } - } - }' | Out-File -Encoding ASCII $Env:TEMP/log.conf - & "C:/Program Files/Amazon/AmazonCloudWatchAgent/amazon-cloudwatch-agent-ctl.ps1" -a fetch-config -m ec2 -s -c file:$Env:TEMP/log.conf -} -function action () { - cd /actions - $RunnerVersion = Get-Content RUNNER_VERSION -Raw - if ($RunnerVersion -eq "latest") { $RunnerFlags = "" } else { $RunnerFlags = "--disableupdate" } - ./config.cmd --unattended --url "\${registrationUrl}" --token "\${runnerTokenPath}" --ephemeral --work _work --labels "\${labels},cdkghr:started:$(Get-Date -UFormat +%s)" $RunnerFlags --name "\${runnerNamePath}" 2>&1 | Out-File -Encoding ASCII -Append /actions/runner.log - - if ($LASTEXITCODE -ne 0) { return 1 } - ./run.cmd 2>&1 | Out-File -Encoding ASCII -Append /actions/runner.log - if ($LASTEXITCODE -ne 0) { return 2 } - - $STATUS = Select-String -Path './_diag/*.log' -Pattern 'finish job request for job [0-9a-f\\-]+ with result: (.*)' | %{$_.Matches.Groups[1].Value} | Select-Object -Last 1 - - if ($STATUS) { - echo "CDKGHA JOB DONE \${labels} $STATUS" | Out-File -Encoding ASCII -Append /actions/runner.log - } - - return 0 - -} -setup_logs -$r = action -if ($r -eq 0) { - aws stepfunctions send-task-success --task-token "$TASK_TOKEN" --task-output '{ }' -} else { - aws stepfunctions send-task-failure --task-token "$TASK_TOKEN" -} -Start-Sleep -Seconds 10 # give cloudwatch agent its default 5 seconds buffer duration to upload logs -Stop-Computer -ComputerName localhost -Force - -`.replace(/{/g, '\\{').replace(/}/g, '\\}').replace(/\\{\\}/g, '{}'); - /** * Properties for {@link Ec2RunnerProvider} construct. @@ -345,10 +201,12 @@ export class Ec2FleetRunnerProvider extends BaseProvider implements IRunnerProvi private readonly subnets: ec2.ISubnet[]; private readonly securityGroups: ec2.ISecurityGroup[]; private readonly launchTemplate: ec2.LaunchTemplate; + private readonly document: ssm.CfnDocument; constructor(scope: Construct, id: string, props?: Ec2FleetRunnerProviderProps) { super(scope, id, props); + // read parameters this.labels = props?.labels ?? ['ec2']; this.vpc = props?.vpc ?? ec2.Vpc.fromLookup(this, 'Default VPC', { isDefault: true }); this.securityGroups = props?.securityGroup ? [props.securityGroup] : (props?.securityGroups ?? [new ec2.SecurityGroup(this, 'SG', { vpc: this.vpc })]); @@ -358,65 +216,302 @@ export class Ec2FleetRunnerProvider extends BaseProvider implements IRunnerProvi this.spot = props?.spot ?? false; this.spotMaxPrice = props?.spotMaxPrice; + // instance role this.grantPrincipal = this.role = new iam.Role(this, 'Role', { assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'), - }); - this.grantPrincipal.addToPrincipalPolicy(new iam.PolicyStatement({ - actions: ['states:SendTaskFailure', 'states:SendTaskSuccess', 'states:SendTaskHeartbeat'], - resources: ['*'], // no support for stateMachine.stateMachineArn :( - conditions: { - StringEquals: { - 'aws:ResourceTag/aws:cloudformation:stack-id': cdk.Stack.of(this).stackId, - }, + inlinePolicies: { + stepfunctions: new iam.PolicyDocument({ + statements: [ + new iam.PolicyStatement({ + actions: ['states:SendTaskHeartbeat'], + resources: ['*'], // no support for stateMachine.stateMachineArn :( + conditions: { + StringEquals: { + 'aws:ResourceTag/aws:cloudformation:stack-id': cdk.Stack.of(this).stackId, + }, + }, + }), + ], + }), }, - })); + }); this.grantPrincipal.addToPrincipalPolicy(MINIMAL_EC2_SSM_SESSION_MANAGER_POLICY_STATEMENT); - this.logGroup = new logs.LogGroup( - this, - 'Logs', - { - retention: props?.logRetention ?? RetentionDays.ONE_MONTH, - removalPolicy: RemovalPolicy.DESTROY, - }, - ); - this.logGroup.grantWrite(this); + // build ami + this.amiBuilder = props?.imageBuilder ?? props?.amiBuilder ?? Ec2FleetRunnerProvider.imageBuilder(this, 'Ami Builder', { + vpc: props?.vpc, + subnetSelection: props?.subnetSelection, + securityGroups: this.securityGroups, + }); + this.ami = this.amiBuilder.bindAmi(); + + if (!this.ami.architecture.instanceTypeMatch(this.instanceType)) { + throw new Error(`AMI architecture (${this.ami.architecture.name}) doesn't match runner instance type (${this.instanceType} / ${this.instanceType.architecture})`); + } - // TODO FML - // const rootDeviceResource = amiRootDevice(this, this.ami.launchTemplate.launchTemplateId); - // rootDeviceResource.node.addDependency(this.amiBuilder); + // figure out root device + const rootDeviceResource = amiRootDevice(this, this.ami.launchTemplate.launchTemplateId); + rootDeviceResource.node.addDependency(this.amiBuilder); + // launch template (ami will be added later with override) this.launchTemplate = new ec2.LaunchTemplate(this, 'Launch Template', { - instanceType: props?.instanceType ?? ec2.InstanceType.of(ec2.InstanceClass.M5, ec2.InstanceSize.LARGE), + instanceType: this.instanceType, role: this.role, - userData: ec2.UserData.custom(linuxUserDataTemplate), instanceInitiatedShutdownBehavior: ec2.InstanceInitiatedShutdownBehavior.TERMINATE, requireImdsv2: true, instanceMetadataTags: true, blockDevices: [{ - deviceName: '/dev/sda1', // TODO rootDeviceResource.ref, - volume: ec2.BlockDeviceVolume.ebs(this.storageSize.toGibibytes()), + deviceName: rootDeviceResource.ref, + volume: ec2.BlockDeviceVolume.ebs(this.storageSize.toGibibytes(), { + deleteOnTermination: true, + }), }], spotOptions: this.spot ? { requestType: ec2.SpotRequestType.ONE_TIME, maxPrice: this.spotMaxPrice ? Number(this.spotMaxPrice) : undefined, // TODO prop should be number } : undefined, + userData: ec2.UserData.forLinux(), }); - - this.securityGroups.map(sg => this.launchTemplate.addSecurityGroup(sg)); - - this.amiBuilder = props?.imageBuilder ?? props?.amiBuilder ?? Ec2FleetRunnerProvider.imageBuilder(this, 'Ami Builder', { - vpc: props?.vpc, - subnetSelection: props?.subnetSelection, - securityGroups: this.securityGroups, + this.launchTemplate.userData!.addCommands(`{ + sleep 600 + if [ ! -e /home/runner/STARTED ]; then + echo "Runner didn't connect to SSM, powering off" + sudo poweroff + fi + } &`); + + // role specifically for ssm document + const ssmRole = new iam.Role(this, 'Ssm Role', { + assumedBy: new iam.ServicePrincipal('ssm.amazonaws.com'), + inlinePolicies: { + stepfunctions: new iam.PolicyDocument({ + statements: [ + new iam.PolicyStatement({ + actions: ['states:SendTaskFailure', 'states:SendTaskSuccess'], + resources: ['*'], // no support for stateMachine.stateMachineArn :( + conditions: { + StringEquals: { + 'aws:ResourceTag/aws:cloudformation:stack-id': cdk.Stack.of(this).stackId, + }, + }, + }), + ], + }), + ssm: new iam.PolicyDocument({ + statements: [ + new iam.PolicyStatement({ + actions: ['ssm:SendCommand'], + resources: [ + '*', + // { + // 'Fn::Sub': 'arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:instance/*', + // }, + // { + // 'Fn::Sub': 'arn:aws:ssm:${AWS::Region}::document/AWS-RunShellScript', + // }, + // { + // 'Fn::Sub': 'arn:aws:ssm:${AWS::Region}::document/AWS-RunPowerShellScript', + // }, + ], + }), + new iam.PolicyStatement({ + actions: ['ssm:DescribeInstanceInformation'], + resources: ['*'], + }), + new iam.PolicyStatement({ + actions: [ + 'ssm:ListCommands', + 'ssm:ListCommandInvocations', + ], + resources: [ + '*', + // { + // 'Fn::Sub': 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:*', + // }, + ], + }), + ], + }), + ec2: new iam.PolicyDocument({ + statements: [ + new iam.PolicyStatement({ + actions: ['ec2:TerminateInstances'], + resources: ['*'], + conditions: { + StringEquals: { + 'aws:ResourceTag/aws:ec2launchtemplate:id': this.launchTemplate.launchTemplateId, + }, + }, + }), + ], + }), + }, }); - this.ami = this.amiBuilder.bindAmi({ launchTemplate: this.launchTemplate }); - - // TODO using an existing image builder will not update the launch template on deploy -- make AWS::ImageBuilder::Image get new id on adding launch templates (or changing them?) - if (!this.ami.architecture.instanceTypeMatch(this.instanceType)) { - throw new Error(`AMI architecture (${this.ami.architecture.name}) doesn't match runner instance type (${this.instanceType} / ${this.instanceType.architecture})`); - } + // log group for runner + this.logGroup = new logs.LogGroup( + this, + 'Logs', + { + retention: props?.logRetention ?? logs.RetentionDays.ONE_MONTH, + removalPolicy: RemovalPolicy.DESTROY, + }, + ); + this.logGroup.grantWrite(this.role); + this.logGroup.grant(this.role, 'logs:CreateLogGroup'); + + // ssm document that starts runner, updates step function, and terminates the instance + this.document = new ssm.CfnDocument(this, 'Automation', { + // name: 'SfnRunCommandByTargets', + documentType: 'Automation', + targetType: '/AWS::EC2::Host', + content: { + description: 'TODO github runners', + schemaVersion: '0.3', + assumeRole: ssmRole.roleArn, + parameters: { + instanceId: { + type: 'String', + description: 'Instance id where runner should be executed', + }, + taskToken: { + type: 'String', + description: 'Step Function task token for callback response', + }, + runnerName: { + type: 'String', + description: 'Runner name', + }, + runnerToken: { + type: 'String', + description: 'Runner token used to register runner on GitHub', + }, + labels: { + type: 'String', + description: 'Labels to assign to runner', + }, + registrationUrl: { + type: 'String', + description: 'Full URL to use for runner registration', + }, + }, + mainSteps: [ + // { + // name: 'Branch', + // action: 'aws:branch', + // inputs: { + // Choices: [ + // { + // NextStep: 'RunCommand_Powershell', + // Variable: '{{ shell }}', + // StringEquals: 'PowerShell', + // }, + // ], + // Default: 'RunCommand_Shell', + // }, + // }, + { + name: 'RunCommand', + action: 'aws:runCommand', + inputs: { + DocumentName: 'AWS-RunShellScript', + InstanceIds: ['{{ instanceId }}'], + // TODO no execution timeout + Parameters: { + commands: [ + 'set -ex', + // tell user data that we started + 'touch /home/runner/STARTED', + // send heartbeat + `{ + while true; do + aws stepfunctions send-task-heartbeat --task-token "{{ taskToken }}" + sleep 60 + done + } &`, + // decide if we should update runner + `if [ "$(cat RUNNER_VERSION)" = "latest" ]; then + RUNNER_FLAGS="" + else + RUNNER_FLAGS="--disableupdate" + fi`, + // configure runner + 'sudo -Hu runner /home/runner/config.sh --unattended --url "{{ registrationUrl }}" --token "{{ runnerToken }}" --ephemeral --work _work --labels "{{ labels }},cdkghr:started:$(date +%s)" $RUNNER_FLAGS --name "{{ runnerName }}" || exit 1', + // start runner without exposing task token and other possibly sensitive environment variables + 'sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2', + // print whether job was successful for our metric filter + `STATUS=$(grep -Phors "finish job request for job [0-9a-f\\-]+ with result: \\K.*" /home/runner/_diag/ | tail -n1) + [ -n "$STATUS" ] && echo CDKGHA JOB DONE "$labels" "$STATUS"`, + ], + workingDirectory: '/home/runner', + }, + CloudWatchOutputConfig: { + CloudWatchLogGroupName: this.logGroup.logGroupName, + CloudWatchOutputEnabled: true, + }, + }, + nextStep: 'SendTaskSuccess', + onFailure: 'step:SendTaskFailure', + onCancel: 'step:SendTaskFailure', + }, + // { + // name: 'RunCommand_Powershell', + // action: 'aws:runCommand', + // inputs: { + // DocumentName: 'AWS-RunPowerShellScript', + // Parameters: { + // commands: ['echo hello'], + // workingDirectory: 'C:/actions', + // }, + // InstanceIds: ['{{instanceId}}'], + // }, + // nextStep: 'SendTaskSuccess', + // onFailure: 'step:SendTaskFailure_PowerShell', + // onCancel: 'step:SendTaskFailure_PowerShell', + // }, + { + name: 'SendTaskSuccess', + action: 'aws:executeAwsApi', + inputs: { + Service: 'stepfunctions', + Api: 'send_task_success', + taskToken: '{{ taskToken }}', + output: '{}', + }, + timeoutSeconds: 50, + nextStep: 'TerminateInstance', + onFailure: 'step:TerminateInstance', + onCancel: 'step:TerminateInstance', + }, + { + name: 'SendTaskFailure', + action: 'aws:executeAwsApi', + inputs: { + Service: 'stepfunctions', + Api: 'send_task_failure', + taskToken: '{{ taskToken }}', + error: 'Automation document failure', + cause: 'RunCommand failed, check command execution id {{RunCommand.CommandId}} for more details', + }, + timeoutSeconds: 50, + nextStep: 'TerminateInstance', + onFailure: 'step:TerminateInstance', + onCancel: 'step:TerminateInstance', + }, + { + name: 'TerminateInstance', + action: 'aws:executeAwsApi', + inputs: { + Service: 'ec2', + Api: 'terminate_instances', + InstanceIds: ['{{ instanceId }}'], + }, + timeoutSeconds: 50, + isEnd: true, + }, + ], + }, + }); } /** @@ -427,119 +522,77 @@ export class Ec2FleetRunnerProvider extends BaseProvider implements IRunnerProvi * @param parameters workflow job details */ getStepFunctionTask(parameters: RunnerRuntimeParameters): stepfunctions.IChainable { - // we need to build user data in two steps because passing the template as the first parameter to stepfunctions.JsonPath.format fails on syntax - + const stepNamePrefix = this.labels.join(', '); - const passUserData = new stepfunctions.Pass(this, `${this.labels.join(', ')} data`, { + // get ami from builder launch template + const getAmi = new stepfunctions_tasks.CallAwsService(this, `${stepNamePrefix} Get AMI`, { + service: 'ec2', + action: 'describeLaunchTemplateVersions', parameters: { - userdataTemplate: this.ami.os.is(Os.WINDOWS) ? windowsUserDataTemplate : linuxUserDataTemplate, + LaunchTemplateId: this.ami.launchTemplate.launchTemplateId, + Versions: ['$Latest'], + }, + iamResources: ['*'], + resultPath: stepfunctions.JsonPath.stringAt('$.instanceInput'), + resultSelector: { + 'ami.$': '$.LaunchTemplateVersions[0].LaunchTemplateData.ImageId', }, - resultPath: stepfunctions.JsonPath.stringAt('$.ec2'), }); - // we use ec2:RunInstances because we must - // we can't use fleets because they don't let us override user data, security groups or even disk size - // we can't use requestSpotInstances because it doesn't support launch templates, and it's deprecated - // ec2:RunInstances also seemed like the only one to immediately return an error when spot capacity is not available - - // we build a complicated chain of states here because ec2:RunInstances can only try one subnet at a time - // if someone can figure out a good way to use Map for this, please open a PR - - // build a state for each subnet we want to try - const subnetRunners = this.subnets.map((subnet, index) => { - return new stepfunctions_tasks.CallAwsService(this, `${this.labels.join(', ')} subnet${index+1}`, { - comment: subnet.subnetId, - integrationPattern: IntegrationPattern.WAIT_FOR_TASK_TOKEN, - service: 'ec2', - action: 'createFleet', - heartbeatTimeout: stepfunctions.Timeout.duration(Duration.minutes(10)), - // TODO somehow create launch template with security group, profile, user data, tags in imds, etc. - parameters: { - TargetCapacitySpecification: { - TotalTargetCapacity: 1, - // TargetCapacityUnitType: 'units', - DefaultTargetCapacityType: this.spot ? 'spot' : 'on-demand', - }, - LaunchTemplateConfigs: [ - // TODO let user specify these - { - LaunchTemplateSpecification: { - LaunchTemplateId: this.launchTemplate.launchTemplateId, - Version: '$Latest', // TODO latest or default? -- latest for last created by cdk, default for last created by image builder - }, - Overrides: [ - { - SubnetId: subnet.subnetId, - // TODO AMI from image builder launch template ImageId: this.ami.launchTemplate.imageId, - ImageId: 'ami-049d60fa777700d2b', - }, - ], - // TODO Overrides - }, - ], - Type: 'instant', - TagSpecifications: [ - { - ResourceType: 'instance', - Tags: [ - { - Key: 'GitHubRunners:args:taskToken', - // TODO Tag value exceeds the maximum length of 256 characters (Service: Ec2, Status Code: 400, Request ID: a60d7844-a5ae-49d2-aeeb-d36df39cf0fc) - // Value: stepfunctions.JsonPath.taskToken, - Value: 'nope', - }, - { - Key: 'GitHubRunners:args:logGroupName', - Value: this.logGroup.logGroupName, - }, - { - Key: 'GitHubRunners:args:runnerNamePath', - Value: parameters.runnerNamePath, - }, - { - Key: 'GitHubRunners:args:githubDomainPath', - Value: parameters.githubDomainPath, - }, - { - Key: 'GitHubRunners:args:ownerPath', - Value: parameters.ownerPath, - }, - { - Key: 'GitHubRunners:args:repoPath', - Value: parameters.repoPath, - }, - { - Key: 'GitHubRunners:args:runnerTokenPath', - Value: parameters.runnerTokenPath, - }, - { - Key: 'GitHubRunners:args:labels', - Value: this.labels.join(','), - }, - { - Key: 'GitHubRunners:args:registrationURL', - Value: parameters.registrationUrl, - }, - ], - }, - ], + // create fleet with override per subnet + const fleet = new stepfunctions_tasks.CallAwsService(this, `${stepNamePrefix} Fleet`, { + service: 'ec2', + action: 'createFleet', + parameters: { + TargetCapacitySpecification: { + TotalTargetCapacity: 1, + // TargetCapacityUnitType: 'units', + DefaultTargetCapacityType: this.spot ? 'spot' : 'on-demand', }, - iamResources: ['*'], - }); + LaunchTemplateConfigs: [{ + LaunchTemplateSpecification: { + LaunchTemplateId: this.launchTemplate.launchTemplateId, + Version: '$Latest', + }, + Overrides: this.subnets.map(subnet => { + return { + SubnetId: subnet.subnetId, + WeightedCapacity: 1, + ImageId: stepfunctions.JsonPath.stringAt('$.instanceInput.ami'), + }; + }), + }], + Type: 'instant', + }, + iamResources: ['*'], + resultPath: stepfunctions.JsonPath.stringAt('$.instance'), + resultSelector: { + 'id.$': '$.Instances[0].InstanceIds[0]', + }, }); - // start with the first subnet - passUserData.next(subnetRunners[0]); - - // chain up the rest of the subnets - for (let i = 1; i < subnetRunners.length; i++) { - subnetRunners[i-1].addCatch(subnetRunners[i], { - errors: ['Ec2.Ec2Exception', 'States.Timeout'], - resultPath: stepfunctions.JsonPath.stringAt('$.lastSubnetError'), - }); - } + // use ssm to start runner in newly launched instance + const runDocument = new stepfunctions_tasks.CallAwsService(this, `${stepNamePrefix} SSM`, { + // comment: subnet.subnetId, + integrationPattern: stepfunctions.IntegrationPattern.WAIT_FOR_TASK_TOKEN, + service: 'ssm', + action: 'startAutomationExecution', + heartbeatTimeout: stepfunctions.Timeout.duration(Duration.minutes(10)), + parameters: { + DocumentName: this.document.ref, + Parameters: { + instanceId: stepfunctions.JsonPath.array(stepfunctions.JsonPath.stringAt('$.instance.id')), + taskToken: stepfunctions.JsonPath.array(stepfunctions.JsonPath.taskToken), + runnerName: stepfunctions.JsonPath.array(parameters.runnerNamePath), + runnerToken: stepfunctions.JsonPath.array(parameters.runnerTokenPath), + labels: stepfunctions.JsonPath.array(this.labels.join(',')), + registrationUrl: stepfunctions.JsonPath.array(parameters.registrationUrl), + }, + }, + iamResources: ['*'], + }); - return passUserData; + return getAmi.next(fleet).next(runDocument); } grantStateMachine(stateMachineRole: iam.IGrantable) { diff --git a/test/default.integ.ts b/test/default.integ.ts index a5c05d43..a2b4b9ad 100644 --- a/test/default.integ.ts +++ b/test/default.integ.ts @@ -17,6 +17,7 @@ import { Os, RunnerImageComponent, } from '../src'; +import { Ec2FleetRunnerProvider } from '../src/providers/ec2fleet'; const app = new cdk.App(); const stack = new cdk.Stack(app, 'github-runners-test'); @@ -127,6 +128,11 @@ const runners = new GitHubRunners(stack, 'runners', { label: 'codebuild-x64', imageBuilder: codeBuildImageBuilder, }), + new Ec2FleetRunnerProvider(stack, 'Fleet', { + labels: ['ec2fleet', 'linux', 'x64'], + imageBuilder: amiX64Builder, + vpc, + }), new CodeBuildRunnerProvider(stack, 'CodeBuildARM', { labels: ['codebuild', 'linux', 'arm64'], computeType: codebuild.ComputeType.SMALL, From dad49c1098aa34f8c6c1dcdf7f45df71beadbeab Mon Sep 17 00:00:00 2001 From: Amir Szekely Date: Sun, 21 Apr 2024 18:18:01 -0400 Subject: [PATCH 06/11] add windows, replace ec2 provider --- src/providers/ec2.ts | 609 +++++++++++++++++++++-------------- src/providers/ec2fleet.ts | 654 -------------------------------------- src/providers/index.ts | 1 - test/default.integ.ts | 6 - 4 files changed, 364 insertions(+), 906 deletions(-) delete mode 100644 src/providers/ec2fleet.ts diff --git a/src/providers/ec2.ts b/src/providers/ec2.ts index 574a8d39..eb42caa2 100644 --- a/src/providers/ec2.ts +++ b/src/providers/ec2.ts @@ -1,16 +1,15 @@ +// TODO let user specify fleet launch templates? something useful about fleets? + import * as cdk from 'aws-cdk-lib'; import { aws_ec2 as ec2, aws_iam as iam, aws_logs as logs, + aws_ssm as ssm, aws_stepfunctions as stepfunctions, - aws_stepfunctions_tasks as stepfunctions_tasks, - Duration, + aws_stepfunctions_tasks as stepfunctions_tasks, Duration, RemovalPolicy, - Stack, } from 'aws-cdk-lib'; -import { RetentionDays } from 'aws-cdk-lib/aws-logs'; -import { IntegrationPattern } from 'aws-cdk-lib/aws-stepfunctions'; import { Construct } from 'constructs'; import { amiRootDevice, @@ -27,143 +26,6 @@ import { import { IRunnerImageBuilder, RunnerImageBuilder, RunnerImageBuilderProps, RunnerImageBuilderType, RunnerImageComponent } from '../image-builders'; import { MINIMAL_EC2_SSM_SESSION_MANAGER_POLICY_STATEMENT } from '../utils'; -// this script is specifically made so `poweroff` is absolutely always called -// each `{}` is a variable coming from `params` below -const linuxUserDataTemplate = `#!/bin/bash -x -TASK_TOKEN="{}" -logGroupName="{}" -runnerNamePath="{}" -runnerTokenPath="{}" -labels="{}" -registrationURL="{}" - -heartbeat () { - while true; do - aws stepfunctions send-task-heartbeat --task-token "$TASK_TOKEN" - sleep 60 - done -} -setup_logs () { - cat < /tmp/log.conf || exit 1 - { - "logs": { - "log_stream_name": "unknown", - "logs_collected": { - "files": { - "collect_list": [ - { - "file_path": "/var/log/runner.log", - "log_group_name": "$logGroupName", - "log_stream_name": "$runnerNamePath", - "timezone": "UTC" - } - ] - } - } - } - } -EOF - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/tmp/log.conf || exit 2 -} -action () { - # Determine the value of RUNNER_FLAGS - if [ "$(< RUNNER_VERSION)" = "latest" ]; then - RUNNER_FLAGS="" - else - RUNNER_FLAGS="--disableupdate" - fi - - labelsTemplate="$labels,cdkghr:started:$(date +%s)" - - # Execute the configuration command for runner registration - sudo -Hu runner /home/runner/config.sh --unattended --url "$registrationURL" --token "$runnerTokenPath" --ephemeral --work _work --labels "$labelsTemplate" $RUNNER_FLAGS --name "$runnerNamePath" || exit 1 - - # Execute the run command - sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2 - - # Retrieve the status - STATUS=$(grep -Phors "finish job request for job [0-9a-f\\-]+ with result: \K.*" /home/runner/_diag/ | tail -n1) - - # Check and print the job status - [ -n "$STATUS" ] && echo CDKGHA JOB DONE "$labels" "$STATUS" -} -heartbeat & -if setup_logs && action | tee /var/log/runner.log 2>&1; then - aws stepfunctions send-task-success --task-token "$TASK_TOKEN" --task-output '{"ok": true}' -else - aws stepfunctions send-task-failure --task-token "$TASK_TOKEN" -fi -sleep 10 # give cloudwatch agent its default 5 seconds buffer duration to upload logs -poweroff -`.replace(/{/g, '\\{').replace(/}/g, '\\}').replace(/\\{\\}/g, '{}'); - -// this script is specifically made so `poweroff` is absolutely always called -// each `{}` is a variable coming from `params` below and their order should match the linux script -const windowsUserDataTemplate = ` -$TASK_TOKEN = "{}" -$logGroupName="{}" -$runnerNamePath="{}" -$runnerTokenPath="{}" -$labels="{}" -$registrationURL="{}" - -Start-Job -ScriptBlock { - while (1) { - aws stepfunctions send-task-heartbeat --task-token "$using:TASK_TOKEN" - sleep 60 - } -} -function setup_logs () { - echo '{ - "logs": { - "log_stream_name": "unknown", - "logs_collected": { - "files": { - "collect_list": [ - { - "file_path": "/actions/runner.log", - "log_group_name": "$logGroupName", - "log_stream_name": "$runnerNamePath", - "timezone": "UTC" - } - ] - } - } - } - }' | Out-File -Encoding ASCII $Env:TEMP/log.conf - & "C:/Program Files/Amazon/AmazonCloudWatchAgent/amazon-cloudwatch-agent-ctl.ps1" -a fetch-config -m ec2 -s -c file:$Env:TEMP/log.conf -} -function action () { - cd /actions - $RunnerVersion = Get-Content RUNNER_VERSION -Raw - if ($RunnerVersion -eq "latest") { $RunnerFlags = "" } else { $RunnerFlags = "--disableupdate" } - ./config.cmd --unattended --url "\${registrationUrl}" --token "\${runnerTokenPath}" --ephemeral --work _work --labels "\${labels},cdkghr:started:$(Get-Date -UFormat +%s)" $RunnerFlags --name "\${runnerNamePath}" 2>&1 | Out-File -Encoding ASCII -Append /actions/runner.log - - if ($LASTEXITCODE -ne 0) { return 1 } - ./run.cmd 2>&1 | Out-File -Encoding ASCII -Append /actions/runner.log - if ($LASTEXITCODE -ne 0) { return 2 } - - $STATUS = Select-String -Path './_diag/*.log' -Pattern 'finish job request for job [0-9a-f\\-]+ with result: (.*)' | %{$_.Matches.Groups[1].Value} | Select-Object -Last 1 - - if ($STATUS) { - echo "CDKGHA JOB DONE \${labels} $STATUS" | Out-File -Encoding ASCII -Append /actions/runner.log - } - - return 0 - -} -setup_logs -$r = action -if ($r -eq 0) { - aws stepfunctions send-task-success --task-token "$TASK_TOKEN" --task-output '{ }' -} else { - aws stepfunctions send-task-failure --task-token "$TASK_TOKEN" -} -Start-Sleep -Seconds 10 # give cloudwatch agent its default 5 seconds buffer duration to upload logs -Stop-Computer -ComputerName localhost -Force - -`.replace(/{/g, '\\{').replace(/}/g, '\\}').replace(/\\{\\}/g, '{}'); - /** * Properties for {@link Ec2RunnerProvider} construct. @@ -336,10 +198,13 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { private readonly vpc: ec2.IVpc; private readonly subnets: ec2.ISubnet[]; private readonly securityGroups: ec2.ISecurityGroup[]; + private readonly launchTemplate: ec2.LaunchTemplate; + private readonly document: ssm.CfnDocument; constructor(scope: Construct, id: string, props?: Ec2RunnerProviderProps) { super(scope, id, props); + // read parameters this.labels = props?.labels ?? ['ec2']; this.vpc = props?.vpc ?? ec2.Vpc.fromLookup(this, 'Default VPC', { isDefault: true }); this.securityGroups = props?.securityGroup ? [props.securityGroup] : (props?.securityGroups ?? [new ec2.SecurityGroup(this, 'SG', { vpc: this.vpc })]); @@ -349,6 +214,28 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { this.spot = props?.spot ?? false; this.spotMaxPrice = props?.spotMaxPrice; + // instance role + this.grantPrincipal = this.role = new iam.Role(this, 'Role', { + assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'), + inlinePolicies: { + stepfunctions: new iam.PolicyDocument({ + statements: [ + new iam.PolicyStatement({ + actions: ['states:SendTaskHeartbeat'], + resources: ['*'], // no support for stateMachine.stateMachineArn :( + conditions: { + StringEquals: { + 'aws:ResourceTag/aws:cloudformation:stack-id': cdk.Stack.of(this).stackId, + }, + }, + }), + ], + }), + }, + }); + this.grantPrincipal.addToPrincipalPolicy(MINIMAL_EC2_SSM_SESSION_MANAGER_POLICY_STATEMENT); + + // build ami this.amiBuilder = props?.imageBuilder ?? props?.amiBuilder ?? Ec2RunnerProvider.imageBuilder(this, 'Ami Builder', { vpc: props?.vpc, subnetSelection: props?.subnetSelection, @@ -360,29 +247,287 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { throw new Error(`AMI architecture (${this.ami.architecture.name}) doesn't match runner instance type (${this.instanceType} / ${this.instanceType.architecture})`); } - this.grantPrincipal = this.role = new iam.Role(this, 'Role', { - assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'), + // figure out root device + const rootDeviceResource = amiRootDevice(this, this.ami.launchTemplate.launchTemplateId); + rootDeviceResource.node.addDependency(this.amiBuilder); + + // launch template (ami will be added later with override) + this.launchTemplate = new ec2.LaunchTemplate(this, 'Launch Template', { + instanceType: this.instanceType, + role: this.role, + instanceInitiatedShutdownBehavior: ec2.InstanceInitiatedShutdownBehavior.TERMINATE, + requireImdsv2: true, + instanceMetadataTags: true, + blockDevices: [{ + deviceName: rootDeviceResource.ref, + volume: ec2.BlockDeviceVolume.ebs(this.storageSize.toGibibytes(), { + deleteOnTermination: true, + }), + }], + spotOptions: this.spot ? { + requestType: ec2.SpotRequestType.ONE_TIME, + maxPrice: this.spotMaxPrice ? Number(this.spotMaxPrice) : undefined, // TODO prop should be number + } : undefined, + userData: this.generateUserData(), + securityGroup: this.securityGroups.length > 0 ? this.securityGroups[0] : undefined, }); - this.grantPrincipal.addToPrincipalPolicy(new iam.PolicyStatement({ - actions: ['states:SendTaskFailure', 'states:SendTaskSuccess', 'states:SendTaskHeartbeat'], - resources: ['*'], // no support for stateMachine.stateMachineArn :( - conditions: { - StringEquals: { - 'aws:ResourceTag/aws:cloudformation:stack-id': cdk.Stack.of(this).stackId, - }, + this.securityGroups.slice(1).forEach(sg => this.launchTemplate.addSecurityGroup(sg)); + + // role specifically for ssm document + const ssmRole = new iam.Role(this, 'Ssm Role', { + assumedBy: new iam.ServicePrincipal('ssm.amazonaws.com'), + inlinePolicies: { + stepfunctions: new iam.PolicyDocument({ + statements: [ + new iam.PolicyStatement({ + actions: ['states:SendTaskFailure', 'states:SendTaskSuccess'], + resources: ['*'], // no support for stateMachine.stateMachineArn :( + conditions: { + StringEquals: { + 'aws:ResourceTag/aws:cloudformation:stack-id': cdk.Stack.of(this).stackId, + }, + }, + }), + ], + }), + ssm: new iam.PolicyDocument({ + statements: [ + new iam.PolicyStatement({ + actions: ['ssm:SendCommand'], + resources: [ + cdk.Stack.of(this).formatArn({ + service: 'ec2', + resource: 'instance/*', + }), + cdk.Stack.of(this).formatArn({ + service: 'ssm', + account: '', + resource: 'document/AWS-RunShellScript', + }), + cdk.Stack.of(this).formatArn({ + service: 'ssm', + account: '', + resource: 'document/AWS-RunPowerShellScript', + }), + ], + }), + new iam.PolicyStatement({ + actions: ['ssm:DescribeInstanceInformation'], + resources: ['*'], + }), + new iam.PolicyStatement({ + actions: [ + 'ssm:ListCommands', + 'ssm:ListCommandInvocations', + ], + resources: [ + cdk.Stack.of(this).formatArn({ + service: 'ssm', + resource: '*', + }), + ], + }), + ], + }), + ec2: new iam.PolicyDocument({ + statements: [ + new iam.PolicyStatement({ + actions: ['ec2:TerminateInstances'], + resources: ['*'], + conditions: { + StringEquals: { + 'aws:ResourceTag/aws:ec2launchtemplate:id': this.launchTemplate.launchTemplateId, + }, + }, + }), + ], + }), }, - })); - this.grantPrincipal.addToPrincipalPolicy(MINIMAL_EC2_SSM_SESSION_MANAGER_POLICY_STATEMENT); + }); + // log group for runner this.logGroup = new logs.LogGroup( this, 'Logs', { - retention: props?.logRetention ?? RetentionDays.ONE_MONTH, + retention: props?.logRetention ?? logs.RetentionDays.ONE_MONTH, removalPolicy: RemovalPolicy.DESTROY, }, ); - this.logGroup.grantWrite(this); + this.logGroup.grantWrite(this.role); + this.logGroup.grant(this.role, 'logs:CreateLogGroup'); + + // ssm document that starts runner, updates step function, and terminates the instance + this.document = new ssm.CfnDocument(this, 'Automation', { + documentType: 'Automation', + targetType: '/AWS::EC2::Host', + content: { + description: `Run GitHub Runner on EC2 instance for ${this.node.path}`, + schemaVersion: '0.3', + assumeRole: ssmRole.roleArn, + parameters: { + instanceId: { + type: 'String', + description: 'Instance id where runner should be executed', + }, + taskToken: { + type: 'String', + description: 'Step Function task token for callback response', + }, + runnerName: { + type: 'String', + description: 'Runner name', + }, + runnerToken: { + type: 'String', + description: 'Runner token used to register runner on GitHub', + }, + labels: { + type: 'String', + description: 'Labels to assign to runner', + }, + registrationUrl: { + type: 'String', + description: 'Full URL to use for runner registration', + }, + }, + mainSteps: [ + { + name: 'Runner', + action: 'aws:runCommand', + inputs: { + DocumentName: this.ami.os.is(Os.WINDOWS) ? 'AWS-RunPowerShellScript' : 'AWS-RunShellScript', + InstanceIds: ['{{ instanceId }}'], + Parameters: { + // TODO executionTimeout: '0', // no timeout + workingDirectory: this.ami.os.is(Os.WINDOWS) ? 'C:\\actions' : '/home/runner', + commands: this.ami.os.is(Os.WINDOWS) ? [ // *** windows + // tell user data that we started + 'New-Item STARTED', + // send heartbeat + `Start-Job -ScriptBlock { + while (1) { + aws stepfunctions send-task-heartbeat --task-token "{{ taskToken }}" + sleep 60 + } + }`, + // decide if we should update runner + '$RunnerVersion = Get-Content RUNNER_VERSION -Raw', + 'if ($RunnerVersion -eq "latest") { $RunnerFlags = "" } else { $RunnerFlags = "--disableupdate" }', + // configure runner + './config.cmd --unattended --url "{{ registrationUrl }}" --token "{{ runnerToken }}" --ephemeral --work _work --labels "{{ labels }},cdkghr:started:$(Get-Date -UFormat +%s)" $RunnerFlags --name "{{ runnerName }}" 2>&1', + 'if ($LASTEXITCODE -ne 0) { return 1 }', + // start runner + './run.cmd 2>&1', + 'if ($LASTEXITCODE -ne 0) { return 2 }', + // print whether job was successful for our metric filter + `$STATUS = Select-String -Path './_diag/*.log' -Pattern 'finish job request for job [0-9a-f\\-]+ with result: (.*)' | %{$_.Matches.Groups[1].Value} | Select-Object -Last 1 + + if ($STATUS) { + echo "CDKGHA JOB DONE {{ labels }} $STATUS" + }`, + ] : [ // *** linux + 'set -ex', + // tell user data that we started + 'touch /home/runner/STARTED', + // send heartbeat + `{ + while true; do + aws stepfunctions send-task-heartbeat --task-token "{{ taskToken }}" + sleep 60 + done + } &`, + // decide if we should update runner + `if [ "$(cat RUNNER_VERSION)" = "latest" ]; then + RUNNER_FLAGS="" + else + RUNNER_FLAGS="--disableupdate" + fi`, + // configure runner + 'sudo -Hu runner /home/runner/config.sh --unattended --url "{{ registrationUrl }}" --token "{{ runnerToken }}" --ephemeral --work _work --labels "{{ labels }},cdkghr:started:$(date +%s)" $RUNNER_FLAGS --name "{{ runnerName }}" || exit 1', + // start runner without exposing task token and other possibly sensitive environment variables + 'sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2', + // print whether job was successful for our metric filter + `STATUS=$(grep -Phors "finish job request for job [0-9a-f\\-]+ with result: \\K.*" /home/runner/_diag/ | tail -n1) + [ -n "$STATUS" ] && echo CDKGHA JOB DONE "$labels" "$STATUS"`, + ], + }, + CloudWatchOutputConfig: { + CloudWatchLogGroupName: this.logGroup.logGroupName, + CloudWatchOutputEnabled: true, + }, + }, + nextStep: 'SendTaskSuccess', + onFailure: 'step:SendTaskFailure', + onCancel: 'step:SendTaskFailure', + }, + { + name: 'SendTaskSuccess', + action: 'aws:executeAwsApi', + inputs: { + Service: 'stepfunctions', + Api: 'send_task_success', + taskToken: '{{ taskToken }}', + output: '{}', + }, + timeoutSeconds: 50, + nextStep: 'TerminateInstance', + onFailure: 'step:TerminateInstance', + onCancel: 'step:TerminateInstance', + }, + { + name: 'SendTaskFailure', + action: 'aws:executeAwsApi', + inputs: { + Service: 'stepfunctions', + Api: 'send_task_failure', + taskToken: '{{ taskToken }}', + error: 'Automation document failure', + cause: 'Runner failed, check command execution id {{Runner.CommandId}} for more details', + }, + timeoutSeconds: 50, + nextStep: 'TerminateInstance', + onFailure: 'step:TerminateInstance', + onCancel: 'step:TerminateInstance', + }, + { + name: 'TerminateInstance', + action: 'aws:executeAwsApi', + inputs: { + Service: 'ec2', + Api: 'terminate_instances', + InstanceIds: ['{{ instanceId }}'], + }, + timeoutSeconds: 50, + isEnd: true, + }, + ], + }, + }); + } + + private generateUserData() { + if (this.ami.os.is(Os.WINDOWS)) { + const userData = ec2.UserData.forWindows(); + userData.addCommands(`Start-Job -ScriptBlock { + Start-Sleep -Seconds 60 + if (-not (Test-Path C:/actions/STARTED)) { + Write-Output "Runner didn't connect to SSM, powering off" + Stop-Computer -ComputerName localhost -Force + } + }`); + return userData; + } + + const userData = ec2.UserData.forLinux(); + userData.addCommands(`{ + sleep 600 + if [ ! -e /home/runner/STARTED ]; then + echo "Runner didn't connect to SSM, powering off" + sudo poweroff + fi + } &`); + return userData; } /** @@ -393,98 +538,77 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { * @param parameters workflow job details */ getStepFunctionTask(parameters: RunnerRuntimeParameters): stepfunctions.IChainable { - // we need to build user data in two steps because passing the template as the first parameter to stepfunctions.JsonPath.format fails on syntax - - const params = [ - stepfunctions.JsonPath.taskToken, - this.logGroup.logGroupName, - parameters.runnerNamePath, - parameters.runnerTokenPath, - this.labels.join(','), - parameters.registrationUrl, - ]; - - const passUserData = new stepfunctions.Pass(this, `${this.labels.join(', ')} data`, { + const stepNamePrefix = this.labels.join(', '); + + // get ami from builder launch template + const getAmi = new stepfunctions_tasks.CallAwsService(this, `${stepNamePrefix} Get AMI`, { + service: 'ec2', + action: 'describeLaunchTemplateVersions', parameters: { - userdataTemplate: this.ami.os.is(Os.WINDOWS) ? windowsUserDataTemplate : linuxUserDataTemplate, + LaunchTemplateId: this.ami.launchTemplate.launchTemplateId, + Versions: ['$Latest'], + }, + iamResources: ['*'], + resultPath: stepfunctions.JsonPath.stringAt('$.instanceInput'), + resultSelector: { + 'ami.$': '$.LaunchTemplateVersions[0].LaunchTemplateData.ImageId', }, - resultPath: stepfunctions.JsonPath.stringAt('$.ec2'), }); - // we use ec2:RunInstances because we must - // we can't use fleets because they don't let us override user data, security groups or even disk size - // we can't use requestSpotInstances because it doesn't support launch templates, and it's deprecated - // ec2:RunInstances also seemed like the only one to immediately return an error when spot capacity is not available - - // we build a complicated chain of states here because ec2:RunInstances can only try one subnet at a time - // if someone can figure out a good way to use Map for this, please open a PR - - // build a state for each subnet we want to try - const instanceProfile = new iam.CfnInstanceProfile(this, 'Instance Profile', { - roles: [this.role.roleName], - }); - const rootDeviceResource = amiRootDevice(this, this.ami.launchTemplate.launchTemplateId); - rootDeviceResource.node.addDependency(this.amiBuilder); - const subnetRunners = this.subnets.map((subnet, index) => { - return new stepfunctions_tasks.CallAwsService(this, `${this.labels.join(', ')} subnet${index+1}`, { - comment: subnet.subnetId, - integrationPattern: IntegrationPattern.WAIT_FOR_TASK_TOKEN, - service: 'ec2', - action: 'runInstances', - heartbeatTimeout: stepfunctions.Timeout.duration(Duration.minutes(10)), - parameters: { - LaunchTemplate: { - LaunchTemplateId: this.ami.launchTemplate.launchTemplateId, - }, - MinCount: 1, - MaxCount: 1, - InstanceType: this.instanceType.toString(), - UserData: stepfunctions.JsonPath.base64Encode( - stepfunctions.JsonPath.format( - stepfunctions.JsonPath.stringAt('$.ec2.userdataTemplate'), - ...params, - ), - ), - InstanceInitiatedShutdownBehavior: ec2.InstanceInitiatedShutdownBehavior.TERMINATE, - IamInstanceProfile: { - Arn: instanceProfile.attrArn, - }, - MetadataOptions: { - HttpTokens: 'required', - }, - SecurityGroupIds: this.securityGroups.map(sg => sg.securityGroupId), - SubnetId: subnet.subnetId, - BlockDeviceMappings: [{ - DeviceName: rootDeviceResource.ref, - Ebs: { - DeleteOnTermination: true, - VolumeSize: this.storageSize.toGibibytes(), - }, - }], - InstanceMarketOptions: this.spot ? { - MarketType: 'spot', - SpotOptions: { - MaxPrice: this.spotMaxPrice, - SpotInstanceType: 'one-time', - }, - } : undefined, + // create fleet with override per subnet + const fleet = new stepfunctions_tasks.CallAwsService(this, `${stepNamePrefix} Fleet`, { + service: 'ec2', + action: 'createFleet', + parameters: { + TargetCapacitySpecification: { + TotalTargetCapacity: 1, + // TargetCapacityUnitType: 'units', + DefaultTargetCapacityType: this.spot ? 'spot' : 'on-demand', }, - iamResources: ['*'], - }); + LaunchTemplateConfigs: [{ + LaunchTemplateSpecification: { + LaunchTemplateId: this.launchTemplate.launchTemplateId, + Version: '$Latest', + }, + Overrides: this.subnets.map(subnet => { + return { + SubnetId: subnet.subnetId, + WeightedCapacity: 1, + ImageId: stepfunctions.JsonPath.stringAt('$.instanceInput.ami'), + }; + }), + }], + Type: 'instant', + }, + iamResources: ['*'], + resultPath: stepfunctions.JsonPath.stringAt('$.instance'), + resultSelector: { + 'id.$': '$.Instances[0].InstanceIds[0]', + }, }); - // start with the first subnet - passUserData.next(subnetRunners[0]); - - // chain up the rest of the subnets - for (let i = 1; i < subnetRunners.length; i++) { - subnetRunners[i-1].addCatch(subnetRunners[i], { - errors: ['Ec2.Ec2Exception', 'States.Timeout'], - resultPath: stepfunctions.JsonPath.stringAt('$.lastSubnetError'), - }); - } + // use ssm to start runner in newly launched instance + const runDocument = new stepfunctions_tasks.CallAwsService(this, `${stepNamePrefix} SSM`, { + // comment: subnet.subnetId, + integrationPattern: stepfunctions.IntegrationPattern.WAIT_FOR_TASK_TOKEN, + service: 'ssm', + action: 'startAutomationExecution', + heartbeatTimeout: stepfunctions.Timeout.duration(Duration.minutes(10)), + parameters: { + DocumentName: this.document.ref, + Parameters: { + instanceId: stepfunctions.JsonPath.array(stepfunctions.JsonPath.stringAt('$.instance.id')), + taskToken: stepfunctions.JsonPath.array(stepfunctions.JsonPath.taskToken), + runnerName: stepfunctions.JsonPath.array(parameters.runnerNamePath), + runnerToken: stepfunctions.JsonPath.array(parameters.runnerTokenPath), + labels: stepfunctions.JsonPath.array(this.labels.join(',')), + registrationUrl: stepfunctions.JsonPath.array(parameters.registrationUrl), + }, + }, + iamResources: ['*'], + }); - return passUserData; + return getAmi.next(fleet).next(runDocument); } grantStateMachine(stateMachineRole: iam.IGrantable) { @@ -500,7 +624,7 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { stateMachineRole.grantPrincipal.addToPrincipalPolicy(new iam.PolicyStatement({ actions: ['ec2:createTags'], - resources: [Stack.of(this).formatArn({ + resources: [cdk.Stack.of(this).formatArn({ service: 'ec2', resource: '*', })], @@ -544,8 +668,3 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { } } -/** - * @deprecated use {@link Ec2RunnerProvider} - */ -export class Ec2Runner extends Ec2RunnerProvider { -} diff --git a/src/providers/ec2fleet.ts b/src/providers/ec2fleet.ts deleted file mode 100644 index 5955b8be..00000000 --- a/src/providers/ec2fleet.ts +++ /dev/null @@ -1,654 +0,0 @@ -// TODO do we really need a separate ec2 and fleet provider? -// TODO let user specify fleet launch templates? something useful - -import * as cdk from 'aws-cdk-lib'; -import { - aws_ec2 as ec2, - aws_iam as iam, - aws_logs as logs, - aws_ssm as ssm, - aws_stepfunctions as stepfunctions, - aws_stepfunctions_tasks as stepfunctions_tasks, Duration, - RemovalPolicy, - Stack, -} from 'aws-cdk-lib'; -import { Construct } from 'constructs'; -import { - amiRootDevice, - Architecture, - BaseProvider, - IRunnerProvider, - IRunnerProviderStatus, - Os, - RunnerAmi, - RunnerProviderProps, - RunnerRuntimeParameters, - RunnerVersion, -} from './common'; -import { IRunnerImageBuilder, RunnerImageBuilder, RunnerImageBuilderProps, RunnerImageBuilderType, RunnerImageComponent } from '../image-builders'; -import { MINIMAL_EC2_SSM_SESSION_MANAGER_POLICY_STATEMENT } from '../utils'; - - -/** - * Properties for {@link Ec2RunnerProvider} construct. - */ -export interface Ec2FleetRunnerProviderProps extends RunnerProviderProps { - /** - * Runner image builder used to build AMI containing GitHub Runner and all requirements. - * - * The image builder determines the OS and architecture of the runner. - * - * @default Ec2RunnerProvider.imageBuilder() - */ - readonly imageBuilder?: IRunnerImageBuilder; - - /** - * @deprecated use imageBuilder - */ - readonly amiBuilder?: IRunnerImageBuilder; - - /** - * GitHub Actions labels used for this provider. - * - * These labels are used to identify which provider should spawn a new on-demand runner. Every job sends a webhook with the labels it's looking for - * based on runs-on. We match the labels from the webhook with the labels specified here. If all the labels specified here are present in the - * job's labels, this provider will be chosen and spawn a new runner. - * - * @default ['ec2'] - */ - readonly labels?: string[]; - - /** - * Instance type for launched runner instances. - * - * @default m5.large - */ - readonly instanceType?: ec2.InstanceType; - - /** - * Size of volume available for launched runner instances. This modifies the boot volume size and doesn't add any additional volumes. - * - * @default 30GB - */ - readonly storageSize?: cdk.Size; - - /** - * Security Group to assign to launched runner instances. - * - * @default a new security group - * - * @deprecated use {@link securityGroups} - */ - readonly securityGroup?: ec2.ISecurityGroup; - - /** - * Security groups to assign to launched runner instances. - * - * @default a new security group - */ - readonly securityGroups?: ec2.ISecurityGroup[]; - - /** - * Subnet where the runner instances will be launched. - * - * @default default subnet of account's default VPC - * - * @deprecated use {@link vpc} and {@link subnetSelection} - */ - readonly subnet?: ec2.ISubnet; - - /** - * VPC where runner instances will be launched. - * - * @default default account VPC - */ - readonly vpc?: ec2.IVpc; - - /** - * Where to place the network interfaces within the VPC. Only the first matched subnet will be used. - * - * @default default VPC subnet - */ - readonly subnetSelection?: ec2.SubnetSelection; - - /** - * Use spot instances to save money. Spot instances are cheaper but not always available and can be stopped prematurely. - * - * @default false - */ - readonly spot?: boolean; - - /** - * Set a maximum price for spot instances. - * - * @default no max price (you will pay current spot price) - */ - readonly spotMaxPrice?: string; -} - -/** - * GitHub Actions runner provider using EC2 to execute jobs. - * - * This construct is not meant to be used by itself. It should be passed in the providers property for GitHubRunners. - */ -export class Ec2FleetRunnerProvider extends BaseProvider implements IRunnerProvider { - /** - * Create new image builder that builds EC2 specific runner images. - * - * You can customize the OS, architecture, VPC, subnet, security groups, etc. by passing in props. - * - * You can add components to the image builder by calling `imageBuilder.addComponent()`. - * - * The default OS is Ubuntu running on x64 architecture. - * - * Included components: - * * `RunnerImageComponent.requiredPackages()` - * * `RunnerImageComponent.runnerUser()` - * * `RunnerImageComponent.git()` - * * `RunnerImageComponent.githubCli()` - * * `RunnerImageComponent.awsCli()` - * * `RunnerImageComponent.docker()` - * * `RunnerImageComponent.githubRunner()` - */ - public static imageBuilder(scope: Construct, id: string, props?: RunnerImageBuilderProps) { - return RunnerImageBuilder.new(scope, id, { - os: Os.LINUX_UBUNTU, - architecture: Architecture.X86_64, - builderType: RunnerImageBuilderType.AWS_IMAGE_BUILDER, - components: [ - RunnerImageComponent.requiredPackages(), - RunnerImageComponent.runnerUser(), - RunnerImageComponent.git(), - RunnerImageComponent.githubCli(), - RunnerImageComponent.awsCli(), - RunnerImageComponent.docker(), - RunnerImageComponent.githubRunner(props?.runnerVersion ?? RunnerVersion.latest()), - ], - ...props, - }); - } - - /** - * Labels associated with this provider. - */ - readonly labels: string[]; - - /** - * Grant principal used to add permissions to the runner role. - */ - readonly grantPrincipal: iam.IPrincipal; - - /** - * Log group where provided runners will save their logs. - * - * Note that this is not the job log, but the runner itself. It will not contain output from the GitHub Action but only metadata on its execution. - */ - readonly logGroup: logs.ILogGroup; - - readonly retryableErrors = [ - 'Ec2.Ec2Exception', - 'States.Timeout', - ]; - - private readonly amiBuilder: IRunnerImageBuilder; - private readonly ami: RunnerAmi; - private readonly role: iam.Role; - private readonly instanceType: ec2.InstanceType; - private readonly storageSize: cdk.Size; - private readonly spot: boolean; - private readonly spotMaxPrice: string | undefined; - private readonly vpc: ec2.IVpc; - private readonly subnets: ec2.ISubnet[]; - private readonly securityGroups: ec2.ISecurityGroup[]; - private readonly launchTemplate: ec2.LaunchTemplate; - private readonly document: ssm.CfnDocument; - - constructor(scope: Construct, id: string, props?: Ec2FleetRunnerProviderProps) { - super(scope, id, props); - - // read parameters - this.labels = props?.labels ?? ['ec2']; - this.vpc = props?.vpc ?? ec2.Vpc.fromLookup(this, 'Default VPC', { isDefault: true }); - this.securityGroups = props?.securityGroup ? [props.securityGroup] : (props?.securityGroups ?? [new ec2.SecurityGroup(this, 'SG', { vpc: this.vpc })]); - this.subnets = props?.subnet ? [props.subnet] : this.vpc.selectSubnets(props?.subnetSelection).subnets; - this.instanceType = props?.instanceType ?? ec2.InstanceType.of(ec2.InstanceClass.M5, ec2.InstanceSize.LARGE); - this.storageSize = props?.storageSize ?? cdk.Size.gibibytes(30); // 30 is the minimum for Windows - this.spot = props?.spot ?? false; - this.spotMaxPrice = props?.spotMaxPrice; - - // instance role - this.grantPrincipal = this.role = new iam.Role(this, 'Role', { - assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'), - inlinePolicies: { - stepfunctions: new iam.PolicyDocument({ - statements: [ - new iam.PolicyStatement({ - actions: ['states:SendTaskHeartbeat'], - resources: ['*'], // no support for stateMachine.stateMachineArn :( - conditions: { - StringEquals: { - 'aws:ResourceTag/aws:cloudformation:stack-id': cdk.Stack.of(this).stackId, - }, - }, - }), - ], - }), - }, - }); - this.grantPrincipal.addToPrincipalPolicy(MINIMAL_EC2_SSM_SESSION_MANAGER_POLICY_STATEMENT); - - // build ami - this.amiBuilder = props?.imageBuilder ?? props?.amiBuilder ?? Ec2FleetRunnerProvider.imageBuilder(this, 'Ami Builder', { - vpc: props?.vpc, - subnetSelection: props?.subnetSelection, - securityGroups: this.securityGroups, - }); - this.ami = this.amiBuilder.bindAmi(); - - if (!this.ami.architecture.instanceTypeMatch(this.instanceType)) { - throw new Error(`AMI architecture (${this.ami.architecture.name}) doesn't match runner instance type (${this.instanceType} / ${this.instanceType.architecture})`); - } - - // figure out root device - const rootDeviceResource = amiRootDevice(this, this.ami.launchTemplate.launchTemplateId); - rootDeviceResource.node.addDependency(this.amiBuilder); - - // launch template (ami will be added later with override) - this.launchTemplate = new ec2.LaunchTemplate(this, 'Launch Template', { - instanceType: this.instanceType, - role: this.role, - instanceInitiatedShutdownBehavior: ec2.InstanceInitiatedShutdownBehavior.TERMINATE, - requireImdsv2: true, - instanceMetadataTags: true, - blockDevices: [{ - deviceName: rootDeviceResource.ref, - volume: ec2.BlockDeviceVolume.ebs(this.storageSize.toGibibytes(), { - deleteOnTermination: true, - }), - }], - spotOptions: this.spot ? { - requestType: ec2.SpotRequestType.ONE_TIME, - maxPrice: this.spotMaxPrice ? Number(this.spotMaxPrice) : undefined, // TODO prop should be number - } : undefined, - userData: ec2.UserData.forLinux(), - }); - this.launchTemplate.userData!.addCommands(`{ - sleep 600 - if [ ! -e /home/runner/STARTED ]; then - echo "Runner didn't connect to SSM, powering off" - sudo poweroff - fi - } &`); - - // role specifically for ssm document - const ssmRole = new iam.Role(this, 'Ssm Role', { - assumedBy: new iam.ServicePrincipal('ssm.amazonaws.com'), - inlinePolicies: { - stepfunctions: new iam.PolicyDocument({ - statements: [ - new iam.PolicyStatement({ - actions: ['states:SendTaskFailure', 'states:SendTaskSuccess'], - resources: ['*'], // no support for stateMachine.stateMachineArn :( - conditions: { - StringEquals: { - 'aws:ResourceTag/aws:cloudformation:stack-id': cdk.Stack.of(this).stackId, - }, - }, - }), - ], - }), - ssm: new iam.PolicyDocument({ - statements: [ - new iam.PolicyStatement({ - actions: ['ssm:SendCommand'], - resources: [ - '*', - // { - // 'Fn::Sub': 'arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:instance/*', - // }, - // { - // 'Fn::Sub': 'arn:aws:ssm:${AWS::Region}::document/AWS-RunShellScript', - // }, - // { - // 'Fn::Sub': 'arn:aws:ssm:${AWS::Region}::document/AWS-RunPowerShellScript', - // }, - ], - }), - new iam.PolicyStatement({ - actions: ['ssm:DescribeInstanceInformation'], - resources: ['*'], - }), - new iam.PolicyStatement({ - actions: [ - 'ssm:ListCommands', - 'ssm:ListCommandInvocations', - ], - resources: [ - '*', - // { - // 'Fn::Sub': 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:*', - // }, - ], - }), - ], - }), - ec2: new iam.PolicyDocument({ - statements: [ - new iam.PolicyStatement({ - actions: ['ec2:TerminateInstances'], - resources: ['*'], - conditions: { - StringEquals: { - 'aws:ResourceTag/aws:ec2launchtemplate:id': this.launchTemplate.launchTemplateId, - }, - }, - }), - ], - }), - }, - }); - - // log group for runner - this.logGroup = new logs.LogGroup( - this, - 'Logs', - { - retention: props?.logRetention ?? logs.RetentionDays.ONE_MONTH, - removalPolicy: RemovalPolicy.DESTROY, - }, - ); - this.logGroup.grantWrite(this.role); - this.logGroup.grant(this.role, 'logs:CreateLogGroup'); - - // ssm document that starts runner, updates step function, and terminates the instance - this.document = new ssm.CfnDocument(this, 'Automation', { - // name: 'SfnRunCommandByTargets', - documentType: 'Automation', - targetType: '/AWS::EC2::Host', - content: { - description: 'TODO github runners', - schemaVersion: '0.3', - assumeRole: ssmRole.roleArn, - parameters: { - instanceId: { - type: 'String', - description: 'Instance id where runner should be executed', - }, - taskToken: { - type: 'String', - description: 'Step Function task token for callback response', - }, - runnerName: { - type: 'String', - description: 'Runner name', - }, - runnerToken: { - type: 'String', - description: 'Runner token used to register runner on GitHub', - }, - labels: { - type: 'String', - description: 'Labels to assign to runner', - }, - registrationUrl: { - type: 'String', - description: 'Full URL to use for runner registration', - }, - }, - mainSteps: [ - // { - // name: 'Branch', - // action: 'aws:branch', - // inputs: { - // Choices: [ - // { - // NextStep: 'RunCommand_Powershell', - // Variable: '{{ shell }}', - // StringEquals: 'PowerShell', - // }, - // ], - // Default: 'RunCommand_Shell', - // }, - // }, - { - name: 'RunCommand', - action: 'aws:runCommand', - inputs: { - DocumentName: 'AWS-RunShellScript', - InstanceIds: ['{{ instanceId }}'], - // TODO no execution timeout - Parameters: { - commands: [ - 'set -ex', - // tell user data that we started - 'touch /home/runner/STARTED', - // send heartbeat - `{ - while true; do - aws stepfunctions send-task-heartbeat --task-token "{{ taskToken }}" - sleep 60 - done - } &`, - // decide if we should update runner - `if [ "$(cat RUNNER_VERSION)" = "latest" ]; then - RUNNER_FLAGS="" - else - RUNNER_FLAGS="--disableupdate" - fi`, - // configure runner - 'sudo -Hu runner /home/runner/config.sh --unattended --url "{{ registrationUrl }}" --token "{{ runnerToken }}" --ephemeral --work _work --labels "{{ labels }},cdkghr:started:$(date +%s)" $RUNNER_FLAGS --name "{{ runnerName }}" || exit 1', - // start runner without exposing task token and other possibly sensitive environment variables - 'sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2', - // print whether job was successful for our metric filter - `STATUS=$(grep -Phors "finish job request for job [0-9a-f\\-]+ with result: \\K.*" /home/runner/_diag/ | tail -n1) - [ -n "$STATUS" ] && echo CDKGHA JOB DONE "$labels" "$STATUS"`, - ], - workingDirectory: '/home/runner', - }, - CloudWatchOutputConfig: { - CloudWatchLogGroupName: this.logGroup.logGroupName, - CloudWatchOutputEnabled: true, - }, - }, - nextStep: 'SendTaskSuccess', - onFailure: 'step:SendTaskFailure', - onCancel: 'step:SendTaskFailure', - }, - // { - // name: 'RunCommand_Powershell', - // action: 'aws:runCommand', - // inputs: { - // DocumentName: 'AWS-RunPowerShellScript', - // Parameters: { - // commands: ['echo hello'], - // workingDirectory: 'C:/actions', - // }, - // InstanceIds: ['{{instanceId}}'], - // }, - // nextStep: 'SendTaskSuccess', - // onFailure: 'step:SendTaskFailure_PowerShell', - // onCancel: 'step:SendTaskFailure_PowerShell', - // }, - { - name: 'SendTaskSuccess', - action: 'aws:executeAwsApi', - inputs: { - Service: 'stepfunctions', - Api: 'send_task_success', - taskToken: '{{ taskToken }}', - output: '{}', - }, - timeoutSeconds: 50, - nextStep: 'TerminateInstance', - onFailure: 'step:TerminateInstance', - onCancel: 'step:TerminateInstance', - }, - { - name: 'SendTaskFailure', - action: 'aws:executeAwsApi', - inputs: { - Service: 'stepfunctions', - Api: 'send_task_failure', - taskToken: '{{ taskToken }}', - error: 'Automation document failure', - cause: 'RunCommand failed, check command execution id {{RunCommand.CommandId}} for more details', - }, - timeoutSeconds: 50, - nextStep: 'TerminateInstance', - onFailure: 'step:TerminateInstance', - onCancel: 'step:TerminateInstance', - }, - { - name: 'TerminateInstance', - action: 'aws:executeAwsApi', - inputs: { - Service: 'ec2', - Api: 'terminate_instances', - InstanceIds: ['{{ instanceId }}'], - }, - timeoutSeconds: 50, - isEnd: true, - }, - ], - }, - }); - } - - /** - * Generate step function task(s) to start a new runner. - * - * Called by GithubRunners and shouldn't be called manually. - * - * @param parameters workflow job details - */ - getStepFunctionTask(parameters: RunnerRuntimeParameters): stepfunctions.IChainable { - const stepNamePrefix = this.labels.join(', '); - - // get ami from builder launch template - const getAmi = new stepfunctions_tasks.CallAwsService(this, `${stepNamePrefix} Get AMI`, { - service: 'ec2', - action: 'describeLaunchTemplateVersions', - parameters: { - LaunchTemplateId: this.ami.launchTemplate.launchTemplateId, - Versions: ['$Latest'], - }, - iamResources: ['*'], - resultPath: stepfunctions.JsonPath.stringAt('$.instanceInput'), - resultSelector: { - 'ami.$': '$.LaunchTemplateVersions[0].LaunchTemplateData.ImageId', - }, - }); - - // create fleet with override per subnet - const fleet = new stepfunctions_tasks.CallAwsService(this, `${stepNamePrefix} Fleet`, { - service: 'ec2', - action: 'createFleet', - parameters: { - TargetCapacitySpecification: { - TotalTargetCapacity: 1, - // TargetCapacityUnitType: 'units', - DefaultTargetCapacityType: this.spot ? 'spot' : 'on-demand', - }, - LaunchTemplateConfigs: [{ - LaunchTemplateSpecification: { - LaunchTemplateId: this.launchTemplate.launchTemplateId, - Version: '$Latest', - }, - Overrides: this.subnets.map(subnet => { - return { - SubnetId: subnet.subnetId, - WeightedCapacity: 1, - ImageId: stepfunctions.JsonPath.stringAt('$.instanceInput.ami'), - }; - }), - }], - Type: 'instant', - }, - iamResources: ['*'], - resultPath: stepfunctions.JsonPath.stringAt('$.instance'), - resultSelector: { - 'id.$': '$.Instances[0].InstanceIds[0]', - }, - }); - - // use ssm to start runner in newly launched instance - const runDocument = new stepfunctions_tasks.CallAwsService(this, `${stepNamePrefix} SSM`, { - // comment: subnet.subnetId, - integrationPattern: stepfunctions.IntegrationPattern.WAIT_FOR_TASK_TOKEN, - service: 'ssm', - action: 'startAutomationExecution', - heartbeatTimeout: stepfunctions.Timeout.duration(Duration.minutes(10)), - parameters: { - DocumentName: this.document.ref, - Parameters: { - instanceId: stepfunctions.JsonPath.array(stepfunctions.JsonPath.stringAt('$.instance.id')), - taskToken: stepfunctions.JsonPath.array(stepfunctions.JsonPath.taskToken), - runnerName: stepfunctions.JsonPath.array(parameters.runnerNamePath), - runnerToken: stepfunctions.JsonPath.array(parameters.runnerTokenPath), - labels: stepfunctions.JsonPath.array(this.labels.join(',')), - registrationUrl: stepfunctions.JsonPath.array(parameters.registrationUrl), - }, - }, - iamResources: ['*'], - }); - - return getAmi.next(fleet).next(runDocument); - } - - grantStateMachine(stateMachineRole: iam.IGrantable) { - stateMachineRole.grantPrincipal.addToPrincipalPolicy(new iam.PolicyStatement({ - actions: ['iam:PassRole'], - resources: [this.role.roleArn], - conditions: { - StringEquals: { - 'iam:PassedToService': 'ec2.amazonaws.com', - }, - }, - })); - - stateMachineRole.grantPrincipal.addToPrincipalPolicy(new iam.PolicyStatement({ - actions: ['ec2:createTags'], - resources: [Stack.of(this).formatArn({ - service: 'ec2', - resource: '*', - })], - })); - - stateMachineRole.grantPrincipal.addToPrincipalPolicy(new iam.PolicyStatement({ - actions: ['iam:CreateServiceLinkedRole'], - resources: ['*'], - conditions: { - StringEquals: { - 'iam:AWSServiceName': 'spot.amazonaws.com', - }, - }, - })); - } - - status(statusFunctionRole: iam.IGrantable): IRunnerProviderStatus { - statusFunctionRole.grantPrincipal.addToPrincipalPolicy(new iam.PolicyStatement({ - actions: ['ec2:DescribeLaunchTemplateVersions'], - resources: ['*'], - })); - - return { - type: this.constructor.name, - labels: this.labels, - securityGroups: this.securityGroups.map(sg => sg.securityGroupId), - roleArn: this.role.roleArn, - logGroup: this.logGroup.logGroupName, - ami: { - launchTemplate: this.ami.launchTemplate.launchTemplateId || 'unknown', - amiBuilderLogGroup: this.ami.logGroup?.logGroupName, - }, - }; - } - - /** - * The network connections associated with this resource. - */ - public get connections(): ec2.Connections { - return new ec2.Connections({ securityGroups: this.securityGroups }); - } -} - diff --git a/src/providers/index.ts b/src/providers/index.ts index 0d5e8efc..5bf48d59 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -1,5 +1,4 @@ export * from './ec2'; -export * from './ec2fleet'; export * from './ecs'; export * from './codebuild'; export * from './lambda'; diff --git a/test/default.integ.ts b/test/default.integ.ts index a2b4b9ad..a5c05d43 100644 --- a/test/default.integ.ts +++ b/test/default.integ.ts @@ -17,7 +17,6 @@ import { Os, RunnerImageComponent, } from '../src'; -import { Ec2FleetRunnerProvider } from '../src/providers/ec2fleet'; const app = new cdk.App(); const stack = new cdk.Stack(app, 'github-runners-test'); @@ -128,11 +127,6 @@ const runners = new GitHubRunners(stack, 'runners', { label: 'codebuild-x64', imageBuilder: codeBuildImageBuilder, }), - new Ec2FleetRunnerProvider(stack, 'Fleet', { - labels: ['ec2fleet', 'linux', 'x64'], - imageBuilder: amiX64Builder, - vpc, - }), new CodeBuildRunnerProvider(stack, 'CodeBuildARM', { labels: ['codebuild', 'linux', 'arm64'], computeType: codebuild.ComputeType.SMALL, From 8fe355b0e0579fc7a3c2ce71049f88d45deb459a Mon Sep 17 00:00:00 2001 From: Amir Szekely Date: Sun, 21 Apr 2024 18:20:20 -0400 Subject: [PATCH 07/11] update snapshot --- .../github-runners-test.assets.json | 28 +- .../github-runners-test.template.json | 2916 ++++++++++++----- 2 files changed, 2137 insertions(+), 807 deletions(-) diff --git a/test/default.integ.snapshot/github-runners-test.assets.json b/test/default.integ.snapshot/github-runners-test.assets.json index 24fb3502..cd88a6c2 100644 --- a/test/default.integ.snapshot/github-runners-test.assets.json +++ b/test/default.integ.snapshot/github-runners-test.assets.json @@ -131,54 +131,54 @@ } } }, - "e4ac7e9be9abde086a27b8b2eb2e4b59dde9a52c2cbdef3266d67ceafaf4dafe": { + "97a52dd61d3f204877c7a944f14423bdbdec7587794ab63213506581ffb19976": { "source": { - "path": "asset.e4ac7e9be9abde086a27b8b2eb2e4b59dde9a52c2cbdef3266d67ceafaf4dafe.lambda", + "path": "asset.97a52dd61d3f204877c7a944f14423bdbdec7587794ab63213506581ffb19976.lambda", "packaging": "zip" }, "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "e4ac7e9be9abde086a27b8b2eb2e4b59dde9a52c2cbdef3266d67ceafaf4dafe.zip", + "objectKey": "97a52dd61d3f204877c7a944f14423bdbdec7587794ab63213506581ffb19976.zip", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } }, - "16bb1d322fead6e35092941eced41b07006f6ef31dbe6283dbaad26cffb70fa8": { + "e4ac7e9be9abde086a27b8b2eb2e4b59dde9a52c2cbdef3266d67ceafaf4dafe": { "source": { - "path": "asset.16bb1d322fead6e35092941eced41b07006f6ef31dbe6283dbaad26cffb70fa8.lambda", + "path": "asset.e4ac7e9be9abde086a27b8b2eb2e4b59dde9a52c2cbdef3266d67ceafaf4dafe.lambda", "packaging": "zip" }, "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "16bb1d322fead6e35092941eced41b07006f6ef31dbe6283dbaad26cffb70fa8.zip", + "objectKey": "e4ac7e9be9abde086a27b8b2eb2e4b59dde9a52c2cbdef3266d67ceafaf4dafe.zip", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } }, - "dfe33c6b4de9a62d153ad3025abe39cb85d5fef9d1a6e4b54e79b96cdbfc64ea": { + "16bb1d322fead6e35092941eced41b07006f6ef31dbe6283dbaad26cffb70fa8": { "source": { - "path": "asset.dfe33c6b4de9a62d153ad3025abe39cb85d5fef9d1a6e4b54e79b96cdbfc64ea.lambda", + "path": "asset.16bb1d322fead6e35092941eced41b07006f6ef31dbe6283dbaad26cffb70fa8.lambda", "packaging": "zip" }, "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "dfe33c6b4de9a62d153ad3025abe39cb85d5fef9d1a6e4b54e79b96cdbfc64ea.zip", + "objectKey": "16bb1d322fead6e35092941eced41b07006f6ef31dbe6283dbaad26cffb70fa8.zip", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } }, - "97a52dd61d3f204877c7a944f14423bdbdec7587794ab63213506581ffb19976": { + "dfe33c6b4de9a62d153ad3025abe39cb85d5fef9d1a6e4b54e79b96cdbfc64ea": { "source": { - "path": "asset.97a52dd61d3f204877c7a944f14423bdbdec7587794ab63213506581ffb19976.lambda", + "path": "asset.dfe33c6b4de9a62d153ad3025abe39cb85d5fef9d1a6e4b54e79b96cdbfc64ea.lambda", "packaging": "zip" }, "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "97a52dd61d3f204877c7a944f14423bdbdec7587794ab63213506581ffb19976.zip", + "objectKey": "dfe33c6b4de9a62d153ad3025abe39cb85d5fef9d1a6e4b54e79b96cdbfc64ea.zip", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } @@ -235,7 +235,7 @@ } } }, - "e0a2b1d424418d59ce339e3fc6ec13297efe609e1c74984a369f871a521ac8ad": { + "4b0524dac32ebcde2edd5ab38a5c92126a5e899e826a24f533713795a45390a3": { "source": { "path": "github-runners-test.template.json", "packaging": "file" @@ -243,7 +243,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "e0a2b1d424418d59ce339e3fc6ec13297efe609e1c74984a369f871a521ac8ad.json", + "objectKey": "4b0524dac32ebcde2edd5ab38a5c92126a5e899e826a24f533713795a45390a3.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/test/default.integ.snapshot/github-runners-test.template.json b/test/default.integ.snapshot/github-runners-test.template.json index b01c21fd..f854332d 100644 --- a/test/default.integ.snapshot/github-runners-test.template.json +++ b/test/default.integ.snapshot/github-runners-test.template.json @@ -13383,7 +13383,29 @@ } ], "Version": "2012-10-17" - } + }, + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": "states:SendTaskHeartbeat", + "Condition": { + "StringEquals": { + "aws:ResourceTag/aws:cloudformation:stack-id": { + "Ref": "AWS::StackId" + } + } + }, + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "stepfunctions" + } + ] } }, "EC2LinuxRoleDefaultPolicy1369791B": { @@ -13391,22 +13413,6 @@ "Properties": { "PolicyDocument": { "Statement": [ - { - "Action": [ - "states:SendTaskFailure", - "states:SendTaskSuccess", - "states:SendTaskHeartbeat" - ], - "Condition": { - "StringEquals": { - "aws:ResourceTag/aws:cloudformation:stack-id": { - "Ref": "AWS::StackId" - } - } - }, - "Effect": "Allow", - "Resource": "*" - }, { "Action": [ "ssmmessages:CreateControlChannel", @@ -13431,6 +13437,16 @@ "Arn" ] } + }, + { + "Action": "logs:CreateLogGroup", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "EC2LinuxLogsC4CD8F14", + "Arn" + ] + } } ], "Version": "2012-10-17" @@ -13443,51 +13459,6 @@ ] } }, - "EC2LinuxLogsC4CD8F14": { - "Type": "AWS::Logs::LogGroup", - "Properties": { - "RetentionInDays": 30 - }, - "UpdateReplacePolicy": "Delete", - "DeletionPolicy": "Delete" - }, - "EC2LinuxLogsLogsfilter186C2AB7": { - "Type": "AWS::Logs::MetricFilter", - "Properties": { - "FilterPattern": "[..., marker = \"CDKGHA\", job = \"JOB\", done = \"DONE\", labels, status = \"Succeeded\" || status = \"SucceededWithIssues\" || status = \"Failed\" || status = \"Canceled\" || status = \"Skipped\" || status = \"Abandoned\"]", - "LogGroupName": { - "Ref": "EC2LinuxLogsC4CD8F14" - }, - "MetricTransformations": [ - { - "Dimensions": [ - { - "Key": "ProviderLabels", - "Value": "$labels" - }, - { - "Key": "Status", - "Value": "$status" - } - ], - "MetricName": "JobCompleted", - "MetricNamespace": "GitHubRunners", - "MetricValue": "1", - "Unit": "Count" - } - ] - } - }, - "EC2LinuxInstanceProfile2D2BB473": { - "Type": "AWS::IAM::InstanceProfile", - "Properties": { - "Roles": [ - { - "Ref": "EC2LinuxRole8B6519A2" - } - ] - } - }, "EC2LinuxAMIRootDevice26D5E56E": { "Type": "Custom::AmiRootDevice", "Properties": { @@ -13537,141 +13508,95 @@ "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" }, - "deleteamidcc036c8876b451ea2c1552f9e06e9e1ServiceRole1CC58A6F": { - "Type": "AWS::IAM::Role", + "EC2LinuxLaunchTemplateProfile1127F1E8": { + "Type": "AWS::IAM::InstanceProfile", "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "lambda.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "ManagedPolicyArns": [ + "Roles": [ { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - ] - ] + "Ref": "EC2LinuxRole8B6519A2" } ] } }, - "deleteamidcc036c8876b451ea2c1552f9e06e9e1ServiceRoleDefaultPolicy0C44BF83": { - "Type": "AWS::IAM::Policy", + "EC2LinuxLaunchTemplate36CD9B92": { + "Type": "AWS::EC2::LaunchTemplate", "Properties": { - "PolicyDocument": { - "Statement": [ + "LaunchTemplateData": { + "BlockDeviceMappings": [ { - "Action": "ec2:DescribeImages", - "Effect": "Allow", - "Resource": "*" + "DeviceName": { + "Ref": "EC2LinuxAMIRootDevice26D5E56E" + }, + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 30 + } + } + ], + "IamInstanceProfile": { + "Arn": { + "Fn::GetAtt": [ + "EC2LinuxLaunchTemplateProfile1127F1E8", + "Arn" + ] + } + }, + "InstanceInitiatedShutdownBehavior": "terminate", + "InstanceType": "m5.large", + "MetadataOptions": { + "HttpTokens": "required", + "InstanceMetadataTags": "enabled" + }, + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "EC2LinuxSGF5B89300", + "GroupId" + ] + } + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "Name", + "Value": "github-runners-test/EC2 Linux/Launch Template" + } + ] }, { - "Action": [ - "ec2:DeregisterImage", - "ec2:DeleteSnapshot" - ], - "Condition": { - "StringEquals": { - "aws:ResourceTag/GitHubRunners:Stack": "github-runners-test" + "ResourceType": "volume", + "Tags": [ + { + "Key": "Name", + "Value": "github-runners-test/EC2 Linux/Launch Template" } - }, - "Effect": "Allow", - "Resource": "*" + ] } ], - "Version": "2012-10-17" + "UserData": { + "Fn::Base64": "#!/bin/bash\n{\n sleep 600\n if [ ! -e /home/runner/STARTED ]; then\n echo \"Runner didn't connect to SSM, powering off\"\n sudo poweroff\n fi\n } &" + } }, - "PolicyName": "deleteamidcc036c8876b451ea2c1552f9e06e9e1ServiceRoleDefaultPolicy0C44BF83", - "Roles": [ + "TagSpecifications": [ { - "Ref": "deleteamidcc036c8876b451ea2c1552f9e06e9e1ServiceRole1CC58A6F" + "ResourceType": "launch-template", + "Tags": [ + { + "Key": "Name", + "Value": "github-runners-test/EC2 Linux/Launch Template" + } + ] } ] - } - }, - "deleteamidcc036c8876b451ea2c1552f9e06e9e1BE713303": { - "Type": "AWS::Lambda::Function", - "Properties": { - "Code": { - "S3Bucket": { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "S3Key": "b1862470ac31c934803dff83368cf5cdec42c544fd77a6e8fc25d6d7e55befa4.zip" - }, - "Description": "Delete old GitHub Runner AMIs", - "Environment": { - "Variables": { - "AWS_NODEJS_CONNECTION_REUSE_ENABLED": "1" - } - }, - "Handler": "index.handler", - "Role": { - "Fn::GetAtt": [ - "deleteamidcc036c8876b451ea2c1552f9e06e9e1ServiceRole1CC58A6F", - "Arn" - ] - }, - "Runtime": "nodejs18.x", - "Timeout": 300 }, "DependsOn": [ - "deleteamidcc036c8876b451ea2c1552f9e06e9e1ServiceRoleDefaultPolicy0C44BF83", - "deleteamidcc036c8876b451ea2c1552f9e06e9e1ServiceRole1CC58A6F" + "EC2LinuxRoleDefaultPolicy1369791B", + "EC2LinuxRole8B6519A2" ] }, - "deleteamidcc036c8876b451ea2c1552f9e06e9e1LogRetention85F808EB": { - "Type": "Custom::LogRetention", - "Properties": { - "ServiceToken": { - "Fn::GetAtt": [ - "LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8aFD4BFC8A", - "Arn" - ] - }, - "LogGroupName": { - "Fn::Join": [ - "", - [ - "/aws/lambda/", - { - "Ref": "deleteamidcc036c8876b451ea2c1552f9e06e9e1BE713303" - } - ] - ] - }, - "RetentionInDays": 30 - } - }, - "EC2SpotLinuxSG8D846B64": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "GroupDescription": "github-runners-test/EC2 Spot Linux/SG", - "SecurityGroupEgress": [ - { - "CidrIp": "0.0.0.0/0", - "Description": "Allow all outbound traffic by default", - "IpProtocol": "-1" - } - ], - "VpcId": { - "Ref": "Vpc8378EB38" - } - } - }, - "EC2SpotLinuxRole86333E5D": { + "EC2LinuxSsmRole4825D4ED": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { @@ -13680,72 +13605,158 @@ "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { - "Service": "ec2.amazonaws.com" + "Service": "ssm.amazonaws.com" } } ], "Version": "2012-10-17" - } - } - }, - "EC2SpotLinuxRoleDefaultPolicy061AD1D0": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "states:SendTaskFailure", - "states:SendTaskSuccess", - "states:SendTaskHeartbeat" + }, + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "states:SendTaskFailure", + "states:SendTaskSuccess" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/aws:cloudformation:stack-id": { + "Ref": "AWS::StackId" + } + } + }, + "Effect": "Allow", + "Resource": "*" + } ], - "Condition": { - "StringEquals": { - "aws:ResourceTag/aws:cloudformation:stack-id": { - "Ref": "AWS::StackId" + "Version": "2012-10-17" + }, + "PolicyName": "stepfunctions" + }, + { + "PolicyDocument": { + "Statement": [ + { + "Action": "ssm:SendCommand", + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":instance/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ssm:", + { + "Ref": "AWS::Region" + }, + "::document/AWS-RunShellScript" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ssm:", + { + "Ref": "AWS::Region" + }, + "::document/AWS-RunPowerShellScript" + ] + ] + } + ] + }, + { + "Action": "ssm:DescribeInstanceInformation", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ssm:ListCommands", + "ssm:ListCommandInvocations" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ssm:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":*" + ] + ] } } - }, - "Effect": "Allow", - "Resource": "*" - }, - { - "Action": [ - "ssmmessages:CreateControlChannel", - "ssmmessages:CreateDataChannel", - "ssmmessages:OpenControlChannel", - "ssmmessages:OpenDataChannel", - "s3:GetEncryptionConfiguration", - "ssm:UpdateInstanceInformation" ], - "Effect": "Allow", - "Resource": "*" + "Version": "2012-10-17" }, - { - "Action": [ - "logs:CreateLogStream", - "logs:PutLogEvents" - ], - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "EC2SpotLinuxLogsF78D5F0E", - "Arn" - ] - } - } - ], - "Version": "2012-10-17" - }, - "PolicyName": "EC2SpotLinuxRoleDefaultPolicy061AD1D0", - "Roles": [ + "PolicyName": "ssm" + }, { - "Ref": "EC2SpotLinuxRole86333E5D" + "PolicyDocument": { + "Statement": [ + { + "Action": "ec2:TerminateInstances", + "Condition": { + "StringEquals": { + "aws:ResourceTag/aws:ec2launchtemplate:id": { + "Ref": "EC2LinuxLaunchTemplate36CD9B92" + } + } + }, + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "ec2" } ] } }, - "EC2SpotLinuxLogsF78D5F0E": { + "EC2LinuxLogsC4CD8F14": { "Type": "AWS::Logs::LogGroup", "Properties": { "RetentionInDays": 30 @@ -13753,12 +13764,12 @@ "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" }, - "EC2SpotLinuxLogsLogsfilter89CE719F": { + "EC2LinuxLogsLogsfilter186C2AB7": { "Type": "AWS::Logs::MetricFilter", "Properties": { "FilterPattern": "[..., marker = \"CDKGHA\", job = \"JOB\", done = \"DONE\", labels, status = \"Succeeded\" || status = \"SucceededWithIssues\" || status = \"Failed\" || status = \"Canceled\" || status = \"Skipped\" || status = \"Abandoned\"]", "LogGroupName": { - "Ref": "EC2SpotLinuxLogsF78D5F0E" + "Ref": "EC2LinuxLogsC4CD8F14" }, "MetricTransformations": [ { @@ -13780,28 +13791,469 @@ ] } }, - "EC2SpotLinuxInstanceProfileB12320D4": { - "Type": "AWS::IAM::InstanceProfile", + "EC2LinuxAutomation2C6F2077": { + "Type": "AWS::SSM::Document", + "Properties": { + "Content": { + "description": "Run GitHub Runner on EC2 instance for github-runners-test/EC2 Linux", + "schemaVersion": "0.3", + "assumeRole": { + "Fn::GetAtt": [ + "EC2LinuxSsmRole4825D4ED", + "Arn" + ] + }, + "parameters": { + "instanceId": { + "type": "String", + "description": "Instance id where runner should be executed" + }, + "taskToken": { + "type": "String", + "description": "Step Function task token for callback response" + }, + "runnerName": { + "type": "String", + "description": "Runner name" + }, + "runnerToken": { + "type": "String", + "description": "Runner token used to register runner on GitHub" + }, + "labels": { + "type": "String", + "description": "Labels to assign to runner" + }, + "registrationUrl": { + "type": "String", + "description": "Full URL to use for runner registration" + } + }, + "mainSteps": [ + { + "name": "Runner", + "action": "aws:runCommand", + "inputs": { + "DocumentName": "AWS-RunShellScript", + "InstanceIds": [ + "{{ instanceId }}" + ], + "Parameters": { + "workingDirectory": "/home/runner", + "commands": [ + "set -ex", + "touch /home/runner/STARTED", + "{\n while true; do\n aws stepfunctions send-task-heartbeat --task-token \"{{ taskToken }}\"\n sleep 60\n done\n } &", + "if [ \"$(cat RUNNER_VERSION)\" = \"latest\" ]; then\n RUNNER_FLAGS=\"\"\n else\n RUNNER_FLAGS=\"--disableupdate\"\n fi", + "sudo -Hu runner /home/runner/config.sh --unattended --url \"{{ registrationUrl }}\" --token \"{{ runnerToken }}\" --ephemeral --work _work --labels \"{{ labels }},cdkghr:started:$(date +%s)\" $RUNNER_FLAGS --name \"{{ runnerName }}\" || exit 1", + "sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2", + "STATUS=$(grep -Phors \"finish job request for job [0-9a-f\\-]+ with result: \\K.*\" /home/runner/_diag/ | tail -n1)\n [ -n \"$STATUS\" ] && echo CDKGHA JOB DONE \"$labels\" \"$STATUS\"" + ] + }, + "CloudWatchOutputConfig": { + "CloudWatchLogGroupName": { + "Ref": "EC2LinuxLogsC4CD8F14" + }, + "CloudWatchOutputEnabled": true + } + }, + "nextStep": "SendTaskSuccess", + "onFailure": "step:SendTaskFailure", + "onCancel": "step:SendTaskFailure" + }, + { + "name": "SendTaskSuccess", + "action": "aws:executeAwsApi", + "inputs": { + "Service": "stepfunctions", + "Api": "send_task_success", + "taskToken": "{{ taskToken }}", + "output": "{}" + }, + "timeoutSeconds": 50, + "nextStep": "TerminateInstance", + "onFailure": "step:TerminateInstance", + "onCancel": "step:TerminateInstance" + }, + { + "name": "SendTaskFailure", + "action": "aws:executeAwsApi", + "inputs": { + "Service": "stepfunctions", + "Api": "send_task_failure", + "taskToken": "{{ taskToken }}", + "error": "Automation document failure", + "cause": "Runner failed, check command execution id {{Runner.CommandId}} for more details" + }, + "timeoutSeconds": 50, + "nextStep": "TerminateInstance", + "onFailure": "step:TerminateInstance", + "onCancel": "step:TerminateInstance" + }, + { + "name": "TerminateInstance", + "action": "aws:executeAwsApi", + "inputs": { + "Service": "ec2", + "Api": "terminate_instances", + "InstanceIds": [ + "{{ instanceId }}" + ] + }, + "timeoutSeconds": 50, + "isEnd": true + } + ] + }, + "DocumentType": "Automation", + "TargetType": "/AWS::EC2::Host" + } + }, + "deleteamidcc036c8876b451ea2c1552f9e06e9e1ServiceRole1CC58A6F": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + } + }, + "deleteamidcc036c8876b451ea2c1552f9e06e9e1ServiceRoleDefaultPolicy0C44BF83": { + "Type": "AWS::IAM::Policy", "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "ec2:DescribeImages", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:DeregisterImage", + "ec2:DeleteSnapshot" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/GitHubRunners:Stack": "github-runners-test" + } + }, + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "deleteamidcc036c8876b451ea2c1552f9e06e9e1ServiceRoleDefaultPolicy0C44BF83", "Roles": [ { - "Ref": "EC2SpotLinuxRole86333E5D" + "Ref": "deleteamidcc036c8876b451ea2c1552f9e06e9e1ServiceRole1CC58A6F" } ] } }, - "EC2SpotLinuxAMIRootDeviceC8A06843": { - "Type": "Custom::AmiRootDevice", + "deleteamidcc036c8876b451ea2c1552f9e06e9e1BE713303": { + "Type": "AWS::Lambda::Function", "Properties": { - "ServiceToken": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "b1862470ac31c934803dff83368cf5cdec42c544fd77a6e8fc25d6d7e55befa4.zip" + }, + "Description": "Delete old GitHub Runner AMIs", + "Environment": { + "Variables": { + "AWS_NODEJS_CONNECTION_REUSE_ENABLED": "1" + } + }, + "Handler": "index.handler", + "Role": { "Fn::GetAtt": [ - "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1C64D247C", + "deleteamidcc036c8876b451ea2c1552f9e06e9e1ServiceRole1CC58A6F", "Arn" ] }, - "Ami": { - "Ref": "AMILinuxBuilderLaunchtemplateA29452C4" - } + "Runtime": "nodejs18.x", + "Timeout": 300 + }, + "DependsOn": [ + "deleteamidcc036c8876b451ea2c1552f9e06e9e1ServiceRoleDefaultPolicy0C44BF83", + "deleteamidcc036c8876b451ea2c1552f9e06e9e1ServiceRole1CC58A6F" + ] + }, + "deleteamidcc036c8876b451ea2c1552f9e06e9e1LogRetention85F808EB": { + "Type": "Custom::LogRetention", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8aFD4BFC8A", + "Arn" + ] + }, + "LogGroupName": { + "Fn::Join": [ + "", + [ + "/aws/lambda/", + { + "Ref": "deleteamidcc036c8876b451ea2c1552f9e06e9e1BE713303" + } + ] + ] + }, + "RetentionInDays": 30 + } + }, + "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1ServiceRole69906A36": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + } + }, + "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1ServiceRoleDefaultPolicyDC1762B6": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ssm:GetParameter", + "ec2:DescribeImages", + "ec2:DescribeLaunchTemplateVersions" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1ServiceRoleDefaultPolicyDC1762B6", + "Roles": [ + { + "Ref": "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1ServiceRole69906A36" + } + ] + } + }, + "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1C64D247C": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "97a52dd61d3f204877c7a944f14423bdbdec7587794ab63213506581ffb19976.zip" + }, + "Description": "Custom resource handler that triggers CodeBuild to build runner images, and cleans-up images on deletion", + "Environment": { + "Variables": { + "AWS_NODEJS_CONNECTION_REUSE_ENABLED": "1" + } + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1ServiceRole69906A36", + "Arn" + ] + }, + "Runtime": "nodejs18.x", + "Timeout": 60 + }, + "DependsOn": [ + "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1ServiceRoleDefaultPolicyDC1762B6", + "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1ServiceRole69906A36" + ] + }, + "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1LogRetentionFC67AFCF": { + "Type": "Custom::LogRetention", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8aFD4BFC8A", + "Arn" + ] + }, + "LogGroupName": { + "Fn::Join": [ + "", + [ + "/aws/lambda/", + { + "Ref": "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1C64D247C" + } + ] + ] + }, + "RetentionInDays": 30 + } + }, + "EC2SpotLinuxSG8D846B64": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "github-runners-test/EC2 Spot Linux/SG", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "VpcId": { + "Ref": "Vpc8378EB38" + } + } + }, + "EC2SpotLinuxRole86333E5D": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": "states:SendTaskHeartbeat", + "Condition": { + "StringEquals": { + "aws:ResourceTag/aws:cloudformation:stack-id": { + "Ref": "AWS::StackId" + } + } + }, + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "stepfunctions" + } + ] + } + }, + "EC2SpotLinuxRoleDefaultPolicy061AD1D0": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ssmmessages:CreateControlChannel", + "ssmmessages:CreateDataChannel", + "ssmmessages:OpenControlChannel", + "ssmmessages:OpenDataChannel", + "s3:GetEncryptionConfiguration", + "ssm:UpdateInstanceInformation" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "EC2SpotLinuxLogsF78D5F0E", + "Arn" + ] + } + }, + { + "Action": "logs:CreateLogGroup", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "EC2SpotLinuxLogsF78D5F0E", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "EC2SpotLinuxRoleDefaultPolicy061AD1D0", + "Roles": [ + { + "Ref": "EC2SpotLinuxRole86333E5D" + } + ] + } + }, + "EC2SpotLinuxAMIRootDeviceC8A06843": { + "Type": "Custom::AmiRootDevice", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1C64D247C", + "Arn" + ] + }, + "Ami": { + "Ref": "AMILinuxBuilderLaunchtemplateA29452C4" + } }, "DependsOn": [ "AMILinuxBuilderAMIDeleter58658716", @@ -13839,10 +14291,973 @@ "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" }, - "EC2Linuxarm64SG550ECD6C": { + "EC2SpotLinuxLaunchTemplateProfile23B96A0D": { + "Type": "AWS::IAM::InstanceProfile", + "Properties": { + "Roles": [ + { + "Ref": "EC2SpotLinuxRole86333E5D" + } + ] + } + }, + "EC2SpotLinuxLaunchTemplate9ACE58E5": { + "Type": "AWS::EC2::LaunchTemplate", + "Properties": { + "LaunchTemplateData": { + "BlockDeviceMappings": [ + { + "DeviceName": { + "Ref": "EC2SpotLinuxAMIRootDeviceC8A06843" + }, + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 30 + } + } + ], + "IamInstanceProfile": { + "Arn": { + "Fn::GetAtt": [ + "EC2SpotLinuxLaunchTemplateProfile23B96A0D", + "Arn" + ] + } + }, + "InstanceInitiatedShutdownBehavior": "terminate", + "InstanceMarketOptions": { + "MarketType": "spot", + "SpotOptions": { + "SpotInstanceType": "one-time" + } + }, + "InstanceType": "m5.large", + "MetadataOptions": { + "HttpTokens": "required", + "InstanceMetadataTags": "enabled" + }, + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "EC2SpotLinuxSG8D846B64", + "GroupId" + ] + } + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "Name", + "Value": "github-runners-test/EC2 Spot Linux/Launch Template" + } + ] + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "Name", + "Value": "github-runners-test/EC2 Spot Linux/Launch Template" + } + ] + } + ], + "UserData": { + "Fn::Base64": "#!/bin/bash\n{\n sleep 600\n if [ ! -e /home/runner/STARTED ]; then\n echo \"Runner didn't connect to SSM, powering off\"\n sudo poweroff\n fi\n } &" + } + }, + "TagSpecifications": [ + { + "ResourceType": "launch-template", + "Tags": [ + { + "Key": "Name", + "Value": "github-runners-test/EC2 Spot Linux/Launch Template" + } + ] + } + ] + }, + "DependsOn": [ + "EC2SpotLinuxRoleDefaultPolicy061AD1D0", + "EC2SpotLinuxRole86333E5D" + ] + }, + "EC2SpotLinuxSsmRoleD31ED667": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ssm.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "states:SendTaskFailure", + "states:SendTaskSuccess" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/aws:cloudformation:stack-id": { + "Ref": "AWS::StackId" + } + } + }, + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "stepfunctions" + }, + { + "PolicyDocument": { + "Statement": [ + { + "Action": "ssm:SendCommand", + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":instance/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ssm:", + { + "Ref": "AWS::Region" + }, + "::document/AWS-RunShellScript" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ssm:", + { + "Ref": "AWS::Region" + }, + "::document/AWS-RunPowerShellScript" + ] + ] + } + ] + }, + { + "Action": "ssm:DescribeInstanceInformation", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ssm:ListCommands", + "ssm:ListCommandInvocations" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ssm:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":*" + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "ssm" + }, + { + "PolicyDocument": { + "Statement": [ + { + "Action": "ec2:TerminateInstances", + "Condition": { + "StringEquals": { + "aws:ResourceTag/aws:ec2launchtemplate:id": { + "Ref": "EC2SpotLinuxLaunchTemplate9ACE58E5" + } + } + }, + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "ec2" + } + ] + } + }, + "EC2SpotLinuxLogsF78D5F0E": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "RetentionInDays": 30 + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "EC2SpotLinuxLogsLogsfilter89CE719F": { + "Type": "AWS::Logs::MetricFilter", + "Properties": { + "FilterPattern": "[..., marker = \"CDKGHA\", job = \"JOB\", done = \"DONE\", labels, status = \"Succeeded\" || status = \"SucceededWithIssues\" || status = \"Failed\" || status = \"Canceled\" || status = \"Skipped\" || status = \"Abandoned\"]", + "LogGroupName": { + "Ref": "EC2SpotLinuxLogsF78D5F0E" + }, + "MetricTransformations": [ + { + "Dimensions": [ + { + "Key": "ProviderLabels", + "Value": "$labels" + }, + { + "Key": "Status", + "Value": "$status" + } + ], + "MetricName": "JobCompleted", + "MetricNamespace": "GitHubRunners", + "MetricValue": "1", + "Unit": "Count" + } + ] + } + }, + "EC2SpotLinuxAutomation1255B0E6": { + "Type": "AWS::SSM::Document", + "Properties": { + "Content": { + "description": "Run GitHub Runner on EC2 instance for github-runners-test/EC2 Spot Linux", + "schemaVersion": "0.3", + "assumeRole": { + "Fn::GetAtt": [ + "EC2SpotLinuxSsmRoleD31ED667", + "Arn" + ] + }, + "parameters": { + "instanceId": { + "type": "String", + "description": "Instance id where runner should be executed" + }, + "taskToken": { + "type": "String", + "description": "Step Function task token for callback response" + }, + "runnerName": { + "type": "String", + "description": "Runner name" + }, + "runnerToken": { + "type": "String", + "description": "Runner token used to register runner on GitHub" + }, + "labels": { + "type": "String", + "description": "Labels to assign to runner" + }, + "registrationUrl": { + "type": "String", + "description": "Full URL to use for runner registration" + } + }, + "mainSteps": [ + { + "name": "Runner", + "action": "aws:runCommand", + "inputs": { + "DocumentName": "AWS-RunShellScript", + "InstanceIds": [ + "{{ instanceId }}" + ], + "Parameters": { + "workingDirectory": "/home/runner", + "commands": [ + "set -ex", + "touch /home/runner/STARTED", + "{\n while true; do\n aws stepfunctions send-task-heartbeat --task-token \"{{ taskToken }}\"\n sleep 60\n done\n } &", + "if [ \"$(cat RUNNER_VERSION)\" = \"latest\" ]; then\n RUNNER_FLAGS=\"\"\n else\n RUNNER_FLAGS=\"--disableupdate\"\n fi", + "sudo -Hu runner /home/runner/config.sh --unattended --url \"{{ registrationUrl }}\" --token \"{{ runnerToken }}\" --ephemeral --work _work --labels \"{{ labels }},cdkghr:started:$(date +%s)\" $RUNNER_FLAGS --name \"{{ runnerName }}\" || exit 1", + "sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2", + "STATUS=$(grep -Phors \"finish job request for job [0-9a-f\\-]+ with result: \\K.*\" /home/runner/_diag/ | tail -n1)\n [ -n \"$STATUS\" ] && echo CDKGHA JOB DONE \"$labels\" \"$STATUS\"" + ] + }, + "CloudWatchOutputConfig": { + "CloudWatchLogGroupName": { + "Ref": "EC2SpotLinuxLogsF78D5F0E" + }, + "CloudWatchOutputEnabled": true + } + }, + "nextStep": "SendTaskSuccess", + "onFailure": "step:SendTaskFailure", + "onCancel": "step:SendTaskFailure" + }, + { + "name": "SendTaskSuccess", + "action": "aws:executeAwsApi", + "inputs": { + "Service": "stepfunctions", + "Api": "send_task_success", + "taskToken": "{{ taskToken }}", + "output": "{}" + }, + "timeoutSeconds": 50, + "nextStep": "TerminateInstance", + "onFailure": "step:TerminateInstance", + "onCancel": "step:TerminateInstance" + }, + { + "name": "SendTaskFailure", + "action": "aws:executeAwsApi", + "inputs": { + "Service": "stepfunctions", + "Api": "send_task_failure", + "taskToken": "{{ taskToken }}", + "error": "Automation document failure", + "cause": "Runner failed, check command execution id {{Runner.CommandId}} for more details" + }, + "timeoutSeconds": 50, + "nextStep": "TerminateInstance", + "onFailure": "step:TerminateInstance", + "onCancel": "step:TerminateInstance" + }, + { + "name": "TerminateInstance", + "action": "aws:executeAwsApi", + "inputs": { + "Service": "ec2", + "Api": "terminate_instances", + "InstanceIds": [ + "{{ instanceId }}" + ] + }, + "timeoutSeconds": 50, + "isEnd": true + } + ] + }, + "DocumentType": "Automation", + "TargetType": "/AWS::EC2::Host" + } + }, + "EC2Linuxarm64SG550ECD6C": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "github-runners-test/EC2 Linux arm64/SG", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "VpcId": { + "Ref": "Vpc8378EB38" + } + } + }, + "EC2Linuxarm64Role242F68FF": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": "states:SendTaskHeartbeat", + "Condition": { + "StringEquals": { + "aws:ResourceTag/aws:cloudformation:stack-id": { + "Ref": "AWS::StackId" + } + } + }, + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "stepfunctions" + } + ] + } + }, + "EC2Linuxarm64RoleDefaultPolicyDE9193C3": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ssmmessages:CreateControlChannel", + "ssmmessages:CreateDataChannel", + "ssmmessages:OpenControlChannel", + "ssmmessages:OpenDataChannel", + "s3:GetEncryptionConfiguration", + "ssm:UpdateInstanceInformation" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "EC2Linuxarm64Logs577E371E", + "Arn" + ] + } + }, + { + "Action": "logs:CreateLogGroup", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "EC2Linuxarm64Logs577E371E", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "EC2Linuxarm64RoleDefaultPolicyDE9193C3", + "Roles": [ + { + "Ref": "EC2Linuxarm64Role242F68FF" + } + ] + } + }, + "EC2Linuxarm64AMIRootDevice3046D37D": { + "Type": "Custom::AmiRootDevice", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1C64D247C", + "Arn" + ] + }, + "Ami": { + "Ref": "AMILinuxarm64BuilderLaunchtemplate8F5EFF44" + } + }, + "DependsOn": [ + "AMILinuxarm64BuilderAMIDeleter27E72E27", + "AMILinuxarm64BuilderAMIDistribution2BDAF717", + "AMILinuxarm64BuilderAMIImage40078F9E", + "AMILinuxarm64BuilderAmiLog84A9D94A", + "AMILinuxarm64BuilderAMIPipeline9CC1354C", + "AMILinuxarm64BuilderAmiRecipe6A6ED38F", + "AMILinuxarm64BuilderAmiRecipeVersion429E7646", + "AMILinuxarm64BuilderComponent0RequiredPackagesComponent2ABFFC8F", + "AMILinuxarm64BuilderComponent0RequiredPackagesVersion7CCB5268", + "AMILinuxarm64BuilderComponent1RunnerUserComponent4BA674F3", + "AMILinuxarm64BuilderComponent1RunnerUserVersion7CC2642F", + "AMILinuxarm64BuilderComponent2GitComponent4229D34D", + "AMILinuxarm64BuilderComponent2GitVersion6653A045", + "AMILinuxarm64BuilderComponent3GithubCliComponentF183CE19", + "AMILinuxarm64BuilderComponent3GithubCliVersion0BCEFC78", + "AMILinuxarm64BuilderComponent4AwsCliComponent48566B83", + "AMILinuxarm64BuilderComponent4AwsCliVersion78036929", + "AMILinuxarm64BuilderComponent5DockerComponent9F2EEB10", + "AMILinuxarm64BuilderComponent5DockerVersion844A4107", + "AMILinuxarm64BuilderComponent6GithubRunnerComponentF4B81DE9", + "AMILinuxarm64BuilderComponent6GithubRunnerVersionC728248D", + "AMILinuxarm64BuilderComponent7CustomUndefinedComponent6F6F62D0", + "AMILinuxarm64BuilderComponent7CustomUndefinedVersion5AA71C6E", + "AMILinuxarm64BuilderInfrastructure80FA16D6", + "AMILinuxarm64BuilderInstanceProfileCE3B6B09", + "AMILinuxarm64BuilderLaunchtemplate8F5EFF44", + "AMILinuxarm64BuilderLifecyclePolicyAMI44B6EABD", + "AMILinuxarm64BuilderLifecyclePolicyAMIRole1D81BF43", + "AMILinuxarm64BuilderRoleDefaultPolicy113305EE", + "AMILinuxarm64BuilderRole40D54E29", + "AMILinuxarm64BuilderSG94315968" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "EC2Linuxarm64LaunchTemplateProfile27EF4C0D": { + "Type": "AWS::IAM::InstanceProfile", + "Properties": { + "Roles": [ + { + "Ref": "EC2Linuxarm64Role242F68FF" + } + ] + } + }, + "EC2Linuxarm64LaunchTemplateDBA523C3": { + "Type": "AWS::EC2::LaunchTemplate", + "Properties": { + "LaunchTemplateData": { + "BlockDeviceMappings": [ + { + "DeviceName": { + "Ref": "EC2Linuxarm64AMIRootDevice3046D37D" + }, + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 30 + } + } + ], + "IamInstanceProfile": { + "Arn": { + "Fn::GetAtt": [ + "EC2Linuxarm64LaunchTemplateProfile27EF4C0D", + "Arn" + ] + } + }, + "InstanceInitiatedShutdownBehavior": "terminate", + "InstanceType": "m6g.large", + "MetadataOptions": { + "HttpTokens": "required", + "InstanceMetadataTags": "enabled" + }, + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "EC2Linuxarm64SG550ECD6C", + "GroupId" + ] + } + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "Name", + "Value": "github-runners-test/EC2 Linux arm64/Launch Template" + } + ] + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "Name", + "Value": "github-runners-test/EC2 Linux arm64/Launch Template" + } + ] + } + ], + "UserData": { + "Fn::Base64": "#!/bin/bash\n{\n sleep 600\n if [ ! -e /home/runner/STARTED ]; then\n echo \"Runner didn't connect to SSM, powering off\"\n sudo poweroff\n fi\n } &" + } + }, + "TagSpecifications": [ + { + "ResourceType": "launch-template", + "Tags": [ + { + "Key": "Name", + "Value": "github-runners-test/EC2 Linux arm64/Launch Template" + } + ] + } + ] + }, + "DependsOn": [ + "EC2Linuxarm64RoleDefaultPolicyDE9193C3", + "EC2Linuxarm64Role242F68FF" + ] + }, + "EC2Linuxarm64SsmRole7A88520A": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ssm.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "states:SendTaskFailure", + "states:SendTaskSuccess" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/aws:cloudformation:stack-id": { + "Ref": "AWS::StackId" + } + } + }, + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "stepfunctions" + }, + { + "PolicyDocument": { + "Statement": [ + { + "Action": "ssm:SendCommand", + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":instance/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ssm:", + { + "Ref": "AWS::Region" + }, + "::document/AWS-RunShellScript" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ssm:", + { + "Ref": "AWS::Region" + }, + "::document/AWS-RunPowerShellScript" + ] + ] + } + ] + }, + { + "Action": "ssm:DescribeInstanceInformation", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ssm:ListCommands", + "ssm:ListCommandInvocations" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ssm:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":*" + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "ssm" + }, + { + "PolicyDocument": { + "Statement": [ + { + "Action": "ec2:TerminateInstances", + "Condition": { + "StringEquals": { + "aws:ResourceTag/aws:ec2launchtemplate:id": { + "Ref": "EC2Linuxarm64LaunchTemplateDBA523C3" + } + } + }, + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "ec2" + } + ] + } + }, + "EC2Linuxarm64Logs577E371E": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "RetentionInDays": 30 + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "EC2Linuxarm64LogsLogsfilter0A247938": { + "Type": "AWS::Logs::MetricFilter", + "Properties": { + "FilterPattern": "[..., marker = \"CDKGHA\", job = \"JOB\", done = \"DONE\", labels, status = \"Succeeded\" || status = \"SucceededWithIssues\" || status = \"Failed\" || status = \"Canceled\" || status = \"Skipped\" || status = \"Abandoned\"]", + "LogGroupName": { + "Ref": "EC2Linuxarm64Logs577E371E" + }, + "MetricTransformations": [ + { + "Dimensions": [ + { + "Key": "ProviderLabels", + "Value": "$labels" + }, + { + "Key": "Status", + "Value": "$status" + } + ], + "MetricName": "JobCompleted", + "MetricNamespace": "GitHubRunners", + "MetricValue": "1", + "Unit": "Count" + } + ] + } + }, + "EC2Linuxarm64Automation0232B49F": { + "Type": "AWS::SSM::Document", + "Properties": { + "Content": { + "description": "Run GitHub Runner on EC2 instance for github-runners-test/EC2 Linux arm64", + "schemaVersion": "0.3", + "assumeRole": { + "Fn::GetAtt": [ + "EC2Linuxarm64SsmRole7A88520A", + "Arn" + ] + }, + "parameters": { + "instanceId": { + "type": "String", + "description": "Instance id where runner should be executed" + }, + "taskToken": { + "type": "String", + "description": "Step Function task token for callback response" + }, + "runnerName": { + "type": "String", + "description": "Runner name" + }, + "runnerToken": { + "type": "String", + "description": "Runner token used to register runner on GitHub" + }, + "labels": { + "type": "String", + "description": "Labels to assign to runner" + }, + "registrationUrl": { + "type": "String", + "description": "Full URL to use for runner registration" + } + }, + "mainSteps": [ + { + "name": "Runner", + "action": "aws:runCommand", + "inputs": { + "DocumentName": "AWS-RunShellScript", + "InstanceIds": [ + "{{ instanceId }}" + ], + "Parameters": { + "workingDirectory": "/home/runner", + "commands": [ + "set -ex", + "touch /home/runner/STARTED", + "{\n while true; do\n aws stepfunctions send-task-heartbeat --task-token \"{{ taskToken }}\"\n sleep 60\n done\n } &", + "if [ \"$(cat RUNNER_VERSION)\" = \"latest\" ]; then\n RUNNER_FLAGS=\"\"\n else\n RUNNER_FLAGS=\"--disableupdate\"\n fi", + "sudo -Hu runner /home/runner/config.sh --unattended --url \"{{ registrationUrl }}\" --token \"{{ runnerToken }}\" --ephemeral --work _work --labels \"{{ labels }},cdkghr:started:$(date +%s)\" $RUNNER_FLAGS --name \"{{ runnerName }}\" || exit 1", + "sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2", + "STATUS=$(grep -Phors \"finish job request for job [0-9a-f\\-]+ with result: \\K.*\" /home/runner/_diag/ | tail -n1)\n [ -n \"$STATUS\" ] && echo CDKGHA JOB DONE \"$labels\" \"$STATUS\"" + ] + }, + "CloudWatchOutputConfig": { + "CloudWatchLogGroupName": { + "Ref": "EC2Linuxarm64Logs577E371E" + }, + "CloudWatchOutputEnabled": true + } + }, + "nextStep": "SendTaskSuccess", + "onFailure": "step:SendTaskFailure", + "onCancel": "step:SendTaskFailure" + }, + { + "name": "SendTaskSuccess", + "action": "aws:executeAwsApi", + "inputs": { + "Service": "stepfunctions", + "Api": "send_task_success", + "taskToken": "{{ taskToken }}", + "output": "{}" + }, + "timeoutSeconds": 50, + "nextStep": "TerminateInstance", + "onFailure": "step:TerminateInstance", + "onCancel": "step:TerminateInstance" + }, + { + "name": "SendTaskFailure", + "action": "aws:executeAwsApi", + "inputs": { + "Service": "stepfunctions", + "Api": "send_task_failure", + "taskToken": "{{ taskToken }}", + "error": "Automation document failure", + "cause": "Runner failed, check command execution id {{Runner.CommandId}} for more details" + }, + "timeoutSeconds": 50, + "nextStep": "TerminateInstance", + "onFailure": "step:TerminateInstance", + "onCancel": "step:TerminateInstance" + }, + { + "name": "TerminateInstance", + "action": "aws:executeAwsApi", + "inputs": { + "Service": "ec2", + "Api": "terminate_instances", + "InstanceIds": [ + "{{ instanceId }}" + ] + }, + "timeoutSeconds": 50, + "isEnd": true + } + ] + }, + "DocumentType": "Automation", + "TargetType": "/AWS::EC2::Host" + } + }, + "EC2WindowsSG13E24976": { "Type": "AWS::EC2::SecurityGroup", "Properties": { - "GroupDescription": "github-runners-test/EC2 Linux arm64/SG", + "GroupDescription": "github-runners-test/EC2 Windows/SG", "SecurityGroupEgress": [ { "CidrIp": "0.0.0.0/0", @@ -13855,7 +15270,7 @@ } } }, - "EC2Linuxarm64Role242F68FF": { + "EC2WindowsRoleC0D850D2": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { @@ -13869,30 +15284,36 @@ } ], "Version": "2012-10-17" - } + }, + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": "states:SendTaskHeartbeat", + "Condition": { + "StringEquals": { + "aws:ResourceTag/aws:cloudformation:stack-id": { + "Ref": "AWS::StackId" + } + } + }, + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "stepfunctions" + } + ] } }, - "EC2Linuxarm64RoleDefaultPolicyDE9193C3": { + "EC2WindowsRoleDefaultPolicyB6A15409": { "Type": "AWS::IAM::Policy", "Properties": { "PolicyDocument": { "Statement": [ - { - "Action": [ - "states:SendTaskFailure", - "states:SendTaskSuccess", - "states:SendTaskHeartbeat" - ], - "Condition": { - "StringEquals": { - "aws:ResourceTag/aws:cloudformation:stack-id": { - "Ref": "AWS::StackId" - } - } - }, - "Effect": "Allow", - "Resource": "*" - }, { "Action": [ "ssmmessages:CreateControlChannel", @@ -13913,7 +15334,17 @@ "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "EC2Linuxarm64Logs577E371E", + "EC2WindowsLogsDC1F2ABF", + "Arn" + ] + } + }, + { + "Action": "logs:CreateLogGroup", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "EC2WindowsLogsDC1F2ABF", "Arn" ] } @@ -13921,60 +15352,15 @@ ], "Version": "2012-10-17" }, - "PolicyName": "EC2Linuxarm64RoleDefaultPolicyDE9193C3", - "Roles": [ - { - "Ref": "EC2Linuxarm64Role242F68FF" - } - ] - } - }, - "EC2Linuxarm64Logs577E371E": { - "Type": "AWS::Logs::LogGroup", - "Properties": { - "RetentionInDays": 30 - }, - "UpdateReplacePolicy": "Delete", - "DeletionPolicy": "Delete" - }, - "EC2Linuxarm64LogsLogsfilter0A247938": { - "Type": "AWS::Logs::MetricFilter", - "Properties": { - "FilterPattern": "[..., marker = \"CDKGHA\", job = \"JOB\", done = \"DONE\", labels, status = \"Succeeded\" || status = \"SucceededWithIssues\" || status = \"Failed\" || status = \"Canceled\" || status = \"Skipped\" || status = \"Abandoned\"]", - "LogGroupName": { - "Ref": "EC2Linuxarm64Logs577E371E" - }, - "MetricTransformations": [ - { - "Dimensions": [ - { - "Key": "ProviderLabels", - "Value": "$labels" - }, - { - "Key": "Status", - "Value": "$status" - } - ], - "MetricName": "JobCompleted", - "MetricNamespace": "GitHubRunners", - "MetricValue": "1", - "Unit": "Count" - } - ] - } - }, - "EC2Linuxarm64InstanceProfile1E6F8D53": { - "Type": "AWS::IAM::InstanceProfile", - "Properties": { + "PolicyName": "EC2WindowsRoleDefaultPolicyB6A15409", "Roles": [ { - "Ref": "EC2Linuxarm64Role242F68FF" + "Ref": "EC2WindowsRoleC0D850D2" } ] } }, - "EC2Linuxarm64AMIRootDevice3046D37D": { + "EC2WindowsAMIRootDevice9FFC971A": { "Type": "Custom::AmiRootDevice", "Properties": { "ServiceToken": { @@ -13984,62 +15370,134 @@ ] }, "Ami": { - "Ref": "AMILinuxarm64BuilderLaunchtemplate8F5EFF44" + "Ref": "WindowsEC2BuilderLaunchtemplate0A66E9C2" } }, "DependsOn": [ - "AMILinuxarm64BuilderAMIDeleter27E72E27", - "AMILinuxarm64BuilderAMIDistribution2BDAF717", - "AMILinuxarm64BuilderAMIImage40078F9E", - "AMILinuxarm64BuilderAmiLog84A9D94A", - "AMILinuxarm64BuilderAMIPipeline9CC1354C", - "AMILinuxarm64BuilderAmiRecipe6A6ED38F", - "AMILinuxarm64BuilderAmiRecipeVersion429E7646", - "AMILinuxarm64BuilderComponent0RequiredPackagesComponent2ABFFC8F", - "AMILinuxarm64BuilderComponent0RequiredPackagesVersion7CCB5268", - "AMILinuxarm64BuilderComponent1RunnerUserComponent4BA674F3", - "AMILinuxarm64BuilderComponent1RunnerUserVersion7CC2642F", - "AMILinuxarm64BuilderComponent2GitComponent4229D34D", - "AMILinuxarm64BuilderComponent2GitVersion6653A045", - "AMILinuxarm64BuilderComponent3GithubCliComponentF183CE19", - "AMILinuxarm64BuilderComponent3GithubCliVersion0BCEFC78", - "AMILinuxarm64BuilderComponent4AwsCliComponent48566B83", - "AMILinuxarm64BuilderComponent4AwsCliVersion78036929", - "AMILinuxarm64BuilderComponent5DockerComponent9F2EEB10", - "AMILinuxarm64BuilderComponent5DockerVersion844A4107", - "AMILinuxarm64BuilderComponent6GithubRunnerComponentF4B81DE9", - "AMILinuxarm64BuilderComponent6GithubRunnerVersionC728248D", - "AMILinuxarm64BuilderComponent7CustomUndefinedComponent6F6F62D0", - "AMILinuxarm64BuilderComponent7CustomUndefinedVersion5AA71C6E", - "AMILinuxarm64BuilderInfrastructure80FA16D6", - "AMILinuxarm64BuilderInstanceProfileCE3B6B09", - "AMILinuxarm64BuilderLaunchtemplate8F5EFF44", - "AMILinuxarm64BuilderLifecyclePolicyAMI44B6EABD", - "AMILinuxarm64BuilderLifecyclePolicyAMIRole1D81BF43", - "AMILinuxarm64BuilderRoleDefaultPolicy113305EE", - "AMILinuxarm64BuilderRole40D54E29", - "AMILinuxarm64BuilderSG94315968" + "WindowsEC2BuilderAMIDeleterF9643905", + "WindowsEC2BuilderAMIDistributionC7E428C1", + "WindowsEC2BuilderAMIImage9D812188", + "WindowsEC2BuilderAmiLog126E54B2", + "WindowsEC2BuilderAMIPipelineE3836949", + "WindowsEC2BuilderAmiRecipe41A137FF", + "WindowsEC2BuilderAmiRecipeVersionEE5C2F0C", + "WindowsEC2BuilderComponent0RequiredPackagesComponent42A20182", + "WindowsEC2BuilderComponent0RequiredPackagesVersion08C0B23D", + "WindowsEC2BuilderComponent1RunnerUserComponent90CB91A0", + "WindowsEC2BuilderComponent1RunnerUserVersionBB304040", + "WindowsEC2BuilderComponent2GitComponentBDE86E60", + "WindowsEC2BuilderComponent2GitVersion6AACF128", + "WindowsEC2BuilderComponent3GithubCliComponent44942D8F", + "WindowsEC2BuilderComponent3GithubCliVersion1BB3282E", + "WindowsEC2BuilderComponent4AwsCliComponent2112B01A", + "WindowsEC2BuilderComponent4AwsCliVersion235F6DDE", + "WindowsEC2BuilderComponent5DockerComponent043F7B9D", + "WindowsEC2BuilderComponent5DockerVersion40844B58", + "WindowsEC2BuilderComponent6GithubRunnerComponent6342E200", + "WindowsEC2BuilderComponent6GithubRunnerVersion068CC361", + "WindowsEC2BuilderComponent7CustomUndefinedComponent83F7C59D", + "WindowsEC2BuilderComponent7CustomUndefinedVersion96AF0127", + "WindowsEC2BuilderInfrastructure757D4FD7", + "WindowsEC2BuilderInstanceProfileA8DBA763", + "WindowsEC2BuilderLaunchtemplate0A66E9C2", + "WindowsEC2BuilderLifecyclePolicyAMIF732B3A7", + "WindowsEC2BuilderLifecyclePolicyAMIRoleE85A7F2C", + "WindowsEC2BuilderRoleDefaultPolicy8A5B4E85", + "WindowsEC2BuilderRole1BA7D3E7", + "WindowsEC2BuilderSGE3F67842" ], "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" }, - "EC2WindowsSG13E24976": { - "Type": "AWS::EC2::SecurityGroup", + "EC2WindowsLaunchTemplateProfile15A35900": { + "Type": "AWS::IAM::InstanceProfile", "Properties": { - "GroupDescription": "github-runners-test/EC2 Windows/SG", - "SecurityGroupEgress": [ + "Roles": [ + { + "Ref": "EC2WindowsRoleC0D850D2" + } + ] + } + }, + "EC2WindowsLaunchTemplateCF55C6AE": { + "Type": "AWS::EC2::LaunchTemplate", + "Properties": { + "LaunchTemplateData": { + "BlockDeviceMappings": [ + { + "DeviceName": { + "Ref": "EC2WindowsAMIRootDevice9FFC971A" + }, + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 30 + } + } + ], + "IamInstanceProfile": { + "Arn": { + "Fn::GetAtt": [ + "EC2WindowsLaunchTemplateProfile15A35900", + "Arn" + ] + } + }, + "InstanceInitiatedShutdownBehavior": "terminate", + "InstanceType": "m5.large", + "MetadataOptions": { + "HttpTokens": "required", + "InstanceMetadataTags": "enabled" + }, + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "EC2WindowsSG13E24976", + "GroupId" + ] + } + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "Name", + "Value": "github-runners-test/EC2 Windows/Launch Template" + } + ] + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "Name", + "Value": "github-runners-test/EC2 Windows/Launch Template" + } + ] + } + ], + "UserData": { + "Fn::Base64": "Start-Job -ScriptBlock {\n Start-Sleep -Seconds 60\n if (-not (Test-Path C:/actions/STARTED)) {\n Write-Output \"Runner didn't connect to SSM, powering off\"\n Stop-Computer -ComputerName localhost -Force\n }\n }" + } + }, + "TagSpecifications": [ { - "CidrIp": "0.0.0.0/0", - "Description": "Allow all outbound traffic by default", - "IpProtocol": "-1" + "ResourceType": "launch-template", + "Tags": [ + { + "Key": "Name", + "Value": "github-runners-test/EC2 Windows/Launch Template" + } + ] } - ], - "VpcId": { - "Ref": "Vpc8378EB38" - } - } + ] + }, + "DependsOn": [ + "EC2WindowsRoleDefaultPolicyB6A15409", + "EC2WindowsRoleC0D850D2" + ] }, - "EC2WindowsRoleC0D850D2": { + "EC2WindowsSsmRole1EC56ECB": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { @@ -14048,67 +15506,153 @@ "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { - "Service": "ec2.amazonaws.com" + "Service": "ssm.amazonaws.com" } } ], "Version": "2012-10-17" - } - } - }, - "EC2WindowsRoleDefaultPolicyB6A15409": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "states:SendTaskFailure", - "states:SendTaskSuccess", - "states:SendTaskHeartbeat" + }, + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "states:SendTaskFailure", + "states:SendTaskSuccess" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/aws:cloudformation:stack-id": { + "Ref": "AWS::StackId" + } + } + }, + "Effect": "Allow", + "Resource": "*" + } ], - "Condition": { - "StringEquals": { - "aws:ResourceTag/aws:cloudformation:stack-id": { - "Ref": "AWS::StackId" + "Version": "2012-10-17" + }, + "PolicyName": "stepfunctions" + }, + { + "PolicyDocument": { + "Statement": [ + { + "Action": "ssm:SendCommand", + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":instance/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ssm:", + { + "Ref": "AWS::Region" + }, + "::document/AWS-RunShellScript" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ssm:", + { + "Ref": "AWS::Region" + }, + "::document/AWS-RunPowerShellScript" + ] + ] + } + ] + }, + { + "Action": "ssm:DescribeInstanceInformation", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ssm:ListCommands", + "ssm:ListCommandInvocations" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ssm:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":*" + ] + ] } } - }, - "Effect": "Allow", - "Resource": "*" - }, - { - "Action": [ - "ssmmessages:CreateControlChannel", - "ssmmessages:CreateDataChannel", - "ssmmessages:OpenControlChannel", - "ssmmessages:OpenDataChannel", - "s3:GetEncryptionConfiguration", - "ssm:UpdateInstanceInformation" ], - "Effect": "Allow", - "Resource": "*" + "Version": "2012-10-17" }, - { - "Action": [ - "logs:CreateLogStream", - "logs:PutLogEvents" - ], - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "EC2WindowsLogsDC1F2ABF", - "Arn" - ] - } - } - ], - "Version": "2012-10-17" - }, - "PolicyName": "EC2WindowsRoleDefaultPolicyB6A15409", - "Roles": [ + "PolicyName": "ssm" + }, { - "Ref": "EC2WindowsRoleC0D850D2" + "PolicyDocument": { + "Statement": [ + { + "Action": "ec2:TerminateInstances", + "Condition": { + "StringEquals": { + "aws:ResourceTag/aws:ec2launchtemplate:id": { + "Ref": "EC2WindowsLaunchTemplateCF55C6AE" + } + } + }, + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "ec2" } ] } @@ -14148,64 +15692,125 @@ ] } }, - "EC2WindowsInstanceProfileDCA59D9C": { - "Type": "AWS::IAM::InstanceProfile", - "Properties": { - "Roles": [ - { - "Ref": "EC2WindowsRoleC0D850D2" - } - ] - } - }, - "EC2WindowsAMIRootDevice9FFC971A": { - "Type": "Custom::AmiRootDevice", + "EC2WindowsAutomationC493D3FB": { + "Type": "AWS::SSM::Document", "Properties": { - "ServiceToken": { - "Fn::GetAtt": [ - "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1C64D247C", - "Arn" + "Content": { + "description": "Run GitHub Runner on EC2 instance for github-runners-test/EC2 Windows", + "schemaVersion": "0.3", + "assumeRole": { + "Fn::GetAtt": [ + "EC2WindowsSsmRole1EC56ECB", + "Arn" + ] + }, + "parameters": { + "instanceId": { + "type": "String", + "description": "Instance id where runner should be executed" + }, + "taskToken": { + "type": "String", + "description": "Step Function task token for callback response" + }, + "runnerName": { + "type": "String", + "description": "Runner name" + }, + "runnerToken": { + "type": "String", + "description": "Runner token used to register runner on GitHub" + }, + "labels": { + "type": "String", + "description": "Labels to assign to runner" + }, + "registrationUrl": { + "type": "String", + "description": "Full URL to use for runner registration" + } + }, + "mainSteps": [ + { + "name": "Runner", + "action": "aws:runCommand", + "inputs": { + "DocumentName": "AWS-RunPowerShellScript", + "InstanceIds": [ + "{{ instanceId }}" + ], + "Parameters": { + "workingDirectory": "C:\\actions", + "commands": [ + "New-Item STARTED", + "Start-Job -ScriptBlock {\n while (1) {\n aws stepfunctions send-task-heartbeat --task-token \"{{ taskToken }}\"\n sleep 60\n }\n }", + "$RunnerVersion = Get-Content RUNNER_VERSION -Raw", + "if ($RunnerVersion -eq \"latest\") { $RunnerFlags = \"\" } else { $RunnerFlags = \"--disableupdate\" }", + "./config.cmd --unattended --url \"{{ registrationUrl }}\" --token \"{{ runnerToken }}\" --ephemeral --work _work --labels \"{{ labels }},cdkghr:started:$(Get-Date -UFormat +%s)\" $RunnerFlags --name \"{{ runnerName }}\" 2>&1", + "if ($LASTEXITCODE -ne 0) { return 1 }", + "./run.cmd 2>&1", + "if ($LASTEXITCODE -ne 0) { return 2 }", + "$STATUS = Select-String -Path './_diag/*.log' -Pattern 'finish job request for job [0-9a-f\\-]+ with result: (.*)' | %{$_.Matches.Groups[1].Value} | Select-Object -Last 1\n\n if ($STATUS) {\n echo \"CDKGHA JOB DONE {{ labels }} $STATUS\"\n }" + ] + }, + "CloudWatchOutputConfig": { + "CloudWatchLogGroupName": { + "Ref": "EC2WindowsLogsDC1F2ABF" + }, + "CloudWatchOutputEnabled": true + } + }, + "nextStep": "SendTaskSuccess", + "onFailure": "step:SendTaskFailure", + "onCancel": "step:SendTaskFailure" + }, + { + "name": "SendTaskSuccess", + "action": "aws:executeAwsApi", + "inputs": { + "Service": "stepfunctions", + "Api": "send_task_success", + "taskToken": "{{ taskToken }}", + "output": "{}" + }, + "timeoutSeconds": 50, + "nextStep": "TerminateInstance", + "onFailure": "step:TerminateInstance", + "onCancel": "step:TerminateInstance" + }, + { + "name": "SendTaskFailure", + "action": "aws:executeAwsApi", + "inputs": { + "Service": "stepfunctions", + "Api": "send_task_failure", + "taskToken": "{{ taskToken }}", + "error": "Automation document failure", + "cause": "Runner failed, check command execution id {{Runner.CommandId}} for more details" + }, + "timeoutSeconds": 50, + "nextStep": "TerminateInstance", + "onFailure": "step:TerminateInstance", + "onCancel": "step:TerminateInstance" + }, + { + "name": "TerminateInstance", + "action": "aws:executeAwsApi", + "inputs": { + "Service": "ec2", + "Api": "terminate_instances", + "InstanceIds": [ + "{{ instanceId }}" + ] + }, + "timeoutSeconds": 50, + "isEnd": true + } ] }, - "Ami": { - "Ref": "WindowsEC2BuilderLaunchtemplate0A66E9C2" - } - }, - "DependsOn": [ - "WindowsEC2BuilderAMIDeleterF9643905", - "WindowsEC2BuilderAMIDistributionC7E428C1", - "WindowsEC2BuilderAMIImage9D812188", - "WindowsEC2BuilderAmiLog126E54B2", - "WindowsEC2BuilderAMIPipelineE3836949", - "WindowsEC2BuilderAmiRecipe41A137FF", - "WindowsEC2BuilderAmiRecipeVersionEE5C2F0C", - "WindowsEC2BuilderComponent0RequiredPackagesComponent42A20182", - "WindowsEC2BuilderComponent0RequiredPackagesVersion08C0B23D", - "WindowsEC2BuilderComponent1RunnerUserComponent90CB91A0", - "WindowsEC2BuilderComponent1RunnerUserVersionBB304040", - "WindowsEC2BuilderComponent2GitComponentBDE86E60", - "WindowsEC2BuilderComponent2GitVersion6AACF128", - "WindowsEC2BuilderComponent3GithubCliComponent44942D8F", - "WindowsEC2BuilderComponent3GithubCliVersion1BB3282E", - "WindowsEC2BuilderComponent4AwsCliComponent2112B01A", - "WindowsEC2BuilderComponent4AwsCliVersion235F6DDE", - "WindowsEC2BuilderComponent5DockerComponent043F7B9D", - "WindowsEC2BuilderComponent5DockerVersion40844B58", - "WindowsEC2BuilderComponent6GithubRunnerComponent6342E200", - "WindowsEC2BuilderComponent6GithubRunnerVersion068CC361", - "WindowsEC2BuilderComponent7CustomUndefinedComponent83F7C59D", - "WindowsEC2BuilderComponent7CustomUndefinedVersion96AF0127", - "WindowsEC2BuilderInfrastructure757D4FD7", - "WindowsEC2BuilderInstanceProfileA8DBA763", - "WindowsEC2BuilderLaunchtemplate0A66E9C2", - "WindowsEC2BuilderLifecyclePolicyAMIF732B3A7", - "WindowsEC2BuilderLifecyclePolicyAMIRoleE85A7F2C", - "WindowsEC2BuilderRoleDefaultPolicy8A5B4E85", - "WindowsEC2BuilderRole1BA7D3E7", - "WindowsEC2BuilderSGE3F67842" - ], - "UpdateReplacePolicy": "Delete", - "DeletionPolicy": "Delete" + "DocumentType": "Automation", + "TargetType": "/AWS::EC2::Host" + } }, "runnersSecretsWebhook7AF0D74E": { "Type": "AWS::SecretsManager::Secret", @@ -16091,7 +17696,17 @@ ] }, { - "Action": "ec2:runInstances", + "Action": "ec2:describeLaunchTemplateVersions", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "ec2:createFleet", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "ssm:startAutomationExecution", "Effect": "Allow", "Resource": "*" }, @@ -16222,7 +17837,7 @@ "Arn" ] }, - "\"},\"Choose provider\":{\"Type\":\"Choice\",\"Choices\":[{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/CodeBuildx64\"}],\"Next\":\"codebuild-x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/CodeBuildARM\"}],\"Next\":\"codebuild, linux, arm64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/CodeBuildWindows\"}],\"Next\":\"codebuild, windows, x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/ECS\"}],\"Next\":\"ecs, linux, x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/ECS ARM64\"}],\"Next\":\"ecs, linux, arm64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/ECS Windows\"}],\"Next\":\"ecs, windows, x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/Lambda\"}],\"Next\":\"lambda, x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/LambdaARM\"}],\"Next\":\"lambda, arm64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/Fargate\"}],\"Next\":\"fargate, linux, x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/Fargate-x64-spot\"}],\"Next\":\"fargate-spot, linux, x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/Fargate-arm64\"}],\"Next\":\"fargate, linux, arm64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/Fargate-arm64-spot\"}],\"Next\":\"fargate-spot, linux, arm64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/Fargate-Windows\"}],\"Next\":\"fargate, windows, x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/EC2 Linux\"}],\"Next\":\"ec2, linux, x64 data\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/EC2 Spot Linux\"}],\"Next\":\"ec2-spot, linux, x64 data\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/EC2 Linux arm64\"}],\"Next\":\"ec2, linux, arm64 data\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/EC2 Windows\"}],\"Next\":\"ec2, windows, x64 data\"}],\"Default\":\"Unknown label\"},\"Unknown label\":{\"Type\":\"Succeed\"},\"codebuild-x64\":{\"End\":true,\"Type\":\"Task\",\"Resource\":\"arn:", + "\"},\"Choose provider\":{\"Type\":\"Choice\",\"Choices\":[{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/CodeBuildx64\"}],\"Next\":\"codebuild-x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/CodeBuildARM\"}],\"Next\":\"codebuild, linux, arm64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/CodeBuildWindows\"}],\"Next\":\"codebuild, windows, x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/ECS\"}],\"Next\":\"ecs, linux, x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/ECS ARM64\"}],\"Next\":\"ecs, linux, arm64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/ECS Windows\"}],\"Next\":\"ecs, windows, x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/Lambda\"}],\"Next\":\"lambda, x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/LambdaARM\"}],\"Next\":\"lambda, arm64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/Fargate\"}],\"Next\":\"fargate, linux, x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/Fargate-x64-spot\"}],\"Next\":\"fargate-spot, linux, x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/Fargate-arm64\"}],\"Next\":\"fargate, linux, arm64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/Fargate-arm64-spot\"}],\"Next\":\"fargate-spot, linux, arm64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/Fargate-Windows\"}],\"Next\":\"fargate, windows, x64\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/EC2 Linux\"}],\"Next\":\"ec2, linux, x64 Get AMI\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/EC2 Spot Linux\"}],\"Next\":\"ec2-spot, linux, x64 Get AMI\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/EC2 Linux arm64\"}],\"Next\":\"ec2, linux, arm64 Get AMI\"},{\"And\":[{\"Variable\":\"$.provider\",\"StringEquals\":\"github-runners-test/EC2 Windows\"}],\"Next\":\"ec2, windows, x64 Get AMI\"}],\"Default\":\"Unknown label\"},\"Unknown label\":{\"Type\":\"Succeed\"},\"codebuild-x64\":{\"End\":true,\"Type\":\"Task\",\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, @@ -16443,311 +18058,135 @@ "GroupId" ] }, - "\"]}},\"Overrides\":{\"ContainerOverrides\":[{\"Name\":\"runner\",\"Environment\":[{\"Name\":\"RUNNER_TOKEN\",\"Value.$\":\"$.runner.token\"},{\"Name\":\"RUNNER_NAME\",\"Value.$\":\"$$.Execution.Name\"},{\"Name\":\"RUNNER_LABEL\",\"Value\":\"fargate,windows,x64\"},{\"Name\":\"GITHUB_DOMAIN\",\"Value.$\":\"$.runner.domain\"},{\"Name\":\"OWNER\",\"Value.$\":\"$.owner\"},{\"Name\":\"REPO\",\"Value.$\":\"$.repo\"},{\"Name\":\"REGISTRATION_URL\",\"Value.$\":\"$.runner.registrationUrl\"}]}]},\"PropagateTags\":\"TASK_DEFINITION\",\"EnableExecuteCommand\":false,\"CapacityProviderStrategy\":[{\"CapacityProvider\":\"FARGATE\"}]}},\"ec2, linux, x64 data\":{\"Type\":\"Pass\",\"ResultPath\":\"$.ec2\",\"Parameters\":{\"userdataTemplate\":\"#!/bin/bash -x\\nTASK_TOKEN=\\\"{}\\\"\\nlogGroupName=\\\"{}\\\"\\nrunnerNamePath=\\\"{}\\\"\\ngithubDomainPath=\\\"{}\\\"\\nownerPath=\\\"{}\\\"\\nrepoPath=\\\"{}\\\"\\nrunnerTokenPath=\\\"{}\\\"\\nlabels=\\\"{}\\\"\\nregistrationURL=\\\"{}\\\"\\n\\nheartbeat () \\\\{\\n while true; do\\n aws stepfunctions send-task-heartbeat --task-token \\\"$TASK_TOKEN\\\"\\n sleep 60\\n done\\n\\\\}\\nsetup_logs () \\\\{\\n cat < /tmp/log.conf || exit 1\\n \\\\{\\n \\\"logs\\\": \\\\{\\n \\\"log_stream_name\\\": \\\"unknown\\\",\\n \\\"logs_collected\\\": \\\\{\\n \\\"files\\\": \\\\{\\n \\\"collect_list\\\": [\\n \\\\{\\n \\\"file_path\\\": \\\"/var/log/runner.log\\\",\\n \\\"log_group_name\\\": \\\"$logGroupName\\\",\\n \\\"log_stream_name\\\": \\\"$runnerNamePath\\\",\\n \\\"timezone\\\": \\\"UTC\\\"\\n \\\\}\\n ]\\n \\\\}\\n \\\\}\\n \\\\}\\n \\\\}\\nEOF\\n /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/tmp/log.conf || exit 2\\n\\\\}\\naction () \\\\{\\n # Determine the value of RUNNER_FLAGS\\n if [ \\\"$(< RUNNER_VERSION)\\\" = \\\"latest\\\" ]; then\\n RUNNER_FLAGS=\\\"\\\"\\n else\\n RUNNER_FLAGS=\\\"--disableupdate\\\"\\n fi\\n\\n labelsTemplate=\\\"$labels,cdkghr:started:$(date +%s)\\\"\\n\\n # Execute the configuration command for runner registration\\n sudo -Hu runner /home/runner/config.sh --unattended --url \\\"$registrationURL\\\" --token \\\"$runnerTokenPath\\\" --ephemeral --work _work --labels \\\"$labelsTemplate\\\" $RUNNER_FLAGS --name \\\"$runnerNamePath\\\" || exit 1\\n\\n # Execute the run command\\n sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2\\n\\n # Retrieve the status\\n STATUS=$(grep -Phors \\\"finish job request for job [0-9a-f\\\\-]+ with result: K.*\\\" /home/runner/_diag/ | tail -n1)\\n\\n # Check and print the job status\\n [ -n \\\"$STATUS\\\" ] && echo CDKGHA JOB DONE \\\"$labels\\\" \\\"$STATUS\\\"\\n\\\\}\\nheartbeat &\\nif setup_logs && action | tee /var/log/runner.log 2>&1; then\\n aws stepfunctions send-task-success --task-token \\\"$TASK_TOKEN\\\" --task-output '\\\\{\\\"ok\\\": true\\\\}'\\nelse\\n aws stepfunctions send-task-failure --task-token \\\"$TASK_TOKEN\\\"\\nfi\\nsleep 10 # give cloudwatch agent its default 5 seconds buffer duration to upload logs\\npoweroff\\n\"},\"Next\":\"ec2, linux, x64 subnet1\"},\"ec2, linux, x64 subnet1\":{\"End\":true,\"Catch\":[{\"ErrorEquals\":[\"Ec2.Ec2Exception\",\"States.Timeout\"],\"ResultPath\":\"$.lastSubnetError\",\"Next\":\"ec2, linux, x64 subnet2\"}],\"Type\":\"Task\",\"Comment\":\"", - { - "Ref": "VpcPublicSubnet1Subnet5C2D37C4" - }, - "\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", + "\"]}},\"Overrides\":{\"ContainerOverrides\":[{\"Name\":\"runner\",\"Environment\":[{\"Name\":\"RUNNER_TOKEN\",\"Value.$\":\"$.runner.token\"},{\"Name\":\"RUNNER_NAME\",\"Value.$\":\"$$.Execution.Name\"},{\"Name\":\"RUNNER_LABEL\",\"Value\":\"fargate,windows,x64\"},{\"Name\":\"GITHUB_DOMAIN\",\"Value.$\":\"$.runner.domain\"},{\"Name\":\"OWNER\",\"Value.$\":\"$.owner\"},{\"Name\":\"REPO\",\"Value.$\":\"$.repo\"},{\"Name\":\"REGISTRATION_URL\",\"Value.$\":\"$.runner.registrationUrl\"}]}]},\"PropagateTags\":\"TASK_DEFINITION\",\"EnableExecuteCommand\":false,\"CapacityProviderStrategy\":[{\"CapacityProvider\":\"FARGATE\"}]}},\"ec2, linux, x64 Get AMI\":{\"Next\":\"ec2, linux, x64 Fleet\",\"Type\":\"Task\",\"ResultPath\":\"$.instanceInput\",\"ResultSelector\":{\"ami.$\":\"$.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"},\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, - ":states:::aws-sdk:ec2:runInstances.waitForTaskToken\",\"Parameters\":{\"LaunchTemplate\":{\"LaunchTemplateId\":\"", + ":states:::aws-sdk:ec2:describeLaunchTemplateVersions\",\"Parameters\":{\"LaunchTemplateId\":\"", { "Ref": "AMILinuxBuilderLaunchtemplateA29452C4" }, - "\"},\"MinCount\":1,\"MaxCount\":1,\"InstanceType\":\"m5.large\",\"UserData.$\":\"States.Base64Encode(States.Format($.ec2.userdataTemplate, $$.Task.Token, '", - { - "Ref": "EC2LinuxLogsC4CD8F14" - }, - "', $$.Execution.Name, $.runner.domain, $.owner, $.repo, $.runner.token, 'ec2,linux,x64', $.runner.registrationUrl))\",\"InstanceInitiatedShutdownBehavior\":\"terminate\",\"IamInstanceProfile\":{\"Arn\":\"", + "\",\"Versions\":[\"$Latest\"]}},\"ec2, linux, x64 Fleet\":{\"Next\":\"ec2, linux, x64 SSM\",\"Type\":\"Task\",\"ResultPath\":\"$.instance\",\"ResultSelector\":{\"id.$\":\"$.Instances[0].InstanceIds[0]\"},\"Resource\":\"arn:", { - "Fn::GetAtt": [ - "EC2LinuxInstanceProfile2D2BB473", - "Arn" - ] + "Ref": "AWS::Partition" }, - "\"},\"MetadataOptions\":{\"HttpTokens\":\"required\"},\"SecurityGroupIds\":[\"", + ":states:::aws-sdk:ec2:createFleet\",\"Parameters\":{\"TargetCapacitySpecification\":{\"TotalTargetCapacity\":1,\"DefaultTargetCapacityType\":\"on-demand\"},\"LaunchTemplateConfigs\":[{\"LaunchTemplateSpecification\":{\"LaunchTemplateId\":\"", { - "Fn::GetAtt": [ - "EC2LinuxSGF5B89300", - "GroupId" - ] + "Ref": "EC2LinuxLaunchTemplate36CD9B92" }, - "\"],\"SubnetId\":\"", + "\",\"Version\":\"$Latest\"},\"Overrides\":[{\"SubnetId\":\"", { "Ref": "VpcPublicSubnet1Subnet5C2D37C4" }, - "\",\"BlockDeviceMappings\":[{\"DeviceName\":\"", - { - "Ref": "EC2LinuxAMIRootDevice26D5E56E" - }, - "\",\"Ebs\":{\"DeleteOnTermination\":true,\"VolumeSize\":30}}]}},\"ec2, linux, x64 subnet2\":{\"End\":true,\"Type\":\"Task\",\"Comment\":\"", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"},{\"SubnetId\":\"", { "Ref": "VpcPublicSubnet2Subnet691E08A3" }, - "\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"}]}],\"Type\":\"instant\"}},\"ec2, linux, x64 SSM\":{\"End\":true,\"Type\":\"Task\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, - ":states:::aws-sdk:ec2:runInstances.waitForTaskToken\",\"Parameters\":{\"LaunchTemplate\":{\"LaunchTemplateId\":\"", - { - "Ref": "AMILinuxBuilderLaunchtemplateA29452C4" - }, - "\"},\"MinCount\":1,\"MaxCount\":1,\"InstanceType\":\"m5.large\",\"UserData.$\":\"States.Base64Encode(States.Format($.ec2.userdataTemplate, $$.Task.Token, '", - { - "Ref": "EC2LinuxLogsC4CD8F14" - }, - "', $$.Execution.Name, $.runner.domain, $.owner, $.repo, $.runner.token, 'ec2,linux,x64', $.runner.registrationUrl))\",\"InstanceInitiatedShutdownBehavior\":\"terminate\",\"IamInstanceProfile\":{\"Arn\":\"", - { - "Fn::GetAtt": [ - "EC2LinuxInstanceProfile2D2BB473", - "Arn" - ] - }, - "\"},\"MetadataOptions\":{\"HttpTokens\":\"required\"},\"SecurityGroupIds\":[\"", - { - "Fn::GetAtt": [ - "EC2LinuxSGF5B89300", - "GroupId" - ] - }, - "\"],\"SubnetId\":\"", - { - "Ref": "VpcPublicSubnet2Subnet691E08A3" - }, - "\",\"BlockDeviceMappings\":[{\"DeviceName\":\"", - { - "Ref": "EC2LinuxAMIRootDevice26D5E56E" - }, - "\",\"Ebs\":{\"DeleteOnTermination\":true,\"VolumeSize\":30}}]}},\"ec2-spot, linux, x64 data\":{\"Type\":\"Pass\",\"ResultPath\":\"$.ec2\",\"Parameters\":{\"userdataTemplate\":\"#!/bin/bash -x\\nTASK_TOKEN=\\\"{}\\\"\\nlogGroupName=\\\"{}\\\"\\nrunnerNamePath=\\\"{}\\\"\\ngithubDomainPath=\\\"{}\\\"\\nownerPath=\\\"{}\\\"\\nrepoPath=\\\"{}\\\"\\nrunnerTokenPath=\\\"{}\\\"\\nlabels=\\\"{}\\\"\\nregistrationURL=\\\"{}\\\"\\n\\nheartbeat () \\\\{\\n while true; do\\n aws stepfunctions send-task-heartbeat --task-token \\\"$TASK_TOKEN\\\"\\n sleep 60\\n done\\n\\\\}\\nsetup_logs () \\\\{\\n cat < /tmp/log.conf || exit 1\\n \\\\{\\n \\\"logs\\\": \\\\{\\n \\\"log_stream_name\\\": \\\"unknown\\\",\\n \\\"logs_collected\\\": \\\\{\\n \\\"files\\\": \\\\{\\n \\\"collect_list\\\": [\\n \\\\{\\n \\\"file_path\\\": \\\"/var/log/runner.log\\\",\\n \\\"log_group_name\\\": \\\"$logGroupName\\\",\\n \\\"log_stream_name\\\": \\\"$runnerNamePath\\\",\\n \\\"timezone\\\": \\\"UTC\\\"\\n \\\\}\\n ]\\n \\\\}\\n \\\\}\\n \\\\}\\n \\\\}\\nEOF\\n /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/tmp/log.conf || exit 2\\n\\\\}\\naction () \\\\{\\n # Determine the value of RUNNER_FLAGS\\n if [ \\\"$(< RUNNER_VERSION)\\\" = \\\"latest\\\" ]; then\\n RUNNER_FLAGS=\\\"\\\"\\n else\\n RUNNER_FLAGS=\\\"--disableupdate\\\"\\n fi\\n\\n labelsTemplate=\\\"$labels,cdkghr:started:$(date +%s)\\\"\\n\\n # Execute the configuration command for runner registration\\n sudo -Hu runner /home/runner/config.sh --unattended --url \\\"$registrationURL\\\" --token \\\"$runnerTokenPath\\\" --ephemeral --work _work --labels \\\"$labelsTemplate\\\" $RUNNER_FLAGS --name \\\"$runnerNamePath\\\" || exit 1\\n\\n # Execute the run command\\n sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2\\n\\n # Retrieve the status\\n STATUS=$(grep -Phors \\\"finish job request for job [0-9a-f\\\\-]+ with result: K.*\\\" /home/runner/_diag/ | tail -n1)\\n\\n # Check and print the job status\\n [ -n \\\"$STATUS\\\" ] && echo CDKGHA JOB DONE \\\"$labels\\\" \\\"$STATUS\\\"\\n\\\\}\\nheartbeat &\\nif setup_logs && action | tee /var/log/runner.log 2>&1; then\\n aws stepfunctions send-task-success --task-token \\\"$TASK_TOKEN\\\" --task-output '\\\\{\\\"ok\\\": true\\\\}'\\nelse\\n aws stepfunctions send-task-failure --task-token \\\"$TASK_TOKEN\\\"\\nfi\\nsleep 10 # give cloudwatch agent its default 5 seconds buffer duration to upload logs\\npoweroff\\n\"},\"Next\":\"ec2-spot, linux, x64 subnet1\"},\"ec2-spot, linux, x64 subnet1\":{\"End\":true,\"Catch\":[{\"ErrorEquals\":[\"Ec2.Ec2Exception\",\"States.Timeout\"],\"ResultPath\":\"$.lastSubnetError\",\"Next\":\"ec2-spot, linux, x64 subnet2\"}],\"Type\":\"Task\",\"Comment\":\"", + ":states:::aws-sdk:ssm:startAutomationExecution.waitForTaskToken\",\"Parameters\":{\"DocumentName\":\"", { - "Ref": "VpcPublicSubnet1Subnet5C2D37C4" + "Ref": "EC2LinuxAutomation2C6F2077" }, - "\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", + "\",\"Parameters\":{\"instanceId.$\":\"States.Array($.instance.id)\",\"taskToken.$\":\"States.Array($$.Task.Token)\",\"runnerName.$\":\"States.Array($$.Execution.Name)\",\"runnerToken.$\":\"States.Array($.runner.token)\",\"labels.$\":\"States.Array('ec2,linux,x64')\",\"registrationUrl.$\":\"States.Array($.runner.registrationUrl)\"}}},\"ec2-spot, linux, x64 Get AMI\":{\"Next\":\"ec2-spot, linux, x64 Fleet\",\"Type\":\"Task\",\"ResultPath\":\"$.instanceInput\",\"ResultSelector\":{\"ami.$\":\"$.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"},\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, - ":states:::aws-sdk:ec2:runInstances.waitForTaskToken\",\"Parameters\":{\"LaunchTemplate\":{\"LaunchTemplateId\":\"", + ":states:::aws-sdk:ec2:describeLaunchTemplateVersions\",\"Parameters\":{\"LaunchTemplateId\":\"", { "Ref": "AMILinuxBuilderLaunchtemplateA29452C4" }, - "\"},\"MinCount\":1,\"MaxCount\":1,\"InstanceType\":\"m5.large\",\"UserData.$\":\"States.Base64Encode(States.Format($.ec2.userdataTemplate, $$.Task.Token, '", - { - "Ref": "EC2SpotLinuxLogsF78D5F0E" - }, - "', $$.Execution.Name, $.runner.domain, $.owner, $.repo, $.runner.token, 'ec2-spot,linux,x64', $.runner.registrationUrl))\",\"InstanceInitiatedShutdownBehavior\":\"terminate\",\"IamInstanceProfile\":{\"Arn\":\"", + "\",\"Versions\":[\"$Latest\"]}},\"ec2-spot, linux, x64 Fleet\":{\"Next\":\"ec2-spot, linux, x64 SSM\",\"Type\":\"Task\",\"ResultPath\":\"$.instance\",\"ResultSelector\":{\"id.$\":\"$.Instances[0].InstanceIds[0]\"},\"Resource\":\"arn:", { - "Fn::GetAtt": [ - "EC2SpotLinuxInstanceProfileB12320D4", - "Arn" - ] + "Ref": "AWS::Partition" }, - "\"},\"MetadataOptions\":{\"HttpTokens\":\"required\"},\"SecurityGroupIds\":[\"", + ":states:::aws-sdk:ec2:createFleet\",\"Parameters\":{\"TargetCapacitySpecification\":{\"TotalTargetCapacity\":1,\"DefaultTargetCapacityType\":\"spot\"},\"LaunchTemplateConfigs\":[{\"LaunchTemplateSpecification\":{\"LaunchTemplateId\":\"", { - "Fn::GetAtt": [ - "EC2SpotLinuxSG8D846B64", - "GroupId" - ] + "Ref": "EC2SpotLinuxLaunchTemplate9ACE58E5" }, - "\"],\"SubnetId\":\"", + "\",\"Version\":\"$Latest\"},\"Overrides\":[{\"SubnetId\":\"", { "Ref": "VpcPublicSubnet1Subnet5C2D37C4" }, - "\",\"BlockDeviceMappings\":[{\"DeviceName\":\"", - { - "Ref": "EC2SpotLinuxAMIRootDeviceC8A06843" - }, - "\",\"Ebs\":{\"DeleteOnTermination\":true,\"VolumeSize\":30}}],\"InstanceMarketOptions\":{\"MarketType\":\"spot\",\"SpotOptions\":{\"SpotInstanceType\":\"one-time\"}}}},\"ec2-spot, linux, x64 subnet2\":{\"End\":true,\"Type\":\"Task\",\"Comment\":\"", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"},{\"SubnetId\":\"", { "Ref": "VpcPublicSubnet2Subnet691E08A3" }, - "\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"}]}],\"Type\":\"instant\"}},\"ec2-spot, linux, x64 SSM\":{\"End\":true,\"Type\":\"Task\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, - ":states:::aws-sdk:ec2:runInstances.waitForTaskToken\",\"Parameters\":{\"LaunchTemplate\":{\"LaunchTemplateId\":\"", - { - "Ref": "AMILinuxBuilderLaunchtemplateA29452C4" - }, - "\"},\"MinCount\":1,\"MaxCount\":1,\"InstanceType\":\"m5.large\",\"UserData.$\":\"States.Base64Encode(States.Format($.ec2.userdataTemplate, $$.Task.Token, '", - { - "Ref": "EC2SpotLinuxLogsF78D5F0E" - }, - "', $$.Execution.Name, $.runner.domain, $.owner, $.repo, $.runner.token, 'ec2-spot,linux,x64', $.runner.registrationUrl))\",\"InstanceInitiatedShutdownBehavior\":\"terminate\",\"IamInstanceProfile\":{\"Arn\":\"", - { - "Fn::GetAtt": [ - "EC2SpotLinuxInstanceProfileB12320D4", - "Arn" - ] - }, - "\"},\"MetadataOptions\":{\"HttpTokens\":\"required\"},\"SecurityGroupIds\":[\"", - { - "Fn::GetAtt": [ - "EC2SpotLinuxSG8D846B64", - "GroupId" - ] - }, - "\"],\"SubnetId\":\"", - { - "Ref": "VpcPublicSubnet2Subnet691E08A3" - }, - "\",\"BlockDeviceMappings\":[{\"DeviceName\":\"", - { - "Ref": "EC2SpotLinuxAMIRootDeviceC8A06843" - }, - "\",\"Ebs\":{\"DeleteOnTermination\":true,\"VolumeSize\":30}}],\"InstanceMarketOptions\":{\"MarketType\":\"spot\",\"SpotOptions\":{\"SpotInstanceType\":\"one-time\"}}}},\"ec2, linux, arm64 data\":{\"Type\":\"Pass\",\"ResultPath\":\"$.ec2\",\"Parameters\":{\"userdataTemplate\":\"#!/bin/bash -x\\nTASK_TOKEN=\\\"{}\\\"\\nlogGroupName=\\\"{}\\\"\\nrunnerNamePath=\\\"{}\\\"\\ngithubDomainPath=\\\"{}\\\"\\nownerPath=\\\"{}\\\"\\nrepoPath=\\\"{}\\\"\\nrunnerTokenPath=\\\"{}\\\"\\nlabels=\\\"{}\\\"\\nregistrationURL=\\\"{}\\\"\\n\\nheartbeat () \\\\{\\n while true; do\\n aws stepfunctions send-task-heartbeat --task-token \\\"$TASK_TOKEN\\\"\\n sleep 60\\n done\\n\\\\}\\nsetup_logs () \\\\{\\n cat < /tmp/log.conf || exit 1\\n \\\\{\\n \\\"logs\\\": \\\\{\\n \\\"log_stream_name\\\": \\\"unknown\\\",\\n \\\"logs_collected\\\": \\\\{\\n \\\"files\\\": \\\\{\\n \\\"collect_list\\\": [\\n \\\\{\\n \\\"file_path\\\": \\\"/var/log/runner.log\\\",\\n \\\"log_group_name\\\": \\\"$logGroupName\\\",\\n \\\"log_stream_name\\\": \\\"$runnerNamePath\\\",\\n \\\"timezone\\\": \\\"UTC\\\"\\n \\\\}\\n ]\\n \\\\}\\n \\\\}\\n \\\\}\\n \\\\}\\nEOF\\n /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/tmp/log.conf || exit 2\\n\\\\}\\naction () \\\\{\\n # Determine the value of RUNNER_FLAGS\\n if [ \\\"$(< RUNNER_VERSION)\\\" = \\\"latest\\\" ]; then\\n RUNNER_FLAGS=\\\"\\\"\\n else\\n RUNNER_FLAGS=\\\"--disableupdate\\\"\\n fi\\n\\n labelsTemplate=\\\"$labels,cdkghr:started:$(date +%s)\\\"\\n\\n # Execute the configuration command for runner registration\\n sudo -Hu runner /home/runner/config.sh --unattended --url \\\"$registrationURL\\\" --token \\\"$runnerTokenPath\\\" --ephemeral --work _work --labels \\\"$labelsTemplate\\\" $RUNNER_FLAGS --name \\\"$runnerNamePath\\\" || exit 1\\n\\n # Execute the run command\\n sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2\\n\\n # Retrieve the status\\n STATUS=$(grep -Phors \\\"finish job request for job [0-9a-f\\\\-]+ with result: K.*\\\" /home/runner/_diag/ | tail -n1)\\n\\n # Check and print the job status\\n [ -n \\\"$STATUS\\\" ] && echo CDKGHA JOB DONE \\\"$labels\\\" \\\"$STATUS\\\"\\n\\\\}\\nheartbeat &\\nif setup_logs && action | tee /var/log/runner.log 2>&1; then\\n aws stepfunctions send-task-success --task-token \\\"$TASK_TOKEN\\\" --task-output '\\\\{\\\"ok\\\": true\\\\}'\\nelse\\n aws stepfunctions send-task-failure --task-token \\\"$TASK_TOKEN\\\"\\nfi\\nsleep 10 # give cloudwatch agent its default 5 seconds buffer duration to upload logs\\npoweroff\\n\"},\"Next\":\"ec2, linux, arm64 subnet1\"},\"ec2, linux, arm64 subnet1\":{\"End\":true,\"Catch\":[{\"ErrorEquals\":[\"Ec2.Ec2Exception\",\"States.Timeout\"],\"ResultPath\":\"$.lastSubnetError\",\"Next\":\"ec2, linux, arm64 subnet2\"}],\"Type\":\"Task\",\"Comment\":\"", + ":states:::aws-sdk:ssm:startAutomationExecution.waitForTaskToken\",\"Parameters\":{\"DocumentName\":\"", { - "Ref": "VpcPublicSubnet1Subnet5C2D37C4" + "Ref": "EC2SpotLinuxAutomation1255B0E6" }, - "\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", + "\",\"Parameters\":{\"instanceId.$\":\"States.Array($.instance.id)\",\"taskToken.$\":\"States.Array($$.Task.Token)\",\"runnerName.$\":\"States.Array($$.Execution.Name)\",\"runnerToken.$\":\"States.Array($.runner.token)\",\"labels.$\":\"States.Array('ec2-spot,linux,x64')\",\"registrationUrl.$\":\"States.Array($.runner.registrationUrl)\"}}},\"ec2, linux, arm64 Get AMI\":{\"Next\":\"ec2, linux, arm64 Fleet\",\"Type\":\"Task\",\"ResultPath\":\"$.instanceInput\",\"ResultSelector\":{\"ami.$\":\"$.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"},\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, - ":states:::aws-sdk:ec2:runInstances.waitForTaskToken\",\"Parameters\":{\"LaunchTemplate\":{\"LaunchTemplateId\":\"", + ":states:::aws-sdk:ec2:describeLaunchTemplateVersions\",\"Parameters\":{\"LaunchTemplateId\":\"", { "Ref": "AMILinuxarm64BuilderLaunchtemplate8F5EFF44" }, - "\"},\"MinCount\":1,\"MaxCount\":1,\"InstanceType\":\"m6g.large\",\"UserData.$\":\"States.Base64Encode(States.Format($.ec2.userdataTemplate, $$.Task.Token, '", - { - "Ref": "EC2Linuxarm64Logs577E371E" - }, - "', $$.Execution.Name, $.runner.domain, $.owner, $.repo, $.runner.token, 'ec2,linux,arm64', $.runner.registrationUrl))\",\"InstanceInitiatedShutdownBehavior\":\"terminate\",\"IamInstanceProfile\":{\"Arn\":\"", + "\",\"Versions\":[\"$Latest\"]}},\"ec2, linux, arm64 Fleet\":{\"Next\":\"ec2, linux, arm64 SSM\",\"Type\":\"Task\",\"ResultPath\":\"$.instance\",\"ResultSelector\":{\"id.$\":\"$.Instances[0].InstanceIds[0]\"},\"Resource\":\"arn:", { - "Fn::GetAtt": [ - "EC2Linuxarm64InstanceProfile1E6F8D53", - "Arn" - ] + "Ref": "AWS::Partition" }, - "\"},\"MetadataOptions\":{\"HttpTokens\":\"required\"},\"SecurityGroupIds\":[\"", + ":states:::aws-sdk:ec2:createFleet\",\"Parameters\":{\"TargetCapacitySpecification\":{\"TotalTargetCapacity\":1,\"DefaultTargetCapacityType\":\"on-demand\"},\"LaunchTemplateConfigs\":[{\"LaunchTemplateSpecification\":{\"LaunchTemplateId\":\"", { - "Fn::GetAtt": [ - "EC2Linuxarm64SG550ECD6C", - "GroupId" - ] + "Ref": "EC2Linuxarm64LaunchTemplateDBA523C3" }, - "\"],\"SubnetId\":\"", + "\",\"Version\":\"$Latest\"},\"Overrides\":[{\"SubnetId\":\"", { "Ref": "VpcPublicSubnet1Subnet5C2D37C4" }, - "\",\"BlockDeviceMappings\":[{\"DeviceName\":\"", - { - "Ref": "EC2Linuxarm64AMIRootDevice3046D37D" - }, - "\",\"Ebs\":{\"DeleteOnTermination\":true,\"VolumeSize\":30}}]}},\"ec2, linux, arm64 subnet2\":{\"End\":true,\"Type\":\"Task\",\"Comment\":\"", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"},{\"SubnetId\":\"", { "Ref": "VpcPublicSubnet2Subnet691E08A3" }, - "\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"}]}],\"Type\":\"instant\"}},\"ec2, linux, arm64 SSM\":{\"End\":true,\"Type\":\"Task\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, - ":states:::aws-sdk:ec2:runInstances.waitForTaskToken\",\"Parameters\":{\"LaunchTemplate\":{\"LaunchTemplateId\":\"", - { - "Ref": "AMILinuxarm64BuilderLaunchtemplate8F5EFF44" - }, - "\"},\"MinCount\":1,\"MaxCount\":1,\"InstanceType\":\"m6g.large\",\"UserData.$\":\"States.Base64Encode(States.Format($.ec2.userdataTemplate, $$.Task.Token, '", - { - "Ref": "EC2Linuxarm64Logs577E371E" - }, - "', $$.Execution.Name, $.runner.domain, $.owner, $.repo, $.runner.token, 'ec2,linux,arm64', $.runner.registrationUrl))\",\"InstanceInitiatedShutdownBehavior\":\"terminate\",\"IamInstanceProfile\":{\"Arn\":\"", + ":states:::aws-sdk:ssm:startAutomationExecution.waitForTaskToken\",\"Parameters\":{\"DocumentName\":\"", { - "Fn::GetAtt": [ - "EC2Linuxarm64InstanceProfile1E6F8D53", - "Arn" - ] - }, - "\"},\"MetadataOptions\":{\"HttpTokens\":\"required\"},\"SecurityGroupIds\":[\"", - { - "Fn::GetAtt": [ - "EC2Linuxarm64SG550ECD6C", - "GroupId" - ] - }, - "\"],\"SubnetId\":\"", - { - "Ref": "VpcPublicSubnet2Subnet691E08A3" - }, - "\",\"BlockDeviceMappings\":[{\"DeviceName\":\"", - { - "Ref": "EC2Linuxarm64AMIRootDevice3046D37D" - }, - "\",\"Ebs\":{\"DeleteOnTermination\":true,\"VolumeSize\":30}}]}},\"ec2, windows, x64 data\":{\"Type\":\"Pass\",\"ResultPath\":\"$.ec2\",\"Parameters\":{\"userdataTemplate\":\"\\n$TASK_TOKEN = \\\"{}\\\"\\n$logGroupName=\\\"{}\\\"\\n$runnerNamePath=\\\"{}\\\"\\n$githubDomainPath=\\\"{}\\\"\\n$ownerPath=\\\"{}\\\"\\n$repoPath=\\\"{}\\\"\\n$runnerTokenPath=\\\"{}\\\"\\n$labels=\\\"{}\\\"\\n$registrationURL=\\\"{}\\\"\\n\\nStart-Job -ScriptBlock \\\\{\\n while (1) \\\\{\\n aws stepfunctions send-task-heartbeat --task-token \\\"$using:TASK_TOKEN\\\"\\n sleep 60\\n \\\\}\\n\\\\}\\nfunction setup_logs () \\\\{\\n echo '\\\\{\\n \\\"logs\\\": \\\\{\\n \\\"log_stream_name\\\": \\\"unknown\\\",\\n \\\"logs_collected\\\": \\\\{\\n \\\"files\\\": \\\\{\\n \\\"collect_list\\\": [\\n \\\\{\\n \\\"file_path\\\": \\\"/actions/runner.log\\\",\\n \\\"log_group_name\\\": \\\"$logGroupName\\\",\\n \\\"log_stream_name\\\": \\\"$runnerNamePath\\\",\\n \\\"timezone\\\": \\\"UTC\\\"\\n \\\\}\\n ]\\n \\\\}\\n \\\\}\\n \\\\}\\n \\\\}' | Out-File -Encoding ASCII $Env:TEMP/log.conf\\n & \\\"C:/Program Files/Amazon/AmazonCloudWatchAgent/amazon-cloudwatch-agent-ctl.ps1\\\" -a fetch-config -m ec2 -s -c file:$Env:TEMP/log.conf\\n\\\\}\\nfunction action () \\\\{\\n cd /actions\\n $RunnerVersion = Get-Content RUNNER_VERSION -Raw\\n if ($RunnerVersion -eq \\\"latest\\\") \\\\{ $RunnerFlags = \\\"\\\" \\\\} else \\\\{ $RunnerFlags = \\\"--disableupdate\\\" \\\\}\\n ./config.cmd --unattended --url \\\"$\\\\{registrationUrl\\\\}\\\" --token \\\"$\\\\{runnerTokenPath\\\\}\\\" --ephemeral --work _work --labels \\\"$\\\\{labels\\\\},cdkghr:started:$(Get-Date -UFormat +%s)\\\" $RunnerFlags --name \\\"$\\\\{runnerNamePath\\\\}\\\" 2>&1 | Out-File -Encoding ASCII -Append /actions/runner.log\\n\\n if ($LASTEXITCODE -ne 0) \\\\{ return 1 \\\\}\\n ./run.cmd 2>&1 | Out-File -Encoding ASCII -Append /actions/runner.log\\n if ($LASTEXITCODE -ne 0) \\\\{ return 2 \\\\}\\n\\n $STATUS = Select-String -Path './_diag/*.log' -Pattern 'finish job request for job [0-9a-f\\\\-]+ with result: (.*)' | %\\\\{$_.Matches.Groups[1].Value\\\\} | Select-Object -Last 1\\n\\n if ($STATUS) \\\\{\\n echo \\\"CDKGHA JOB DONE $\\\\{labels\\\\} $STATUS\\\" | Out-File -Encoding ASCII -Append /actions/runner.log\\n \\\\}\\n\\n return 0\\n\\n\\\\}\\nsetup_logs\\n$r = action\\nif ($r -eq 0) \\\\{\\n aws stepfunctions send-task-success --task-token \\\"$TASK_TOKEN\\\" --task-output '\\\\{ \\\\}'\\n\\\\} else \\\\{\\n aws stepfunctions send-task-failure --task-token \\\"$TASK_TOKEN\\\"\\n\\\\}\\nStart-Sleep -Seconds 10 # give cloudwatch agent its default 5 seconds buffer duration to upload logs\\nStop-Computer -ComputerName localhost -Force\\n\\n\"},\"Next\":\"ec2, windows, x64 subnet1\"},\"ec2, windows, x64 subnet1\":{\"End\":true,\"Catch\":[{\"ErrorEquals\":[\"Ec2.Ec2Exception\",\"States.Timeout\"],\"ResultPath\":\"$.lastSubnetError\",\"Next\":\"ec2, windows, x64 subnet2\"}],\"Type\":\"Task\",\"Comment\":\"", - { - "Ref": "VpcPublicSubnet1Subnet5C2D37C4" + "Ref": "EC2Linuxarm64Automation0232B49F" }, - "\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", + "\",\"Parameters\":{\"instanceId.$\":\"States.Array($.instance.id)\",\"taskToken.$\":\"States.Array($$.Task.Token)\",\"runnerName.$\":\"States.Array($$.Execution.Name)\",\"runnerToken.$\":\"States.Array($.runner.token)\",\"labels.$\":\"States.Array('ec2,linux,arm64')\",\"registrationUrl.$\":\"States.Array($.runner.registrationUrl)\"}}},\"ec2, windows, x64 Get AMI\":{\"Next\":\"ec2, windows, x64 Fleet\",\"Type\":\"Task\",\"ResultPath\":\"$.instanceInput\",\"ResultSelector\":{\"ami.$\":\"$.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"},\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, - ":states:::aws-sdk:ec2:runInstances.waitForTaskToken\",\"Parameters\":{\"LaunchTemplate\":{\"LaunchTemplateId\":\"", + ":states:::aws-sdk:ec2:describeLaunchTemplateVersions\",\"Parameters\":{\"LaunchTemplateId\":\"", { "Ref": "WindowsEC2BuilderLaunchtemplate0A66E9C2" }, - "\"},\"MinCount\":1,\"MaxCount\":1,\"InstanceType\":\"m5.large\",\"UserData.$\":\"States.Base64Encode(States.Format($.ec2.userdataTemplate, $$.Task.Token, '", - { - "Ref": "EC2WindowsLogsDC1F2ABF" - }, - "', $$.Execution.Name, $.runner.domain, $.owner, $.repo, $.runner.token, 'ec2,windows,x64', $.runner.registrationUrl))\",\"InstanceInitiatedShutdownBehavior\":\"terminate\",\"IamInstanceProfile\":{\"Arn\":\"", + "\",\"Versions\":[\"$Latest\"]}},\"ec2, windows, x64 Fleet\":{\"Next\":\"ec2, windows, x64 SSM\",\"Type\":\"Task\",\"ResultPath\":\"$.instance\",\"ResultSelector\":{\"id.$\":\"$.Instances[0].InstanceIds[0]\"},\"Resource\":\"arn:", { - "Fn::GetAtt": [ - "EC2WindowsInstanceProfileDCA59D9C", - "Arn" - ] + "Ref": "AWS::Partition" }, - "\"},\"MetadataOptions\":{\"HttpTokens\":\"required\"},\"SecurityGroupIds\":[\"", + ":states:::aws-sdk:ec2:createFleet\",\"Parameters\":{\"TargetCapacitySpecification\":{\"TotalTargetCapacity\":1,\"DefaultTargetCapacityType\":\"on-demand\"},\"LaunchTemplateConfigs\":[{\"LaunchTemplateSpecification\":{\"LaunchTemplateId\":\"", { - "Fn::GetAtt": [ - "EC2WindowsSG13E24976", - "GroupId" - ] + "Ref": "EC2WindowsLaunchTemplateCF55C6AE" }, - "\"],\"SubnetId\":\"", + "\",\"Version\":\"$Latest\"},\"Overrides\":[{\"SubnetId\":\"", { "Ref": "VpcPublicSubnet1Subnet5C2D37C4" }, - "\",\"BlockDeviceMappings\":[{\"DeviceName\":\"", - { - "Ref": "EC2WindowsAMIRootDevice9FFC971A" - }, - "\",\"Ebs\":{\"DeleteOnTermination\":true,\"VolumeSize\":30}}]}},\"ec2, windows, x64 subnet2\":{\"End\":true,\"Type\":\"Task\",\"Comment\":\"", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"},{\"SubnetId\":\"", { "Ref": "VpcPublicSubnet2Subnet691E08A3" }, - "\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"}]}],\"Type\":\"instant\"}},\"ec2, windows, x64 SSM\":{\"End\":true,\"Type\":\"Task\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, - ":states:::aws-sdk:ec2:runInstances.waitForTaskToken\",\"Parameters\":{\"LaunchTemplate\":{\"LaunchTemplateId\":\"", - { - "Ref": "WindowsEC2BuilderLaunchtemplate0A66E9C2" - }, - "\"},\"MinCount\":1,\"MaxCount\":1,\"InstanceType\":\"m5.large\",\"UserData.$\":\"States.Base64Encode(States.Format($.ec2.userdataTemplate, $$.Task.Token, '", - { - "Ref": "EC2WindowsLogsDC1F2ABF" - }, - "', $$.Execution.Name, $.runner.domain, $.owner, $.repo, $.runner.token, 'ec2,windows,x64', $.runner.registrationUrl))\",\"InstanceInitiatedShutdownBehavior\":\"terminate\",\"IamInstanceProfile\":{\"Arn\":\"", - { - "Fn::GetAtt": [ - "EC2WindowsInstanceProfileDCA59D9C", - "Arn" - ] - }, - "\"},\"MetadataOptions\":{\"HttpTokens\":\"required\"},\"SecurityGroupIds\":[\"", - { - "Fn::GetAtt": [ - "EC2WindowsSG13E24976", - "GroupId" - ] - }, - "\"],\"SubnetId\":\"", - { - "Ref": "VpcPublicSubnet2Subnet691E08A3" - }, - "\",\"BlockDeviceMappings\":[{\"DeviceName\":\"", + ":states:::aws-sdk:ssm:startAutomationExecution.waitForTaskToken\",\"Parameters\":{\"DocumentName\":\"", { - "Ref": "EC2WindowsAMIRootDevice9FFC971A" + "Ref": "EC2WindowsAutomationC493D3FB" }, - "\",\"Ebs\":{\"DeleteOnTermination\":true,\"VolumeSize\":30}}]}}}}]},\"Delete Failed Runner\":{\"End\":true,\"Retry\":[{\"ErrorEquals\":[\"Lambda.ClientExecutionTimeoutException\",\"Lambda.ServiceException\",\"Lambda.AWSLambdaException\",\"Lambda.SdkClientException\"],\"IntervalSeconds\":2,\"MaxAttempts\":6,\"BackoffRate\":2},{\"ErrorEquals\":[\"RunnerBusy\"],\"IntervalSeconds\":60,\"MaxAttempts\":60,\"BackoffRate\":1}],\"Type\":\"Task\",\"ResultPath\":\"$.delete\",\"Resource\":\"", + "\",\"Parameters\":{\"instanceId.$\":\"States.Array($.instance.id)\",\"taskToken.$\":\"States.Array($$.Task.Token)\",\"runnerName.$\":\"States.Array($$.Execution.Name)\",\"runnerToken.$\":\"States.Array($.runner.token)\",\"labels.$\":\"States.Array('ec2,windows,x64')\",\"registrationUrl.$\":\"States.Array($.runner.registrationUrl)\"}}}}}]},\"Delete Failed Runner\":{\"End\":true,\"Retry\":[{\"ErrorEquals\":[\"Lambda.ClientExecutionTimeoutException\",\"Lambda.ServiceException\",\"Lambda.AWSLambdaException\",\"Lambda.SdkClientException\"],\"IntervalSeconds\":2,\"MaxAttempts\":6,\"BackoffRate\":2},{\"ErrorEquals\":[\"RunnerBusy\"],\"IntervalSeconds\":60,\"MaxAttempts\":60,\"BackoffRate\":1}],\"Type\":\"Task\",\"ResultPath\":\"$.delete\",\"Resource\":\"", { "Fn::GetAtt": [ "runnersdeleterunner7F8D5293", @@ -19020,115 +20459,6 @@ "Name": "GitHub Runners/Webhook started runners", "QueryString": "fields @timestamp, @message\n| filter jobUrl like /http.*/\n| sort @timestamp desc\n| limit 100" } - }, - "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1ServiceRole69906A36": { - "Type": "AWS::IAM::Role", - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "lambda.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "ManagedPolicyArns": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - ] - ] - } - ] - } - }, - "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1ServiceRoleDefaultPolicyDC1762B6": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "ssm:GetParameter", - "ec2:DescribeImages", - "ec2:DescribeLaunchTemplateVersions" - ], - "Effect": "Allow", - "Resource": "*" - } - ], - "Version": "2012-10-17" - }, - "PolicyName": "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1ServiceRoleDefaultPolicyDC1762B6", - "Roles": [ - { - "Ref": "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1ServiceRole69906A36" - } - ] - } - }, - "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1C64D247C": { - "Type": "AWS::Lambda::Function", - "Properties": { - "Code": { - "S3Bucket": { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "S3Key": "97a52dd61d3f204877c7a944f14423bdbdec7587794ab63213506581ffb19976.zip" - }, - "Description": "Custom resource handler that triggers CodeBuild to build runner images, and cleans-up images on deletion", - "Environment": { - "Variables": { - "AWS_NODEJS_CONNECTION_REUSE_ENABLED": "1" - } - }, - "Handler": "index.handler", - "Role": { - "Fn::GetAtt": [ - "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1ServiceRole69906A36", - "Arn" - ] - }, - "Runtime": "nodejs18.x", - "Timeout": 60 - }, - "DependsOn": [ - "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1ServiceRoleDefaultPolicyDC1762B6", - "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1ServiceRole69906A36" - ] - }, - "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1LogRetentionFC67AFCF": { - "Type": "Custom::LogRetention", - "Properties": { - "ServiceToken": { - "Fn::GetAtt": [ - "LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8aFD4BFC8A", - "Arn" - ] - }, - "LogGroupName": { - "Fn::Join": [ - "", - [ - "/aws/lambda/", - { - "Ref": "AMIRootDeviceReaderdcc036c8876b451ea2c1552f9e06e9e1C64D247C" - } - ] - ] - }, - "RetentionInDays": 30 - } } }, "Parameters": { From afe170dfa9324592769620eec2d53510d755f046 Mon Sep 17 00:00:00 2001 From: Amir Szekely Date: Sun, 21 Apr 2024 18:21:27 -0400 Subject: [PATCH 08/11] oops that's gone --- test/imagebuilder.test.ts | 47 +-------------------------------------- 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/test/imagebuilder.test.ts b/test/imagebuilder.test.ts index f0ef3f07..53c417f5 100644 --- a/test/imagebuilder.test.ts +++ b/test/imagebuilder.test.ts @@ -1,5 +1,5 @@ import * as cdk from 'aws-cdk-lib'; -import { aws_ec2 as ec2, CfnElement } from 'aws-cdk-lib'; +import { aws_ec2 as ec2 } from 'aws-cdk-lib'; import { Match, Template } from 'aws-cdk-lib/assertions'; import { AmiBuilder, @@ -338,48 +338,3 @@ test('Unused builder doesn\'t throw exceptions', () => { app.synth(); }); - -test('Adding more launch templates to the same builder', () => { - const app = new cdk.App(); - const stack = new cdk.Stack(app, 'test'); - - const vpc = new ec2.Vpc(stack, 'vpc'); - - const builder = Ec2RunnerProvider.imageBuilder(stack, 'builder', { vpc }); - - const boundAmi = builder.bindAmi(); - const lt1 = new ec2.LaunchTemplate(stack, 'lt1'); - const lt2 = new ec2.LaunchTemplate(stack, 'lt2'); - - builder.bindAmi(); - builder.bindAmi({ launchTemplate: lt1 }); - builder.bindAmi({ launchTemplate: lt2 }); - - const builder2 = Ec2RunnerProvider.imageBuilder(stack, 'builder2', { vpc }); - const boundAmi2 = builder2.bindAmi({ launchTemplate: lt1 }); - - const template = Template.fromStack(stack); - - template.hasResourceProperties('AWS::ImageBuilder::DistributionConfiguration', { - Distributions: [ - { - LaunchTemplateConfigurations: [ - { LaunchTemplateId: { Ref: stack.getLogicalId(boundAmi.launchTemplate.node.defaultChild as CfnElement) } }, - { LaunchTemplateId: { Ref: stack.getLogicalId(lt1.node.defaultChild as CfnElement) } }, - { LaunchTemplateId: { Ref: stack.getLogicalId(lt2.node.defaultChild as CfnElement) } }, - ], - }, - ], - }); - - template.hasResourceProperties('AWS::ImageBuilder::DistributionConfiguration', { - Distributions: [ - { - LaunchTemplateConfigurations: [ - { LaunchTemplateId: { Ref: stack.getLogicalId(boundAmi2.launchTemplate.node.defaultChild as CfnElement) } }, - { LaunchTemplateId: { Ref: stack.getLogicalId(lt1.node.defaultChild as CfnElement) } }, - ], - }, - ], - }); -}); From 1cbb93c499a1dc194555d4bca686c6964eef0371 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 21 Apr 2024 22:24:48 +0000 Subject: [PATCH 09/11] chore: self mutation Signed-off-by: github-actions --- API.md | 278 +-------------------------------------------------------- 1 file changed, 1 insertion(+), 277 deletions(-) diff --git a/API.md b/API.md index 3ed289bb..e3658f52 100644 --- a/API.md +++ b/API.md @@ -1467,282 +1467,6 @@ public readonly repository: IRepository; --- -### Ec2Runner - -#### Initializers - -```typescript -import { Ec2Runner } from '@cloudsnorkel/cdk-github-runners' - -new Ec2Runner(scope: Construct, id: string, props?: Ec2RunnerProviderProps) -``` - -| **Name** | **Type** | **Description** | -| --- | --- | --- | -| scope | constructs.Construct | *No description.* | -| id | string | *No description.* | -| props | Ec2RunnerProviderProps | *No description.* | - ---- - -##### `scope`Required - -- *Type:* constructs.Construct - ---- - -##### `id`Required - -- *Type:* string - ---- - -##### `props`Optional - -- *Type:* Ec2RunnerProviderProps - ---- - -#### Methods - -| **Name** | **Description** | -| --- | --- | -| toString | Returns a string representation of this construct. | -| getStepFunctionTask | Generate step function task(s) to start a new runner. | -| grantStateMachine | An optional method that modifies the role of the state machine after all the tasks have been generated. | -| status | Return status of the runner provider to be used in the main status function. | - ---- - -##### ~~`toString`~~ - -```typescript -public toString(): string -``` - -Returns a string representation of this construct. - -##### ~~`getStepFunctionTask`~~ - -```typescript -public getStepFunctionTask(parameters: RunnerRuntimeParameters): IChainable -``` - -Generate step function task(s) to start a new runner. - -Called by GithubRunners and shouldn't be called manually. - -###### `parameters`Required - -- *Type:* RunnerRuntimeParameters - -workflow job details. - ---- - -##### ~~`grantStateMachine`~~ - -```typescript -public grantStateMachine(stateMachineRole: IGrantable): void -``` - -An optional method that modifies the role of the state machine after all the tasks have been generated. - -This can be used to add additional policy -statements to the state machine role that are not automatically added by the task returned from {@link getStepFunctionTask}. - -###### `stateMachineRole`Required - -- *Type:* aws-cdk-lib.aws_iam.IGrantable - ---- - -##### ~~`status`~~ - -```typescript -public status(statusFunctionRole: IGrantable): IRunnerProviderStatus -``` - -Return status of the runner provider to be used in the main status function. - -Also gives the status function any needed permissions to query the Docker image or AMI. - -###### `statusFunctionRole`Required - -- *Type:* aws-cdk-lib.aws_iam.IGrantable - ---- - -#### Static Functions - -| **Name** | **Description** | -| --- | --- | -| isConstruct | Checks if `x` is a construct. | -| imageBuilder | Create new image builder that builds EC2 specific runner images. | - ---- - -##### ~~`isConstruct`~~ - -```typescript -import { Ec2Runner } from '@cloudsnorkel/cdk-github-runners' - -Ec2Runner.isConstruct(x: any) -``` - -Checks if `x` is a construct. - -###### `x`Required - -- *Type:* any - -Any object. - ---- - -##### ~~`imageBuilder`~~ - -```typescript -import { Ec2Runner } from '@cloudsnorkel/cdk-github-runners' - -Ec2Runner.imageBuilder(scope: Construct, id: string, props?: RunnerImageBuilderProps) -``` - -Create new image builder that builds EC2 specific runner images. - -You can customize the OS, architecture, VPC, subnet, security groups, etc. by passing in props. - -You can add components to the image builder by calling `imageBuilder.addComponent()`. - -The default OS is Ubuntu running on x64 architecture. - -Included components: - * `RunnerImageComponent.requiredPackages()` - * `RunnerImageComponent.runnerUser()` - * `RunnerImageComponent.git()` - * `RunnerImageComponent.githubCli()` - * `RunnerImageComponent.awsCli()` - * `RunnerImageComponent.docker()` - * `RunnerImageComponent.githubRunner()` - -###### `scope`Required - -- *Type:* constructs.Construct - ---- - -###### `id`Required - -- *Type:* string - ---- - -###### `props`Optional - -- *Type:* RunnerImageBuilderProps - ---- - -#### Properties - -| **Name** | **Type** | **Description** | -| --- | --- | --- | -| node | constructs.Node | The tree node. | -| connections | aws-cdk-lib.aws_ec2.Connections | The network connections associated with this resource. | -| grantPrincipal | aws-cdk-lib.aws_iam.IPrincipal | Grant principal used to add permissions to the runner role. | -| labels | string[] | Labels associated with this provider. | -| logGroup | aws-cdk-lib.aws_logs.ILogGroup | Log group where provided runners will save their logs. | -| retryableErrors | string[] | List of step functions errors that should be retried. | - ---- - -##### ~~`node`~~Required - -- *Deprecated:* use {@link Ec2RunnerProvider } - -```typescript -public readonly node: Node; -``` - -- *Type:* constructs.Node - -The tree node. - ---- - -##### ~~`connections`~~Required - -- *Deprecated:* use {@link Ec2RunnerProvider } - -```typescript -public readonly connections: Connections; -``` - -- *Type:* aws-cdk-lib.aws_ec2.Connections - -The network connections associated with this resource. - ---- - -##### ~~`grantPrincipal`~~Required - -- *Deprecated:* use {@link Ec2RunnerProvider } - -```typescript -public readonly grantPrincipal: IPrincipal; -``` - -- *Type:* aws-cdk-lib.aws_iam.IPrincipal - -Grant principal used to add permissions to the runner role. - ---- - -##### ~~`labels`~~Required - -- *Deprecated:* use {@link Ec2RunnerProvider } - -```typescript -public readonly labels: string[]; -``` - -- *Type:* string[] - -Labels associated with this provider. - ---- - -##### ~~`logGroup`~~Required - -- *Deprecated:* use {@link Ec2RunnerProvider } - -```typescript -public readonly logGroup: ILogGroup; -``` - -- *Type:* aws-cdk-lib.aws_logs.ILogGroup - -Log group where provided runners will save their logs. - -Note that this is not the job log, but the runner itself. It will not contain output from the GitHub Action but only metadata on its execution. - ---- - -##### ~~`retryableErrors`~~Required - -- *Deprecated:* use {@link Ec2RunnerProvider } - -```typescript -public readonly retryableErrors: string[]; -``` - -- *Type:* string[] - -List of step functions errors that should be retried. - ---- - - ### Ec2RunnerProvider - *Implements:* IRunnerProvider @@ -9847,7 +9571,7 @@ Log group name for the image builder where history of image builds can be analyz - *Extends:* aws-cdk-lib.aws_ec2.IConnectable, aws-cdk-lib.aws_iam.IGrantable, constructs.IConstruct -- *Implemented By:* CodeBuildRunner, CodeBuildRunnerProvider, Ec2Runner, Ec2RunnerProvider, EcsRunnerProvider, FargateRunner, FargateRunnerProvider, LambdaRunner, LambdaRunnerProvider, IRunnerProvider +- *Implemented By:* CodeBuildRunner, CodeBuildRunnerProvider, Ec2RunnerProvider, EcsRunnerProvider, FargateRunner, FargateRunnerProvider, LambdaRunner, LambdaRunnerProvider, IRunnerProvider Interface for all runner providers. From 042c6ced852028a4ab43b633851e5b17d929f355 Mon Sep 17 00:00:00 2001 From: Amir Szekely Date: Sun, 21 Apr 2024 18:30:55 -0400 Subject: [PATCH 10/11] missing permissions --- src/providers/ec2.ts | 18 ++++-- .../github-runners-test.assets.json | 4 +- .../github-runners-test.template.json | 61 +++++++++++++------ 3 files changed, 56 insertions(+), 27 deletions(-) diff --git a/src/providers/ec2.ts b/src/providers/ec2.ts index eb42caa2..8b5a3143 100644 --- a/src/providers/ec2.ts +++ b/src/providers/ec2.ts @@ -583,6 +583,7 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { iamResources: ['*'], resultPath: stepfunctions.JsonPath.stringAt('$.instance'), resultSelector: { + // TODO retry on this failing? if the call fails, there is nothing here 'id.$': '$.Instances[0].InstanceIds[0]', }, }); @@ -623,11 +624,18 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { })); stateMachineRole.grantPrincipal.addToPrincipalPolicy(new iam.PolicyStatement({ - actions: ['ec2:createTags'], - resources: [cdk.Stack.of(this).formatArn({ - service: 'ec2', - resource: '*', - })], + actions: ['ec2:CreateTags', 'ec2:RunInstances'], + resources: [ + cdk.Stack.of(this).formatArn({ + service: 'ec2', + resource: '*', + }), + cdk.Stack.of(this).formatArn({ + service: 'ec2', + account: '', + resource: 'image/*', + }), + ], })); stateMachineRole.grantPrincipal.addToPrincipalPolicy(new iam.PolicyStatement({ diff --git a/test/default.integ.snapshot/github-runners-test.assets.json b/test/default.integ.snapshot/github-runners-test.assets.json index cd88a6c2..775d83f0 100644 --- a/test/default.integ.snapshot/github-runners-test.assets.json +++ b/test/default.integ.snapshot/github-runners-test.assets.json @@ -235,7 +235,7 @@ } } }, - "4b0524dac32ebcde2edd5ab38a5c92126a5e899e826a24f533713795a45390a3": { + "3341cfa842c8d99e84b9c4532dba2cb1d86ab7edf660ebe2a4838462913a02cd": { "source": { "path": "github-runners-test.template.json", "packaging": "file" @@ -243,7 +243,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "4b0524dac32ebcde2edd5ab38a5c92126a5e899e826a24f533713795a45390a3.json", + "objectKey": "3341cfa842c8d99e84b9c4532dba2cb1d86ab7edf660ebe2a4838462913a02cd.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/test/default.integ.snapshot/github-runners-test.template.json b/test/default.integ.snapshot/github-runners-test.template.json index f854332d..26657f5a 100644 --- a/test/default.integ.snapshot/github-runners-test.template.json +++ b/test/default.integ.snapshot/github-runners-test.template.json @@ -17726,28 +17726,49 @@ } }, { - "Action": "ec2:createTags", + "Action": [ + "ec2:CreateTags", + "ec2:RunInstances" + ], "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":ec2:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":*" + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":*" + ] ] - ] - } + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + "::image/*" + ] + ] + } + ] }, { "Action": "iam:CreateServiceLinkedRole", From 9dd8b34ee4e9aeea07f989f8bf07713fb2d86135 Mon Sep 17 00:00:00 2001 From: Amir Szekely Date: Thu, 25 Apr 2024 21:38:22 -0400 Subject: [PATCH 11/11] fix error handling, linux never stopping, linux status metric filter --- src/providers/ec2.ts | 19 +++---- .../github-runners-test.assets.json | 4 +- .../github-runners-test.template.json | 50 ++++++++++--------- 3 files changed, 36 insertions(+), 37 deletions(-) diff --git a/src/providers/ec2.ts b/src/providers/ec2.ts index 8b5a3143..21acc568 100644 --- a/src/providers/ec2.ts +++ b/src/providers/ec2.ts @@ -399,6 +399,8 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { DocumentName: this.ami.os.is(Os.WINDOWS) ? 'AWS-RunPowerShellScript' : 'AWS-RunShellScript', InstanceIds: ['{{ instanceId }}'], Parameters: { + // TODO let ec2 instance send success/failure as ssm document is time limited + // TODO also terminate from step function on failure // TODO executionTimeout: '0', // no timeout workingDirectory: this.ami.os.is(Os.WINDOWS) ? 'C:\\actions' : '/home/runner', commands: this.ami.os.is(Os.WINDOWS) ? [ // *** windows @@ -436,7 +438,7 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { aws stepfunctions send-task-heartbeat --task-token "{{ taskToken }}" sleep 60 done - } &`, + } >/dev/null 2>&1 &`, // decide if we should update runner `if [ "$(cat RUNNER_VERSION)" = "latest" ]; then RUNNER_FLAGS="" @@ -449,7 +451,7 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { 'sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2', // print whether job was successful for our metric filter `STATUS=$(grep -Phors "finish job request for job [0-9a-f\\-]+ with result: \\K.*" /home/runner/_diag/ | tail -n1) - [ -n "$STATUS" ] && echo CDKGHA JOB DONE "$labels" "$STATUS"`, + [ -n "$STATUS" ] && echo CDKGHA JOB DONE "{{ labels }}" "$STATUS"`, ], }, CloudWatchOutputConfig: { @@ -549,10 +551,7 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { Versions: ['$Latest'], }, iamResources: ['*'], - resultPath: stepfunctions.JsonPath.stringAt('$.instanceInput'), - resultSelector: { - 'ami.$': '$.LaunchTemplateVersions[0].LaunchTemplateData.ImageId', - }, + resultPath: stepfunctions.JsonPath.stringAt('$.ami'), }); // create fleet with override per subnet @@ -574,7 +573,7 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { return { SubnetId: subnet.subnetId, WeightedCapacity: 1, - ImageId: stepfunctions.JsonPath.stringAt('$.instanceInput.ami'), + ImageId: stepfunctions.JsonPath.stringAt('$.ami.LaunchTemplateVersions[0].LaunchTemplateData.ImageId'), }; }), }], @@ -582,10 +581,6 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { }, iamResources: ['*'], resultPath: stepfunctions.JsonPath.stringAt('$.instance'), - resultSelector: { - // TODO retry on this failing? if the call fails, there is nothing here - 'id.$': '$.Instances[0].InstanceIds[0]', - }, }); // use ssm to start runner in newly launched instance @@ -598,7 +593,7 @@ export class Ec2RunnerProvider extends BaseProvider implements IRunnerProvider { parameters: { DocumentName: this.document.ref, Parameters: { - instanceId: stepfunctions.JsonPath.array(stepfunctions.JsonPath.stringAt('$.instance.id')), + instanceId: stepfunctions.JsonPath.array(stepfunctions.JsonPath.stringAt('$.instance.Instances[0].InstanceIds[0]')), taskToken: stepfunctions.JsonPath.array(stepfunctions.JsonPath.taskToken), runnerName: stepfunctions.JsonPath.array(parameters.runnerNamePath), runnerToken: stepfunctions.JsonPath.array(parameters.runnerTokenPath), diff --git a/test/default.integ.snapshot/github-runners-test.assets.json b/test/default.integ.snapshot/github-runners-test.assets.json index 775d83f0..02b0cd2c 100644 --- a/test/default.integ.snapshot/github-runners-test.assets.json +++ b/test/default.integ.snapshot/github-runners-test.assets.json @@ -235,7 +235,7 @@ } } }, - "3341cfa842c8d99e84b9c4532dba2cb1d86ab7edf660ebe2a4838462913a02cd": { + "9d5cb37590fe22ae742baf8430ac7240db2041d276aa965cc71c114180251ece": { "source": { "path": "github-runners-test.template.json", "packaging": "file" @@ -243,7 +243,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "3341cfa842c8d99e84b9c4532dba2cb1d86ab7edf660ebe2a4838462913a02cd.json", + "objectKey": "9d5cb37590fe22ae742baf8430ac7240db2041d276aa965cc71c114180251ece.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/test/default.integ.snapshot/github-runners-test.template.json b/test/default.integ.snapshot/github-runners-test.template.json index 26657f5a..e7ecd13f 100644 --- a/test/default.integ.snapshot/github-runners-test.template.json +++ b/test/default.integ.snapshot/github-runners-test.template.json @@ -13839,15 +13839,16 @@ "{{ instanceId }}" ], "Parameters": { + "executionTimeout": "30", "workingDirectory": "/home/runner", "commands": [ "set -ex", "touch /home/runner/STARTED", - "{\n while true; do\n aws stepfunctions send-task-heartbeat --task-token \"{{ taskToken }}\"\n sleep 60\n done\n } &", + "{\n while true; do\n aws stepfunctions send-task-heartbeat --task-token \"{{ taskToken }}\"\n sleep 60\n done\n } >/dev/null 2>&1 &", "if [ \"$(cat RUNNER_VERSION)\" = \"latest\" ]; then\n RUNNER_FLAGS=\"\"\n else\n RUNNER_FLAGS=\"--disableupdate\"\n fi", "sudo -Hu runner /home/runner/config.sh --unattended --url \"{{ registrationUrl }}\" --token \"{{ runnerToken }}\" --ephemeral --work _work --labels \"{{ labels }},cdkghr:started:$(date +%s)\" $RUNNER_FLAGS --name \"{{ runnerName }}\" || exit 1", "sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2", - "STATUS=$(grep -Phors \"finish job request for job [0-9a-f\\-]+ with result: \\K.*\" /home/runner/_diag/ | tail -n1)\n [ -n \"$STATUS\" ] && echo CDKGHA JOB DONE \"$labels\" \"$STATUS\"" + "STATUS=$(grep -Phors \"finish job request for job [0-9a-f\\-]+ with result: \\K.*\" /home/runner/_diag/ | tail -n1)\n [ -n \"$STATUS\" ] && echo CDKGHA JOB DONE \"{{ labels }}\" \"$STATUS\"" ] }, "CloudWatchOutputConfig": { @@ -14628,15 +14629,16 @@ "{{ instanceId }}" ], "Parameters": { + "executionTimeout": "30", "workingDirectory": "/home/runner", "commands": [ "set -ex", "touch /home/runner/STARTED", - "{\n while true; do\n aws stepfunctions send-task-heartbeat --task-token \"{{ taskToken }}\"\n sleep 60\n done\n } &", + "{\n while true; do\n aws stepfunctions send-task-heartbeat --task-token \"{{ taskToken }}\"\n sleep 60\n done\n } >/dev/null 2>&1 &", "if [ \"$(cat RUNNER_VERSION)\" = \"latest\" ]; then\n RUNNER_FLAGS=\"\"\n else\n RUNNER_FLAGS=\"--disableupdate\"\n fi", "sudo -Hu runner /home/runner/config.sh --unattended --url \"{{ registrationUrl }}\" --token \"{{ runnerToken }}\" --ephemeral --work _work --labels \"{{ labels }},cdkghr:started:$(date +%s)\" $RUNNER_FLAGS --name \"{{ runnerName }}\" || exit 1", "sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2", - "STATUS=$(grep -Phors \"finish job request for job [0-9a-f\\-]+ with result: \\K.*\" /home/runner/_diag/ | tail -n1)\n [ -n \"$STATUS\" ] && echo CDKGHA JOB DONE \"$labels\" \"$STATUS\"" + "STATUS=$(grep -Phors \"finish job request for job [0-9a-f\\-]+ with result: \\K.*\" /home/runner/_diag/ | tail -n1)\n [ -n \"$STATUS\" ] && echo CDKGHA JOB DONE \"{{ labels }}\" \"$STATUS\"" ] }, "CloudWatchOutputConfig": { @@ -15184,15 +15186,16 @@ "{{ instanceId }}" ], "Parameters": { + "executionTimeout": "30", "workingDirectory": "/home/runner", "commands": [ "set -ex", "touch /home/runner/STARTED", - "{\n while true; do\n aws stepfunctions send-task-heartbeat --task-token \"{{ taskToken }}\"\n sleep 60\n done\n } &", + "{\n while true; do\n aws stepfunctions send-task-heartbeat --task-token \"{{ taskToken }}\"\n sleep 60\n done\n } >/dev/null 2>&1 &", "if [ \"$(cat RUNNER_VERSION)\" = \"latest\" ]; then\n RUNNER_FLAGS=\"\"\n else\n RUNNER_FLAGS=\"--disableupdate\"\n fi", "sudo -Hu runner /home/runner/config.sh --unattended --url \"{{ registrationUrl }}\" --token \"{{ runnerToken }}\" --ephemeral --work _work --labels \"{{ labels }},cdkghr:started:$(date +%s)\" $RUNNER_FLAGS --name \"{{ runnerName }}\" || exit 1", "sudo --preserve-env=AWS_REGION -Hu runner /home/runner/run.sh || exit 2", - "STATUS=$(grep -Phors \"finish job request for job [0-9a-f\\-]+ with result: \\K.*\" /home/runner/_diag/ | tail -n1)\n [ -n \"$STATUS\" ] && echo CDKGHA JOB DONE \"$labels\" \"$STATUS\"" + "STATUS=$(grep -Phors \"finish job request for job [0-9a-f\\-]+ with result: \\K.*\" /home/runner/_diag/ | tail -n1)\n [ -n \"$STATUS\" ] && echo CDKGHA JOB DONE \"{{ labels }}\" \"$STATUS\"" ] }, "CloudWatchOutputConfig": { @@ -15740,6 +15743,7 @@ "{{ instanceId }}" ], "Parameters": { + "executionTimeout": "30", "workingDirectory": "C:\\actions", "commands": [ "New-Item STARTED", @@ -18079,7 +18083,7 @@ "GroupId" ] }, - "\"]}},\"Overrides\":{\"ContainerOverrides\":[{\"Name\":\"runner\",\"Environment\":[{\"Name\":\"RUNNER_TOKEN\",\"Value.$\":\"$.runner.token\"},{\"Name\":\"RUNNER_NAME\",\"Value.$\":\"$$.Execution.Name\"},{\"Name\":\"RUNNER_LABEL\",\"Value\":\"fargate,windows,x64\"},{\"Name\":\"GITHUB_DOMAIN\",\"Value.$\":\"$.runner.domain\"},{\"Name\":\"OWNER\",\"Value.$\":\"$.owner\"},{\"Name\":\"REPO\",\"Value.$\":\"$.repo\"},{\"Name\":\"REGISTRATION_URL\",\"Value.$\":\"$.runner.registrationUrl\"}]}]},\"PropagateTags\":\"TASK_DEFINITION\",\"EnableExecuteCommand\":false,\"CapacityProviderStrategy\":[{\"CapacityProvider\":\"FARGATE\"}]}},\"ec2, linux, x64 Get AMI\":{\"Next\":\"ec2, linux, x64 Fleet\",\"Type\":\"Task\",\"ResultPath\":\"$.instanceInput\",\"ResultSelector\":{\"ami.$\":\"$.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"},\"Resource\":\"arn:", + "\"]}},\"Overrides\":{\"ContainerOverrides\":[{\"Name\":\"runner\",\"Environment\":[{\"Name\":\"RUNNER_TOKEN\",\"Value.$\":\"$.runner.token\"},{\"Name\":\"RUNNER_NAME\",\"Value.$\":\"$$.Execution.Name\"},{\"Name\":\"RUNNER_LABEL\",\"Value\":\"fargate,windows,x64\"},{\"Name\":\"GITHUB_DOMAIN\",\"Value.$\":\"$.runner.domain\"},{\"Name\":\"OWNER\",\"Value.$\":\"$.owner\"},{\"Name\":\"REPO\",\"Value.$\":\"$.repo\"},{\"Name\":\"REGISTRATION_URL\",\"Value.$\":\"$.runner.registrationUrl\"}]}]},\"PropagateTags\":\"TASK_DEFINITION\",\"EnableExecuteCommand\":false,\"CapacityProviderStrategy\":[{\"CapacityProvider\":\"FARGATE\"}]}},\"ec2, linux, x64 Get AMI\":{\"Next\":\"ec2, linux, x64 Fleet\",\"Type\":\"Task\",\"ResultPath\":\"$.ami\",\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, @@ -18087,7 +18091,7 @@ { "Ref": "AMILinuxBuilderLaunchtemplateA29452C4" }, - "\",\"Versions\":[\"$Latest\"]}},\"ec2, linux, x64 Fleet\":{\"Next\":\"ec2, linux, x64 SSM\",\"Type\":\"Task\",\"ResultPath\":\"$.instance\",\"ResultSelector\":{\"id.$\":\"$.Instances[0].InstanceIds[0]\"},\"Resource\":\"arn:", + "\",\"Versions\":[\"$Latest\"]}},\"ec2, linux, x64 Fleet\":{\"Next\":\"ec2, linux, x64 SSM\",\"Type\":\"Task\",\"ResultPath\":\"$.instance\",\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, @@ -18099,11 +18103,11 @@ { "Ref": "VpcPublicSubnet1Subnet5C2D37C4" }, - "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"},{\"SubnetId\":\"", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.ami.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"},{\"SubnetId\":\"", { "Ref": "VpcPublicSubnet2Subnet691E08A3" }, - "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"}]}],\"Type\":\"instant\"}},\"ec2, linux, x64 SSM\":{\"End\":true,\"Type\":\"Task\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.ami.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"}]}],\"Type\":\"instant\"}},\"ec2, linux, x64 SSM\":{\"End\":true,\"Type\":\"Task\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, @@ -18111,7 +18115,7 @@ { "Ref": "EC2LinuxAutomation2C6F2077" }, - "\",\"Parameters\":{\"instanceId.$\":\"States.Array($.instance.id)\",\"taskToken.$\":\"States.Array($$.Task.Token)\",\"runnerName.$\":\"States.Array($$.Execution.Name)\",\"runnerToken.$\":\"States.Array($.runner.token)\",\"labels.$\":\"States.Array('ec2,linux,x64')\",\"registrationUrl.$\":\"States.Array($.runner.registrationUrl)\"}}},\"ec2-spot, linux, x64 Get AMI\":{\"Next\":\"ec2-spot, linux, x64 Fleet\",\"Type\":\"Task\",\"ResultPath\":\"$.instanceInput\",\"ResultSelector\":{\"ami.$\":\"$.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"},\"Resource\":\"arn:", + "\",\"Parameters\":{\"instanceId.$\":\"States.Array($.instance.Instances[0].InstanceIds[0])\",\"taskToken.$\":\"States.Array($$.Task.Token)\",\"runnerName.$\":\"States.Array($$.Execution.Name)\",\"runnerToken.$\":\"States.Array($.runner.token)\",\"labels.$\":\"States.Array('ec2,linux,x64')\",\"registrationUrl.$\":\"States.Array($.runner.registrationUrl)\"}}},\"ec2-spot, linux, x64 Get AMI\":{\"Next\":\"ec2-spot, linux, x64 Fleet\",\"Type\":\"Task\",\"ResultPath\":\"$.ami\",\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, @@ -18119,7 +18123,7 @@ { "Ref": "AMILinuxBuilderLaunchtemplateA29452C4" }, - "\",\"Versions\":[\"$Latest\"]}},\"ec2-spot, linux, x64 Fleet\":{\"Next\":\"ec2-spot, linux, x64 SSM\",\"Type\":\"Task\",\"ResultPath\":\"$.instance\",\"ResultSelector\":{\"id.$\":\"$.Instances[0].InstanceIds[0]\"},\"Resource\":\"arn:", + "\",\"Versions\":[\"$Latest\"]}},\"ec2-spot, linux, x64 Fleet\":{\"Next\":\"ec2-spot, linux, x64 SSM\",\"Type\":\"Task\",\"ResultPath\":\"$.instance\",\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, @@ -18131,11 +18135,11 @@ { "Ref": "VpcPublicSubnet1Subnet5C2D37C4" }, - "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"},{\"SubnetId\":\"", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.ami.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"},{\"SubnetId\":\"", { "Ref": "VpcPublicSubnet2Subnet691E08A3" }, - "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"}]}],\"Type\":\"instant\"}},\"ec2-spot, linux, x64 SSM\":{\"End\":true,\"Type\":\"Task\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.ami.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"}]}],\"Type\":\"instant\"}},\"ec2-spot, linux, x64 SSM\":{\"End\":true,\"Type\":\"Task\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, @@ -18143,7 +18147,7 @@ { "Ref": "EC2SpotLinuxAutomation1255B0E6" }, - "\",\"Parameters\":{\"instanceId.$\":\"States.Array($.instance.id)\",\"taskToken.$\":\"States.Array($$.Task.Token)\",\"runnerName.$\":\"States.Array($$.Execution.Name)\",\"runnerToken.$\":\"States.Array($.runner.token)\",\"labels.$\":\"States.Array('ec2-spot,linux,x64')\",\"registrationUrl.$\":\"States.Array($.runner.registrationUrl)\"}}},\"ec2, linux, arm64 Get AMI\":{\"Next\":\"ec2, linux, arm64 Fleet\",\"Type\":\"Task\",\"ResultPath\":\"$.instanceInput\",\"ResultSelector\":{\"ami.$\":\"$.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"},\"Resource\":\"arn:", + "\",\"Parameters\":{\"instanceId.$\":\"States.Array($.instance.Instances[0].InstanceIds[0])\",\"taskToken.$\":\"States.Array($$.Task.Token)\",\"runnerName.$\":\"States.Array($$.Execution.Name)\",\"runnerToken.$\":\"States.Array($.runner.token)\",\"labels.$\":\"States.Array('ec2-spot,linux,x64')\",\"registrationUrl.$\":\"States.Array($.runner.registrationUrl)\"}}},\"ec2, linux, arm64 Get AMI\":{\"Next\":\"ec2, linux, arm64 Fleet\",\"Type\":\"Task\",\"ResultPath\":\"$.ami\",\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, @@ -18151,7 +18155,7 @@ { "Ref": "AMILinuxarm64BuilderLaunchtemplate8F5EFF44" }, - "\",\"Versions\":[\"$Latest\"]}},\"ec2, linux, arm64 Fleet\":{\"Next\":\"ec2, linux, arm64 SSM\",\"Type\":\"Task\",\"ResultPath\":\"$.instance\",\"ResultSelector\":{\"id.$\":\"$.Instances[0].InstanceIds[0]\"},\"Resource\":\"arn:", + "\",\"Versions\":[\"$Latest\"]}},\"ec2, linux, arm64 Fleet\":{\"Next\":\"ec2, linux, arm64 SSM\",\"Type\":\"Task\",\"ResultPath\":\"$.instance\",\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, @@ -18163,11 +18167,11 @@ { "Ref": "VpcPublicSubnet1Subnet5C2D37C4" }, - "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"},{\"SubnetId\":\"", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.ami.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"},{\"SubnetId\":\"", { "Ref": "VpcPublicSubnet2Subnet691E08A3" }, - "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"}]}],\"Type\":\"instant\"}},\"ec2, linux, arm64 SSM\":{\"End\":true,\"Type\":\"Task\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.ami.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"}]}],\"Type\":\"instant\"}},\"ec2, linux, arm64 SSM\":{\"End\":true,\"Type\":\"Task\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, @@ -18175,7 +18179,7 @@ { "Ref": "EC2Linuxarm64Automation0232B49F" }, - "\",\"Parameters\":{\"instanceId.$\":\"States.Array($.instance.id)\",\"taskToken.$\":\"States.Array($$.Task.Token)\",\"runnerName.$\":\"States.Array($$.Execution.Name)\",\"runnerToken.$\":\"States.Array($.runner.token)\",\"labels.$\":\"States.Array('ec2,linux,arm64')\",\"registrationUrl.$\":\"States.Array($.runner.registrationUrl)\"}}},\"ec2, windows, x64 Get AMI\":{\"Next\":\"ec2, windows, x64 Fleet\",\"Type\":\"Task\",\"ResultPath\":\"$.instanceInput\",\"ResultSelector\":{\"ami.$\":\"$.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"},\"Resource\":\"arn:", + "\",\"Parameters\":{\"instanceId.$\":\"States.Array($.instance.Instances[0].InstanceIds[0])\",\"taskToken.$\":\"States.Array($$.Task.Token)\",\"runnerName.$\":\"States.Array($$.Execution.Name)\",\"runnerToken.$\":\"States.Array($.runner.token)\",\"labels.$\":\"States.Array('ec2,linux,arm64')\",\"registrationUrl.$\":\"States.Array($.runner.registrationUrl)\"}}},\"ec2, windows, x64 Get AMI\":{\"Next\":\"ec2, windows, x64 Fleet\",\"Type\":\"Task\",\"ResultPath\":\"$.ami\",\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, @@ -18183,7 +18187,7 @@ { "Ref": "WindowsEC2BuilderLaunchtemplate0A66E9C2" }, - "\",\"Versions\":[\"$Latest\"]}},\"ec2, windows, x64 Fleet\":{\"Next\":\"ec2, windows, x64 SSM\",\"Type\":\"Task\",\"ResultPath\":\"$.instance\",\"ResultSelector\":{\"id.$\":\"$.Instances[0].InstanceIds[0]\"},\"Resource\":\"arn:", + "\",\"Versions\":[\"$Latest\"]}},\"ec2, windows, x64 Fleet\":{\"Next\":\"ec2, windows, x64 SSM\",\"Type\":\"Task\",\"ResultPath\":\"$.instance\",\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, @@ -18195,11 +18199,11 @@ { "Ref": "VpcPublicSubnet1Subnet5C2D37C4" }, - "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"},{\"SubnetId\":\"", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.ami.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"},{\"SubnetId\":\"", { "Ref": "VpcPublicSubnet2Subnet691E08A3" }, - "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.instanceInput.ami\"}]}],\"Type\":\"instant\"}},\"ec2, windows, x64 SSM\":{\"End\":true,\"Type\":\"Task\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", + "\",\"WeightedCapacity\":1,\"ImageId.$\":\"$.ami.LaunchTemplateVersions[0].LaunchTemplateData.ImageId\"}]}],\"Type\":\"instant\"}},\"ec2, windows, x64 SSM\":{\"End\":true,\"Type\":\"Task\",\"HeartbeatSeconds\":600,\"Resource\":\"arn:", { "Ref": "AWS::Partition" }, @@ -18207,7 +18211,7 @@ { "Ref": "EC2WindowsAutomationC493D3FB" }, - "\",\"Parameters\":{\"instanceId.$\":\"States.Array($.instance.id)\",\"taskToken.$\":\"States.Array($$.Task.Token)\",\"runnerName.$\":\"States.Array($$.Execution.Name)\",\"runnerToken.$\":\"States.Array($.runner.token)\",\"labels.$\":\"States.Array('ec2,windows,x64')\",\"registrationUrl.$\":\"States.Array($.runner.registrationUrl)\"}}}}}]},\"Delete Failed Runner\":{\"End\":true,\"Retry\":[{\"ErrorEquals\":[\"Lambda.ClientExecutionTimeoutException\",\"Lambda.ServiceException\",\"Lambda.AWSLambdaException\",\"Lambda.SdkClientException\"],\"IntervalSeconds\":2,\"MaxAttempts\":6,\"BackoffRate\":2},{\"ErrorEquals\":[\"RunnerBusy\"],\"IntervalSeconds\":60,\"MaxAttempts\":60,\"BackoffRate\":1}],\"Type\":\"Task\",\"ResultPath\":\"$.delete\",\"Resource\":\"", + "\",\"Parameters\":{\"instanceId.$\":\"States.Array($.instance.Instances[0].InstanceIds[0])\",\"taskToken.$\":\"States.Array($$.Task.Token)\",\"runnerName.$\":\"States.Array($$.Execution.Name)\",\"runnerToken.$\":\"States.Array($.runner.token)\",\"labels.$\":\"States.Array('ec2,windows,x64')\",\"registrationUrl.$\":\"States.Array($.runner.registrationUrl)\"}}}}}]},\"Delete Failed Runner\":{\"End\":true,\"Retry\":[{\"ErrorEquals\":[\"Lambda.ClientExecutionTimeoutException\",\"Lambda.ServiceException\",\"Lambda.AWSLambdaException\",\"Lambda.SdkClientException\"],\"IntervalSeconds\":2,\"MaxAttempts\":6,\"BackoffRate\":2},{\"ErrorEquals\":[\"RunnerBusy\"],\"IntervalSeconds\":60,\"MaxAttempts\":60,\"BackoffRate\":1}],\"Type\":\"Task\",\"ResultPath\":\"$.delete\",\"Resource\":\"", { "Fn::GetAtt": [ "runnersdeleterunner7F8D5293",