From 27e974da12e5c009732b5dd6adc0b7a7711fba14 Mon Sep 17 00:00:00 2001 From: Niek Palm Date: Wed, 5 Jan 2022 16:56:10 +0100 Subject: [PATCH] feat: Replace run instance API by create fleet API (#1556) * Replace creating instance by the fleet api * Enable create fleet API * Enable create fleet API * refactor * refactor * fix tests * cleanup * refactor * refactor, prepare for creating muliple runners * revert and typo * revert and typo * Adjust for ARM * refactor * Update README.md Co-authored-by: Scott Guymer * Update modules/runners/scale-up.tf Co-authored-by: Scott Guymer * Update modules/runners/variables.tf Co-authored-by: Scott Guymer * Update modules/runners/variables.tf Co-authored-by: Scott Guymer * Update variables.tf Co-authored-by: Scott Guymer * Update README.md Co-authored-by: Scott Guymer * review * typo Co-authored-by: Scott Guymer --- README.md | 21 +- examples/ephemeral/main.tf | 4 +- main.tf | 16 +- modules/runner-binaries-syncer/README.md | 2 +- modules/runner-binaries-syncer/variables.tf | 9 +- modules/runners/README.md | 5 +- .../runners/lambdas/runners/jest.config.js | 14 +- .../lambdas/runners/src/aws/runners.test.ts | 431 ++++++++++++++++++ .../lambdas/runners/src/aws/runners.ts | 196 ++++++++ .../src/{scale-runners => aws}/ssm.test.ts | 18 + .../runners/src/{scale-runners => aws}/ssm.ts | 0 .../gh-auth.test.ts | 4 +- .../src/{scale-runners => gh-auth}/gh-auth.ts | 8 +- .../lambdas/runners/src/lambda.test.ts | 7 +- modules/runners/lambdas/runners/src/lambda.ts | 4 +- modules/runners/lambdas/runners/src/local.ts | 18 +- .../runners/src/{scale-runners => }/logger.ts | 0 .../src/{scale-runners => }/modules.d.ts | 8 + .../runners/src/scale-runners/runners.test.ts | 231 ---------- .../runners/src/scale-runners/runners.ts | 134 ------ .../src/scale-runners/scale-down.test.ts | 8 +- .../runners/src/scale-runners/scale-down.ts | 6 +- .../src/scale-runners/scale-up.test.ts | 108 ++--- .../runners/src/scale-runners/scale-up.ts | 226 +++++---- modules/runners/lambdas/runners/tsconfig.json | 8 +- modules/runners/main.tf | 19 +- modules/runners/policies/lambda-scale-up.json | 18 +- modules/runners/scale-up.tf | 6 +- modules/runners/variables.tf | 35 +- outputs.tf | 6 +- variables.tf | 63 ++- 31 files changed, 1005 insertions(+), 628 deletions(-) create mode 100644 modules/runners/lambdas/runners/src/aws/runners.test.ts create mode 100644 modules/runners/lambdas/runners/src/aws/runners.ts rename modules/runners/lambdas/runners/src/{scale-runners => aws}/ssm.test.ts (69%) rename modules/runners/lambdas/runners/src/{scale-runners => aws}/ssm.ts (100%) rename modules/runners/lambdas/runners/src/{scale-runners => gh-auth}/gh-auth.test.ts (98%) rename modules/runners/lambdas/runners/src/{scale-runners => gh-auth}/gh-auth.ts (92%) rename modules/runners/lambdas/runners/src/{scale-runners => }/logger.ts (100%) rename modules/runners/lambdas/runners/src/{scale-runners => }/modules.d.ts (67%) delete mode 100644 modules/runners/lambdas/runners/src/scale-runners/runners.test.ts delete mode 100644 modules/runners/lambdas/runners/src/scale-runners/runners.ts diff --git a/README.md b/README.md index a43e92a23c..c235c979c5 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ To be able to support a number of use-cases the module has quite a lot configura - Linux vs Windows. you can configure the os types linux and win. Linux will be used by default. - Re-use vs Ephemeral. By default runners are re-used for till detected idle, once idle they will be removed from the pool. To improve security we are introducing ephemeral runners. Those runners are only used for one job. Ephemeral runners are only working in combination with the workflow job event. We also suggest to use a pre-build AMI to improve the start time of jobs. - GitHub cloud vs GitHub enterprise server (GHES). The runner support GitHub cloud as well GitHub enterprise service. For GHES we rely on our community to test and support. We have no possibility to test ourselves on GHES. +- Spot vs on-demand. The runners using either the EC2 spot or on-demand life cycle. Runners will be created via the AWS [CreateFleet API](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html). The module (scale up lambda) will request an instance via the create fleet API in one of the subnets and matching one of the specified instance types. #### ARM64 support via Graviton/Graviton2 instance-types @@ -325,15 +326,7 @@ The following sub modules are optional and are provided as example or utility: ### ARM64 configuration for submodules -When not using the top-level module and specifying an `a1`, `t4g` or `*6g*` (6th-gen Graviton2) `instance_type`, the `runner-binaries-syncer` and `runners` submodules need to be configured appropriately for pulling the ARM64 GitHub action runner binary and leveraging the arm64 AMI for the runners. - -When configuring `runner-binaries-syncer` - -- _runner_architecture_ - set to `arm64`, defaults to `x64` - -When configuring `runners` - -- _ami_filter_ - set to `["amzn2-ami-hvm-2*-arm64-gp2"]`, defaults to `["amzn2-ami-hvm-2.*-x86_64-ebs"]` +When using the top-level module configure `runner_architecture = arm64` and ensure the list of `instance_types` matches. When not using the top-level ensure the bot properties are set on the submodules. ## Debugging @@ -401,9 +394,12 @@ In case the setup does not work as intended follow the trace of events: | [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. Example: https://github.internal.co - DO NOT SET IF USING PUBLIC GITHUB | `string` | `null` | no | | [github\_app](#input\_github\_app) | GitHub app parameters, see your github app. Ensure the key is the base64-encoded `.pem` file (the output of `base64 app.private-key.pem`, not the content of `private-key.pem`). | 
object({
    key_base64     = string
    id             = string
    webhook_secret = string
  })
| n/a | yes | | [idle\_config](#input\_idle\_config) | List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    cron      = string
    timeZone  = string
    idleCount = number
  }))
| `[]` | no | +| [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | +| [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot intances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no | | [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the environment name will be used. | `string` | `null` | no | -| [instance\_type](#input\_instance\_type) | [DEPRECATED] See instance\_types. | `string` | `"m5.large"` | no | -| [instance\_types](#input\_instance\_types) | List of instance types for the action runner. Defaults are based on runner\_os (amzn2 for linux and Windows Server Core for win). | `list(string)` | `null` | no | +| [instance\_target\_capacity\_type](#input\_instance\_target\_capacity\_type) | Default lifecycle used for runner instances, can be either `spot` or `on-demand`. | `string` | `"spot"` | no | +| [instance\_type](#input\_instance\_type) | [DEPRECATED] See instance\_types. | `string` | `null` | no | +| [instance\_types](#input\_instance\_types) | List of instance types for the action runner. Defaults are based on runner\_os (amzn2 for linux and Windows Server Core for win). | `list(string)` | 
[
  "m5.large",
  "c5.large"
]
| no | | [job\_queue\_retention\_in\_seconds](#input\_job\_queue\_retention\_in\_seconds) | The number of seconds the job is held in the queue before it is purged | `number` | `86400` | no | | [key\_name](#input\_key\_name) | Key pair name | `string` | `null` | no | | [kms\_key\_arn](#input\_kms\_key\_arn) | Optional CMK Key ARN to be used for Parameter Store. This key must be in the current account. | `string` | `null` | no | @@ -414,7 +410,7 @@ In case the setup does not work as intended follow the trace of events: | [log\_level](#input\_log\_level) | Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. | `string` | `"info"` | no | | [log\_type](#input\_log\_type) | Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. | `string` | `"pretty"` | no | | [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no | -| [market\_options](#input\_market\_options) | Market options for the action runner instances. Setting the value to `null` let the scaler create on-demand instances instead of spot instances. | `string` | `"spot"` | no | +| [market\_options](#input\_market\_options) | DEPCRECATED: Replaced by `instance_target_capacity_type`. | `string` | `null` | no | | [minimum\_running\_time\_in\_minutes](#input\_minimum\_running\_time\_in\_minutes) | The time an ec2 action runner should be running at minimum before terminated if not busy. | `number` | `null` | no | | [redrive\_build\_queue](#input\_redrive\_build\_queue) | Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting, `enalbed' to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries.` | 
object({
    enabled         = bool
    maxReceiveCount = number
  })
| 
{
  "enabled": false,
  "maxReceiveCount": null
}
| no | | [repository\_white\_list](#input\_repository\_white\_list) | List of repositories allowed to use the github app | `list(string)` | `[]` | no | @@ -422,6 +418,7 @@ In case the setup does not work as intended follow the trace of events: | [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created roles. | `string` | `null` | no | | [runner\_additional\_security\_group\_ids](#input\_runner\_additional\_security\_group\_ids) | (optional) List of additional security groups IDs to apply to the runner | `list(string)` | `[]` | no | | [runner\_allow\_prerelease\_binaries](#input\_runner\_allow\_prerelease\_binaries) | Allow the runners to update to prerelease binaries. | `bool` | `false` | no | +| [runner\_architecture](#input\_runner\_architecture) | The platform architecture of the runner instance\_type. | `string` | `"x64"` | no | | [runner\_as\_root](#input\_runner\_as\_root) | Run the action runner under the root user. Variable `runner_run_as` will be ingored. | `bool` | `false` | no | | [runner\_binaries\_s3\_sse\_configuration](#input\_runner\_binaries\_s3\_sse\_configuration) | Map containing server-side encryption configuration for runner-binaries S3 bucket. | `any` | `{}` | no | | [runner\_binaries\_syncer\_lambda\_timeout](#input\_runner\_binaries\_syncer\_lambda\_timeout) | Time out of the binaries sync lambda in seconds. | `number` | `300` | no | diff --git a/examples/ephemeral/main.tf b/examples/ephemeral/main.tf index b394034a3e..d53c0f7869 100644 --- a/examples/ephemeral/main.tf +++ b/examples/ephemeral/main.tf @@ -1,5 +1,5 @@ locals { - environment = "ephemeraal" + environment = "ephemeral" aws_region = "eu-west-1" } @@ -60,7 +60,7 @@ module "runners" { # ami_owners = [data.aws_caller_identity.current.account_id] # Enable logging - # log_level = "debug" + # log_level = "debug" # Setup a dead letter queue, by default scale up lambda will kepp retrying to process event in case of scaling error. # redrive_policy_build_queue = { diff --git a/main.tf b/main.tf index 798bc7e8cf..db29db37fb 100644 --- a/main.tf +++ b/main.tf @@ -5,7 +5,6 @@ locals { }) s3_action_runner_url = "s3://${module.runner_binaries.bucket.id}/${module.runner_binaries.runner_distribution_object_key}" - runner_architecture = substr(var.instance_type, 0, 2) == "a1" || substr(var.instance_type, 0, 3) == "t4g" || substr(var.instance_type, 1, 2) == "6g" ? "arm64" : "x64" github_app_parameters = { id = module.ssm.parameters.github_app_id key_base64 = module.ssm.parameters.github_app_key_base64 @@ -91,13 +90,14 @@ module "runners" { s3_bucket_runner_binaries = module.runner_binaries.bucket s3_location_runner_binaries = local.s3_action_runner_url - runner_os = var.runner_os - instance_type = var.instance_type - instance_types = var.instance_types - market_options = var.market_options - block_device_mappings = var.block_device_mappings + runner_os = var.runner_os + instance_types = var.instance_types + instance_target_capacity_type = var.instance_target_capacity_type + instance_allocation_strategy = var.instance_allocation_strategy + instance_max_spot_price = var.instance_max_spot_price + block_device_mappings = var.block_device_mappings - runner_architecture = local.runner_architecture + runner_architecture = var.runner_architecture ami_filter = var.ami_filter ami_owners = var.ami_owners @@ -169,7 +169,7 @@ module "runner_binaries" { distribution_bucket_name = "${var.environment}-dist-${random_string.random.result}" runner_os = var.runner_os - runner_architecture = local.runner_architecture + runner_architecture = var.runner_architecture runner_allow_prerelease_binaries = var.runner_allow_prerelease_binaries lambda_s3_bucket = var.lambda_s3_bucket diff --git a/modules/runner-binaries-syncer/README.md b/modules/runner-binaries-syncer/README.md index 7dbbc5945b..5bd2cf408a 100644 --- a/modules/runner-binaries-syncer/README.md +++ b/modules/runner-binaries-syncer/README.md @@ -91,7 +91,7 @@ No modules. | [role\_path](#input\_role\_path) | The path that will be added to the role, if not set the environment name will be used. | `string` | `null` | no | | [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no | | [runner\_allow\_prerelease\_binaries](#input\_runner\_allow\_prerelease\_binaries) | Allow the runners to update to prerelease binaries. | `bool` | `false` | no | -| [runner\_architecture](#input\_runner\_architecture) | The platform architecture for the runner instance (x64, arm64), defaults to 'x64' | `string` | `"x64"` | no | +| [runner\_architecture](#input\_runner\_architecture) | The platform architecture of the runner instance\_type. | `string` | `"x64"` | no | | [runner\_os](#input\_runner\_os) | The operating system for the runner instance (linux, win), defaults to 'linux' | `string` | `"linux"` | no | | [server\_side\_encryption\_configuration](#input\_server\_side\_encryption\_configuration) | Map containing server-side encryption configuration. | `any` | `{}` | no | | [syncer\_lambda\_s3\_key](#input\_syncer\_lambda\_s3\_key) | S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no | diff --git a/modules/runner-binaries-syncer/variables.tf b/modules/runner-binaries-syncer/variables.tf index 4e9547a9ac..e338105bbe 100644 --- a/modules/runner-binaries-syncer/variables.tf +++ b/modules/runner-binaries-syncer/variables.tf @@ -61,9 +61,16 @@ variable "runner_os" { } variable "runner_architecture" { - description = "The platform architecture for the runner instance (x64, arm64), defaults to 'x64'" + description = "The platform architecture of the runner instance_type." type = string default = "x64" + validation { + condition = anytrue([ + var.runner_architecture == "x64", + var.runner_architecture == "arm64", + ]) + error_message = "`runner_architecture` value not valid, valid values are: `x64` and `arm64`." + } } variable "logging_retention_in_days" { diff --git a/modules/runners/README.md b/modules/runners/README.md index 7f070635a9..0f0b528e5f 100644 --- a/modules/runners/README.md +++ b/modules/runners/README.md @@ -128,7 +128,10 @@ No modules. | [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. DO NOT SET IF USING PUBLIC GITHUB | `string` | `null` | no | | [github\_app\_parameters](#input\_github\_app\_parameters) | Parameter Store for GitHub App Parameters. | 
object({
    key_base64 = map(string)
    id         = map(string)
  })
| n/a | yes | | [idle\_config](#input\_idle\_config) | List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    cron      = string
    timeZone  = string
    idleCount = number
  }))
| `[]` | no | +| [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | +| [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot intances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no | | [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the environment name will be used. | `string` | `null` | no | +| [instance\_target\_capacity\_type](#input\_instance\_target\_capacity\_type) | Default lifecyle used runner instances, can be either `spot` or `on-demand`. | `string` | `"spot"` | no | | [instance\_type](#input\_instance\_type) | [DEPRECATED] See instance\_types. | `string` | `"m5.large"` | no | | [instance\_types](#input\_instance\_types) | List of instance types for the action runner. Defaults are based on runner\_os (amzn2 for linux and Windows Server Core for win). | `list(string)` | `null` | no | | [key\_name](#input\_key\_name) | Key pair name | `string` | `null` | no | @@ -142,7 +145,7 @@ No modules. | [log\_level](#input\_log\_level) | Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. | `string` | `"info"` | no | | [log\_type](#input\_log\_type) | Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. | `string` | `"pretty"` | no | | [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no | -| [market\_options](#input\_market\_options) | Market options for the action runner instances. | `string` | `"spot"` | no | +| [market\_options](#input\_market\_options) | DEPCRECATED: Replaced by `instance_target_capacity_type`. | `string` | `null` | no | | [metadata\_options](#input\_metadata\_options) | Metadata options for the ec2 runner instances. | `map(any)` | 
{
  "http_endpoint": "enabled",
  "http_put_response_hop_limit": 1,
  "http_tokens": "optional"
}
| no | | [minimum\_running\_time\_in\_minutes](#input\_minimum\_running\_time\_in\_minutes) | The time an ec2 action runner should be running at minimum before terminated if non busy. If not set the default is calculated based on the OS. | `number` | `null` | no | | [overrides](#input\_overrides) | This map provides the possibility to override some defaults. The following attributes are supported: `name_sg` overrides the `Name` tag for all security groups created by this module. `name_runner_agent_instance` overrides the `Name` tag for the ec2 instance defined in the auto launch configuration. `name_docker_machine_runners` overrides the `Name` tag spot instances created by the runner agent. | `map(string)` | 
{
  "name_runner": "",
  "name_sg": ""
}
| no | diff --git a/modules/runners/lambdas/runners/jest.config.js b/modules/runners/lambdas/runners/jest.config.js index 8c7a9f17c5..165378f519 100644 --- a/modules/runners/lambdas/runners/jest.config.js +++ b/modules/runners/lambdas/runners/jest.config.js @@ -2,13 +2,13 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', collectCoverage: true, - collectCoverageFrom: ['src/**/*.{ts,js,jsx}','!src/**/*local*.ts'], + collectCoverageFrom: ['src/**/*.{ts,js,jsx}', '!src/**/*local*.ts', 'src/**/*.d.ts'], coverageThreshold: { global: { - branches: 80, - functions: 80, - lines: 80, - statements: 80 - } - } + branches: 92, + functions: 92, + lines: 92, + statements: 92, + }, + }, }; diff --git a/modules/runners/lambdas/runners/src/aws/runners.test.ts b/modules/runners/lambdas/runners/src/aws/runners.test.ts new file mode 100644 index 0000000000..a836e5f6ea --- /dev/null +++ b/modules/runners/lambdas/runners/src/aws/runners.test.ts @@ -0,0 +1,431 @@ +import { EC2 } from 'aws-sdk'; +import { listEC2Runners, createRunner, terminateRunner, RunnerInfo, RunnerInputParameters } from './runners'; +import ScaleError from './../scale-runners/ScaleError'; + +const mockEC2 = { describeInstances: jest.fn(), createFleet: jest.fn(), terminateInstances: jest.fn() }; +const mockSSM = { putParameter: jest.fn() }; +jest.mock('aws-sdk', () => ({ + EC2: jest.fn().mockImplementation(() => mockEC2), + SSM: jest.fn().mockImplementation(() => mockSSM), +})); + +const LAUNCH_TEMPLATE = 'lt-1'; +const ORG_NAME = 'SomeAwesomeCoder'; +const REPO_NAME = `${ORG_NAME}/some-amazing-library`; +const ENVIRONMENT = 'unit-test-environment'; + +describe('list instances', () => { + const mockDescribeInstances = { promise: jest.fn() }; + beforeEach(() => { + jest.clearAllMocks(); + mockEC2.describeInstances.mockImplementation(() => mockDescribeInstances); + const mockRunningInstances: AWS.EC2.DescribeInstancesResult = { + Reservations: [ + { + Instances: [ + { + LaunchTime: new Date('2020-10-10T14:48:00.000+09:00'), + InstanceId: 'i-1234', + Tags: [ + { Key: 'Application', Value: 'github-action-runner' }, + { Key: 'Type', Value: 'Org' }, + { Key: 'Owner', Value: 'CoderToCat' }, + ], + }, + { + LaunchTime: new Date('2020-10-11T14:48:00.000+09:00'), + InstanceId: 'i-5678', + Tags: [ + { Key: 'Owner', Value: REPO_NAME }, + { Key: 'Type', Value: 'Repo' }, + { Key: 'Application', Value: 'github-action-runner' }, + ], + }, + ], + }, + ], + }; + mockDescribeInstances.promise.mockReturnValue(mockRunningInstances); + }); + + it('returns a list of instances', async () => { + const resp = await listEC2Runners(); + expect(resp.length).toBe(2); + expect(resp).toContainEqual({ + instanceId: 'i-1234', + launchTime: new Date('2020-10-10T14:48:00.000+09:00'), + type: 'Org', + owner: 'CoderToCat', + }); + expect(resp).toContainEqual({ + instanceId: 'i-5678', + launchTime: new Date('2020-10-11T14:48:00.000+09:00'), + type: 'Repo', + owner: REPO_NAME, + }); + }); + + it('calls EC2 describe instances', async () => { + await listEC2Runners(); + expect(mockEC2.describeInstances).toBeCalled(); + }); + + it('filters instances on repo name', async () => { + await listEC2Runners({ runnerType: 'Repo', runnerOwner: REPO_NAME, environment: undefined }); + expect(mockEC2.describeInstances).toBeCalledWith({ + Filters: [ + { Name: 'tag:Application', Values: ['github-action-runner'] }, + { Name: 'instance-state-name', Values: ['running', 'pending'] }, + { Name: 'tag:Type', Values: ['Repo'] }, + { Name: 'tag:Owner', Values: [REPO_NAME] }, + ], + }); + }); + + it('filters instances on org name', async () => { + await listEC2Runners({ runnerType: 'Org', runnerOwner: ORG_NAME, environment: undefined }); + expect(mockEC2.describeInstances).toBeCalledWith({ + Filters: [ + { Name: 'tag:Application', Values: ['github-action-runner'] }, + { Name: 'instance-state-name', Values: ['running', 'pending'] }, + { Name: 'tag:Type', Values: ['Org'] }, + { Name: 'tag:Owner', Values: [ORG_NAME] }, + ], + }); + }); + + it('filters instances on environment', async () => { + await listEC2Runners({ environment: ENVIRONMENT }); + expect(mockEC2.describeInstances).toBeCalledWith({ + Filters: [ + { Name: 'tag:Application', Values: ['github-action-runner'] }, + { Name: 'instance-state-name', Values: ['running', 'pending'] }, + { Name: 'tag:Environment', Values: [ENVIRONMENT] }, + ], + }); + }); + + it('No instances, undefined reservations list.', async () => { + const noInstances: AWS.EC2.DescribeInstancesResult = { + Reservations: undefined, + }; + mockDescribeInstances.promise.mockReturnValue(noInstances); + const resp = await listEC2Runners(); + expect(resp.length).toBe(0); + }); + + it('No instances, undefined instance list.', async () => { + const noInstances: AWS.EC2.DescribeInstancesResult = { + Reservations: [ + { + Instances: undefined, + }, + ], + }; + mockDescribeInstances.promise.mockReturnValue(noInstances); + const resp = await listEC2Runners(); + expect(resp.length).toBe(0); + }); + + it('Instances with no tags.', async () => { + const noInstances: AWS.EC2.DescribeInstancesResult = { + Reservations: [ + { + Instances: [ + { + LaunchTime: new Date('2020-10-11T14:48:00.000+09:00'), + InstanceId: 'i-5678', + Tags: undefined, + }, + ], + }, + ], + }; + mockDescribeInstances.promise.mockReturnValue(noInstances); + const resp = await listEC2Runners(); + expect(resp.length).toBe(1); + }); +}); + +describe('terminate runner', () => { + const mockTerminateInstances = { promise: jest.fn() }; + beforeEach(() => { + jest.clearAllMocks(); + mockEC2.terminateInstances.mockImplementation(() => mockTerminateInstances); + mockTerminateInstances.promise.mockReturnThis(); + }); + it('calls terminate instances with the right instance ids', async () => { + const runner: RunnerInfo = { + instanceId: 'instance-2', + owner: 'owner-2', + type: 'Repo', + }; + await terminateRunner(runner.instanceId); + + expect(mockEC2.terminateInstances).toBeCalledWith({ InstanceIds: [runner.instanceId] }); + }); +}); + +describe('create runner', () => { + const mockCreateFleet = { promise: jest.fn() }; + const mockPutParameter = { promise: jest.fn() }; + const defaultRunnerConfig: RunnerConfig = { + allocationStrategy: 'capacity-optimized', + capacityType: 'spot', + type: 'Org', + }; + const defaultExpectedFleetRequestValues: ExpectedFleetRequestValues = { + type: 'Org', + capacityType: 'spot', + allocationStrategy: 'capacity-optimized', + totalTargetCapacity: 1, + }; + + beforeEach(() => { + jest.clearAllMocks(); + + mockEC2.createFleet.mockImplementation(() => mockCreateFleet); + + mockCreateFleet.promise.mockReturnValue({ + Instances: [{ InstanceIds: ['i-1234'] }], + }); + mockSSM.putParameter.mockImplementation(() => mockPutParameter); + }); + + it('calls create fleet of 1 instance with the correct config for repo', async () => { + await createRunner(createRunnerConfig({ ...defaultRunnerConfig, type: 'Repo' })); + expect(mockEC2.createFleet).toBeCalledWith( + expectedCreateFleetRequest({ ...defaultExpectedFleetRequestValues, type: 'Repo' }), + ); + expect(mockSSM.putParameter).toBeCalledTimes(1); + }); + + it('calls create fleet of 2 instances with the correct config for org ', async () => { + const instances = [{ InstanceIds: ['i-1234', 'i-5678'] }]; + mockCreateFleet.promise.mockReturnValue({ + Instances: instances, + }); + + await createRunner({ ...createRunnerConfig(defaultRunnerConfig), numberOfRunners: 2 }); + + expect(mockEC2.createFleet).toBeCalledWith( + expectedCreateFleetRequest({ ...defaultExpectedFleetRequestValues, totalTargetCapacity: 2 }), + ); + expect(mockSSM.putParameter).toBeCalledTimes(2); + for (const instance of instances[0].InstanceIds) { + expect(mockSSM.putParameter).toBeCalledWith({ + Name: `unit-test-environment-${instance}`, + Type: 'SecureString', + Value: 'bla', + }); + } + }); + + it('calls create fleet of 1 instance with the correct config for org', async () => { + await createRunner(createRunnerConfig(defaultRunnerConfig)); + expect(mockEC2.createFleet).toBeCalledWith(expectedCreateFleetRequest(defaultExpectedFleetRequestValues)); + expect(mockSSM.putParameter).toBeCalledTimes(1); + }); + + it('calls create fleet of 1 instance with the on-demand capacity', async () => { + await createRunner(createRunnerConfig({ ...defaultRunnerConfig, capacityType: 'on-demand' })); + expect(mockEC2.createFleet).toBeCalledWith( + expectedCreateFleetRequest({ ...defaultExpectedFleetRequestValues, capacityType: 'on-demand' }), + ); + expect(mockSSM.putParameter).toBeCalledTimes(1); + }); + + it('calls run instances with the on-demand capacity', async () => { + await createRunner(createRunnerConfig({ ...defaultRunnerConfig, maxSpotPrice: '0.1' })); + expect(mockEC2.createFleet).toBeCalledWith( + expectedCreateFleetRequest({ ...defaultExpectedFleetRequestValues, maxSpotPrice: '0.1' }), + ); + }); + + it('creates ssm parameters for each created instance', async () => { + await createRunner(createRunnerConfig(defaultRunnerConfig)); + expect(mockSSM.putParameter).toBeCalledWith({ + Name: `${ENVIRONMENT}-i-1234`, + Value: 'bla', + Type: 'SecureString', + }); + }); + + it('does not create ssm parameters when no instance is created', async () => { + mockCreateFleet.promise.mockReturnValue({ + Instances: [], + }); + await expect(createRunner(createRunnerConfig(defaultRunnerConfig))).rejects; + expect(mockSSM.putParameter).not.toBeCalled(); + }); +}); + +describe('create runner with errors', () => { + const defaultRunnerConfig: RunnerConfig = { + allocationStrategy: 'capacity-optimized', + capacityType: 'spot', + type: 'Repo', + }; + const defaultExpectedFleetRequestValues: ExpectedFleetRequestValues = { + type: 'Repo', + capacityType: 'spot', + allocationStrategy: 'capacity-optimized', + totalTargetCapacity: 1, + }; + beforeEach(() => { + jest.clearAllMocks(); + + const mockPutParameter = { promise: jest.fn() }; + + mockSSM.putParameter.mockImplementation(() => mockPutParameter); + }); + + it('test ScaleError with one error.', async () => { + createFleetMockWithErrors(['UnfulfillableCapacity']); + + await expect(createRunner(createRunnerConfig(defaultRunnerConfig))).rejects.toBeInstanceOf(ScaleError); + expect(mockEC2.createFleet).toBeCalledWith(expectedCreateFleetRequest(defaultExpectedFleetRequestValues)); + expect(mockSSM.putParameter).not.toBeCalled(); + }); + + it('test ScaleError with multiple error.', async () => { + createFleetMockWithErrors(['UnfulfillableCapacity', 'SomeError']); + + await expect(createRunner(createRunnerConfig(defaultRunnerConfig))).rejects.toBeInstanceOf(ScaleError); + expect(mockEC2.createFleet).toBeCalledWith(expectedCreateFleetRequest(defaultExpectedFleetRequestValues)); + expect(mockSSM.putParameter).not.toBeCalled(); + }); + + it('test default Error', async () => { + createFleetMockWithErrors(['NonMappedError']); + + await expect(createRunner(createRunnerConfig(defaultRunnerConfig))).rejects.toBeInstanceOf(Error); + expect(mockEC2.createFleet).toBeCalledWith(expectedCreateFleetRequest(defaultExpectedFleetRequestValues)); + expect(mockSSM.putParameter).not.toBeCalled(); + }); + + it('test now error is thrown if an instance is created', async () => { + createFleetMockWithErrors(['NonMappedError'], ['i-123']); + + expect(await createRunner(createRunnerConfig(defaultRunnerConfig))).resolves; + expect(mockEC2.createFleet).toBeCalledWith(expectedCreateFleetRequest(defaultExpectedFleetRequestValues)); + expect(mockSSM.putParameter).toBeCalled(); + }); + + it('test error by create fleet call is thrown.', async () => { + mockEC2.createFleet.mockImplementation(() => { + return { + promise: jest.fn().mockImplementation(() => { + throw Error(''); + }), + }; + }); + + await expect(createRunner(createRunnerConfig(defaultRunnerConfig))).rejects.toBeInstanceOf(Error); + expect(mockEC2.createFleet).toBeCalledWith(expectedCreateFleetRequest(defaultExpectedFleetRequestValues)); + expect(mockSSM.putParameter).not.toBeCalled(); + }); +}); + +function createFleetMockWithErrors(errors: string[], instances?: string[]) { + let result: AWS.EC2.CreateFleetResult = { + Errors: errors.map((e) => ({ ErrorCode: e })), + }; + + if (instances) { + result = { + ...result, + Instances: [ + { + InstanceIds: instances.map((i) => i), + }, + ], + }; + } + + mockEC2.createFleet.mockImplementation(() => { + return { promise: jest.fn().mockReturnValue(result) }; + }); +} + +interface RunnerConfig { + type: 'Repo' | 'Org'; + capacityType: EC2.DefaultTargetCapacityType; + allocationStrategy: EC2.AllocationStrategy; + maxSpotPrice?: string; +} + +function createRunnerConfig(runnerConfig: RunnerConfig): RunnerInputParameters { + return { + runnerServiceConfig: 'bla', + environment: ENVIRONMENT, + runnerType: runnerConfig.type, + runnerOwner: REPO_NAME, + launchTemplateName: LAUNCH_TEMPLATE, + ec2instanceCriteria: { + instanceTypes: ['m5.large', 'c5.large'], + targetCapacityType: runnerConfig.capacityType, + maxSpotPrice: runnerConfig.maxSpotPrice, + instanceAllocationStrategy: runnerConfig.allocationStrategy, + }, + subnets: ['subnet-123', 'subnet-456'], + }; +} + +interface ExpectedFleetRequestValues { + type: 'Repo' | 'Org'; + capacityType: EC2.DefaultTargetCapacityType; + allocationStrategy: EC2.AllocationStrategy; + maxSpotPrice?: string; + totalTargetCapacity: number; +} + +function expectedCreateFleetRequest(expectedValues: ExpectedFleetRequestValues): AWS.EC2.CreateFleetRequest { + return { + LaunchTemplateConfigs: [ + { + LaunchTemplateSpecification: { + LaunchTemplateName: 'lt-1', + Version: '$Default', + }, + Overrides: [ + { + InstanceType: 'm5.large', + SubnetId: 'subnet-123', + }, + { + InstanceType: 'c5.large', + SubnetId: 'subnet-123', + }, + { + InstanceType: 'm5.large', + SubnetId: 'subnet-456', + }, + { + InstanceType: 'c5.large', + SubnetId: 'subnet-456', + }, + ], + }, + ], + SpotOptions: { + AllocationStrategy: expectedValues.allocationStrategy, + MaxTotalPrice: expectedValues.maxSpotPrice, + }, + TagSpecifications: [ + { + ResourceType: 'instance', + Tags: [ + { Key: 'Application', Value: 'github-action-runner' }, + { Key: 'Type', Value: expectedValues.type }, + { Key: 'Owner', Value: REPO_NAME }, + ], + }, + ], + TargetCapacitySpecification: { + DefaultTargetCapacityType: expectedValues.capacityType, + TotalTargetCapacity: expectedValues.totalTargetCapacity, + }, + Type: 'instant', + }; +} diff --git a/modules/runners/lambdas/runners/src/aws/runners.ts b/modules/runners/lambdas/runners/src/aws/runners.ts new file mode 100644 index 0000000000..1ac66b907d --- /dev/null +++ b/modules/runners/lambdas/runners/src/aws/runners.ts @@ -0,0 +1,196 @@ +import { EC2, SSM } from 'aws-sdk'; +import { logger as rootLogger, LogFields } from '../logger'; +import ScaleError from './../scale-runners/ScaleError'; + +const logger = rootLogger.getChildLogger({ name: 'runners' }); + +export interface RunnerList { + instanceId: string; + launchTime?: Date; + owner?: string; + type?: string; + repo?: string; + org?: string; +} + +export interface RunnerInfo { + instanceId: string; + launchTime?: Date; + owner: string; + type: string; +} + +export interface ListRunnerFilters { + runnerType?: 'Org' | 'Repo'; + runnerOwner?: string; + environment?: string; +} + +export interface RunnerInputParameters { + runnerServiceConfig: string; + environment: string; + runnerType: 'Org' | 'Repo'; + runnerOwner: string; + subnets: string[]; + launchTemplateName: string; + ec2instanceCriteria: { + instanceTypes: string[]; + targetCapacityType: EC2.DefaultTargetCapacityType; + maxSpotPrice?: string; + instanceAllocationStrategy: EC2.SpotAllocationStrategy; + }; + numberOfRunners?: number; +} + +export async function listEC2Runners(filters: ListRunnerFilters | undefined = undefined): Promise { + const ec2 = new EC2(); + const ec2Filters = [ + { Name: 'tag:Application', Values: ['github-action-runner'] }, + { Name: 'instance-state-name', Values: ['running', 'pending'] }, + ]; + if (filters) { + if (filters.environment !== undefined) { + ec2Filters.push({ Name: 'tag:Environment', Values: [filters.environment] }); + } + if (filters.runnerType && filters.runnerOwner) { + ec2Filters.push({ Name: `tag:Type`, Values: [filters.runnerType] }); + ec2Filters.push({ Name: `tag:Owner`, Values: [filters.runnerOwner] }); + } + } + const runningInstances = await ec2.describeInstances({ Filters: ec2Filters }).promise(); + const runners: RunnerList[] = []; + if (runningInstances.Reservations) { + for (const r of runningInstances.Reservations) { + if (r.Instances) { + for (const i of r.Instances) { + runners.push({ + instanceId: i.InstanceId as string, + launchTime: i.LaunchTime, + owner: i.Tags?.find((e) => e.Key === 'Owner')?.Value as string, + type: i.Tags?.find((e) => e.Key === 'Type')?.Value as string, + repo: i.Tags?.find((e) => e.Key === 'Repo')?.Value as string, + org: i.Tags?.find((e) => e.Key === 'Org')?.Value as string, + }); + } + } + } + } + return runners; +} + +export async function terminateRunner(instanceId: string): Promise { + const ec2 = new EC2(); + await ec2 + .terminateInstances({ + InstanceIds: [instanceId], + }) + .promise(); + logger.info(`Runner ${instanceId} has been terminated.`, LogFields.print()); +} + +function generateFleeOverrides( + subnetIds: string[], + instancesTypes: string[], +): EC2.FleetLaunchTemplateOverridesListRequest { + const result: EC2.FleetLaunchTemplateOverridesListRequest = []; + subnetIds.forEach((s) => { + instancesTypes.forEach((i) => { + result.push({ + SubnetId: s, + InstanceType: i, + }); + }); + }); + return result; +} + +export async function createRunner(runnerParameters: RunnerInputParameters): Promise { + logger.debug('Runner configuration: ' + JSON.stringify(runnerParameters), LogFields.print()); + + const ec2 = new EC2(); + const numberOfRunners = runnerParameters.numberOfRunners ? runnerParameters.numberOfRunners : 1; + + let fleet: AWS.EC2.CreateFleetResult; + try { + // see for spec https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + fleet = await ec2 + .createFleet({ + LaunchTemplateConfigs: [ + { + LaunchTemplateSpecification: { + LaunchTemplateName: runnerParameters.launchTemplateName, + Version: '$Default', + }, + Overrides: generateFleeOverrides( + runnerParameters.subnets, + runnerParameters.ec2instanceCriteria.instanceTypes, + ), + }, + ], + SpotOptions: { + MaxTotalPrice: runnerParameters.ec2instanceCriteria.maxSpotPrice, + AllocationStrategy: 'capacity-optimized', + }, + TargetCapacitySpecification: { + TotalTargetCapacity: numberOfRunners, + DefaultTargetCapacityType: runnerParameters.ec2instanceCriteria.targetCapacityType, + }, + TagSpecifications: [ + { + ResourceType: 'instance', + Tags: [ + { Key: 'Application', Value: 'github-action-runner' }, + { Key: 'Type', Value: runnerParameters.runnerType }, + { Key: 'Owner', Value: runnerParameters.runnerOwner }, + ], + }, + ], + Type: 'instant', + }) + .promise(); + } catch (e) { + logger.warn('Create fleet request failed.', e); + throw e; + } + + const instances: string[] = fleet.Instances?.flatMap((i) => i.InstanceIds?.flatMap((j) => j) || []) || []; + + if (instances.length === 0) { + logger.warn(`No instances created by fleet request. Check configuration! Response:`, fleet); + const errors = fleet.Errors?.flatMap((e) => e.ErrorCode || '') || []; + + // Educated guess of errors that would make sense to retry based on the list + // https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html + const scaleErrors = [ + 'UnfulfillableCapacity', + 'MaxSpotInstanceCountExceeded', + 'TargetCapacityLimitExceededException', + 'RequestLimitExceeded', + 'ResourceLimitExceeded', + 'MaxSpotInstanceCountExceeded', + 'MaxSpotFleetRequestCountExceeded', + ]; + + if (errors.some((e) => scaleErrors.includes(e))) { + logger.warn('Create fleet failed, ScaleError will be thrown to trigger retry for ephemeral runners.'); + logger.debug('Create fleet failed.', fleet.Errors); + throw new ScaleError('Failed to create instance, create fleet failed.'); + } else { + logger.warn('Create fleet failed, error not recognized as scaling error.', fleet.Errors); + throw Error('Create fleet failed, no instance created.'); + } + } + + logger.info('Created instance(s): ', instances.join(','), LogFields.print()); + + const ssm = new SSM(); + for (const instance of instances) { + await ssm + .putParameter({ + Name: `${runnerParameters.environment}-${instance}`, + Value: runnerParameters.runnerServiceConfig, + Type: 'SecureString', + }) + .promise(); + } +} diff --git a/modules/runners/lambdas/runners/src/scale-runners/ssm.test.ts b/modules/runners/lambdas/runners/src/aws/ssm.test.ts similarity index 69% rename from modules/runners/lambdas/runners/src/scale-runners/ssm.test.ts rename to modules/runners/lambdas/runners/src/aws/ssm.test.ts index 6d54d43e91..817308d89c 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/ssm.test.ts +++ b/modules/runners/lambdas/runners/src/aws/ssm.test.ts @@ -37,4 +37,22 @@ describe('Test getParameterValue', () => { // Assert expect(result).toBe(parameterValue); }); + + test('Gets invalid parameters and returns string', async () => { + // Arrange + const parameterName = 'invalid'; + const output: GetParameterCommandOutput = { + $metadata: { + httpStatusCode: 200, + }, + }; + + SSM.prototype.getParameter = jest.fn().mockResolvedValue(output); + + // Act + const result = await getParameterValue(parameterName); + + // Assert + expect(result).toBe(undefined); + }); }); diff --git a/modules/runners/lambdas/runners/src/scale-runners/ssm.ts b/modules/runners/lambdas/runners/src/aws/ssm.ts similarity index 100% rename from modules/runners/lambdas/runners/src/scale-runners/ssm.ts rename to modules/runners/lambdas/runners/src/aws/ssm.ts diff --git a/modules/runners/lambdas/runners/src/scale-runners/gh-auth.test.ts b/modules/runners/lambdas/runners/src/gh-auth/gh-auth.test.ts similarity index 98% rename from modules/runners/lambdas/runners/src/scale-runners/gh-auth.test.ts rename to modules/runners/lambdas/runners/src/gh-auth/gh-auth.test.ts index 1f9c2f4af9..22734f8a41 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/gh-auth.test.ts +++ b/modules/runners/lambdas/runners/src/gh-auth/gh-auth.test.ts @@ -3,14 +3,14 @@ import nock from 'nock'; import { createAppAuth } from '@octokit/auth-app'; import { StrategyOptions } from '@octokit/auth-app/dist-types/types'; -import { getParameterValue } from './ssm'; +import { getParameterValue } from './../aws/ssm'; import { RequestInterface } from '@octokit/types'; import { mock, MockProxy } from 'jest-mock-extended'; import { request } from '@octokit/request'; import { mocked } from 'ts-jest/utils'; -jest.mock('./ssm'); +jest.mock('./../aws/ssm'); jest.mock('@octokit/auth-app'); const cleanEnv = process.env; diff --git a/modules/runners/lambdas/runners/src/scale-runners/gh-auth.ts b/modules/runners/lambdas/runners/src/gh-auth/gh-auth.ts similarity index 92% rename from modules/runners/lambdas/runners/src/scale-runners/gh-auth.ts rename to modules/runners/lambdas/runners/src/gh-auth/gh-auth.ts index d70ec66927..8195438a60 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/gh-auth.ts +++ b/modules/runners/lambdas/runners/src/gh-auth/gh-auth.ts @@ -10,8 +10,8 @@ import { AuthInterface, } from '@octokit/auth-app/dist-types/types'; import { OctokitOptions } from '@octokit/core/dist-types/types'; -import { getParameterValue } from './ssm'; -import { logger as rootLogger, LogFields } from './logger'; +import { getParameterValue } from '../aws/ssm'; +import { logger as rootLogger, LogFields } from '../logger'; const logger = rootLogger.getChildLogger({ name: 'gh-auth' }); @@ -32,7 +32,7 @@ export async function createGithubAppAuth( ): Promise { const auth = await createAuth(installationId, ghesApiUrl); const appAuthOptions: AppAuthOptions = { type: 'app' }; - return await auth(appAuthOptions); + return auth(appAuthOptions); } export async function createGithubInstallationAuth( @@ -41,7 +41,7 @@ export async function createGithubInstallationAuth( ): Promise { const auth = await createAuth(installationId, ghesApiUrl); const installationAuthOptions: InstallationAuthOptions = { type: 'installation', installationId }; - return await auth(installationAuthOptions); + return auth(installationAuthOptions); } async function createAuth(installationId: number | undefined, ghesApiUrl: string): Promise { diff --git a/modules/runners/lambdas/runners/src/lambda.test.ts b/modules/runners/lambdas/runners/src/lambda.test.ts index 43f5ffdff3..b8c163a00f 100644 --- a/modules/runners/lambdas/runners/src/lambda.test.ts +++ b/modules/runners/lambdas/runners/src/lambda.test.ts @@ -1,10 +1,9 @@ -import { fail } from 'assert'; import { Context, SQSEvent, SQSRecord } from 'aws-lambda'; import { mocked } from 'ts-jest/utils'; import { scaleUpHandler } from './lambda'; import { ActionRequestMessage, scaleUp } from './scale-runners/scale-up'; import ScaleError from './scale-runners/ScaleError'; -import { logger } from './scale-runners/logger'; +import { logger } from './logger'; import { scaleDown } from './scale-runners/scale-down'; const body: ActionRequestMessage = { @@ -59,7 +58,7 @@ const context: Context = { jest.mock('./scale-runners/scale-up'); jest.mock('./scale-runners/scale-down'); -jest.mock('./scale-runners/logger'); +jest.mock('./logger'); describe('Test scale up lambda wrapper.', () => { it('Do not handle multiple record sets.', async () => { @@ -73,7 +72,7 @@ describe('Test scale up lambda wrapper.', () => { it('Scale without error should resolve.', async () => { const mock = mocked(scaleUp); mock.mockImplementation(() => { - return new Promise((resolve, reject) => { + return new Promise((resolve) => { resolve(); }); }); diff --git a/modules/runners/lambdas/runners/src/lambda.ts b/modules/runners/lambdas/runners/src/lambda.ts index 20e1c40135..d767595d61 100644 --- a/modules/runners/lambdas/runners/src/lambda.ts +++ b/modules/runners/lambdas/runners/src/lambda.ts @@ -1,7 +1,7 @@ import { scaleUp } from './scale-runners/scale-up'; import { scaleDown } from './scale-runners/scale-down'; -import { SQSEvent, ScheduledEvent, Context, Callback } from 'aws-lambda'; -import { LogFields, logger } from './scale-runners/logger'; +import { SQSEvent, ScheduledEvent, Context } from 'aws-lambda'; +import { LogFields, logger } from './logger'; import ScaleError from './scale-runners/ScaleError'; import 'source-map-support/register'; diff --git a/modules/runners/lambdas/runners/src/local.ts b/modules/runners/lambdas/runners/src/local.ts index e81f1dfe3a..1cd8e4f6cc 100644 --- a/modules/runners/lambdas/runners/src/local.ts +++ b/modules/runners/lambdas/runners/src/local.ts @@ -1,3 +1,4 @@ +import { logger } from './logger'; import { scaleUp, ActionRequestMessage } from './scale-runners/scale-up'; const sqsEvent = { @@ -8,11 +9,11 @@ const sqsEvent = { // eslint-disable-next-line max-len 'AQEBCpLYzDEKq4aKSJyFQCkJduSKZef8SJVOperbYyNhXqqnpFG5k74WygVAJ4O0+9nybRyeOFThvITOaS21/jeHiI5fgaM9YKuI0oGYeWCIzPQsluW5CMDmtvqv1aA8sXQ5n2x0L9MJkzgdIHTC3YWBFLQ2AxSveOyIHwW+cHLIFCAcZlOaaf0YtaLfGHGkAC4IfycmaijV8NSlzYgDuxrC9sIsWJ0bSvk5iT4ru/R4+0cjm7qZtGlc04k9xk5Fu6A+wRxMaIyiFRY+Ya19ykcevQldidmEjEWvN6CRToLgclk=', body: { - id: 19072, - repositoryName: 'ErrBud', - repositoryOwner: 'ActionsTest', - eventType: 'check_run', - installationId: 5, + repositoryName: 'self-hosted', + repositoryOwner: 'test-runners', + eventType: 'workflow_job', + id: 987654, + installationId: 123456789, }, attributes: { ApproximateReceiveCount: '1', @@ -31,8 +32,13 @@ const sqsEvent = { }, ], }; + export function run(): void { - scaleUp(sqsEvent.Records[0].eventSource, sqsEvent.Records[0].body as ActionRequestMessage); + scaleUp(sqsEvent.Records[0].eventSource, sqsEvent.Records[0].body as ActionRequestMessage) + .then() + .catch((e) => { + logger.error(e); + }); } run(); diff --git a/modules/runners/lambdas/runners/src/scale-runners/logger.ts b/modules/runners/lambdas/runners/src/logger.ts similarity index 100% rename from modules/runners/lambdas/runners/src/scale-runners/logger.ts rename to modules/runners/lambdas/runners/src/logger.ts diff --git a/modules/runners/lambdas/runners/src/scale-runners/modules.d.ts b/modules/runners/lambdas/runners/src/modules.d.ts similarity index 67% rename from modules/runners/lambdas/runners/src/scale-runners/modules.d.ts rename to modules/runners/lambdas/runners/src/modules.d.ts index 6b11e1a626..59d49e5b08 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/modules.d.ts +++ b/modules/runners/lambdas/runners/src/modules.d.ts @@ -13,5 +13,13 @@ declare namespace NodeJS { PARAMETER_GITHUB_APP_KEY_BASE64_NAME: string; SCALE_DOWN_CONFIG: string; SUBNET_IDS: string; + INSTANCE_TYPES: string; + INSTANCE_TARGET_CAPACITY_TYPE: 'on-demand' | 'spot'; + INSTANCE_MAX_SPOT_PRICE: string | undefined; + INSTANCE_ALLOCATION_STRATEGY: + | 'lowest-price' + | 'diversified' + | 'capacity-optimized' + | 'capacity-optimized-prioritized'; } } diff --git a/modules/runners/lambdas/runners/src/scale-runners/runners.test.ts b/modules/runners/lambdas/runners/src/scale-runners/runners.test.ts deleted file mode 100644 index 94e4aff0e2..0000000000 --- a/modules/runners/lambdas/runners/src/scale-runners/runners.test.ts +++ /dev/null @@ -1,231 +0,0 @@ -import { listEC2Runners, createRunner, terminateRunner, RunnerInfo } from './runners'; - -const mockEC2 = { describeInstances: jest.fn(), runInstances: jest.fn(), terminateInstances: jest.fn() }; -const mockSSM = { putParameter: jest.fn() }; -jest.mock('aws-sdk', () => ({ - EC2: jest.fn().mockImplementation(() => mockEC2), - SSM: jest.fn().mockImplementation(() => mockSSM), -})); - -const LAUNCH_TEMPLATE = 'lt-1'; -const ORG_NAME = 'SomeAwesomeCoder'; -const REPO_NAME = `${ORG_NAME}/some-amazing-library`; -const ENVIRONMENT = 'unit-test-environment'; - -describe('list instances', () => { - const mockDescribeInstances = { promise: jest.fn() }; - beforeEach(() => { - jest.clearAllMocks(); - mockEC2.describeInstances.mockImplementation(() => mockDescribeInstances); - const mockRunningInstances: AWS.EC2.DescribeInstancesResult = { - Reservations: [ - { - Instances: [ - { - LaunchTime: new Date('2020-10-10T14:48:00.000+09:00'), - InstanceId: 'i-1234', - Tags: [ - { Key: 'Application', Value: 'github-action-runner' }, - { Key: 'Type', Value: 'Org' }, - { Key: 'Owner', Value: 'CoderToCat' }, - ], - }, - { - LaunchTime: new Date('2020-10-11T14:48:00.000+09:00'), - InstanceId: 'i-5678', - Tags: [ - { Key: 'Owner', Value: REPO_NAME }, - { Key: 'Type', Value: 'Repo' }, - { Key: 'Application', Value: 'github-action-runner' }, - ], - }, - ], - }, - ], - }; - mockDescribeInstances.promise.mockReturnValue(mockRunningInstances); - }); - - it('returns a list of instances', async () => { - const resp = await listEC2Runners(); - expect(resp.length).toBe(2); - expect(resp).toContainEqual({ - instanceId: 'i-1234', - launchTime: new Date('2020-10-10T14:48:00.000+09:00'), - type: 'Org', - owner: 'CoderToCat', - }); - expect(resp).toContainEqual({ - instanceId: 'i-5678', - launchTime: new Date('2020-10-11T14:48:00.000+09:00'), - type: 'Repo', - owner: REPO_NAME, - }); - }); - - it('calls EC2 describe instances', async () => { - await listEC2Runners(); - expect(mockEC2.describeInstances).toBeCalled(); - }); - - it('filters instances on repo name', async () => { - await listEC2Runners({ runnerType: 'Repo', runnerOwner: REPO_NAME, environment: undefined }); - expect(mockEC2.describeInstances).toBeCalledWith({ - Filters: [ - { Name: 'tag:Application', Values: ['github-action-runner'] }, - { Name: 'instance-state-name', Values: ['running', 'pending'] }, - { Name: 'tag:Type', Values: ['Repo'] }, - { Name: 'tag:Owner', Values: [REPO_NAME] }, - ], - }); - }); - - it('filters instances on org name', async () => { - await listEC2Runners({ runnerType: 'Org', runnerOwner: ORG_NAME, environment: undefined }); - expect(mockEC2.describeInstances).toBeCalledWith({ - Filters: [ - { Name: 'tag:Application', Values: ['github-action-runner'] }, - { Name: 'instance-state-name', Values: ['running', 'pending'] }, - { Name: 'tag:Type', Values: ['Org'] }, - { Name: 'tag:Owner', Values: [ORG_NAME] }, - ], - }); - }); - - it('filters instances on environment', async () => { - await listEC2Runners({ environment: ENVIRONMENT }); - expect(mockEC2.describeInstances).toBeCalledWith({ - Filters: [ - { Name: 'tag:Application', Values: ['github-action-runner'] }, - { Name: 'instance-state-name', Values: ['running', 'pending'] }, - { Name: 'tag:Environment', Values: [ENVIRONMENT] }, - ], - }); - }); -}); - -describe('terminate runner', () => { - const mockTerminateInstances = { promise: jest.fn() }; - beforeEach(() => { - jest.clearAllMocks(); - mockEC2.terminateInstances.mockImplementation(() => mockTerminateInstances); - mockTerminateInstances.promise.mockReturnThis(); - }); - it('calls terminate instances with the right instance ids', async () => { - const runner: RunnerInfo = { - instanceId: 'instance-2', - owner: 'owner-2', - type: 'Repo', - }; - await terminateRunner(runner.instanceId); - - expect(mockEC2.terminateInstances).toBeCalledWith({ InstanceIds: [runner.instanceId] }); - }); -}); - -describe('create runner', () => { - const mockRunInstances = { promise: jest.fn() }; - const mockPutParameter = { promise: jest.fn() }; - beforeEach(() => { - jest.clearAllMocks(); - mockEC2.runInstances.mockImplementation(() => mockRunInstances); - mockRunInstances.promise.mockReturnValue({ - Instances: [ - { - InstanceId: 'i-1234', - }, - ], - }); - mockSSM.putParameter.mockImplementation(() => mockPutParameter); - process.env.SUBNET_IDS = 'sub-1234'; - }); - - it('calls run instances with the correct config for repo', async () => { - await createRunner( - { - runnerServiceConfig: 'bla', - environment: ENVIRONMENT, - runnerType: 'Repo', - runnerOwner: REPO_NAME, - }, - LAUNCH_TEMPLATE, - ); - expect(mockEC2.runInstances).toBeCalledWith({ - MaxCount: 1, - MinCount: 1, - LaunchTemplate: { LaunchTemplateName: LAUNCH_TEMPLATE, Version: '$Default' }, - SubnetId: 'sub-1234', - TagSpecifications: [ - { - ResourceType: 'instance', - Tags: [ - { Key: 'Application', Value: 'github-action-runner' }, - { Key: 'Type', Value: 'Repo' }, - { Key: 'Owner', Value: REPO_NAME }, - ], - }, - ], - }); - }); - - it('calls run instances with the correct config for org', async () => { - await createRunner( - { - runnerServiceConfig: 'bla', - environment: ENVIRONMENT, - runnerType: 'Org', - runnerOwner: ORG_NAME, - }, - LAUNCH_TEMPLATE, - ); - expect(mockEC2.runInstances).toBeCalledWith({ - MaxCount: 1, - MinCount: 1, - LaunchTemplate: { LaunchTemplateName: LAUNCH_TEMPLATE, Version: '$Default' }, - SubnetId: 'sub-1234', - TagSpecifications: [ - { - ResourceType: 'instance', - Tags: [ - { Key: 'Application', Value: 'github-action-runner' }, - { Key: 'Type', Value: 'Org' }, - { Key: 'Owner', Value: ORG_NAME }, - ], - }, - ], - }); - }); - - it('creates ssm parameters for each created instance', async () => { - await createRunner( - { - runnerServiceConfig: 'bla', - environment: ENVIRONMENT, - runnerType: 'Org', - runnerOwner: ORG_NAME, - }, - LAUNCH_TEMPLATE, - ); - expect(mockSSM.putParameter).toBeCalledWith({ - Name: `${ENVIRONMENT}-i-1234`, - Value: 'bla', - Type: 'SecureString', - }); - }); - - it('does not create ssm parameters when no instance is created', async () => { - mockRunInstances.promise.mockReturnValue({ - Instances: [], - }); - await createRunner( - { - runnerServiceConfig: 'bla', - environment: ENVIRONMENT, - runnerType: 'Org', - runnerOwner: ORG_NAME, - }, - LAUNCH_TEMPLATE, - ); - expect(mockSSM.putParameter).not.toBeCalled(); - }); -}); diff --git a/modules/runners/lambdas/runners/src/scale-runners/runners.ts b/modules/runners/lambdas/runners/src/scale-runners/runners.ts deleted file mode 100644 index 9670f4e025..0000000000 --- a/modules/runners/lambdas/runners/src/scale-runners/runners.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { EC2, SSM } from 'aws-sdk'; -import { logger as rootLogger, LogFields } from './logger'; - -const logger = rootLogger.getChildLogger({ name: 'runners' }); - -export interface RunnerList { - instanceId: string; - launchTime?: Date; - owner?: string; - type?: string; - repo?: string; - org?: string; -} - -export interface RunnerInfo { - instanceId: string; - launchTime?: Date; - owner: string; - type: string; -} - -export interface ListRunnerFilters { - runnerType?: 'Org' | 'Repo'; - runnerOwner?: string; - environment?: string; -} - -export interface RunnerInputParameters { - runnerServiceConfig: string; - environment: string; - runnerType: 'Org' | 'Repo'; - runnerOwner: string; -} - -export async function listEC2Runners(filters: ListRunnerFilters | undefined = undefined): Promise { - const ec2 = new EC2(); - const ec2Filters = [ - { Name: 'tag:Application', Values: ['github-action-runner'] }, - { Name: 'instance-state-name', Values: ['running', 'pending'] }, - ]; - if (filters) { - if (filters.environment !== undefined) { - ec2Filters.push({ Name: 'tag:Environment', Values: [filters.environment] }); - } - if (filters.runnerType && filters.runnerOwner) { - ec2Filters.push({ Name: `tag:Type`, Values: [filters.runnerType] }); - ec2Filters.push({ Name: `tag:Owner`, Values: [filters.runnerOwner] }); - } - } - const runningInstances = await ec2.describeInstances({ Filters: ec2Filters }).promise(); - const runners: RunnerList[] = []; - if (runningInstances.Reservations) { - for (const r of runningInstances.Reservations) { - if (r.Instances) { - for (const i of r.Instances) { - runners.push({ - instanceId: i.InstanceId as string, - launchTime: i.LaunchTime, - owner: i.Tags?.find((e) => e.Key === 'Owner')?.Value as string, - type: i.Tags?.find((e) => e.Key === 'Type')?.Value as string, - repo: i.Tags?.find((e) => e.Key === 'Repo')?.Value as string, - org: i.Tags?.find((e) => e.Key === 'Org')?.Value as string, - }); - } - } - } - } - return runners; -} - -export async function terminateRunner(instanceId: string): Promise { - const ec2 = new EC2(); - await ec2 - .terminateInstances({ - InstanceIds: [instanceId], - }) - .promise(); - logger.info(`Runner ${instanceId} has been terminated.`, LogFields.print()); -} - -export async function createRunner(runnerParameters: RunnerInputParameters, launchTemplateName: string): Promise { - logger.debug('Runner configuration: ' + JSON.stringify(runnerParameters), LogFields.print()); - const ec2 = new EC2(); - const runInstancesResponse = await ec2 - .runInstances(getInstanceParams(launchTemplateName, runnerParameters)) - .promise(); - logger.info( - 'Created instance(s): ', - runInstancesResponse.Instances?.map((i) => i.InstanceId).join(','), - LogFields.print(), - ); - const ssm = new SSM(); - if (runInstancesResponse.Instances) { - for (let i = 0; i < runInstancesResponse.Instances?.length; i++) { - await ssm - .putParameter({ - Name: runnerParameters.environment + '-' + (runInstancesResponse.Instances[i].InstanceId as string), - Value: runnerParameters.runnerServiceConfig, - Type: 'SecureString', - }) - .promise(); - } - } -} - -function getInstanceParams( - launchTemplateName: string, - runnerParameters: RunnerInputParameters, -): EC2.RunInstancesRequest { - return { - MaxCount: 1, - MinCount: 1, - LaunchTemplate: { - LaunchTemplateName: launchTemplateName, - Version: '$Default', - }, - SubnetId: getSubnet(), - TagSpecifications: [ - { - ResourceType: 'instance', - Tags: [ - { Key: 'Application', Value: 'github-action-runner' }, - { Key: 'Type', Value: runnerParameters.runnerType }, - { Key: 'Owner', Value: runnerParameters.runnerOwner }, - ], - }, - ], - }; -} - -function getSubnet(): string { - const subnets = process.env.SUBNET_IDS.split(','); - return subnets[Math.floor(Math.random() * subnets.length)]; -} diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts index 72616ee033..629be733c9 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/scale-down.test.ts @@ -1,8 +1,8 @@ import moment from 'moment'; import { mocked } from 'ts-jest/utils'; -import { listEC2Runners, terminateRunner, RunnerInfo, RunnerList } from './runners'; +import { listEC2Runners, terminateRunner, RunnerInfo, RunnerList } from './../aws/runners'; import { scaleDown } from './scale-down'; -import * as ghAuth from './gh-auth'; +import * as ghAuth from '../gh-auth/gh-auth'; import nock from 'nock'; import { Octokit } from '@octokit/rest'; import { githubCache } from './cache'; @@ -24,8 +24,8 @@ jest.mock('@octokit/rest', () => ({ Octokit: jest.fn().mockImplementation(() => mockOctokit), })); -jest.mock('./runners'); -jest.mock('./gh-auth'); +jest.mock('./../aws/runners'); +jest.mock('./../gh-auth/gh-auth'); jest.mock('./cache'); const mocktokit = Octokit as jest.MockedClass; diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts index 137ccac436..a26b833619 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts @@ -1,10 +1,10 @@ import { Octokit } from '@octokit/rest'; import moment from 'moment'; -import { listEC2Runners, RunnerInfo, RunnerList, terminateRunner } from './runners'; +import { listEC2Runners, RunnerInfo, RunnerList, terminateRunner } from './../aws/runners'; import { getIdleRunnerCount, ScalingDownConfig } from './scale-down-config'; -import { createOctoClient, createGithubAppAuth, createGithubInstallationAuth } from './gh-auth'; +import { createOctoClient, createGithubAppAuth, createGithubInstallationAuth } from '../gh-auth/gh-auth'; import { githubCache, GhRunners } from './cache'; -import { logger as rootLogger, LogFields } from './logger'; +import { logger as rootLogger, LogFields } from '../logger'; const logger = rootLogger.getChildLogger({ name: 'scale-down' }); diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts index 6bd0449f0b..6d5d61662a 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts @@ -1,7 +1,7 @@ import { mocked } from 'ts-jest/utils'; import * as scaleUpModule from './scale-up'; -import { listEC2Runners, createRunner, RunnerInputParameters } from './runners'; -import * as ghAuth from './gh-auth'; +import { listEC2Runners, createRunner, RunnerInputParameters } from './../aws/runners'; +import * as ghAuth from '../gh-auth/gh-auth'; import nock from 'nock'; import { Octokit } from '@octokit/rest'; import ScaleError from './ScaleError'; @@ -23,8 +23,8 @@ jest.mock('@octokit/rest', () => ({ Octokit: jest.fn().mockImplementation(() => mockOctokit), })); -jest.mock('./runners'); -jest.mock('./gh-auth'); +jest.mock('./../aws/runners'); +jest.mock('./../gh-auth/gh-auth'); const mocktokit = Octokit as jest.MockedClass; const mockedAppAuth = mocked(ghAuth.createGithubAppAuth, true); @@ -48,8 +48,6 @@ const TEST_DATA_WITH_ZERO_INSTALL_ID: scaleUpModule.ActionRequestMessage = { installationId: 0, }; -const LAUNCH_TEMPLATE = 'lt-1'; - const cleanEnv = process.env; const EXPECTED_RUNNER_PARAMS: RunnerInputParameters = { @@ -57,6 +55,13 @@ const EXPECTED_RUNNER_PARAMS: RunnerInputParameters = { runnerServiceConfig: `--url https://github.enterprise.something/${TEST_DATA.repositoryOwner} --token 1234abcd`, runnerType: 'Org', runnerOwner: TEST_DATA.repositoryOwner, + launchTemplateName: 'lt-1', + ec2instanceCriteria: { + instanceTypes: ['m5.large'], + targetCapacityType: 'spot', + instanceAllocationStrategy: 'lowest-price', + }, + subnets: ['subnet-123'], }; let expectedRunnerParams: RunnerInputParameters; @@ -71,7 +76,10 @@ beforeEach(() => { process.env.GITHUB_APP_CLIENT_SECRET = 'TEST_CLIENT_SECRET'; process.env.RUNNERS_MAXIMUM_COUNT = '3'; process.env.ENVIRONMENT = 'unit-test-environment'; - process.env.LAUNCH_TEMPLATE_NAME = 'lt-1,lt-2'; + process.env.LAUNCH_TEMPLATE_NAME = 'lt-1'; + process.env.SUBNET_IDS = 'subnet-123'; + process.env.INSTANCE_TYPES = 'm5.large'; + process.env.INSTANCE_TARGET_CAPACITY_TYPE = 'spot'; mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ data: { @@ -214,12 +222,12 @@ describe('scaleUp with GHES', () => { it('creates a runner with correct config', async () => { await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams, 'lt-1'); + expect(createRunner).toBeCalledWith(expectedRunnerParams); }); it('creates a runner with legacy event check_run', async () => { await scaleUpModule.scaleUp('aws:sqs', { ...TEST_DATA, eventType: 'check_run' }); - expect(createRunner).toBeCalledWith(expectedRunnerParams, 'lt-1'); + expect(createRunner).toBeCalledWith(expectedRunnerParams); }); it('creates a runner with labels in a specific group', async () => { @@ -228,27 +236,7 @@ describe('scaleUp with GHES', () => { await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expectedRunnerParams.runnerServiceConfig = expectedRunnerParams.runnerServiceConfig + ` --labels label1,label2 --runnergroup TEST_GROUP`; - expect(createRunner).toBeCalledWith(expectedRunnerParams, 'lt-1'); - }); - - it('attempts next launch template if first fails', async () => { - const mockCreateRunners = mocked(createRunner); - mockCreateRunners.mockRejectedValueOnce(new Error('no capactiy')); - await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); - expect(createRunner).toBeCalledTimes(2); - expect(createRunner).toHaveBeenNthCalledWith(1, expectedRunnerParams, 'lt-1'); - expect(createRunner).toHaveBeenNthCalledWith(2, expectedRunnerParams, 'lt-2'); - mockCreateRunners.mockReset(); - }); - - it('all launch templates fail', async () => { - const mockCreateRunners = mocked(createRunner); - mockCreateRunners.mockRejectedValue(new Error('All launch templates failed')); - await expect(scaleUpModule.scaleUp('aws:sqs', TEST_DATA)).rejects.toThrow('All launch templates failed'); - expect(createRunner).toBeCalledTimes(2); - expect(createRunner).toHaveBeenNthCalledWith(1, expectedRunnerParams, 'lt-1'); - expect(createRunner).toHaveBeenNthCalledWith(2, expectedRunnerParams, 'lt-2'); - mockCreateRunners.mockReset(); + expect(createRunner).toBeCalledWith(expectedRunnerParams); }); }); @@ -328,7 +316,7 @@ describe('scaleUp with GHES', () => { process.env.RUNNER_EXTRA_LABELS = 'label1,label2'; await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expectedRunnerParams.runnerServiceConfig = expectedRunnerParams.runnerServiceConfig + ` --labels label1,label2`; - expect(createRunner).toBeCalledWith(expectedRunnerParams, 'lt-1'); + expect(createRunner).toBeCalledWith(expectedRunnerParams); }); it('creates a runner and ensure the group argument is ignored', async () => { @@ -336,25 +324,13 @@ describe('scaleUp with GHES', () => { process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expectedRunnerParams.runnerServiceConfig = expectedRunnerParams.runnerServiceConfig + ` --labels label1,label2`; - expect(createRunner).toBeCalledWith(expectedRunnerParams, 'lt-1'); - }); - - it('attempts next launch template if first fails', async () => { - const mockCreateRunners = mocked(createRunner); - mockCreateRunners.mockRejectedValueOnce(new Error('no capactiy')); - await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); - expect(createRunner).toBeCalledTimes(2); - expect(createRunner).toHaveBeenNthCalledWith(1, expectedRunnerParams, 'lt-1'); - expect(createRunner).toHaveBeenNthCalledWith(2, expectedRunnerParams, 'lt-2'); + expect(createRunner).toBeCalledWith(expectedRunnerParams); }); - it('all launch templates fail', async () => { + it('Check error is thrown', async () => { const mockCreateRunners = mocked(createRunner); - mockCreateRunners.mockRejectedValue(new Error('All launch templates failed')); - await expect(scaleUpModule.scaleUp('aws:sqs', TEST_DATA)).rejects.toThrow('All launch templates failed'); - expect(createRunner).toBeCalledTimes(2); - expect(createRunner).toHaveBeenNthCalledWith(1, expectedRunnerParams, 'lt-1'); - expect(createRunner).toHaveBeenNthCalledWith(2, expectedRunnerParams, 'lt-2'); + mockCreateRunners.mockRejectedValue(new Error('no retry')); + await expect(scaleUpModule.scaleUp('aws:sqs', TEST_DATA)).rejects.toThrow('no retry'); mockCreateRunners.mockReset(); }); }); @@ -403,6 +379,14 @@ describe('scaleUp with public GH', () => { expect(listEC2Runners).not.toBeCalled(); }); + it('does not list runners when no workflows are queued (check_run)', async () => { + mockOctokit.checks.get.mockImplementation(() => ({ + data: { status: 'completed' }, + })); + await scaleUpModule.scaleUp('aws:sqs', { ...TEST_DATA, eventType: 'check_run' }); + expect(listEC2Runners).not.toBeCalled(); + }); + describe('on org level', () => { beforeEach(() => { process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; @@ -437,12 +421,12 @@ describe('scaleUp with public GH', () => { it('creates a runner with correct config', async () => { await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams, LAUNCH_TEMPLATE); + expect(createRunner).toBeCalledWith(expectedRunnerParams); }); it('creates a runner with legacy event check_run', async () => { await scaleUpModule.scaleUp('aws:sqs', { ...TEST_DATA, eventType: 'check_run' }); - expect(createRunner).toBeCalledWith(expectedRunnerParams, LAUNCH_TEMPLATE); + expect(createRunner).toBeCalledWith(expectedRunnerParams); }); it('creates a runner with labels in s specific group', async () => { @@ -451,16 +435,7 @@ describe('scaleUp with public GH', () => { await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expectedRunnerParams.runnerServiceConfig = expectedRunnerParams.runnerServiceConfig + ` --labels label1,label2 --runnergroup TEST_GROUP`; - expect(createRunner).toBeCalledWith(expectedRunnerParams, LAUNCH_TEMPLATE); - }); - - it('attempts next launch template if first fails', async () => { - const mockCreateRunners = mocked(createRunner); - mockCreateRunners.mockRejectedValueOnce(new Error('no capactiy')); - await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); - expect(createRunner).toBeCalledTimes(2); - expect(createRunner).toHaveBeenNthCalledWith(1, expectedRunnerParams, 'lt-1'); - expect(createRunner).toHaveBeenNthCalledWith(2, expectedRunnerParams, 'lt-2'); + expect(createRunner).toBeCalledWith(expectedRunnerParams); }); }); @@ -523,7 +498,7 @@ describe('scaleUp with public GH', () => { process.env.RUNNER_EXTRA_LABELS = 'label1,label2'; await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expectedRunnerParams.runnerServiceConfig = expectedRunnerParams.runnerServiceConfig + ` --labels label1,label2`; - expect(createRunner).toBeCalledWith(expectedRunnerParams, LAUNCH_TEMPLATE); + expect(createRunner).toBeCalledWith(expectedRunnerParams); }); it('creates a runner and ensure the group argument is ignored', async () => { @@ -531,16 +506,7 @@ describe('scaleUp with public GH', () => { process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expectedRunnerParams.runnerServiceConfig = expectedRunnerParams.runnerServiceConfig + ` --labels label1,label2`; - expect(createRunner).toBeCalledWith(expectedRunnerParams, LAUNCH_TEMPLATE); - }); - - it('attempts next launch template if first fails', async () => { - const mockCreateRunners = mocked(createRunner); - mockCreateRunners.mockRejectedValueOnce(new Error('no capactiy')); - await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); - expect(createRunner).toBeCalledTimes(2); - expect(createRunner).toHaveBeenNthCalledWith(1, expectedRunnerParams, 'lt-1'); - expect(createRunner).toHaveBeenNthCalledWith(2, expectedRunnerParams, 'lt-2'); + expect(createRunner).toBeCalledWith(expectedRunnerParams); }); it('ephemeral runners only run with workflow_job event, others should fail.', async () => { @@ -557,7 +523,7 @@ describe('scaleUp with public GH', () => { process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expectedRunnerParams.runnerServiceConfig = expectedRunnerParams.runnerServiceConfig + ` --ephemeral`; - expect(createRunner).toBeCalledWith(expectedRunnerParams, LAUNCH_TEMPLATE); + expect(createRunner).toBeCalledWith(expectedRunnerParams); }); it('Scaling error should cause reject so retry can be triggered.', async () => { diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts index 8422c09540..296219e274 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts @@ -1,8 +1,8 @@ -import { listEC2Runners, createRunner, RunnerInputParameters } from './runners'; -import { createOctoClient, createGithubAppAuth, createGithubInstallationAuth } from './gh-auth'; +import { listEC2Runners, createRunner, RunnerInputParameters } from './../aws/runners'; +import { createOctoClient, createGithubAppAuth, createGithubInstallationAuth } from '../gh-auth/gh-auth'; import yn from 'yn'; import { Octokit } from '@octokit/rest'; -import { LogFields, logger as rootLogger } from './logger'; +import { LogFields, logger as rootLogger } from '../logger'; import ScaleError from './ScaleError'; const logger = rootLogger.getChildLogger({ name: 'scale-up' }); @@ -15,6 +15,109 @@ export interface ActionRequestMessage { installationId: number; } +interface CreateGitHubRunnerConfig { + ephemeral: boolean; + ghesBaseUrl: string; + runnerExtraLabels: string | undefined; + runnerGroup: string | undefined; + runnerOwner: string; + runnerType: 'Org' | 'Repo'; +} + +interface CreateEC2RunnerConfig { + environment: string; + subnets: string[]; + launchTemplateName: string; + ec2instanceCriteria: RunnerInputParameters['ec2instanceCriteria']; +} + +function generateRunnerServiceConfig(githubRunnerConfig: CreateGitHubRunnerConfig, token: any) { + const labelsArgument = + githubRunnerConfig.runnerExtraLabels !== undefined ? `--labels ${githubRunnerConfig.runnerExtraLabels} ` : ''; + const runnerGroupArgument = + githubRunnerConfig.runnerGroup !== undefined ? `--runnergroup ${githubRunnerConfig.runnerGroup} ` : ''; + const configBaseUrl = githubRunnerConfig.ghesBaseUrl ? githubRunnerConfig.ghesBaseUrl : 'https://github.com'; + const ephemeralArgument = githubRunnerConfig.ephemeral ? '--ephemeral ' : ''; + const runnerArgs = `--token ${token} ${labelsArgument}${ephemeralArgument}`; + return githubRunnerConfig.runnerType === 'Org' + ? `--url ${configBaseUrl}/${githubRunnerConfig.runnerOwner} ${runnerArgs}${runnerGroupArgument}`.trim() + : `--url ${configBaseUrl}/${githubRunnerConfig.runnerOwner} ${runnerArgs}`.trim(); +} + +async function getGithubRunnerRegistrationToken(githubRunnerConfig: CreateGitHubRunnerConfig, ghClient: Octokit) { + const registrationToken = + githubRunnerConfig.runnerType === 'Org' + ? await ghClient.actions.createRegistrationTokenForOrg({ org: githubRunnerConfig.runnerOwner }) + : await ghClient.actions.createRegistrationTokenForRepo({ + owner: githubRunnerConfig.runnerOwner.split('/')[0], + repo: githubRunnerConfig.runnerOwner.split('/')[1], + }); + return registrationToken.data.token; +} + +async function getInstallationId(ghesApiUrl: string, enableOrgLevel: boolean, payload: ActionRequestMessage) { + if (payload.installationId !== 0) { + return payload.installationId; + } + + const ghAuth = await createGithubAppAuth(undefined, ghesApiUrl); + const githubClient = await createOctoClient(ghAuth.token, ghesApiUrl); + return enableOrgLevel + ? ( + await githubClient.apps.getOrgInstallation({ + org: payload.repositoryOwner, + }) + ).data.id + : ( + await githubClient.apps.getRepoInstallation({ + owner: payload.repositoryOwner, + repo: payload.repositoryName, + }) + ).data.id; +} + +async function isJobQueued(githubInstallationClient: Octokit, payload: ActionRequestMessage): Promise { + let isQueued = false; + if (payload.eventType === 'workflow_job') { + const jobForWorkflowRun = await githubInstallationClient.actions.getJobForWorkflowRun({ + job_id: payload.id, + owner: payload.repositoryOwner, + repo: payload.repositoryName, + }); + isQueued = jobForWorkflowRun.data.status === 'queued'; + } else if (payload.eventType === 'check_run') { + const checkRun = await githubInstallationClient.checks.get({ + check_run_id: payload.id, + owner: payload.repositoryOwner, + repo: payload.repositoryName, + }); + isQueued = checkRun.data.status === 'queued'; + } else { + throw Error(`Event ${payload.eventType} is not supported`); + } + if (!isQueued) { + logger.info(`Job not queued`, LogFields.print()); + } + return isQueued; +} + +export async function createRunners( + githubRunnerConfig: CreateGitHubRunnerConfig, + ec2RunnerConfig: CreateEC2RunnerConfig, + ghClient: Octokit, +): Promise { + const token = await getGithubRunnerRegistrationToken(githubRunnerConfig, ghClient); + + const runnerServiceConfig = generateRunnerServiceConfig(githubRunnerConfig, token); + + await createRunner({ + runnerServiceConfig, + runnerType: githubRunnerConfig.runnerType, + runnerOwner: githubRunnerConfig.runnerOwner, + ...ec2RunnerConfig, + }); +} + export async function scaleUp(eventSource: string, payload: ActionRequestMessage): Promise { logger.info( `Received ${payload.eventType} from ${payload.repositoryOwner}/${payload.repositoryName}`, @@ -28,7 +131,13 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage const runnerGroup = process.env.RUNNER_GROUP_NAME; const environment = process.env.ENVIRONMENT; const ghesBaseUrl = process.env.GHES_URL; + const subnets = process.env.SUBNET_IDS.split(','); + const instanceTypes = process.env.INSTANCE_TYPES.split(','); + const instanceTargetTargetCapacityType = process.env.INSTANCE_TARGET_CAPACITY_TYPE; const ephemeralEnabled = yn(process.env.ENABLE_EPHEMERAL_RUNNERS, { default: false }); + const launchTemplateName = process.env.LAUNCH_TEMPLATE_NAME; + const instanceMaxSpotPrice = process.env.INSTANCE_MAX_SPOT_PRICE; + const instanceAllocationStrategy = process.env.INSTANCE_ALLOCATION_STRATEGY || 'lowest-price'; // same as AWS default if (ephemeralEnabled && payload.eventType !== 'workflow_job') { logger.warn( @@ -57,28 +166,11 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage ghesApiUrl = `${ghesBaseUrl}/api/v3`; } - let installationId = payload.installationId; - if (installationId == 0) { - const ghAuth = await createGithubAppAuth(undefined, ghesApiUrl); - const githubClient = await createOctoClient(ghAuth.token, ghesApiUrl); - installationId = enableOrgLevel - ? ( - await githubClient.apps.getOrgInstallation({ - org: payload.repositoryOwner, - }) - ).data.id - : ( - await githubClient.apps.getRepoInstallation({ - owner: payload.repositoryOwner, - repo: payload.repositoryName, - }) - ).data.id; - } - + const installationId = await getInstallationId(ghesApiUrl, enableOrgLevel, payload); const ghAuth = await createGithubInstallationAuth(installationId, ghesApiUrl); const githubInstallationClient = await createOctoClient(ghAuth.token, ghesApiUrl); - if (ephemeral || (await getJobStatus(githubInstallationClient, payload))) { + if (ephemeral || (await isJobQueued(githubInstallationClient, payload))) { const currentRunners = await listEC2Runners({ environment, runnerType, @@ -88,29 +180,29 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage if (currentRunners.length < maximumRunners) { logger.info(`Attempting to launch a new runner`, LogFields.print()); - // create token - const registrationToken = enableOrgLevel - ? await githubInstallationClient.actions.createRegistrationTokenForOrg({ org: payload.repositoryOwner }) - : await githubInstallationClient.actions.createRegistrationTokenForRepo({ - owner: payload.repositoryOwner, - repo: payload.repositoryName, - }); - const token = registrationToken.data.token; - - const labelsArgument = runnerExtraLabels !== undefined ? `--labels ${runnerExtraLabels} ` : ''; - const runnerGroupArgument = runnerGroup !== undefined ? `--runnergroup ${runnerGroup} ` : ''; - const configBaseUrl = ghesBaseUrl ? ghesBaseUrl : 'https://github.com'; - const ephemeralArgument = ephemeral ? '--ephemeral ' : ''; - const runnerArgs = `--token ${token} ${labelsArgument}${ephemeralArgument}`; - - await createRunnerLoop({ - environment, - runnerServiceConfig: enableOrgLevel - ? `--url ${configBaseUrl}/${payload.repositoryOwner} ${runnerArgs}${runnerGroupArgument}`.trim() - : `--url ${configBaseUrl}/${payload.repositoryOwner}/${payload.repositoryName} ${runnerArgs}`.trim(), - runnerOwner, - runnerType, - }); + + await createRunners( + { + ephemeral, + ghesBaseUrl, + runnerExtraLabels, + runnerGroup, + runnerOwner, + runnerType, + }, + { + ec2instanceCriteria: { + instanceTypes, + targetCapacityType: instanceTargetTargetCapacityType, + maxSpotPrice: instanceMaxSpotPrice, + instanceAllocationStrategy: instanceAllocationStrategy, + }, + environment, + launchTemplateName, + subnets, + }, + githubInstallationClient, + ); } else { logger.info('No runner will be created, maximum number of runners reached.', LogFields.print()); if (ephemeral) { @@ -119,47 +211,3 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage } } } - -async function getJobStatus(githubInstallationClient: Octokit, payload: ActionRequestMessage): Promise { - let isQueued = false; - if (payload.eventType === 'workflow_job') { - const jobForWorkflowRun = await githubInstallationClient.actions.getJobForWorkflowRun({ - job_id: payload.id, - owner: payload.repositoryOwner, - repo: payload.repositoryName, - }); - isQueued = jobForWorkflowRun.data.status === 'queued'; - } else if (payload.eventType === 'check_run') { - const checkRun = await githubInstallationClient.checks.get({ - check_run_id: payload.id, - owner: payload.repositoryOwner, - repo: payload.repositoryName, - }); - isQueued = checkRun.data.status === 'queued'; - } else { - throw Error(`Event ${payload.eventType} is not supported`); - } - if (!isQueued) { - logger.info(`Job not queued`, LogFields.print()); - } - return isQueued; -} - -export async function createRunnerLoop(runnerParameters: RunnerInputParameters): Promise { - const launchTemplateNames = process.env.LAUNCH_TEMPLATE_NAME?.split(',') as string[]; - let launched = false; - for (let i = 0; i < launchTemplateNames.length; i++) { - logger.info(`Attempt '${i}' to launch instance using ${launchTemplateNames[i]}.`, LogFields.print()); - try { - await createRunner(runnerParameters, launchTemplateNames[i]); - launched = true; - break; - } catch (error) { - logger.debug(`Attempt '${i}' to launch instance using ${launchTemplateNames[i]} FAILED.`, LogFields.print()); - logger.error(error, LogFields.print()); - } - } - if (launched == false) { - throw new ScaleError('All launch templates failed'); - } -} diff --git a/modules/runners/lambdas/runners/tsconfig.json b/modules/runners/lambdas/runners/tsconfig.json index 764f2f6ae0..eab8079cbb 100644 --- a/modules/runners/lambdas/runners/tsconfig.json +++ b/modules/runners/lambdas/runners/tsconfig.json @@ -62,5 +62,9 @@ }, "include": [ "src/**/*" - ] -} + ], + // required to process types defined in module.d.ts + "ts-node": { + "files": true + } +} \ No newline at end of file diff --git a/modules/runners/main.tf b/modules/runners/main.tf index a6deec5b05..cf73ab22bf 100644 --- a/modules/runners/main.tf +++ b/modules/runners/main.tf @@ -13,7 +13,6 @@ locals { lambda_zip = var.lambda_zip == null ? "${path.module}/lambdas/runners/runners.zip" : var.lambda_zip userdata_template = var.userdata_template == null ? local.default_userdata_template[var.runner_os] : var.userdata_template userdata_arm_patch = "${path.module}/templates/arm-runner-patch.tpl" - instance_types = distinct(var.instance_types == null ? [var.instance_type] : var.instance_types) kms_key_arn = var.kms_key_arn != null ? var.kms_key_arn : "" default_ami = { @@ -54,9 +53,7 @@ data "aws_ami" "runner" { } resource "aws_launch_template" "runner" { - count = length(local.instance_types) - - name = "${var.environment}-action-runner-${local.instance_types[count.index]}" + name = "${var.environment}-action-runner" dynamic "block_device_mappings" { for_each = [var.block_device_mappings] @@ -88,18 +85,8 @@ resource "aws_launch_template" "runner" { } instance_initiated_shutdown_behavior = "terminate" - - dynamic "instance_market_options" { - for_each = var.market_options != null ? [var.market_options] : [] - - content { - market_type = instance_market_options.value - } - } - - image_id = data.aws_ami.runner.id - instance_type = local.instance_types[count.index] - key_name = var.key_name + image_id = data.aws_ami.runner.id + key_name = var.key_name vpc_security_group_ids = compact(concat( [aws_security_group.runner_sg.id], diff --git a/modules/runners/policies/lambda-scale-up.json b/modules/runners/policies/lambda-scale-up.json index 057bb77068..e194535bc9 100644 --- a/modules/runners/policies/lambda-scale-up.json +++ b/modules/runners/policies/lambda-scale-up.json @@ -6,25 +6,13 @@ "Action": [ "ec2:DescribeInstances", "ec2:DescribeTags", - "ec2:RunInstances" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet", "ec2:CreateTags" ], "Resource": [ "*" - ], - "Condition": { - "StringEquals": { - "ec2:CreateAction": "RunInstances" - } - } + ] }, { "Effect": "Allow", diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf index e217cf4b74..632439520e 100644 --- a/modules/runners/scale-up.tf +++ b/modules/runners/scale-up.tf @@ -18,7 +18,7 @@ resource "aws_lambda_function" "scale_up" { ENABLE_ORGANIZATION_RUNNERS = var.enable_organization_runners ENVIRONMENT = var.environment GHES_URL = var.ghes_url - LAUNCH_TEMPLATE_NAME = join(",", [for template in aws_launch_template.runner : template.name]) + LAUNCH_TEMPLATE_NAME = aws_launch_template.runner.name LOG_LEVEL = var.log_level LOG_TYPE = var.log_type NODE_TLS_REJECT_UNAUTHORIZED = var.ghes_url != null && !var.ghes_ssl_verify ? 0 : 1 @@ -29,6 +29,10 @@ resource "aws_lambda_function" "scale_up" { RUNNERS_MAXIMUM_COUNT = var.runners_maximum_count SUBNET_IDS = join(",", var.subnet_ids) ENABLE_EPHEMERAL_RUNNERS = var.enable_ephemeral_runners + INSTANCE_TYPES = join(",", var.instance_types) + INSTANCE_TARGET_CAPACITY_TYPE = var.instance_target_capacity_type + INSTANCE_MAX_SPOT_PRICE = var.instance_max_spot_price + INSTANCE_ALLOCATION_STRATEGY = var.instance_allocation_strategy } } diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index 6e092f48ab..7b6606b91c 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -52,9 +52,42 @@ variable "block_device_mappings" { } variable "market_options" { - description = "Market options for the action runner instances." + description = "DEPCRECATED: Replaced by `instance_target_capacity_type`." + type = string + default = null + + validation { + condition = anytrue([var.market_options == null]) + error_message = "Deprecated, replaced by `instance_target_capacity_type`." + } +} + +variable "instance_target_capacity_type" { + description = "Default lifecyle used runner instances, can be either `spot` or `on-demand`." type = string default = "spot" + + validation { + condition = contains(["spot", "on-demand"], var.instance_target_capacity_type) + error_message = "The instance target capacity should be either spot or on-demand." + } +} + +variable "instance_allocation_strategy" { + description = "The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`." + type = string + default = "lowest-price" + + validation { + condition = contains(["lowest-price", "diversified", "capacity-optimized", "capacity-optimized-prioritized"], var.instance_allocation_strategy) + error_message = "The instance allocation strategy does not match the allowed values." + } +} + +variable "instance_max_spot_price" { + description = "Max price price for spot intances per hour. This variable will be passed to the create fleet as max spot price for the fleet." + type = string + default = null } variable "runner_os" { diff --git a/outputs.tf b/outputs.tf index 8dcc418b77..f650ac2ece 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,8 +1,8 @@ output "runners" { value = { - launch_template_name = [for template in module.runners.launch_template : template.name] - launch_template_id = [for template in module.runners.launch_template : template.id] - launch_template_version = [for template in module.runners.launch_template : template.latest_version] + launch_template_name = module.runners.launch_template.name + launch_template_id = module.runners.launch_template.id + launch_template_version = module.runners.launch_template.latest_version lambda_up = module.runners.lambda_scale_up lambda_down = module.runners.lambda_scale_down role_runner = module.runners.role_runner diff --git a/variables.tf b/variables.tf index 02c200ba7f..3ff01c3bd5 100644 --- a/variables.tf +++ b/variables.tf @@ -141,12 +141,6 @@ variable "instance_profile_path" { default = null } -variable "instance_type" { - description = "[DEPRECATED] See instance_types." - type = string - default = "m5.large" -} - variable "runner_as_root" { description = "Run the action runner under the root user. Variable `runner_run_as` will be ingored." type = bool @@ -346,9 +340,40 @@ variable "runner_additional_security_group_ids" { } variable "market_options" { - description = "Market options for the action runner instances. Setting the value to `null` let the scaler create on-demand instances instead of spot instances." + description = "DEPCRECATED: Replaced by `instance_target_capacity_type`." + type = string + default = null + + validation { + condition = anytrue([var.market_options == null]) + error_message = "Deprecated, replaced by `instance_target_capacity_type`." + } +} + +variable "instance_target_capacity_type" { + description = "Default lifecycle used for runner instances, can be either `spot` or `on-demand`." type = string default = "spot" + validation { + condition = contains(["spot", "on-demand"], var.instance_target_capacity_type) + error_message = "The instance target capacity should be either spot or on-demand." + } +} + +variable "instance_allocation_strategy" { + description = "The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`." + type = string + default = "lowest-price" + validation { + condition = contains(["lowest-price", "diversified", "capacity-optimized", "capacity-optimized-prioritized"], var.instance_allocation_strategy) + error_message = "The instance allocation strategy does not match the allowed values." + } +} + +variable "instance_max_spot_price" { + description = "Max price price for spot intances per hour. This variable will be passed to the create fleet as max spot price for the fleet." + type = string + default = null } variable "volume_size" { @@ -357,10 +382,21 @@ variable "volume_size" { default = 30 } +variable "instance_type" { + description = "[DEPRECATED] See instance_types." + type = string + default = null + + validation { + condition = anytrue([var.instance_type == null]) + error_message = "Deprecated, replaced by `instance_types`." + } +} + variable "instance_types" { description = "List of instance types for the action runner. Defaults are based on runner_os (amzn2 for linux and Windows Server Core for win)." type = list(string) - default = null + default = ["m5.large", "c5.large"] } variable "repository_white_list" { @@ -506,3 +542,14 @@ variable "redrive_build_queue" { error_message = "Ensure you have set the maxReceiveCount when enabled." } } + + +variable "runner_architecture" { + description = "The platform architecture of the runner instance_type." + type = string + default = "x64" + validation { + condition = contains(["x64", "arm64"], var.runner_architecture) + error_message = "`runner_architecture` value not valid, valid values are: `x64` and `arm64`." + } +}