fix: Honor runner boot time when topping up pools

Rebased off of the patch provided by Michael Poutre <[email protected]> in philips-labs/terraform-aws-github-runner#2590 and re-targeted against the develop branch. Co-authored-by: Ryan McKern <[email protected]>
github-aws-runners · Jan 11, 2023 · 8a8701a · 8a8701a
1 parent 680f9aa
commit 8a8701a
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 5 deletions.
diff --git a/modules/runners/lambdas/runners/src/pool/pool.ts b/modules/runners/lambdas/runners/src/pool/pool.ts
@@ -1,9 +1,10 @@
 import yn from 'yn';
 
-import { listEC2Runners } from '../aws/runners';
+import { listEC2Runners, RunnerList } from '../aws/runners';
 import { createGithubAppAuth, createGithubInstallationAuth, createOctoClient } from '../gh-auth/gh-auth';
 import { logger as rootLogger } from '../logger';
 import { createRunners } from '../scale-runners/scale-up';
+import moment from "moment";
 
 const logger = rootLogger.getChildLogger({ name: 'pool' });
 
@@ -45,7 +46,7 @@ export async function adjust(event: PoolEvent): Promise<void> {
       per_page: 100,
     },
   );
-  const idleRunners = runners.filter((r) => !r.busy && r.status === 'online').map((r) => r.name);
+  const githubIdleRunners = runners.filter((r) => !r.busy && r.status === 'online');
 
   // Look up the managed ec2 runners in AWS, but running does not mean idle
   const ec2runners = (
@@ -55,9 +56,13 @@ export async function adjust(event: PoolEvent): Promise<void> {
       runnerType: 'Org',
       statuses: ['running'],
     })
-  ).map((r) => r.instanceId);
+  );
+
+  // Runner should be considered idle if it is still booting, or is idle in GitHub
+  const managedIdleRunners = ec2runners.filter((r) => {
+    return githubIdleRunners.some(ghRunner => ghRunner.name == r.instanceId) || !bootTimeExceeded(r);
+  }).map((r) => r.instanceId);
 
-  const managedIdleRunners = ec2runners.filter((r) => idleRunners.includes(r));
   const topUp = event.poolSize - managedIdleRunners.length;
   if (topUp > 0) {
     logger.info(`The pool will be topped up with ${topUp} runners.`);
@@ -87,7 +92,7 @@ export async function adjust(event: PoolEvent): Promise<void> {
       githubInstallationClient,
     );
   } else {
-    logger.info(`Pool will not be topped up. Find ${managedIdleRunners} managed idle runners.`);
+    logger.info(`Pool will not be topped up. Found ${managedIdleRunners} managed idle runners.`);
   }
 }
 
@@ -101,3 +106,9 @@ async function getInstallationId(ghesApiUrl: string, org: string): Promise<numbe
     })
   ).data.id;
 }
+
+function bootTimeExceeded(ec2Runner: RunnerList): boolean {
+  const runnerBootTimeInMinutes = process.env.RUNNER_BOOT_TIME_IN_MINUTES;
+  const launchTimePlusBootTime = moment(ec2Runner.launchTime).utc().add(runnerBootTimeInMinutes, 'minutes');
+  return launchTimePlusBootTime < moment(new Date()).utc();
+}
diff --git a/modules/runners/pool.tf b/modules/runners/pool.tf
@@ -37,6 +37,7 @@ module "pool" {
     runner = {
       disable_runner_autoupdate = var.disable_runner_autoupdate
       ephemeral                 = var.enable_ephemeral_runners
+      boot_time_in_minutes      = var.runner_boot_time_in_minutes
       extra_labels              = var.runner_extra_labels
       launch_template           = aws_launch_template.runner
       group_name                = var.runner_group_name

diff --git a/modules/runners/pool/main.tf b/modules/runners/pool/main.tf
@@ -31,6 +31,7 @@ resource "aws_lambda_function" "pool" {
       NODE_TLS_REJECT_UNAUTHORIZED         = var.config.ghes.url != null && !var.config.ghes.ssl_verify ? 0 : 1
       PARAMETER_GITHUB_APP_ID_NAME         = var.config.github_app_parameters.id.name
       PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github_app_parameters.key_base64.name
+      RUNNER_BOOT_TIME_IN_MINUTES          = var.config.runner.boot_time_in_minutes
       RUNNER_EXTRA_LABELS                  = var.config.runner.extra_labels
       RUNNER_GROUP_NAME                    = var.config.runner.group_name
       RUNNER_OWNER                         = var.config.runner.pool_owner

diff --git a/modules/runners/pool/variables.tf b/modules/runners/pool/variables.tf
@@ -29,6 +29,7 @@ variable "config" {
     runner = object({
       disable_runner_autoupdate = bool
       ephemeral                 = bool
+      boot_time_in_minutes      = number
       extra_labels              = string
       launch_template = object({
         name = string