limit messages delay to 900s on retry (#1966)

pytorch · Jan 31, 2023 · 698132e · 698132e
1 parent 9afcf35
commit 698132e
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 5 deletions.
diff --git a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/lambda.ts b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/lambda.ts
@@ -39,10 +39,13 @@ export async function scaleUp(event: SQSEvent, context: Context, callback: any)
           (Config.Instance.retryScaleUpRecordQueueUrl?.length ?? 0) > 0
         ) {
           body.retryCount = retryCount + 1;
-          body.delaySeconds = getDelayWithJitterRetryCount(
-            retryCount,
-            Math.max(Config.Instance.retryScaleUpRecordDelayS, 20),
-            Config.Instance.retryScaleUpRecordJitterPct,
+          body.delaySeconds = Math.min(
+            900,
+            getDelayWithJitterRetryCount(
+              retryCount,
+              Math.max(Config.Instance.retryScaleUpRecordDelayS, 20),
+              Config.Instance.retryScaleUpRecordJitterPct,
+            ),
           );
 
           const sqsPayload: SQS.SendMessageRequest = {

diff --git a/terraform-aws-github-runner/modules/runners/scale-up.tf b/terraform-aws-github-runner/modules/runners/scale-up.tf
@@ -48,7 +48,7 @@ resource "aws_lambda_function" "scale_up" {
       LAUNCH_TEMPLATE_VERSION_WINDOWS       = aws_launch_template.windows_runner.latest_version
       MAX_RETRY_SCALEUP_RECORD              = "12"
       MUST_HAVE_ISSUES_LABELS               = join(",", var.must_have_issues_labels)
-      RETRY_SCALE_UP_RECORD_DELAY_S         = "30"
+      RETRY_SCALE_UP_RECORD_DELAY_S         = "90"
       RETRY_SCALE_UP_RECORD_JITTER_PCT      = "0.15"
       RETRY_SCALE_UP_RECORD_QUEUE_URL       = var.sqs_build_queue_retry.url
       RUNNER_EXTRA_LABELS                   = var.runner_extra_labels