gitlab-runner-autoscaling.ts

import * as path from 'path';
import * as cdk from 'aws-cdk-lib';
import * as asg from 'aws-cdk-lib/aws-autoscaling';
import { FunctionHook } from 'aws-cdk-lib/aws-autoscaling-hooktargets';
import * as ec2 from 'aws-cdk-lib/aws-ec2';
import * as iam from 'aws-cdk-lib/aws-iam';
import * as lambda from 'aws-cdk-lib/aws-lambda';
import * as logs from 'aws-cdk-lib/aws-logs';
import * as assets from 'aws-cdk-lib/aws-s3-assets';
import * as sns from 'aws-cdk-lib/aws-sns';
import * as subscriptions from 'aws-cdk-lib/aws-sns-subscriptions';
import * as cr from 'aws-cdk-lib/custom-resources';
// eslint-disable-next-line import/no-extraneous-dependencies
import { compare } from 'compare-versions';
import { Construct } from 'constructs';
import { DockerVolumes } from './gitlab-runner-interfaces';

/**
 * GitlabRunnerAutoscaling Props.
 */
export interface GitlabRunnerAutoscalingProps {
  /**
   * Gitlab Runner version
   * Please give me gitlab runner version.
   */
  readonly gitlabRunnerVersion: string;
  /**
   * Gitlab token.
   *
   * @example
   * new GitlabRunnerAutoscaling(stack, 'runner', { gitlabToken: 'GITLAB_TOKEN' });
   */
  readonly gitlabToken: string;

  /**
   * Image URL of Gitlab Runner.
   *
   * @example
   * new GitlabRunnerAutoscaling(stack, 'runner', { gitlabToken: 'GITLAB_TOKEN', gitlabRunnerImage: 'gitlab/gitlab-runner:alpine' });
   *
   * @default public.ecr.aws/gitlab/gitlab-runner:latest
   *
   */
  readonly gitlabRunnerImage?: string;

  /**
   * Runner default EC2 instance type.
   *
   * @example
   * new GitlabRunnerAutoscaling(stack, 'runner', { gitlabToken: 'GITLAB_TOKEN', instanceType: 't3.small' });
   *
   * @default - t3.micro
   *
   */
  readonly instanceType?: string;

  /**
   * VPC for the Gitlab Runner .
   *
   * @example
   * const newVpc = new Vpc(stack, 'NewVPC', {
   *   ipAddresses: IpAddresses.cidr('10.0.0.0/16'),
   *   maxAzs: 2,
   *   subnetConfiguration: [{
   *     cidrMask: 26,
   *     name: 'RunnerVPC',
   *     subnetType: SubnetType.PUBLIC,
   *   }],
   *   natGateways: 0,
   * });
   *
   * new GitlabRunnerAutoscaling(stack, 'runner', { gitlabToken: 'GITLAB_TOKEN', vpc: newVpc });
   *
   * @default - A new VPC will be created.
   *
   */
  readonly vpc?: ec2.IVpc;

  /**
   * IAM role for the Gitlab Runner Instance .
   *
   * @example
   * const role = new Role(stack, 'runner-role', {
   *   assumedBy: new ServicePrincipal('ec2.amazonaws.com'),
   *   description: 'For Gitlab Runner Test Role',
   *   roleName: 'Runner-Role',
   * });
   *
   * new GitlabRunnerAutoscaling(stack, 'runner', { gitlabToken: 'GITLAB_TOKEN', instanceRole: role });
   *
   * @default - new Role for Gitlab Runner Instance , attach AmazonSSMManagedInstanceCore Policy .
   *
   */
  readonly instanceRole?: iam.IRole;

  /**
   * Run worker nodes as EC2 Spot
   *
   * @default - false
   */
  readonly spotInstance?: boolean;

  /**
   * Minimum capacity limit for autoscaling group.
   *
   * @example
   * new GitlabRunnerAutoscaling(stack, 'runner', { gitlabToken: 'GITLAB_TOKEN', minCapacity: 2 });
   *
   * @default - minCapacity: 1
   *
   */
  readonly minCapacity?: number;

  /**
   * Maximum capacity limit for autoscaling group.
   *
   * @example
   * new GitlabRunnerAutoscaling(stack, 'runner', { gitlabToken: 'GITLAB_TOKEN', maxCapacity: 4 });
   *
   * @default - desiredCapacity
   *
   */
  readonly maxCapacity?: number;

  /**
   * Desired capacity limit for autoscaling group.
   *
   * @example
   * new GitlabRunnerAutoscaling(stack, 'runner', { gitlabToken: 'GITLAB_TOKEN', desiredCapacity: 2 });
   *
   * @default - minCapacity, and leave unchanged during deployment
   *
   */
  readonly desiredCapacity?: number;

  /**
   * tags for the runner
   *
   * @default - ['runner', 'gitlab', 'awscdk']
   */
  readonly tags?: string[];

  /**
   * Gitlab Runner register url .
   *
   * @example
   * const runner = new GitlabRunnerAutoscaling(stack, 'runner', { gitlabToken: 'GITLAB_TOKEN',gitlabUrl: 'https://gitlab.com/'});
   *
   * @default - https://gitlab.com/ , The trailing slash is mandatory.
   *
   */
  readonly gitlabUrl?: string;

  /**
   * Gitlab Runner instance EBS size .
   *
   * @deprecated , use ebsConfig
   *
   */
  readonly ebsSize?: number;

  /**
   * Gitlab Runner instance EBS config.
   *
   * @example
   * const runner = new GitlabRunnerAutoscaling(stack, 'runner', { gitlabToken: 'GITLAB_TOKEN', ebsConfig: { volumeSize: 60}});
   *
   * @default - ebsConfig={ volumeSize: 60}
   *
   */
  readonly ebsConfig?: ec2.CfnLaunchTemplate.EbsProperty;

  /**
   * VPC subnet
   *
   * @example
   * const vpc = new Vpc(stack, 'nat', {
   * natGateways: 1,
   * maxAzs: 2,
   * });
   * const runner = new GitlabRunnerAutoscaling(stack, 'testing', {
   *   gitlabToken: 'GITLAB_TOKEN',
   *   instanceType: 't3.large',
   *   instanceRole: role,
   *   vpc: vpc,
   *   vpcSubnet: {
   *     subnetType: SubnetType.PUBLIC,
   *   },
   * });
   *
   * @default - SubnetType.PRIVATE subnet
   */
  readonly vpcSubnet?: ec2.SubnetSelection;

  /**
   * add another Gitlab Container Runner Docker Volumes Path at job runner runtime.
   *
   * more detail see https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runnersdocker-section
   *
   * @default - already mount "/var/run/docker.sock:/var/run/docker.sock"
   *
   * @example
   * dockerVolumes: [
   *   {
   *     hostPath: '/tmp/cache',
   *     containerPath: '/tmp/cache',
   *   },
   * ],
   */
  readonly dockerVolumes?: DockerVolumes[];

  /**
   * Parameters of put_metric_alarm function
   *
   * https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudwatch.html#CloudWatch.Client.put_metric_alarm
   *
   * @default - [{
   *     AlarmName: 'GitlabRunnerDiskUsage',
   *     MetricName: 'disk_used_percent',
   * }]
   *
   */
  readonly alarms?: object[];
}

/**
 * GitlabRunnerAutoscaling Construct for create Autoscaling Gitlab Runner.
 */
export class GitlabRunnerAutoscaling extends Construct {
  /**
   * The IAM role assumed by the Runner instance.
   */
  public readonly instanceRole: iam.IRole;

  /**
   * This represents a Runner Auto Scaling Group
   */
  public readonly autoscalingGroup: asg.AutoScalingGroup;

  /**
   * The EC2 runner's VPC.
   */
  public readonly vpc: ec2.IVpc;

  /**
   * The EC2 runner's default SecurityGroup.
   */
  public readonly securityGroup: ec2.ISecurityGroup;

  /**
   * The SNS topic to suscribe alarms for EC2 runner's metrics.
   */
  public readonly topicAlarm: sns.ITopic;


  constructor(scope: Construct, id: string, props: GitlabRunnerAutoscalingProps) {
    super(scope, id);
    const defaultProps = {
      instanceType: 't3.micro',
      tags: ['gitlab', 'awscdk', 'runner'],
      gitlabUrl: 'https://gitlab.com/',
      gitlabRunnerImage: 'public.ecr.aws/gitlab/gitlab-runner:latest',
      alarms: [
        {
          AlarmName: 'GitlabRunnerDiskUsage',
          MetricName: 'disk_used_percent',
        },
      ],
    };
    const runnerProps = { ...defaultProps, ...props };
    if (compare(props.gitlabRunnerVersion, '15.10', '>=') && props.gitlabToken.includes('glrt-') === false) {
      throw new Error('If gitlabRunnerVersion >= 15.10, gitlabtoken please give glrt-xxxxxxx @see https://docs.gitlab.com/ee/ci/runners/new_creation_workflow.html');
    }
    const asset = new assets.Asset(this, 'GitlabRunnerUserDataAsset', {
      path: path.join(__dirname, '../assets/userdata/amazon-cloudwatch-agent.json'),
    });

    const userData = ec2.UserData.forLinux();
    userData.addS3DownloadCommand({
      bucket: asset.bucket,
      bucketKey: asset.s3ObjectKey,
      localFile: '/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json',
    });
    userData.addCommands(...this.createUserData(runnerProps));

    this.instanceRole =
      runnerProps.instanceRole ??
      new iam.Role(this, 'GitlabRunnerInstanceRole', {
        assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'),
        description: 'For EC2 Instance (Gitlab Runner) Role',
        managedPolicies: [
          iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'),
          iam.ManagedPolicy.fromAwsManagedPolicyName('CloudWatchAgentServerPolicy'),
          iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonS3ReadOnlyAccess'),
        ],
      });

    this.vpc = runnerProps.vpc ?? new ec2.Vpc(this, 'VPC');

    this.securityGroup = new ec2.SecurityGroup(this, 'GitlabRunnerSecurityGroup', {
      vpc: this.vpc,
    });
    const instanceProfile = new iam.CfnInstanceProfile(this, 'InstanceProfile', {
      roles: [this.instanceRole.roleName],
    });
    const lt = new ec2.CfnLaunchTemplate(this, 'GitlabRunnerLaunchTemplate', {
      launchTemplateData: {
        imageId: ec2.MachineImage.latestAmazonLinux2().getImage(this).imageId,
        instanceType: runnerProps.instanceType,
        instanceMarketOptions: {
          marketType: runnerProps.spotInstance ? 'spot' : undefined,
          spotOptions: runnerProps.spotInstance ? {
            spotInstanceType: 'one-time',
          } : undefined,
        },
        userData: cdk.Fn.base64(userData.render()),
        blockDeviceMappings: [
          {
            deviceName: '/dev/xvda',
            ebs: runnerProps.ebsConfig ?? {
              volumeSize: 60,
            },
          },
        ],
        iamInstanceProfile: {
          arn: instanceProfile.attrArn,
        },
        securityGroupIds: this.securityGroup.connections.securityGroups.map(
          (m) => m.securityGroupId,
        ),
      },
    });

    this.autoscalingGroup = new asg.AutoScalingGroup(this, 'GitlabRunnerAutoscalingGroup', {
      instanceType: new ec2.InstanceType(runnerProps.instanceType),
      autoScalingGroupName: `Gitlab Runners (${runnerProps.instanceType})`,
      vpc: this.vpc,
      vpcSubnets: runnerProps.vpcSubnet,
      machineImage: ec2.MachineImage.latestAmazonLinux2(),
      minCapacity: runnerProps.minCapacity,
      maxCapacity: runnerProps.maxCapacity,
      desiredCapacity: runnerProps.desiredCapacity,
    });

    const cfnAsg = this.autoscalingGroup.node.tryFindChild('ASG') as asg.CfnAutoScalingGroup;
    cfnAsg.addPropertyDeletionOverride('LaunchConfigurationName');
    cfnAsg.addPropertyOverride('LaunchTemplate', {
      LaunchTemplateId: lt.ref,
      Version: lt.attrLatestVersionNumber,
    });
    this.autoscalingGroup.node.tryRemoveChild('LaunchConfig');

    this.topicAlarm = new sns.Topic(this, 'GitlabRunnerAlarm');
    const alarms = JSON.stringify(runnerProps.alarms);

    // Put alarms at launch
    const registerFunction = new lambda.Function(this, 'GitlabRunnerRegisterFunction', {
      code: lambda.Code.fromAsset(path.join(__dirname, '../assets/functions')),
      handler: 'autoscaling_events.register',
      runtime: lambda.Runtime.PYTHON_3_8,
      timeout: cdk.Duration.seconds(60),
      logRetention: logs.RetentionDays.ONE_DAY,
      environment: {
        ALARMS: alarms,
        SNS_TOPIC_ARN: this.topicAlarm.topicArn,
      },
    });
    registerFunction.role?.addToPrincipalPolicy(
      new iam.PolicyStatement({
        effect: iam.Effect.ALLOW,
        resources: ['*'],
        actions: [
          'cloudwatch:PutMetricAlarm',
        ],
      }),
    );

    this.autoscalingGroup.addLifecycleHook('GitlabRunnerLifeCycleHookLaunching', {
      lifecycleTransition: asg.LifecycleTransition.INSTANCE_LAUNCHING,
      notificationTarget: new FunctionHook(registerFunction),
      defaultResult: asg.DefaultResult.CONTINUE,
      heartbeatTimeout: cdk.Duration.seconds(60),
    });

    // Add an alarm action to terminate invalid instances
    const alarmAction = new lambda.Function(this, 'GitlabRunnerAlarmAction', {
      code: lambda.Code.fromAsset(path.join(__dirname, '../assets/functions')),
      handler: 'autoscaling_events.on_alarm',
      runtime: lambda.Runtime.PYTHON_3_8,
      timeout: cdk.Duration.seconds(60),
      logRetention: logs.RetentionDays.ONE_DAY,
    });
    alarmAction.role?.addToPrincipalPolicy(
      new iam.PolicyStatement({
        effect: iam.Effect.ALLOW,
        resources: ['*'],
        actions: [
          'autoscaling:SetInstanceHealth',
        ],
      }),
    );
    const alarmSubscription = new subscriptions.LambdaSubscription(alarmAction);
    this.topicAlarm.addSubscription(alarmSubscription);

    // Unregister gitlab runners and remove alarms on instance termination or CFn stack deletion
    const unregisterRole = new iam.Role(this, 'GitlabRunnerUnregisterRole', {
      assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'),
      description: 'For Gitlab Runner Unregistering Function Role',
      managedPolicies: [
        iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole'),
      ],
    });
    unregisterRole.addToPrincipalPolicy(
      new iam.PolicyStatement({
        effect: iam.Effect.ALLOW,
        resources: ['*'],
        actions: [
          'ssm:SendCommand',
          'autoscaling:DescribeAutoScalingGroups',
          'cloudwatch:DeleteAlarms',
        ],
      }),
    );

    const unregisterFunction = new lambda.Function(this, 'GitlabRunnerUnregisterFunction', {
      code: lambda.Code.fromAsset(path.join(__dirname, '../assets/functions')),
      handler: 'autoscaling_events.unregister',
      runtime: lambda.Runtime.PYTHON_3_8,
      timeout: cdk.Duration.seconds(60),
      role: unregisterRole,
      logRetention: logs.RetentionDays.ONE_DAY,
      environment: {
        ALARMS: alarms,
      },
    });

    this.autoscalingGroup.addLifecycleHook('GitlabRunnerLifeCycleHookTerminating', {
      lifecycleTransition: asg.LifecycleTransition.INSTANCE_TERMINATING,
      notificationTarget: new FunctionHook(unregisterFunction),
      defaultResult: asg.DefaultResult.CONTINUE,
      heartbeatTimeout: cdk.Duration.seconds(60),
    });

    const unregisterCustomResource = new lambda.Function(this, 'GitlabRunnerUnregisterCustomResource', {
      code: lambda.Code.fromAsset(path.join(__dirname, '../assets/functions')),
      handler: 'autoscaling_events.on_event',
      runtime: lambda.Runtime.PYTHON_3_8,
      role: unregisterRole,
      logRetention: logs.RetentionDays.ONE_DAY,
      environment: {
        ALARMS: alarms,
      },
    });

    const unregisterProvider = new cr.Provider(this, 'GitlabRunnerUnregisterProvider', {
      onEventHandler: unregisterCustomResource,
    });

    const customResource = new cdk.CustomResource(this, 'GitlabRunnerCustomResource', {
      serviceToken: unregisterProvider.serviceToken,
      properties: {
        AutoScalingGroupNames: [this.autoscalingGroup.autoScalingGroupName],
      },
    });
    customResource.node.addDependency(unregisterProvider);

    new cdk.CfnOutput(this, 'GitlabRunnerAutoScalingGroupArn', {
      value: this.autoscalingGroup.autoScalingGroupArn,
    });
  }

  private dockerVolumesList(dockerVolume: DockerVolumes[] | undefined): string {
    let tempString: string = '--docker-volumes "/var/run/docker.sock:/var/run/docker.sock"';
    if (dockerVolume) {
      let tempList: string[] = [];
      dockerVolume.forEach(e => {
        tempList.push(`"${e.hostPath}:${e.containerPath}"`);
      });
      tempList.forEach(e => {
        tempString = `${tempString} --docker-volumes ${e}`;
      });
    }
    return tempString;
  }
  /**
   * @param props
   * @returns Array.
   */
  public createUserData(props: GitlabRunnerAutoscalingProps): string[] {
    return [
      'yum update -y',
      'sleep 15 && amazon-linux-extras install docker && yum install -y amazon-cloudwatch-agent && systemctl start docker && usermod -aG docker ec2-user && chmod 777 /var/run/docker.sock',
      'systemctl restart docker && systemctl enable docker && systemctl start amazon-cloudwatch-agent && systemctl enable amazon-cloudwatch-agent',
      `docker run -d -v /home/ec2-user/.gitlab-runner:/etc/gitlab-runner -v /var/run/docker.sock:/var/run/docker.sock \
      --name gitlab-runner-register ${props.gitlabRunnerImage} register --non-interactive --url ${props.gitlabUrl} ${compare(props.gitlabRunnerVersion, '15.10', '>=') ? '--token' : '--registration-token'} ${props.gitlabToken} \
      --docker-pull-policy if-not-present ${this.dockerVolumesList(props?.dockerVolumes)} \
      --executor docker --docker-image "alpine:latest" --description "A Runner on EC2 Instance (${props.instanceType})" \
      ${compare(props.gitlabRunnerVersion, '15.10', '>=') ? undefined : `--tag-list "${props.tags?.join(',')}" `} --docker-privileged`,
      `sleep 2 && docker run --restart always -d -v /home/ec2-user/.gitlab-runner:/etc/gitlab-runner -v /var/run/docker.sock:/var/run/docker.sock --name gitlab-runner ${props.gitlabRunnerImage}`,
    ];
  }
}