diff --git a/examples/default/main.tf b/examples/default/main.tf index e69de29bb2..6b96a39a98 100644 --- a/examples/default/main.tf +++ b/examples/default/main.tf @@ -0,0 +1,18 @@ +locals { + environment = "default-action-runners" + aws_region = "eu-west-1" +} + +module "runners" { + source = "../../" + + aws_region = local.aws_region + vpc_id = module.vpc.vpc_id + + environment = local.environment + tags = { + Project = "ProjectX" + } + +} + diff --git a/examples/default/outputs.tf b/examples/default/outputs.tf new file mode 100644 index 0000000000..08e04595ed --- /dev/null +++ b/examples/default/outputs.tf @@ -0,0 +1,5 @@ +output "action_runners" { + value = { + runners = module.runners.runners + } +} diff --git a/examples/default/providers.tf b/examples/default/providers.tf new file mode 100644 index 0000000000..582241f0ed --- /dev/null +++ b/examples/default/providers.tf @@ -0,0 +1,5 @@ +provider "aws" { + region = local.aws_region + version = "2.59" +} + diff --git a/examples/default/vpc.tf b/examples/default/vpc.tf new file mode 100644 index 0000000000..3a9fba6c1f --- /dev/null +++ b/examples/default/vpc.tf @@ -0,0 +1,7 @@ +module "vpc" { + source = "git::https://github.com/philips-software/terraform-aws-vpc.git?ref=2.1.0" + + environment = local.environment + aws_region = local.aws_region +} + diff --git a/main.tf b/main.tf index e69de29bb2..73913bb76c 100644 --- a/main.tf +++ b/main.tf @@ -0,0 +1,64 @@ +resource "random_string" "random" { + length = 24 + special = false + upper = false +} + +module "dsitrubtion_cache" { + source = "./modules/action-runner-binary-cache" + + aws_region = var.aws_region + environment = var.environment + tags = var.tags + + distribution_bucket_name = "${var.environment}-dist-${random_string.random.result}" +} + +module "runners" { + source = "./modules/runners" + + aws_region = var.aws_region + vpc_id = var.vpc_id + environment = var.environment + tags = var.tags + + s3_location_runner_distribution = module.dsitrubtion_cache.s3_location_runner_distribution +} + + +resource "aws_iam_policy" "dist_bucket" { + name = "${var.environment}-gh-distribution-bucket" + path = "/" + description = "Policy for the runner to download the github action runner." + + policy = templatefile("${path.module}/policies/action-runner-s3-policy.json", + { + s3_arn = module.dsitrubtion_cache.distribution_bucket.arn + } + ) +} + +resource "aws_iam_role_policy_attachment" "dist_bucket" { + role = module.runners.role.name + policy_arn = aws_iam_policy.dist_bucket.arn +} + +resource "aws_resourcegroups_group" "resourcegroups_group" { + name = "${var.environment}-group" + + resource_query { + query = <<-JSON +{ + "ResourceTypeFilters": [ + "AWS::AllSupported" + ], + "TagFilters": [ + { + "Key": "Environment", + "Values": ["${var.environment}"] + } + ] +} + JSON + } +} diff --git a/modules/action-runner-binary-cache/main.tf b/modules/action-runner-binary-cache/main.tf new file mode 100644 index 0000000000..d2fc57bf2f --- /dev/null +++ b/modules/action-runner-binary-cache/main.tf @@ -0,0 +1,11 @@ +locals { + action_runner_distribution_object_key = "actions-runner-linux.tar.gz" +} + +resource "aws_s3_bucket" "action_dist" { + bucket = var.distribution_bucket_name + acl = "private" + force_destroy = true + tags = var.tags +} + diff --git a/modules/action-runner-binary-cache/outputs.tf b/modules/action-runner-binary-cache/outputs.tf new file mode 100644 index 0000000000..66873c4807 --- /dev/null +++ b/modules/action-runner-binary-cache/outputs.tf @@ -0,0 +1,7 @@ +output "distribution_bucket" { + value = aws_s3_bucket.action_dist +} + +output "s3_location_runner_distribution" { + value = "s3://${aws_s3_bucket.action_dist.id}/${local.action_runner_distribution_object_key}" +} diff --git a/modules/action-runner-binary-cache/variables.tf b/modules/action-runner-binary-cache/variables.tf new file mode 100644 index 0000000000..8fc974a157 --- /dev/null +++ b/modules/action-runner-binary-cache/variables.tf @@ -0,0 +1,21 @@ +variable "aws_region" { + description = "AWS region." + type = string +} + +variable "tags" { + description = "Map of tags that will be added to created resources. By default resources will be tagged with name and environment." + type = map(string) + default = {} +} + +variable "environment" { + description = "A name that identifies the environment, used as prefix and for tagging." + type = string +} + +variable "distribution_bucket_name" { + description = "Bucket for storing the action runner distribution." + type = string +} + diff --git a/modules/runners/README.md b/modules/runners/README.md new file mode 100644 index 0000000000..e19418db31 --- /dev/null +++ b/modules/runners/README.md @@ -0,0 +1,8 @@ +# Action runner module + +The module create resources to facilitate the `orchestrator labmda` to recreate action runners. + +- *launch template* : A launch template is created that can create an action runner, by default a spot instance is requested. For configuration parameters SSM is used. +- *security group* : Security groups attached to the action runner. +- *s3 bucket* : To avoid the action runner distribution to be downloaded from Github every time (which could be slow), a version is cached in a S3 bucket. +- *policies and roles* : Policies and roles for the action runner. By default the session manager is enabled diff --git a/modules/runners/main.tf b/modules/runners/main.tf new file mode 100644 index 0000000000..f9e91d6b91 --- /dev/null +++ b/modules/runners/main.tf @@ -0,0 +1,93 @@ +locals { + name_sg = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"] + + tags = merge( + { + "Name" = format("%s", var.environment) + }, + { + "Environment" = format("%s", var.environment) + }, + var.tags, + ) +} + +data "aws_ami" "runner" { + most_recent = "true" + + dynamic "filter" { + for_each = var.ami_filter + content { + name = filter.key + values = filter.value + } + } + + owners = var.ami_owners +} + +resource "aws_launch_template" "runner" { + name = "${var.environment}-action-runner" + + dynamic "block_device_mappings" { + for_each = [var.block_device_mappings] + content { + device_name = "/dev/xvda" + + ebs { + delete_on_termination = lookup(block_device_mappings.value, "delete_on_termination", true) + volume_type = lookup(block_device_mappings.value, "volume_type", "gp2") + volume_size = lookup(block_device_mappings.value, "volume_size", 30) + encrypted = lookup(block_device_mappings.value, "encrypted", true) + iops = lookup(block_device_mappings.value, "iops", null) + } + } + } + + iam_instance_profile { + name = aws_iam_instance_profile.runner.name + } + + instance_initiated_shutdown_behavior = "terminate" + + instance_market_options { + market_type = var.market_options + } + + image_id = data.aws_ami.runner.id + instance_type = var.instance_type + + vpc_security_group_ids = [aws_security_group.runner_sg.id] + + tag_specifications { + resource_type = "instance" + tags = local.tags + } + + user_data = base64encode(templatefile("${path.module}/templates/user-data.sh", { + environment = var.environment + pre_install = var.userdata_pre_install + post_install = var.userdata_post_install + s3_location_runner_distribution = var.s3_location_runner_distribution + })) +} + +resource "aws_security_group" "runner_sg" { + name_prefix = "${var.environment}-github-actions-runner-sg" + description = "Github Actions Runner security group" + + vpc_id = var.vpc_id + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + tags = merge( + local.tags, + { + "Name" = format("%s", local.name_sg) + }, + ) +} diff --git a/modules/runners/outputs.tf b/modules/runners/outputs.tf new file mode 100644 index 0000000000..2417f29b0e --- /dev/null +++ b/modules/runners/outputs.tf @@ -0,0 +1,7 @@ +output "launch_template" { + value = aws_launch_template.runner +} + +output "role" { + value = aws_iam_role.runner +} diff --git a/modules/runners/policies.tf b/modules/runners/policies.tf new file mode 100644 index 0000000000..080bbbd98e --- /dev/null +++ b/modules/runners/policies.tf @@ -0,0 +1,47 @@ +data "aws_caller_identity" "current" {} + +resource "aws_iam_role" "runner" { + name = "${var.environment}-github-action-runners-runner-role" + assume_role_policy = templatefile("${path.module}/policies/instance-role-trust-policy.json", {}) + tags = local.tags +} + +resource "aws_iam_instance_profile" "runner" { + name = "${var.environment}-github-action-runners-profile" + role = aws_iam_role.runner.name +} + +resource "aws_iam_policy" "runner_session_manager_policy" { + name = "${var.environment}-github-action-runners-session-manager" + path = "/" + description = "Policy session manager." + + policy = templatefile("${path.module}/policies/instance-session-manager-policy.json", {}) +} + +resource "aws_iam_role_policy_attachment" "runner_session_manager_policy" { + role = aws_iam_role.runner.name + policy_arn = aws_iam_policy.runner_session_manager_policy.arn +} + +resource "aws_iam_role_policy_attachment" "runner_session_manager_aws_managed" { + role = aws_iam_role.runner.name + policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} + +resource "aws_iam_policy" "ssm_parameters" { + name = "${var.environment}-runner-ssm-parameters" + path = "/" + description = "Policy for the runner to download the github action runner." + + policy = templatefile("${path.module}/policies/instance-ssm-parameters-policy.json", + { + arn_ssm_parameters = "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter/${var.environment}-*" + } + ) +} + +resource "aws_iam_role_policy_attachment" "ssm_parameters" { + role = aws_iam_role.runner.name + policy_arn = aws_iam_policy.ssm_parameters.arn +} diff --git a/modules/runners/policies/instance-role-trust-policy.json b/modules/runners/policies/instance-role-trust-policy.json new file mode 100644 index 0000000000..45bf82b081 --- /dev/null +++ b/modules/runners/policies/instance-role-trust-policy.json @@ -0,0 +1,13 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/modules/runners/policies/instance-session-manager-policy.json b/modules/runners/policies/instance-session-manager-policy.json new file mode 100644 index 0000000000..63ac8ca941 --- /dev/null +++ b/modules/runners/policies/instance-session-manager-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ssmmessages:CreateControlChannel", + "ssmmessages:CreateDataChannel", + "ssmmessages:OpenControlChannel", + "ssmmessages:OpenDataChannel" + ], + "Resource": "*" + } + ] +} \ No newline at end of file diff --git a/modules/runners/policies/instance-ssm-parameters-policy.json b/modules/runners/policies/instance-ssm-parameters-policy.json new file mode 100644 index 0000000000..bb43708b64 --- /dev/null +++ b/modules/runners/policies/instance-ssm-parameters-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["ssm:DeleteParameter"], + "Resource": "${arn_ssm_parameters}" + }, + { + "Effect": "Allow", + "Action": ["ssm:GetParameters"], + "Resource": "${arn_ssm_parameters}" + } + ] +} diff --git a/modules/runners/templates/user-data.sh b/modules/runners/templates/user-data.sh new file mode 100644 index 0000000000..3d390fb174 --- /dev/null +++ b/modules/runners/templates/user-data.sh @@ -0,0 +1,55 @@ +#!/bin/bash -e +exec > >(tee /var/log/user-data.log | logger -t user-data -s 2>/dev/console) 2>&1 + +yum update -y + +# Install docker +amazon-linux-extras install docker +service docker start +usermod -a -G docker ec2-user + +# Install runner +yum install -y curl jq git +cd /home/ec2-user +mkdir actions-runner && cd actions-runner +#!/bin/bash -ex +exec > >(tee /var/log/user-data.log | logger -t user-data -s 2>/dev/console) 2>&1 + +yum update -y + +${pre_install} + +# Install docker +amazon-linux-extras install docker +service docker start +usermod -a -G docker ec2-user + +# Install runner +yum install -y curl jq git + +cd /home/ec2-user +mkdir actions-runner && cd actions-runner +aws s3 cp ${s3_location_runner_distribution} actions-runner.tar.gz +tar xzf ./actions-runner.tar.gz +rm actions-runner.tar.gz + +INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id) +REGION=$(curl -s 169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region) + +echo wait for configuration +while [[ $(aws ssm get-parameters --names ${environment}-$INSTANCE_ID --with-decryption --region $REGION | jq -r ".Parameters | .[0] | .Value") == null ]]; do + echo Waiting for configuration ... + sleep 1 +done +CONFIG=$(aws ssm get-parameters --names ${environment}-$INSTANCE_ID --with-decryption --region $REGION | jq -r ".Parameters | .[0] | .Value") +aws ssm delete-parameter --name ${environment}-$INSTANCE_ID --region $REGION + +export RUNNER_ALLOW_RUNASROOT=1 +./config.sh --unattended --name $INSTANCE_ID --work "_work" $CONFIG + +chown -R ec2-user:ec2-user . +./svc.sh install ec2-user + +${post_install} + +./svc.sh start diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf new file mode 100644 index 0000000000..19a8785043 --- /dev/null +++ b/modules/runners/variables.tf @@ -0,0 +1,80 @@ +variable "aws_region" { + description = "AWS region." + type = string +} + +variable "vpc_id" { + description = "The VPC for the security groupss." + type = string +} + +variable "overrides" { + description = "This maps provides the possibility to override some defaults. The following attributes are supported: `name_sg` overwrite the `Name` tag for all security groups created by this module. `name_runner_agent_instance` override the `Name` tag for the ec2 instance defined in the auto launch configuration. `name_docker_machine_runners` ovverrid the `Name` tag spot instances created by the runner agent." + type = map(string) + + default = { + name_sg = "" + } +} + +variable "tags" { + description = "Map of tags that will be added to created resources. By default resources will be tagged with name and environment." + type = map(string) + default = {} +} + +variable "environment" { + description = "A name that identifies the environment, used as prefix and for tagging." + type = string +} + +variable "s3_location_runner_distribution" { + description = "S3 location of runner distribution." + type = string +} + +variable "block_device_mappings" { + description = "The EC2 instance block device configuration. Takes the following keys: `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`" + type = map(string) + default = {} +} + +variable "market_options" { + description = "Market options for the action runner instances." + type = string + default = "spot" +} + +variable "instance_type" { + description = "Default instance type for the action runner." + type = string + default = "m5.large" +} + +variable "ami_filter" { + description = "List of maps used to create the AMI filter for the action runner AMI." + type = map(list(string)) + + default = { + name = ["amzn2-ami-hvm-2.*-x86_64-ebs"] + } +} + +variable "ami_owners" { + description = "The list of owners used to select the AMI of action runner instances." + type = list(string) + default = ["amazon"] +} + + +variable "userdata_pre_install" { + description = "User-data script snippet to insert before GitHub acton runner install" + type = string + default = "" +} + +variable "userdata_post_install" { + description = "User-data script snippet to insert after GitHub acton runner install" + type = string + default = "" +} diff --git a/outputs.tf b/outputs.tf index e69de29bb2..24c1ba34a9 100644 --- a/outputs.tf +++ b/outputs.tf @@ -0,0 +1,8 @@ +output "runners" { + value = { + launch_template_name = module.runners.launch_template.name + launch_template_id = module.runners.launch_template.id + launch_template_version = module.runners.launch_template.latest_version + action_runner_distribution = module.dsitrubtion_cache.s3_location_runner_distribution + } +} diff --git a/policies/action-runner-s3-policy.json b/policies/action-runner-s3-policy.json new file mode 100644 index 0000000000..68d21ebce6 --- /dev/null +++ b/policies/action-runner-s3-policy.json @@ -0,0 +1,11 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "githubActionDist", + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:GetObjectAcl"], + "Resource": ["${s3_arn}/*"] + } + ] +} diff --git a/variables.tf b/variables.tf index e69de29bb2..3b664c05a5 100644 --- a/variables.tf +++ b/variables.tf @@ -0,0 +1,21 @@ +variable "aws_region" { + description = "AWS region." + type = string +} + +variable "vpc_id" { + description = "The VPC for security groups of the action runners." + type = string +} + +variable "tags" { + description = "Map of tags that will be added to created resources. By default resources will be tagged with name and environment." + type = map(string) + default = {} +} + +variable "environment" { + description = "A name that identifies the environment, used as prefix and for tagging." + type = string +} +