Merge pull request #8 from alphagov/persistent-storage

Persistent storage

terraform/projects/app-ecs-instances/ecs-iam-policy.tf

@@ -24,6 +24,8 @@ resource "aws_iam_role" "instance_iam_role" {
 EOF
 }
 
+data "aws_caller_identity" "current" {}
+
 data "aws_iam_policy_document" "ecs_instance_document" {
   statement {
     sid = "ECSInstancePolicy"
@@ -46,6 +48,18 @@ data "aws_iam_policy_document" "ecs_instance_document" {
       "logs:PutLogEvents",
     ]
   }
+
+  statement {
+    resources = [
+      "${aws_ebs_volume.prometheus_ebs_volume.arn}",
+      "arn:aws:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:instance/*",
+    ]
+
+    actions = [
+      "ec2:AttachVolume",
+      "ec2:DetachVolume",
+    ]
+  }
 }
 
 resource "aws_iam_policy" "ecs_instance_policy" {

terraform/projects/app-ecs-instances/instance-user-data.tpl

@@ -1,8 +1,44 @@
 #!/bin/bash
+# Attach EBS volume to instance
+echo "[$(date '+%H:%M:%S %d-%m-%Y')] installing dependencies for volume attaching"
+sudo yum install -y aws-cli wget
 
-echo 'ECS_CLUSTER=${cluster_name}' >> /etc/ecs/ecs.config
+REGION="${region}"
+DEVICE="xvdf"
+VOLUME_ID="${volume_id}"
+
+echo "[$(date '+%H:%M:%S %d-%m-%Y')] finding current instance ID"
+INSTANCE_ID="`wget -q -O - http://169.254.169.254/latest/meta-data/instance-id`"
+
+echo "[$(date '+%H:%M:%S %d-%m-%Y')] attaching volume"
+aws ec2 attach-volume --volume-id $VOLUME_ID --instance-id $INSTANCE_ID --device /dev/$DEVICE --region $REGION
+
+# Waiting for volume to finish attaching
+x=0
+while [[ $x -lt 15 ]]; do
+  if ! [[ -e /dev/$DEVICE ]] ; then
+    sleep 1
+  else
+    break
+  fi
+  x=$((x+1))
+done
+
+# Format and mount volume
+if file -s /dev/$DEVICE | grep -q "/dev/$DEVICE: data"; then
+  echo "[$(date '+%H:%M:%S %d-%m-%Y')] attach-volume: /dev/$DEVICE does not contain any partition, beginning to format disk"
+  mkfs -t ext4 /dev/$DEVICE
+else
+  echo "[$(date '+%H:%M:%S %d-%m-%Y')] attach-volume: /dev/$DEVICE is already formatted: $(file -s /dev/$DEVICE)"
+fi
+
+# Allow prometheus container user access to read/write/execute within container
+mkdir -p /ecs/prometheus_data
+mount /dev/$DEVICE /ecs/prometheus_data
+chmod 777 /ecs/prometheus_data
 
 # Set any ECS agent configuration options
+echo 'ECS_CLUSTER=${cluster_name}' >> /etc/ecs/ecs.config
 yum install -y ecs-init
 start ecs
 service docker start

terraform/projects/app-ecs-instances/main.tf

@@ -138,6 +138,8 @@ data "template_file" "instance_user_data" {
 
   vars {
     cluster_name = "${local.cluster_name}"
+    volume_id    = "${aws_ebs_volume.prometheus_ebs_volume.id}"
+    region       = "${var.aws_region}"
   }
 }
 
@@ -182,6 +184,23 @@ module "ecs_instance" {
   )}"
 }
 
+resource "aws_ebs_volume" "prometheus_ebs_volume" {
+  availability_zone = "${element(data.terraform_remote_state.infra_networking.az_names, 0)}"
+  size              = 500
+  type              = "gp2"
+
+  lifecycle {
+    prevent_destroy = true
+  }
+
+  tags = "${merge(
+    local.default_tags,
+    var.additional_tags,
+    map("Stackname", "${var.stack_name}"),
+    map("Name", "${var.stack_name}-prometheus-ebs-volume")
+  )}"
+}
+
 ## Outputs
 
 output "ecs_instance_asg_id" {

terraform/projects/app-ecs-services/prometheus-service.tf

@@ -89,6 +89,12 @@ resource "aws_ecs_task_definition" "prometheus_server" {
     name      = "prometheus-config"
     host_path = "/ecs/config-from-s3/prometheus"
   }
+
+  # We mount this at /prometheus which is the expected location for the prom/prometheus docker image
+  volume {
+    name      = "prometheus-timeseries-storage"
+    host_path = "/ecs/prometheus_data"
+  }
 }
 
 resource "aws_ecs_service" "prometheus_server" {

terraform/projects/app-ecs-services/task-definitions/prometheus-server.json

@@ -14,6 +14,10 @@
       {
         "sourceVolume": "prometheus-config",
         "containerPath": "/etc/prometheus"
+      },
+      {
+        "sourceVolume": "prometheus-timeseries-storage",
+        "containerPath": "/prometheus"
       }
     ],
     "logConfiguration": {

terraform/projects/infra-networking/README.md

@@ -17,6 +17,7 @@ related services. You will often have multiple VPCs in an account
 
 | Name | Description |
 |------|-------------|
+| az_names | Names of available availability zones |
 | private_subnets | List of private subnet IDs |
 | public_subnets | List of public subnet IDs |
 | vpc_id | VPC ID where the stack resources are created |

terraform/projects/infra-networking/main.tf

@@ -82,6 +82,11 @@ module "vpc" {
 
 ## Outputs
 
+output "az_names" {
+  value       = "${data.aws_availability_zones.available.names}"
+  description = "Names of available availability zones"
+}
+
 output "vpc_id" {
   value       = "${module.vpc.vpc_id}"
   description = "VPC ID where the stack resources are created"