add proxy for scraping paas apps

This has to be a separate container because container links can't form cycles.
alphagov · May 17, 2018 · 8b3fe02 · 8b3fe02
1 parent 38773ff
commit 8b3fe02
Show file tree

Hide file tree

Showing 4 changed files with 66 additions and 12 deletions.
diff --git a/terraform/projects/app-ecs-services/config/prometheus.yml b/terraform/projects/app-ecs-services/config/prometheus.yml
@@ -13,7 +13,7 @@ scrape_configs:
       - targets: ['prometheus-server.sd.ecs-monitoring.com:9090']
   - job_name: paas-targets
     scheme: http
-    proxy_url: 'http://metrics-nginx.sd.ecs-monitoring.com:8080'
+    proxy_url: 'http://paas-proxy:8080'
     file_sd_configs:
       - files: ['/etc/prometheus/targets/*.json']
         refresh_interval: 30s

diff --git a/terraform/projects/app-ecs-services/config/vhosts/paas-proxy.conf b/terraform/projects/app-ecs-services/config/vhosts/paas-proxy.conf
@@ -0,0 +1,10 @@
+server {
+  listen 8080;
+
+  location / {
+    proxy_pass https://$host$uri;
+    proxy_ssl_server_name on;
+    proxy_set_header X-CF-APP-INSTANCE $arg_cf_app_guid:$arg_cf_app_instance_index;
+    proxy_set_header Authorization "Bearer $arg_cf_app_guid";
+  }
+}
diff --git a/terraform/projects/app-ecs-services/prometheus-service.tf b/terraform/projects/app-ecs-services/prometheus-service.tf
@@ -91,8 +91,13 @@ resource "aws_ecs_task_definition" "prometheus_server" {
   }
 
   volume {
-    name      = "nginx-vhosts"
-    host_path = "/ecs/config-from-s3/nginx/conf.d"
+    name      = "auth-proxy"
+    host_path = "/ecs/config-from-s3/auth-proxy/conf.d"
+  }
+
+  volume {
+    name      = "paas-proxy"
+    host_path = "/ecs/config-from-s3/paas-proxy/conf.d"
   }
 }
 
@@ -104,7 +109,7 @@ resource "aws_ecs_service" "prometheus_server" {
 
   load_balancer {
     target_group_arn = "${data.terraform_remote_state.app_ecs_albs.monitoring_external_tg}"
-    container_name   = "nginx"
+    container_name   = "auth-proxy"
     container_port   = 9090
   }
 }
@@ -120,7 +125,7 @@ resource "aws_s3_bucket_object" "prometheus-config" {
 
 resource "aws_s3_bucket_object" "nginx-reverse-proxy" {
   bucket = "${aws_s3_bucket.config_bucket.id}"
-  key    = "prometheus/nginx/conf.d/prometheus-auth-proxy.conf"
+  key    = "prometheus/auth-proxy/conf.d/prometheus-auth-proxy.conf"
   source = "config/vhosts/auth-proxy.conf"
   etag   = "${md5(file("config/vhosts/auth-proxy.conf"))}"
 }
@@ -130,7 +135,16 @@ resource "aws_s3_bucket_object" "nginx-reverse-proxy" {
 # https://github.com/nginxinc/docker-nginx/issues/29
 resource "aws_s3_bucket_object" "nginx-htpasswd" {
   bucket = "${aws_s3_bucket.config_bucket.id}"
-  key    = "prometheus/nginx/conf.d/.htpasswd"
+  key    = "prometheus/auth-proxy/conf.d/.htpasswd"
   source = "config/vhosts/.htpasswd"
   etag   = "${md5(file("config/vhosts/.htpasswd"))}"
 }
+
+#### paas proxy
+
+resource "aws_s3_bucket_object" "nginx-paas-proxy" {
+  bucket = "${aws_s3_bucket.config_bucket.id}"
+  key    = "prometheus/paas-proxy/conf.d/prometheus-paas-proxy.conf"
+  source = "config/vhosts/paas-proxy.conf"
+  etag   = "${md5(file("config/vhosts/paas-proxy.conf"))}"
+}
diff --git a/terraform/projects/app-ecs-services/task-definitions/prometheus-server.json b/terraform/projects/app-ecs-services/task-definitions/prometheus-server.json
@@ -11,6 +11,9 @@
         "containerPath": "/etc/prometheus"
       }
     ],
+    "links": [
+      "paas-proxy"
+    ],
     "logConfiguration": {
       "logDriver": "awslogs",
       "options": {
@@ -23,8 +26,8 @@
   {
     "name": "s3-config-grabber",
     "image": "mesosphere/aws-cli",
-    "cpu": 256,
-    "memory": 256,
+    "cpu": 128,
+    "memory": 128,
     "essential": false,
     "mountPoints": [
       {
@@ -43,10 +46,10 @@
     }
   },
   {
-    "name": "nginx",
+    "name": "auth-proxy",
     "image": "nginx:alpine",
-    "cpu": 256,
-    "memory": 256,
+    "cpu": 128,
+    "memory": 128,
     "essential": true,
     "portMappings": [
       {
@@ -55,7 +58,7 @@
     ],
     "mountPoints": [
       {
-        "sourceVolume": "nginx-vhosts",
+        "sourceVolume": "auth-proxy",
         "containerPath": "/etc/nginx/conf.d",
         "readOnly": true
       }
@@ -71,5 +74,32 @@
         "awslogs-stream-prefix": "prometheus"
       }
     }    
+  },
+  {
+    "name": "paas-proxy",
+    "image": "nginx:alpine",
+    "cpu": 128,
+    "memory": 128,
+    "essential": true,
+    "portMappings": [
+      {
+        "containerPort": 8080
+      }
+    ],
+    "mountPoints": [
+      {
+        "sourceVolume": "paas-proxy",
+        "containerPath": "/etc/nginx/conf.d",
+        "readOnly": true
+      }
+    ],
+    "logConfiguration": {
+      "logDriver": "awslogs",
+      "options": {
+        "awslogs-group": "${log_group}",
+        "awslogs-region": "${region}",
+        "awslogs-stream-prefix": "prometheus"
+      }
+    }
   }
 ]