use nginx for prometheus basic auth

terraform/projects/app-ecs-albs/main.tf

@@ -108,7 +108,7 @@ resource "aws_lb_target_group" "monitoring_external_tg" {
 
   health_check {
     interval            = "10"
-    path                = "/graph" # path chosen that 200s as '/' does not return 200
+    path                = "/health" # static health check on nginx auth proxy
     matcher             = "200"
     protocol            = "HTTP"
     healthy_threshold   = 2

terraform/projects/app-ecs-services/config/prometheus.yml

@@ -13,7 +13,7 @@ scrape_configs:
       - targets: ['localhost:9090']
   - job_name: paas-targets
     scheme: http
-    proxy_url: 'http://metrics-nginx.sd.ecs-monitoring.com:8080'
+    proxy_url: 'http://paas-proxy:8080'
     file_sd_configs:
       - files: ['/etc/prometheus/targets/*.json']
         refresh_interval: 30s

terraform/projects/app-ecs-services/config/vhosts/.htpasswd

@@ -0,0 +1 @@
+grafana:$2y$05$KtkeBzc53efo5u10r3A1gexcml34YiMNVtKd8CqaXPEac4hELCERK

terraform/projects/app-ecs-services/config/vhosts/auth-proxy.conf

@@ -0,0 +1,16 @@
+server {
+  listen 9090 default_server;
+  auth_basic "Prometheus";
+  auth_basic_user_file /etc/nginx/conf.d/.htpasswd;
+
+  location / {
+    proxy_pass  http://prometheus:9090;
+  }
+  location /status {
+    auth_basic off;
+    proxy_pass http://prometheus:9090/status;
+  }
+  location /health {
+    return 200 "Static health check";
+  }
+}

terraform/projects/app-ecs-services/config/vhosts/paas-proxy.conf

@@ -0,0 +1,10 @@
+server {
+  listen 8080;
+
+  location / {
+    proxy_pass https://$host$uri;
+    proxy_ssl_server_name on;
+    proxy_set_header X-CF-APP-INSTANCE $arg_cf_app_guid:$arg_cf_app_instance_index;
+    proxy_set_header Authorization "Bearer $arg_cf_app_guid";
+  }
+}

terraform/projects/app-ecs-services/prometheus-service.tf

@@ -34,7 +34,7 @@ data "aws_iam_policy_document" "prometheus_policy_doc" {
   statement {
     sid = "GetPrometheusFiles"
 
-    resources = ["arn:aws:s3:::${aws_s3_bucket.config_bucket.id}/etc/prometheus/*"]
+    resources = ["arn:aws:s3:::${aws_s3_bucket.config_bucket.id}/prometheus/*"]
 
     actions = [
       "s3:Get*",
@@ -90,6 +90,16 @@ resource "aws_ecs_task_definition" "prometheus_server" {
     host_path = "/ecs/config-from-s3/prometheus"
   }
 
+  volume {
+    name      = "auth-proxy"
+    host_path = "/ecs/config-from-s3/auth-proxy/conf.d"
+  }
+
+  volume {
+    name      = "paas-proxy"
+    host_path = "/ecs/config-from-s3/paas-proxy/conf.d"
+  }
+
   # We mount this at /prometheus which is the expected location for the prom/prometheus docker image
   volume {
     name      = "prometheus-timeseries-storage"
@@ -105,14 +115,42 @@ resource "aws_ecs_service" "prometheus_server" {
 
   load_balancer {
     target_group_arn = "${data.terraform_remote_state.app_ecs_albs.monitoring_external_tg}"
-    container_name   = "prometheus"
+    container_name   = "auth-proxy"
     container_port   = 9090
   }
 }
 
 resource "aws_s3_bucket_object" "prometheus-config" {
   bucket = "${aws_s3_bucket.config_bucket.id}"
-  key    = "etc/prometheus/prometheus.yml"
+  key    = "prometheus/prometheus/prometheus.yml"
   source = "config/prometheus.yml"
   etag   = "${md5(file("config/prometheus.yml"))}"
 }
+
+#### nginx reverse proxy
+
+resource "aws_s3_bucket_object" "nginx-reverse-proxy" {
+  bucket = "${aws_s3_bucket.config_bucket.id}"
+  key    = "prometheus/auth-proxy/conf.d/prometheus-auth-proxy.conf"
+  source = "config/vhosts/auth-proxy.conf"
+  etag   = "${md5(file("config/vhosts/auth-proxy.conf"))}"
+}
+
+# The htpasswd file is in bcrypt format, which is only supported
+# by the nginx:alpine image, not the plain nginx image
+# https://github.com/nginxinc/docker-nginx/issues/29
+resource "aws_s3_bucket_object" "nginx-htpasswd" {
+  bucket = "${aws_s3_bucket.config_bucket.id}"
+  key    = "prometheus/auth-proxy/conf.d/.htpasswd"
+  source = "config/vhosts/.htpasswd"
+  etag   = "${md5(file("config/vhosts/.htpasswd"))}"
+}
+
+#### paas proxy
+
+resource "aws_s3_bucket_object" "nginx-paas-proxy" {
+  bucket = "${aws_s3_bucket.config_bucket.id}"
+  key    = "prometheus/paas-proxy/conf.d/prometheus-paas-proxy.conf"
+  source = "config/vhosts/paas-proxy.conf"
+  etag   = "${md5(file("config/vhosts/paas-proxy.conf"))}"
+}

terraform/projects/app-ecs-services/task-definitions/prometheus-server.json

@@ -2,14 +2,9 @@
   {
     "name": "prometheus",
     "image": "prom/prometheus",
+    "cpu": 1024,
     "memoryReservation": 4096,
     "essential": true,
-    "portMappings": [
-      {
-        "containerPort": 9090,
-        "hostPort": 9090
-      }
-    ],
     "mountPoints": [
       {
         "sourceVolume": "prometheus-config",
@@ -20,6 +15,9 @@
         "containerPath": "/prometheus"
       }
     ],
+    "links": [
+      "paas-proxy"
+    ],
     "logConfiguration": {
       "logDriver": "awslogs",
       "options": {
@@ -32,15 +30,73 @@
   {
     "name": "s3-config-grabber",
     "image": "mesosphere/aws-cli",
-    "memory": 256,
+    "cpu": 128,
+    "memory": 128,
     "essential": false,
     "mountPoints": [
       {
         "sourceVolume": "config-from-s3",
         "containerPath": "/configs"
       }
     ],
-    "command": ["s3", "sync", "s3://${config_bucket}/etc/prometheus", "/configs/prometheus"],
+    "command": ["s3", "sync", "s3://${config_bucket}/prometheus", "/configs"],
+    "logConfiguration": {
+      "logDriver": "awslogs",
+      "options": {
+        "awslogs-group": "${log_group}",
+        "awslogs-region": "${region}",
+        "awslogs-stream-prefix": "prometheus"
+      }
+    }
+  },
+  {
+    "name": "auth-proxy",
+    "image": "nginx:alpine",
+    "cpu": 128,
+    "memory": 128,
+    "essential": true,
+    "portMappings": [
+      {
+        "containerPort": 9090
+      }
+    ],
+    "mountPoints": [
+      {
+        "sourceVolume": "auth-proxy",
+        "containerPath": "/etc/nginx/conf.d",
+        "readOnly": true
+      }
+    ],
+    "links": [
+      "prometheus"
+    ],
+    "logConfiguration": {
+      "logDriver": "awslogs",
+      "options": {
+        "awslogs-group": "${log_group}",
+        "awslogs-region": "${region}",
+        "awslogs-stream-prefix": "prometheus"
+      }
+    }
+  },
+  {
+    "name": "paas-proxy",
+    "image": "nginx:alpine",
+    "cpu": 128,
+    "memory": 128,
+    "essential": true,
+    "portMappings": [
+      {
+        "containerPort": 8080
+      }
+    ],
+    "mountPoints": [
+      {
+        "sourceVolume": "paas-proxy",
+        "containerPath": "/etc/nginx/conf.d",
+        "readOnly": true
+      }
+    ],
     "logConfiguration": {
       "logDriver": "awslogs",
       "options": {