diff --git a/terraform/projects/app-ecs-albs/main.tf b/terraform/projects/app-ecs-albs/main.tf index 0b84a703..4ca6708b 100644 --- a/terraform/projects/app-ecs-albs/main.tf +++ b/terraform/projects/app-ecs-albs/main.tf @@ -108,7 +108,7 @@ resource "aws_lb_target_group" "monitoring_external_tg" { health_check { interval = "10" - path = "/graph" # path chosen that 200s as '/' does not return 200 + path = "/health" # static health check on nginx auth proxy matcher = "200" protocol = "HTTP" healthy_threshold = 2 diff --git a/terraform/projects/app-ecs-services/config/prometheus.yml b/terraform/projects/app-ecs-services/config/prometheus.yml index aa65cefe..0ee3bec5 100644 --- a/terraform/projects/app-ecs-services/config/prometheus.yml +++ b/terraform/projects/app-ecs-services/config/prometheus.yml @@ -13,7 +13,7 @@ scrape_configs: - targets: ['localhost:9090'] - job_name: paas-targets scheme: http - proxy_url: 'http://metrics-nginx.sd.ecs-monitoring.com:8080' + proxy_url: 'http://paas-proxy:8080' file_sd_configs: - files: ['/etc/prometheus/targets/*.json'] refresh_interval: 30s diff --git a/terraform/projects/app-ecs-services/config/vhosts/.htpasswd b/terraform/projects/app-ecs-services/config/vhosts/.htpasswd new file mode 100644 index 00000000..19621115 --- /dev/null +++ b/terraform/projects/app-ecs-services/config/vhosts/.htpasswd @@ -0,0 +1 @@ +grafana:$2y$05$KtkeBzc53efo5u10r3A1gexcml34YiMNVtKd8CqaXPEac4hELCERK diff --git a/terraform/projects/app-ecs-services/config/vhosts/auth-proxy.conf b/terraform/projects/app-ecs-services/config/vhosts/auth-proxy.conf new file mode 100644 index 00000000..f4a46dd3 --- /dev/null +++ b/terraform/projects/app-ecs-services/config/vhosts/auth-proxy.conf @@ -0,0 +1,16 @@ +server { + listen 9090 default_server; + auth_basic "Prometheus"; + auth_basic_user_file /etc/nginx/conf.d/.htpasswd; + + location / { + proxy_pass http://prometheus:9090; + } + location /status { + auth_basic off; + proxy_pass http://prometheus:9090/status; + } + location /health { + return 200 "Static health check"; + } +} diff --git a/terraform/projects/app-ecs-services/config/vhosts/paas-proxy.conf b/terraform/projects/app-ecs-services/config/vhosts/paas-proxy.conf new file mode 100644 index 00000000..f947b6b7 --- /dev/null +++ b/terraform/projects/app-ecs-services/config/vhosts/paas-proxy.conf @@ -0,0 +1,10 @@ +server { + listen 8080; + + location / { + proxy_pass https://$host$uri; + proxy_ssl_server_name on; + proxy_set_header X-CF-APP-INSTANCE $arg_cf_app_guid:$arg_cf_app_instance_index; + proxy_set_header Authorization "Bearer $arg_cf_app_guid"; + } +} diff --git a/terraform/projects/app-ecs-services/prometheus-service.tf b/terraform/projects/app-ecs-services/prometheus-service.tf index f3897150..aa1a171a 100644 --- a/terraform/projects/app-ecs-services/prometheus-service.tf +++ b/terraform/projects/app-ecs-services/prometheus-service.tf @@ -34,7 +34,7 @@ data "aws_iam_policy_document" "prometheus_policy_doc" { statement { sid = "GetPrometheusFiles" - resources = ["arn:aws:s3:::${aws_s3_bucket.config_bucket.id}/etc/prometheus/*"] + resources = ["arn:aws:s3:::${aws_s3_bucket.config_bucket.id}/prometheus/*"] actions = [ "s3:Get*", @@ -90,6 +90,16 @@ resource "aws_ecs_task_definition" "prometheus_server" { host_path = "/ecs/config-from-s3/prometheus" } + volume { + name = "auth-proxy" + host_path = "/ecs/config-from-s3/auth-proxy/conf.d" + } + + volume { + name = "paas-proxy" + host_path = "/ecs/config-from-s3/paas-proxy/conf.d" + } + # We mount this at /prometheus which is the expected location for the prom/prometheus docker image volume { name = "prometheus-timeseries-storage" @@ -105,14 +115,42 @@ resource "aws_ecs_service" "prometheus_server" { load_balancer { target_group_arn = "${data.terraform_remote_state.app_ecs_albs.monitoring_external_tg}" - container_name = "prometheus" + container_name = "auth-proxy" container_port = 9090 } } resource "aws_s3_bucket_object" "prometheus-config" { bucket = "${aws_s3_bucket.config_bucket.id}" - key = "etc/prometheus/prometheus.yml" + key = "prometheus/prometheus/prometheus.yml" source = "config/prometheus.yml" etag = "${md5(file("config/prometheus.yml"))}" } + +#### nginx reverse proxy + +resource "aws_s3_bucket_object" "nginx-reverse-proxy" { + bucket = "${aws_s3_bucket.config_bucket.id}" + key = "prometheus/auth-proxy/conf.d/prometheus-auth-proxy.conf" + source = "config/vhosts/auth-proxy.conf" + etag = "${md5(file("config/vhosts/auth-proxy.conf"))}" +} + +# The htpasswd file is in bcrypt format, which is only supported +# by the nginx:alpine image, not the plain nginx image +# https://github.com/nginxinc/docker-nginx/issues/29 +resource "aws_s3_bucket_object" "nginx-htpasswd" { + bucket = "${aws_s3_bucket.config_bucket.id}" + key = "prometheus/auth-proxy/conf.d/.htpasswd" + source = "config/vhosts/.htpasswd" + etag = "${md5(file("config/vhosts/.htpasswd"))}" +} + +#### paas proxy + +resource "aws_s3_bucket_object" "nginx-paas-proxy" { + bucket = "${aws_s3_bucket.config_bucket.id}" + key = "prometheus/paas-proxy/conf.d/prometheus-paas-proxy.conf" + source = "config/vhosts/paas-proxy.conf" + etag = "${md5(file("config/vhosts/paas-proxy.conf"))}" +} diff --git a/terraform/projects/app-ecs-services/task-definitions/prometheus-server.json b/terraform/projects/app-ecs-services/task-definitions/prometheus-server.json index 6c3f19a6..2397410b 100644 --- a/terraform/projects/app-ecs-services/task-definitions/prometheus-server.json +++ b/terraform/projects/app-ecs-services/task-definitions/prometheus-server.json @@ -2,14 +2,9 @@ { "name": "prometheus", "image": "prom/prometheus", + "cpu": 1024, "memoryReservation": 4096, "essential": true, - "portMappings": [ - { - "containerPort": 9090, - "hostPort": 9090 - } - ], "mountPoints": [ { "sourceVolume": "prometheus-config", @@ -20,6 +15,9 @@ "containerPath": "/prometheus" } ], + "links": [ + "paas-proxy" + ], "logConfiguration": { "logDriver": "awslogs", "options": { @@ -32,7 +30,8 @@ { "name": "s3-config-grabber", "image": "mesosphere/aws-cli", - "memory": 256, + "cpu": 128, + "memory": 128, "essential": false, "mountPoints": [ { @@ -40,7 +39,64 @@ "containerPath": "/configs" } ], - "command": ["s3", "sync", "s3://${config_bucket}/etc/prometheus", "/configs/prometheus"], + "command": ["s3", "sync", "s3://${config_bucket}/prometheus", "/configs"], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "${log_group}", + "awslogs-region": "${region}", + "awslogs-stream-prefix": "prometheus" + } + } + }, + { + "name": "auth-proxy", + "image": "nginx:alpine", + "cpu": 128, + "memory": 128, + "essential": true, + "portMappings": [ + { + "containerPort": 9090 + } + ], + "mountPoints": [ + { + "sourceVolume": "auth-proxy", + "containerPath": "/etc/nginx/conf.d", + "readOnly": true + } + ], + "links": [ + "prometheus" + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "${log_group}", + "awslogs-region": "${region}", + "awslogs-stream-prefix": "prometheus" + } + } + }, + { + "name": "paas-proxy", + "image": "nginx:alpine", + "cpu": 128, + "memory": 128, + "essential": true, + "portMappings": [ + { + "containerPort": 8080 + } + ], + "mountPoints": [ + { + "sourceVolume": "paas-proxy", + "containerPath": "/etc/nginx/conf.d", + "readOnly": true + } + ], "logConfiguration": { "logDriver": "awslogs", "options": {