Revert "Revert Prometheus metrics configurations"

This reverts commit ee0e1e5.

jobs/cloud_controller_ng/spec

@@ -31,10 +31,14 @@ templates:
   migrate_db.sh.erb: bin/migrate_db
   mime.types: config/mime.types
   nginx_server_mtls.conf: config/nginx_server_mtls.conf
+  prom_scraper_mtls.conf: config/prom_scraper_mtls.conf
   nginx_server_public_tls.conf: config/nginx_server_public_tls.conf
   mutual_tls.crt.erb: config/certs/mutual_tls.crt
   mutual_tls.key.erb: config/certs/mutual_tls.key
   mutual_tls_ca.crt.erb: config/certs/mutual_tls_ca.crt
+  scrape.crt.erb: config/certs/scrape.crt
+  scrape.key.erb: config/certs/scrape.key
+  scrape_ca.crt.erb: config/certs/scrape_ca.crt
   public_tls.crt.erb: config/certs/public_tls.crt
   public_tls.key.erb: config/certs/public_tls.key
   newrelic.yml.erb: config/newrelic.yml
@@ -60,6 +64,7 @@ templates:
   copilot_ca.crt.erb: config/certs/copilot_ca.crt
   copilot.crt.erb: config/certs/copilot.crt
   copilot.key.erb: config/certs/copilot.key
+  prom_scraper_config.yml.erb: config/prom_scraper_config.yml
 
 packages:
   - capi_utils
@@ -104,6 +109,8 @@ provides:
   - cc.internal_service_hostname
   - cc.tls_port
   - cc.mutual_tls.ca_cert
+  - cc.prom_metrics_server_tls_port
+  - cc.prom_scraper_tls.ca_cert
 - name: cloud_controller_container_networking_info
   type: cloud_controller_container_networking_info
   properties:
@@ -318,6 +325,9 @@ properties:
   cc.public_tls.port:
     description: "Port for TLS with gorouter"
     default: 9024
+  cc.prom_metrics_server_tls_port:
+    description: "Port for internal TLS communication with prom_scraper"
+    default: 9025
   cc.internal_service_hostname:
     description: "Internal hostname used to resolve the address of the Cloud Controller"
     default: "cloud-controller-ng.service.cf.internal"
@@ -1059,6 +1069,13 @@ properties:
   cc.public_tls.private_key:
     description: "PEM-encoded key for secure TLS communication over external endpoints"
 
+  cc.prom_scraper_tls.ca_cert:
+    description: "PEM-encoded CA certificate for secure, mutually authenticated TLS communication with prom_scraper"
+  cc.prom_scraper_tls.public_cert:
+    description: "PEM-encoded certificate for secure, mutually authenticated TLS communication with prom_scraper"
+  cc.prom_scraper_tls.private_key:
+    description: "PEM-encoded key for secure, mutually authenticated TLS communication with prom_scraper"
+
   cc.diego.file_server_url:
     description: "URL of file server"
     default: http://file-server.service.cf.internal:8080

jobs/cloud_controller_ng/templates/nginx.conf.erb

@@ -118,6 +118,88 @@ http {
     }
   }
 
+  server {
+    listen <%= p("cc.prom_metrics_server_tls_port") %> ssl;
+    include prom_scraper_mtls.conf;
+
+    server_name  "--"; # This is yet another invalid catch-all name. See the docs at: http://nginx.org/en/docs/http/server_names.html . search for "In catch-all server examples the strange name “_” can be seen:"
+    server_name_in_redirect off;
+    <% if p("request_timeout_in_seconds").to_i > 0 %>
+      proxy_send_timeout          <%= p("request_timeout_in_seconds") %>;
+      proxy_read_timeout          <%= p("request_timeout_in_seconds") %>;
+    <% end %>
+    proxy_buffering             off;
+    proxy_set_header            Host $host;
+    proxy_set_header            X-Real_IP $remote_addr;
+    proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_redirect              off;
+    proxy_connect_timeout       10;
+
+
+    location /internal/v4/ {
+      proxy_pass http://cloud_controller;
+    }
+
+    location ~ /internal/v3/staging/.*/(droplet_completed|build_completed) {
+      proxy_pass http://cloud_controller;
+    }
+
+    location ~ /internal/v4/(droplets|buildpack_cache)/.*/upload {
+      # Allow download the droplets and buildpacks
+      if ($request_method = GET){
+        proxy_pass http://cloud_controller;
+      }
+
+      # Allow large uploads
+      client_max_body_size <%= p("cc.app_bits_max_body_size") %>; #already enforced upstream/but doesn't hurt.
+
+      # Pass altered request body to this location
+      upload_pass   @cc_uploads;
+
+      # Store files to this directory
+      upload_store /var/vcap/data/cloud_controller_ng/tmp/staged_droplet_uploads;
+
+      # Allow uploaded files to be read only by user
+      upload_store_access user:r;
+
+      # Set specified fields in request body
+      upload_set_form_field "droplet_path" $upload_tmp_path;
+
+      #on any error, delete uploaded files.
+      upload_cleanup 400-505;
+    }
+
+    include local_blobstore_downloads.conf;
+
+    # Pass altered request body to a backend
+    location @cc_uploads {
+      proxy_pass http://cloud_controller;
+    }
+  }
+  <%# server { %>
+  <%#   listen 9024 ssl; %>
+  <%#   include prom_scraper_mtls.conf; %>
+
+  <%#   server_name  _; %>
+  <%#   server_name_in_redirect off; %>
+  <%#   <% if p("request_timeout_in_seconds").to_i > 0 %1> %>
+  <%#     proxy_send_timeout          <%= p("request_timeout_in_seconds") %1>; %>
+  <%#     proxy_read_timeout          <%= p("request_timeout_in_seconds") %1>; %>
+  <%#   <% end %1> %>
+  <%#   proxy_buffering             off; %>
+  <%#   proxy_set_header            Host $host; %>
+  <%#   proxy_set_header            X-Real_IP $remote_addr; %>
+  <%#   proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for; %>
+  <%#   proxy_redirect              off; %>
+  <%#   proxy_connect_timeout       10; %>
+
+
+  <%#   location /internal/v4/metrics { %>
+  <%#     proxy_pass http://cloud_controller; %>
+  <%#   } %>
+  <%# } %>
+
+
   # This block handles public endpoints over TLS
   server {
   listen <%= p("cc.public_tls.port") %> ssl;

jobs/cloud_controller_ng/templates/prom_scraper_config.yml.erb

@@ -0,0 +1,6 @@
+port: <%= p("cc.prom_metrics_server_tls_port") %>
+source_id: "cloud_controller_ng"
+instance_id: <%= spec.id || spec.index.to_s %>
+scheme: https
+server_name: <%= p("cc.internal_service_hostname") %>
+path: /internal/v4/metrics

jobs/cloud_controller_ng/templates/prom_scraper_mtls.conf

@@ -0,0 +1,9 @@
+ssl_ciphers DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384;
+ssl_certificate        /var/vcap/jobs/cloud_controller_ng/config/certs/scrape.crt;
+ssl_certificate_key    /var/vcap/jobs/cloud_controller_ng/config/certs/scrape.key;
+ssl_client_certificate /var/vcap/jobs/cloud_controller_ng/config/certs/scrape_ca.crt;
+ssl_verify_client      on;
+ssl_verify_depth       2;
+ssl_protocols          TLSv1.2;
+ssl_session_cache shared:SSL:10m;
+ssl_session_timeout 10m;

jobs/cloud_controller_ng/templates/scrape.crt.erb

@@ -0,0 +1,4 @@
+<% if_p('cc.prom_scraper_tls.public_cert') do %>
+<%= p('cc.prom_scraper_tls.public_cert') %>
+<% end %>
+

jobs/cloud_controller_ng/templates/scrape.key.erb

@@ -0,0 +1,3 @@
+<% if_p('cc.prom_scraper_tls.private_key') do %>
+<%= p('cc.prom_scraper_tls.private_key') %>
+<% end %>

jobs/cloud_controller_ng/templates/scrape_ca.crt.erb

@@ -0,0 +1,3 @@
+<% if_p("cc.prom_scraper_tls.ca_cert") do %>
+<%= p("cc.prom_scraper_tls.ca_cert") %>
+<% end %>