Adds support for prom_scraper metrics endpoint

Notable changes:

* Nginx has been configured to listen on a second port (controlled by
`cc.prom_metrics_server_tls_port`) and use a special set of certs
(controlled by `cc.prom_scraper_tls`) to communicate with the
`prom_scraper`. `prom_scraper` requires that the certificate of the
server that it's scraping is signed with the same CA cert that every
other endpoint in the deployment that it is scraping is signed with. So
-rather than massively complicate configuration for our existing
server configuration- we have decided to create a second server with its
own configuration.

Co-authored-by: Kenneth Lakin <[email protected]>
Co-authored-by: Michael Oleske <[email protected]>
Co-authored-by: David Alvarado <[email protected]>
Co-authored-by: Merric de Launey <[email protected]>

jobs/cloud_controller_ng/spec

@@ -33,10 +33,14 @@ templates:
   migrate_db.sh.erb: bin/migrate_db
   mime.types: config/mime.types
   nginx_server_mtls.conf: config/nginx_server_mtls.conf
+  prom_scraper_mtls.conf: config/prom_scraper_mtls.conf
   nginx_server_public_tls.conf: config/nginx_server_public_tls.conf
   mutual_tls.crt.erb: config/certs/mutual_tls.crt
   mutual_tls.key.erb: config/certs/mutual_tls.key
   mutual_tls_ca.crt.erb: config/certs/mutual_tls_ca.crt
+  scrape.crt.erb: config/certs/scrape.crt
+  scrape.key.erb: config/certs/scrape.key
+  scrape_ca.crt.erb: config/certs/scrape_ca.crt
   public_tls.crt.erb: config/certs/public_tls.crt
   public_tls.key.erb: config/certs/public_tls.key
   newrelic.yml.erb: config/newrelic.yml
@@ -68,6 +72,7 @@ templates:
   copilot_ca.crt.erb: config/certs/copilot_ca.crt
   copilot.crt.erb: config/certs/copilot.crt
   copilot.key.erb: config/certs/copilot.key
+  prom_scraper_config.yml.erb: config/prom_scraper_config.yml
 
 packages:
   - capi_utils
@@ -112,6 +117,8 @@ provides:
   - cc.internal_service_hostname
   - cc.tls_port
   - cc.mutual_tls.ca_cert
+  - cc.prom_metrics_server_tls_port
+  - cc.prom_scraper_tls.ca_cert
 - name: cloud_controller_container_networking_info
   type: cloud_controller_container_networking_info
   properties:
@@ -335,6 +342,9 @@ properties:
   cc.public_tls.port:
     description: "Port for TLS with gorouter"
     default: 9024
+  cc.prom_metrics_server_tls_port:
+    description: "Port for internal TLS communication with prom_scraper"
+    default: 9025
   cc.internal_service_hostname:
     description: "Internal hostname used to resolve the address of the Cloud Controller"
     default: "cloud-controller-ng.service.cf.internal"
@@ -1098,6 +1108,13 @@ properties:
   cc.public_tls.private_key:
     description: "PEM-encoded key for secure TLS communication over external endpoints"
 
+  cc.prom_scraper_tls.ca_cert:
+    description: "PEM-encoded CA certificate for secure, mutually authenticated TLS communication with prom_scraper"
+  cc.prom_scraper_tls.public_cert:
+    description: "PEM-encoded certificate for secure, mutually authenticated TLS communication with prom_scraper"
+  cc.prom_scraper_tls.private_key:
+    description: "PEM-encoded key for secure, mutually authenticated TLS communication with prom_scraper"
+
   cc.diego.file_server_url:
     description: "URL of file server"
     default: http://file-server.service.cf.internal:8080

jobs/cloud_controller_ng/templates/nginx.conf.erb

@@ -114,6 +114,88 @@ http {
     }
   }
 
+  server {
+    listen <%= p("cc.prom_metrics_server_tls_port") %> ssl;
+    include prom_scraper_mtls.conf;
+
+    server_name  "--"; # This is yet another invalid catch-all name. See the docs at: http://nginx.org/en/docs/http/server_names.html . search for "In catch-all server examples the strange name “_” can be seen:"
+    server_name_in_redirect off;
+    <% if p("request_timeout_in_seconds").to_i > 0 %>
+      proxy_send_timeout          <%= p("request_timeout_in_seconds") %>;
+      proxy_read_timeout          <%= p("request_timeout_in_seconds") %>;
+    <% end %>
+    proxy_buffering             off;
+    proxy_set_header            Host $host;
+    proxy_set_header            X-Real_IP $remote_addr;
+    proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_redirect              off;
+    proxy_connect_timeout       10;
+
+
+    location /internal/v4/ {
+      proxy_pass http://cloud_controller;
+    }
+
+    location ~ /internal/v3/staging/.*/(droplet_completed|build_completed) {
+      proxy_pass http://cloud_controller;
+    }
+
+    location ~ /internal/v4/(droplets|buildpack_cache)/.*/upload {
+      # Allow download the droplets and buildpacks
+      if ($request_method = GET){
+        proxy_pass http://cloud_controller;
+      }
+
+      # Allow large uploads
+      client_max_body_size <%= p("cc.app_bits_max_body_size") %>; #already enforced upstream/but doesn't hurt.
+
+      # Pass altered request body to this location
+      upload_pass   @cc_uploads;
+
+      # Store files to this directory
+      upload_store /var/vcap/data/cloud_controller_ng/tmp/staged_droplet_uploads;
+
+      # Allow uploaded files to be read only by user
+      upload_store_access user:r;
+
+      # Set specified fields in request body
+      upload_set_form_field "droplet_path" $upload_tmp_path;
+
+      #on any error, delete uploaded files.
+      upload_cleanup 400-505;
+    }
+
+    include local_blobstore_downloads.conf;
+
+    # Pass altered request body to a backend
+    location @cc_uploads {
+      proxy_pass http://cloud_controller;
+    }
+  }
+  <%# server { %>
+  <%#   listen 9024 ssl; %>
+  <%#   include prom_scraper_mtls.conf; %>
+
+  <%#   server_name  _; %>
+  <%#   server_name_in_redirect off; %>
+  <%#   <% if p("request_timeout_in_seconds").to_i > 0 %1> %>
+  <%#     proxy_send_timeout          <%= p("request_timeout_in_seconds") %1>; %>
+  <%#     proxy_read_timeout          <%= p("request_timeout_in_seconds") %1>; %>
+  <%#   <% end %1> %>
+  <%#   proxy_buffering             off; %>
+  <%#   proxy_set_header            Host $host; %>
+  <%#   proxy_set_header            X-Real_IP $remote_addr; %>
+  <%#   proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for; %>
+  <%#   proxy_redirect              off; %>
+  <%#   proxy_connect_timeout       10; %>
+
+
+  <%#   location /internal/v4/metrics { %>
+  <%#     proxy_pass http://cloud_controller; %>
+  <%#   } %>
+  <%# } %>
+
+
   # This block handles public endpoints over TLS
   server {
   listen <%= p("cc.public_tls.port") %> ssl;

jobs/cloud_controller_ng/templates/prom_scraper_config.yml.erb

@@ -0,0 +1,6 @@
+port: <%= p("cc.prom_metrics_server_tls_port") %>
+source_id: "cloud_controller_ng"
+instance_id: <%= spec.id || spec.index.to_s %>
+scheme: https
+server_name: <%= p("cc.internal_service_hostname") %>
+path: /internal/v4/metrics

jobs/cloud_controller_ng/templates/prom_scraper_mtls.conf

@@ -0,0 +1,9 @@
+ssl_ciphers DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384;
+ssl_certificate        /var/vcap/jobs/cloud_controller_ng/config/certs/scrape.crt;
+ssl_certificate_key    /var/vcap/jobs/cloud_controller_ng/config/certs/scrape.key;
+ssl_client_certificate /var/vcap/jobs/cloud_controller_ng/config/certs/scrape_ca.crt;
+ssl_verify_client      on;
+ssl_verify_depth       2;
+ssl_protocols          TLSv1.2;
+ssl_session_cache shared:SSL:10m;
+ssl_session_timeout 10m;

jobs/cloud_controller_ng/templates/scrape.crt.erb

@@ -0,0 +1 @@
+<%= p('cc.prom_scraper_tls.public_cert') %>

jobs/cloud_controller_ng/templates/scrape.key.erb

@@ -0,0 +1 @@
+<%= p('cc.prom_scraper_tls.private_key') %>

jobs/cloud_controller_ng/templates/scrape_ca.crt.erb

@@ -0,0 +1 @@
+<%= p('cc.prom_scraper_tls.ca_cert') %>