From 476b938488f03b4d343b817ca098a7abcb4c910f Mon Sep 17 00:00:00 2001 From: Kenneth Lakin Date: Tue, 12 Apr 2022 23:57:52 +0000 Subject: [PATCH] Adds support for prom_scraper metrics endpoint Notable changes: * Nginx has been configured to listen on a second port (controlled by `cc.prom_metrics_server_tls_port`) and use a special set of certs (controlled by `cc.prom_scraper_tls`) to communicate with the `prom_scraper`. `prom_scraper` requires that the certificate of the server that it's scraping is signed with the same CA cert that every other endpoint in the deployment that it is scraping is signed with. So -rather than massively complicate configuration for our existing server configuration- we have decided to create a second server with its own configuration. Co-authored-by: Kenneth Lakin Co-authored-by: Michael Oleske Co-authored-by: David Alvarado Co-authored-by: Merric de Launey --- jobs/cloud_controller_ng/spec | 17 ++++ .../templates/nginx.conf.erb | 82 +++++++++++++++++++ .../templates/prom_scraper_config.yml.erb | 6 ++ .../templates/prom_scraper_mtls.conf | 9 ++ .../templates/scrape.crt.erb | 1 + .../templates/scrape.key.erb | 1 + .../templates/scrape_ca.crt.erb | 1 + src/cloud_controller_ng | 2 +- 8 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 jobs/cloud_controller_ng/templates/prom_scraper_config.yml.erb create mode 100644 jobs/cloud_controller_ng/templates/prom_scraper_mtls.conf create mode 100644 jobs/cloud_controller_ng/templates/scrape.crt.erb create mode 100644 jobs/cloud_controller_ng/templates/scrape.key.erb create mode 100644 jobs/cloud_controller_ng/templates/scrape_ca.crt.erb diff --git a/jobs/cloud_controller_ng/spec b/jobs/cloud_controller_ng/spec index 56d33bb942..27373e3cec 100644 --- a/jobs/cloud_controller_ng/spec +++ b/jobs/cloud_controller_ng/spec @@ -33,10 +33,14 @@ templates: migrate_db.sh.erb: bin/migrate_db mime.types: config/mime.types nginx_server_mtls.conf: config/nginx_server_mtls.conf + prom_scraper_mtls.conf: config/prom_scraper_mtls.conf nginx_server_public_tls.conf: config/nginx_server_public_tls.conf mutual_tls.crt.erb: config/certs/mutual_tls.crt mutual_tls.key.erb: config/certs/mutual_tls.key mutual_tls_ca.crt.erb: config/certs/mutual_tls_ca.crt + scrape.crt.erb: config/certs/scrape.crt + scrape.key.erb: config/certs/scrape.key + scrape_ca.crt.erb: config/certs/scrape_ca.crt public_tls.crt.erb: config/certs/public_tls.crt public_tls.key.erb: config/certs/public_tls.key newrelic.yml.erb: config/newrelic.yml @@ -68,6 +72,7 @@ templates: copilot_ca.crt.erb: config/certs/copilot_ca.crt copilot.crt.erb: config/certs/copilot.crt copilot.key.erb: config/certs/copilot.key + prom_scraper_config.yml.erb: config/prom_scraper_config.yml packages: - capi_utils @@ -112,6 +117,8 @@ provides: - cc.internal_service_hostname - cc.tls_port - cc.mutual_tls.ca_cert + - cc.prom_metrics_server_tls_port + - cc.prom_scraper_tls.ca_cert - name: cloud_controller_container_networking_info type: cloud_controller_container_networking_info properties: @@ -335,6 +342,9 @@ properties: cc.public_tls.port: description: "Port for TLS with gorouter" default: 9024 + cc.prom_metrics_server_tls_port: + description: "Port for internal TLS communication with prom_scraper" + default: 9025 cc.internal_service_hostname: description: "Internal hostname used to resolve the address of the Cloud Controller" default: "cloud-controller-ng.service.cf.internal" @@ -1098,6 +1108,13 @@ properties: cc.public_tls.private_key: description: "PEM-encoded key for secure TLS communication over external endpoints" + cc.prom_scraper_tls.ca_cert: + description: "PEM-encoded CA certificate for secure, mutually authenticated TLS communication with prom_scraper" + cc.prom_scraper_tls.public_cert: + description: "PEM-encoded certificate for secure, mutually authenticated TLS communication with prom_scraper" + cc.prom_scraper_tls.private_key: + description: "PEM-encoded key for secure, mutually authenticated TLS communication with prom_scraper" + cc.diego.file_server_url: description: "URL of file server" default: http://file-server.service.cf.internal:8080 diff --git a/jobs/cloud_controller_ng/templates/nginx.conf.erb b/jobs/cloud_controller_ng/templates/nginx.conf.erb index 81e5939758..8b53660208 100644 --- a/jobs/cloud_controller_ng/templates/nginx.conf.erb +++ b/jobs/cloud_controller_ng/templates/nginx.conf.erb @@ -114,6 +114,88 @@ http { } } + server { + listen <%= p("cc.prom_metrics_server_tls_port") %> ssl; + include prom_scraper_mtls.conf; + + server_name "--"; # This is yet another invalid catch-all name. See the docs at: http://nginx.org/en/docs/http/server_names.html . search for "In catch-all server examples the strange name “_” can be seen:" + server_name_in_redirect off; + <% if p("request_timeout_in_seconds").to_i > 0 %> + proxy_send_timeout <%= p("request_timeout_in_seconds") %>; + proxy_read_timeout <%= p("request_timeout_in_seconds") %>; + <% end %> + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real_IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_redirect off; + proxy_connect_timeout 10; + + + location /internal/v4/ { + proxy_pass http://cloud_controller; + } + + location ~ /internal/v3/staging/.*/(droplet_completed|build_completed) { + proxy_pass http://cloud_controller; + } + + location ~ /internal/v4/(droplets|buildpack_cache)/.*/upload { + # Allow download the droplets and buildpacks + if ($request_method = GET){ + proxy_pass http://cloud_controller; + } + + # Allow large uploads + client_max_body_size <%= p("cc.app_bits_max_body_size") %>; #already enforced upstream/but doesn't hurt. + + # Pass altered request body to this location + upload_pass @cc_uploads; + + # Store files to this directory + upload_store /var/vcap/data/cloud_controller_ng/tmp/staged_droplet_uploads; + + # Allow uploaded files to be read only by user + upload_store_access user:r; + + # Set specified fields in request body + upload_set_form_field "droplet_path" $upload_tmp_path; + + #on any error, delete uploaded files. + upload_cleanup 400-505; + } + + include local_blobstore_downloads.conf; + + # Pass altered request body to a backend + location @cc_uploads { + proxy_pass http://cloud_controller; + } + } + <%# server { %> + <%# listen 9024 ssl; %> + <%# include prom_scraper_mtls.conf; %> + + <%# server_name _; %> + <%# server_name_in_redirect off; %> + <%# <% if p("request_timeout_in_seconds").to_i > 0 %1> %> + <%# proxy_send_timeout <%= p("request_timeout_in_seconds") %1>; %> + <%# proxy_read_timeout <%= p("request_timeout_in_seconds") %1>; %> + <%# <% end %1> %> + <%# proxy_buffering off; %> + <%# proxy_set_header Host $host; %> + <%# proxy_set_header X-Real_IP $remote_addr; %> + <%# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; %> + <%# proxy_redirect off; %> + <%# proxy_connect_timeout 10; %> + + + <%# location /internal/v4/metrics { %> + <%# proxy_pass http://cloud_controller; %> + <%# } %> + <%# } %> + + # This block handles public endpoints over TLS server { listen <%= p("cc.public_tls.port") %> ssl; diff --git a/jobs/cloud_controller_ng/templates/prom_scraper_config.yml.erb b/jobs/cloud_controller_ng/templates/prom_scraper_config.yml.erb new file mode 100644 index 0000000000..d2768cdf38 --- /dev/null +++ b/jobs/cloud_controller_ng/templates/prom_scraper_config.yml.erb @@ -0,0 +1,6 @@ +port: <%= p("cc.prom_metrics_server_tls_port") %> +source_id: "cloud_controller_ng" +instance_id: <%= spec.id || spec.index.to_s %> +scheme: https +server_name: <%= p("cc.internal_service_hostname") %> +path: /internal/v4/metrics diff --git a/jobs/cloud_controller_ng/templates/prom_scraper_mtls.conf b/jobs/cloud_controller_ng/templates/prom_scraper_mtls.conf new file mode 100644 index 0000000000..04b447b5b5 --- /dev/null +++ b/jobs/cloud_controller_ng/templates/prom_scraper_mtls.conf @@ -0,0 +1,9 @@ +ssl_ciphers DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384; +ssl_certificate /var/vcap/jobs/cloud_controller_ng/config/certs/scrape.crt; +ssl_certificate_key /var/vcap/jobs/cloud_controller_ng/config/certs/scrape.key; +ssl_client_certificate /var/vcap/jobs/cloud_controller_ng/config/certs/scrape_ca.crt; +ssl_verify_client on; +ssl_verify_depth 2; +ssl_protocols TLSv1.2; +ssl_session_cache shared:SSL:10m; +ssl_session_timeout 10m; diff --git a/jobs/cloud_controller_ng/templates/scrape.crt.erb b/jobs/cloud_controller_ng/templates/scrape.crt.erb new file mode 100644 index 0000000000..90165db508 --- /dev/null +++ b/jobs/cloud_controller_ng/templates/scrape.crt.erb @@ -0,0 +1 @@ +<%= p('cc.prom_scraper_tls.public_cert') %> diff --git a/jobs/cloud_controller_ng/templates/scrape.key.erb b/jobs/cloud_controller_ng/templates/scrape.key.erb new file mode 100644 index 0000000000..82692f0a32 --- /dev/null +++ b/jobs/cloud_controller_ng/templates/scrape.key.erb @@ -0,0 +1 @@ +<%= p('cc.prom_scraper_tls.private_key') %> diff --git a/jobs/cloud_controller_ng/templates/scrape_ca.crt.erb b/jobs/cloud_controller_ng/templates/scrape_ca.crt.erb new file mode 100644 index 0000000000..74c0f458ad --- /dev/null +++ b/jobs/cloud_controller_ng/templates/scrape_ca.crt.erb @@ -0,0 +1 @@ +<%= p('cc.prom_scraper_tls.ca_cert') %> diff --git a/src/cloud_controller_ng b/src/cloud_controller_ng index 2e1b0cc435..c04fde53e3 160000 --- a/src/cloud_controller_ng +++ b/src/cloud_controller_ng @@ -1 +1 @@ -Subproject commit 2e1b0cc4359d8f836a536f3a61760bc4a1073395 +Subproject commit c04fde53e3d8cfb2b708491eb8b5111d93e64778