diff --git a/jobs/cloud_controller_ng/spec b/jobs/cloud_controller_ng/spec index d94ad19c10..e4bc8eb240 100644 --- a/jobs/cloud_controller_ng/spec +++ b/jobs/cloud_controller_ng/spec @@ -32,10 +32,14 @@ templates: migrate_db.sh.erb: bin/migrate_db mime.types: config/mime.types nginx_server_mtls.conf: config/nginx_server_mtls.conf + prom_scraper_mtls.conf: config/prom_scraper_mtls.conf nginx_server_public_tls.conf: config/nginx_server_public_tls.conf mutual_tls.crt.erb: config/certs/mutual_tls.crt mutual_tls.key.erb: config/certs/mutual_tls.key mutual_tls_ca.crt.erb: config/certs/mutual_tls_ca.crt + scrape.crt.erb: config/certs/scrape.crt + scrape.key.erb: config/certs/scrape.key + scrape_ca.crt.erb: config/certs/scrape_ca.crt public_tls.crt.erb: config/certs/public_tls.crt public_tls.key.erb: config/certs/public_tls.key newrelic.yml.erb: config/newrelic.yml @@ -65,6 +69,7 @@ templates: copilot_ca.crt.erb: config/certs/copilot_ca.crt copilot.crt.erb: config/certs/copilot.crt copilot.key.erb: config/certs/copilot.key + prom_scraper_config.yml.erb: config/prom_scraper_config.yml packages: - capi_utils @@ -109,6 +114,8 @@ provides: - cc.internal_service_hostname - cc.tls_port - cc.mutual_tls.ca_cert + - cc.prom_metrics_server_tls_port + - cc.prom_scraper_tls.ca_cert - name: cloud_controller_container_networking_info type: cloud_controller_container_networking_info properties: @@ -332,6 +339,9 @@ properties: cc.public_tls.port: description: "Port for TLS with gorouter" default: 9024 + cc.prom_metrics_server_tls_port: + description: "Port for internal TLS communication with prom_scraper" + default: 9025 cc.internal_service_hostname: description: "Internal hostname used to resolve the address of the Cloud Controller" default: "cloud-controller-ng.service.cf.internal" @@ -1095,6 +1105,13 @@ properties: cc.public_tls.private_key: description: "PEM-encoded key for secure TLS communication over external endpoints" + cc.prom_scraper_tls.ca_cert: + description: "PEM-encoded CA certificate for secure, mutually authenticated TLS communication with prom_scraper" + cc.prom_scraper_tls.public_cert: + description: "PEM-encoded certificate for secure, mutually authenticated TLS communication with prom_scraper" + cc.prom_scraper_tls.private_key: + description: "PEM-encoded key for secure, mutually authenticated TLS communication with prom_scraper" + cc.diego.file_server_url: description: "URL of file server" default: http://file-server.service.cf.internal:8080 diff --git a/jobs/cloud_controller_ng/templates/nginx.conf.erb b/jobs/cloud_controller_ng/templates/nginx.conf.erb index 81e5939758..8b53660208 100644 --- a/jobs/cloud_controller_ng/templates/nginx.conf.erb +++ b/jobs/cloud_controller_ng/templates/nginx.conf.erb @@ -114,6 +114,88 @@ http { } } + server { + listen <%= p("cc.prom_metrics_server_tls_port") %> ssl; + include prom_scraper_mtls.conf; + + server_name "--"; # This is yet another invalid catch-all name. See the docs at: http://nginx.org/en/docs/http/server_names.html . search for "In catch-all server examples the strange name “_” can be seen:" + server_name_in_redirect off; + <% if p("request_timeout_in_seconds").to_i > 0 %> + proxy_send_timeout <%= p("request_timeout_in_seconds") %>; + proxy_read_timeout <%= p("request_timeout_in_seconds") %>; + <% end %> + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real_IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_redirect off; + proxy_connect_timeout 10; + + + location /internal/v4/ { + proxy_pass http://cloud_controller; + } + + location ~ /internal/v3/staging/.*/(droplet_completed|build_completed) { + proxy_pass http://cloud_controller; + } + + location ~ /internal/v4/(droplets|buildpack_cache)/.*/upload { + # Allow download the droplets and buildpacks + if ($request_method = GET){ + proxy_pass http://cloud_controller; + } + + # Allow large uploads + client_max_body_size <%= p("cc.app_bits_max_body_size") %>; #already enforced upstream/but doesn't hurt. + + # Pass altered request body to this location + upload_pass @cc_uploads; + + # Store files to this directory + upload_store /var/vcap/data/cloud_controller_ng/tmp/staged_droplet_uploads; + + # Allow uploaded files to be read only by user + upload_store_access user:r; + + # Set specified fields in request body + upload_set_form_field "droplet_path" $upload_tmp_path; + + #on any error, delete uploaded files. + upload_cleanup 400-505; + } + + include local_blobstore_downloads.conf; + + # Pass altered request body to a backend + location @cc_uploads { + proxy_pass http://cloud_controller; + } + } + <%# server { %> + <%# listen 9024 ssl; %> + <%# include prom_scraper_mtls.conf; %> + + <%# server_name _; %> + <%# server_name_in_redirect off; %> + <%# <% if p("request_timeout_in_seconds").to_i > 0 %1> %> + <%# proxy_send_timeout <%= p("request_timeout_in_seconds") %1>; %> + <%# proxy_read_timeout <%= p("request_timeout_in_seconds") %1>; %> + <%# <% end %1> %> + <%# proxy_buffering off; %> + <%# proxy_set_header Host $host; %> + <%# proxy_set_header X-Real_IP $remote_addr; %> + <%# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; %> + <%# proxy_redirect off; %> + <%# proxy_connect_timeout 10; %> + + + <%# location /internal/v4/metrics { %> + <%# proxy_pass http://cloud_controller; %> + <%# } %> + <%# } %> + + # This block handles public endpoints over TLS server { listen <%= p("cc.public_tls.port") %> ssl; diff --git a/jobs/cloud_controller_ng/templates/prom_scraper_config.yml.erb b/jobs/cloud_controller_ng/templates/prom_scraper_config.yml.erb new file mode 100644 index 0000000000..d2768cdf38 --- /dev/null +++ b/jobs/cloud_controller_ng/templates/prom_scraper_config.yml.erb @@ -0,0 +1,6 @@ +port: <%= p("cc.prom_metrics_server_tls_port") %> +source_id: "cloud_controller_ng" +instance_id: <%= spec.id || spec.index.to_s %> +scheme: https +server_name: <%= p("cc.internal_service_hostname") %> +path: /internal/v4/metrics diff --git a/jobs/cloud_controller_ng/templates/prom_scraper_mtls.conf b/jobs/cloud_controller_ng/templates/prom_scraper_mtls.conf new file mode 100644 index 0000000000..04b447b5b5 --- /dev/null +++ b/jobs/cloud_controller_ng/templates/prom_scraper_mtls.conf @@ -0,0 +1,9 @@ +ssl_ciphers DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384; +ssl_certificate /var/vcap/jobs/cloud_controller_ng/config/certs/scrape.crt; +ssl_certificate_key /var/vcap/jobs/cloud_controller_ng/config/certs/scrape.key; +ssl_client_certificate /var/vcap/jobs/cloud_controller_ng/config/certs/scrape_ca.crt; +ssl_verify_client on; +ssl_verify_depth 2; +ssl_protocols TLSv1.2; +ssl_session_cache shared:SSL:10m; +ssl_session_timeout 10m; diff --git a/jobs/cloud_controller_ng/templates/scrape.crt.erb b/jobs/cloud_controller_ng/templates/scrape.crt.erb new file mode 100644 index 0000000000..90165db508 --- /dev/null +++ b/jobs/cloud_controller_ng/templates/scrape.crt.erb @@ -0,0 +1 @@ +<%= p('cc.prom_scraper_tls.public_cert') %> diff --git a/jobs/cloud_controller_ng/templates/scrape.key.erb b/jobs/cloud_controller_ng/templates/scrape.key.erb new file mode 100644 index 0000000000..82692f0a32 --- /dev/null +++ b/jobs/cloud_controller_ng/templates/scrape.key.erb @@ -0,0 +1 @@ +<%= p('cc.prom_scraper_tls.private_key') %> diff --git a/jobs/cloud_controller_ng/templates/scrape_ca.crt.erb b/jobs/cloud_controller_ng/templates/scrape_ca.crt.erb new file mode 100644 index 0000000000..74c0f458ad --- /dev/null +++ b/jobs/cloud_controller_ng/templates/scrape_ca.crt.erb @@ -0,0 +1 @@ +<%= p('cc.prom_scraper_tls.ca_cert') %>