From fdc721956f208059a03fe206cfc8b0f13543c34c Mon Sep 17 00:00:00 2001 From: Camille Date: Fri, 20 Oct 2017 14:50:31 +0200 Subject: [PATCH] Add a HTTPS proxy for concourse on GCP (#62) * Add a HTTPS proxy for concourse on GCP using a VIP and self signed certificate * Add letsencrypt support when concourse exposed on public IP * Force to use HTTPS * Update the documentation * Create a specific deployment for ingress * Fine tuning and hotfixes --- lib/kite/render.rb | 46 +++++++- tpl/gcp/README.md | 13 ++- tpl/gcp/bin/base/setup-tunnel.sh.tt | 11 +- tpl/gcp/bin/ingress-deploy.sh.tt | 7 ++ tpl/gcp/deployments/bosh/cloud-config.yml.tt | 18 ++- .../deployments/concourse/concourse.yml.tt | 14 ++- tpl/gcp/deployments/ingress/ingress.yml.erb | 107 ++++++++++++++++++ tpl/gcp/deployments/nginx/nginx.yml.erb | 62 ---------- tpl/gcp/docs/bosh.md | 5 + tpl/gcp/docs/ingress.md | 14 +++ tpl/gcp/docs/vault.md | 3 +- tpl/gcp/terraform/main.tf | 2 +- tpl/skel/config/cloud.yml | 18 ++- 13 files changed, 235 insertions(+), 85 deletions(-) create mode 100755 tpl/gcp/bin/ingress-deploy.sh.tt create mode 100644 tpl/gcp/deployments/ingress/ingress.yml.erb delete mode 100644 tpl/gcp/deployments/nginx/nginx.yml.erb create mode 100644 tpl/gcp/docs/ingress.md diff --git a/lib/kite/render.rb b/lib/kite/render.rb index 89b570d..c002eaf 100644 --- a/lib/kite/render.rb +++ b/lib/kite/render.rb @@ -4,10 +4,44 @@ class Render < Base include Kite::Helpers + no_commands do + def ingress_db_file + "config/ingress.yml" + end + + def ingress_db + @db ||= YAML.load(File.read(ingress_db_file)) rescue {} + end + + def ingress_db_save! + create_file ingress_db_file, YAML.dump(ingress_db), force: true + end + + def ingress_add_entry(hostname, upstreams, args = {}) + raise "upstreams argument should be an array" unless upstreams.is_a?(Array) + args[:port] ||= 80 + args[:protocol] ||= "http" + ingress_db[hostname] = { + upstreams: upstreams, + port: args[:port], + protocol: args[:protocol], + } + ingress_db_save! + end + end + desc "manifest ", "Renders a manifest of selected type" + long_desc <<-LONGDESC + Available types: + \x5 BOSH Render Bosh environement + \x5 CONCOURSE Render Concourse deployment + \x5 VAULT Render Vault deployment + \x5 INGRESS Render Ingress deployment + LONGDESC method_option :cloud, type: :string, desc: "Cloud provider", enum: %w{aws gcp}, required: true # Render a manifest of selected type based on config/cloud.yml and terraform apply results def manifest(type) + type = type.downcase say "Rendering #{type} manifest", :green @values = parse_cloud_config @tf_output = parse_tf_state('terraform/terraform.tfstate') if options[:cloud] == 'aws' @@ -18,6 +52,9 @@ def manifest(type) @private_subnet = IPAddr.new(@values['gcp']['subnet_cidr']).to_range.to_a end + @static_ip_vault = @private_subnet[11].to_s + @static_ips_concourse = [@private_subnet[12]].map(&:to_s) + case type when "bosh" directory("#{options[:cloud]}/deployments/bosh", 'deployments/bosh') @@ -31,15 +68,20 @@ def manifest(type) copy_file("#{options[:cloud]}/docs/concourse.md", "docs/concourse.md") template("#{options[:cloud]}/bin/concourse-deploy.sh.tt", "bin/concourse-deploy.sh") chmod('bin/concourse-deploy.sh', 0755) + ingress_add_entry(@values['concourse']['hostname'], @static_ips_concourse, port: 8080) when "vault" template("#{options[:cloud]}/deployments/vault/vault.yml.erb", "deployments/vault/vault.yml") copy_file("#{options[:cloud]}/docs/vault.md", "docs/vault.md") template("#{options[:cloud]}/bin/vault-deploy.sh.tt", "bin/vault-deploy.sh") chmod('bin/vault-deploy.sh', 0755) + ingress_add_entry(@values['vault']['hostname'], [@static_ip_vault], port: 8200) - when "nginx" - template("#{options[:cloud]}/deployments/nginx/nginx.yml.erb", "deployments/nginx/nginx.yml") + when "ingress" + template("#{options[:cloud]}/deployments/ingress/ingress.yml.erb", "deployments/ingress/ingress.yml") + copy_file("#{options[:cloud]}/docs/ingress.md", "docs/ingress.md") + template("#{options[:cloud]}/bin/ingress-deploy.sh.tt", "bin/ingress-deploy.sh") + chmod('bin/ingress-deploy.sh', 0755) else say "Manifest type not specified" diff --git a/tpl/gcp/README.md b/tpl/gcp/README.md index 53a1e45..8fc4f22 100644 --- a/tpl/gcp/README.md +++ b/tpl/gcp/README.md @@ -1,13 +1,14 @@ -## GCP Cloud +# GCP Cloud -### Setup +## Setup +### Prerequisites Set path to your service account credentials: ``` export GOOGLE_CREDENTIALS=*~/credentials/service-account.json* ``` - +### Setup the basic infrastructure and bastion Apply terraform code ``` pushd terraform && terraform init && terraform apply && popd @@ -16,6 +17,7 @@ pushd terraform && terraform init && terraform apply && popd [Note] To destroy Bastion later, use `terraform destroy -target google_compute_instance.bastion` +### Setup BOSH Render BOSH manifest and related files ``` kite render manifest bosh --cloud gcp @@ -23,6 +25,7 @@ kite render manifest bosh --cloud gcp Prepare BOSH environment using instructions from [docs/bosh.md](docs/bosh.md) +### Setup VAULT Render Vault deployment ``` kite render manifest vault --cloud gcp @@ -30,6 +33,10 @@ kite render manifest vault --cloud gcp Follow instructions from [docs/vault.md](docs/vault.md) to deploy Vault +### Setup CONCOURSE +[Note] +To expose concourse publicly, you must create first (manually) a virtual IP in GCP and create a DNS A entry for the hostname for this IP. Set the IP into config/cloud.yml (concourse.vip). + Render Concourse manifest ``` kite render manifest concourse --cloud gcp diff --git a/tpl/gcp/bin/base/setup-tunnel.sh.tt b/tpl/gcp/bin/base/setup-tunnel.sh.tt index a8ffa7f..fa23845 100644 --- a/tpl/gcp/bin/base/setup-tunnel.sh.tt +++ b/tpl/gcp/bin/base/setup-tunnel.sh.tt @@ -1,8 +1,13 @@ #!/usr/bin/env bash + pushd terraform BASTION_IP="$(terraform output bastion_ip)" popd -ssh -D 5000 -fNC kite@$BASTION_IP -i <%= @values['kite']['private_key_path'] %> - -export BOSH_ALL_PROXY=socks5://localhost:5000 +if [[ -z "${BASTION_IP}" ]]; then + echo "Something goes wrong, please check terraform environement" 1>&2 + false +else + ssh -D 5000 -fNC kite@${BASTION_IP} -i ~/.ssh/kite.key + export BOSH_ALL_PROXY=socks5://localhost:5000 +fi diff --git a/tpl/gcp/bin/ingress-deploy.sh.tt b/tpl/gcp/bin/ingress-deploy.sh.tt new file mode 100755 index 0000000..0183e65 --- /dev/null +++ b/tpl/gcp/bin/ingress-deploy.sh.tt @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -xe + +bosh -e <%= @values['bosh']['name'] %> upload-release https://github.com/cloudfoundry-community/nginx-release/releases/download/v1.12.1/nginx-1.12.1.tgz + +bosh -e <%= @values['bosh']['name'] %> -d ingress deploy deployments/ingress/ingress.yml diff --git a/tpl/gcp/deployments/bosh/cloud-config.yml.tt b/tpl/gcp/deployments/bosh/cloud-config.yml.tt index 2f148a5..e3d9f9c 100644 --- a/tpl/gcp/deployments/bosh/cloud-config.yml.tt +++ b/tpl/gcp/deployments/bosh/cloud-config.yml.tt @@ -10,11 +10,22 @@ vm_types: root_disk_size_gb: 20 root_disk_type: pd-ssd +- name: ingress + cloud_properties: + machine_type: g1-small + root_disk_size_gb: 20 + root_disk_type: pd-ssd + tags: + - http-server + - https-server + - name: worker cloud_properties: - machine_type: n1-standard-4 + machine_type: n1-standard-2 root_disk_size_gb: 100 root_disk_type: pd-ssd + tags: + - no-ip # vm_extensions: # - name: concourse-lb @@ -27,10 +38,12 @@ compilation: reuse_compilation_vms: true az: z1 cloud_properties: - machine_type: n1-standard-4 + machine_type: n1-standard-2 root_disk_size_gb: 100 root_disk_type: pd-ssd preemptible: true + tags: + - no-ip networks: - name: public @@ -46,7 +59,6 @@ networks: subnetwork_name: <%= @values['gcp']['subnet_name'] %> ephemeral_external_ip: false tags: - - no-ip - platform-internal - concourse-public - concourse-internal diff --git a/tpl/gcp/deployments/concourse/concourse.yml.tt b/tpl/gcp/deployments/concourse/concourse.yml.tt index bf2b1c0..f28291b 100644 --- a/tpl/gcp/deployments/concourse/concourse.yml.tt +++ b/tpl/gcp/deployments/concourse/concourse.yml.tt @@ -15,14 +15,14 @@ instance_groups: stemcell: trusty networks: - name: public - static_ips: [<%= @private_subnet[12] %>] + static_ips: <%= @static_ips_concourse %> default: [dns, gateway] jobs: - name: atc release: concourse properties: - bind_port: 80 + bind_port: 8080 external_url: <%= @values['concourse']['url'] %> basic_auth_username: <%= @values['concourse']['auth_username'] %> basic_auth_password: ((auth_password)) @@ -33,7 +33,7 @@ instance_groups: backend: token client_token: ((vault_token)) path_prefix: /concourse - url: "http://<%= @private_subnet[11] %>:8200" # expecting Vault to be deployed first + url: "http://<%= @static_ip_vault %>:8200" # expecting Vault to be deployed first postgresql_database: &atc_db atc @@ -47,7 +47,9 @@ instance_groups: azs: [z1] stemcell: trusty persistent_disk_type: database - networks: [{name: public}] + networks: + - name: public + default: [dns, gateway] jobs: - name: postgresql release: concourse @@ -62,7 +64,9 @@ instance_groups: vm_type: worker azs: [z1] stemcell: trusty - networks: [{name: public}] + networks: + - name: public + default: [dns, gateway] jobs: - name: groundcrew release: concourse diff --git a/tpl/gcp/deployments/ingress/ingress.yml.erb b/tpl/gcp/deployments/ingress/ingress.yml.erb new file mode 100644 index 0000000..6a280c3 --- /dev/null +++ b/tpl/gcp/deployments/ingress/ingress.yml.erb @@ -0,0 +1,107 @@ +--- +name: ingress + +releases: +- name: nginx + version: latest + +instance_groups: +- name: ingress + instances: 1 + vm_type: ingress-tiny + azs: [z1] + stemcell: trusty + networks: + - name: public + static_ips: [<%= @private_subnet[13] %>] + default: [dns, gateway] + + - name: vip + static_ips: [<%= @values['ingress']['vip'] %>] + + jobs: + - name: nginx + release: nginx + properties: + nginx_conf: | + worker_processes 1; + error_log /var/vcap/sys/log/nginx/error.log info; + events { + worker_connections 1024; + } + + http { + include /var/vcap/packages/nginx/conf/mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + server_names_hash_bucket_size 64; + + server { + listen 80; + return 301 https://$host$request_uri; + } +<% ingress_db.each do |hostname, config| %> +<% upstream_name = hostname.gsub('.', '-') %> + upstream <%= upstream_name %> { + <%- config[:upstreams].each do |upstream| -%> + server <%= upstream %>:<%= config[:port] %>; + <%- end -%> + } + server { + listen 443; + server_name <%= hostname %>; + ssl_certificate_key /var/vcap/jobs/nginx/etc/<%= hostname %>/key.pem; + ssl_certificate /var/vcap/jobs/nginx/etc/<%= hostname %>/cert.pem; + ssl on; + ssl_session_cache builtin:1000 shared:SSL:10m; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers HIGH:!aNULL:!eNULL:!EXPORT:!CAMELLIA:!DES:!MD5:!PSK:!RC4; + ssl_prefer_server_ciphers on; + + access_log /var/vcap/sys/log/nginx/<%= hostname %>-access.log; + error_log /var/vcap/sys/log/nginx/<%= hostname %>-error.log; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_pass <%= config[:protocol] %>://<%= upstream_name%>; + proxy_read_timeout 90; + } + } +<% end %> + } + + pre_start: | + #!/bin/bash + set -x + source /etc/profile + export HOME=/root + export USER=root + echo "Running pre_start script as ${USER} with ${SHELL} with home ${HOME}" + if [[ ! -f ${HOME}/.acme.sh/acme.sh.env ]]; then + curl -s https://get.acme.sh | sh + fi + source ${HOME}/.acme.sh/acme.sh.env +<% ingress_db.each do |hostname, config| -%> + mkdir -p /var/vcap/jobs/nginx/etc/<%= hostname %>/ + ${HOME}/.acme.sh/acme.sh --issue --tls -d <%= hostname %> + ${HOME}/.acme.sh/acme.sh --install-cert -d <%= hostname %> \ + --key-file /var/vcap/jobs/nginx/etc/<%= hostname %>/key.pem \ + --fullchain-file /var/vcap/jobs/nginx/etc/<%= hostname %>/cert.pem +<%- end -%> + +stemcells: +- alias: trusty + os: ubuntu-trusty + version: latest + +update: + canaries: 1 + max_in_flight: 1 + serial: false + canary_watch_time: 1000-60000 + update_watch_time: 1000-60000 diff --git a/tpl/gcp/deployments/nginx/nginx.yml.erb b/tpl/gcp/deployments/nginx/nginx.yml.erb deleted file mode 100644 index 0b14cf5..0000000 --- a/tpl/gcp/deployments/nginx/nginx.yml.erb +++ /dev/null @@ -1,62 +0,0 @@ ---- -name: nginx - -releases: -- name: nginx - version: latest - - -instance_groups: -- name: nginx - instances: 1 - vm_type: default - azs: [z1] - stemcell: trusty - networks: - - name: public - static_ips: [<%= @private_subnet[13] %>] - default: [dns, gateway] - - jobs: - - name: nginx - release: nginx - properties: - nginx_conf: | - worker_processes 1; - error_log /var/vcap/sys/log/nginx/error.log info; - events { - worker_connections 1024; - } - - http { - include /var/vcap/packages/nginx/conf/mime.types; - default_type application/octet-stream; - sendfile on; - keepalive_timeout 65; - server_names_hash_bucket_size 64; - server { - server_name kite-nginx; - - location / { - proxy_pass http://<%= @private_subnet[13] %>; # Concourse web panel IP - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $http_host; - proxy_redirect off; - } - - access_log /var/vcap/sys/log/nginx/kite-access.log; - error_log /var/vcap/sys/log/nginx/kite-error.log; - } - } - -stemcells: -- alias: trusty - os: ubuntu-trusty - version: latest - -update: - canaries: 1 - max_in_flight: 1 - serial: false - canary_watch_time: 1000-60000 - update_watch_time: 1000-60000 diff --git a/tpl/gcp/docs/bosh.md b/tpl/gcp/docs/bosh.md index 215b01b..b1ed97a 100644 --- a/tpl/gcp/docs/bosh.md +++ b/tpl/gcp/docs/bosh.md @@ -29,3 +29,8 @@ Connect to the Director . bin/set-env.sh ``` + +Update the cloud configuration +``` +bosh -e bosh-director ucc deployments/bosh/cloud-config.yml +``` diff --git a/tpl/gcp/docs/ingress.md b/tpl/gcp/docs/ingress.md new file mode 100644 index 0000000..c9411d1 --- /dev/null +++ b/tpl/gcp/docs/ingress.md @@ -0,0 +1,14 @@ +#### [Back](../README.md) + +## Ingress + +### Prerequisites + +Before deploying the Ingress ensure: + +- All hostnames resolve to the VIP configured in cloud.yml (this is mandatory to issue SSL certificates) + +### Deployment + +To deploy Vault, use `./bin/ingress-deploy.sh` + diff --git a/tpl/gcp/docs/vault.md b/tpl/gcp/docs/vault.md index 7c8fe08..413ef5b 100644 --- a/tpl/gcp/docs/vault.md +++ b/tpl/gcp/docs/vault.md @@ -16,7 +16,8 @@ To deploy Vault, use `./bin/vault-deploy.sh` ### Connection -- Export your Vault's IP using `export VAULT_ADDR=http://*vault_ip*:8200` +- You can now deploy the ingress to access vault +- Export your Vault's address using `export VAULT_ADDR=https://*vault_host*` - Run `vault init` to initialize the vault - Store the keys displayed after init - Unseal the vault by running `vault unseal` three times using three keys from the previous step diff --git a/tpl/gcp/terraform/main.tf b/tpl/gcp/terraform/main.tf index 866c57f..1fecd52 100644 --- a/tpl/gcp/terraform/main.tf +++ b/tpl/gcp/terraform/main.tf @@ -29,7 +29,7 @@ resource "google_compute_address" "bastion" { resource "google_compute_instance" "bastion" { name = "bastion" - machine_type = "n1-standard-1" + machine_type = "g1-small" zone = "${var.zone}" tags = ["bastion", "platform-internal"] diff --git a/tpl/skel/config/cloud.yml b/tpl/skel/config/cloud.yml index 58fa536..7c15bd5 100644 --- a/tpl/skel/config/cloud.yml +++ b/tpl/skel/config/cloud.yml @@ -2,7 +2,7 @@ kite: keypair_name: "kitekey" public_key_path: "~/.ssh/kite.key.pub" private_key_path: "~/.ssh/kite.key" - bucket_name: "kite-state" + bucket_name: "kite-state-project-example" aws: access_key: "enter your amazon key" @@ -24,18 +24,26 @@ aws: gcp: project: "gcp-project" region: "europe-west1" - zone: "europe-west1-b" + zone: "europe-west1-c" service_account: "~/safe/terraform.json" vpc_name: "platform-tools" subnet_name: "platform-net" - subnet_cidr: "10.0.0.0/24" - internal_gw: "10.0.0.2" + subnet_cidr: "10.0.20.0/24" + internal_gw: "10.0.20.2" bosh: name: "bosh-director" static_ip: "10.0.20.10" +ingress: + vip: "42.42.42.42" + +vault: + hostname: "vault.example.com" + concourse: hostname: "concourse.example.com" - url: "http://concourse.example.com" + url: "https://concourse.example.com" auth_username: "concourse" + auth_password: "concourse" + db_password: "concourse"