E2E: provide options for reverse proxy for web UI

e2e/terraform/README.md

@@ -84,6 +84,27 @@ folder and upload them to the cluster during provisioning.
 * `etc/consul.d` are the Consul agent configuration files.
 * `etc/acls` are ACL policy files for Consul and Vault.
 
+## Web UI
+
+To access the web UI, deploy a reverse proxy to the cluster. All
+clients have a TLS proxy certificate at `/etc/nomad.d/tls_proxy.crt`
+and a self-signed cert at `/etc/nomad.d/self_signed.crt`. See
+`../ui/inputs/proxy.nomad` for an example of using this. Deploy as follows:
+
+```sh
+nomad namespace apply proxy
+nomad job run ../ui/input/proxy.nomad
+```
+
+You can get the public IP for the proxy allocation from the following
+nested query:
+
+```sh
+nomad node status -json -verbose \
+    $(nomad operator api '/v1/allocations?namespace=proxy' | jq -r '.[] | select(.JobID == "nomad-proxy") | .NodeID') \
+    | jq '.Attributes."unique.platform.aws.public-ipv4"'
+```
+
 ## Outputs
 
 After deploying the infrastructure, you can get connection information

e2e/terraform/network.tf

@@ -34,6 +34,14 @@ resource "aws_security_group" "primary" {
     cidr_blocks = [local.ingress_cidr]
   }
 
+  # UI reverse proxy
+  ingress {
+    from_port   = 6464
+    to_port     = 6464
+    protocol    = "tcp"
+    cidr_blocks = [local.ingress_cidr]
+  }
+
   # Fabio
   ingress {
     from_port   = 9998

e2e/terraform/provision-nomad/install-linux.tf

@@ -91,6 +91,10 @@ resource "null_resource" "install_nomad_configs_linux" {
       "sudo mv /tmp/tls.hcl /etc/nomad.d/tls.hcl",
       "sudo mv /tmp/agent-${var.instance.public_ip}.key /etc/nomad.d/tls/agent.key",
       "sudo mv /tmp/agent-${var.instance.public_ip}.crt /etc/nomad.d/tls/agent.crt",
+      "sudo mv /tmp/tls_proxy.key /etc/nomad.d/tls/tls_proxy.key",
+      "sudo mv /tmp/tls_proxy.crt /etc/nomad.d/tls/tls_proxy.crt",
+      "sudo mv /tmp/self_signed.key /etc/nomad.d/tls/self_signed.key",
+      "sudo mv /tmp/self_signed.crt /etc/nomad.d/tls/self_signed.crt",
       "sudo mv /tmp/ca.crt /etc/nomad.d/tls/ca.crt",
 
       "sudo mv /tmp/nomad.service /etc/systemd/system/nomad.service",

e2e/terraform/provision-nomad/main.tf

@@ -129,8 +129,25 @@ resource "null_resource" "upload_nomad_configs" {
     source      = local_sensitive_file.nomad_client_cert.filename
     destination = "/tmp/agent-${var.instance.public_ip}.crt"
   }
+  provisioner "file" {
+    source      = "keys/tls_api_client.key"
+    destination = "/tmp/tls_proxy.key"
+  }
+  provisioner "file" {
+    source      = "keys/tls_api_client.crt"
+    destination = "/tmp/tls_proxy.crt"
+  }
   provisioner "file" {
     source      = "keys/tls_ca.crt"
     destination = "/tmp/ca.crt"
   }
+  provisioner "file" {
+    source      = "keys/self_signed.key"
+    destination = "/tmp/self_signed.key"
+  }
+  provisioner "file" {
+    source      = "keys/self_signed.crt"
+    destination = "/tmp/self_signed.crt"
+  }
+
 }

e2e/terraform/tls_client.tf

@@ -38,3 +38,35 @@ resource "local_sensitive_file" "api_client_cert" {
   content  = tls_locally_signed_cert.api_client.cert_pem
   filename = "keys/tls_api_client.crt"
 }
+
+# Self signed cert for reverse proxy
+
+resource "tls_private_key" "self_signed" {
+  algorithm   = "ECDSA"
+  ecdsa_curve = "P384"
+}
+
+resource "tls_self_signed_cert" "self_signed" {
+  private_key_pem = tls_private_key.self_signed.private_key_pem
+  subject {
+    common_name  = "${local.random_name}.local"
+    organization = "HashiCorp, Inc."
+  }
+
+  ip_addresses = toset(aws_instance.client_ubuntu_bionic_amd64.*.public_ip)
+
+  validity_period_hours = 720
+  allowed_uses = [
+    "server_auth"
+  ]
+}
+
+resource "local_sensitive_file" "self_signed_key" {
+  content  = tls_private_key.self_signed.private_key_pem
+  filename = "keys/self_signed.key"
+}
+
+resource "local_sensitive_file" "self_signed_cert" {
+  content  = tls_self_signed_cert.self_signed.cert_pem
+  filename = "keys/self_signed.crt"
+}

e2e/ui/input/proxy.nomad

@@ -0,0 +1,119 @@
+job "nomad-proxy" {
+  datacenters = ["dc1", "dc2"]
+  namespace   = "proxy"
+
+  group "proxy" {
+
+    network {
+      port "www" {
+        static = 6464
+        to     = 443
+      }
+    }
+
+    task "nginx" {
+
+      driver = "docker"
+
+      config {
+        image = "nginx:latest"
+        ports = ["www"]
+
+        mount {
+          type   = "bind"
+          source = "local/nginx.conf"
+          target = "/etc/nginx/nginx.conf"
+        }
+
+        mount {
+          type   = "bind"
+          source = "/etc/nomad.d/tls/tls_proxy.key"
+          target = "/etc/ssl/tls_proxy.key"
+        }
+
+        mount {
+          type   = "bind"
+          source = "/etc/nomad.d/tls/tls_proxy.crt"
+          target = "/etc/ssl/tls_proxy.crt"
+        }
+
+        mount {
+          type   = "bind"
+          source = "/etc/nomad.d/tls/self_signed.key"
+          target = "/etc/ssl/self_signed.key"
+        }
+
+        mount {
+          type   = "bind"
+          source = "/etc/nomad.d/tls/self_signed.crt"
+          target = "/etc/ssl/self_signed.crt"
+        }
+      }
+
+      resources {
+        cpu    = 256
+        memory = 128
+      }
+
+      # this template is mostly lifted from the Learn Guide:
+      # https://learn.hashicorp.com/tutorials/nomad/reverse-proxy-ui
+      template {
+        destination = "local/nginx.conf"
+        data        = <<EOT
+
+events {}
+
+http {
+  server {
+
+    listen              443 ssl;
+    server_name         _;
+    ssl_certificate     /etc/ssl/self_signed.crt;
+    ssl_certificate_key /etc/ssl/self_signed.key;
+
+    location / {
+      proxy_pass https://nomad-ws;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_ssl_certificate     /etc/ssl/tls_proxy.crt;
+      proxy_ssl_certificate_key /etc/ssl/tls_proxy.key;
+
+      # Nomad blocking queries will remain open for a default of 5 minutes.
+      # Increase the proxy timeout to accommodate this timeout with an
+      # additional grace period.
+      proxy_read_timeout 310s;
+
+      # Nomad log streaming uses streaming HTTP requests. In order to
+      # synchronously stream logs from Nomad to NGINX to the browser
+      # proxy buffering needs to be turned off.
+      proxy_buffering off;
+
+      # The Upgrade and Connection headers are used to establish
+      # a WebSockets connection.
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
+
+      # The default Origin header will be the proxy address, which
+      # will be rejected by Nomad. It must be rewritten to be the
+      # host address instead.
+      proxy_set_header Origin "${scheme}://${proxy_host}";
+    }
+  }
+
+  # WebSockets are stateful connections but we're deploying only one proxy
+  # and proxying to the local Nomad client. That client will stream RPCs
+  # from the server. But we've left ip_hash here in case someone comes
+  # along and copy-and-pastes this configuration elsewhere without reading
+  # the Learn Guide.
+  upstream nomad-ws {
+    ip_hash;
+    server {{ env "attr.unique.network.ip-address" }}:4646;
+  }
+}
+
+EOT
+      }
+
+
+    }
+  }
+}