From aafcf979844cc5af0c21b7dc4381163644fdf737 Mon Sep 17 00:00:00 2001
From: Tim Gross <tgross@hashicorp.com>
Date: Tue, 19 Apr 2022 16:55:05 -0400
Subject: [PATCH] E2E: provide options for reverse proxy for web UI (#12671)

Our E2E test environment is deployed with mTLS, but it's impractical
for us to use mTLS in headless browsers for automated testing (or even
in manual testing). Provide certificates for proxying the web UI via
Nginx. This proxy uses client certs for proxying to the HTTP endpoint
and a self-signed cert for the browser-facing endpoint. We can accept
certificate errors in the automated tests we'll be adding in the next
step of this work.
---
 e2e/terraform/README.md                       |  21 ++++
 e2e/terraform/network.tf                      |   8 ++
 .../provision-nomad/install-linux.tf          |   4 +
 e2e/terraform/provision-nomad/main.tf         |  17 +++
 e2e/terraform/tls_client.tf                   |  32 +++++
 e2e/ui/input/proxy.nomad                      | 119 ++++++++++++++++++
 6 files changed, 201 insertions(+)
 create mode 100644 e2e/ui/input/proxy.nomad

diff --git a/e2e/terraform/README.md b/e2e/terraform/README.md
index c94c5c27201..937168de251 100644
--- a/e2e/terraform/README.md
+++ b/e2e/terraform/README.md
@@ -84,6 +84,27 @@ folder and upload them to the cluster during provisioning.
 * `etc/consul.d` are the Consul agent configuration files.
 * `etc/acls` are ACL policy files for Consul and Vault.
 
+## Web UI
+
+To access the web UI, deploy a reverse proxy to the cluster. All
+clients have a TLS proxy certificate at `/etc/nomad.d/tls_proxy.crt`
+and a self-signed cert at `/etc/nomad.d/self_signed.crt`. See
+`../ui/inputs/proxy.nomad` for an example of using this. Deploy as follows:
+
+```sh
+nomad namespace apply proxy
+nomad job run ../ui/input/proxy.nomad
+```
+
+You can get the public IP for the proxy allocation from the following
+nested query:
+
+```sh
+nomad node status -json -verbose \
+    $(nomad operator api '/v1/allocations?namespace=proxy' | jq -r '.[] | select(.JobID == "nomad-proxy") | .NodeID') \
+    | jq '.Attributes."unique.platform.aws.public-ipv4"'
+```
+
 ## Outputs
 
 After deploying the infrastructure, you can get connection information
diff --git a/e2e/terraform/network.tf b/e2e/terraform/network.tf
index d3c6fea9f2b..79cd6b73c2a 100644
--- a/e2e/terraform/network.tf
+++ b/e2e/terraform/network.tf
@@ -34,6 +34,14 @@ resource "aws_security_group" "primary" {
     cidr_blocks = [local.ingress_cidr]
   }
 
+  # UI reverse proxy
+  ingress {
+    from_port   = 6464
+    to_port     = 6464
+    protocol    = "tcp"
+    cidr_blocks = [local.ingress_cidr]
+  }
+
   # Fabio
   ingress {
     from_port   = 9998
diff --git a/e2e/terraform/provision-nomad/install-linux.tf b/e2e/terraform/provision-nomad/install-linux.tf
index 3f1e65f56cd..b61a4ca87d1 100644
--- a/e2e/terraform/provision-nomad/install-linux.tf
+++ b/e2e/terraform/provision-nomad/install-linux.tf
@@ -91,6 +91,10 @@ resource "null_resource" "install_nomad_configs_linux" {
       "sudo mv /tmp/tls.hcl /etc/nomad.d/tls.hcl",
       "sudo mv /tmp/agent-${var.instance.public_ip}.key /etc/nomad.d/tls/agent.key",
       "sudo mv /tmp/agent-${var.instance.public_ip}.crt /etc/nomad.d/tls/agent.crt",
+      "sudo mv /tmp/tls_proxy.key /etc/nomad.d/tls/tls_proxy.key",
+      "sudo mv /tmp/tls_proxy.crt /etc/nomad.d/tls/tls_proxy.crt",
+      "sudo mv /tmp/self_signed.key /etc/nomad.d/tls/self_signed.key",
+      "sudo mv /tmp/self_signed.crt /etc/nomad.d/tls/self_signed.crt",
       "sudo mv /tmp/ca.crt /etc/nomad.d/tls/ca.crt",
 
       "sudo mv /tmp/nomad.service /etc/systemd/system/nomad.service",
diff --git a/e2e/terraform/provision-nomad/main.tf b/e2e/terraform/provision-nomad/main.tf
index f6d190dca5c..f51d97719cb 100644
--- a/e2e/terraform/provision-nomad/main.tf
+++ b/e2e/terraform/provision-nomad/main.tf
@@ -129,8 +129,25 @@ resource "null_resource" "upload_nomad_configs" {
     source      = local_sensitive_file.nomad_client_cert.filename
     destination = "/tmp/agent-${var.instance.public_ip}.crt"
   }
+  provisioner "file" {
+    source      = "keys/tls_api_client.key"
+    destination = "/tmp/tls_proxy.key"
+  }
+  provisioner "file" {
+    source      = "keys/tls_api_client.crt"
+    destination = "/tmp/tls_proxy.crt"
+  }
   provisioner "file" {
     source      = "keys/tls_ca.crt"
     destination = "/tmp/ca.crt"
   }
+  provisioner "file" {
+    source      = "keys/self_signed.key"
+    destination = "/tmp/self_signed.key"
+  }
+  provisioner "file" {
+    source      = "keys/self_signed.crt"
+    destination = "/tmp/self_signed.crt"
+  }
+
 }
diff --git a/e2e/terraform/tls_client.tf b/e2e/terraform/tls_client.tf
index db9f7aff0d5..794987fff79 100644
--- a/e2e/terraform/tls_client.tf
+++ b/e2e/terraform/tls_client.tf
@@ -38,3 +38,35 @@ resource "local_sensitive_file" "api_client_cert" {
   content  = tls_locally_signed_cert.api_client.cert_pem
   filename = "keys/tls_api_client.crt"
 }
+
+# Self signed cert for reverse proxy
+
+resource "tls_private_key" "self_signed" {
+  algorithm   = "ECDSA"
+  ecdsa_curve = "P384"
+}
+
+resource "tls_self_signed_cert" "self_signed" {
+  private_key_pem = tls_private_key.self_signed.private_key_pem
+  subject {
+    common_name  = "${local.random_name}.local"
+    organization = "HashiCorp, Inc."
+  }
+
+  ip_addresses = toset(aws_instance.client_ubuntu_bionic_amd64.*.public_ip)
+
+  validity_period_hours = 720
+  allowed_uses = [
+    "server_auth"
+  ]
+}
+
+resource "local_sensitive_file" "self_signed_key" {
+  content  = tls_private_key.self_signed.private_key_pem
+  filename = "keys/self_signed.key"
+}
+
+resource "local_sensitive_file" "self_signed_cert" {
+  content  = tls_self_signed_cert.self_signed.cert_pem
+  filename = "keys/self_signed.crt"
+}
diff --git a/e2e/ui/input/proxy.nomad b/e2e/ui/input/proxy.nomad
new file mode 100644
index 00000000000..a9e7c7764ce
--- /dev/null
+++ b/e2e/ui/input/proxy.nomad
@@ -0,0 +1,119 @@
+job "nomad-proxy" {
+  datacenters = ["dc1", "dc2"]
+  namespace   = "proxy"
+
+  group "proxy" {
+
+    network {
+      port "www" {
+        static = 6464
+        to     = 443
+      }
+    }
+
+    task "nginx" {
+
+      driver = "docker"
+
+      config {
+        image = "nginx:latest"
+        ports = ["www"]
+
+        mount {
+          type   = "bind"
+          source = "local/nginx.conf"
+          target = "/etc/nginx/nginx.conf"
+        }
+
+        mount {
+          type   = "bind"
+          source = "/etc/nomad.d/tls/tls_proxy.key"
+          target = "/etc/ssl/tls_proxy.key"
+        }
+
+        mount {
+          type   = "bind"
+          source = "/etc/nomad.d/tls/tls_proxy.crt"
+          target = "/etc/ssl/tls_proxy.crt"
+        }
+
+        mount {
+          type   = "bind"
+          source = "/etc/nomad.d/tls/self_signed.key"
+          target = "/etc/ssl/self_signed.key"
+        }
+
+        mount {
+          type   = "bind"
+          source = "/etc/nomad.d/tls/self_signed.crt"
+          target = "/etc/ssl/self_signed.crt"
+        }
+      }
+
+      resources {
+        cpu    = 256
+        memory = 128
+      }
+
+      # this template is mostly lifted from the Learn Guide:
+      # https://learn.hashicorp.com/tutorials/nomad/reverse-proxy-ui
+      template {
+        destination = "local/nginx.conf"
+        data        = <<EOT
+
+events {}
+
+http {
+  server {
+
+    listen              443 ssl;
+    server_name         _;
+    ssl_certificate     /etc/ssl/self_signed.crt;
+    ssl_certificate_key /etc/ssl/self_signed.key;
+
+    location / {
+      proxy_pass https://nomad-ws;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_ssl_certificate     /etc/ssl/tls_proxy.crt;
+      proxy_ssl_certificate_key /etc/ssl/tls_proxy.key;
+
+      # Nomad blocking queries will remain open for a default of 5 minutes.
+      # Increase the proxy timeout to accommodate this timeout with an
+      # additional grace period.
+      proxy_read_timeout 310s;
+
+      # Nomad log streaming uses streaming HTTP requests. In order to
+      # synchronously stream logs from Nomad to NGINX to the browser
+      # proxy buffering needs to be turned off.
+      proxy_buffering off;
+
+      # The Upgrade and Connection headers are used to establish
+      # a WebSockets connection.
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
+
+      # The default Origin header will be the proxy address, which
+      # will be rejected by Nomad. It must be rewritten to be the
+      # host address instead.
+      proxy_set_header Origin "${scheme}://${proxy_host}";
+    }
+  }
+
+  # WebSockets are stateful connections but we're deploying only one proxy
+  # and proxying to the local Nomad client. That client will stream RPCs
+  # from the server. But we've left ip_hash here in case someone comes
+  # along and copy-and-pastes this configuration elsewhere without reading
+  # the Learn Guide.
+  upstream nomad-ws {
+    ip_hash;
+    server {{ env "attr.unique.network.ip-address" }}:4646;
+  }
+}
+
+EOT
+      }
+
+
+    }
+  }
+}