e2e: add flag to bootstrap Nomad ACLs (#8961)

Adds a `nomad_acls` flag to our Terraform stack that bootstraps Nomad ACLs via
a `local-exec` provider. There's no way to set the `NOMAD_TOKEN` in the Nomad
TF provider if we're bootstrapping in the same Terraform stack, so instead of
using `resource.nomad_acl_token`, we also bootstrap a wide-open anonymous
policy. The resulting management token is exported as an environment var with
`$(terraform output environment)` and tests that want stricter ACLs will be
able to write them using that token.

This should also provide a basis to do similar work with Consul ACLs in the
future.

e2e/terraform/README.md

@@ -60,6 +60,12 @@ You'll need to pass one of the following variables in either your
 If you want to deploy the Enterprise build of a specific SHA, include
 `-var 'nomad_enterprise=true'`.
 
+If you want to bootstrap Nomad ACLs, include `-var 'nomad_acls=true'`.
+
+> Note: If you bootstrap ACLs you will see "No cluster leader" in the output
+> several times while the ACL bootstrap script polls the cluster to start and
+> and elect a leader.
+
 ## Profiles
 
 The `profile` field selects from a set of configuration files for Nomad,

e2e/terraform/acls/anonymous.policy.hcl

@@ -0,0 +1,24 @@
+namespace "*" {
+  policy       = "write"
+  capabilities = ["alloc-node-exec"]
+}
+
+agent {
+  policy = "write"
+}
+
+operator {
+  policy = "write"
+}
+
+quota {
+  policy = "write"
+}
+
+node {
+  policy = "write"
+}
+
+host_volume "*" {
+  policy = "write"
+}

e2e/terraform/acls/bootstrap-nomad.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+while true :
+do
+    ROOT_TOKEN=$(nomad acl bootstrap | awk '/Secret ID/{print $4}')
+    if [ ! -z $ROOT_TOKEN ]; then break; fi
+    sleep 5
+done
+set -e
+
+export NOMAD_TOKEN="$ROOT_TOKEN"
+
+mkdir -p ../keys
+echo $NOMAD_TOKEN > "${DIR}/../keys/nomad_root_token"
+
+# Our default policy after bootstrapping will be full-access. Without
+# further policy, we only test that we're hitting the ACL code
+# Tests can set their own ACL policy using the management token so
+# long as they clean up the ACLs afterwards.
+nomad acl policy apply \
+      -description "Anonymous policy (full-access)" \
+      anonymous \
+      "${DIR}/anonymous.policy.hcl"

e2e/terraform/config/shared/README.md

@@ -0,0 +1,5 @@
+### Shared configs
+
+The only configurations that should go here are ones that we want to be able
+to toggle on/off for any profile. Adding a new configuration here requires
+adding a flag to the provision scripts as well to symlink it.

e2e/terraform/config/shared/nomad-acl.hcl

@@ -0,0 +1,3 @@
+acl {
+  enabled = true
+}

e2e/terraform/nomad-acls.tf

@@ -0,0 +1,29 @@
+# Bootstrapping Nomad ACLs:
+# We can't both bootstrap the ACLs and use the Nomad TF provider's
+# resource.nomad_acl_token in the same Terraform run, because there's no way
+# to get the management token into the provider's environment after we bootstrap.
+# So we run a bootstrapping script and write our management token into a file
+# that we read in for the output of $(terraform output environment) later.
+
+resource "null_resource" "bootstrap_nomad_acls" {
+  depends_on = [module.nomad_server]
+  triggers = {
+    script = data.template_file.bootstrap_script.rendered
+  }
+
+  provisioner "local-exec" {
+    command = data.template_file.bootstrap_script.rendered
+  }
+}
+
+# write the bootstrap token to the keys/ directory (where the ssh key is)
+# so that we can read it into the data.local_file later. If not set,
+# ensure that it's empty.
+data "template_file" "bootstrap_script" {
+  template = var.nomad_acls ? "NOMAD_ADDR=http://${aws_instance.server.0.public_ip}:4646 ./acls/bootstrap-nomad.sh" : "mkdir -p ${path.root}/keys; echo > ${path.root}/keys/nomad_root_token"
+}
+
+data "local_file" "nomad_token" {
+  depends_on = [null_resource.bootstrap_nomad_acls]
+  filename   = "${path.root}/keys/nomad_root_token"
+}

e2e/terraform/nomad.tf

@@ -20,6 +20,7 @@ module "nomad_server" {
   nomad_local_binary = count.index < length(var.nomad_local_binary_server) ? var.nomad_local_binary_server[count.index] : var.nomad_local_binary
 
   nomad_enterprise = var.nomad_enterprise
+  nomad_acls       = var.nomad_acls
 
   connection = {
     type        = "ssh"
@@ -54,6 +55,7 @@ module "nomad_client_linux" {
   nomad_local_binary = count.index < length(var.nomad_local_binary_client_linux) ? var.nomad_local_binary_client_linux[count.index] : var.nomad_local_binary
 
   nomad_enterprise = var.nomad_enterprise
+  nomad_acls       = false
 
   connection = {
     type        = "ssh"
@@ -89,6 +91,7 @@ module "nomad_client_windows" {
   nomad_local_binary = count.index < length(var.nomad_local_binary_client_windows) ? var.nomad_local_binary_client_windows[count.index] : ""
 
   nomad_enterprise = var.nomad_enterprise
+  nomad_acls       = false
 
   connection = {
     type        = "ssh"

e2e/terraform/outputs.tf

@@ -39,5 +39,6 @@ output "environment" {
 export NOMAD_ADDR=http://${aws_instance.server[0].public_ip}:4646
 export CONSUL_HTTP_ADDR=http://${aws_instance.server[0].public_ip}:8500
 export NOMAD_E2E=1
+export NOMAD_TOKEN=${data.local_file.nomad_token.content}
 EOM
 }

e2e/terraform/packer/linux/provision.sh

@@ -18,6 +18,7 @@ Options for configuration:
  --index INDEX              count of instance, for profiles with per-instance config
  --nostart                  do not start or restart Nomad
  --enterprise               if nomad_sha is passed, use the ENT version
+ --nomad_acls               write Nomad ACL configuration
 
 EOF
 
@@ -35,6 +36,7 @@ NOMAD_PROFILE=
 NOMAD_ROLE=
 NOMAD_INDEX=
 BUILD_FOLDER="builds-oss"
+ACLS=0
 
 install_from_s3() {
     # check that we don't already have this version
@@ -110,6 +112,10 @@ install_config_profile() {
         sym "${NOMAD_PROFILE}/consul/${NOMAD_ROLE}/indexed/" "*${NOMAD_INDEX}*" /etc/consul.d
         sym "${NOMAD_PROFILE}/vault/${NOMAD_ROLE}/indexed/" "*${NOMAD_INDEX}*" /etc/vault.d
     fi
+
+    if [ $ACLS == "1" ]; then
+        sudo ln -fs /opt/config/shared/nomad-acl.hcl /etc/nomad.d/acl.hcl
+    fi
 }
 
 
@@ -159,6 +165,10 @@ opt="$1"
             BUILD_FOLDER="builds-ent"
             shift
             ;;
+        --nomad_acls)
+            ACLS=1
+            shift
+            ;;
         *) usage ;;
     esac
 done

e2e/terraform/packer/windows/provision.ps1

@@ -3,6 +3,7 @@ param(
     [string]$nomad_version,
     [string]$nomad_binary,
     [switch]$enterprise = $false,
+    [switch]$nomad_acls = $false,
     [string]$config_profile,
     [string]$role,
     [string]$index,

e2e/terraform/provision-nomad/main.tf

@@ -46,7 +46,7 @@ resource "null_resource" "provision_nomad" {
 }
 
 data "template_file" "provision_script" {
-  template = "${local.provision_script}${data.template_file.arg_nomad_sha.rendered}${data.template_file.arg_nomad_version.rendered}${data.template_file.arg_nomad_binary.rendered}${data.template_file.arg_nomad_enterprise.rendered}${data.template_file.arg_profile.rendered}${data.template_file.arg_role.rendered}${data.template_file.arg_index.rendered}"
+  template = "${local.provision_script}${data.template_file.arg_nomad_sha.rendered}${data.template_file.arg_nomad_version.rendered}${data.template_file.arg_nomad_binary.rendered}${data.template_file.arg_nomad_enterprise.rendered}${data.template_file.arg_nomad_acls.rendered}${data.template_file.arg_profile.rendered}${data.template_file.arg_role.rendered}${data.template_file.arg_index.rendered}"
 }
 
 data "template_file" "arg_nomad_sha" {
@@ -62,7 +62,11 @@ data "template_file" "arg_nomad_binary" {
 }
 
 data "template_file" "arg_nomad_enterprise" {
-  template = var.nomad_enterprise != false ? " ${local._arg}enterprise" : ""
+  template = var.nomad_enterprise ? " ${local._arg}enterprise" : ""
+}
+
+data "template_file" "arg_nomad_acls" {
+  template = var.nomad_acls ? " ${local._arg}nomad_acls" : ""
 }
 
 data "template_file" "arg_profile" {

e2e/terraform/provision-nomad/variables.tf

@@ -28,6 +28,12 @@ variable "nomad_enterprise" {
   default     = false
 }
 
+variable "nomad_acls" {
+  type        = bool
+  description = "Bootstrap ACLs"
+  default     = false
+}
+
 variable "profile" {
   type        = string
   description = "The name of the configuration profile (ex. 'full-cluster')"

e2e/terraform/terraform.tfvars

@@ -4,3 +4,5 @@ server_count         = "3"
 client_count         = "4"
 windows_client_count = "1"
 profile              = "full-cluster"
+nomad_enterprise     = true
+nomad_acls           = true

e2e/terraform/variables.tf

@@ -78,6 +78,13 @@ variable "nomad_enterprise" {
   description = "If nomad_sha is used, deploy Nomad Enterprise"
   default     = false
 }
+
+variable "nomad_acls" {
+  type        = bool
+  description = "Bootstrap ACLs"
+  default     = false
+}
+
 # ----------------------------------------
 # If you want to deploy multiple versions you can use these variables to
 # provide a list of builds to override the values of nomad_sha, nomad_version,