e2e: refactor Consul configurations (#6559)

Ensure that we're reusing the base configuration between client and servers without the possibility of drift. Reduce the amount of `sed` mangling of the configuration file, and make recommended changes from `shellcheck` for this section of the provisioning script. Fixes some rebase errors on the Nomad config as well.
hashicorp · Oct 28, 2019 · 4abb6b5 · 4abb6b5
1 parent 56bfcdd
commit 4abb6b5
Show file tree

Hide file tree

Showing 14 changed files with 53 additions and 27 deletions.
diff --git a/e2e/terraform/compute.tf b/e2e/terraform/compute.tf
@@ -31,7 +31,7 @@ resource "aws_instance" "server" {
   provisioner "remote-exec" {
     inline = [
       "chmod +x /ops/shared/config/provision-server.sh",
-      "/ops/shared/config/provision-server.sh aws ${var.server_count} '${var.retry_join}' '${var.nomad_sha}' '${var.indexed == false ? "server.hcl" : "indexed/server-${count.index}.hcl"}'",
+      "/ops/shared/config/provision-server.sh aws ${var.server_count} '${var.nomad_sha}' '${var.indexed == false ? "server.hcl" : "indexed/server-${count.index}.hcl"}'",
     ]
 
     connection {
@@ -84,7 +84,7 @@ resource "aws_instance" "client" {
   provisioner "remote-exec" {
     inline = [
       "chmod +x /ops/shared/config/provision-client.sh",
-      "/ops/shared/config/provision-client.sh aws '${var.retry_join}' '${var.nomad_sha}' '${var.indexed == false ? "client.hcl" : "indexed/client-${count.index}.hcl"}'",
+      "/ops/shared/config/provision-client.sh aws '${var.nomad_sha}' '${var.indexed == false ? "client.hcl" : "indexed/client-${count.index}.hcl"}'",
     ]
 
     connection {

diff --git a/e2e/terraform/main.tf b/e2e/terraform/main.tf
@@ -28,11 +28,6 @@ variable "client_count" {
   default     = "4"
 }
 
-variable "retry_join" {
-  description = "Used by Consul to automatically form a cluster."
-  default     = "provider=aws tag_key=ConsulAutoJoin tag_value=auto-join"
-}
-
 variable "nomad_sha" {
   description = "The sha of Nomad to run"
 }

diff --git a/e2e/terraform/shared/config/README.md b/e2e/terraform/shared/config/README.md
diff --git a/e2e/terraform/shared/config/provision-client.sh b/e2e/terraform/shared/config/provision-client.sh
@@ -5,7 +5,6 @@ set -o nounset
 
 CONFIGDIR=/ops/shared/config
 
-CONSULCONFIGDIR=/etc/consul.d
 HADOOP_VERSION=hadoop-2.7.7
 HADOOPCONFIGDIR=/usr/local/$HADOOP_VERSION/etc/hadoop
 HOME_DIR=ubuntu
@@ -14,14 +13,16 @@ IP_ADDRESS=$(/usr/local/bin/sockaddr eval 'GetPrivateIP')
 DOCKER_BRIDGE_IP_ADDRESS=$(/usr/local/bin/sockaddr eval 'GetInterfaceIP "docker0"')
 
 CLOUD="$1"
-RETRY_JOIN="$2"
-NOMAD_SHA="$3"
-NOMAD_CONFIG="$4"
+NOMAD_SHA="$2"
+NOMAD_CONFIG="$3"
 
 # Consul
-sed -i "s/RETRY_JOIN/$RETRY_JOIN/g" $CONFIGDIR/consul_client.json
-sudo cp $CONFIGDIR/consul_client.json $CONSULCONFIGDIR/consul.json
-sudo cp $CONFIGDIR/consul_$CLOUD.service /etc/systemd/system/consul.service
+CONSUL_SRC=/ops/shared/consul
+CONSUL_DEST=/etc/consul.d
+
+sudo cp "$CONSUL_SRC/base.json" "$CONSUL_DEST/"
+sudo cp "$CONSUL_SRC/retry_$CLOUD.json" "$CONSUL_DEST/"
+sudo cp "$CONSUL_SRC/consul_$CLOUD.service" /etc/systemd/system/consul.service
 
 sudo systemctl enable consul.service
 sudo systemctl start  consul.service
@@ -54,6 +55,7 @@ echo "export PATH=$PATH:/usr/local/bin/spark/bin:/usr/local/$HADOOP_VERSION/bin"
 
 NOMAD_SRC=/ops/shared/nomad
 NOMAD_DEST=/etc/nomad.d
+NOMAD_CONFIG_FILENAME=$(basename "$NOMAD_CONFIG")
 
 # download
 aws s3 cp "s3://nomad-team-test-binary/builds-oss/nomad_linux_amd64_${NOMAD_SHA}.tar.gz" nomad.tar.gz
@@ -64,7 +66,7 @@ sudo chmod 0755 /usr/local/bin/nomad
 sudo chown root:root /usr/local/bin/nomad
 
 sudo cp "$NOMAD_SRC/base.hcl" "$NOMAD_DEST/"
-sudo cp "$NOMAD_SRC/$NOMAD_CONFIG" "$NOMAD_DEST/"
+sudo cp "$NOMAD_SRC/$NOMAD_CONFIG" "$NOMAD_DEST/$NOMAD_CONFIG_FILENAME"
 
 # Setup Host Volumes
 sudo mkdir /tmp/data

diff --git a/e2e/terraform/shared/config/provision-server.sh b/e2e/terraform/shared/config/provision-server.sh
@@ -5,7 +5,6 @@ set -o nounset
 
 CONFIGDIR=/ops/shared/config
 
-CONSULCONFIGDIR=/etc/consul.d
 HADOOP_VERSION=hadoop-2.7.7
 HADOOPCONFIGDIR=/usr/local/$HADOOP_VERSION/etc/hadoop
 HOME_DIR=ubuntu
@@ -15,15 +14,18 @@ DOCKER_BRIDGE_IP_ADDRESS=$(/usr/local/bin/sockaddr eval 'GetInterfaceIP "docker0
 
 CLOUD="$1"
 SERVER_COUNT="$2"
-RETRY_JOIN="$3"
-NOMAD_SHA="$4"
-NOMAD_CONFIG="$5"
+NOMAD_SHA="$3"
+NOMAD_CONFIG="$4"
 
 # Consul
-sed -i "s/SERVER_COUNT/$SERVER_COUNT/g" $CONFIGDIR/consul.json
-sed -i "s/RETRY_JOIN/$RETRY_JOIN/g" $CONFIGDIR/consul.json
-sudo cp $CONFIGDIR/consul.json $CONSULCONFIGDIR
-sudo cp $CONFIGDIR/consul_$CLOUD.service /etc/systemd/system/consul.service
+CONSUL_SRC=/ops/shared/consul
+CONSUL_DEST=/etc/consul.d
+
+sed "s/SERVER_COUNT/$SERVER_COUNT/g" "$CONSUL_SRC/server.json" > /tmp/server.json
+sudo mv /tmp/server.json "$CONSUL_DEST/server.json"
+sudo cp "$CONSUL_SRC/base.json" "$CONSUL_DEST/"
+sudo cp "$CONSUL_SRC/retry_$CLOUD.json" "$CONSUL_DEST/"
+sudo cp "$CONSUL_SRC/consul_$CLOUD.service" /etc/systemd/system/consul.service
 
 sudo systemctl enable consul.service
 sudo systemctl start  consul.service
@@ -72,6 +74,7 @@ echo "export PATH=$PATH:/usr/local/bin/spark/bin:/usr/local/$HADOOP_VERSION/bin"
 
 NOMAD_SRC=/ops/shared/nomad
 NOMAD_DEST=/etc/nomad.d
+NOMAD_CONFIG_FILENAME=$(basename "$NOMAD_CONFIG")
 
 # download
 aws s3 cp "s3://nomad-team-test-binary/builds-oss/nomad_linux_amd64_${NOMAD_SHA}.tar.gz" nomad.tar.gz
@@ -83,8 +86,9 @@ sudo chown root:root /usr/local/bin/nomad
 
 sudo cp "$NOMAD_SRC/base.hcl" "$NOMAD_DEST/"
 
-sed "s/3 # SERVER_COUNT/$SERVER_COUNT/g" "$NOMAD_SRC/$NOMAD_CONFIG" > "/tmp/$NOMAD_CONFIG"
-sudo mv "/tmp/$NOMAD_CONFIG" "$NOMAD_DEST/$NOMAD_CONFIG"
+sed "s/3 # SERVER_COUNT/$SERVER_COUNT/g" "$NOMAD_SRC/$NOMAD_CONFIG" \
+    > "/tmp/$NOMAD_CONFIG_FILENAME"
+sudo mv "/tmp/$NOMAD_CONFIG_FILENAME" "$NOMAD_DEST/$NOMAD_CONFIG_FILENAME"
 
 # enable as a systemd service
 sudo cp "$NOMAD_SRC/nomad.service" /etc/systemd/system/nomad.service

diff --git a/e2e/terraform/shared/consul/base.json b/e2e/terraform/shared/consul/base.json
@@ -0,0 +1,13 @@
+{
+  "log_level": "INFO",
+  "data_dir": "/opt/consul/data",
+  "bind_addr": "0.0.0.0",
+  "client_addr": "0.0.0.0",
+  "advertise_addr": "{{ GetPrivateIP }}",
+  "connect": {
+    "enabled": true
+  },
+  "ports": {
+    "grpc": 8502
+  }
+}
diff --git a/e2e/terraform/shared/config/consul.json → e2e/terraform/shared/consul/consul.json b/e2e/terraform/shared/config/consul.json → e2e/terraform/shared/consul/consul.json
diff --git a/...erraform/shared/config/consul_aws.service → ...erraform/shared/consul/consul_aws.service b/...erraform/shared/config/consul_aws.service → ...erraform/shared/consul/consul_aws.service
diff --git a/...raform/shared/config/consul_azure.service → ...raform/shared/consul/consul_azure.service b/...raform/shared/config/consul_azure.service → ...raform/shared/consul/consul_azure.service
diff --git a/...erraform/shared/config/consul_client.json → ...erraform/shared/consul/consul_client.json b/...erraform/shared/config/consul_client.json → ...erraform/shared/consul/consul_client.json
diff --git a/e2e/terraform/shared/consul/retry_aws.json b/e2e/terraform/shared/consul/retry_aws.json
@@ -0,0 +1,3 @@
+{
+  "retry_join": ["provider=aws tag_key=ConsulAutoJoin tag_value=auto-join"]
+}
diff --git a/e2e/terraform/shared/consul/retry_azure.json b/e2e/terraform/shared/consul/retry_azure.json
@@ -0,0 +1,3 @@
+{
+  "retry_join": ["provider=azure tag_key=ConsulAutoJoin tag_value=auto-join"]
+}
diff --git a/e2e/terraform/shared/consul/server.json b/e2e/terraform/shared/consul/server.json
@@ -0,0 +1,8 @@
+{
+  "server": true,
+  "ui": true,
+  "bootstrap_expect": SERVER_COUNT,
+  "service": {
+    "name": "consul"
+  }
+}
diff --git a/e2e/terraform/shared/config/nomad.service → e2e/terraform/shared/nomad/nomad.service b/e2e/terraform/shared/config/nomad.service → e2e/terraform/shared/nomad/nomad.service