Terratest e2e test (#93)

Introduce go-based E2E testing

Signed-off-by: Jeff McCoy <[email protected]>

.gitignore

@@ -16,3 +16,8 @@ data/
 bundle/
 charts/
 .idea/
+.tool-versions
+test/tf/public-ec2-instance/.test-data
+test/tf/public-ec2-instance/.terraform
+terraform.tfstate
+terraform.tfstate.backup

Makefile

@@ -14,35 +14,47 @@ ifneq ($(UNAME_S),Linux)
 	endif
 endif
 
-# remove all zarf packages recursively
-remove-packages:
+.DEFAULT_GOAL := help
+
+.PHONY: help
+help: ## Show a list of all targets
+	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) \
+	| sed -n 's/^\(.*\): \(.*\)##\(.*\)/\1:\3/p' \
+	| column -t -s ":"
+
+remove-packages: ## remove all zarf packages recursively
 	find . -type f -name 'zarf-package-*' -delete
 
-# usage: make test OS=ubuntu
-test:
+vm-init: ## usage -> make vm-init OS=ubuntu
 	vagrant destroy -f
 	vagrant up --no-color ${OS}
 	echo -e "\n\n\n\033[1;93m  ✅ BUILD COMPLETE.  To access this environment, run \"vagrant ssh ${OS}\"\n\n\n"
 
-test-close:
+vm-destroy: ## Destroy the VM
 	vagrant destroy -f
 
-init-package:
-	$(ZARF_BIN) package create --confirm
-	mv zarf-init.tar.zst build
+test-e2e: ## Run E2E tests. Requires access to an AWS account. Costs money. Make sure you ran the `build-cli` and `init-package` targets first
+	cd test/e2e && go test ./... -v -timeout 1200s
 
-	cd build && sha256sum -b zarf* > zarf.sha256
-	ls -lh build
+e2e-ssh: ## Run this if you set SKIP_teardown=1 and want to SSH into the still-running test server. Don't forget to unset SKIP_teardown when you're done
+	cd test/tf/public-ec2-instance/.test-data && cat Ec2KeyPair.json | jq -r .PrivateKey > privatekey.pem && chmod 600 privatekey.pem
+	cd test/tf/public-ec2-instance && ssh -i .test-data/privatekey.pem ubuntu@$$(terraform output public_instance_ip)
 
-build-cli:
+build-cli: ## Build the CLI
 	rm -fr build
 	cd cli && $(MAKE) build
 	cd cli && $(MAKE) build-mac
 
-build-test: build-cli init-package
+init-package: ## Create the zarf init package
+	$(ZARF_BIN) package create --confirm
+	mv zarf-init.tar.zst build
+
+	cd build && sha256sum -b zarf* > zarf.sha256
+	ls -lh build
+
+build-test: build-cli init-package ## Build the CLI and create the init package
 
-ci-release: init-package
+ci-release: init-package ## Create the init package
 
-# automatically package all example directories and add the tarballs to the build directory
-package-examples:
+package-examples: ## automatically package all example directories and add the tarballs to the examples/sync directory
 	cd examples && $(MAKE) package-examples

cli/Makefile

@@ -1,6 +1,6 @@
 # Tests have been purged for now due to large refactor deleted all tested code
 # deps:
-# 	go get gotest.tools/gotestsum 
+# 	go get gotest.tools/gotestsum
 # 	go mod download
 
 # test: deps

test/e2e/go.mod

@@ -0,0 +1,8 @@
+module github.com/defenseunicorns/zarf/test/e2e
+
+go 1.16
+
+require (
+	github.com/gruntwork-io/terratest v0.37.5
+	github.com/stretchr/testify v1.4.0
+)

test/e2e/terraform_ssh_e2e_test.go

@@ -0,0 +1,196 @@
+package test
+
+import (
+  "bufio"
+  "encoding/base64"
+  "fmt"
+  "github.com/gruntwork-io/terratest/modules/random"
+  "github.com/gruntwork-io/terratest/modules/retry"
+  "github.com/stretchr/testify/require"
+  "io/ioutil"
+  "os"
+  "testing"
+  "time"
+
+  "github.com/gruntwork-io/terratest/modules/aws"
+  "github.com/gruntwork-io/terratest/modules/ssh"
+  "github.com/gruntwork-io/terratest/modules/terraform"
+  teststructure "github.com/gruntwork-io/terratest/modules/test-structure"
+)
+
+func TestTerraformSshExample(t *testing.T) {
+  t.Parallel()
+
+  // Copy the terraform folder to a temp directory so we can run multiple tests in parallel
+  tmpFolder := teststructure.CopyTerraformFolderToTemp(t, "..", "tf/public-ec2-instance")
+
+  // At the end of the test, run `terraform destroy` to clean up any resources that were created
+  defer teststructure.RunTestStage(t, "TEARDOWN", func() {
+    keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder)
+    aws.DeleteEC2KeyPair(t, keyPair)
+
+    terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder)
+    terraform.Destroy(t, terraformOptions)
+  })
+
+  // Deploy the terraform infra
+  teststructure.RunTestStage(t, "SETUP", func() {
+    terraformOptions, keyPair, err := configureTerraformOptions(t, tmpFolder)
+    require.NoError(t, err)
+
+    // Save the options and key pair so later test stages can use them
+    teststructure.SaveTerraformOptions(t, tmpFolder, terraformOptions)
+    teststructure.SaveEc2KeyPair(t, tmpFolder, keyPair)
+
+    // This will run `terraform init` and `terraform apply` and fail the test if there are any errors
+    terraform.InitAndApply(t, terraformOptions)
+  })
+
+  // Upload the Zarf artifacts
+  teststructure.RunTestStage(t, "UPLOAD", func() {
+    terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder)
+    keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder)
+
+    // This will upload the Zarf binary, init package, and other necessary files to the server so we can use them for
+    // tests
+    syncFilesToRemoteServer(t, terraformOptions, keyPair)
+  })
+
+  // Make sure we can SSH to the public Instance directly from the public Internet
+  teststructure.RunTestStage(t, "TEST", func() {
+    terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder)
+    keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder)
+
+    // Finally run the actual test
+    test(t, terraformOptions, keyPair)
+  })
+}
+
+func configureTerraformOptions(t *testing.T, tmpFolder string) (*terraform.Options, *aws.Ec2Keypair, error) {
+  // A unique ID we can use to namespace resources so we don't clash with anything already in the AWS account or
+  // tests running in parallel
+  uniqueID := random.UniqueId()
+  namespace := "zarf"
+  stage := "terratest"
+  name := fmt.Sprintf("e2e-%s", uniqueID)
+
+  // Get the region to use from the system's environment
+  awsRegion, err := getAwsRegion()
+  if err != nil {
+    return nil, nil, err
+  }
+
+  // Some AWS regions are missing certain instance types, so pick an available type based on the region we picked
+  instanceType := aws.GetRecommendedInstanceType(t, awsRegion, []string{"t3a.large", "t3.large", "t2.large"})
+
+  // Create an EC2 KeyPair that we can use for SSH access
+  keyPairName := fmt.Sprintf("%s-%s-%s", namespace, stage, name)
+  keyPair := aws.CreateAndImportEC2KeyPair(t, awsRegion, keyPairName)
+
+  // Construct the terraform options with default retryable errors to handle the most common retryable errors in
+  // terraform testing.
+  terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{
+    // The path to where our Terraform code is located
+    TerraformDir: tmpFolder,
+
+    // Variables to pass to our Terraform code using -var options
+    Vars: map[string]interface{}{
+      "aws_region":    awsRegion,
+      "namespace":     namespace,
+      "stage":         stage,
+      "name":          name,
+      "instance_type": instanceType,
+      "key_pair_name": keyPairName,
+    },
+  })
+
+  return terraformOptions, keyPair, nil
+}
+
+func syncFilesToRemoteServer(t *testing.T, terraformOptions *terraform.Options, keyPair *aws.Ec2Keypair) {
+  // Run `terraform output` to get the value of an output variable
+  publicInstanceIP := terraform.Output(t, terraformOptions, "public_instance_ip")
+
+  // We're going to try to SSH to the instance IP, using the Key Pair we created earlier, and the user "ubuntu",
+  // as we know the Instance is running an Ubuntu AMI that has such a user
+  publicHost := ssh.Host{
+    Hostname:    publicInstanceIP,
+    SshKeyPair:  keyPair.KeyPair,
+    SshUserName: "ubuntu",
+  }
+
+  // It can take a minute or so for the Instance to boot up, so retry a few times
+  maxRetries := 15
+  timeBetweenRetries, err := time.ParseDuration("5s")
+  require.NoError(t, err)
+
+  // Wait for the instance to be ready
+  _, err = retry.DoWithRetryE(t, "Wait for the instance to be ready", maxRetries, timeBetweenRetries, func() (string, error){
+    _, err := ssh.CheckSshCommandE(t, publicHost, "whoami")
+    if err != nil {
+      return "", err
+    }
+    return "", nil
+  })
+  require.NoError(t, err)
+
+  // Upload the compiled Zarf binary to the server. The ssh lib only supports sending strings so we'll base64encode it
+  // first
+  f, err := os.Open("../../build/zarf")
+  require.NoError(t, err)
+  reader := bufio.NewReader(f)
+  content, err := ioutil.ReadAll(reader)
+  require.NoError(t, err)
+  encodedZarfBinary := base64.StdEncoding.EncodeToString(content)
+  err = ssh.ScpFileToE(t, publicHost, 0644, "$HOME/zarf.b64", encodedZarfBinary)
+  require.NoError(t, err)
+  output, err := ssh.CheckSshCommandE(t, publicHost, "cd $HOME && sudo bash -c 'base64 -d zarf.b64 > /usr/local/bin/zarf && chmod 0777 /usr/local/bin/zarf'")
+  require.NoError(t, err, output)
+
+  // Upload zarf-init.tar.zst
+  f, err = os.Open("../../build/zarf-init.tar.zst")
+  require.NoError(t, err)
+  reader = bufio.NewReader(f)
+  content, err = ioutil.ReadAll(reader)
+  require.NoError(t, err)
+  encodedZarfInit := base64.StdEncoding.EncodeToString(content)
+  err = ssh.ScpFileToE(t, publicHost, 0644, "$HOME/zarf-init.tar.zst.b64", encodedZarfInit)
+  require.NoError(t, err)
+  output, err = ssh.CheckSshCommandE(t, publicHost, "cd $HOME && base64 -d zarf-init.tar.zst.b64 > zarf-init.tar.zst")
+  require.NoError(t, err, output)
+}
+
+func test(t *testing.T, terraformOptions *terraform.Options, keyPair *aws.Ec2Keypair) {
+  // Run `terraform output` to get the value of an output variable
+  publicInstanceIP := terraform.Output(t, terraformOptions, "public_instance_ip")
+
+  // We're going to try to SSH to the instance IP, using the Key Pair we created earlier, and the user "ubuntu",
+  // as we know the Instance is running an Ubuntu AMI that has such a user
+  publicHost := ssh.Host{
+    Hostname:    publicInstanceIP,
+    SshKeyPair:  keyPair.KeyPair,
+    SshUserName: "ubuntu",
+  }
+
+  // Make sure `zarf --help` doesn't error
+  output, err := ssh.CheckSshCommandE(t, publicHost, "zarf --help")
+  require.NoError(t, err, output)
+
+  // Test `zarf init just to make sure it returns a zero exit code.`
+  output, err = ssh.CheckSshCommandE(t, publicHost, "sudo bash -c 'zarf init --confirm --components management,logging,gitops-service --host localhost'")
+  require.NoError(t, err, output)
+}
+
+// getAwsRegion returns the desired AWS region to use by first checking the env var AWS_REGION, then checking
+//AWS_DEFAULT_REGION if AWS_REGION isn't set. If neither is set it returns an error
+func getAwsRegion() (string, error) {
+  val, present := os.LookupEnv("AWS_REGION")
+  if !present {
+    val, present = os.LookupEnv("AWS_DEFAULT_REGION")
+  }
+  if !present {
+    return "", fmt.Errorf("expected either AWS_REGION or AWS_DEFAULT_REGION env var to be set, but they were not")
+  } else {
+    return val, nil
+  }
+}

test/tf/public-ec2-instance/main.tf

@@ -0,0 +1,83 @@
+terraform {
+  required_version = "0.13.7"
+}
+
+locals {
+  fullname = "${var.namespace}-${var.stage}-${var.name}"
+}
+
+provider "aws" {
+  region = var.aws_region
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# CREATE A PUBLIC EC2 INSTANCE
+# ---------------------------------------------------------------------------------------------------------------------
+
+resource "aws_instance" "public" {
+  ami                    = data.aws_ami.ubuntu.id
+  instance_type          = var.instance_type
+  vpc_security_group_ids = [aws_security_group.public.id]
+  key_name               = var.key_pair_name
+
+  # This EC2 Instance has a public IP and will be accessible directly from the public Internet
+  associate_public_ip_address = true
+
+  tags = {
+    Name = "${local.fullname}-public"
+  }
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# CREATE A SECURITY GROUP TO CONTROL WHAT REQUESTS CAN GO IN AND OUT OF THE EC2 INSTANCES
+# ---------------------------------------------------------------------------------------------------------------------
+
+resource "aws_security_group" "public" {
+  name = local.fullname
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port = 22
+    to_port   = 22
+    protocol  = "tcp"
+
+    # To keep this example simple, we allow incoming SSH requests from any IP. In real-world usage, you should only
+    # allow SSH requests from trusted servers, such as a bastion host or VPN server.
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# LOOK UP THE LATEST UBUNTU AMI
+# ---------------------------------------------------------------------------------------------------------------------
+
+data "aws_ami" "ubuntu" {
+  most_recent = true
+  owners      = ["099720109477"] # Canonical
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "architecture"
+    values = ["x86_64"]
+  }
+
+  filter {
+    name   = "image-type"
+    values = ["machine"]
+  }
+
+  filter {
+    name   = "name"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"]
+  }
+}

test/tf/public-ec2-instance/outputs.tf

@@ -0,0 +1,7 @@
+output "public_instance_id" {
+  value = aws_instance.public.id
+}
+
+output "public_instance_ip" {
+  value = aws_instance.public.public_ip
+}