From b4756a2337cf16c96e5b6aa7c3581160e216ae87 Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Mon, 26 Jul 2021 17:02:19 -0700 Subject: [PATCH 01/22] Support for packer init --- .circleci/config.yml | 2 +- examples/packer-basic-example/README.md | 16 ++- examples/packer-basic-example/build.json | 61 --------- examples/packer-basic-example/build.pkr.hcl | 122 ++++++++++++++++++ examples/packer-docker-example/README.md | 18 ++- examples/packer-docker-example/build.json | 55 -------- examples/packer-docker-example/build.pkr.hcl | 68 ++++++++++ examples/packer-hello-world-example/README.md | 18 ++- .../packer-hello-world-example/build.json | 18 --- .../packer-hello-world-example/build.pkr.hcl | 18 +++ go.mod | 3 +- go.sum | 2 + modules/packer/packer.go | 55 ++++++++ test/packer_basic_example_test.go | 8 +- test/packer_hello_world_example_test.go | 2 +- 15 files changed, 308 insertions(+), 158 deletions(-) delete mode 100644 examples/packer-basic-example/build.json create mode 100644 examples/packer-basic-example/build.pkr.hcl delete mode 100644 examples/packer-docker-example/build.json create mode 100644 examples/packer-docker-example/build.pkr.hcl delete mode 100644 examples/packer-hello-world-example/build.json create mode 100644 examples/packer-hello-world-example/build.pkr.hcl diff --git a/.circleci/config.yml b/.circleci/config.yml index b4ac6e2b3..1cf3eccc2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ env: &env MODULE_GCP_CI_VERSION: v0.1.1 TERRAFORM_VERSION: 1.0.3 TERRAGRUNT_VERSION: v0.28.24 - PACKER_VERSION: 1.6.6 + PACKER_VERSION: 1.7.4 GO_VERSION: 1.16.3 GO111MODULE: auto K8S_VERSION: v1.15.0 # Same as EKS diff --git a/examples/packer-basic-example/README.md b/examples/packer-basic-example/README.md index 2f3508206..ca1ac657f 100644 --- a/examples/packer-basic-example/README.md +++ b/examples/packer-basic-example/README.md @@ -2,7 +2,8 @@ This folder contains a very simple Packer template to demonstrate how you can use Terratest to write automated tests for your Packer templates. The template just creates an up-to-date Ubuntu AMI by running `apt-get update` and -`apt-get upgrade`. +`apt-get upgrade`. It also uses the [packer-provisioner-comment plugin](https://github.com/sylviamoss/packer-plugin-comment) to +demonstrate how plugins work (this is primarily for testing purposes). Check out [test/packer_basic_example_test.go](/test/packer_basic_example_test.go) to see how you can write automated tests for this simple template. @@ -14,16 +15,19 @@ Terratest. For slightly more complicated, real-world examples of Packer template - -## Building the Packer template manually - +## Installation steps 1. Sign up for [AWS](https://aws.amazon.com/). 1. Configure your AWS credentials using one of the [supported methods for AWS CLI tools](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html), such as setting the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables. 1. Install [Packer](https://www.packer.io/) and make sure it's on your `PATH`. -1. Run `packer build build.json`. +## Building the Packer template manually (Packer >= 1.7.0) +1. Run `packer init build.pkr.hcl`. +1. Run `packer build build.pkr.hcl`. + +## Building the Packer template manually (Packer < 1.7.0) +1. Run `packer build build.json`. @@ -61,7 +65,7 @@ Terratest. For slightly more complicated, real-world examples of Packer template 1. Sign up for [OCI](https://cloud.oracle.com/cloud-infrastructure). 1. Configure your OCI credentials via [CLI Configuration Information](https://docs.cloud.oracle.com/iaas/Content/API/Concepts/sdkconfig.htm). -1. Create [VCN](https://docs.cloud.oracle.com/iaas/Content/GSG/Tasks/creatingnetwork.htm) and subnet +1. Create [VCN](https://docs.cloud.oracle.com/iaas/Content/GSG/Tasks/creatingnetwork.htm) and subnet resources in your tenancy (a.k.a. a root compartment). 1. (Optional) Create `TF_VAR_pass_phrase` environment property with the pass phrase for decrypting of the OCI [API signing key](https://docs.cloud.oracle.com/iaas/Content/API/Concepts/apisigningkey.htm) (can be omitted diff --git a/examples/packer-basic-example/build.json b/examples/packer-basic-example/build.json deleted file mode 100644 index 97c5bc8c6..000000000 --- a/examples/packer-basic-example/build.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "min_packer_version": "1.0.4", - "variables": { - "aws_region": "us-east-1", - "ami_base_name": "", - "gcp_project_id": "", - "gcp_zone": "us-central1-a", - "oci_availability_domain": "", - "oci_base_image_ocid": "", - "oci_compartment_ocid": "", - "oci_pass_phrase": "", - "oci_subnet_ocid": "", - "instance_type": "t2.micro" - }, - "builders": [{ - "type": "amazon-ebs", - "ami_name": "{{user `ami_base_name`}}-terratest-packer-example", - "ami_description": "An example of how to create a custom AMI on top of Ubuntu", - "instance_type": "{{user `instance_type`}}", - "region": "{{user `aws_region`}}", - "source_ami_filter": { - "filters": { - "virtualization-type": "hvm", - "architecture": "x86_64", - "name": "*ubuntu-xenial-16.04-amd64-server-*", - "block-device-mapping.volume-type": "gp2", - "root-device-type": "ebs" - }, - "owners": ["099720109477"], - "most_recent": true - }, - "ssh_username": "ubuntu", - "encrypt_boot": false - }, { - "type": "googlecompute", - "image_name": "terratest-packer-example-{{isotime | clean_resource_name}}", - "image_family": "terratest", - "project_id": "{{user `gcp_project_id`}}", - "source_image_family": "ubuntu-1804-lts", - "zone": "{{user `gcp_zone`}}", - "ssh_username": "ubuntu" - }, { - "type": "oracle-oci", - "image_name": "terratest-packer-example-{{isotime}}", - "availability_domain": "{{user `oci_availability_domain`}}", - "base_image_ocid": "{{user `oci_base_image_ocid`}}", - "compartment_ocid": "{{user `oci_compartment_ocid`}}", - "pass_phrase": "{{user `oci_pass_phrase`}}", - "shape": "VM.Standard2.1", - "ssh_username": "ubuntu", - "subnet_ocid": "{{user `oci_subnet_ocid`}}" - }], - "provisioners": [{ - "type": "shell", - "inline": [ - "sudo DEBIAN_FRONTEND=noninteractive apt-get update", - "sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y" - ], - "pause_before": "30s" - }] -} diff --git a/examples/packer-basic-example/build.pkr.hcl b/examples/packer-basic-example/build.pkr.hcl new file mode 100644 index 000000000..eed84f8b2 --- /dev/null +++ b/examples/packer-basic-example/build.pkr.hcl @@ -0,0 +1,122 @@ +packer { + required_version = ">= 1.7.0" + required_plugins { + # This plugin is primarily used for testing the new required_plugins feature introduced in Packer 1.7.0. + comment = { + version = ">=v0.2.23" + source = "github.com/sylviamoss/comment" + } + } +} + +variable "ami_base_name" { + type = string + default = "" +} + +variable "aws_region" { + type = string + default = "us-east-1" +} + +variable "gcp_project_id" { + type = string + default = "" +} + +variable "gcp_zone" { + type = string + default = "us-central1-a" +} + +variable "instance_type" { + type = string + default = "t2.micro" +} + +variable "oci_availability_domain" { + type = string + default = "" +} + +variable "oci_base_image_ocid" { + type = string + default = "" +} + +variable "oci_compartment_ocid" { + type = string + default = "" +} + +variable "oci_pass_phrase" { + type = string + default = "" +} + +variable "oci_subnet_ocid" { + type = string + default = "" +} + +data "amazon-ami" "ubuntu-xenial" { + filters = { + architecture = "x86_64" + "block-device-mapping.volume-type" = "gp2" + name = "*ubuntu-xenial-16.04-amd64-server-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["099720109477"] + region = var.aws_region +} + +source "amazon-ebs" "aws" { + ami_description = "An example of how to create a custom AMI on top of Ubuntu" + ami_name = "${var.ami_base_name}-terratest-packer-example" + encrypt_boot = false + instance_type = var.instance_type + region = var.aws_region + source_ami = data.amazon-ami.ubuntu-xenial.id + ssh_username = "ubuntu" +} + +source "googlecompute" "gcp" { + image_family = "terratest" + image_name = "terratest-packer-example-${formatdate("YYYYMMDD-hhmm", timestamp())}" + project_id = var.gcp_project_id + source_image_family = "ubuntu-1804-lts" + ssh_username = "ubuntu" + zone = var.gcp_zone +} + +source "oracle-oci" "oracle" { + availability_domain = var.oci_availability_domain + base_image_ocid = var.oci_base_image_ocid + compartment_ocid = var.oci_compartment_ocid + image_name = "terratest-packer-example-${formatdate("YYYYMMDD-hhmm", timestamp())}" + pass_phrase = var.oci_pass_phrase + shape = "VM.Standard2.1" + ssh_username = "ubuntu" + subnet_ocid = var.oci_subnet_ocid +} + +build { + sources = [ + "source.amazon-ebs.aws", + "source.googlecompute.gcp", + "source.oracle-oci.oracle" + ] + + provisioner "comment" { + comment = "Basic comment example" + ui = false + } + + provisioner "shell" { + inline = ["sudo DEBIAN_FRONTEND=noninteractive apt-get update", "sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y"] + pause_before = "30s" + } + +} diff --git a/examples/packer-docker-example/README.md b/examples/packer-docker-example/README.md index 2610c2b62..4bc95d0e0 100644 --- a/examples/packer-docker-example/README.md +++ b/examples/packer-docker-example/README.md @@ -14,13 +14,21 @@ The Docker-based tests in this folder are in some sense "unit tests" for the Pac [terraform-packer-example](/examples/terraform-packer-example). +## Installation steps +1. Install [Packer](https://www.packer.io/) and make sure it's on your `PATH`. +1. Install [Docker](https://www.docker.com/) and make sure it's on your `PATH`. -## Building a Docker image for local testing +## Building a Docker image for local testing (Packer >= 1.7.0) +1. Run `packer init build.pkr.hcl`. +1. Run `packer build build.pkr.hcl`. -1. Install [Packer](https://www.packer.io/) and make sure it's on your `PATH`. -1. Install [Docker](https://www.docker.com/) and make sure it's on your `PATH`. -1. Run `packer build -only=ubuntu-docker build.json`. + +## Building a Docker image for local testing (Packer < 1.7.0) +1. Run `packer build build.json`. + + +## Run the container 1. Run `docker-compose up`. 1. You should now be able to access the sample web app at http://localhost:8080 @@ -53,4 +61,4 @@ The Docker-based tests in this folder are in some sense "unit tests" for the Pac ## Running automated tests in AWS against this Packer template -See [terraform-packer-example](/examples/terraform-packer-example). \ No newline at end of file +See [terraform-packer-example](/examples/terraform-packer-example). diff --git a/examples/packer-docker-example/build.json b/examples/packer-docker-example/build.json deleted file mode 100644 index fde5ab0c3..000000000 --- a/examples/packer-docker-example/build.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "variables": { - "aws_region": "us-east-1", - "ami_name_base": "terratest-packer-docker-example", - "instance_type": "t2.micro" - }, - "builders": [{ - "name": "ubuntu-ami", - "ami_name": "{{user `ami_name_base`}}-{{isotime | clean_resource_name}}", - "ami_description": "An example of how to create a custom AMI with a simple web app on top of Ubuntu", - "instance_type": "{{user `instance_type`}}", - "region": "{{user `aws_region`}}", - "type": "amazon-ebs", - "source_ami_filter": { - "filters": { - "virtualization-type": "hvm", - "architecture": "x86_64", - "name": "*ubuntu-xenial-16.04-amd64-server-*", - "block-device-mapping.volume-type": "gp2", - "root-device-type": "ebs" - }, - "owners": ["099720109477"], - "most_recent": true - }, - "ssh_username": "ubuntu", - "encrypt_boot": false - },{ - "name": "ubuntu-docker", - "type": "docker", - "image": "gruntwork/ubuntu-test:16.04", - "commit": true, - "changes": ["ENTRYPOINT [\"\"]"] - }], - "provisioners": [{ - "type": "shell", - "inline": [ - "echo 'Sleeping for a few seconds to give Ubuntu time to boot up'", - "sleep 30" - ], - "only": ["ubuntu-ami"] - },{ - "type": "file", - "source": "{{template_dir}}", - "destination": "/tmp/packer-docker-example" - },{ - "type": "shell", - "inline": ["/tmp/packer-docker-example/configure-sinatra-app.sh"] - }], - "post-processors": [{ - "type": "docker-tag", - "repository": "gruntwork/packer-docker-example", - "tag": "latest", - "only": ["ubuntu-docker"] - }] -} diff --git a/examples/packer-docker-example/build.pkr.hcl b/examples/packer-docker-example/build.pkr.hcl new file mode 100644 index 000000000..303eaad4b --- /dev/null +++ b/examples/packer-docker-example/build.pkr.hcl @@ -0,0 +1,68 @@ + +variable "ami_name_base" { + type = string + default = "terratest-packer-docker-example" +} + +variable "aws_region" { + type = string + default = "us-east-1" +} + +variable "instance_type" { + type = string + default = "t2.micro" +} + +data "amazon-ami" "aws" { + filters = { + architecture = "x86_64" + "block-device-mapping.volume-type" = "gp2" + name = "*ubuntu-xenial-16.04-amd64-server-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["099720109477"] + region = var.aws_region +} + +source "amazon-ebs" "ubuntu-ami" { + ami_description = "An example of how to create a custom AMI with a simple web app on top of Ubuntu" + ami_name = "${var.ami_name_base}-${formatdate("YYYYMMDD-hhmm", timestamp())}" + encrypt_boot = false + instance_type = var.instance_type + region = var.aws_region + source_ami = data.amazon-ami.aws.id + ssh_username = "ubuntu" +} + +source "docker" "ubuntu-docker" { + changes = ["ENTRYPOINT [\"\"]"] + commit = true + image = "gruntwork/ubuntu-test:16.04" +} + +build { + sources = ["source.amazon-ebs.ubuntu-ami", "source.docker.ubuntu-docker"] + + provisioner "shell" { + inline = ["echo 'Sleeping for a few seconds to give Ubuntu time to boot up'", "sleep 30"] + only = ["ubuntu-ami"] + } + + provisioner "file" { + destination = "/tmp/packer-docker-example" + source = path.root + } + + provisioner "shell" { + inline = ["/tmp/packer-docker-example/configure-sinatra-app.sh"] + } + + post-processor "docker-tag" { + only = ["ubuntu-docker"] + repository = "gruntwork/packer-docker-example" + tag = "latest" + } +} diff --git a/examples/packer-hello-world-example/README.md b/examples/packer-hello-world-example/README.md index be4afc2ff..313f6212e 100644 --- a/examples/packer-hello-world-example/README.md +++ b/examples/packer-hello-world-example/README.md @@ -1,23 +1,29 @@ # Packer "Hello, World" Example This folder contains the simplest possible Packer template—one that builds a Docker image with a text file that says -"Hello, World"!—to demonstrate how you can use Terratest to write automated tests for your Packer templates. +"Hello, World"!—to demonstrate how you can use Terratest to write automated tests for your Packer templates. Check out [test/packer_hello_world_example_test.go](/test/packer_hello_world_example_test.go) to see how you can write automated tests for this simple template. +## Installation steps +1. Install [Packer](https://www.packer.io/) and make sure it's on your `PATH`. +1. Install [Docker](https://www.docker.com/) and make sure it's on your `PATH`. -## Building the Packer template +## Building the Packer template (Packer >= 1.7.0) +1. Run `packer init build.pkr.hcl`. +1. Run `packer build build.pkr.hcl`. -1. Install [Packer](https://www.packer.io/) and make sure it's on your `PATH`. -1. Install [Docker](https://www.docker.com/) and make sure it's on your `PATH`. + +## Building the Packer template (Packer < 1.7.0) 1. Run `packer build build.json`. -1. Run `docker run -it --rm gruntwork/packer-hello-world-example cat /test.txt`. -1. You should see the text "Hello, World!" +## Run Docker +1. Run `docker run -it --rm gruntwork/packer-hello-world-example cat /test.txt`. +1. You should see the text "Hello, World!" ## Running automated tests against the Packer template diff --git a/examples/packer-hello-world-example/build.json b/examples/packer-hello-world-example/build.json deleted file mode 100644 index 4947b931a..000000000 --- a/examples/packer-hello-world-example/build.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "builders": [{ - "name": "ubuntu-docker", - "type": "docker", - "image": "gruntwork/ubuntu-test:16.04", - "commit": true, - "changes": ["ENTRYPOINT [\"\"]"] - }], - "provisioners": [{ - "type": "shell", - "inline": ["echo 'Hello, World!' > /test.txt"] - }], - "post-processors": [{ - "type": "docker-tag", - "repository": "gruntwork/packer-hello-world-example", - "tag": "latest" - }] -} diff --git a/examples/packer-hello-world-example/build.pkr.hcl b/examples/packer-hello-world-example/build.pkr.hcl new file mode 100644 index 000000000..301e0902f --- /dev/null +++ b/examples/packer-hello-world-example/build.pkr.hcl @@ -0,0 +1,18 @@ +source "docker" "ubuntu-docker" { + changes = ["ENTRYPOINT [\"\"]"] + commit = true + image = "gruntwork/ubuntu-test:16.04" +} + +build { + sources = ["source.docker.ubuntu-docker"] + + provisioner "shell" { + inline = ["echo 'Hello, World!' > /test.txt"] + } + + post-processor "docker-tag" { + repository = "gruntwork/packer-hello-world-example" + tag = "latest" + } +} diff --git a/go.mod b/go.mod index 731224d57..4a9052483 100644 --- a/go.mod +++ b/go.mod @@ -22,13 +22,14 @@ require ( github.com/google/uuid v1.1.1 github.com/gruntwork-io/go-commons v0.8.0 github.com/hashicorp/go-multierror v1.1.0 + github.com/hashicorp/go-version v1.3.0 github.com/hashicorp/hcl/v2 v2.8.2 github.com/hashicorp/terraform-json v0.12.0 github.com/imdario/mergo v0.3.7 // indirect github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/jstemmer/go-junit-report v0.9.1 github.com/magiconair/properties v1.8.0 - github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect + github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 github.com/miekg/dns v1.1.31 github.com/mitchellh/go-homedir v1.1.0 github.com/oracle/oci-go-sdk v7.1.0+incompatible diff --git a/go.sum b/go.sum index 43ca0c598..f7982af01 100644 --- a/go.sum +++ b/go.sum @@ -258,6 +258,8 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw= +github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= diff --git a/modules/packer/packer.go b/modules/packer/packer.go index 879878d8d..070fca15c 100644 --- a/modules/packer/packer.go +++ b/modules/packer/packer.go @@ -14,6 +14,7 @@ import ( "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/shell" "github.com/gruntwork-io/terratest/modules/testing" + "github.com/hashicorp/go-version" ) // Options are the options for Packer. @@ -92,6 +93,11 @@ func BuildArtifact(t testing.TestingT, options *Options) string { func BuildArtifactE(t testing.TestingT, options *Options) (string, error) { options.Logger.Logf(t, "Running Packer to generate a custom artifact for template %s", options.Template) + err := packerInit(t, options) + if err != nil { + return "", err + } + cmd := shell.Command{ Command: "packer", Args: formatPackerArgs(options), @@ -145,6 +151,55 @@ func extractArtifactID(packerLogOutput string) (string, error) { return "", errors.New("Could not find Artifact ID pattern in Packer output") } +// packerInit checks if init is supported in the current version of Packer, then runs packer init to download any required plugins. +func packerInit(t testing.TestingT, options *Options) error { + // The init command was introduced in Packer 1.7.0 + const packerInitVersion = "1.7.0" + minInitVersion, err := version.NewVersion(packerInitVersion) + if err != nil { + return err + } + cmd := shell.Command{ + Command: "packer", + Args: []string{"-version"}, + WorkingDir: options.WorkingDir, + } + description := fmt.Sprintf("%s %v", cmd.Command, cmd.Args) + installedVersion, err := retry.DoWithRetryableErrorsE(t, description, options.RetryableErrors, options.MaxRetries, options.TimeBetweenRetries, func() (string, error) { + return shell.RunCommandAndGetOutputE(t, cmd) + }) + if err != nil { + return err + } + thisVersion, err := version.NewVersion(installedVersion) + if err != nil { + return err + } + if thisVersion.LessThan(minInitVersion) { + options.Logger.Logf(t, "Skipping 'packer init' because it is not present in this version (%s)", thisVersion) + return nil + } + + options.Logger.Logf(t, "Running Packer init") + + cmd = shell.Command{ + Command: "packer", + Args: []string{"init", options.Template}, + WorkingDir: options.WorkingDir, + } + + description = fmt.Sprintf("%s %v", cmd.Command, cmd.Args) + _, err = retry.DoWithRetryableErrorsE(t, description, options.RetryableErrors, options.MaxRetries, options.TimeBetweenRetries, func() (string, error) { + return shell.RunCommandAndGetOutputE(t, cmd) + }) + + if err != nil { + return err + } + + return nil +} + // Convert the inputs to a format palatable to packer. The build command should have the format: // // packer build [OPTIONS] template diff --git a/test/packer_basic_example_test.go b/test/packer_basic_example_test.go index 418177e7a..1982be80c 100644 --- a/test/packer_basic_example_test.go +++ b/test/packer_basic_example_test.go @@ -40,7 +40,7 @@ func TestPackerBasicExample(t *testing.T) { // website::tag::1::Read Packer's template and set AWS Region variable. packerOptions := &packer.Options{ // The path to where the Packer template is located - Template: "../examples/packer-basic-example/build.json", + Template: "../examples/packer-basic-example/build.pkr.hcl", // Variables to pass to our Packer build using -var options Vars: map[string]string{ @@ -50,7 +50,7 @@ func TestPackerBasicExample(t *testing.T) { }, // Only build the AWS AMI - Only: "amazon-ebs", + Only: "amazon-ebs.aws", // Configure retries for intermittent errors RetryableErrors: DefaultRetryablePackerErrors, @@ -116,7 +116,7 @@ func TestPackerBasicExampleWithVarFile(t *testing.T) { }, // Only build the AWS AMI - Only: "amazon-ebs", + Only: "amazon-ebs.aws", // Configure retries for intermittent errors RetryableErrors: DefaultRetryablePackerErrors, @@ -171,7 +171,7 @@ func TestPackerMultipleConcurrentAmis(t *testing.T) { }, // Only build the AWS AMI - Only: "amazon-ebs", + Only: "amazon-ebs.aws", // Configure retries for intermittent errors RetryableErrors: DefaultRetryablePackerErrors, diff --git a/test/packer_hello_world_example_test.go b/test/packer_hello_world_example_test.go index 56aacafc7..5607caef8 100644 --- a/test/packer_hello_world_example_test.go +++ b/test/packer_hello_world_example_test.go @@ -11,7 +11,7 @@ import ( func TestPackerHelloWorldExample(t *testing.T) { packerOptions := &packer.Options{ // website::tag::1:: The path to where the Packer template is located - Template: "../examples/packer-hello-world-example/build.json", + Template: "../examples/packer-hello-world-example/build.json.pkr.hcl", } // website::tag::2:: Build the Packer template. This template will create a Docker image. From 583608d39214a3e6c59c18f8c6101cee14111944 Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Mon, 26 Jul 2021 17:18:40 -0700 Subject: [PATCH 02/22] Fix missing build.json refs --- examples/packer-docker-example/docker-compose.yml | 2 +- examples/terraform-packer-example/user-data/user-data.sh | 4 ++-- examples/terraform-packer-example/variables.tf | 2 +- test/gcp/packer_gcp_basic_example_test.go | 2 +- test/packer_basic_example_test.go | 4 ++-- test/packer_docker_example_test.go | 2 +- test/packer_hello_world_example_test.go | 2 +- test/packer_oci_example_test.go | 2 +- test/terraform_packer_example_test.go | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/packer-docker-example/docker-compose.yml b/examples/packer-docker-example/docker-compose.yml index 4d2244027..5ea8e5f83 100644 --- a/examples/packer-docker-example/docker-compose.yml +++ b/examples/packer-docker-example/docker-compose.yml @@ -4,7 +4,7 @@ version: '3' services: web_app: - # The name we use for the Docker image in build.json + # The name we use for the Docker image in build.json (or build.pkr.hcl) image: gruntwork/packer-docker-example # Run the sample web app on port 8080 diff --git a/examples/terraform-packer-example/user-data/user-data.sh b/examples/terraform-packer-example/user-data/user-data.sh index da1699ec1..e197deacc 100644 --- a/examples/terraform-packer-example/user-data/user-data.sh +++ b/examples/terraform-packer-example/user-data/user-data.sh @@ -1,7 +1,7 @@ #!/bin/bash # This script is meant to be run in the User Data of an EC2 Instance while it's booting. It starts a Ruby web app. -# This script assumes it is running in an AMI built from the Packer template -# in examples/packer-docker-example/build.json. +# This script assumes it is running in an AMI built from the Packer templates in examples/packer-docker-example +# (either build.json or build.pkr.hcl). set -e diff --git a/examples/terraform-packer-example/variables.tf b/examples/terraform-packer-example/variables.tf index cb1f3b96e..ee7133857 100644 --- a/examples/terraform-packer-example/variables.tf +++ b/examples/terraform-packer-example/variables.tf @@ -12,7 +12,7 @@ # --------------------------------------------------------------------------------------------------------------------- variable "ami_id" { - description = "The ID of the AMI to run on each EC2 Instance. Should be an AMI built from the Packer template in examples/packer-docker-example/build.json." + description = "The ID of the AMI to run on each EC2 Instance. Should be an AMI built from the Packer templates in examples/packer-docker-example (build.json or build.pkr.hcl)." type = string } diff --git a/test/gcp/packer_gcp_basic_example_test.go b/test/gcp/packer_gcp_basic_example_test.go index 99c6729a1..8ed51803f 100644 --- a/test/gcp/packer_gcp_basic_example_test.go +++ b/test/gcp/packer_gcp_basic_example_test.go @@ -36,7 +36,7 @@ func TestPackerGCPBasicExample(t *testing.T) { packerOptions := &packer.Options{ // The path to where the Packer template is located - Template: "../../examples/packer-basic-example/build.json", + Template: "../../examples/packer-basic-example/build.pkr.hcl", // Variables to pass to our Packer build using -var options Vars: map[string]string{ diff --git a/test/packer_basic_example_test.go b/test/packer_basic_example_test.go index 1982be80c..4ac56815e 100644 --- a/test/packer_basic_example_test.go +++ b/test/packer_basic_example_test.go @@ -108,7 +108,7 @@ func TestPackerBasicExampleWithVarFile(t *testing.T) { packerOptions := &packer.Options{ // The path to where the Packer template is located - Template: "../examples/packer-basic-example/build.json", + Template: "../examples/packer-basic-example/build.pkr.hcl", // Variable file to to pass to our Packer build using -var-file option VarFiles: []string{ @@ -161,7 +161,7 @@ func TestPackerMultipleConcurrentAmis(t *testing.T) { packerOptions := &packer.Options{ // The path to where the Packer template is located - Template: "../examples/packer-basic-example/build.json", + Template: "../examples/packer-basic-example/build.pkr.hcl", // Variables to pass to our Packer build using -var options Vars: map[string]string{ diff --git a/test/packer_docker_example_test.go b/test/packer_docker_example_test.go index 34a547ab3..7f0d3711a 100644 --- a/test/packer_docker_example_test.go +++ b/test/packer_docker_example_test.go @@ -21,7 +21,7 @@ func TestPackerDockerExampleLocal(t *testing.T) { // website::tag::1::Configure Packer to build Docker image. packerOptions := &packer.Options{ // The path to where the Packer template is located - Template: "../examples/packer-docker-example/build.json", + Template: "../examples/packer-docker-example/build.pkr.hcl", // Only build the Docker image for local testing Only: "ubuntu-docker", diff --git a/test/packer_hello_world_example_test.go b/test/packer_hello_world_example_test.go index 5607caef8..515050861 100644 --- a/test/packer_hello_world_example_test.go +++ b/test/packer_hello_world_example_test.go @@ -11,7 +11,7 @@ import ( func TestPackerHelloWorldExample(t *testing.T) { packerOptions := &packer.Options{ // website::tag::1:: The path to where the Packer template is located - Template: "../examples/packer-hello-world-example/build.json.pkr.hcl", + Template: "../examples/packer-hello-world-example/build.pkr.hcl", } // website::tag::2:: Build the Packer template. This template will create a Docker image. diff --git a/test/packer_oci_example_test.go b/test/packer_oci_example_test.go index 46ab46c1b..801dfcb8a 100644 --- a/test/packer_oci_example_test.go +++ b/test/packer_oci_example_test.go @@ -26,7 +26,7 @@ func TestPackerOciExample(t *testing.T) { packerOptions := &packer.Options{ // The path to where the Packer template is located - Template: "../examples/packer-basic-example/build.json", + Template: "../examples/packer-basic-example/build.pkr.hcl", // Variables to pass to our Packer build using -var options Vars: map[string]string{ diff --git a/test/terraform_packer_example_test.go b/test/terraform_packer_example_test.go index ec82247db..15bc946d8 100644 --- a/test/terraform_packer_example_test.go +++ b/test/terraform_packer_example_test.go @@ -71,7 +71,7 @@ func buildAMI(t *testing.T, awsRegion string, workingDir string) { packerOptions := &packer.Options{ // The path to where the Packer template is located - Template: "../examples/packer-docker-example/build.json", + Template: "../examples/packer-docker-example/build.pkr.hcl", // Only build the AMI Only: "ubuntu-ami", From 9b4928d1a0c07e27431888edb008c13de47c8d6a Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Mon, 26 Jul 2021 17:53:02 -0700 Subject: [PATCH 03/22] Break out GCP to its own template --- examples/packer-basic-example/README.md | 7 ++--- .../packer-basic-example/build-gcp.pkr.hcl | 31 +++++++++++++++++++ examples/packer-basic-example/build.pkr.hcl | 15 +++------ test/gcp/packer_gcp_basic_example_test.go | 2 +- 4 files changed, 39 insertions(+), 16 deletions(-) create mode 100644 examples/packer-basic-example/build-gcp.pkr.hcl diff --git a/examples/packer-basic-example/README.md b/examples/packer-basic-example/README.md index ca1ac657f..a105de181 100644 --- a/examples/packer-basic-example/README.md +++ b/examples/packer-basic-example/README.md @@ -2,8 +2,7 @@ This folder contains a very simple Packer template to demonstrate how you can use Terratest to write automated tests for your Packer templates. The template just creates an up-to-date Ubuntu AMI by running `apt-get update` and -`apt-get upgrade`. It also uses the [packer-provisioner-comment plugin](https://github.com/sylviamoss/packer-plugin-comment) to -demonstrate how plugins work (this is primarily for testing purposes). +`apt-get upgrade`. Check out [test/packer_basic_example_test.go](/test/packer_basic_example_test.go) to see how you can write automated tests for this simple template. @@ -23,8 +22,8 @@ Terratest. For slightly more complicated, real-world examples of Packer template 1. Install [Packer](https://www.packer.io/) and make sure it's on your `PATH`. ## Building the Packer template manually (Packer >= 1.7.0) -1. Run `packer init build.pkr.hcl`. -1. Run `packer build build.pkr.hcl`. +1. Run `packer init build.pkr.hcl`. # Use build-gcp.pkr.hcl if using GCP +1. Run `packer build build.pkr.hcl`. # Use build-gcp.pkr.hcl if using GCP ## Building the Packer template manually (Packer < 1.7.0) 1. Run `packer build build.json`. diff --git a/examples/packer-basic-example/build-gcp.pkr.hcl b/examples/packer-basic-example/build-gcp.pkr.hcl new file mode 100644 index 000000000..f8c31e370 --- /dev/null +++ b/examples/packer-basic-example/build-gcp.pkr.hcl @@ -0,0 +1,31 @@ +variable "gcp_project_id" { + type = string + default = "" +} + +variable "gcp_zone" { + type = string + default = "us-central1-a" +} + +source "googlecompute" "gcp" { + image_family = "terratest" + image_name = "terratest-packer-example-${formatdate("YYYYMMDD-hhmm", timestamp())}" + project_id = var.gcp_project_id + source_image_family = "ubuntu-1804-lts" + ssh_username = "ubuntu" + zone = var.gcp_zone +} + + +build { + sources = [ + "source.googlecompute.gcp" + ] + + provisioner "shell" { + inline = ["sudo DEBIAN_FRONTEND=noninteractive apt-get update", "sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y"] + pause_before = "30s" + } + +} diff --git a/examples/packer-basic-example/build.pkr.hcl b/examples/packer-basic-example/build.pkr.hcl index eed84f8b2..374d699b0 100644 --- a/examples/packer-basic-example/build.pkr.hcl +++ b/examples/packer-basic-example/build.pkr.hcl @@ -1,10 +1,8 @@ packer { - required_version = ">= 1.7.0" required_plugins { - # This plugin is primarily used for testing the new required_plugins feature introduced in Packer 1.7.0. - comment = { - version = ">=v0.2.23" - source = "github.com/sylviamoss/comment" + amazon = { + version = ">= 0.0.1" + source = "github.com/hashicorp/amazon" } } } @@ -84,7 +82,7 @@ source "amazon-ebs" "aws" { source "googlecompute" "gcp" { image_family = "terratest" - image_name = "terratest-packer-example-${formatdate("YYYYMMDD-hhmm", timestamp())}" + image_name = "terratest-packer-example-${formatdate("YYYYMMDD-hhmm", timestamp())}" project_id = var.gcp_project_id source_image_family = "ubuntu-1804-lts" ssh_username = "ubuntu" @@ -109,11 +107,6 @@ build { "source.oracle-oci.oracle" ] - provisioner "comment" { - comment = "Basic comment example" - ui = false - } - provisioner "shell" { inline = ["sudo DEBIAN_FRONTEND=noninteractive apt-get update", "sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y"] pause_before = "30s" diff --git a/test/gcp/packer_gcp_basic_example_test.go b/test/gcp/packer_gcp_basic_example_test.go index 8ed51803f..e9a87bbb9 100644 --- a/test/gcp/packer_gcp_basic_example_test.go +++ b/test/gcp/packer_gcp_basic_example_test.go @@ -36,7 +36,7 @@ func TestPackerGCPBasicExample(t *testing.T) { packerOptions := &packer.Options{ // The path to where the Packer template is located - Template: "../../examples/packer-basic-example/build.pkr.hcl", + Template: "../../examples/packer-basic-example/build-gcp.pkr.hcl", // Variables to pass to our Packer build using -var options Vars: map[string]string{ From 409765572d1d5ea28f670f0e6815a9f54ea6f3c8 Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Mon, 26 Jul 2021 19:01:56 -0700 Subject: [PATCH 04/22] Fix required_providers --- examples/packer-basic-example/build-gcp.pkr.hcl | 9 +++++++++ examples/packer-basic-example/build.pkr.hcl | 2 +- examples/packer-docker-example/build.pkr.hcl | 8 ++++++++ examples/packer-hello-world-example/build.pkr.hcl | 11 ++++++++++- test/gcp/packer_gcp_basic_example_test.go | 2 +- 5 files changed, 29 insertions(+), 3 deletions(-) diff --git a/examples/packer-basic-example/build-gcp.pkr.hcl b/examples/packer-basic-example/build-gcp.pkr.hcl index f8c31e370..9494d9946 100644 --- a/examples/packer-basic-example/build-gcp.pkr.hcl +++ b/examples/packer-basic-example/build-gcp.pkr.hcl @@ -1,3 +1,12 @@ +packer { + required_plugins { + googlecompute = { + version = ">=v1.0.0" + source = "github.com/hashicorp/googlecompute" + } + } +} + variable "gcp_project_id" { type = string default = "" diff --git a/examples/packer-basic-example/build.pkr.hcl b/examples/packer-basic-example/build.pkr.hcl index 374d699b0..d01a96cd4 100644 --- a/examples/packer-basic-example/build.pkr.hcl +++ b/examples/packer-basic-example/build.pkr.hcl @@ -1,7 +1,7 @@ packer { required_plugins { amazon = { - version = ">= 0.0.1" + version = ">= 1.0.0" source = "github.com/hashicorp/amazon" } } diff --git a/examples/packer-docker-example/build.pkr.hcl b/examples/packer-docker-example/build.pkr.hcl index 303eaad4b..db4a80de7 100644 --- a/examples/packer-docker-example/build.pkr.hcl +++ b/examples/packer-docker-example/build.pkr.hcl @@ -1,3 +1,11 @@ +packer { + required_plugins { + amazon = { + version = ">= 1.0.0" + source = "github.com/hashicorp/amazon" + } + } +} variable "ami_name_base" { type = string diff --git a/examples/packer-hello-world-example/build.pkr.hcl b/examples/packer-hello-world-example/build.pkr.hcl index 301e0902f..2db2cabab 100644 --- a/examples/packer-hello-world-example/build.pkr.hcl +++ b/examples/packer-hello-world-example/build.pkr.hcl @@ -1,3 +1,12 @@ +packer { + required_plugins { + docker = { + version = ">=v1.0.1" + source = "github.com/hashicorp/docker" + } + } +} + source "docker" "ubuntu-docker" { changes = ["ENTRYPOINT [\"\"]"] commit = true @@ -13,6 +22,6 @@ build { post-processor "docker-tag" { repository = "gruntwork/packer-hello-world-example" - tag = "latest" + tag = ["latest"] } } diff --git a/test/gcp/packer_gcp_basic_example_test.go b/test/gcp/packer_gcp_basic_example_test.go index e9a87bbb9..53f4811ad 100644 --- a/test/gcp/packer_gcp_basic_example_test.go +++ b/test/gcp/packer_gcp_basic_example_test.go @@ -45,7 +45,7 @@ func TestPackerGCPBasicExample(t *testing.T) { }, // Only build the Google Compute Image - Only: "googlecompute", + Only: "googlecompute.gcp", // Configure retries for intermittent errors RetryableErrors: DefaultRetryablePackerErrors, From e482d7321e87e9d88fd450ff214eaedba58a1afe Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 09:01:27 -0700 Subject: [PATCH 05/22] Fix required version --- examples/packer-docker-example/build.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/packer-docker-example/build.pkr.hcl b/examples/packer-docker-example/build.pkr.hcl index db4a80de7..dec6843d6 100644 --- a/examples/packer-docker-example/build.pkr.hcl +++ b/examples/packer-docker-example/build.pkr.hcl @@ -1,7 +1,7 @@ packer { required_plugins { amazon = { - version = ">= 1.0.0" + version = ">=v1.0.0" source = "github.com/hashicorp/amazon" } } From 63ec966ff726c9dc6931c69511e4805fa63c21ef Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 09:02:03 -0700 Subject: [PATCH 06/22] Fix required version --- examples/packer-basic-example/build.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/packer-basic-example/build.pkr.hcl b/examples/packer-basic-example/build.pkr.hcl index d01a96cd4..59671cf4a 100644 --- a/examples/packer-basic-example/build.pkr.hcl +++ b/examples/packer-basic-example/build.pkr.hcl @@ -1,7 +1,7 @@ packer { required_plugins { amazon = { - version = ">= 1.0.0" + version = ">= v1.0.0" source = "github.com/hashicorp/amazon" } } From 41f099950869118cdbf419055999daa585d9096b Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 09:47:28 -0700 Subject: [PATCH 07/22] [skip ci] fix hclfmt --- examples/packer-basic-example/build-gcp.pkr.hcl | 2 +- examples/packer-basic-example/build.pkr.hcl | 2 +- examples/packer-docker-example/build.pkr.hcl | 2 +- examples/packer-hello-world-example/build.pkr.hcl | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/packer-basic-example/build-gcp.pkr.hcl b/examples/packer-basic-example/build-gcp.pkr.hcl index 9494d9946..46960b476 100644 --- a/examples/packer-basic-example/build-gcp.pkr.hcl +++ b/examples/packer-basic-example/build-gcp.pkr.hcl @@ -2,7 +2,7 @@ packer { required_plugins { googlecompute = { version = ">=v1.0.0" - source = "github.com/hashicorp/googlecompute" + source = "github.com/hashicorp/googlecompute" } } } diff --git a/examples/packer-basic-example/build.pkr.hcl b/examples/packer-basic-example/build.pkr.hcl index 59671cf4a..3d44939c2 100644 --- a/examples/packer-basic-example/build.pkr.hcl +++ b/examples/packer-basic-example/build.pkr.hcl @@ -2,7 +2,7 @@ packer { required_plugins { amazon = { version = ">= v1.0.0" - source = "github.com/hashicorp/amazon" + source = "github.com/hashicorp/amazon" } } } diff --git a/examples/packer-docker-example/build.pkr.hcl b/examples/packer-docker-example/build.pkr.hcl index dec6843d6..8b9d81ffe 100644 --- a/examples/packer-docker-example/build.pkr.hcl +++ b/examples/packer-docker-example/build.pkr.hcl @@ -2,7 +2,7 @@ packer { required_plugins { amazon = { version = ">=v1.0.0" - source = "github.com/hashicorp/amazon" + source = "github.com/hashicorp/amazon" } } } diff --git a/examples/packer-hello-world-example/build.pkr.hcl b/examples/packer-hello-world-example/build.pkr.hcl index 2db2cabab..96c6832a8 100644 --- a/examples/packer-hello-world-example/build.pkr.hcl +++ b/examples/packer-hello-world-example/build.pkr.hcl @@ -2,7 +2,7 @@ packer { required_plugins { docker = { version = ">=v1.0.1" - source = "github.com/hashicorp/docker" + source = "github.com/hashicorp/docker" } } } From b4550aba939dcfec4a76b9e7eddec26b92c06349 Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 10:39:17 -0700 Subject: [PATCH 08/22] simplify required_plugin sources --- .../packer-basic-example/build-gcp.pkr.hcl | 2 +- examples/packer-basic-example/build.pkr.hcl | 18 ++++++------------ examples/packer-docker-example/build.pkr.hcl | 2 +- .../packer-hello-world-example/build.pkr.hcl | 2 +- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/examples/packer-basic-example/build-gcp.pkr.hcl b/examples/packer-basic-example/build-gcp.pkr.hcl index 46960b476..c956ee0ad 100644 --- a/examples/packer-basic-example/build-gcp.pkr.hcl +++ b/examples/packer-basic-example/build-gcp.pkr.hcl @@ -2,7 +2,7 @@ packer { required_plugins { googlecompute = { version = ">=v1.0.0" - source = "github.com/hashicorp/googlecompute" + source = "hashicorp/googlecompute" } } } diff --git a/examples/packer-basic-example/build.pkr.hcl b/examples/packer-basic-example/build.pkr.hcl index 3d44939c2..9138daf60 100644 --- a/examples/packer-basic-example/build.pkr.hcl +++ b/examples/packer-basic-example/build.pkr.hcl @@ -1,8 +1,12 @@ packer { required_plugins { amazon = { - version = ">= v1.0.0" - source = "github.com/hashicorp/amazon" + version = ">=v1.0.0" + source = "hashicorp/amazon" + } + oracle = { + version = ">=v1.0.0" + source = "hashicorp/oracle" } } } @@ -80,15 +84,6 @@ source "amazon-ebs" "aws" { ssh_username = "ubuntu" } -source "googlecompute" "gcp" { - image_family = "terratest" - image_name = "terratest-packer-example-${formatdate("YYYYMMDD-hhmm", timestamp())}" - project_id = var.gcp_project_id - source_image_family = "ubuntu-1804-lts" - ssh_username = "ubuntu" - zone = var.gcp_zone -} - source "oracle-oci" "oracle" { availability_domain = var.oci_availability_domain base_image_ocid = var.oci_base_image_ocid @@ -103,7 +98,6 @@ source "oracle-oci" "oracle" { build { sources = [ "source.amazon-ebs.aws", - "source.googlecompute.gcp", "source.oracle-oci.oracle" ] diff --git a/examples/packer-docker-example/build.pkr.hcl b/examples/packer-docker-example/build.pkr.hcl index 8b9d81ffe..80cbc9ee2 100644 --- a/examples/packer-docker-example/build.pkr.hcl +++ b/examples/packer-docker-example/build.pkr.hcl @@ -2,7 +2,7 @@ packer { required_plugins { amazon = { version = ">=v1.0.0" - source = "github.com/hashicorp/amazon" + source = "hashicorp/amazon" } } } diff --git a/examples/packer-hello-world-example/build.pkr.hcl b/examples/packer-hello-world-example/build.pkr.hcl index 96c6832a8..16c8d9dad 100644 --- a/examples/packer-hello-world-example/build.pkr.hcl +++ b/examples/packer-hello-world-example/build.pkr.hcl @@ -2,7 +2,7 @@ packer { required_plugins { docker = { version = ">=v1.0.1" - source = "github.com/hashicorp/docker" + source = "hashicorp/docker" } } } From 8beca4c81386da8c47c33f4d051558076d1c63d2 Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 10:58:42 -0700 Subject: [PATCH 09/22] Revert "simplify required_plugin sources" This reverts commit b4550aba939dcfec4a76b9e7eddec26b92c06349. --- .../packer-basic-example/build-gcp.pkr.hcl | 2 +- examples/packer-basic-example/build.pkr.hcl | 18 ++++++++++++------ examples/packer-docker-example/build.pkr.hcl | 2 +- .../packer-hello-world-example/build.pkr.hcl | 2 +- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/examples/packer-basic-example/build-gcp.pkr.hcl b/examples/packer-basic-example/build-gcp.pkr.hcl index c956ee0ad..46960b476 100644 --- a/examples/packer-basic-example/build-gcp.pkr.hcl +++ b/examples/packer-basic-example/build-gcp.pkr.hcl @@ -2,7 +2,7 @@ packer { required_plugins { googlecompute = { version = ">=v1.0.0" - source = "hashicorp/googlecompute" + source = "github.com/hashicorp/googlecompute" } } } diff --git a/examples/packer-basic-example/build.pkr.hcl b/examples/packer-basic-example/build.pkr.hcl index 9138daf60..3d44939c2 100644 --- a/examples/packer-basic-example/build.pkr.hcl +++ b/examples/packer-basic-example/build.pkr.hcl @@ -1,12 +1,8 @@ packer { required_plugins { amazon = { - version = ">=v1.0.0" - source = "hashicorp/amazon" - } - oracle = { - version = ">=v1.0.0" - source = "hashicorp/oracle" + version = ">= v1.0.0" + source = "github.com/hashicorp/amazon" } } } @@ -84,6 +80,15 @@ source "amazon-ebs" "aws" { ssh_username = "ubuntu" } +source "googlecompute" "gcp" { + image_family = "terratest" + image_name = "terratest-packer-example-${formatdate("YYYYMMDD-hhmm", timestamp())}" + project_id = var.gcp_project_id + source_image_family = "ubuntu-1804-lts" + ssh_username = "ubuntu" + zone = var.gcp_zone +} + source "oracle-oci" "oracle" { availability_domain = var.oci_availability_domain base_image_ocid = var.oci_base_image_ocid @@ -98,6 +103,7 @@ source "oracle-oci" "oracle" { build { sources = [ "source.amazon-ebs.aws", + "source.googlecompute.gcp", "source.oracle-oci.oracle" ] diff --git a/examples/packer-docker-example/build.pkr.hcl b/examples/packer-docker-example/build.pkr.hcl index 80cbc9ee2..8b9d81ffe 100644 --- a/examples/packer-docker-example/build.pkr.hcl +++ b/examples/packer-docker-example/build.pkr.hcl @@ -2,7 +2,7 @@ packer { required_plugins { amazon = { version = ">=v1.0.0" - source = "hashicorp/amazon" + source = "github.com/hashicorp/amazon" } } } diff --git a/examples/packer-hello-world-example/build.pkr.hcl b/examples/packer-hello-world-example/build.pkr.hcl index 16c8d9dad..96c6832a8 100644 --- a/examples/packer-hello-world-example/build.pkr.hcl +++ b/examples/packer-hello-world-example/build.pkr.hcl @@ -2,7 +2,7 @@ packer { required_plugins { docker = { version = ">=v1.0.1" - source = "hashicorp/docker" + source = "github.com/hashicorp/docker" } } } From 473e7b4e57bc1d9fc2a6bd57901bda110703b147 Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 11:00:34 -0700 Subject: [PATCH 10/22] Drop space in version --- examples/packer-basic-example/build.pkr.hcl | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/examples/packer-basic-example/build.pkr.hcl b/examples/packer-basic-example/build.pkr.hcl index 3d44939c2..bd6c27336 100644 --- a/examples/packer-basic-example/build.pkr.hcl +++ b/examples/packer-basic-example/build.pkr.hcl @@ -1,9 +1,13 @@ packer { required_plugins { amazon = { - version = ">= v1.0.0" + version = ">=v1.0.0" source = "github.com/hashicorp/amazon" } + oracle = { + version = ">=v1.0.0" + source = "github.com/hashicorp/oracle" + } } } @@ -80,15 +84,6 @@ source "amazon-ebs" "aws" { ssh_username = "ubuntu" } -source "googlecompute" "gcp" { - image_family = "terratest" - image_name = "terratest-packer-example-${formatdate("YYYYMMDD-hhmm", timestamp())}" - project_id = var.gcp_project_id - source_image_family = "ubuntu-1804-lts" - ssh_username = "ubuntu" - zone = var.gcp_zone -} - source "oracle-oci" "oracle" { availability_domain = var.oci_availability_domain base_image_ocid = var.oci_base_image_ocid @@ -103,7 +98,6 @@ source "oracle-oci" "oracle" { build { sources = [ "source.amazon-ebs.aws", - "source.googlecompute.gcp", "source.oracle-oci.oracle" ] From 477fa36da02002595e1368647fc4b42b335c2a35 Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 11:55:59 -0700 Subject: [PATCH 11/22] explicitly set plugin path --- .circleci/config.yml | 1 + examples/packer-basic-example/build-gcp.pkr.hcl | 4 ++-- examples/packer-basic-example/build.pkr.hcl | 15 ++------------- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1cf3eccc2..b33bc4259 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,7 @@ env: &env HELM_VERSION: v3.1.1 KUBECONFIG: /home/circleci/.kube/config BIN_BUILD_PARALLELISM: 3 + PACKER_PLUGIN_PATH: /home/circleci/.packer.d/plugins defaults: &defaults machine: diff --git a/examples/packer-basic-example/build-gcp.pkr.hcl b/examples/packer-basic-example/build-gcp.pkr.hcl index 46960b476..59c462969 100644 --- a/examples/packer-basic-example/build-gcp.pkr.hcl +++ b/examples/packer-basic-example/build-gcp.pkr.hcl @@ -17,7 +17,7 @@ variable "gcp_zone" { default = "us-central1-a" } -source "googlecompute" "gcp" { +source "googlecompute" "ubuntu-bionic" { image_family = "terratest" image_name = "terratest-packer-example-${formatdate("YYYYMMDD-hhmm", timestamp())}" project_id = var.gcp_project_id @@ -29,7 +29,7 @@ source "googlecompute" "gcp" { build { sources = [ - "source.googlecompute.gcp" + "source.googlecompute.ubuntu-bionic" ] provisioner "shell" { diff --git a/examples/packer-basic-example/build.pkr.hcl b/examples/packer-basic-example/build.pkr.hcl index bd6c27336..1771417a8 100644 --- a/examples/packer-basic-example/build.pkr.hcl +++ b/examples/packer-basic-example/build.pkr.hcl @@ -21,16 +21,6 @@ variable "aws_region" { default = "us-east-1" } -variable "gcp_project_id" { - type = string - default = "" -} - -variable "gcp_zone" { - type = string - default = "us-central1-a" -} - variable "instance_type" { type = string default = "t2.micro" @@ -74,7 +64,7 @@ data "amazon-ami" "ubuntu-xenial" { region = var.aws_region } -source "amazon-ebs" "aws" { +source "amazon-ebs" "example" { ami_description = "An example of how to create a custom AMI on top of Ubuntu" ami_name = "${var.ami_base_name}-terratest-packer-example" encrypt_boot = false @@ -97,7 +87,7 @@ source "oracle-oci" "oracle" { build { sources = [ - "source.amazon-ebs.aws", + "source.amazon-ebs.example", "source.oracle-oci.oracle" ] @@ -105,5 +95,4 @@ build { inline = ["sudo DEBIAN_FRONTEND=noninteractive apt-get update", "sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y"] pause_before = "30s" } - } From 44a77e506f441fa8226dd888da662176c2df877d Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 13:30:30 -0700 Subject: [PATCH 12/22] align package image source names --- examples/packer-basic-example/build.pkr.hcl | 8 ++++---- test/gcp/packer_gcp_basic_example_test.go | 2 +- test/packer_basic_example_test.go | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/packer-basic-example/build.pkr.hcl b/examples/packer-basic-example/build.pkr.hcl index 1771417a8..a38de4b81 100644 --- a/examples/packer-basic-example/build.pkr.hcl +++ b/examples/packer-basic-example/build.pkr.hcl @@ -64,7 +64,7 @@ data "amazon-ami" "ubuntu-xenial" { region = var.aws_region } -source "amazon-ebs" "example" { +source "amazon-ebs" "ubuntu-example" { ami_description = "An example of how to create a custom AMI on top of Ubuntu" ami_name = "${var.ami_base_name}-terratest-packer-example" encrypt_boot = false @@ -74,7 +74,7 @@ source "amazon-ebs" "example" { ssh_username = "ubuntu" } -source "oracle-oci" "oracle" { +source "oracle-oci" "oracle-example" { availability_domain = var.oci_availability_domain base_image_ocid = var.oci_base_image_ocid compartment_ocid = var.oci_compartment_ocid @@ -87,8 +87,8 @@ source "oracle-oci" "oracle" { build { sources = [ - "source.amazon-ebs.example", - "source.oracle-oci.oracle" + "source.amazon-ebs.ubuntu-example", + "source.oracle-oci.oracle-example" ] provisioner "shell" { diff --git a/test/gcp/packer_gcp_basic_example_test.go b/test/gcp/packer_gcp_basic_example_test.go index 53f4811ad..bfc3b35b2 100644 --- a/test/gcp/packer_gcp_basic_example_test.go +++ b/test/gcp/packer_gcp_basic_example_test.go @@ -45,7 +45,7 @@ func TestPackerGCPBasicExample(t *testing.T) { }, // Only build the Google Compute Image - Only: "googlecompute.gcp", + Only: "googlecompute.ubuntu-bionic", // Configure retries for intermittent errors RetryableErrors: DefaultRetryablePackerErrors, diff --git a/test/packer_basic_example_test.go b/test/packer_basic_example_test.go index 4ac56815e..9cb2d91a7 100644 --- a/test/packer_basic_example_test.go +++ b/test/packer_basic_example_test.go @@ -50,7 +50,7 @@ func TestPackerBasicExample(t *testing.T) { }, // Only build the AWS AMI - Only: "amazon-ebs.aws", + Only: "amazon-ebs.ubuntu-example", // Configure retries for intermittent errors RetryableErrors: DefaultRetryablePackerErrors, @@ -116,7 +116,7 @@ func TestPackerBasicExampleWithVarFile(t *testing.T) { }, // Only build the AWS AMI - Only: "amazon-ebs.aws", + Only: "amazon-ebs.ubuntu-example", // Configure retries for intermittent errors RetryableErrors: DefaultRetryablePackerErrors, @@ -171,7 +171,7 @@ func TestPackerMultipleConcurrentAmis(t *testing.T) { }, // Only build the AWS AMI - Only: "amazon-ebs.aws", + Only: "amazon-ebs.ubuntu-example", // Configure retries for intermittent errors RetryableErrors: DefaultRetryablePackerErrors, From 91468bfd9b1dc3fcfc7f14c25f3ed5447a630cba Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 15:20:14 -0700 Subject: [PATCH 13/22] Do not set plugin config dir --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b33bc4259..1cf3eccc2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,7 +13,6 @@ env: &env HELM_VERSION: v3.1.1 KUBECONFIG: /home/circleci/.kube/config BIN_BUILD_PARALLELISM: 3 - PACKER_PLUGIN_PATH: /home/circleci/.packer.d/plugins defaults: &defaults machine: From 0fac53358505ab43464fa7c1cc60b8356c159d12 Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 15:20:35 -0700 Subject: [PATCH 14/22] Reduce number of files to upload to S3 --- modules/aws/s3_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/aws/s3_test.go b/modules/aws/s3_test.go index 2d4ad117a..947fd4b80 100644 --- a/modules/aws/s3_test.go +++ b/modules/aws/s3_test.go @@ -147,7 +147,7 @@ func TestAssertS3BucketPolicyExists(t *testing.T) { } func testEmptyBucket(t *testing.T, s3Client *s3.S3, region string, s3BucketName string) { - expectedFileCount := rand.Intn(10000) + expectedFileCount := rand.Intn(1000) logger.Logf(t, "Uploading %s files to bucket %s", strconv.Itoa(expectedFileCount), s3BucketName) deleted := 0 From 2e0a0bf2b91070d385f00d4e797f3286e40d87d9 Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 16:15:23 -0700 Subject: [PATCH 15/22] use temporary plugin dirs --- modules/packer/packer.go | 41 ++++++++++++++++++------- test/packer_basic_example_test.go | 8 +++++ test/packer_docker_example_test.go | 3 ++ test/packer_hello_world_example_test.go | 3 ++ test/terraform_packer_example_test.go | 5 ++- 5 files changed, 48 insertions(+), 12 deletions(-) diff --git a/modules/packer/packer.go b/modules/packer/packer.go index 070fca15c..e6bb02700 100644 --- a/modules/packer/packer.go +++ b/modules/packer/packer.go @@ -4,6 +4,9 @@ package packer import ( "errors" "fmt" + "io/ioutil" + "log" + "os" "regexp" "sync" "time" @@ -17,19 +20,23 @@ import ( "github.com/hashicorp/go-version" ) +// PackerPluginPathEnvVar is the built-in env variable defining where plugins are downloaded +const PackerPluginPathEnvVar = "PACKER_PLUGIN_PATH" + // Options are the options for Packer. type Options struct { - Template string // The path to the Packer template - Vars map[string]string // The custom vars to pass when running the build command - VarFiles []string // Var file paths to pass Packer using -var-file option - Only string // If specified, only run the build of this name - Except string // Runs the build excluding the specified builds and post-processors - Env map[string]string // Custom environment variables to set when running Packer - RetryableErrors map[string]string // If packer build fails with one of these (transient) errors, retry. The keys are a regexp to match against the error and the message is what to display to a user if that error is matched. - MaxRetries int // Maximum number of times to retry errors matching RetryableErrors - TimeBetweenRetries time.Duration // The amount of time to wait between retries - WorkingDir string // The directory to run packer in - Logger *logger.Logger // If set, use a non-default logger + Template string // The path to the Packer template + Vars map[string]string // The custom vars to pass when running the build command + VarFiles []string // Var file paths to pass Packer using -var-file option + Only string // If specified, only run the build of this name + Except string // Runs the build excluding the specified builds and post-processors + Env map[string]string // Custom environment variables to set when running Packer + RetryableErrors map[string]string // If packer build fails with one of these (transient) errors, retry. The keys are a regexp to match against the error and the message is what to display to a user if that error is matched. + MaxRetries int // Maximum number of times to retry errors matching RetryableErrors + TimeBetweenRetries time.Duration // The amount of time to wait between retries + WorkingDir string // The directory to run packer in + Logger *logger.Logger // If set, use a non-default logger + DisposablePluginPath bool // If set, download plugins to a temporary directory and remove them at the end of the test } // BuildArtifacts can take a map of identifierName <-> Options and then parallelize @@ -93,6 +100,16 @@ func BuildArtifact(t testing.TestingT, options *Options) string { func BuildArtifactE(t testing.TestingT, options *Options) (string, error) { options.Logger.Logf(t, "Running Packer to generate a custom artifact for template %s", options.Template) + if options.DisposablePluginPath { + options.Logger.Logf(t, "Creating a temporary directory for Packer plugins") + pluginDir, err := ioutil.TempDir("", "terratest-packer-") + if err != nil { + log.Fatal(err) + } + options.Env[PackerPluginPathEnvVar] = pluginDir + defer os.RemoveAll(pluginDir) + } + err := packerInit(t, options) if err != nil { return "", err @@ -162,6 +179,7 @@ func packerInit(t testing.TestingT, options *Options) error { cmd := shell.Command{ Command: "packer", Args: []string{"-version"}, + Env: options.Env, WorkingDir: options.WorkingDir, } description := fmt.Sprintf("%s %v", cmd.Command, cmd.Args) @@ -185,6 +203,7 @@ func packerInit(t testing.TestingT, options *Options) error { cmd = shell.Command{ Command: "packer", Args: []string{"init", options.Template}, + Env: options.Env, WorkingDir: options.WorkingDir, } diff --git a/test/packer_basic_example_test.go b/test/packer_basic_example_test.go index 9cb2d91a7..61587d412 100644 --- a/test/packer_basic_example_test.go +++ b/test/packer_basic_example_test.go @@ -115,6 +115,11 @@ func TestPackerBasicExampleWithVarFile(t *testing.T) { varFile.Name(), }, + // Environment settings to avoid plugin conflicts + Env: map[string]string{ + "PACKER_PLUGIN_PATH": "../examples/packer-basic-example/.packer.d/plugins", + }, + // Only build the AWS AMI Only: "amazon-ebs.ubuntu-example", @@ -177,6 +182,9 @@ func TestPackerMultipleConcurrentAmis(t *testing.T) { RetryableErrors: DefaultRetryablePackerErrors, TimeBetweenRetries: DefaultTimeBetweenPackerRetries, MaxRetries: DefaultMaxPackerRetries, + + // Use a temporary directory for Packer plugins + DisposablePluginPath: true, } identifierToOptions[random.UniqueId()] = packerOptions diff --git a/test/packer_docker_example_test.go b/test/packer_docker_example_test.go index 7f0d3711a..ce4de7fe5 100644 --- a/test/packer_docker_example_test.go +++ b/test/packer_docker_example_test.go @@ -26,6 +26,9 @@ func TestPackerDockerExampleLocal(t *testing.T) { // Only build the Docker image for local testing Only: "ubuntu-docker", + // Use a temporary directory for Packer plugins + DisposablePluginPath: true, + // Configure retries for intermittent errors RetryableErrors: DefaultRetryablePackerErrors, TimeBetweenRetries: DefaultTimeBetweenPackerRetries, diff --git a/test/packer_hello_world_example_test.go b/test/packer_hello_world_example_test.go index 515050861..72bf52279 100644 --- a/test/packer_hello_world_example_test.go +++ b/test/packer_hello_world_example_test.go @@ -12,6 +12,9 @@ func TestPackerHelloWorldExample(t *testing.T) { packerOptions := &packer.Options{ // website::tag::1:: The path to where the Packer template is located Template: "../examples/packer-hello-world-example/build.pkr.hcl", + + // Use a temporary directory for Packer plugins + DisposablePluginPath: true, } // website::tag::2:: Build the Packer template. This template will create a Docker image. diff --git a/test/terraform_packer_example_test.go b/test/terraform_packer_example_test.go index 15bc946d8..0369a3bb6 100644 --- a/test/terraform_packer_example_test.go +++ b/test/terraform_packer_example_test.go @@ -82,6 +82,9 @@ func buildAMI(t *testing.T, awsRegion string, workingDir string) { "instance_type": instanceType, }, + // Use a temporary directory for Packer plugins + DisposablePluginPath: true, + // Configure retries for intermittent errors RetryableErrors: DefaultRetryablePackerErrors, TimeBetweenRetries: DefaultTimeBetweenPackerRetries, @@ -92,7 +95,7 @@ func buildAMI(t *testing.T, awsRegion string, workingDir string) { test_structure.SavePackerOptions(t, workingDir, packerOptions) // Build the AMI - amiID := packer.BuildAmi(t, packerOptions) + amiID := packer.BuildArtifact(t, packerOptions) // Save the AMI ID so future test stages can use them test_structure.SaveAmiId(t, workingDir, amiID) From bfbf10edc1d3f76b7708b22823927b20792be19e Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 16:38:10 -0700 Subject: [PATCH 16/22] Default to using a temp dir --- modules/packer/packer.go | 29 ++++++++++++++----------- test/packer_basic_example_test.go | 3 --- test/packer_docker_example_test.go | 3 --- test/packer_hello_world_example_test.go | 3 --- test/terraform_packer_example_test.go | 3 --- 5 files changed, 16 insertions(+), 25 deletions(-) diff --git a/modules/packer/packer.go b/modules/packer/packer.go index e6bb02700..16199c8a7 100644 --- a/modules/packer/packer.go +++ b/modules/packer/packer.go @@ -25,18 +25,18 @@ const PackerPluginPathEnvVar = "PACKER_PLUGIN_PATH" // Options are the options for Packer. type Options struct { - Template string // The path to the Packer template - Vars map[string]string // The custom vars to pass when running the build command - VarFiles []string // Var file paths to pass Packer using -var-file option - Only string // If specified, only run the build of this name - Except string // Runs the build excluding the specified builds and post-processors - Env map[string]string // Custom environment variables to set when running Packer - RetryableErrors map[string]string // If packer build fails with one of these (transient) errors, retry. The keys are a regexp to match against the error and the message is what to display to a user if that error is matched. - MaxRetries int // Maximum number of times to retry errors matching RetryableErrors - TimeBetweenRetries time.Duration // The amount of time to wait between retries - WorkingDir string // The directory to run packer in - Logger *logger.Logger // If set, use a non-default logger - DisposablePluginPath bool // If set, download plugins to a temporary directory and remove them at the end of the test + Template string // The path to the Packer template + Vars map[string]string // The custom vars to pass when running the build command + VarFiles []string // Var file paths to pass Packer using -var-file option + Only string // If specified, only run the build of this name + Except string // Runs the build excluding the specified builds and post-processors + Env map[string]string // Custom environment variables to set when running Packer + RetryableErrors map[string]string // If packer build fails with one of these (transient) errors, retry. The keys are a regexp to match against the error and the message is what to display to a user if that error is matched. + MaxRetries int // Maximum number of times to retry errors matching RetryableErrors + TimeBetweenRetries time.Duration // The amount of time to wait between retries + WorkingDir string // The directory to run packer in + Logger *logger.Logger // If set, use a non-default logger + DisableTemporaryPluginPath bool // If set, do not use a temporary directory for Packer plugins. } // BuildArtifacts can take a map of identifierName <-> Options and then parallelize @@ -100,12 +100,15 @@ func BuildArtifact(t testing.TestingT, options *Options) string { func BuildArtifactE(t testing.TestingT, options *Options) (string, error) { options.Logger.Logf(t, "Running Packer to generate a custom artifact for template %s", options.Template) - if options.DisposablePluginPath { + if !options.DisableTemporaryPluginPath { options.Logger.Logf(t, "Creating a temporary directory for Packer plugins") pluginDir, err := ioutil.TempDir("", "terratest-packer-") if err != nil { log.Fatal(err) } + if len(options.Env) == 0 { + options.Env = make(map[string]string) + } options.Env[PackerPluginPathEnvVar] = pluginDir defer os.RemoveAll(pluginDir) } diff --git a/test/packer_basic_example_test.go b/test/packer_basic_example_test.go index 61587d412..37558ed9c 100644 --- a/test/packer_basic_example_test.go +++ b/test/packer_basic_example_test.go @@ -182,9 +182,6 @@ func TestPackerMultipleConcurrentAmis(t *testing.T) { RetryableErrors: DefaultRetryablePackerErrors, TimeBetweenRetries: DefaultTimeBetweenPackerRetries, MaxRetries: DefaultMaxPackerRetries, - - // Use a temporary directory for Packer plugins - DisposablePluginPath: true, } identifierToOptions[random.UniqueId()] = packerOptions diff --git a/test/packer_docker_example_test.go b/test/packer_docker_example_test.go index ce4de7fe5..7f0d3711a 100644 --- a/test/packer_docker_example_test.go +++ b/test/packer_docker_example_test.go @@ -26,9 +26,6 @@ func TestPackerDockerExampleLocal(t *testing.T) { // Only build the Docker image for local testing Only: "ubuntu-docker", - // Use a temporary directory for Packer plugins - DisposablePluginPath: true, - // Configure retries for intermittent errors RetryableErrors: DefaultRetryablePackerErrors, TimeBetweenRetries: DefaultTimeBetweenPackerRetries, diff --git a/test/packer_hello_world_example_test.go b/test/packer_hello_world_example_test.go index 72bf52279..515050861 100644 --- a/test/packer_hello_world_example_test.go +++ b/test/packer_hello_world_example_test.go @@ -12,9 +12,6 @@ func TestPackerHelloWorldExample(t *testing.T) { packerOptions := &packer.Options{ // website::tag::1:: The path to where the Packer template is located Template: "../examples/packer-hello-world-example/build.pkr.hcl", - - // Use a temporary directory for Packer plugins - DisposablePluginPath: true, } // website::tag::2:: Build the Packer template. This template will create a Docker image. diff --git a/test/terraform_packer_example_test.go b/test/terraform_packer_example_test.go index 0369a3bb6..ad837877e 100644 --- a/test/terraform_packer_example_test.go +++ b/test/terraform_packer_example_test.go @@ -82,9 +82,6 @@ func buildAMI(t *testing.T, awsRegion string, workingDir string) { "instance_type": instanceType, }, - // Use a temporary directory for Packer plugins - DisposablePluginPath: true, - // Configure retries for intermittent errors RetryableErrors: DefaultRetryablePackerErrors, TimeBetweenRetries: DefaultTimeBetweenPackerRetries, From 36b3004845065617f9cac00a63c832f7fc08bb01 Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 16:42:01 -0700 Subject: [PATCH 17/22] Test packer only [go-test-args=-run '^TestPackerBasicExample$'] From 85786f320cad3d87231f0aefe318a654e10696e0 Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 16:49:02 -0700 Subject: [PATCH 18/22] Run all tests From 5f1a9c5d5318cc4fdba18eb3dad178c16ae202e5 Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 18:47:40 -0700 Subject: [PATCH 19/22] Slight refactoring --- modules/packer/packer.go | 47 +++++++++++++++++++-------- test/packer_docker_example_test.go | 2 +- test/terraform_packer_example_test.go | 2 +- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/modules/packer/packer.go b/modules/packer/packer.go index 16199c8a7..7d301568d 100644 --- a/modules/packer/packer.go +++ b/modules/packer/packer.go @@ -20,9 +20,6 @@ import ( "github.com/hashicorp/go-version" ) -// PackerPluginPathEnvVar is the built-in env variable defining where plugins are downloaded -const PackerPluginPathEnvVar = "PACKER_PLUGIN_PATH" - // Options are the options for Packer. type Options struct { Template string // The path to the Packer template @@ -100,7 +97,13 @@ func BuildArtifact(t testing.TestingT, options *Options) string { func BuildArtifactE(t testing.TestingT, options *Options) (string, error) { options.Logger.Logf(t, "Running Packer to generate a custom artifact for template %s", options.Template) + // By default, we download packer plugins to a temporary directory rather than use the global plugin path. + // This prevents race conditions when multiple tests are running in parallel and each of them attempt + // to download the same plugin at the same time to the global path. + // Set DisableTemporaryPluginPath to disable this behavior. if !options.DisableTemporaryPluginPath { + // The built-in env variable defining where plugins are downloaded + const packerPluginPathEnvVar = "PACKER_PLUGIN_PATH" options.Logger.Logf(t, "Creating a temporary directory for Packer plugins") pluginDir, err := ioutil.TempDir("", "terratest-packer-") if err != nil { @@ -109,7 +112,7 @@ func BuildArtifactE(t testing.TestingT, options *Options) (string, error) { if len(options.Env) == 0 { options.Env = make(map[string]string) } - options.Env[PackerPluginPathEnvVar] = pluginDir + options.Env[packerPluginPathEnvVar] = pluginDir defer os.RemoveAll(pluginDir) } @@ -171,14 +174,15 @@ func extractArtifactID(packerLogOutput string) (string, error) { return "", errors.New("Could not find Artifact ID pattern in Packer output") } -// packerInit checks if init is supported in the current version of Packer, then runs packer init to download any required plugins. -func packerInit(t testing.TestingT, options *Options) error { +// Check if the local version of Packer has init +func hasPackerInit(t testing.TestingT, options *Options) (bool, error) { // The init command was introduced in Packer 1.7.0 const packerInitVersion = "1.7.0" minInitVersion, err := version.NewVersion(packerInitVersion) if err != nil { - return err + return false, err } + cmd := shell.Command{ Command: "packer", Args: []string{"-version"}, @@ -186,31 +190,46 @@ func packerInit(t testing.TestingT, options *Options) error { WorkingDir: options.WorkingDir, } description := fmt.Sprintf("%s %v", cmd.Command, cmd.Args) - installedVersion, err := retry.DoWithRetryableErrorsE(t, description, options.RetryableErrors, options.MaxRetries, options.TimeBetweenRetries, func() (string, error) { + localVersion, err := retry.DoWithRetryableErrorsE(t, description, options.RetryableErrors, options.MaxRetries, options.TimeBetweenRetries, func() (string, error) { return shell.RunCommandAndGetOutputE(t, cmd) }) if err != nil { - return err + return false, err } - thisVersion, err := version.NewVersion(installedVersion) + + thisVersion, err := version.NewVersion(localVersion) if err != nil { - return err + return false, err } + if thisVersion.LessThan(minInitVersion) { - options.Logger.Logf(t, "Skipping 'packer init' because it is not present in this version (%s)", thisVersion) + return false, nil + } + + return true, nil +} + +// packerInit runs 'packer init' if it is supported by the local packer +func packerInit(t testing.TestingT, options *Options) error { + hasInit, err := hasPackerInit(t, options) + if err != nil { + return err + } + if !hasInit { + options.Logger.Logf(t, "Skipping 'packer init' because it is not present in this version") return nil } options.Logger.Logf(t, "Running Packer init") - cmd = shell.Command{ + cmd := shell.Command{ Command: "packer", Args: []string{"init", options.Template}, Env: options.Env, WorkingDir: options.WorkingDir, } - description = fmt.Sprintf("%s %v", cmd.Command, cmd.Args) + description := fmt.Sprintf("%s %v", cmd.Command, cmd.Args) _, err = retry.DoWithRetryableErrorsE(t, description, options.RetryableErrors, options.MaxRetries, options.TimeBetweenRetries, func() (string, error) { return shell.RunCommandAndGetOutputE(t, cmd) }) diff --git a/test/packer_docker_example_test.go b/test/packer_docker_example_test.go index 7f0d3711a..f038d753a 100644 --- a/test/packer_docker_example_test.go +++ b/test/packer_docker_example_test.go @@ -24,7 +24,7 @@ func TestPackerDockerExampleLocal(t *testing.T) { Template: "../examples/packer-docker-example/build.pkr.hcl", // Only build the Docker image for local testing - Only: "ubuntu-docker", + Only: "docker.ubuntu-docker", // Configure retries for intermittent errors RetryableErrors: DefaultRetryablePackerErrors, diff --git a/test/terraform_packer_example_test.go b/test/terraform_packer_example_test.go index ad837877e..74bea07a7 100644 --- a/test/terraform_packer_example_test.go +++ b/test/terraform_packer_example_test.go @@ -74,7 +74,7 @@ func buildAMI(t *testing.T, awsRegion string, workingDir string) { Template: "../examples/packer-docker-example/build.pkr.hcl", // Only build the AMI - Only: "ubuntu-ami", + Only: "amazon-ebs.ubuntu-ami", // Variables to pass to our Packer build using -var options Vars: map[string]string{ From 0da2db672a5545482424d766918ca073b744ba1b Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 20:03:53 -0700 Subject: [PATCH 20/22] Fix only condition --- examples/packer-docker-example/build.json | 55 ++++++++++++++++++++ examples/packer-docker-example/build.pkr.hcl | 4 +- 2 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 examples/packer-docker-example/build.json diff --git a/examples/packer-docker-example/build.json b/examples/packer-docker-example/build.json new file mode 100644 index 000000000..fde5ab0c3 --- /dev/null +++ b/examples/packer-docker-example/build.json @@ -0,0 +1,55 @@ +{ + "variables": { + "aws_region": "us-east-1", + "ami_name_base": "terratest-packer-docker-example", + "instance_type": "t2.micro" + }, + "builders": [{ + "name": "ubuntu-ami", + "ami_name": "{{user `ami_name_base`}}-{{isotime | clean_resource_name}}", + "ami_description": "An example of how to create a custom AMI with a simple web app on top of Ubuntu", + "instance_type": "{{user `instance_type`}}", + "region": "{{user `aws_region`}}", + "type": "amazon-ebs", + "source_ami_filter": { + "filters": { + "virtualization-type": "hvm", + "architecture": "x86_64", + "name": "*ubuntu-xenial-16.04-amd64-server-*", + "block-device-mapping.volume-type": "gp2", + "root-device-type": "ebs" + }, + "owners": ["099720109477"], + "most_recent": true + }, + "ssh_username": "ubuntu", + "encrypt_boot": false + },{ + "name": "ubuntu-docker", + "type": "docker", + "image": "gruntwork/ubuntu-test:16.04", + "commit": true, + "changes": ["ENTRYPOINT [\"\"]"] + }], + "provisioners": [{ + "type": "shell", + "inline": [ + "echo 'Sleeping for a few seconds to give Ubuntu time to boot up'", + "sleep 30" + ], + "only": ["ubuntu-ami"] + },{ + "type": "file", + "source": "{{template_dir}}", + "destination": "/tmp/packer-docker-example" + },{ + "type": "shell", + "inline": ["/tmp/packer-docker-example/configure-sinatra-app.sh"] + }], + "post-processors": [{ + "type": "docker-tag", + "repository": "gruntwork/packer-docker-example", + "tag": "latest", + "only": ["ubuntu-docker"] + }] +} diff --git a/examples/packer-docker-example/build.pkr.hcl b/examples/packer-docker-example/build.pkr.hcl index 8b9d81ffe..4bcf88363 100644 --- a/examples/packer-docker-example/build.pkr.hcl +++ b/examples/packer-docker-example/build.pkr.hcl @@ -56,7 +56,7 @@ build { provisioner "shell" { inline = ["echo 'Sleeping for a few seconds to give Ubuntu time to boot up'", "sleep 30"] - only = ["ubuntu-ami"] + only = ["amazon-ebs.ubuntu-ami"] } provisioner "file" { @@ -69,7 +69,7 @@ build { } post-processor "docker-tag" { - only = ["ubuntu-docker"] + only = ["docker.ubuntu-docker"] repository = "gruntwork/packer-docker-example" tag = "latest" } From a166b4041046b1a6ece06348d7a522912eff9c57 Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Tue, 27 Jul 2021 21:05:49 -0700 Subject: [PATCH 21/22] Update tag to list of string --- examples/packer-docker-example/build.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/packer-docker-example/build.pkr.hcl b/examples/packer-docker-example/build.pkr.hcl index 4bcf88363..50e36495b 100644 --- a/examples/packer-docker-example/build.pkr.hcl +++ b/examples/packer-docker-example/build.pkr.hcl @@ -71,6 +71,6 @@ build { post-processor "docker-tag" { only = ["docker.ubuntu-docker"] repository = "gruntwork/packer-docker-example" - tag = "latest" + tag = ["latest"] } } From 950fddf5dc96ed85bec57e4f21aae59c818ef1f8 Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Wed, 28 Jul 2021 08:34:21 -0700 Subject: [PATCH 22/22] Incorporate review feedback --- modules/packer/packer.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/modules/packer/packer.go b/modules/packer/packer.go index 7d301568d..dd1f418ce 100644 --- a/modules/packer/packer.go +++ b/modules/packer/packer.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io/ioutil" - "log" "os" "regexp" "sync" @@ -13,6 +12,7 @@ import ( "github.com/gruntwork-io/terratest/modules/retry" "github.com/hashicorp/go-multierror" + "github.com/stretchr/testify/require" "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/shell" @@ -106,9 +106,7 @@ func BuildArtifactE(t testing.TestingT, options *Options) (string, error) { const packerPluginPathEnvVar = "PACKER_PLUGIN_PATH" options.Logger.Logf(t, "Creating a temporary directory for Packer plugins") pluginDir, err := ioutil.TempDir("", "terratest-packer-") - if err != nil { - log.Fatal(err) - } + require.NoError(t, err) if len(options.Env) == 0 { options.Env = make(map[string]string) } @@ -189,14 +187,10 @@ func hasPackerInit(t testing.TestingT, options *Options) (bool, error) { Env: options.Env, WorkingDir: options.WorkingDir, } - description := fmt.Sprintf("%s %v", cmd.Command, cmd.Args) - localVersion, err := retry.DoWithRetryableErrorsE(t, description, options.RetryableErrors, options.MaxRetries, options.TimeBetweenRetries, func() (string, error) { - return shell.RunCommandAndGetOutputE(t, cmd) - }) + localVersion, err := shell.RunCommandAndGetOutputE(t, cmd) if err != nil { return false, err } - thisVersion, err := version.NewVersion(localVersion) if err != nil { return false, err @@ -220,8 +214,6 @@ func packerInit(t testing.TestingT, options *Options) error { return nil } - options.Logger.Logf(t, "Running Packer init") - cmd := shell.Command{ Command: "packer", Args: []string{"init", options.Template}, @@ -229,7 +221,7 @@ func packerInit(t testing.TestingT, options *Options) error { WorkingDir: options.WorkingDir, } - description := fmt.Sprintf("%s %v", cmd.Command, cmd.Args) + description := "Running Packer init" _, err = retry.DoWithRetryableErrorsE(t, description, options.RetryableErrors, options.MaxRetries, options.TimeBetweenRetries, func() (string, error) { return shell.RunCommandAndGetOutputE(t, cmd) })