From 961b446d6e843cbe47e48a69eab01cc96fd1b3b1 Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Mon, 1 Feb 2021 12:59:14 +0000 Subject: [PATCH 1/6] Add pause_before to packer-basic-example Without this, we sometimes get build failures such as: ``` TestBuildPackerArtifactWithGit/ref-sha 2021-01-29T15:45:17Z logger.go:66: 1611935117,,ui,message, amazon-ebs: Reading package lists... TestBuildPackerArtifactWithGit/ref-sha 2021-01-29T15:45:17Z logger.go:66: 1611935117,,ui,message, amazon-ebs: Reading package lists... TestBuildPackerArtifactWithGit/ref-sha 2021-01-29T15:45:17Z logger.go:66: 1611935117,,ui,message, amazon-ebs: Building dependency tree... TestBuildPackerArtifactWithGit/ref-sha 2021-01-29T15:45:17Z logger.go:66: 1611935117,,ui,message, amazon-ebs: Reading state information... TestBuildPackerArtifactWithGit/ref-sha 2021-01-29T15:45:17Z logger.go:66: 1611935117,,ui,message, amazon-ebs: Calculating upgrade... TestBuildPackerArtifactWithGit/ref-sha 2021-01-29T15:45:17Z logger.go:66: 1611935117,,ui,message, amazon-ebs: The following packages have been kept back: TestBuildPackerArtifactWithGit/ref-sha 2021-01-29T15:45:17Z logger.go:66: 1611935117,,ui,message, amazon-ebs: linux-aws linux-headers-aws linux-image-aws TestBuildPackerArtifactWithGit/ref-sha 2021-01-29T15:45:18Z logger.go:66: 1611935118,,ui,message, amazon-ebs: The following packages will be upgraded: TestBuildPackerArtifactWithGit/ref-sha 2021-01-29T15:45:18Z logger.go:66: 1611935118,,ui,message, amazon-ebs: sudo tzdata TestBuildPackerArtifactWithGit/ref-sha 2021-01-29T15:45:18Z logger.go:66: 1611935118,,ui,error,==> amazon-ebs: E: Could not open file /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_xenial-backports_universe_i18n_Translation-en - open (2: No such file or directory) TestBuildPackerArtifactWithGit/ref-sha 2021-01-29T15:45:18Z logger.go:66: 1611935118,,ui,error,==> amazon-ebs: E: Could not open file /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_xenial-backports_universe_binary-amd64_Packages - open (2: No such file or directory) TestBuildPackerArtifactWithGit/ref-sha 2021-01-29T15:45:18Z logger.go:66: 1611935118,,ui,message, amazon-ebs: 2 upgraded%!!(MISSING)(PACKER_COMMA) 0 newly installed%!!(MISSING)(PACKER_COMMA) 0 to remove and 3 not upgraded. TestBuildPackerArtifactWithGit/ref-sha 2021-01-29T15:45:18Z logger.go:66: 1611935118,,ui,error,==> amazon-ebs: E: Could not open file /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_xenial-backports_main_i18n_Translation-en - open (2: No such file or directory) TestBuildPackerArtifactWithGit/ref-sha 2021-01-29T15:45:18Z logger.go:66: 1611935118,,ui,error,==> amazon-ebs: E: Could not open file /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_xenial-backports_main_binary-amd64_Packages - open (2: No such file or directory) TestBuildPackerArtifactWithGit/ref-sha 2021-01-29T15:45:18Z logger.go:66: 1611935118,,ui,error,==> amazon-ebs: E: Could not open file ``` I _think_ this is due to the issue where you run `apt` too soon after boot... --- examples/packer-basic-example/build.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/packer-basic-example/build.json b/examples/packer-basic-example/build.json index bb3d0e67d..d479d2b70 100644 --- a/examples/packer-basic-example/build.json +++ b/examples/packer-basic-example/build.json @@ -54,6 +54,7 @@ "inline": [ "sudo DEBIAN_FRONTEND=noninteractive apt-get update", "sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y" - ] + ], + "pause_before": "30s" }] } From 4020591f7228f934375c2f1df83c64430a322ee5 Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Mon, 1 Feb 2021 13:30:38 +0000 Subject: [PATCH 2/6] Switch to terraform-google-ci for gcp-helpers module --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2cc2edf60..b57afa0eb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,7 +35,7 @@ install_gruntwork_utils: &install_gruntwork_utils curl -Ls https://raw.githubusercontent.com/gruntwork-io/gruntwork-installer/master/bootstrap-gruntwork-installer.sh | bash /dev/stdin --version "${GRUNTWORK_INSTALLER_VERSION}" gruntwork-install --module-name "gruntwork-module-circleci-helpers" --repo "https://github.com/gruntwork-io/module-ci" --tag "${MODULE_CI_VERSION}" gruntwork-install --module-name "kubernetes-circleci-helpers" --repo "https://github.com/gruntwork-io/module-ci" --tag "${MODULE_CI_VERSION}" - gruntwork-install --module-name "gcp-helpers" --repo "https://github.com/gruntwork-io/module-ci" --branch "gcp-helpers" + gruntwork-install --module-name "gcp-helpers" --repo "https://github.com/gruntwork-io/terraform-google-ci" --branch "gcp-helpers" configure-environment-for-gruntwork-module \ --terraform-version ${TERRAFORM_VERSION} \ --terragrunt-version ${TERRAGRUNT_VERSION} \ From c41f16e9930ca27c433434f61633f2482853ff85 Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Mon, 1 Feb 2021 14:11:07 +0000 Subject: [PATCH 3/6] Bump gruntwork-installer version --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b57afa0eb..ff5509587 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ env: &env environment: - GRUNTWORK_INSTALLER_VERSION: v0.0.30 + GRUNTWORK_INSTALLER_VERSION: v0.0.31 MODULE_CI_VERSION: v0.29.2 TERRAFORM_VERSION: 0.14.0 TERRAGRUNT_VERSION: v0.24.2 @@ -35,7 +35,10 @@ install_gruntwork_utils: &install_gruntwork_utils curl -Ls https://raw.githubusercontent.com/gruntwork-io/gruntwork-installer/master/bootstrap-gruntwork-installer.sh | bash /dev/stdin --version "${GRUNTWORK_INSTALLER_VERSION}" gruntwork-install --module-name "gruntwork-module-circleci-helpers" --repo "https://github.com/gruntwork-io/module-ci" --tag "${MODULE_CI_VERSION}" gruntwork-install --module-name "kubernetes-circleci-helpers" --repo "https://github.com/gruntwork-io/module-ci" --tag "${MODULE_CI_VERSION}" + + # TODO: Switch to a release version when https://github.com/gruntwork-io/terraform-google-ci/pull/10 is merged!! gruntwork-install --module-name "gcp-helpers" --repo "https://github.com/gruntwork-io/terraform-google-ci" --branch "gcp-helpers" + configure-environment-for-gruntwork-module \ --terraform-version ${TERRAFORM_VERSION} \ --terragrunt-version ${TERRAGRUNT_VERSION} \ From deda55a1f9ccb4ad0d50f2c9398426b8360cac67 Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Thu, 4 Feb 2021 10:08:56 +0000 Subject: [PATCH 4/6] Update to latest gruntwork installer version This pulls in the latest version of fetch, which has a fix for the --branch param. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ff5509587..3a6f6802c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ env: &env environment: - GRUNTWORK_INSTALLER_VERSION: v0.0.31 + GRUNTWORK_INSTALLER_VERSION: v0.0.33 MODULE_CI_VERSION: v0.29.2 TERRAFORM_VERSION: 0.14.0 TERRAGRUNT_VERSION: v0.24.2 From 033a20f04592a2eaa0947b8c7d0bc8b021d5c3c4 Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Thu, 4 Feb 2021 11:07:28 +0000 Subject: [PATCH 5/6] Pick available instance type for Packer builds --- examples/packer-basic-example/build.json | 5 +++-- examples/packer-docker-example/build.json | 5 +++-- test/packer_basic_example_test.go | 15 +++++++++++++-- test/terraform_packer_example_test.go | 6 +++++- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/examples/packer-basic-example/build.json b/examples/packer-basic-example/build.json index d479d2b70..97c5bc8c6 100644 --- a/examples/packer-basic-example/build.json +++ b/examples/packer-basic-example/build.json @@ -9,13 +9,14 @@ "oci_base_image_ocid": "", "oci_compartment_ocid": "", "oci_pass_phrase": "", - "oci_subnet_ocid": "" + "oci_subnet_ocid": "", + "instance_type": "t2.micro" }, "builders": [{ "type": "amazon-ebs", "ami_name": "{{user `ami_base_name`}}-terratest-packer-example", "ami_description": "An example of how to create a custom AMI on top of Ubuntu", - "instance_type": "t2.micro", + "instance_type": "{{user `instance_type`}}", "region": "{{user `aws_region`}}", "source_ami_filter": { "filters": { diff --git a/examples/packer-docker-example/build.json b/examples/packer-docker-example/build.json index b038715ed..fde5ab0c3 100644 --- a/examples/packer-docker-example/build.json +++ b/examples/packer-docker-example/build.json @@ -1,13 +1,14 @@ { "variables": { "aws_region": "us-east-1", - "ami_name_base": "terratest-packer-docker-example" + "ami_name_base": "terratest-packer-docker-example", + "instance_type": "t2.micro" }, "builders": [{ "name": "ubuntu-ami", "ami_name": "{{user `ami_name_base`}}-{{isotime | clean_resource_name}}", "ami_description": "An example of how to create a custom AMI with a simple web app on top of Ubuntu", - "instance_type": "t2.micro", + "instance_type": "{{user `instance_type`}}", "region": "{{user `aws_region`}}", "type": "amazon-ebs", "source_ami_filter": { diff --git a/test/packer_basic_example_test.go b/test/packer_basic_example_test.go index 2c4085345..418177e7a 100644 --- a/test/packer_basic_example_test.go +++ b/test/packer_basic_example_test.go @@ -34,6 +34,9 @@ func TestPackerBasicExample(t *testing.T) { // Pick a random AWS region to test in. This helps ensure your code works in all regions. awsRegion := terratest_aws.GetRandomStableRegion(t, nil, nil) + // Some AWS regions are missing certain instance types, so pick an available type based on the region we picked + instanceType := terratest_aws.GetRecommendedInstanceType(t, awsRegion, []string{"t2.micro", "t3.micro"}) + // website::tag::1::Read Packer's template and set AWS Region variable. packerOptions := &packer.Options{ // The path to where the Packer template is located @@ -43,6 +46,7 @@ func TestPackerBasicExample(t *testing.T) { Vars: map[string]string{ "aws_region": awsRegion, "ami_base_name": fmt.Sprintf("%s", random.UniqueId()), + "instance_type": instanceType, }, // Only build the AWS AMI @@ -87,6 +91,9 @@ func TestPackerBasicExampleWithVarFile(t *testing.T) { // Pick a random AWS region to test in. This helps ensure your code works in all regions. awsRegion := terratest_aws.GetRandomStableRegion(t, nil, nil) + // Some AWS regions are missing certain instance types, so pick an available type based on the region we picked + instanceType := terratest_aws.GetRecommendedInstanceType(t, awsRegion, []string{"t2.micro", "t3.micro"}) + // Create temporary packer variable file to store aws region varFile, err := ioutil.TempFile("", "*.json") require.NoError(t, err, "Did not expect temp file creation to cause error") @@ -94,8 +101,8 @@ func TestPackerBasicExampleWithVarFile(t *testing.T) { // Be sure to clean up temp file defer os.Remove(varFile.Name()) - // Write random generated aws region to temporary json file - varFileContent := []byte(fmt.Sprintf("{ \"aws_region\": \"%s\" }", awsRegion)) + // Write the vars we need to a temporary json file + varFileContent := []byte(fmt.Sprintf(`{"aws_region": "%s", "instance_type": "%s"}`, awsRegion, instanceType)) _, err = varFile.Write(varFileContent) require.NoError(t, err, "Did not expect writing to temp file %s to cause error", varFile.Name()) @@ -149,6 +156,9 @@ func TestPackerMultipleConcurrentAmis(t *testing.T) { // Pick a random AWS region to test in. This helps ensure your code works in all regions. awsRegion := terratest_aws.GetRandomStableRegion(t, nil, nil) + // Some AWS regions are missing certain instance types, so pick an available type based on the region we picked + instanceType := terratest_aws.GetRecommendedInstanceType(t, awsRegion, []string{"t2.micro", "t3.micro"}) + packerOptions := &packer.Options{ // The path to where the Packer template is located Template: "../examples/packer-basic-example/build.json", @@ -157,6 +167,7 @@ func TestPackerMultipleConcurrentAmis(t *testing.T) { Vars: map[string]string{ "aws_region": awsRegion, "ami_base_name": fmt.Sprintf("%s", random.UniqueId()), + "instance_type": instanceType, }, // Only build the AWS AMI diff --git a/test/terraform_packer_example_test.go b/test/terraform_packer_example_test.go index b9431d1ba..c3d50433f 100644 --- a/test/terraform_packer_example_test.go +++ b/test/terraform_packer_example_test.go @@ -66,6 +66,9 @@ func TestTerraformPackerExample(t *testing.T) { // Build the AMI in packer-docker-example func buildAMI(t *testing.T, awsRegion string, workingDir string) { + // Some AWS regions are missing certain instance types, so pick an available type based on the region we picked + instanceType := aws.GetRecommendedInstanceType(t, awsRegion, []string{"t2.micro", "t3.micro"}) + packerOptions := &packer.Options{ // The path to where the Packer template is located Template: "../examples/packer-docker-example/build.json", @@ -75,7 +78,8 @@ func buildAMI(t *testing.T, awsRegion string, workingDir string) { // Variables to pass to our Packer build using -var options Vars: map[string]string{ - "aws_region": awsRegion, + "aws_region": awsRegion, + "instance_type": instanceType, }, // Configure retries for intermittent errors From dc3c93a6b8fe7a561f633b45b7c959819918e20d Mon Sep 17 00:00:00 2001 From: Yevgeniy Brikman Date: Thu, 4 Feb 2021 12:25:13 +0000 Subject: [PATCH 6/6] Add more retries for more transient provider issues --- modules/terraform/apply_test.go | 36 ++++++++++++++++----------------- modules/terraform/options.go | 1 + 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/modules/terraform/apply_test.go b/modules/terraform/apply_test.go index d838413cc..1e3288bc1 100644 --- a/modules/terraform/apply_test.go +++ b/modules/terraform/apply_test.go @@ -15,10 +15,10 @@ func TestApplyNoError(t *testing.T) { testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-no-error", t.Name()) require.NoError(t, err) - options := &Options{ + options := WithDefaultRetryableErrors(t, &Options{ TerraformDir: testFolder, NoColor: true, - } + }) out := InitAndApply(t, options) @@ -34,9 +34,9 @@ func TestApplyWithErrorNoRetry(t *testing.T) { testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-with-error", t.Name()) require.NoError(t, err) - options := &Options{ + options := WithDefaultRetryableErrors(t, &Options{ TerraformDir: testFolder, - } + }) out, err := InitAndApplyE(t, options) @@ -50,13 +50,13 @@ func TestApplyWithErrorWithRetry(t *testing.T) { testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-with-error", t.Name()) require.NoError(t, err) - options := &Options{ + options := WithDefaultRetryableErrors(t, &Options{ TerraformDir: testFolder, MaxRetries: 1, RetryableTerraformErrors: map[string]string{ "This is the first run, exiting with an error": "Intentional failure in test fixture", }, - } + }) out := InitAndApply(t, options) @@ -68,10 +68,10 @@ func TestTgApplyAllTgError(t *testing.T) { testFolder, err := files.CopyTerragruntFolderToTemp("../../test/fixtures/terragrunt/terragrunt-no-error", t.Name()) require.NoError(t, err) - options := &Options{ + options := WithDefaultRetryableErrors(t, &Options{ TerraformDir: testFolder, TerraformBinary: "terragrunt", - } + }) out := TgApplyAll(t, options) @@ -84,14 +84,14 @@ func TestTgApplyAllError(t *testing.T) { testFolder, err := files.CopyTerragruntFolderToTemp("../../test/fixtures/terragrunt/terragrunt-with-error", t.Name()) require.NoError(t, err) - options := &Options{ + options := WithDefaultRetryableErrors(t, &Options{ TerraformDir: testFolder, TerraformBinary: "terragrunt", MaxRetries: 1, RetryableTerraformErrors: map[string]string{ "This is the first run, exiting with an error": "Intentional failure in test fixture", }, - } + }) out := TgApplyAll(t, options) @@ -101,10 +101,10 @@ func TestTgApplyAllError(t *testing.T) { func TestTgApplyOutput(t *testing.T) { t.Parallel() - options := &Options{ + options := WithDefaultRetryableErrors(t, &Options{ TerraformDir: "../../test/fixtures/terragrunt/terragrunt-output", TerraformBinary: "terragrunt", - } + }) Apply(t, options) @@ -127,10 +127,10 @@ func TestIdempotentNoChanges(t *testing.T) { testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-no-error", t.Name()) require.NoError(t, err) - options := &Options{ + options := WithDefaultRetryableErrors(t, &Options{ TerraformDir: testFolder, NoColor: true, - } + }) InitAndApplyAndIdempotentE(t, options) } @@ -141,10 +141,10 @@ func TestIdempotentWithChanges(t *testing.T) { testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-not-idempotent", t.Name()) require.NoError(t, err) - options := &Options{ + options := WithDefaultRetryableErrors(t, &Options{ TerraformDir: testFolder, NoColor: true, - } + }) out, err := InitAndApplyAndIdempotentE(t, options) @@ -159,10 +159,10 @@ func TestParallelism(t *testing.T) { testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-parallelism", t.Name()) require.NoError(t, err) - options := &Options{ + options := WithDefaultRetryableErrors(t, &Options{ TerraformDir: testFolder, NoColor: true, - } + }) Init(t, options) diff --git a/modules/terraform/options.go b/modules/terraform/options.go index 596ab398d..048e9b4a1 100644 --- a/modules/terraform/options.go +++ b/modules/terraform/options.go @@ -27,6 +27,7 @@ var ( ".*Failed to query available provider packages.*": "Failed to retrieve plugin due to transient network error.", ".*timeout while waiting for plugin to start.*": "Failed to retrieve plugin due to transient network error.", ".*timed out waiting for server handshake.*": "Failed to retrieve plugin due to transient network error.", + "could not query provider registry for": "Failed to retrieve plugin due to transient network error.", // Provider bugs where the data after apply is not propagated. This is usually an eventual consistency issue, so // retrying should self resolve it.