From cff1cb8939e568fc9feb421dd29429859c7a0276 Mon Sep 17 00:00:00 2001 From: Brian Flad Date: Thu, 31 Aug 2023 05:44:39 -0400 Subject: [PATCH] tfexec: Initial test command support (#400) Reference: https://github.com/hashicorp/terraform-exec/issues/398 Reference: https://github.com/hashicorp/terraform/pull/33454 Adds support for the `terraform test` command, which currently supports JSON machine-readable output and one flag for configuring the tests directory away from the command default of `tests`. The command will return a non-zero status if any of the tests fail, which returns an error back to callers of the `Test` function. If consumers need access to the pass/fail test results, the terraform-json Go module will need to be enhanced to support the test summary JSON, e.g. ``` {"@level":"info","@message":"Failure! 0 passed, 1 failed.","@module":"terraform.ui","@timestamp":"2023-07-25T10:03:42.980799-04:00","test_summary":{"status":"fail","passed":0,"failed":1,"errored":0,"skipped":0},"type":"test_summary"} ``` Output of new end-to-end testing: ``` $ TFEXEC_E2ETEST_VERSIONS=1.5.3,1.6.0-alpha20230719 go test -count=1 -run='TestTest' -v ./tfexec/internal/e2etest ... --- PASS: TestTest (9.50s) --- SKIP: TestTest/test_command_passing-1.5.3 (4.06s) --- PASS: TestTest/test_command_passing-1.6.0-alpha20230719 (5.44s) ... --- PASS: TestTestError (0.48s) --- SKIP: TestTestError/test_command_failing-1.5.3 (0.27s) --- PASS: TestTestError/test_command_failing-1.6.0-alpha20230719 (0.21s) ``` --- tfexec/internal/e2etest/test_test.go | 56 ++++++++++++++++ .../testdata/test_command_failing/main.tf | 11 ++++ .../tests/passthrough.tftest | 12 ++++ .../testdata/test_command_passing/main.tf | 11 ++++ .../tests/passthrough.tftest | 12 ++++ tfexec/internal/testutil/tfcache.go | 2 + tfexec/options.go | 9 +++ tfexec/test.go | 66 +++++++++++++++++++ tfexec/test_test.go | 43 ++++++++++++ tfexec/version.go | 1 + 10 files changed, 223 insertions(+) create mode 100644 tfexec/internal/e2etest/test_test.go create mode 100644 tfexec/internal/e2etest/testdata/test_command_failing/main.tf create mode 100644 tfexec/internal/e2etest/testdata/test_command_failing/tests/passthrough.tftest create mode 100644 tfexec/internal/e2etest/testdata/test_command_passing/main.tf create mode 100644 tfexec/internal/e2etest/testdata/test_command_passing/tests/passthrough.tftest create mode 100644 tfexec/test.go create mode 100644 tfexec/test_test.go diff --git a/tfexec/internal/e2etest/test_test.go b/tfexec/internal/e2etest/test_test.go new file mode 100644 index 00000000..f925e228 --- /dev/null +++ b/tfexec/internal/e2etest/test_test.go @@ -0,0 +1,56 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package e2etest + +import ( + "context" + "io" + "regexp" + "testing" + + "github.com/hashicorp/go-version" + + "github.com/hashicorp/terraform-exec/tfexec" +) + +var ( + testMinVersion = version.Must(version.NewVersion("1.6.0")) +) + +func TestTest(t *testing.T) { + runTest(t, "test_command_passing", func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) { + // Use Core() to enable pre-release support + if tfv.Core().LessThan(testMinVersion) { + t.Skip("test command is not available in this Terraform version") + } + + err := tf.Test(context.Background(), nil) + + if err != nil { + t.Fatalf("error running test command: %s", err) + } + }) +} + +func TestTestError(t *testing.T) { + runTest(t, "test_command_failing", func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) { + // Use Core() to enable pre-release support + if tfv.Core().LessThan(testMinVersion) { + t.Skip("test command is not available in this Terraform version") + } + + err := tf.Test(context.Background(), io.Discard) + + if err == nil { + t.Fatal("expected error, got none") + } + + got := err.Error() + expected := regexp.MustCompile("exit status 1") + + if !expected.MatchString(got) { + t.Fatalf("expected error matching '%s', got: %s", expected, got) + } + }) +} diff --git a/tfexec/internal/e2etest/testdata/test_command_failing/main.tf b/tfexec/internal/e2etest/testdata/test_command_failing/main.tf new file mode 100644 index 00000000..b4652a07 --- /dev/null +++ b/tfexec/internal/e2etest/testdata/test_command_failing/main.tf @@ -0,0 +1,11 @@ +variable "test" { + type = string +} + +resource "terraform_data" "test" { + input = var.test +} + +output "test" { + value = terraform_data.test.output +} diff --git a/tfexec/internal/e2etest/testdata/test_command_failing/tests/passthrough.tftest b/tfexec/internal/e2etest/testdata/test_command_failing/tests/passthrough.tftest new file mode 100644 index 00000000..25d30260 --- /dev/null +++ b/tfexec/internal/e2etest/testdata/test_command_failing/tests/passthrough.tftest @@ -0,0 +1,12 @@ +variables { + test = "test value" +} + +run "variable_output_passthrough" { + command = apply + + assert { + condition = output.test == "not test value" # intentionally incorrect + error_message = "variable was not passed through to output" + } +} diff --git a/tfexec/internal/e2etest/testdata/test_command_passing/main.tf b/tfexec/internal/e2etest/testdata/test_command_passing/main.tf new file mode 100644 index 00000000..b4652a07 --- /dev/null +++ b/tfexec/internal/e2etest/testdata/test_command_passing/main.tf @@ -0,0 +1,11 @@ +variable "test" { + type = string +} + +resource "terraform_data" "test" { + input = var.test +} + +output "test" { + value = terraform_data.test.output +} diff --git a/tfexec/internal/e2etest/testdata/test_command_passing/tests/passthrough.tftest b/tfexec/internal/e2etest/testdata/test_command_passing/tests/passthrough.tftest new file mode 100644 index 00000000..3ebaf8d2 --- /dev/null +++ b/tfexec/internal/e2etest/testdata/test_command_passing/tests/passthrough.tftest @@ -0,0 +1,12 @@ +variables { + test = "test value" +} + +run "variable_output_passthrough" { + command = apply + + assert { + condition = output.test == "test value" + error_message = "variable was not passed through to output" + } +} diff --git a/tfexec/internal/testutil/tfcache.go b/tfexec/internal/testutil/tfcache.go index e2f87b29..ff9ceb00 100644 --- a/tfexec/internal/testutil/tfcache.go +++ b/tfexec/internal/testutil/tfcache.go @@ -22,6 +22,8 @@ const ( Latest015 = "0.15.5" Latest_v1 = "1.0.11" Latest_v1_1 = "1.1.9" + Latest_v1_5 = "1.5.3" + Latest_v1_6 = "1.6.0-alpha20230719" ) const appendUserAgent = "tfexec-testutil" diff --git a/tfexec/options.go b/tfexec/options.go index a9bade05..5cccde3e 100644 --- a/tfexec/options.go +++ b/tfexec/options.go @@ -365,6 +365,15 @@ func Target(resource string) *TargetOption { return &TargetOption{resource} } +type TestsDirectoryOption struct { + testsDirectory string +} + +// TestsDirectory represents the -tests-directory option (path to tests files) +func TestsDirectory(testsDirectory string) *TestsDirectoryOption { + return &TestsDirectoryOption{testsDirectory} +} + type GraphTypeOption struct { graphType string } diff --git a/tfexec/test.go b/tfexec/test.go new file mode 100644 index 00000000..5e0bb635 --- /dev/null +++ b/tfexec/test.go @@ -0,0 +1,66 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package tfexec + +import ( + "context" + "fmt" + "io" + "os/exec" +) + +type testConfig struct { + testsDirectory string +} + +var defaultTestOptions = testConfig{} + +type TestOption interface { + configureTest(*testConfig) +} + +func (opt *TestsDirectoryOption) configureTest(conf *testConfig) { + conf.testsDirectory = opt.testsDirectory +} + +// Test represents the terraform test -json subcommand. +// +// The given io.Writer, if specified, will receive +// [machine-readable](https://developer.hashicorp.com/terraform/internals/machine-readable-ui) +// JSON from Terraform including test results. +func (tf *Terraform) Test(ctx context.Context, w io.Writer, opts ...TestOption) error { + err := tf.compatible(ctx, tf1_6_0, nil) + + if err != nil { + return fmt.Errorf("terraform test was added in 1.6.0: %w", err) + } + + tf.SetStdout(w) + + testCmd := tf.testCmd(ctx) + + err = tf.runTerraformCmd(ctx, testCmd) + + if err != nil { + return err + } + + return nil +} + +func (tf *Terraform) testCmd(ctx context.Context, opts ...TestOption) *exec.Cmd { + c := defaultTestOptions + + for _, o := range opts { + o.configureTest(&c) + } + + args := []string{"test", "-json"} + + if c.testsDirectory != "" { + args = append(args, "-tests-directory="+c.testsDirectory) + } + + return tf.buildTerraformCmd(ctx, nil, args...) +} diff --git a/tfexec/test_test.go b/tfexec/test_test.go new file mode 100644 index 00000000..aa2b4508 --- /dev/null +++ b/tfexec/test_test.go @@ -0,0 +1,43 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package tfexec + +import ( + "context" + "testing" + + "github.com/hashicorp/terraform-exec/tfexec/internal/testutil" +) + +func TestTestCmd(t *testing.T) { + td := t.TempDir() + + tf, err := NewTerraform(td, tfVersion(t, testutil.Latest_v1_6)) + + if err != nil { + t.Fatal(err) + } + + // empty env, to avoid environ mismatch in testing + tf.SetEnv(map[string]string{}) + + t.Run("defaults", func(t *testing.T) { + testCmd := tf.testCmd(context.Background()) + + assertCmd(t, []string{ + "test", + "-json", + }, nil, testCmd) + }) + + t.Run("override all defaults", func(t *testing.T) { + testCmd := tf.testCmd(context.Background(), TestsDirectory("test")) + + assertCmd(t, []string{ + "test", + "-json", + "-tests-directory=test", + }, nil, testCmd) + }) +} diff --git a/tfexec/version.go b/tfexec/version.go index ec4d5221..62332122 100644 --- a/tfexec/version.go +++ b/tfexec/version.go @@ -31,6 +31,7 @@ var ( tf0_15_3 = version.Must(version.NewVersion("0.15.3")) tf1_1_0 = version.Must(version.NewVersion("1.1.0")) tf1_4_0 = version.Must(version.NewVersion("1.4.0")) + tf1_6_0 = version.Must(version.NewVersion("1.6.0")) ) // Version returns structured output from the terraform version command including both the Terraform CLI version