Allow testing against refs in addition to releases

.circleci/config.yml

@@ -101,6 +101,7 @@ jobs:
 
 workflows:
   version: 2
+  # TODO: add a `nightly` run for checking against TF master branch using TFEXEC_E2ETEST_VERSIONS
   pr:
     jobs:
       - winbuild

tfexec/internal/e2etest/util_test.go

@@ -3,6 +3,7 @@ package e2etest
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -18,29 +19,34 @@ import (
 )
 
 const testFixtureDir = "testdata"
+const masterRef = "refs/heads/master"
 
 func runTest(t *testing.T, fixtureName string, cb func(t *testing.T, tfVersion *version.Version, tf *tfexec.Terraform)) {
 	t.Helper()
-	runTestVersions(t, []string{
+
+	versions := []string{
 		testutil.Latest011,
 		testutil.Latest012,
 		testutil.Latest013,
-	}, fixtureName, cb)
+	}
+	if override := os.Getenv("TFEXEC_E2ETEST_VERSIONS"); override != "" {
+		versions = strings.Split(override, ",")
+	}
+
+	runTestVersions(t, versions, fixtureName, cb)
 }
 
 func runTestVersions(t *testing.T, versions []string, fixtureName string, cb func(t *testing.T, tfVersion *version.Version, tf *tfexec.Terraform)) {
 	t.Helper()
 
-	// TODO: if overriding latest for master code, skip all other tests
-
 	alreadyRunVersions := map[string]bool{}
 	for _, tfv := range versions {
-		if alreadyRunVersions[tfv] {
-			t.Skipf("already run version %q", tfv)
-		}
-		alreadyRunVersions[tfv] = true
-
 		t.Run(fmt.Sprintf("%s-%s", fixtureName, tfv), func(t *testing.T) {
+			if alreadyRunVersions[tfv] {
+				t.Skipf("already run version %q", tfv)
+			}
+			alreadyRunVersions[tfv] = true
+
 			td, err := ioutil.TempDir("", "tf")
 			if err != nil {
 				t.Fatalf("error creating temporary test directory: %s", err)
@@ -49,11 +55,25 @@ func runTestVersions(t *testing.T, versions []string, fixtureName string, cb fun
 				os.RemoveAll(td)
 			})
 
-			tf, err := tfexec.NewTerraform(td, tfcache.Version(t, tfv))
+			// TODO: do this in a cleaner way than string comparison?
+			var execPath string
+			switch {
+			case strings.HasPrefix(tfv, "refs/"):
+				execPath = tfcache.GitRef(t, tfv)
+			default:
+				execPath = tfcache.Version(t, tfv)
+			}
+
+			tf, err := tfexec.NewTerraform(td, execPath)
 			if err != nil {
 				t.Fatal(err)
 			}
 
+			runningVersion, _, err := tf.Version(context.Background(), false)
+			if err != nil {
+				t.Fatalf("unable to determin running version (expected %q): %s", tfv, err)
+			}
+
 			if fixtureName != "" {
 				err = copyFiles(filepath.Join(testFixtureDir, fixtureName), td)
 				if err != nil {
@@ -68,7 +88,7 @@ func runTestVersions(t *testing.T, versions []string, fixtureName string, cb fun
 			tf.SetLogger(&testingPrintfer{t})
 
 			// TODO: capture panics here?
-			cb(t, version.Must(version.NewVersion(tfv)), tf)
+			cb(t, runningVersion, tf)
 
 			t.Logf("CLI Output:\n%s", stdouterr.String())
 		})

tfexec/internal/testutil/tfcache.go

@@ -2,8 +2,10 @@ package testutil
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 	"testing"
 
@@ -19,42 +21,65 @@ const (
 type TFCache struct {
 	sync.Mutex
 
-	dir      string
-	versions map[string]string
+	dir   string
+	execs map[string]string
 }
 
 func NewTFCache(dir string) *TFCache {
 	return &TFCache{
-		dir:      dir,
-		versions: map[string]string{},
+		dir:   dir,
+		execs: map[string]string{},
 	}
 }
 
+func (tf *TFCache) GitRef(t *testing.T, ref string) string {
+	t.Helper()
+	return tf.find(t, "gitref:"+ref, func(dir string) tfinstall.ExecPathFinder {
+		return tfinstall.GitRef(ref, "", dir)
+	})
+}
+
 func (tf *TFCache) Version(t *testing.T, v string) string {
 	t.Helper()
+	return tf.find(t, "v:"+v, func(dir string) tfinstall.ExecPathFinder {
+		return tfinstall.ExactVersion(v, dir)
+	})
+}
+
+func (tf *TFCache) find(t *testing.T, key string, finder func(dir string) tfinstall.ExecPathFinder) string {
+
+	t.Helper()
 
 	if tf.dir == "" {
-		t.Fatalf("installDir not yet configured")
+		// panic instead of t.fatal as this is going to affect all downstream tests reusing the cache entry
+		panic("installDir not yet configured")
 	}
 
 	tf.Lock()
 	defer tf.Unlock()
 
-	path, ok := tf.versions[v]
+	path, ok := tf.execs[key]
 	if !ok {
-		t.Logf("caching version %s", v)
+		keyDir := key
+		keyDir = strings.ReplaceAll(keyDir, ":", "-")
+		keyDir = strings.ReplaceAll(keyDir, "/", "-")
+
+		dir := filepath.Join(tf.dir, keyDir)
+
+		t.Logf("caching exec %q in dir %q", key, dir)
 
-		dir := filepath.Join(tf.dir, v)
 		err := os.MkdirAll(dir, 0777)
 		if err != nil {
-			t.Fatal(err)
+			// panic instead of t.fatal as this is going to affect all downstream tests reusing the cache entry
+			panic(fmt.Sprintf("unable to mkdir %q: %s", dir, err))
 		}
 
-		path, err = tfinstall.Find(context.Background(), tfinstall.ExactVersion(v, dir))
+		path, err = tfinstall.Find(context.Background(), finder(dir))
 		if err != nil {
-			t.Fatalf("error installing terraform version %q: %s", v, err)
+			// panic instead of t.fatal as this is going to affect all downstream tests reusing the cache entry
+			panic(fmt.Sprintf("error installing terraform %q: %s", key, err))
 		}
-		tf.versions[v] = path
+		tf.execs[key] = path
 	}
 
 	return path