command/init: Read, respect, and update provider dependency locks

This changes the approach used by the provider installer to remember between runs which selections it has previously made, using the lock file format implemented in internal/depsfile. This means that version constraints in the configuration are considered only for providers we've not seen before or when -upgrade mode is active.
hashicorp · Oct 9, 2020 · b3f5c7f · b3f5c7f
1 parent 4a1b081
commit b3f5c7f
Show file tree

Hide file tree

Showing 20 changed files with 800 additions and 519 deletions.
diff --git a/command/e2etest/init_test.go b/command/e2etest/init_test.go
@@ -44,8 +44,8 @@ func TestInitProviders(t *testing.T) {
 		t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)")
 	}
 
-	if !strings.Contains(stdout, "* hashicorp/template: version = ") {
-		t.Errorf("provider pinning recommendation is missing from output:\n%s", stdout)
+	if !strings.Contains(stdout, "Terraform has created a lock file") {
+		t.Errorf("lock file notification is missing from output:\n%s", stdout)
 	}
 
 }

diff --git a/command/init.go b/command/init.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"sort"
 	"strings"
 
 	"github.com/hashicorp/hcl/v2"
@@ -422,9 +421,9 @@ the backend configuration is present and valid.
 func (c *InitCommand) getProviders(config *configs.Config, state *states.State, upgrade bool, pluginDirs []string) (output, abort bool, diags tfdiags.Diagnostics) {
 	// First we'll collect all the provider dependencies we can see in the
 	// configuration and the state.
-	reqs, moreDiags := config.ProviderRequirements()
-	diags = diags.Append(moreDiags)
-	if moreDiags.HasErrors() {
+	reqs, hclDiags := config.ProviderRequirements()
+	diags = diags.Append(hclDiags)
+	if hclDiags.HasErrors() {
 		return false, true, diags
 	}
 	if state != nil {
@@ -444,6 +443,10 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 			))
 		}
 	}
+
+	previousLocks, moreDiags := c.lockedDependencies()
+	diags = diags.Append(moreDiags)
+
 	if diags.HasErrors() {
 		return false, true, diags
 	}
@@ -729,7 +732,7 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 	ctx, done := c.InterruptibleContext()
 	defer done()
 	ctx = evts.OnContext(ctx)
-	selected, err := inst.EnsureProviderVersions(ctx, reqs, mode)
+	newLocks, err := inst.EnsureProviderVersions(ctx, previousLocks, reqs, mode)
 	if ctx.Err() == context.Canceled {
 		c.showDiagnostics(diags)
 		c.Ui.Error("Provider installation was canceled by an interrupt signal.")
@@ -746,29 +749,41 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 		return true, true, diags
 	}
 
-	// If any providers have "floating" versions (completely unconstrained)
-	// we'll suggest the user constrain with a pessimistic constraint to
-	// avoid implicitly adopting a later major release.
-	constraintSuggestions := make(map[string]string)
-	for addr, version := range selected {
-		req := reqs[addr]
-
-		if len(req) == 0 {
-			constraintSuggestions[addr.ForDisplay()] = "~> " + version.String()
+	// If the provider dependencies have changed since the last run then we'll
+	// say a little about that in case the reader wasn't expecting a change.
+	// (When we later integrate module dependencies into the lock file we'll
+	// probably want to refactor this so that we produce one lock-file related
+	// message for all changes together, but this is here for now just because
+	// it's the smallest change relative to what came before it, which was
+	// a hidden JSON file specifically for tracking providers.)
+	if !newLocks.Equal(previousLocks) {
+		if previousLocks.Empty() {
+			// A change from empty to non-empty is special because it suggests
+			// we're running "terraform init" for the first time against a
+			// new configuration. In that case we'll take the opportunity to
+			// say a little about what the dependency lock file is, for new
+			// users or those who are upgrading from a previous Terraform
+			// version that didn't have dependency lock files.
+			c.Ui.Output(c.Colorize().Color(`
+Terraform has created a lock file [bold].terraform.lock.hcl[reset] to record the provider
+selections it made above. Include this file in your version control repository
+so that Terraform can guarantee to make the same selections by default when
+you run "terraform init" in the future.`))
+		} else {
+			c.Ui.Output(c.Colorize().Color(`
+Terraform has made some changes to the provider dependency selections recorded
+in the .terraform.lock.hcl file. Review those changes and commit them to your
+version control system if they represent changes you intended to make.`))
 		}
 	}
-	if len(constraintSuggestions) != 0 {
-		names := make([]string, 0, len(constraintSuggestions))
-		for name := range constraintSuggestions {
-			names = append(names, name)
-		}
-		sort.Strings(names)
 
-		c.Ui.Output(outputInitProvidersUnconstrained)
-		for _, name := range names {
-			c.Ui.Output(fmt.Sprintf("* %s: version = %q", name, constraintSuggestions[name]))
-		}
-	}
+	// TODO: Check whether newLocks is different from previousLocks and mention
+	// in the UI if so. We should emit a different message if previousLocks was
+	// empty, because that indicates we were creating a lock file for the first
+	// time and so we need to introduce the user to the idea of it.
+
+	moreDiags = c.replaceLockedDependencies(newLocks)
+	diags = diags.Append(moreDiags)
 
 	return true, false, diags
 }

diff --git a/command/init_test.go b/command/init_test.go
@@ -1,6 +1,7 @@
 package command
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -20,6 +21,7 @@ import (
 	"github.com/hashicorp/terraform/addrs"
 	"github.com/hashicorp/terraform/configs"
 	"github.com/hashicorp/terraform/configs/configschema"
+	"github.com/hashicorp/terraform/internal/depsfile"
 	"github.com/hashicorp/terraform/internal/getproviders"
 	"github.com/hashicorp/terraform/internal/providercache"
 	"github.com/hashicorp/terraform/states"
@@ -1126,13 +1128,13 @@ func TestInit_getProviderInvalidPackage(t *testing.T) {
 	}
 
 	wantErrors := []string{
-		"Failed to validate installed provider",
+		"Failed to install provider",
 		"could not find executable file starting with terraform-provider-package",
 	}
 	got := ui.ErrorWriter.String()
 	for _, wantError := range wantErrors {
 		if !strings.Contains(got, wantError) {
-			t.Fatalf("missing error:\nwant: %q\n got: %q", wantError, got)
+			t.Fatalf("missing error:\nwant: %q\ngot:\n%s", wantError, got)
 		}
 	}
 }
@@ -1267,29 +1269,38 @@ func TestInit_providerSource(t *testing.T) {
 		t.Errorf("wrong cache directory contents after upgrade\n%s", diff)
 	}
 
-	inst := m.providerInstaller()
-	gotSelected, err := inst.SelectedPackages()
+	locks, err := m.lockedDependencies()
 	if err != nil {
-		t.Fatalf("failed to get selected packages from installer: %s", err)
-	}
-	wantSelected := map[addrs.Provider]*providercache.CachedProvider{
-		addrs.NewDefaultProvider("test-beta"): {
-			Provider:   addrs.NewDefaultProvider("test-beta"),
-			Version:    getproviders.MustParseVersion("1.2.4"),
-			PackageDir: expectedPackageInstallPath("test-beta", "1.2.4", false),
-		},
-		addrs.NewDefaultProvider("test"): {
-			Provider:   addrs.NewDefaultProvider("test"),
-			Version:    getproviders.MustParseVersion("1.2.3"),
-			PackageDir: expectedPackageInstallPath("test", "1.2.3", false),
-		},
-		addrs.NewDefaultProvider("source"): {
-			Provider:   addrs.NewDefaultProvider("source"),
-			Version:    getproviders.MustParseVersion("1.2.3"),
-			PackageDir: expectedPackageInstallPath("source", "1.2.3", false),
-		},
+		t.Fatalf("failed to get locked dependencies: %s", err)
+	}
+	gotProviderLocks := locks.AllProviders()
+	wantProviderLocks := map[addrs.Provider]*depsfile.ProviderLock{
+		addrs.NewDefaultProvider("test-beta"): depsfile.NewProviderLock(
+			addrs.NewDefaultProvider("test-beta"),
+			getproviders.MustParseVersion("1.2.4"),
+			getproviders.MustParseVersionConstraints("= 1.2.4"),
+			[]getproviders.Hash{
+				getproviders.HashScheme1.New("see6W06w09Ea+AobFJ+mbvPTie6ASqZAAdlFZbs8BSM="),
+			},
+		),
+		addrs.NewDefaultProvider("test"): depsfile.NewProviderLock(
+			addrs.NewDefaultProvider("test"),
+			getproviders.MustParseVersion("1.2.3"),
+			getproviders.MustParseVersionConstraints("= 1.2.3"),
+			[]getproviders.Hash{
+				getproviders.HashScheme1.New("wlbEC2mChQZ2hhgUhl6SeVLPP7fMqOFUZAQhQ9GIIno="),
+			},
+		),
+		addrs.NewDefaultProvider("source"): depsfile.NewProviderLock(
+			addrs.NewDefaultProvider("source"),
+			getproviders.MustParseVersion("1.2.3"),
+			getproviders.MustParseVersionConstraints("= 1.2.3"),
+			[]getproviders.Hash{
+				getproviders.HashScheme1.New("myS3qb3px3tRBq1ZWRYJeUH+kySWpBc0Yy8rw6W7/p4="),
+			},
+		),
 	}
-	if diff := cmp.Diff(wantSelected, gotSelected); diff != "" {
+	if diff := cmp.Diff(gotProviderLocks, wantProviderLocks, depsfile.ProviderLockComparer); diff != "" {
 		t.Errorf("wrong version selections after upgrade\n%s", diff)
 	}
 
@@ -1435,32 +1446,40 @@ func TestInit_getUpgradePlugins(t *testing.T) {
 		t.Errorf("wrong cache directory contents after upgrade\n%s", diff)
 	}
 
-	inst := m.providerInstaller()
-	gotSelected, err := inst.SelectedPackages()
+	locks, err := m.lockedDependencies()
 	if err != nil {
-		t.Fatalf("failed to get selected packages from installer: %s", err)
-	}
-	wantSelected := map[addrs.Provider]*providercache.CachedProvider{
-		addrs.NewDefaultProvider("between"): {
-			Provider:   addrs.NewDefaultProvider("between"),
-			Version:    getproviders.MustParseVersion("2.3.4"),
-			PackageDir: expectedPackageInstallPath("between", "2.3.4", false),
-		},
-		addrs.NewDefaultProvider("exact"): {
-			Provider:   addrs.NewDefaultProvider("exact"),
-			Version:    getproviders.MustParseVersion("1.2.3"),
-			PackageDir: expectedPackageInstallPath("exact", "1.2.3", false),
-		},
-		addrs.NewDefaultProvider("greater-than"): {
-			Provider:   addrs.NewDefaultProvider("greater-than"),
-			Version:    getproviders.MustParseVersion("2.3.4"),
-			PackageDir: expectedPackageInstallPath("greater-than", "2.3.4", false),
-		},
+		t.Fatalf("failed to get locked dependencies: %s", err)
+	}
+	gotProviderLocks := locks.AllProviders()
+	wantProviderLocks := map[addrs.Provider]*depsfile.ProviderLock{
+		addrs.NewDefaultProvider("between"): depsfile.NewProviderLock(
+			addrs.NewDefaultProvider("between"),
+			getproviders.MustParseVersion("2.3.4"),
+			getproviders.MustParseVersionConstraints("> 1.0.0, < 3.0.0"),
+			[]getproviders.Hash{
+				getproviders.HashScheme1.New("JVqAvZz88A+hS2wHVtTWQkHaxoA/LrUAz0H3jPBWPIA="),
+			},
+		),
+		addrs.NewDefaultProvider("exact"): depsfile.NewProviderLock(
+			addrs.NewDefaultProvider("exact"),
+			getproviders.MustParseVersion("1.2.3"),
+			getproviders.MustParseVersionConstraints("= 1.2.3"),
+			[]getproviders.Hash{
+				getproviders.HashScheme1.New("H1TxWF8LyhBb6B4iUdKhLc/S9sC/jdcrCykpkbGcfbg="),
+			},
+		),
+		addrs.NewDefaultProvider("greater-than"): depsfile.NewProviderLock(
+			addrs.NewDefaultProvider("greater-than"),
+			getproviders.MustParseVersion("2.3.4"),
+			getproviders.MustParseVersionConstraints(">= 2.3.3"),
+			[]getproviders.Hash{
+				getproviders.HashScheme1.New("SJPpXx/yoFE/W+7eCipjJ+G21xbdnTBD7lWodZ8hWkU="),
+			},
+		),
 	}
-	if diff := cmp.Diff(wantSelected, gotSelected); diff != "" {
+	if diff := cmp.Diff(gotProviderLocks, wantProviderLocks, depsfile.ProviderLockComparer); diff != "" {
 		t.Errorf("wrong version selections after upgrade\n%s", diff)
 	}
-
 }
 
 func TestInit_getProviderMissing(t *testing.T) {
@@ -1537,7 +1556,7 @@ func TestInit_providerLockFile(t *testing.T) {
 	defer testChdir(t, td)()
 
 	providerSource, close := newMockProviderSource(t, map[string][]string{
-		"test": []string{"1.2.3"},
+		"test": {"1.2.3"},
 	})
 	defer close()
 
@@ -1557,23 +1576,26 @@ func TestInit_providerLockFile(t *testing.T) {
 		t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
 	}
 
-	selectionsFile := ".terraform/plugins/selections.json"
-	buf, err := ioutil.ReadFile(selectionsFile)
+	lockFile := ".terraform.lock.hcl"
+	buf, err := ioutil.ReadFile(lockFile)
 	if err != nil {
-		t.Fatalf("failed to read provider selections file %s: %s", selectionsFile, err)
+		t.Fatalf("failed to read dependency lock file %s: %s", lockFile, err)
 	}
+	buf = bytes.TrimSpace(buf)
 	// The hash in here is for the fake package that newMockProviderSource produces
 	// (so it'll change if newMockProviderSource starts producing different contents)
 	wantLockFile := strings.TrimSpace(`
-{
-  "registry.terraform.io/hashicorp/test": {
-    "hash": "h1:wlbEC2mChQZ2hhgUhl6SeVLPP7fMqOFUZAQhQ9GIIno=",
-    "version": "1.2.3"
-  }
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/test" {
+  version     = "1.2.3"
+  constraints = "1.2.3"
+  hashes      = ["h1:wlbEC2mChQZ2hhgUhl6SeVLPP7fMqOFUZAQhQ9GIIno="]
 }
 `)
-	if string(buf) != wantLockFile {
-		t.Errorf("wrong provider selections file contents\ngot:  %s\nwant: %s", buf, wantLockFile)
+	if diff := cmp.Diff(wantLockFile, string(buf)); diff != "" {
+		t.Errorf("wrong dependency lock file contents\n%s", diff)
 	}
 }
 
@@ -1697,29 +1719,38 @@ func TestInit_pluginDirProviders(t *testing.T) {
 		t.Fatalf("bad: \n%s", ui.ErrorWriter)
 	}
 
-	inst := m.providerInstaller()
-	gotSelected, err := inst.SelectedPackages()
+	locks, err := m.lockedDependencies()
 	if err != nil {
-		t.Fatalf("failed to get selected packages from installer: %s", err)
-	}
-	wantSelected := map[addrs.Provider]*providercache.CachedProvider{
-		addrs.NewDefaultProvider("between"): {
-			Provider:   addrs.NewDefaultProvider("between"),
-			Version:    getproviders.MustParseVersion("2.3.4"),
-			PackageDir: expectedPackageInstallPath("between", "2.3.4", false),
-		},
-		addrs.NewDefaultProvider("exact"): {
-			Provider:   addrs.NewDefaultProvider("exact"),
-			Version:    getproviders.MustParseVersion("1.2.3"),
-			PackageDir: expectedPackageInstallPath("exact", "1.2.3", false),
-		},
-		addrs.NewDefaultProvider("greater-than"): {
-			Provider:   addrs.NewDefaultProvider("greater-than"),
-			Version:    getproviders.MustParseVersion("2.3.4"),
-			PackageDir: expectedPackageInstallPath("greater-than", "2.3.4", false),
-		},
+		t.Fatalf("failed to get locked dependencies: %s", err)
+	}
+	gotProviderLocks := locks.AllProviders()
+	wantProviderLocks := map[addrs.Provider]*depsfile.ProviderLock{
+		addrs.NewDefaultProvider("between"): depsfile.NewProviderLock(
+			addrs.NewDefaultProvider("between"),
+			getproviders.MustParseVersion("2.3.4"),
+			getproviders.MustParseVersionConstraints("> 1.0.0, < 3.0.0"),
+			[]getproviders.Hash{
+				getproviders.HashScheme1.New("JVqAvZz88A+hS2wHVtTWQkHaxoA/LrUAz0H3jPBWPIA="),
+			},
+		),
+		addrs.NewDefaultProvider("exact"): depsfile.NewProviderLock(
+			addrs.NewDefaultProvider("exact"),
+			getproviders.MustParseVersion("1.2.3"),
+			getproviders.MustParseVersionConstraints("= 1.2.3"),
+			[]getproviders.Hash{
+				getproviders.HashScheme1.New("H1TxWF8LyhBb6B4iUdKhLc/S9sC/jdcrCykpkbGcfbg="),
+			},
+		),
+		addrs.NewDefaultProvider("greater-than"): depsfile.NewProviderLock(
+			addrs.NewDefaultProvider("greater-than"),
+			getproviders.MustParseVersion("2.3.4"),
+			getproviders.MustParseVersionConstraints(">= 2.3.3"),
+			[]getproviders.Hash{
+				getproviders.HashScheme1.New("SJPpXx/yoFE/W+7eCipjJ+G21xbdnTBD7lWodZ8hWkU="),
+			},
+		),
 	}
-	if diff := cmp.Diff(wantSelected, gotSelected); diff != "" {
+	if diff := cmp.Diff(gotProviderLocks, wantProviderLocks, depsfile.ProviderLockComparer); diff != "" {
 		t.Errorf("wrong version selections after upgrade\n%s", diff)
 	}
 
@@ -1983,7 +2014,7 @@ func installFakeProviderPackagesElsewhere(t *testing.T, cacheDir *providercache.
 			if err != nil {
 				t.Fatalf("failed to prepare fake package for %s %s: %s", name, versionStr, err)
 			}
-			_, err = cacheDir.InstallPackage(context.Background(), meta)
+			_, err = cacheDir.InstallPackage(context.Background(), meta, nil)
 			if err != nil {
 				t.Fatalf("failed to install fake package for %s %s: %s", name, versionStr, err)
 			}

diff --git a/command/meta.go b/command/meta.go
@@ -410,7 +410,12 @@ func (m *Meta) contextOpts() (*terraform.ContextOpts, error) {
 			// This situation shouldn't arise commonly in practice because
 			// the selections file is generated programmatically.
 			log.Printf("[WARN] Failed to determine selected providers: %s", err)
-			providerFactories = nil
+
+			// variable providerFactories may now be incomplete, which could
+			// lead to errors reported downstream from here. providerFactories
+			// tries to populate as many providers as possible even in an
+			// error case, so that operations not using problematic providers
+			// can still succeed.
 		}
 		opts.Providers = providerFactories
 		opts.Provisioners = m.provisionerFactories()