Merge remote-tracking branch 'origin/master' into validate-ignore-emp…

…ty-provider

CHANGELOG.md

@@ -21,6 +21,8 @@ BUG FIXES:
 * cli: Exit with an error if unable to gather input from the UI. For example, this may happen when running in a non-interactive environment but without `-input=false`. Previously Terraform would interpret these errors as empty strings, which could be confusing. [GH-26509]
 * cli: TF_LOG levels other than `trace` will now work correctly [GH-26632]
 * cli: Core and Provider logs can now be enabled separately for debugging, using `TF_LOG_CORE` and `TF_LOG_PROVIDER` [GH-26685]
+* command/console: expressions using `path` (`path.root`, `path.module`) now return the same result as they would in a configuration [GH-27263]
+* command/state list: fix bug where nested modules' resources were missing from `state list` output [GH-27268]
 
 ## Previous Releases
 

backend/local/backend_apply_test.go

@@ -241,6 +241,30 @@ test_instance.foo:
 	assertBackendStateUnlocked(t, b)
 }
 
+func TestLocal_applyRefreshFalse(t *testing.T) {
+	b, cleanup := TestLocal(t)
+	defer cleanup()
+
+	p := TestLocalProvider(t, b, "test", planFixtureSchema())
+	testStateFile(t, b.StatePath, testPlanState())
+
+	op, configCleanup := testOperationApply(t, "./testdata/plan")
+	defer configCleanup()
+
+	run, err := b.Operation(context.Background(), op)
+	if err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+	<-run.Done()
+	if run.Result != backend.OperationSuccess {
+		t.Fatalf("plan operation failed")
+	}
+
+	if p.ReadResourceCalled {
+		t.Fatal("ReadResource should not be called")
+	}
+}
+
 type backendWithFailingState struct {
 	Local
 }

backend/local/backend_local.go

@@ -78,7 +78,7 @@ func (b *Local) context(op *backend.Operation) (*terraform.Context, *configload.
 	opts.Targets = op.Targets
 	opts.UIInput = op.UIIn
 
-	opts.SkipRefresh = op.Type == backend.OperationTypePlan && !op.PlanRefresh
+	opts.SkipRefresh = op.Type != backend.OperationTypeRefresh && !op.PlanRefresh
 	if opts.SkipRefresh {
 		log.Printf("[DEBUG] backend/local: skipping refresh of managed resources")
 	}

backend/remote/backend.go

@@ -641,7 +641,10 @@ func (b *Remote) StateMgr(name string) (statemgr.Full, error) {
 	// accidentally upgrade state with a new code path, and the version check
 	// logic is coarser and simpler.
 	if !b.ignoreVersionConflict {
-		if workspace.TerraformVersion != tfversion.String() {
+		wsv := workspace.TerraformVersion
+		// Explicitly ignore the pseudo-version "latest" here, as it will cause
+		// plan and apply to always fail.
+		if wsv != tfversion.String() && wsv != "latest" {
 			return nil, fmt.Errorf("Remote workspace Terraform version %q does not match local Terraform version %q", workspace.TerraformVersion, tfversion.String())
 		}
 	}
@@ -890,6 +893,13 @@ func (b *Remote) VerifyWorkspaceTerraformVersion(workspaceName string) tfdiags.D
 		return diags
 	}
 
+	// If the workspace has the pseudo-version "latest", all bets are off. We
+	// cannot reasonably determine what the intended Terraform version is, so
+	// we'll skip version verification.
+	if workspace.TerraformVersion == "latest" {
+		return nil
+	}
+
 	remoteVersion, err := version.NewSemver(workspace.TerraformVersion)
 	if err != nil {
 		diags = diags.Append(tfdiags.Sourceless(

backend/remote/backend_test.go

@@ -515,6 +515,45 @@ func TestRemote_StateMgr_versionCheck(t *testing.T) {
 	}
 }
 
+func TestRemote_StateMgr_versionCheckLatest(t *testing.T) {
+	b, bCleanup := testBackendDefault(t)
+	defer bCleanup()
+
+	v0140 := version.Must(version.NewSemver("0.14.0"))
+
+	// Save original local version state and restore afterwards
+	p := tfversion.Prerelease
+	v := tfversion.Version
+	s := tfversion.SemVer
+	defer func() {
+		tfversion.Prerelease = p
+		tfversion.Version = v
+		tfversion.SemVer = s
+	}()
+
+	// For this test, the local Terraform version is set to 0.14.0
+	tfversion.Prerelease = ""
+	tfversion.Version = v0140.String()
+	tfversion.SemVer = v0140
+
+	// Update the remote workspace to the pseudo-version "latest"
+	if _, err := b.client.Workspaces.Update(
+		context.Background(),
+		b.organization,
+		b.workspace,
+		tfe.WorkspaceUpdateOptions{
+			TerraformVersion: tfe.String("latest"),
+		},
+	); err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	// This should succeed despite not being a string match
+	if _, err := b.StateMgr(backend.DefaultStateName); err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+}
+
 func TestRemote_VerifyWorkspaceTerraformVersion(t *testing.T) {
 	testCases := []struct {
 		local   string
@@ -528,14 +567,14 @@ func TestRemote_VerifyWorkspaceTerraformVersion(t *testing.T) {
 		{"0.14.0", "1.1.0", true},
 		{"1.2.0", "1.2.99", false},
 		{"1.2.0", "1.3.0", true},
+		{"0.15.0", "latest", false},
 	}
 	for _, tc := range testCases {
 		t.Run(fmt.Sprintf("local %s, remote %s", tc.local, tc.remote), func(t *testing.T) {
 			b, bCleanup := testBackendDefault(t)
 			defer bCleanup()
 
 			local := version.Must(version.NewSemver(tc.local))
-			remote := version.Must(version.NewSemver(tc.remote))
 
 			// Save original local version state and restore afterwards
 			p := tfversion.Prerelease
@@ -559,7 +598,7 @@ func TestRemote_VerifyWorkspaceTerraformVersion(t *testing.T) {
 				b.organization,
 				b.workspace,
 				tfe.WorkspaceUpdateOptions{
-					TerraformVersion: tfe.String(remote.String()),
+					TerraformVersion: tfe.String(tc.remote),
 				},
 			); err != nil {
 				t.Fatalf("error: %v", err)

builtin/providers/terraform/data_source_state.go

@@ -5,6 +5,7 @@ import (
 	"log"
 
 	"github.com/hashicorp/terraform/backend"
+	"github.com/hashicorp/terraform/backend/remote"
 	"github.com/hashicorp/terraform/configs/configschema"
 	"github.com/hashicorp/terraform/providers"
 	"github.com/hashicorp/terraform/tfdiags"
@@ -233,6 +234,12 @@ func getBackend(cfg cty.Value) (backend.Backend, cty.Value, tfdiags.Diagnostics)
 		return nil, cty.NilVal, diags
 	}
 
+	// If this is the enhanced remote backend, we want to disable the version
+	// check, because this is a read-only operation
+	if rb, ok := b.(*remote.Remote); ok {
+		rb.IgnoreVersionConflict()
+	}
+
 	return b, newVal, diags
 }
 

command/console.go

@@ -35,6 +35,7 @@ func (c *ConsoleCommand) Run(args []string) int {
 		c.Ui.Error(err.Error())
 		return 1
 	}
+	configPath = c.Meta.normalizePath(configPath)
 
 	// Check for user-supplied plugin path
 	if c.pluginPath, err = c.loadPluginPath(); err != nil {

command/init.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/hashicorp/hcl/v2"
@@ -31,14 +30,11 @@ import (
 // module and clones it to the working directory.
 type InitCommand struct {
 	Meta
-
-	// getPlugins is for the -get-plugins flag
-	getPlugins bool
 }
 
 func (c *InitCommand) Run(args []string) int {
 	var flagFromModule string
-	var flagBackend, flagGet, flagUpgrade bool
+	var flagBackend, flagGet, flagUpgrade, getPlugins bool
 	var flagPluginPath FlagStringSlice
 	var flagVerifyPlugins bool
 	flagConfigExtra := newRawFlags("-backend-config")
@@ -49,7 +45,7 @@ func (c *InitCommand) Run(args []string) int {
 	cmdFlags.Var(flagConfigExtra, "backend-config", "")
 	cmdFlags.StringVar(&flagFromModule, "from-module", "", "copy the source of the given module into the directory before init")
 	cmdFlags.BoolVar(&flagGet, "get", true, "")
-	cmdFlags.BoolVar(&c.getPlugins, "get-plugins", true, "")
+	cmdFlags.BoolVar(&getPlugins, "get-plugins", true, "no-op flag, use provider_installation blocks to customize provider installation")
 	cmdFlags.BoolVar(&c.forceInitCopy, "force-copy", false, "suppress prompts about copying state data")
 	cmdFlags.BoolVar(&c.Meta.stateLock, "lock", true, "lock state")
 	cmdFlags.DurationVar(&c.Meta.stateLockTimeout, "lock-timeout", 0, "lock timeout")
@@ -66,7 +62,16 @@ func (c *InitCommand) Run(args []string) int {
 
 	if len(flagPluginPath) > 0 {
 		c.pluginPath = flagPluginPath
-		c.getPlugins = false
+	}
+
+	// If users are setting the no-op get-plugins command, give them a warning,
+	// this should allow us to remove the flag in the future.
+	if !getPlugins {
+		diags = diags.Append(tfdiags.Sourceless(
+			tfdiags.Warning,
+			"No-op -get-plugins flag used",
+			`As of Terraform 0.13+, the -get-plugins=false command is a no-op flag. If you would like to customize provider installation, use a provider_installation block or other available Terraform settings.`,
+		))
 	}
 
 	// Validate the arg count
@@ -468,12 +473,15 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 		log.Printf("[DEBUG] will search for provider plugins in %s", pluginDirs)
 	}
 
+	// Installation can be aborted by interruption signals
+	ctx, done := c.InterruptibleContext()
+	defer done()
+
 	// Because we're currently just streaming a series of events sequentially
 	// into the terminal, we're showing only a subset of the events to keep
 	// things relatively concise. Later it'd be nice to have a progress UI
 	// where statuses update in-place, but we can't do that as long as we
 	// are shimming our vt100 output to the legacy console API on Windows.
-	missingProviders := make(map[addrs.Provider]struct{})
 	evts := &providercache.InstallerEvents{
 		PendingProviders: func(reqs map[addrs.Provider]getproviders.VersionConstraints) {
 			c.Ui.Output(c.Colorize().Color(
@@ -511,10 +519,6 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 			c.Ui.Info(fmt.Sprintf("- Installing %s v%s...", provider.ForDisplay(), version))
 		},
 		QueryPackagesFailure: func(provider addrs.Provider, err error) {
-			// We track providers that had missing metadata because we might
-			// generate additional hints for some of them at the end.
-			missingProviders[provider] = struct{}{}
-
 			switch errorTy := err.(type) {
 			case getproviders.ErrProviderNotFound:
 				sources := errorTy.Sources
@@ -530,11 +534,22 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 					),
 				))
 			case getproviders.ErrRegistryProviderNotKnown:
+				// We might be able to suggest an alternative provider to use
+				// instead of this one.
+				var suggestion string
+				alternative := getproviders.MissingProviderSuggestion(ctx, provider, inst.ProviderSource())
+				if alternative != provider {
+					suggestion = fmt.Sprintf(
+						"\n\nDid you intend to use %s? If so, you must specify that source address in each module which requires that provider. To see which modules are currently depending on %s, run the following command:\n    terraform providers",
+						alternative.ForDisplay(), provider.ForDisplay(),
+					)
+				}
+
 				diags = diags.Append(tfdiags.Sourceless(
 					tfdiags.Error,
 					"Failed to query available provider packages",
-					fmt.Sprintf("Could not retrieve the list of available versions for provider %s: %s",
-						provider.ForDisplay(), err,
+					fmt.Sprintf("Could not retrieve the list of available versions for provider %s: %s%s",
+						provider.ForDisplay(), err, suggestion,
 					),
 				))
 			case getproviders.ErrHostNoProviders:
@@ -730,6 +745,7 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 			))
 		},
 	}
+	ctx = evts.OnContext(ctx)
 
 	// Dev overrides cause the result of "terraform init" to be irrelevant for
 	// any overridden providers, so we'll warn about it to avoid later
@@ -741,70 +757,12 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
 	if upgrade {
 		mode = providercache.InstallUpgrades
 	}
-	// Installation can be aborted by interruption signals
-	ctx, done := c.InterruptibleContext()
-	defer done()
-	ctx = evts.OnContext(ctx)
 	newLocks, err := inst.EnsureProviderVersions(ctx, previousLocks, reqs, mode)
 	if ctx.Err() == context.Canceled {
 		c.showDiagnostics(diags)
 		c.Ui.Error("Provider installation was canceled by an interrupt signal.")
 		return true, true, diags
 	}
-	if len(missingProviders) > 0 {
-		// If we encountered requirements for one or more providers where we
-		// weren't able to find any metadata, that _might_ be because a
-		// user had previously (before 0.14) been incorrectly using the
-		// .terraform/plugins directory as if it were a local filesystem
-		// mirror, rather than as the main cache directory.
-		//
-		// We no longer allow that because it'd be ambiguous whether plugins in
-		// there are explictly intended to be a local mirror or if they are
-		// just leftover cache entries from provider installation in
-		// Terraform 0.13.
-		//
-		// To help those users migrate we have a specialized warning message
-		// for it, which we'll produce only if one of the missing providers can
-		// be seen in the "legacy" cache directory, which is what we're now
-		// considering .terraform/plugins to be. (The _current_ cache directory
-		// is .terraform/providers.)
-		//
-		// This is only a heuristic, so it might potentially produce false
-		// positives if a user happens to encounter another sort of error
-		// while they are upgrading from Terraform 0.13 to 0.14. Aside from
-		// upgrading users should not end up in here because they won't
-		// have a legacy cache directory at all.
-		legacyDir := c.providerLegacyCacheDir()
-		if legacyDir != nil { // if the legacy directory is present at all
-			for missingProvider := range missingProviders {
-				if missingProvider.IsDefault() {
-					// If we get here for a default provider then it's more
-					// likely that something _else_ went wrong, like a network
-					// problem, so we'll skip the warning in this case to
-					// avoid potentially misleading the user into creating an
-					// unnecessary local mirror for an official provider.
-					continue
-				}
-				entry := legacyDir.ProviderLatestVersion(missingProvider)
-				if entry == nil {
-					continue
-				}
-				// If we get here then the missing provider was cached, which
-				// implies that it might be an in-house provider the user
-				// placed manually to try to make Terraform use it as if it
-				// were a local mirror directory.
-				wantDir := filepath.FromSlash(fmt.Sprintf("terraform.d/plugins/%s/%s/%s", missingProvider, entry.Version, getproviders.CurrentPlatform))
-				diags = diags.Append(tfdiags.Sourceless(
-					tfdiags.Warning,
-					"Missing provider is in legacy cache directory",
-					fmt.Sprintf(
-						"Terraform supports a number of local directories that can serve as automatic local filesystem mirrors, but .terraform/plugins is not one of them because Terraform v0.13 and earlier used this directory to cache copies of provider plugins retrieved from elsewhere.\n\nIf you intended to use this directory as a filesystem mirror for %s, place it instead in the following directory:\n  %s",
-						missingProvider, wantDir,
-					),
-				))
-			}
-		}
-	}
 	if err != nil {
 		// The errors captured in "err" should be redundant with what we
 		// received via the InstallerEvents callbacks above, so we'll
@@ -971,7 +929,6 @@ func (c *InitCommand) AutocompleteFlags() complete.Flags {
 		"-force-copy":     complete.PredictNothing,
 		"-from-module":    completePredictModuleSource,
 		"-get":            completePredictBoolean,
-		"-get-plugins":    completePredictBoolean,
 		"-input":          completePredictBoolean,
 		"-lock":           completePredictBoolean,
 		"-lock-timeout":   complete.PredictAnything,
@@ -1024,6 +981,9 @@ Options:
   -get=true            Download any modules for this configuration.
 
   -get-plugins=true    Download any missing plugins for this configuration.
+                       This command is a no-op in Terraform 0.13+: use
+                       -plugin-dir settings or provider_installation blocks
+                       instead.
 
   -input=true          Ask for input if necessary. If false, will error if
                        input was required.
@@ -1042,8 +1002,8 @@ Options:
   -reconfigure         Reconfigure the backend, ignoring any saved
                        configuration.
 
-  -upgrade=false       If installing modules (-get) or plugins (-get-plugins),
-                       ignore previously-downloaded objects and install the
+  -upgrade=false       If installing modules (-get) or plugins, ignore
+                       previously-downloaded objects and install the
                        latest version allowed within configured constraints.
 
   -verify-plugins=true Verify the authenticity and integrity of automatically