Merge remote-tracking branch 'origin/master' into aws_secret_root_rot…

…ation

CHANGELOG.md

@@ -3,8 +3,14 @@
 BUG FIXES:
 
  * core: Re-add deprecated capabilities information for now [GH-5360]
+ * core: Fix handling of cyclic token relationships [GH-4803]
  * storage/mysql: Fix locking on MariaDB [GH-5343]
 
+IMPROVEMENTS:
+
+ * plugins: Add `env` parameter when registering plugins to the catalog to allow
+   operators to include environment variables during plugin execution. [GH-5359]
+
 ## 0.11.1.1 (September 17th, 2018) (Enterprise Only)
 
 BUG FIXES:

Makefile

@@ -147,9 +147,9 @@ proto:
 	protoc helper/identity/types.proto --go_out=plugins=grpc:../../..
 	protoc builtin/logical/database/dbplugin/*.proto --go_out=plugins=grpc:../../..
 	protoc logical/plugin/pb/*.proto --go_out=plugins=grpc:../../..
-	sed -i -e 's/Idp/IDP/' -e 's/Url/URL/' -e 's/Id/ID/' -e 's/IDentity/Identity/' -e 's/EntityId/EntityID/' -e 's/Api/API/' -e 's/Qr/QR/' -e 's/Totp/TOTP/' -e 's/Mfa/MFA/' -e 's/Pingid/PingID/' -e 's/protobuf:"/sentinel:"" protobuf:"/' -e 's/namespaceId/namespaceID/' -e 's/Ttl/TTL/' helper/identity/types.pb.go helper/storagepacker/types.pb.go logical/plugin/pb/backend.pb.go logical/identity.pb.go
 	sed -i '1s;^;// +build !enterprise\n;' physical/types.pb.go
 	sed -i '1s;^;// +build !enterprise\n;' helper/identity/mfa/types.pb.go
+	sed -i -e 's/Idp/IDP/' -e 's/Url/URL/' -e 's/Id/ID/' -e 's/IDentity/Identity/' -e 's/EntityId/EntityID/' -e 's/Api/API/' -e 's/Qr/QR/' -e 's/Totp/TOTP/' -e 's/Mfa/MFA/' -e 's/Pingid/PingID/' -e 's/protobuf:"/sentinel:"" protobuf:"/' -e 's/namespaceId/namespaceID/' -e 's/Ttl/TTL/' -e 's/BoundCidrs/BoundCIDRs/' helper/identity/types.pb.go helper/storagepacker/types.pb.go logical/plugin/pb/backend.pb.go logical/identity.pb.go
 
 fmtcheck:
 	@true

builtin/logical/database/backend_test.go

@@ -101,7 +101,7 @@ func getCluster(t *testing.T) (*vault.TestCluster, logical.SystemView) {
 	os.Setenv(pluginutil.PluginCACertPEMEnv, cluster.CACertPEMFile)
 
 	sys := vault.TestDynamicSystemView(cores[0].Core)
-	vault.TestAddTestPlugin(t, cores[0].Core, "postgresql-database-plugin", "TestBackend_PluginMain")
+	vault.TestAddTestPlugin(t, cores[0].Core, "postgresql-database-plugin", "TestBackend_PluginMain", []string{}, "")
 
 	return cluster, sys
 }

builtin/logical/database/dbplugin/plugin_test.go

@@ -94,8 +94,8 @@ func getCluster(t *testing.T) (*vault.TestCluster, logical.SystemView) {
 	cores := cluster.Cores
 
 	sys := vault.TestDynamicSystemView(cores[0].Core)
-	vault.TestAddTestPlugin(t, cores[0].Core, "test-plugin", "TestPlugin_GRPC_Main")
-	vault.TestAddTestPlugin(t, cores[0].Core, "test-plugin-netRPC", "TestPlugin_NetRPC_Main")
+	vault.TestAddTestPlugin(t, cores[0].Core, "test-plugin", "TestPlugin_GRPC_Main", []string{}, "")
+	vault.TestAddTestPlugin(t, cores[0].Core, "test-plugin-netRPC", "TestPlugin_NetRPC_Main", []string{}, "")
 
 	return cluster, sys
 }

builtin/plugin/backend_test.go

@@ -89,7 +89,7 @@ func testConfig(t *testing.T) (*logical.BackendConfig, func()) {
 
 	os.Setenv(pluginutil.PluginCACertPEMEnv, cluster.CACertPEMFile)
 
-	vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMain")
+	vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMain", []string{}, "")
 
 	return config, func() {
 		cluster.Cleanup()

helper/pluginutil/runner.go

@@ -43,6 +43,7 @@ type PluginRunner struct {
 	Name           string                      `json:"name" structs:"name"`
 	Command        string                      `json:"command" structs:"command"`
 	Args           []string                    `json:"args" structs:"args"`
+	Env            []string                    `json:"env" structs:"env"`
 	Sha256         []byte                      `json:"sha256" structs:"sha256"`
 	Builtin        bool                        `json:"builtin" structs:"builtin"`
 	BuiltinFactory func() (interface{}, error) `json:"-" structs:"-"`
@@ -65,6 +66,10 @@ func (r *PluginRunner) RunMetadataMode(ctx context.Context, wrapper RunnerUtil,
 
 func (r *PluginRunner) runCommon(ctx context.Context, wrapper RunnerUtil, pluginMap map[string]plugin.Plugin, hs plugin.HandshakeConfig, env []string, logger log.Logger, isMetadataMode bool) (*plugin.Client, error) {
 	cmd := exec.Command(r.Command, r.Args...)
+
+	// `env` should always go last to avoid overwriting internal values that might
+	// have been provided externally.
+	cmd.Env = append(cmd.Env, r.Env...)
 	cmd.Env = append(cmd.Env, env...)
 
 	// Add the mlock setting to the ENV of the plugin

http/plugin_test.go

@@ -50,7 +50,7 @@ func getPluginClusterAndCore(t testing.TB, logger log.Logger) (*vault.TestCluste
 	os.Setenv(pluginutil.PluginCACertPEMEnv, cluster.CACertPEMFile)
 
 	vault.TestWaitActive(t, core.Core)
-	vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestPlugin_PluginMain")
+	vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestPlugin_PluginMain", []string{}, "")
 
 	// Mount the mock plugin
 	err = core.Client.Sys().Mount("mock", &api.MountInput{

logical/plugin/pb/translation.go

@@ -499,7 +499,7 @@ func LogicalAuthToProtoAuth(a *logical.Auth) (*Auth, error) {
 		EntityID:         a.EntityID,
 		Alias:            a.Alias,
 		GroupAliases:     a.GroupAliases,
-		BoundCidrs:       boundCIDRs,
+		BoundCIDRs:       boundCIDRs,
 	}, nil
 }
 
@@ -519,7 +519,7 @@ func ProtoAuthToLogicalAuth(a *Auth) (*logical.Auth, error) {
 		return nil, err
 	}
 
-	boundCIDRs, err := parseutil.ParseAddrs(a.BoundCidrs)
+	boundCIDRs, err := parseutil.ParseAddrs(a.BoundCIDRs)
 	if err != nil {
 		return nil, err
 	}
@@ -573,7 +573,7 @@ func LogicalTokenEntryToProtoTokenEntry(t *logical.TokenEntry) *TokenEntry {
 		Role:           t.Role,
 		Period:         int64(t.Period),
 		EntityID:       t.EntityID,
-		BoundCidrs:     boundCIDRs,
+		BoundCIDRs:     boundCIDRs,
 		NamespaceID:    t.NamespaceID,
 		CubbyholeID:    t.CubbyholeID,
 	}
@@ -584,7 +584,7 @@ func ProtoTokenEntryToLogicalTokenEntry(t *TokenEntry) (*logical.TokenEntry, err
 		return nil, nil
 	}
 
-	boundCIDRs, err := parseutil.ParseAddrs(t.BoundCidrs)
+	boundCIDRs, err := parseutil.ParseAddrs(t.BoundCIDRs)
 	if err != nil {
 		return nil, err
 	}

vault/logical_system.go

@@ -269,7 +269,7 @@ func (b *SystemBackend) handlePluginCatalogUpdate(ctx context.Context, req *logi
 		return logical.ErrorResponse("missing command value"), nil
 	}
 
-	// For backwards compatibility, also accept args as part of command.  Don't
+	// For backwards compatibility, also accept args as part of command. Don't
 	// accepts args in both command and args.
 	args := d.Get("args").([]string)
 	parts := strings.Split(command, " ")
@@ -281,12 +281,14 @@ func (b *SystemBackend) handlePluginCatalogUpdate(ctx context.Context, req *logi
 		args = parts[1:]
 	}
 
+	env := d.Get("env").([]string)
+
 	sha256Bytes, err := hex.DecodeString(sha256)
 	if err != nil {
 		return logical.ErrorResponse("Could not decode SHA-256 value from Hex"), err
 	}
 
-	err = b.Core.pluginCatalog.Set(ctx, pluginName, parts[0], args, sha256Bytes)
+	err = b.Core.pluginCatalog.Set(ctx, pluginName, parts[0], args, env, sha256Bytes)
 	if err != nil {
 		return nil, err
 	}
@@ -3526,6 +3528,11 @@ plugin directory.`,
 		`The args passed to plugin command.`,
 		"",
 	},
+	"plugin-catalog_env": {
+		`The environment variables passed to plugin command.
+Each entry is of the form "key=value".`,
+		"",
+	},
 	"leases": {
 		`View or list lease metadata.`,
 		`

vault/logical_system_integ_test.go

@@ -20,6 +20,11 @@ import (
 	"github.com/hashicorp/vault/vault"
 )
 
+const (
+	expectedEnvKey   = "FOO"
+	expectedEnvValue = "BAR"
+)
+
 func TestSystemBackend_Plugin_secret(t *testing.T) {
 	cluster := testSystemBackendMock(t, 1, 1, logical.TypeLogical)
 	defer cluster.Cleanup()
@@ -103,7 +108,7 @@ func TestSystemBackend_Plugin_MismatchType(t *testing.T) {
 	core := cluster.Cores[0]
 
 	// Replace the plugin with a credential backend
-	vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainCredentials")
+	vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainCredentials", []string{}, "")
 
 	// Make a request to lazy load the now-credential plugin
 	// and expect an error
@@ -178,7 +183,7 @@ func testPlugin_CatalogRemoved(t *testing.T, btype logical.BackendType, testMoun
 		switch btype {
 		case logical.TypeLogical:
 			// Add plugin back to the catalog
-			vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainLogical")
+			vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainLogical", []string{}, "")
 			_, err = core.Client.Logical().Write("sys/mounts/mock-0", map[string]interface{}{
 				"type": "plugin",
 				"config": map[string]interface{}{
@@ -187,7 +192,7 @@ func testPlugin_CatalogRemoved(t *testing.T, btype logical.BackendType, testMoun
 			})
 		case logical.TypeCredential:
 			// Add plugin back to the catalog
-			vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainCredentials")
+			vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainCredentials", []string{}, "")
 			_, err = core.Client.Logical().Write("sys/auth/mock-0", map[string]interface{}{
 				"type":        "plugin",
 				"plugin_name": "mock-plugin",
@@ -283,9 +288,9 @@ func testPlugin_continueOnError(t *testing.T, btype logical.BackendType, mismatc
 	// Re-add the plugin to the catalog
 	switch btype {
 	case logical.TypeLogical:
-		vault.TestAddTestPluginTempDir(t, core.Core, "mock-plugin", "TestBackend_PluginMainLogical", cluster.TempDir)
+		vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainLogical", []string{}, cluster.TempDir)
 	case logical.TypeCredential:
-		vault.TestAddTestPluginTempDir(t, core.Core, "mock-plugin", "TestBackend_PluginMainCredentials", cluster.TempDir)
+		vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainCredentials", []string{}, cluster.TempDir)
 	}
 
 	// Reload the plugin
@@ -480,7 +485,7 @@ func testSystemBackendMock(t *testing.T, numCores, numMounts int, backendType lo
 
 	switch backendType {
 	case logical.TypeLogical:
-		vault.TestAddTestPluginTempDir(t, core.Core, "mock-plugin", "TestBackend_PluginMainLogical", tempDir)
+		vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainLogical", []string{}, tempDir)
 		for i := 0; i < numMounts; i++ {
 			// Alternate input styles for plugin_name on every other mount
 			options := map[string]interface{}{
@@ -502,7 +507,7 @@ func testSystemBackendMock(t *testing.T, numCores, numMounts int, backendType lo
 			}
 		}
 	case logical.TypeCredential:
-		vault.TestAddTestPluginTempDir(t, core.Core, "mock-plugin", "TestBackend_PluginMainCredentials", tempDir)
+		vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainCredentials", []string{}, tempDir)
 		for i := 0; i < numMounts; i++ {
 			// Alternate input styles for plugin_name on every other mount
 			options := map[string]interface{}{
@@ -530,6 +535,58 @@ func testSystemBackendMock(t *testing.T, numCores, numMounts int, backendType lo
 	return cluster
 }
 
+func TestSystemBackend_Plugin_Env(t *testing.T) {
+	kvPair := fmt.Sprintf("%s=%s", expectedEnvKey, expectedEnvValue)
+	cluster := testSystemBackend_SingleCluster_Env(t, []string{kvPair})
+	defer cluster.Cleanup()
+}
+
+// testSystemBackend_SingleCluster_Env is a helper func that returns a single
+// cluster and a single mounted plugin logical backend.
+func testSystemBackend_SingleCluster_Env(t *testing.T, env []string) *vault.TestCluster {
+	coreConfig := &vault.CoreConfig{
+		LogicalBackends: map[string]logical.Factory{
+			"plugin": plugin.Factory,
+		},
+	}
+
+	// Create a tempdir, cluster.Cleanup will clean up this directory
+	tempDir, err := ioutil.TempDir("", "vault-test-cluster")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{
+		HandlerFunc:        vaulthttp.Handler,
+		KeepStandbysSealed: true,
+		NumCores:           1,
+		TempDir:            tempDir,
+	})
+	cluster.Start()
+
+	core := cluster.Cores[0]
+	vault.TestWaitActive(t, core.Core)
+	client := core.Client
+
+	os.Setenv(pluginutil.PluginCACertPEMEnv, cluster.CACertPEMFile)
+
+	vault.TestAddTestPlugin(t, core.Core, "mock-plugin", "TestBackend_PluginMainEnv", env, tempDir)
+	options := map[string]interface{}{
+		"type":        "plugin",
+		"plugin_name": "mock-plugin",
+	}
+
+	resp, err := client.Logical().Write("sys/mounts/mock", options)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if resp != nil {
+		t.Fatalf("bad: %v", resp)
+	}
+
+	return cluster
+}
+
 func TestBackend_PluginMainLogical(t *testing.T) {
 	args := []string{}
 	if os.Getenv(pluginutil.PluginUnwrapTokenEnv) == "" && os.Getenv(pluginutil.PluginMetadataModeEnv) != "true" {
@@ -588,6 +645,41 @@ func TestBackend_PluginMainCredentials(t *testing.T) {
 	}
 }
 
+// TestBackend_PluginMainEnv is a mock plugin that simply checks for the existence of FOO env var.
+func TestBackend_PluginMainEnv(t *testing.T) {
+	actual := os.Getenv(expectedEnvKey)
+	if actual != expectedEnvValue {
+		t.Fatalf("expected: %q, got: %q", expectedEnvValue, actual)
+	}
+
+	args := []string{}
+	if os.Getenv(pluginutil.PluginUnwrapTokenEnv) == "" && os.Getenv(pluginutil.PluginMetadataModeEnv) != "true" {
+		return
+	}
+
+	caPEM := os.Getenv(pluginutil.PluginCACertPEMEnv)
+	if caPEM == "" {
+		t.Fatal("CA cert not passed in")
+	}
+	args = append(args, fmt.Sprintf("--ca-cert=%s", caPEM))
+
+	apiClientMeta := &pluginutil.APIClientMeta{}
+	flags := apiClientMeta.FlagSet()
+	flags.Parse(args)
+	tlsConfig := apiClientMeta.GetTLSConfig()
+	tlsProviderFunc := pluginutil.VaultPluginTLSProvider(tlsConfig)
+
+	factoryFunc := mock.FactoryType(logical.TypeLogical)
+
+	err := lplugin.Serve(&lplugin.ServeOpts{
+		BackendFactoryFunc: factoryFunc,
+		TLSProviderFunc:    tlsProviderFunc,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestSystemBackend_InternalUIResultantACL(t *testing.T) {
 	cluster := vault.NewTestCluster(t, nil, &vault.TestClusterOptions{
 		HandlerFunc: vaulthttp.Handler,

vault/logical_system_paths.go

@@ -293,6 +293,10 @@ func (b *SystemBackend) pluginsCatalogPath() *framework.Path {
 				Type:        framework.TypeStringSlice,
 				Description: strings.TrimSpace(sysHelp["plugin-catalog_args"][0]),
 			},
+			"env": &framework.FieldSchema{
+				Type:        framework.TypeStringSlice,
+				Description: strings.TrimSpace(sysHelp["plugin-catalog_env"][0]),
+			},
 		},
 
 		Callbacks: map[logical.Operation]framework.OperationFunc{

vault/plugin_catalog.go

@@ -87,7 +87,7 @@ func (c *PluginCatalog) Get(ctx context.Context, name string) (*pluginutil.Plugi
 
 // Set registers a new external plugin with the catalog, or updates an existing
 // external plugin. It takes the name, command and SHA256 of the plugin.
-func (c *PluginCatalog) Set(ctx context.Context, name, command string, args []string, sha256 []byte) error {
+func (c *PluginCatalog) Set(ctx context.Context, name, command string, args []string, env []string, sha256 []byte) error {
 	if c.directory == "" {
 		return ErrDirectoryNotConfigured
 	}
@@ -122,6 +122,7 @@ func (c *PluginCatalog) Set(ctx context.Context, name, command string, args []st
 		Name:    name,
 		Command: command,
 		Args:    args,
+		Env:     env,
 		Sha256:  sha256,
 		Builtin: false,
 	}