diff --git a/CHANGELOG.md b/CHANGELOG.md index fde8aaf..493c186 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ CHANGES: [`tokenRequests`](https://github.com/kubernetes-sigs/secrets-store-csi-driver/tree/main/charts/secrets-store-csi-driver#configuration) option from the _driver_ helm chart via the flag `--set tokenRequests[0].audience="vault"`. See [CSI TokenRequests documentation](https://kubernetes-csi.github.io/docs/token-requests.html) for further details. +* Vault CSI Provider now creates a Kubernetes secret with an HMAC key to produce consistent hashes for secret versions. [[GH-198](https://github.com/hashicorp/vault-csi-provider/pull/198)] + * Requires RBAC permissions to create secrets, and read the same specific secret back. Versions are not generated otherwise and a warning + is logged on each mount that fails to generate a version. + * Supports creating the secret with custom name via `-hmac-secret-name` IMPROVEMENTS: diff --git a/Makefile b/Makefile index 2313b35..1401d38 100644 --- a/Makefile +++ b/Makefile @@ -78,6 +78,7 @@ e2e-setup: --set linux.image.pullPolicy="IfNotPresent" \ --set syncSecret.enabled=true \ --set tokenRequests[0].audience="vault" + kubectl apply --namespace=csi -f test/bats/configs/vault/hmac-secret-role.yaml helm install vault-bootstrap test/bats/configs/vault \ --namespace=csi helm install vault vault \ diff --git a/go.mod b/go.mod index 32022b8..e9e9ee9 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( k8s.io/api v0.25.4 k8s.io/apimachinery v0.25.4 k8s.io/client-go v0.25.4 + k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed sigs.k8s.io/secrets-store-csi-driver v1.2.4 ) @@ -84,7 +85,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/klog/v2 v2.70.1 // indirect k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect - k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index 6959c97..9cc3979 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -33,6 +33,8 @@ type FlagsConfig struct { Version bool HealthAddr string + HMACSecretName string + VaultAddr string VaultMount string VaultNamespace string diff --git a/internal/hmac/hmac.go b/internal/hmac/hmac.go new file mode 100644 index 0000000..faca271 --- /dev/null +++ b/internal/hmac/hmac.go @@ -0,0 +1,96 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package hmac + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + hmacKeyName = "key" + hmacKeyLength = 32 +) + +var errDeleteSecret = errors.New("delete the kubernetes secret to trigger an automatic regeneration") + +func NewHMACGenerator(client kubernetes.Interface, secretSpec *corev1.Secret) *HMACGenerator { + return &HMACGenerator{ + client: client, + secretSpec: secretSpec, + } +} + +type HMACGenerator struct { + client kubernetes.Interface + secretSpec *corev1.Secret +} + +// GetOrCreateHMACKey will try to read an HMAC key from a Kubernetes secret and +// race with other pods to create it if not found. The HMAC key is persisted to +// a Kubernetes secret to ensure all pods are deterministically producing the +// same version hashes when given the same inputs. +func (g *HMACGenerator) GetOrCreateHMACKey(ctx context.Context) ([]byte, error) { + // Fast path - most of the time the secret will already be created. + secret, err := g.client.CoreV1().Secrets(g.secretSpec.Namespace).Get(ctx, g.secretSpec.Name, metav1.GetOptions{}) + if err == nil { + return hmacKeyFromSecret(secret) + } + if !apierrors.IsNotFound(err) { + return nil, err + } + + // Secret not found. We'll join the race to create it. + hmacKeyCandidate := make([]byte, hmacKeyLength) + _, err = rand.Read(hmacKeyCandidate) + if err != nil { + return nil, err + } + + // Make a copy of the secretSpec to avoid a data race. + secretSpec := *g.secretSpec + secretSpec.Data = map[string][]byte{ + hmacKeyName: hmacKeyCandidate, + } + + var persistedHMACSecret *corev1.Secret + + // Try to create first + persistedHMACSecret, err = g.client.CoreV1().Secrets(secretSpec.Namespace).Create(ctx, &secretSpec, metav1.CreateOptions{}) + switch { + case err == nil: + // We created the secret, nothing to handle. + case apierrors.IsAlreadyExists(err): + // We lost the race to create the secret. Read the existing secret instead. + persistedHMACSecret, err = g.client.CoreV1().Secrets(secretSpec.Namespace).Get(ctx, secretSpec.Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + default: + // Unexpected error case. + return nil, err + } + + return hmacKeyFromSecret(persistedHMACSecret) +} + +func hmacKeyFromSecret(secret *corev1.Secret) ([]byte, error) { + hmacKey, ok := secret.Data[hmacKeyName] + if !ok { + return nil, fmt.Errorf("expected secret %q to have a key %q; %w", secret.Name, hmacKeyName, errDeleteSecret) + } + + if len(hmacKey) == 0 { + return nil, fmt.Errorf("expected secret %q to have a non-zero HMAC key; %w", secret.Name, errDeleteSecret) + } + + return hmacKey, nil +} diff --git a/internal/hmac/hmac_test.go b/internal/hmac/hmac_test.go new file mode 100644 index 0000000..64f3cc3 --- /dev/null +++ b/internal/hmac/hmac_test.go @@ -0,0 +1,110 @@ +package hmac + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +const ( + secretName = "test-secret" + secretNamespace = "test-namespace" +) + +var secretSpec = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: secretNamespace, + }, + Data: map[string][]byte{ + hmacKeyName: []byte(strings.Repeat("a", 32)), + }, +} + +func setup(t *testing.T) (*HMACGenerator, *fake.Clientset) { + client := fake.NewSimpleClientset() + return NewHMACGenerator(client, secretSpec), client +} + +func TestGenerateSecretIfNoneExists(t *testing.T) { + gen, client := setup(t) + + // Add counter functions. + createCount := countAPICalls(client, "create", "secrets") + getCount := countAPICalls(client, "get", "secrets") + + // Get an HMAC key, which should create the k8s secret. + key, err := gen.GetOrCreateHMACKey(context.Background()) + require.NoError(t, err) + assert.Len(t, key, hmacKeyLength) + assert.Equal(t, 1, *createCount) + assert.Equal(t, 1, *getCount) + assert.NotEqual(t, string(secretSpec.Data[hmacKeyName]), string(key)) + assert.NotEmpty(t, string(key)) +} + +func TestReadSecretIfAlreadyExists(t *testing.T) { + gen, client := setup(t) + + ctx := context.Background() + _, err := client.CoreV1().Secrets(secretNamespace).Create(ctx, secretSpec, metav1.CreateOptions{}) + require.NoError(t, err) + + // Add counter functions. + createCount := countAPICalls(client, "create", "secrets") + getCount := countAPICalls(client, "get", "secrets") + + // Get an HMAC key, which should read the existing k8s secret. + key, err := gen.GetOrCreateHMACKey(ctx) + require.NoError(t, err) + assert.Len(t, key, hmacKeyLength) + assert.Equal(t, 0, *createCount) + assert.Equal(t, 1, *getCount) + assert.Equal(t, string(secretSpec.Data[hmacKeyName]), string(key)) +} + +func TestGracefullyHandlesLosingTheRace(t *testing.T) { + gen, client := setup(t) + + ctx := context.Background() + + client.PrependReactor("create", "secrets", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + // Intercept the create call and create the secret just before. + err = client.Tracker().Create(schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "secrets", + }, secretSpec, secretNamespace) + require.NoError(t, err) + return false, nil, nil + }) + createCount := countAPICalls(client, "create", "secrets") + getCount := countAPICalls(client, "get", "secrets") + + // Get an HMAC key, which should initially find no secret, and then lose the race for creating it. + key, err := gen.GetOrCreateHMACKey(ctx) + require.NoError(t, err) + assert.Len(t, key, hmacKeyLength) + assert.Equal(t, 1, *createCount) + assert.Equal(t, 2, *getCount) + assert.Equal(t, string(secretSpec.Data[hmacKeyName]), string(key)) +} + +// Counts the number of times an API is called. +func countAPICalls(client *fake.Clientset, verb string, resource string) *int { + i := 0 + client.PrependReactor(verb, resource, func(_ k8stesting.Action) (handled bool, ret runtime.Object, err error) { + i++ + return false, nil, nil + }) + return &i +} diff --git a/internal/provider/provider.go b/internal/provider/provider.go index e3392f4..d109f85 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -5,6 +5,7 @@ package provider import ( "context" + "crypto/hmac" "crypto/sha256" "encoding/base64" "encoding/hex" @@ -18,6 +19,7 @@ import ( "github.com/hashicorp/go-hclog" vaultclient "github.com/hashicorp/vault-csi-provider/internal/client" "github.com/hashicorp/vault-csi-provider/internal/config" + hmacgen "github.com/hashicorp/vault-csi-provider/internal/hmac" "github.com/hashicorp/vault/api" authenticationv1 "k8s.io/api/authentication/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,14 +34,16 @@ type provider struct { cache map[cacheKey]*api.Secret // Allows mocking Kubernetes API for tests. - k8sClient kubernetes.Interface + k8sClient kubernetes.Interface + hmacGenerator *hmacgen.HMACGenerator } -func NewProvider(logger hclog.Logger, k8sClient kubernetes.Interface) *provider { +func NewProvider(logger hclog.Logger, k8sClient kubernetes.Interface, hmacGenerator *hmacgen.HMACGenerator) *provider { p := &provider{ - logger: logger, - cache: make(map[cacheKey]*api.Secret), - k8sClient: k8sClient, + logger: logger, + cache: make(map[cacheKey]*api.Secret), + k8sClient: k8sClient, + hmacGenerator: hmacGenerator, } return p @@ -234,7 +238,7 @@ func (p *provider) getSecret(ctx context.Context, client *api.Client, secretConf } for _, w := range secret.Warnings { - p.logger.Warn("warning in response from Vault API", "warning", w) + p.logger.Warn("Warning in response from Vault API", "warning", w) } p.cache[key] = secret @@ -267,6 +271,10 @@ func (p *provider) getSecret(ctx context.Context, client *api.Client, secretConf // MountSecretsStoreObjectContent mounts content of the vault object to target path func (p *provider) HandleMountRequest(ctx context.Context, cfg config.Config, flagsConfig config.FlagsConfig) (*pb.MountResponse, error) { + hmacKey, err := p.hmacGenerator.GetOrCreateHMACKey(ctx) + if err != nil { + p.logger.Warn("Error generating HMAC key. Mounted secrets will not be assigned a version", "error", err) + } client, err := vaultclient.New(cfg.Parameters, flagsConfig) if err != nil { return nil, err @@ -291,7 +299,7 @@ func (p *provider) HandleMountRequest(ctx context.Context, cfg config.Config, fl return nil, err } - version, err := generateObjectVersion(secret, content) + version, err := generateObjectVersion(secret, hmacKey, content) if err != nil { return nil, fmt.Errorf("failed to generate version for object name %q: %w", secret.ObjectName, err) } @@ -311,14 +319,30 @@ func (p *provider) HandleMountRequest(ctx context.Context, cfg config.Config, fl }, nil } -func generateObjectVersion(secret config.Secret, content []byte) (*pb.ObjectVersion, error) { - hash := sha256.New() +func generateObjectVersion(secret config.Secret, hmacKey []byte, content []byte) (*pb.ObjectVersion, error) { + // If something went wrong with generating the HMAC key, we log the error and + // treat generating the version as best-effort instead, as delivering the secret + // is generally more critical to workloads than assigning a version for it. + if hmacKey == nil { + return &pb.ObjectVersion{ + Id: secret.ObjectName, + Version: "", + }, nil + } + // We include the secret config in the hash input to avoid leaking information // about different secrets that could have the same content. - _, err := hash.Write([]byte(fmt.Sprintf("%v:%s", secret, content))) + hash := hmac.New(sha256.New, hmacKey) + cfg, err := json.Marshal(secret) if err != nil { return nil, err } + if _, err := hash.Write(cfg); err != nil { + return nil, err + } + if _, err := hash.Write(content); err != nil { + return nil, err + } return &pb.ObjectVersion{ Id: secret.ObjectName, diff --git a/internal/provider/provider_test.go b/internal/provider/provider_test.go index 8180849..788d3a7 100644 --- a/internal/provider/provider_test.go +++ b/internal/provider/provider_test.go @@ -14,6 +14,7 @@ import ( "github.com/hashicorp/go-hclog" "github.com/hashicorp/vault-csi-provider/internal/config" + "github.com/hashicorp/vault-csi-provider/internal/hmac" "github.com/hashicorp/vault/api" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -291,20 +292,8 @@ func TestHandleMountRequest(t *testing.T) { Contents: []byte("secret v1 from: /v1/path/three"), }, } - expectedVersions := []*pb.ObjectVersion{ - { - Id: "object-one", - Version: "7eM6I4jvRmoPuY8XiQsUuJtEVDQlSE5JCPbXQWXN2tE=", - }, - { - Id: "object-two", - Version: "V7eu3GtXFYYNJkbDDEfTNalWWpZl-VTu3Pu-qF9sWi4=", - }, - { - Id: "object-three", - Version: "95O8POIdARplTKNAtExps-7jm8jETgDB4idsUA9KcL8=", - }, - } + expectedVersionIDs := []string{"object-one", "object-two", "object-three"} + versionsSeen := map[string]struct{}{} // SETUP mockVaultServer := httptest.NewServer(http.HandlerFunc(mockVaultHandler( @@ -329,31 +318,39 @@ func TestHandleMountRequest(t *testing.T) { &corev1.ServiceAccount{}, &authenticationv1.TokenRequest{}, ) + hmacGenerator := hmac.NewHMACGenerator(k8sClient, &corev1.Secret{}) // While we hit the cache, the secret contents and versions should remain the same. - provider := NewProvider(hclog.Default(), k8sClient) + provider := NewProvider(hclog.Default(), k8sClient, hmacGenerator) for i := 0; i < 3; i++ { resp, err := provider.HandleMountRequest(context.Background(), spcConfig, flagsConfig) require.NoError(t, err) assert.Equal(t, (*v1alpha1.Error)(nil), resp.Error) assert.Equal(t, expectedFiles, resp.Files) - assert.Equal(t, expectedVersions, resp.ObjectVersion) + assert.Equal(t, expectedVersionIDs[i], resp.ObjectVersion[i].Id) + assert.NotEmpty(t, resp.ObjectVersion[i].Version) + _, seen := versionsSeen[resp.ObjectVersion[i].Version] + assert.False(t, seen) + versionsSeen[resp.ObjectVersion[i].Version] = struct{}{} } // The mockVaultHandler function below includes a dynamic counter in the content of secrets. // That means mounting again with a fresh provider will update the contents of the secrets, which should update the version. - resp, err := NewProvider(hclog.Default(), k8sClient).HandleMountRequest(context.Background(), spcConfig, flagsConfig) + resp, err := NewProvider(hclog.Default(), k8sClient, hmacGenerator).HandleMountRequest(context.Background(), spcConfig, flagsConfig) require.NoError(t, err) assert.Equal(t, (*v1alpha1.Error)(nil), resp.Error) expectedFiles[0].Contents = []byte("secret v2 from: /v1/path/one") expectedFiles[1].Contents = []byte(`{"request_id":"","lease_id":"","lease_duration":0,"renewable":false,"data":{"the-key":"secret v2 from: /v1/path/two"},"warnings":null}`) expectedFiles[2].Contents = []byte("secret v2 from: /v1/path/three") - expectedVersions[0].Version = "R-NY6w6nGg5vX510c7i28A5sLZtxlDbu8y9zY92AUPY=" - expectedVersions[1].Version = "6hCb1c_dfqXbIdYYh7zEuqSG_f8ROpuE_5OmSja5pIk=" - expectedVersions[2].Version = "rKthxBOUCu5jDLuU6ZwabWnN4OWOiSPG8cnT2PtHqik=" assert.Equal(t, expectedFiles, resp.Files) - assert.Equal(t, expectedVersions, resp.ObjectVersion) + for i := 0; i < len(expectedFiles); i++ { + assert.Equal(t, expectedVersionIDs[i], resp.ObjectVersion[i].Id) + assert.NotEmpty(t, resp.ObjectVersion[i].Version) + _, seen := versionsSeen[resp.ObjectVersion[i].Version] + assert.False(t, seen) + versionsSeen[resp.ObjectVersion[i].Version] = struct{}{} + } } func mockVaultHandler(pathMapping map[string]func(numberOfCalls int) (string, interface{})) func(w http.ResponseWriter, req *http.Request) { diff --git a/internal/server/server.go b/internal/server/server.go index af259a4..e9806d2 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -9,6 +9,7 @@ import ( "github.com/hashicorp/go-hclog" "github.com/hashicorp/vault-csi-provider/internal/config" + "github.com/hashicorp/vault-csi-provider/internal/hmac" "github.com/hashicorp/vault-csi-provider/internal/provider" "github.com/hashicorp/vault-csi-provider/internal/version" "k8s.io/client-go/kubernetes" @@ -19,16 +20,18 @@ var _ pb.CSIDriverProviderServer = (*Server)(nil) // Server implements the secrets-store-csi-driver provider gRPC service interface. type Server struct { - logger hclog.Logger - flagsConfig config.FlagsConfig - k8sClient kubernetes.Interface + logger hclog.Logger + flagsConfig config.FlagsConfig + k8sClient kubernetes.Interface + hmacGenerator *hmac.HMACGenerator } -func NewServer(logger hclog.Logger, flagsConfig config.FlagsConfig, k8sClient kubernetes.Interface) *Server { +func NewServer(logger hclog.Logger, flagsConfig config.FlagsConfig, k8sClient kubernetes.Interface, hmacGenerator *hmac.HMACGenerator) *Server { return &Server{ - logger: logger, - flagsConfig: flagsConfig, - k8sClient: k8sClient, + logger: logger, + flagsConfig: flagsConfig, + k8sClient: k8sClient, + hmacGenerator: hmacGenerator, } } @@ -46,7 +49,7 @@ func (s *Server) Mount(ctx context.Context, req *pb.MountRequest) (*pb.MountResp return nil, err } - provider := provider.NewProvider(s.logger.Named("provider"), s.k8sClient) + provider := provider.NewProvider(s.logger.Named("provider"), s.k8sClient, s.hmacGenerator) resp, err := provider.HandleMountRequest(ctx, cfg, s.flagsConfig) if err != nil { return nil, fmt.Errorf("error making mount request: %w", err) diff --git a/main.go b/main.go index 752d7a1..c3c44eb 100644 --- a/main.go +++ b/main.go @@ -16,15 +16,23 @@ import ( "github.com/hashicorp/go-hclog" "github.com/hashicorp/vault-csi-provider/internal/config" + "github.com/hashicorp/vault-csi-provider/internal/hmac" providerserver "github.com/hashicorp/vault-csi-provider/internal/server" "github.com/hashicorp/vault-csi-provider/internal/version" "google.golang.org/grpc" "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/utils/pointer" pb "sigs.k8s.io/secrets-store-csi-driver/provider/v1alpha1" ) +const ( + namespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" +) + func main() { logger := hclog.Default() err := realMain(logger) @@ -41,6 +49,8 @@ func realMain(logger hclog.Logger) error { flag.BoolVar(&flags.Version, "version", false, "Prints the version information.") flag.StringVar(&flags.HealthAddr, "health-addr", ":8080", "Configure http listener for reporting health.") + flag.StringVar(&flags.HMACSecretName, "hmac-secret-name", "vault-csi-provider-hmac-key", "Configure the Kubernetes secret name that the provider creates to store an HMAC key for generating secret version hashes") + flag.StringVar(&flags.VaultAddr, "vault-addr", "", "Default address for connecting to Vault. Can also be specified via the VAULT_ADDR environment variable.") flag.StringVar(&flags.VaultMount, "vault-mount", "kubernetes", "Default Vault mount path for Kubernetes authentication.") flag.StringVar(&flags.VaultNamespace, "vault-namespace", "", "Default Vault namespace for Vault requests. Can also be specified via the VAULT_NAMESPACE environment variable.") @@ -95,16 +105,30 @@ func realMain(logger hclog.Logger) error { } defer listener.Close() - config, err := rest.InClusterConfig() + cfg, err := rest.InClusterConfig() if err != nil { return err } - clientset, err := kubernetes.NewForConfig(config) + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { return err } - srv := providerserver.NewServer(serverLogger, flags, clientset) + namespace, err := os.ReadFile(namespaceFile) + if err != nil { + return fmt.Errorf("failed to read namespace from file: %w", err) + } + hmacSecretSpec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: flags.HMACSecretName, + Namespace: string(namespace), + // TODO: Configurable labels and annotations? + }, + Immutable: pointer.Bool(true), + } + hmacGenerator := hmac.NewHMACGenerator(clientset, hmacSecretSpec) + + srv := providerserver.NewServer(serverLogger, flags, clientset, hmacGenerator) pb.RegisterCSIDriverProviderServer(server, srv) // Create health handler diff --git a/manifest_staging/deployment/vault-csi-provider.yaml b/manifest_staging/deployment/vault-csi-provider.yaml index dedec6a..f03b76d 100644 --- a/manifest_staging/deployment/vault-csi-provider.yaml +++ b/manifest_staging/deployment/vault-csi-provider.yaml @@ -1,6 +1,11 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: MPL-2.0 +apiVersion: v1 +kind: Namespace +metadata: + name: csi +--- apiVersion: v1 kind: ServiceAccount metadata: @@ -32,11 +37,35 @@ subjects: name: vault-csi-provider namespace: csi --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: vault-csi-provider-role +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "get"] + resourceNames: + - vault-csi-provider-hmac-key +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: vault-csi-provider-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vault-csi-provider-role +subjects: +- kind: ServiceAccount + name: vault-csi-provider + namespace: csi +--- apiVersion: apps/v1 kind: DaemonSet metadata: labels: - app: vault-csi-provider + app.kubernetes.io/name: vault-csi-provider name: vault-csi-provider namespace: csi spec: @@ -44,17 +73,17 @@ spec: type: RollingUpdate selector: matchLabels: - app: vault-csi-provider + app.kubernetes.io/name: vault-csi-provider template: metadata: labels: - app: vault-csi-provider + app.kubernetes.io/name: vault-csi-provider spec: serviceAccountName: vault-csi-provider tolerations: containers: - name: provider-vault-installer - image: hashicorp/vault-csi-provider:1.1.0 + image: hashicorp/vault-csi-provider:1.2.1 imagePullPolicy: Always args: - -endpoint=/provider/vault.sock diff --git a/test/bats/configs/nginx/templates/nginix.yaml b/test/bats/configs/nginx/templates/nginx.yaml similarity index 100% rename from test/bats/configs/nginx/templates/nginix.yaml rename to test/bats/configs/nginx/templates/nginx.yaml diff --git a/test/bats/configs/vault/hmac-secret-role.yaml b/test/bats/configs/vault/hmac-secret-role.yaml new file mode 100644 index 0000000..cce66ed --- /dev/null +++ b/test/bats/configs/vault/hmac-secret-role.yaml @@ -0,0 +1,30 @@ +# TODO: Remove when helm chart has this role baked in. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: vault-csi-provider-role +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + resourceNames: + - vault-csi-provider-hmac-key +# 'create' permissions cannot be restricted by resource name: +# https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources +- apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: vault-csi-provider-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vault-csi-provider-role +subjects: +- kind: ServiceAccount + name: vault-csi-provider + namespace: csi +--- \ No newline at end of file diff --git a/test/bats/configs/vault/vault.values.yaml b/test/bats/configs/vault/vault.values.yaml index 9372faa..fe41b9a 100644 --- a/test/bats/configs/vault/vault.values.yaml +++ b/test/bats/configs/vault/vault.values.yaml @@ -65,7 +65,21 @@ csi: - name: vault-client-tls secret: secretName: vault-client-tls + # TODO: Delete this volume when helm chart has it baked in. + - name: metadata + downwardAPI: + items: + - path: "labels" + fieldRef: + fieldPath: metadata.labels + - path: "annotations" + fieldRef: + fieldPath: metadata.annotations volumeMounts: - name: vault-client-tls mountPath: /mnt/tls readOnly: true + # TODO: Delete this mount when helm chart has it baked in. + - name: metadata + mountPath: "/var/run/metadata/kubernetes.io/pod" + readOnly: true diff --git a/test/bats/provider.bats b/test/bats/provider.bats index 4dba06a..822372a 100644 --- a/test/bats/provider.bats +++ b/test/bats/provider.bats @@ -341,3 +341,46 @@ teardown(){ result=$(kubectl --namespace=test exec nginx-kv-custom-audience -- cat /mnt/secrets-store/secret) [[ "$result" == "hello-custom-audience" ]] } + +@test "11 Consistent version hashes" { + helm --namespace=test install nginx $CONFIGS/nginx \ + --set engine=kv --set sa=kv \ + --wait --timeout=5m + + # HMAC secret should exist. + kubectl --namespace=csi get secrets vault-csi-provider-hmac-key + + # Save the status UID and secret versions. + statusUID1=$(kubectl --namespace=test get secretproviderclasspodstatus nginx-kv-test-vault-kv -o jsonpath='{.metadata.uid}') + versions1=$(kubectl --namespace=test get secretproviderclasspodstatus nginx-kv-test-vault-kv -o jsonpath='{.status.objects[*].version}') + + # Recreate the pod, which should remount the secrets and recreate the status object. + helm --namespace=test uninstall nginx + helm --namespace=test install nginx $CONFIGS/nginx \ + --set engine=kv --set sa=kv \ + --wait --timeout=5m + + # Now the uid should be different, but versions should still be the same. + statusUID2=$(kubectl --namespace=test get secretproviderclasspodstatus nginx-kv-test-vault-kv -o jsonpath='{.metadata.uid}') + versions2=$(kubectl --namespace=test get secretproviderclasspodstatus nginx-kv-test-vault-kv -o jsonpath='{.status.objects[*].version}') + + [[ "$statusUID1" != "$statusUID2" ]] + [[ "$versions1" == "$versions2" ]] + + # Finally, delete the HMAC secret and recreate the pod one more time. + # The HMAC secret should get regenerated and the secret versions should then change. + kubectl --namespace=csi delete secret vault-csi-provider-hmac-key + helm --namespace=test uninstall nginx + helm --namespace=test install nginx $CONFIGS/nginx \ + --set engine=kv --set sa=kv \ + --wait --timeout=5m + + kubectl --namespace=csi get secrets vault-csi-provider-hmac-key + + statusUID3=$(kubectl --namespace=test get secretproviderclasspodstatus nginx-kv-test-vault-kv -o jsonpath='{.metadata.uid}') + versions3=$(kubectl --namespace=test get secretproviderclasspodstatus nginx-kv-test-vault-kv -o jsonpath='{.status.objects[*].version}') + + [[ "$statusUID1" != "$statusUID3" ]] + [[ "$statusUID2" != "$statusUID3" ]] + [[ "$versions2" != "$versions3" ]] +}