Skip to content

Commit

Permalink
Support Vault via vault-k8s (#846)
Browse files Browse the repository at this point in the history 
Co-authored-by: Marcial Rosales <[email protected]>
  • Loading branch information
@ansd @MarcialRosales
ansd and MarcialRosales authored Sep 28, 2021
1 parent 82cfd8b commit e4e4271
Show file tree
Hide file tree
Showing 24 changed files with 1,240 additions and 229 deletions.
86 changes: 85 additions & 1 deletion api/v1beta1/rabbitmqcluster_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
package v1beta1

import (
"fmt"
"strconv"
"strings"

Expand Down Expand Up @@ -84,6 +85,70 @@ type RabbitmqClusterSpec struct {
// +kubebuilder:validation:Minimum:=0
// +kubebuilder:default:=604800
TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
// Secret backend configuration for the RabbitmqCluster.
// Enables to fetch default user credentials and certificates from K8s external secret stores.
SecretBackend SecretBackend `json:"secretBackend,omitempty"`
}

// SecretBackend configures a single secret backend.
// Today, only Vault exists as supported secret backend.
// Future secret backends could be Secrets Store CSI Driver.
// If not configured, K8s Secrets will be used.
type SecretBackend struct {
Vault *VaultSpec `json:"vault,omitempty"`
}

// VaultSpec will add Vault annotations (see https://www.vaultproject.io/docs/platform/k8s/injector/annotations)
// to RabbitMQ Pods. It requires a Vault Agent Sidecar Injector (https://www.vaultproject.io/docs/platform/k8s/injector)
// to be installed in the K8s cluster. The injector is a K8s Mutation Webhook Controller that alters RabbitMQ Pod specifications
// (based on the added Vault annotations) to include Vault Agent containers that render Vault secrets to the volume.
type VaultSpec struct {
// Role in Vault.
// If vault.defaultUserPath is set, this role must have capability to read the pre-created default user credential in Vault.
// If vault.tls is set, this role must have capability to create and update certificates in the Vault PKI engine for the domains
// "<namespace>" and "<namespace>.svc".
Role string `json:"role,omitempty"`
// Vault annotations that override the Vault annotations set by the cluster-operator.
// For a list of valid Vault annotations, see https://www.vaultproject.io/docs/platform/k8s/injector/annotations
// +optional
Annotations map[string]string `json:"annotations,omitempty"`
// Path in Vault to access a KV (Key-Value) secret with the fields username and password for the default user.
// For example "secret/data/rabbitmq/config".
DefaultUserPath string `json:"defaultUserPath,omitempty"`
// Sidecar container that updates the default user's password in RabbitMQ when it changes in Vault.
// Additionally, it updates /var/lib/rabbitmq/.rabbitmqadmin.conf (used by rabbitmqadmin CLI).
// Set to empty string to disable the sidecar container.
// +kubebuilder:default:="rabbitmqoperator/default-user-credential-updater:0.1.1"
DefaultUserUpdaterImage *string `json:"defaultUserUpdaterImage,omitempty"`
TLS VaultTLSSpec `json:"tls,omitempty"`
}

type VaultTLSSpec struct {
// Path in Vault PKI engine.
// For example "pki/issue/hashicorp-com".
// required
PKIIssuerPath string `json:"pkiIssuerPath,omitempty"`
// Specifies the requested certificate Common Name (CN).
// Defaults to <serviceName>.<namespace>.svc if not provided.
// +optional
CommonName string `json:"commonName,omitempty"`
// Specifies the requested Subject Alternative Names (SANs), in a comma-delimited list.
// These will be appended to the SANs added by the cluster-operator.
// The cluster-operator will add SANs:
// "<RabbitmqCluster name>-server-<index>.<RabbitmqCluster name>-nodes.<namespace>" for each pod,
// e.g. "myrabbit-server-0.myrabbit-nodes.default".
// +optional
AltNames string `json:"altNames,omitempty"`
// Specifies the requested IP Subject Alternative Names, in a comma-delimited list.
// +optional
IpSans string `json:"ipSans,omitempty"`
}

func (spec *VaultSpec) TLSEnabled() bool {
return spec.TLS.PKIIssuerPath != ""
}
func (spec *VaultSpec) DefaultUserSecretEnabled() bool {
return spec.DefaultUserPath != ""
}

// Provides the ability to override the generated manifest of several child resources.
Expand Down Expand Up @@ -323,11 +388,14 @@ type RabbitmqClusterServiceSpec struct {
}

func (cluster *RabbitmqCluster) TLSEnabled() bool {
return cluster.SecretTLSEnabled() || cluster.VaultTLSEnabled()
}
func (cluster *RabbitmqCluster) SecretTLSEnabled() bool {
return cluster.Spec.TLS.SecretName != ""
}

func (cluster *RabbitmqCluster) MutualTLSEnabled() bool {
return cluster.TLSEnabled() && cluster.Spec.TLS.CaSecretName != ""
return (cluster.SecretTLSEnabled() && cluster.Spec.TLS.CaSecretName != "") || cluster.VaultTLSEnabled()
}

func (cluster *RabbitmqCluster) MemoryLimited() bool {
Expand Down Expand Up @@ -356,6 +424,22 @@ func (cluster *RabbitmqCluster) StreamNeeded() bool {
return cluster.AdditionalPluginEnabled("rabbitmq_stream") || cluster.AdditionalPluginEnabled("rabbitmq_multi_dc_replication")
}

func (cluster *RabbitmqCluster) VaultEnabled() bool {
return cluster.Spec.SecretBackend.Vault != nil
}

func (cluster *RabbitmqCluster) VaultDefaultUserSecretEnabled() bool {
return cluster.VaultEnabled() && cluster.Spec.SecretBackend.Vault.DefaultUserSecretEnabled()
}

func (cluster *RabbitmqCluster) VaultTLSEnabled() bool {
return cluster.VaultEnabled() && cluster.Spec.SecretBackend.Vault.TLSEnabled()
}

func (cluster *RabbitmqCluster) ServiceSubDomain() string {
return fmt.Sprintf("%s.%s.svc", cluster.Name, cluster.Namespace)
}

// +kubebuilder:object:root=true

// RabbitmqClusterList contains a list of RabbitmqClusters.
Expand Down
136 changes: 86 additions & 50 deletions api/v1beta1/rabbitmqcluster_types_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ package v1beta1
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
. "github.com/onsi/gomega/gstruct"
"github.com/rabbitmq/cluster-operator/internal/status"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand All @@ -31,39 +32,39 @@ var _ = Describe("RabbitmqCluster", func() {
It("can be created with a single replica", func() {
created := generateRabbitmqClusterObject("rabbit1")
created.Spec.Replicas = pointer.Int32Ptr(1)
Expect(k8sClient.Create(context.TODO(), created)).To(Succeed())
Expect(k8sClient.Create(context.Background(), created)).To(Succeed())

fetched := &RabbitmqCluster{}
Expect(k8sClient.Get(context.TODO(), getKey(created), fetched)).To(Succeed())
Expect(k8sClient.Get(context.Background(), getKey(created), fetched)).To(Succeed())
Expect(fetched).To(Equal(created))
})

It("can be created with three replicas", func() {
created := generateRabbitmqClusterObject("rabbit2")
created.Spec.Replicas = pointer.Int32Ptr(3)
Expect(k8sClient.Create(context.TODO(), created)).To(Succeed())
Expect(k8sClient.Create(context.Background(), created)).To(Succeed())

fetched := &RabbitmqCluster{}
Expect(k8sClient.Get(context.TODO(), getKey(created), fetched)).To(Succeed())
Expect(k8sClient.Get(context.Background(), getKey(created), fetched)).To(Succeed())
Expect(fetched).To(Equal(created))
})

It("can be created with five replicas", func() {
created := generateRabbitmqClusterObject("rabbit3")
created.Spec.Replicas = pointer.Int32Ptr(5)
Expect(k8sClient.Create(context.TODO(), created)).To(Succeed())
Expect(k8sClient.Create(context.Background(), created)).To(Succeed())

fetched := &RabbitmqCluster{}
Expect(k8sClient.Get(context.TODO(), getKey(created), fetched)).To(Succeed())
Expect(k8sClient.Get(context.Background(), getKey(created), fetched)).To(Succeed())
Expect(fetched).To(Equal(created))
})

It("can be deleted", func() {
created := generateRabbitmqClusterObject("rabbit4")
Expect(k8sClient.Create(context.TODO(), created)).To(Succeed())
Expect(k8sClient.Create(context.Background(), created)).To(Succeed())

Expect(k8sClient.Delete(context.TODO(), created)).To(Succeed())
Expect(k8sClient.Get(context.TODO(), getKey(created), created)).ToNot(Succeed())
Expect(k8sClient.Delete(context.Background(), created)).To(Succeed())
Expect(k8sClient.Get(context.Background(), getKey(created), created)).ToNot(Succeed())
})

It("can be created with resource requests", func() {
Expand All @@ -78,13 +79,13 @@ var _ = Describe("RabbitmqCluster", func() {
corev1.ResourceMemory: k8sresource.MustParse("100Mi"),
},
}
Expect(k8sClient.Create(context.TODO(), created)).To(Succeed())
Expect(k8sClient.Create(context.Background(), created)).To(Succeed())
})

It("can be created with server side TLS", func() {
created := generateRabbitmqClusterObject("rabbit-tls")
created.Spec.TLS.SecretName = "tls-secret-name"
Expect(k8sClient.Create(context.TODO(), created)).To(Succeed())
Expect(k8sClient.Create(context.Background(), created)).To(Succeed())
})

It("can be queried if TLS is enabled", func() {
Expand Down Expand Up @@ -125,15 +126,15 @@ var _ = Describe("RabbitmqCluster", func() {
By("checking the replica count", func() {
invalidReplica := generateRabbitmqClusterObject("rabbit4")
invalidReplica.Spec.Replicas = pointer.Int32Ptr(-1)
Expect(apierrors.IsInvalid(k8sClient.Create(context.TODO(), invalidReplica))).To(BeTrue())
Expect(k8sClient.Create(context.TODO(), invalidReplica)).To(MatchError(ContainSubstring("spec.replicas in body should be greater than or equal to 0")))
Expect(apierrors.IsInvalid(k8sClient.Create(context.Background(), invalidReplica))).To(BeTrue())
Expect(k8sClient.Create(context.Background(), invalidReplica)).To(MatchError(ContainSubstring("spec.replicas in body should be greater than or equal to 0")))
})

By("checking the service type", func() {
invalidService := generateRabbitmqClusterObject("rabbit5")
invalidService.Spec.Service.Type = "ihateservices"
Expect(apierrors.IsInvalid(k8sClient.Create(context.TODO(), invalidService))).To(BeTrue())
Expect(k8sClient.Create(context.TODO(), invalidService)).To(MatchError(ContainSubstring("supported values: \"ClusterIP\", \"LoadBalancer\", \"NodePort\"")))
Expect(apierrors.IsInvalid(k8sClient.Create(context.Background(), invalidService))).To(BeTrue())
Expect(k8sClient.Create(context.Background(), invalidService)).To(MatchError(ContainSubstring("supported values: \"ClusterIP\", \"LoadBalancer\", \"NodePort\"")))
})
})

Expand Down Expand Up @@ -163,12 +164,9 @@ var _ = Describe("RabbitmqCluster", func() {
},
}

Expect(k8sClient.Create(context.TODO(), &rmqClusterInstance)).To(Succeed())
Expect(k8sClient.Create(context.Background(), &rmqClusterInstance)).To(Succeed())
fetchedRabbit := &RabbitmqCluster{}
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{
Name: "rabbitmq-defaults",
Namespace: "default",
}, fetchedRabbit)).To(Succeed())
Expect(k8sClient.Get(context.Background(), getKey(&rmqClusterInstance), fetchedRabbit)).To(Succeed())
Expect(fetchedRabbit.Spec).To(Equal(expectedClusterInstance.Spec))
})
})
Expand Down Expand Up @@ -241,12 +239,9 @@ var _ = Describe("RabbitmqCluster", func() {
},
}

Expect(k8sClient.Create(context.TODO(), &rmqClusterInstance)).To(Succeed())
Expect(k8sClient.Create(context.Background(), &rmqClusterInstance)).To(Succeed())
fetchedRabbit := &RabbitmqCluster{}
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{
Name: "rabbitmq-full-manifest",
Namespace: "default",
}, fetchedRabbit)).To(Succeed())
Expect(k8sClient.Get(context.Background(), getKey(&rmqClusterInstance), fetchedRabbit)).To(Succeed())
Expect(fetchedRabbit.Spec).To(Equal(rmqClusterInstance.Spec))
})
})
Expand All @@ -266,12 +261,9 @@ var _ = Describe("RabbitmqCluster", func() {

expectedClusterInstance.Spec.Image = "test-image"

Expect(k8sClient.Create(context.TODO(), &rmqClusterInstance)).To(Succeed())
Expect(k8sClient.Create(context.Background(), &rmqClusterInstance)).To(Succeed())
fetchedRabbit := &RabbitmqCluster{}
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{
Name: "rabbitmq-image",
Namespace: "default",
}, fetchedRabbit)).To(Succeed())
Expect(k8sClient.Get(context.Background(), getKey(&rmqClusterInstance), fetchedRabbit)).To(Succeed())
Expect(fetchedRabbit.Spec).To(Equal(expectedClusterInstance.Spec))
})

Expand All @@ -289,12 +281,9 @@ var _ = Describe("RabbitmqCluster", func() {

expectedClusterInstance.Spec.Resources = expectedResources

Expect(k8sClient.Create(context.TODO(), &rmqClusterInstance)).To(Succeed())
Expect(k8sClient.Create(context.Background(), &rmqClusterInstance)).To(Succeed())
fetchedRabbit := &RabbitmqCluster{}
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{
Name: "rabbitmq-empty-resource",
Namespace: "default",
}, fetchedRabbit)).To(Succeed())
Expect(k8sClient.Get(context.Background(), getKey(&rmqClusterInstance), fetchedRabbit)).To(Succeed())
Expect(fetchedRabbit.Spec).To(Equal(expectedClusterInstance.Spec))
})

Expand All @@ -316,12 +305,9 @@ var _ = Describe("RabbitmqCluster", func() {

expectedClusterInstance.Spec.Resources = expectedResources

Expect(k8sClient.Create(context.TODO(), &rmqClusterInstance)).To(Succeed())
Expect(k8sClient.Create(context.Background(), &rmqClusterInstance)).To(Succeed())
fetchedRabbit := &RabbitmqCluster{}
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{
Name: "rabbitmq-partial-resource",
Namespace: "default",
}, fetchedRabbit)).To(Succeed())
Expect(k8sClient.Get(context.Background(), getKey(&rmqClusterInstance), fetchedRabbit)).To(Succeed())
Expect(fetchedRabbit.Spec).To(Equal(expectedClusterInstance.Spec))
})

Expand All @@ -343,12 +329,9 @@ var _ = Describe("RabbitmqCluster", func() {
Type: "ClusterIP",
}

Expect(k8sClient.Create(context.TODO(), &rmqClusterInstance)).To(Succeed())
Expect(k8sClient.Create(context.Background(), &rmqClusterInstance)).To(Succeed())
fetchedRabbit := &RabbitmqCluster{}
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{
Name: "rabbit-service-type",
Namespace: "default",
}, fetchedRabbit)).To(Succeed())
Expect(k8sClient.Get(context.Background(), getKey(&rmqClusterInstance), fetchedRabbit)).To(Succeed())
Expect(fetchedRabbit.Spec).To(Equal(expectedClusterInstance.Spec))
})

Expand All @@ -372,16 +355,69 @@ var _ = Describe("RabbitmqCluster", func() {
Storage: &tenGi,
}

Expect(k8sClient.Create(context.TODO(), &rmqClusterInstance)).To(Succeed())
Expect(k8sClient.Create(context.Background(), &rmqClusterInstance)).To(Succeed())
fetchedRabbit := &RabbitmqCluster{}
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{
Name: "rabbit-storage",
Namespace: "default",
}, fetchedRabbit)).To(Succeed())
Expect(k8sClient.Get(context.Background(), getKey(&rmqClusterInstance), fetchedRabbit)).To(Succeed())
Expect(fetchedRabbit.Spec).To(Equal(expectedClusterInstance.Spec))
})
})
})
Context("Vault", func() {
It("is disabled by default", func() {
rabbit := generateRabbitmqClusterObject("rabbit-without-vault")
Expect(k8sClient.Create(context.Background(), rabbit)).To(Succeed())
fetchedRabbit := &RabbitmqCluster{}
Expect(k8sClient.Get(context.Background(), getKey(rabbit), fetchedRabbit)).To(Succeed())

Expect(fetchedRabbit.VaultEnabled()).To(BeFalse())
})
When("only default user is configured", func() {
It("sets vault configuration correctly", func() {
rabbit := generateRabbitmqClusterObject("rabbit-vault-default-user")
rabbit.Spec.SecretBackend.Vault = &VaultSpec{
Role: "test-role",
DefaultUserPath: "test-path",
}
Expect(k8sClient.Create(context.Background(), rabbit)).To(Succeed())
fetchedRabbit := &RabbitmqCluster{}
Expect(k8sClient.Get(context.Background(), getKey(rabbit), fetchedRabbit)).To(Succeed())

Expect(fetchedRabbit.Spec.SecretBackend.Vault.Role).To(Equal("test-role"))
Expect(fetchedRabbit.Spec.SecretBackend.Vault.DefaultUserPath).To(Equal("test-path"))
Expect(fetchedRabbit.VaultEnabled()).To(BeTrue())
Expect(fetchedRabbit.VaultDefaultUserSecretEnabled()).To(BeTrue())
Expect(fetchedRabbit.Spec.SecretBackend.Vault.DefaultUserSecretEnabled()).To(BeTrue())
Expect(fetchedRabbit.VaultTLSEnabled()).To(BeFalse())
Expect(fetchedRabbit.Spec.SecretBackend.Vault.TLSEnabled()).To(BeFalse())

By("setting the default-user-credential-updater image by default")
Expect(fetchedRabbit.Spec.SecretBackend.Vault.DefaultUserUpdaterImage).To(
PointTo(HavePrefix("rabbitmqoperator/default-user-credential-updater:")))
})
})
When("only TLS is configured", func() {
It("sets vault configuration correctly", func() {
rabbit := generateRabbitmqClusterObject("rabbit-vault-tls")
rabbit.Spec.SecretBackend.Vault = &VaultSpec{
Role: "test-role",
TLS: VaultTLSSpec{
PKIIssuerPath: "pki/issue/hashicorp-com",
},
}
Expect(k8sClient.Create(context.Background(), rabbit)).To(Succeed())
fetchedRabbit := &RabbitmqCluster{}
Expect(k8sClient.Get(context.Background(), getKey(rabbit), fetchedRabbit)).To(Succeed())

Expect(fetchedRabbit.Spec.SecretBackend.Vault.Role).To(Equal("test-role"))
Expect(fetchedRabbit.Spec.SecretBackend.Vault.TLS.PKIIssuerPath).To(Equal("pki/issue/hashicorp-com"))
Expect(fetchedRabbit.VaultEnabled()).To(BeTrue())
Expect(fetchedRabbit.VaultDefaultUserSecretEnabled()).To(BeFalse())
Expect(fetchedRabbit.Spec.SecretBackend.Vault.DefaultUserSecretEnabled()).To(BeFalse())
Expect(fetchedRabbit.VaultTLSEnabled()).To(BeTrue())
Expect(fetchedRabbit.Spec.SecretBackend.Vault.TLSEnabled()).To(BeTrue())
})
})
})
})
Context("RabbitmqClusterStatus", func() {
It("sets conditions based on inputs", func() {
Expand Down
Loading

0 comments on commit e4e4271

Please sign in to comment.