Skip to content

Commit

Permalink
Merge pull request #121 from hyperspike/external-tls
Browse files Browse the repository at this point in the history 
feat(controller): Support TLS External Access
  • Loading branch information
@dmolik
dmolik authored Nov 26, 2024
2 parents 5d6e890 + de3a55f commit 0509620
Show file tree
Hide file tree
Showing 4 changed files with 172 additions and 22 deletions.
11 changes: 11 additions & 0 deletions api/v1/valkey_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,14 @@ type ExternalAccess struct {
// LoadBalancer Settings
LoadBalancer *LoadBalancerSettings `json:"loadBalancer,omitempty"`

// Cert Issuer for external access TLS certificate
CertIssuer string `json:"certIssuer,omitempty"`

// Cert Issuer Type for external access TLS certificate
// +kubebuilder:default:="ClusterIssuer"
// +kubebuilder:validation:Enum=ClusterIssuer;Issuer
CertIssuerType string `json:"certIssuerType,omitempty"`

// Support External DNS
// +kubebuilder:default:=false
ExternalDNS bool `json:"externalDNS,omitempty"`
Expand All @@ -118,6 +126,9 @@ type ProxySettings struct {
// Replicas for the proxy
// +kubebuilder:default:=1
Replicas *int32 `json:"replicas,omitempty"`

// External Hostname for the proxy
Hostname string `json:"hostname,omitempty"`
}

// LoadBalancerSettings defines the load balancer settings
Expand Down
13 changes: 13 additions & 0 deletions config/crd/bases/hyperspike.io_valkeys.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,16 @@ spec:
externalAccess:
description: External access configuration
properties:
certIssuer:
description: Cert Issuer for external access TLS certificate
type: string
certIssuerType:
default: ClusterIssuer
description: Cert Issuer Type for external access TLS certificate
enum:
- ClusterIssuer
- Issuer
type: string
enabled:
default: false
description: Enable external access
Expand Down Expand Up @@ -110,6 +120,9 @@ spec:
extraConfig:
description: Extra Envoy configuration
type: string
hostname:
description: External Hostname for the proxy
type: string
image:
default: envoyproxy/envoy:v1.32.1
description: Image to use for the proxy
Expand Down
2 changes: 1 addition & 1 deletion config/manager/kustomization.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ kind: Kustomization
images:
- name: controller
newName: localhost:5000/controller
newTag: "24"
newTag: "6"
168 changes: 147 additions & 21 deletions internal/controller/valkey_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,11 @@ func (r *ValkeyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
}
}
if externalAccess && externalType == "Proxy" {
if valkey.Spec.TLS {
if err := r.upsertProxyCertificate(ctx, valkey); err != nil {
return ctrl.Result{}, err
}
}
if err := r.upsertExternalAccessProxySecret(ctx, valkey); err != nil {
return ctrl.Result{}, err
}
Expand Down Expand Up @@ -436,7 +441,7 @@ func (r *ValkeyReconciler) setClusterAnnounceIp(ctx context.Context, valkey *hyp
opt := valkeyClient.ClientOption{
InitAddress: []string{address},
Password: password,
ForceSingleClient: true,
ForceSingleClient: true, // this is necessary to avoid failing through to another shard and setting the wrong ip
}
if valkey.Spec.TLS {
ca, err := r.getCACertificate(ctx, valkey)
Expand Down Expand Up @@ -485,21 +490,21 @@ func (r *ValkeyReconciler) setClusterAnnounceIp(ctx context.Context, valkey *hyp
}
time.Sleep(time.Second * 1)
}
/*
for podName, _ := range ips {
for shard, shardIp := range ips {
if shard == podName {
continue
}
time.Sleep(time.Second * 1)
logger.Info("node meeting peer", "valkey", valkey.Name, "namespace", valkey.Namespace, "peer", shardIp, "pod", podName)
r.Recorder.Event(valkey, "Normal", "Setting",
fmt.Sprintf("Node meeting peer %s on pod %s for %s/%s", shardIp, podName, valkey.Namespace, valkey.Name))
if err := clients[podName].Do(ctx, clients[podName].B().ClusterMeet().Ip(shardIp).Port(6379).Build()).Error(); err != nil {
logger.Error(err, "failed to cluster meet", "valkey", valkey.Name, "namespace", valkey.Namespace, "shard", shard, "ip", shardIp, "pod", podName)
}
/* this might be useful in the future
for podName, _ := range ips {
for shard, shardIp := range ips {
if shard == podName {
continue
}
time.Sleep(time.Second * 1)
logger.Info("node meeting peer", "valkey", valkey.Name, "namespace", valkey.Namespace, "peer", shardIp, "pod", podName)
r.Recorder.Event(valkey, "Normal", "Setting",
fmt.Sprintf("Node meeting peer %s on pod %s for %s/%s", shardIp, podName, valkey.Namespace, valkey.Name))
if err := clients[podName].Do(ctx, clients[podName].B().ClusterMeet().Ip(shardIp).Port(6379).Build()).Error(); err != nil {
logger.Error(err, "failed to cluster meet", "valkey", valkey.Name, "namespace", valkey.Namespace, "shard", shard, "ip", shardIp, "pod", podName)
}
}
}
*/
return nil
}
Expand Down Expand Up @@ -641,21 +646,113 @@ func (r *ValkeyReconciler) upsertExternalAccessProxySvc(ctx context.Context, val
return nil
}

func (r *ValkeyReconciler) upsertProxyCertificate(ctx context.Context, valkey *hyperv1.Valkey) error {
logger := log.FromContext(ctx)

logger.Info("upserting proxy certificate")

issuerName := ""
issuerKind := ""
if valkey.Spec.ExternalAccess != nil {
issuerName = valkey.Spec.ExternalAccess.CertIssuer
issuerKind = valkey.Spec.ExternalAccess.CertIssuerType
}
if issuerKind == "" {
issuerKind = valkey.Spec.CertIssuerType
}
if issuerName == "" {
issuerName = valkey.Spec.CertIssuer
}
cert := &certv1.Certificate{
ObjectMeta: metav1.ObjectMeta{
Name: valkey.Name + "-proxy",
Namespace: valkey.Namespace,
Labels: labels(valkey),
},
Spec: certv1.CertificateSpec{
SecretName: valkey.Name + "-proxy-tls",
IssuerRef: cmetav1.ObjectReference{
Name: issuerName,
Kind: issuerKind,
},
CommonName: valkey.Name + "-proxy",
DNSNames: []string{
valkey.Name + "-proxy",
valkey.Name + "-proxy." + valkey.Namespace,
valkey.Name + "-proxy." + valkey.Namespace + ".svc",
valkey.Name + "-proxy." + valkey.Namespace + ".svc." + valkey.Spec.ClusterDomain,
},
},
}
hostname := valkey.Spec.ExternalAccess.Proxy.Hostname
if hostname != "" {
cert.Spec.DNSNames = append(cert.Spec.DNSNames, hostname)
}
if err := controllerutil.SetControllerReference(valkey, cert, r.Scheme); err != nil {
return err
}
if err := r.Create(ctx, cert); err != nil {
if errors.IsAlreadyExists(err) {
if err := r.Update(ctx, cert); err != nil {
logger.Error(err, "failed to update proxy certificate")
return err
}
} else {
logger.Error(err, "failed to create proxy certificate")
return err
}
} else {
r.Recorder.Event(valkey, "Normal", "Created",
fmt.Sprintf("Certificate %s/%s is created", valkey.Namespace, valkey.Name+"-proxy-cert"))
}
return nil
}

func (r *ValkeyReconciler) upsertExternalAccessProxySecret(ctx context.Context, valkey *hyperv1.Valkey) error {
logger := log.FromContext(ctx)

logger.Info("upserting external proxy configmap")

endpoints := []string{}
for i := 0; i < int(valkey.Spec.Shards); i++ {
host := fmt.Sprintf("%s-%d.%s-headless.%s", valkey.Name, i, valkey.Name, valkey.Namespace)
host := fmt.Sprintf("%s-%d.%s-headless.%s.svc", valkey.Name, i, valkey.Name, valkey.Namespace)
endpoints = append(endpoints, ` - endpoint:
address:
socket_address:
address: `+host+`
port_value: 6379`)
}

tlsServer := ""
tlsClient := ""
if valkey.Spec.TLS {
tlsServer = ` transport_socket:
name: envoy.transport_sockets.tls
typed_config:
"@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext
#require_client_certificate: true # @TODO mtls auth
common_tls_context:
tls_certificates:
- certificate_chain:
filename: /etc/envoy/certs/tls.crt
private_key:
filename: /etc/envoy/certs/tls.key
validation_context:
trusted_ca:
filename: /etc/envoy/certs/ca.crt`
tlsClient = ` transport_socket:
name: envoy.transport_sockets.tls
typed_config:
"@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
common_tls_context:
#tls_certificates:
# certificate_chain:
# filename: "certs/client.crt"
# private_key:
# filename: "/etc/valkey/certs/client.key"
validation_context:
trusted_ca:
filename: "/etc/valkey/certs/ca.crt"`
}
password, err := r.GetPassword(ctx, valkey)
if err != nil {
logger.Error(err, "failed to get password")
Expand Down Expand Up @@ -686,11 +783,13 @@ static_resources:
stat_prefix: egress_redis
settings:
op_timeout: 5s
enable_redirection: false
prefix_routes:
catch_all_route:
cluster: redis_cluster
downstream_auth_password:
inline_string: "` + password + `"
` + tlsServer + `
clusters:
- name: redis_cluster
# type: STRICT_DNS # static
Expand All @@ -713,6 +812,7 @@ static_resources:
value:
auth_password:
inline_string: "` + password + `"
` + tlsClient + `
admin:
address:
socket_address:
Expand Down Expand Up @@ -817,6 +917,32 @@ func (r *ValkeyReconciler) upsertExternalAccessProxyDeployment(ctx context.Conte
},
},
}
if valkey.Spec.TLS {
proxyDeployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(proxyDeployment.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
Name: "envoy-certs",
MountPath: "/etc/envoy/certs",
})
proxyDeployment.Spec.Template.Spec.Volumes = append(proxyDeployment.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "envoy-certs",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: valkey.Name + "-proxy-tls",
},
},
})
proxyDeployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(proxyDeployment.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
Name: "valkey-certs",
MountPath: "/etc/valkey/certs",
})
proxyDeployment.Spec.Template.Spec.Volumes = append(proxyDeployment.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "valkey-certs",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: valkey.Name + "-tls",
},
},
})
}

if err := controllerutil.SetControllerReference(valkey, proxyDeployment, r.Scheme); err != nil {
return err
Expand Down Expand Up @@ -1573,11 +1699,11 @@ func (r *ValkeyReconciler) exporter(valkey *hyperv1.Valkey) corev1.Container {
{
Name: "REDIS_EXPORTER_TLS_CLIENT_CERT_FILE",
Value: "/etc/valkey/certs/tls.crt",
},
{
Name: "REDIS_EXPORTER_TLS_CA_FILE",
Value: "/etc/valkey/certs/ca.crt",
},*/
}, */
{
Name: "REDIS_EXPORTER_TLS_CA_FILE",
Value: "/etc/valkey/certs/ca.crt",
},
}
container.Env = append(container.Env, tlsExporterEnv...)
}
Expand Down

0 comments on commit 0509620

Please sign in to comment.