feat: support replicated architecture w/ sentinel

.github/workflows/test.yaml

@@ -61,4 +61,5 @@ jobs:
       upgrade-flavors: ${{ needs.check-flavor.outputs.upgrade-flavors }}
       flavor: ${{ matrix.flavor }}
       type: ${{ matrix.type }}
-    secrets: inherit # Inherits all secrets from the parent workflow.
+      runsOn: uds-swf-ubuntu-big-boy-8-core
+    secrets: inherit  # Inherits all secrets from the parent workflow.

bundle/uds-bundle.yaml

@@ -24,17 +24,19 @@ packages:
                 - direction: Ingress
                   selector:
                     app.kubernetes.io/name: valkey
-                  remoteNamespace: valkey-cli
+                  remoteNamespace: valkey-test
                   remoteSelector:
-                    app: valkey-cli
+                    app: valkey-test
                   port: 6379
                   description: "Ingress from Valkey CLI (for tests)"
             - path: copyPassword
               value:
                 enabled: true
-                namespace: valkey-cli
-                secretName: valkey
-                secretKey: valkey-password
+                namespace: valkey-test
+                secretName: valkey-standalone
+                # This allows us to mount it in as an env var and the valkey-cli picks it right up. Note: in
+                # production, mount this in as a file instead. It's less secure to use env variables for passwords.
+                secretKey: REDISCLI_AUTH
         valkey:
           variables:
             - name: VALKEY_RESOURCES
@@ -46,3 +48,56 @@ packages:
                 requests:
                   cpu: 100m
                   memory: 300Mi
+  - name: valkey
+    path: ../
+    # x-release-please-start-version
+    ref: 8.0.1-uds.0
+    # x-release-please-end
+    overrides:
+      valkey:
+        uds-valkey-config:
+          namespace: valkey-replicated-w-sentinel
+          values:
+            - path: custom
+              value:
+                - direction: Ingress
+                  selector:
+                    app.kubernetes.io/name: valkey
+                  remoteNamespace: valkey-test
+                  remoteSelector:
+                    app: valkey-test
+                  port: 6379
+                  description: "Ingress from Valkey CLI (for tests)"
+                - direction: Ingress
+                  selector:
+                    app.kubernetes.io/name: valkey
+                  remoteNamespace: valkey-test
+                  remoteSelector:
+                    app: valkey-test
+                  port: 26379
+                  description: "Ingress from Valkey CLI (for tests) sentinel"
+            - path: copyPassword
+              value:
+                enabled: true
+                namespace: valkey-test
+                secretName: valkey-replicated-w-sentinel
+                secretKey: REDISCLI_AUTH
+            - path: replicas
+              value: 3  # Because replication is enabled and will spin up 3 replicas by default
+        valkey:
+          namespace: valkey-replicated-w-sentinel
+          values:
+            - path: architecture
+              value: replication
+            - path: sentinel.enabled  # https://github.com/bitnami/charts/blob/main/bitnami/valkey/values.yaml#L1143
+              value: true
+          variables:
+            - name: VALKEY_RESOURCES
+              path: "master.resources"
+              default:
+                limits:
+                  cpu: 100m
+                  memory: 300Mi
+                requests:
+                  cpu: 100m
+                  memory: 300Mi

chart/templates/networking.yaml

@@ -0,0 +1,36 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+# This removes the NR filter_not_found error that you get when you try to access
+# the primary node.
+{{- $replicas := int .Values.replicas }}
+{{- range $i, $e := until $replicas }}
+---
+apiVersion: networking.istio.io/v1beta1
+kind: ServiceEntry
+metadata:
+  name: valkey-headless-{{ $i }}
+  namespace: {{ $.Release.Namespace }}
+spec:
+  hosts:
+    - valkey-node-{{ $i }}.valkey-headless.{{ $.Release.Namespace }}.svc.cluster.local  # Matches pod-specific DNS names
+  ports:
+    - number: 6379
+      name: redis
+      protocol: TCP
+  location: MESH_INTERNAL
+  resolution: NONE
+---
+# This enables node-to-node communications
+apiVersion: networking.istio.io/v1beta1
+kind: DestinationRule
+metadata:
+  name: valkey-headless-{{ $i }}
+  namespace: {{ $.Release.Namespace }}
+spec:
+  host: valkey-node-{{ $i }}.valkey-headless.{{ $.Release.Namespace }}.svc.cluster.local
+  trafficPolicy:
+    tls:
+      mode: ISTIO_MUTUAL
+      sni: valkey-node-{{ $i }}.valkey-headless.{{ $.Release.Namespace }}.svc.cluster.local
+{{- end }}

chart/templates/uds-package.yaml

@@ -13,7 +13,6 @@ spec:
       targetPort: 9121
       portName: http-metrics
       description: Metrics
-
   network:
     allow:
       - direction: Ingress

chart/values.yaml

@@ -7,4 +7,6 @@ copyPassword:
   secretName: ""
   secretKey: ""
 
+replicas: 0
+
 custom: []

common/zarf.yaml

@@ -21,22 +21,3 @@ components:
         url: oci://registry-1.docker.io/bitnamicharts/valkey
         valuesFiles:
           - ../values/values.yaml
-    actions:
-      onDeploy:
-        after:
-          - description: Validate Valkey Package
-            maxTotalSeconds: 300
-            wait:
-              cluster:
-                kind: packages.uds.dev
-                name: valkey
-                namespace: valkey
-                condition: "'{.status.phase}'=Ready"
-          - description: Valkey to be Healthy
-            maxTotalSeconds: 90
-            wait:
-              cluster:
-                kind: pod
-                name: app.kubernetes.io/name=valkey
-                namespace: valkey
-                condition: Ready

docs/configuration.md

@@ -16,3 +16,76 @@ Valkey is currently configured to expect a single user or workload to be using i
 - `copyPassword.namespace`: the namespace to copy the Kubernetes secret into
 - `copyPassword.secretName`: the name to give the Kubernetes secret in the other namespace
 - `copyPassword.secretKey`: the key to place the password under within the Kubernetes secret
+
+## High Availability
+
+The default Valkey configuration is a single read/write node, which is sufficient for most use cases. For example, GitLab recommends a single-node architecture even in [their 2,000 user reference architecture](https://docs.gitlab.com/ee/administration/reference_architectures/2k_users.html). They only suggest replication starting with [their 3000 user reference architecture](https://docs.gitlab.com/ee/administration/reference_architectures/3k_users.html) (note that the pages linked refer to Redis, for which Valkey is a drop-in replacement). For scenarios requiring higher availability, this package also supports a replicated architecture.
+
+### Configuration Changes
+
+The configuration changes required to switch from the standalone to the replicated (with sentinel) architecture can be derived by comparing the two valkey instances in the [test bundle definition](../bundle/uds-bundle.yaml). The changes are as follows:
+
+1. Add an ingress to the sentinel port in the `custom` network rules in the `uds-valkey-config` chart. This is unnecessary if not enabling the sentinel later (not recommended).
+
+```yaml
+packages:
+  - name: valkey
+    overrides:
+      valkey:
+        uds-valkey-config:
+          values:
+            - direction: Ingress
+                selector:
+                app.kubernetes.io/name: valkey
+                remoteNamespace: <your application's namespace>
+                remoteSelector:
+                app: <your application's UDS Package app name>
+                port: 26379
+                description: "Ingress from <your application> to sentinel"
+```
+
+2. Set the desired number of `replicas` in the `uds-valkey-config` chart. Note this defaults to zero for the standalone instance in order to prevent the creation of network policies which are only needed to support Valkey's clustering behavior.
+
+```yaml
+packages:
+  - name: valkey
+    overrides:
+      valkey:
+        uds-valkey-config:
+          values:
+            - path: replicas
+              value: 3
+```
+
+3. Set the [`architecture` variable in the upstream valkey chart](https://github.com/bitnami/charts/blob/main/bitnami/valkey/values.yaml#L128) to `replication` and turn on the sentinel (recommended).
+
+```yaml
+packages:
+  - name: valkey
+    overrides:
+      valkey:
+        valkey:
+          values:
+            - path: architecture
+              value: replication
+            - path: sentinel.enabled
+              value: true
+```
+
+### Impacts
+
+This high-availability configuration will result in a few changes, some obvious, some less obvious:
+
+1. The single `valkey-master` pod will be replaced by pods named `valkey-node-0`, `valkey-node-1`, and so on per the requested number of `replicas`.
+2. Every `valkey-node` pod will now includes a Sentinel sidecar. It is accessed by contacting the valkey service at the Sentinel port `26379` rather than the read/write port `6379`.
+3. As may be guessed from those two changes, the valkey service name also changes from `valkey-master.<valkey namespace>.svc.cluster.local` to, depending on your use-case:
+
+    - `valkey.<valkey namespace>.svc.cluster.local:26379` if trying to access a Sentinel.
+    - `valkey.<valkey namespace>.svc.cluster.local:6379` if trying to _read_ data.
+    - `valkey-node-<?>.valkey-headless.<valkey namespace>.svc.cluster.local:6379` if trying to _write_ data.
+
+      > Note the `<?>` in that address. The write node (called the Primary node) is only known by asking a Sentinel for the address and can change dynamically. Calling the sentinel to know where the primary node is should be handled by the calling application and so not relevant to most bundle development. If a Redis-ready application is given the address of the Sentinel service and the _read_ service that should be enough. For further clarity, see the Redis or Valkey documentation and review the tests for this application package where the Valkey CLI is used to communicate with both the standalone and replicated instances defined in the test bundle.
+
+### Alternative: Replication without Sentinel
+
+If the `sentinel.enabled` value above is set to `false` then one node will be the primary, the others read-replicas, and valkey will be unable to recover from the loss of that primary node. This is not recommended. The write node address will also need to be given at deploy-time to client applications. This configuration should be entirely possible with minimal if any changes to the UDS Package but is an exercise left to the reader.

tasks.yaml

@@ -34,7 +34,6 @@ tasks:
     actions:
       - task: create:test-bundle
       - task: deploy:test-bundle
-      - task: setup:create-doug-user
       - task: test:all
 
   - name: dev
@@ -60,7 +59,6 @@ tasks:
       - task: upgrade:create-latest-tag-bundle
       - task: setup:k3d-test-cluster
       - task: deploy:test-bundle
-      - task: setup:create-doug-user
       - task: compliance:validate
       - task: create-dev-package
       - task: create-deploy-test-bundle

tasks/test.yaml

@@ -4,22 +4,15 @@
 tasks:
   - name: all
     actions:
-      - task: health-check
-      - task: setup-data-stores
+      - task: create-test-package
+      - task: test-valkey
 
-  - name: setup-data-stores
+  - name: create-test-package
+    description: Create the test package to confirm Valkey is working
     actions:
-      - description: Create the data store test package for the Valkey instance
-        cmd: uds zarf package create tests --confirm --no-progress --architecture="${UDS_ARCH}" --skip-sbom --no-progress
-      - description: Deploy the test package into the cluster
-        cmd: uds zarf package deploy "zarf-package-valkey-test-${UDS_ARCH}-0.1.0.tar.zst" --confirm --no-progress
+      - cmd: uds zarf package create tests --architecture="${UDS_ARCH}" --confirm --skip-sbom
 
-  - name: health-check
+  - name: test-valkey
     actions:
-      - description: Valkey Status
-        wait:
-          cluster:
-            kind: pod
-            name: app.kubernetes.io/name=valkey
-            namespace: valkey
-            condition: Ready
+      - description: Deploy the test package into the cluster
+        cmd: uds zarf package deploy "zarf-package-valkey-test-${UDS_ARCH}-0.1.0.tar.zst" --confirm