feat: config to enable resilient registry (#2440)

## Description

- Provided fields for affinity and toleration of the registry pod
- Added s3 irsa configurability for the docker registry helm chart if
using a compatible image

## Related Issue

Relates to #375

## Checklist before merging

- [x] Test, docs, adr added or updated as needed
- [x] [Contributor Guide
Steps](https://github.com/defenseunicorns/zarf/blob/main/.github/CONTRIBUTING.md#developer-workflow)
followed

---------

Co-authored-by: Zack A <[email protected]>
Co-authored-by: corang <[email protected]>
Co-authored-by: Zack Annexstein <[email protected]>
Co-authored-by: Lucas Rodriguez <[email protected]>
Co-authored-by: Lucas Rodriguez <[email protected]>
Co-authored-by: razzle <[email protected]>
Co-authored-by: Austin Abro <[email protected]>

packages/zarf-registry/chart/templates/_helpers.tpl

@@ -23,6 +23,37 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- end -}}
 {{- end -}}
 
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "docker-registry.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Selector labels
+*/}}
+{{- define "docker-registry.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "docker-registry.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "docker-registry.labels" -}}
+{{ include "docker-registry.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/part-of: {{ .Release.Name }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+helm.sh/chart: {{ include "docker-registry.chart" . }}
+{{- with .Values.customLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 {{/*
 Merge all configmaps
 */}}
@@ -34,3 +65,14 @@ Merge all configmaps
 {{ .Values.caBundle | indent 6 }}
 {{- end }}
 {{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "docker-registry.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "docker-registry.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}

packages/zarf-registry/chart/templates/deployment.yaml

@@ -26,6 +26,7 @@ spec:
       annotations:
         checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
     spec:
+      serviceAccountName: {{ include "docker-registry.serviceAccountName" . }}
       {{- if .Values.imagePullSecrets }}
       imagePullSecrets:
 {{ toYaml .Values.imagePullSecrets | indent 8 }}
@@ -83,7 +84,11 @@ spec:
               subPath: ca-certificates.crt
               readOnly: true
 {{- end }}
+{{- if .Values.affinity.enabled }}
       affinity:
+{{- if .Values.affinity.custom }}
+{{ toYaml .Values.affinity.custom | indent 8 }}
+{{- else }}
 {{- if (eq "ReadWriteMany" .Values.persistence.accessMode) }}
         podAntiAffinity:
 {{- else }}
@@ -99,6 +104,12 @@ spec:
                       values:
                         - {{ template "docker-registry.name" . }}
                 topologyKey: kubernetes.io/hostname
+{{- end }}
+{{- end }}
+{{- if .Values.tolerations}}
+      tolerations:
+{{ toYaml .Values.tolerations | indent 8 }}
+{{- end }}
       volumes:
         - name: config
           secret:

packages/zarf-registry/chart/templates/hpa.yaml

@@ -13,8 +13,13 @@ spec:
     apiVersion: apps/v1
     kind: Deployment
     name: {{ template "docker-registry.fullname" . }}
+{{- if .Values.autoscaling.mapReplicasToNodes }}
+  minReplicas: {{ len (lookup "v1" "Node" "" "") }}
+  maxReplicas: {{ add (len (lookup "v1" "Node" "" "")) 4 }}
+{{- else }}
   minReplicas: {{ .Values.autoscaling.minReplicas }}
   maxReplicas: {{ .Values.autoscaling.maxReplicas }}
+{{- end }}
   metrics:
     - type: Resource
       resource:

packages/zarf-registry/chart/templates/serviceaccount.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: {{ .Values.namespace | default .Release.Namespace }}
+  name: {{ template "docker-registry.serviceAccountName" . }}
+  labels:
+    {{- include "docker-registry.labels" . | nindent 4 }}
+{{- if .Values.serviceAccount.annotations }}
+  annotations:
+{{ toYaml .Values.serviceAccount.annotations | indent 4 }}
+{{- end }}
+{{- end -}}

packages/zarf-registry/chart/values.yaml

@@ -48,8 +48,15 @@ secrets:
 podDisruptionBudget:
   minAvailable: 1
 
+affinity:
+  enabled: true
+  custom: {}
+
+tolerations: []
+
 autoscaling:
   enabled: true
+  mapReplicasToNodes: false
   minReplicas: 1
   maxReplicas: 5
   targetCPUUtilizationPercentage: 80
@@ -75,3 +82,8 @@ extraEnvVars: []
 ## Additional ENV variables to set
 # - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
 #   value: "/var/lib/example"
+
+serviceAccount:
+  create: false
+  name: ""
+  annotations: {}

packages/zarf-registry/registry-values.yaml

@@ -34,14 +34,31 @@ fullnameOverride: "zarf-docker-registry"
 podLabels:
   zarf.dev/agent: "ignore"
 
+affinity:
+  enabled: ###ZARF_VAR_REGISTRY_AFFINITY_ENABLE###
+  custom:
+    ###ZARF_VAR_REGISTRY_AFFINITY_CUSTOM###
+
+tolerations:
+  ###ZARF_VAR_REGISTRY_TOLERATIONS###
+
 autoscaling:
   enabled: ###ZARF_VAR_REGISTRY_HPA_ENABLE###
+  mapReplicasToNodes: ###ZARF_VAR_REGISTRY_HPA_AUTO_SIZE###
   minReplicas: "###ZARF_VAR_REGISTRY_HPA_MIN###"
   maxReplicas: "###ZARF_VAR_REGISTRY_HPA_MAX###"
-  targetCPUUtilizationPercentage: 80
+  targetCPUUtilizationPercentage: ###ZARF_VAR_REGISTRY_HPA_TARGET_CPU###
 
 caBundle: |
   ###ZARF_VAR_REGISTRY_CA_BUNDLE###
 
 extraEnvVars:
   ###ZARF_VAR_REGISTRY_EXTRA_ENVS###
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: ###ZARF_VAR_REGISTRY_CREATE_SERVICE_ACCOUNT###
+  # The name of the service account to use. If name not set and create is true, a name is generated using fullname template
+  name: "###ZARF_VAR_REGISTRY_SERVICE_ACCOUNT_NAME###"
+  annotations:
+    ###ZARF_VAR_REGISTRY_SERVICE_ACCOUNT_ANNOTATIONS###

packages/zarf-registry/zarf.yaml

@@ -58,6 +58,41 @@ variables:
     default: ""
     autoIndent: true
 
+  - name: REGISTRY_CREATE_SERVICE_ACCOUNT
+    description: Toggle the creation of a new service account for the registry
+    default: "false"
+
+  - name: REGISTRY_SERVICE_ACCOUNT_NAME
+    description: The name of the service account to use. If not set and create is true, a name is generated using fullname template
+    default: ""
+
+  - name: REGISTRY_SERVICE_ACCOUNT_ANNOTATIONS
+    description: Map of annotations to add to the created service account
+    default: ""
+    autoIndent: true
+
+  - name: REGISTRY_AFFINITY_ENABLE
+    description: Enable pod affinity for the registry
+    default: "true"
+
+  - name: REGISTRY_AFFINITY_CUSTOM
+    description: Custom pod affinity yaml block for the registry
+    default: ""
+    autoIndent: true
+
+  - name: REGISTRY_TOLERATIONS
+    description: Custom tolerations array for the registry
+    default: ""
+    autoIndent: true
+
+  - name: REGISTRY_HPA_AUTO_SIZE
+    description: Enable to set min and max replicas based on amount of nodes
+    default: "false"
+
+  - name: REGISTRY_HPA_TARGET_CPU
+    description: The target CPU utilization percentage for the registry
+    default: "80"
+
 constants:
   - name: REGISTRY_IMAGE
     value: "###ZARF_PKG_TMPL_REGISTRY_IMAGE###"

site/src/content/docs/ref/init-package.mdx

@@ -104,6 +104,36 @@ The `registry:2` image and the Zarf Agent image can be configured with a custom
 
 :::
 
+## Making the Registry Highly-Available
+
+By default, the registry included in the init package creates a ReadWriteOnce PVC and is only scheduled to run on one node at a time. This setup is usually enough for smaller and simpler deployments. However, for larger deployments or those where nodes are frequently restarted or updated, you may want to make the registry highly-available.
+
+This approach requires certain prerequisites, such as a storage class that supports ReadWriteMany, or being in an environment that allows you to configure the registry to use an S3-compatible backend. Additionally, you must provide custom configuration to the registry to ensure it is distributed across all nodes and has the appropriate number of replicas. Below is an example [configuration file](/ref/config-files/) using a ReadWriteMany storage class:
+
+```yaml
+# zarf-config.yaml
+package:
+  deploy:
+    set:
+      REGISTRY_PVC_ENABLED: "true"
+      REGISTRY_PVC_ACCESS_MODE: "ReadWriteMany"
+      REGISTRY_HPA_AUTO_SIZE: "true"
+      REGISTRY_AFFINITY_CUSTOM: |
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: app
+                      operator: In
+                      values:
+                        - docker-registry
+                topologyKey: kubernetes.io/hostname
+```
+
+Notably, the `REGISTRY_AFFINITY_CUSTOM` variable overrides the default pod anti-affinity, and `REGISTRY_HPA_AUTO_SIZE` automatically adjusts the minimum and maximum replicas for the registry based on the number of nodes in the cluster. If you prefer to manually set the minimum and maximum replicas, you can use `REGISTRY_HPA_MIN` and `REGISTRY_HPA_MAX` to specify the desired values.
+
 ## The `zarf init` Lifecycle
 
 The `zarf init` lifecycle is _very similar_ to the [`zarf package deploy` lifecycle](/ref/deploy/) except that it sets up resources specific to Zarf such as the `zarf-state` and performs special actions such as the injection procedure.
@@ -123,7 +153,7 @@ graph TD
     B12(prompt to confirm components)-->B13
     B13(prompt to choose components in '.group')-->B14
 
-    subgraph  
+    subgraph
     B52 --> |Yes|B14(deploy each component)-->B14
     B14 --> B15{Component is \n zarf-seed-registry}
     B15 --> |Yes|B51(initialize zarf-state secret):::action

src/test/external/docker-registry-values.yaml

@@ -3,7 +3,7 @@ image:
   tag: 2.8.3
   pullPolicy: IfNotPresent
 imagePullSecrets:
-    - name: private-registry
+  - name: private-registry
 
 service:
   name: registry