refactor: change UpdateZarfAgentValues to rolling restart (#2644)

## Description
This PR splits out a refactor that was done in #2625

#2625 (comment)



Here is some context for why I refactored `UpdateZarfAgentValues`.

In our e2e test suite, `TestConnectAndCreds` runs `zarf tools
update-creds --confirm`.

`update-creds` calls `UpdateZarfAgentValues` which used to delete all
agent pods manually based on label selector and immediately return
without waiting for the pods to be ready. `TestMetrics` is ran
immediately after `TestConnectAndCreds`, and it attempts to establish a
tunnel connection to the Zarf agent to test the /metrics endpoint. This
test was failing because there were no running agent pods yet after
deleting them. This is not a problem in our tests right now because we
run assertions against the logging stack in the cluster in
`TestConnectAndCreds`, which buys us enough time for the agent pods to
come back up and running.

I refactored `UpdateZarfAgentValues` to perform a rolling update of the
agent deployment by modifying the Pod template rather than manually
deleting pods, and added a wait to ensure the agent deployment is ready
before returning. This approach ensures a controlled and graceful update
process managed by Kubernetes and simplifies the code.

Co-authored-by: Austin Abro <[email protected]>

src/internal/packager/helm/zarf.go

@@ -74,42 +74,26 @@ func (h *Helm) UpdateZarfAgentValues(ctx context.Context) error {
 	spinner := message.NewProgressSpinner("Gathering information to update Zarf Agent TLS")
 	defer spinner.Stop()
 
-	err := h.createActionConfig(cluster.ZarfNamespaceName, spinner)
-	if err != nil {
-		return fmt.Errorf("unable to initialize the K8s client: %w", err)
-	}
-
-	// Get the current agent image from one of its pods.
-	selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"app": "agent-hook"}})
+	deployment, err := h.cluster.Clientset.AppsV1().Deployments(cluster.ZarfNamespaceName).Get(ctx, "agent-hook", metav1.GetOptions{})
 	if err != nil {
 		return err
 	}
-	listOpts := metav1.ListOptions{
-		LabelSelector: selector.String(),
-	}
-	podList, err := h.cluster.Clientset.CoreV1().Pods(cluster.ZarfNamespaceName).List(ctx, listOpts)
+	agentImage, err := transform.ParseImageRef(deployment.Spec.Template.Spec.Containers[0].Image)
 	if err != nil {
 		return err
 	}
 
-	var currentAgentImage transform.Image
-	if len(podList.Items) > 0 && len(podList.Items[0].Spec.Containers) > 0 {
-		currentAgentImage, err = transform.ParseImageRef(podList.Items[0].Spec.Containers[0].Image)
-		if err != nil {
-			return fmt.Errorf("unable to parse current agent image reference: %w", err)
-		}
-	} else {
-		return fmt.Errorf("unable to get current agent pod")
+	err = h.createActionConfig(cluster.ZarfNamespaceName, spinner)
+	if err != nil {
+		return err
 	}
 
 	// List the releases to find the current agent release name.
 	listClient := action.NewList(h.actionConfig)
-
 	releases, err := listClient.Run()
 	if err != nil {
 		return fmt.Errorf("unable to list helm releases: %w", err)
 	}
-
 	spinner.Success()
 
 	for _, release := range releases {
@@ -122,11 +106,11 @@ func (h *Helm) UpdateZarfAgentValues(ctx context.Context) error {
 			h.variableConfig.SetConstants([]variables.Constant{
 				{
 					Name:  "AGENT_IMAGE",
-					Value: currentAgentImage.Path,
+					Value: agentImage.Path,
 				},
 				{
 					Name:  "AGENT_IMAGE_TAG",
-					Value: currentAgentImage.Tag,
+					Value: agentImage.Tag,
 				},
 			})
 			applicationTemplates, err := template.GetZarfTemplates("zarf-agent", h.state)
@@ -142,30 +126,43 @@ func (h *Helm) UpdateZarfAgentValues(ctx context.Context) error {
 		}
 	}
 
-	spinner = message.NewProgressSpinner("Cleaning up Zarf Agent pods after update")
+	// Trigger a rolling update for the TLS secret update to take effect.
+	// https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment
+	spinner = message.NewProgressSpinner("Performing a rolling update for the Zarf Agent deployment")
 	defer spinner.Stop()
 
-	// Force pods to be recreated to get the updated secret.
-	// TODO: Explain why no grace period is given.
-	deleteGracePeriod := int64(0)
-	deletePolicy := metav1.DeletePropagationForeground
-	deleteOpts := metav1.DeleteOptions{
-		GracePeriodSeconds: &deleteGracePeriod,
-		PropagationPolicy:  &deletePolicy,
+	// Re-fetch the agent deployment before we update since the resourceVersion has changed after updating the Helm release values.
+	// Avoids this error: https://github.com/kubernetes/kubernetes/issues/28149
+	deployment, err = h.cluster.Clientset.AppsV1().Deployments(cluster.ZarfNamespaceName).Get(ctx, "agent-hook", metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if deployment.Spec.Template.Annotations == nil {
+		deployment.Spec.Template.Annotations = map[string]string{}
 	}
-	selector, err = metav1.LabelSelectorAsSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"app": "agent-hook"}})
+	deployment.Spec.Template.Annotations["zarf.dev/restartedAt"] = time.Now().UTC().Format(time.RFC3339)
+	_, err = h.cluster.Clientset.AppsV1().Deployments(cluster.ZarfNamespaceName).Update(ctx, deployment, metav1.UpdateOptions{})
 	if err != nil {
 		return err
 	}
-	listOpts = metav1.ListOptions{
-		LabelSelector: selector.String(),
+
+	objs := []object.ObjMetadata{
+		{
+			GroupKind: schema.GroupKind{
+				Group: "apps",
+				Kind:  "Deployment",
+			},
+			Namespace: cluster.ZarfNamespaceName,
+			Name:      "agent-hook",
+		},
 	}
-	err = h.cluster.Clientset.CoreV1().Pods(cluster.ZarfNamespaceName).DeleteCollection(ctx, deleteOpts, listOpts)
+	waitCtx, waitCancel := context.WithTimeout(ctx, 60*time.Second)
+	defer waitCancel()
+	err = pkgkubernetes.WaitForReady(waitCtx, h.cluster.Watcher, objs)
 	if err != nil {
-		return fmt.Errorf("error recycling pods for the Zarf Agent: %w", err)
+		return err
 	}
 
 	spinner.Success()
-
 	return nil
 }