diff --git a/docs/design/lifecycle_hooks_and_finalizers.md b/docs/design/lifecycle_hooks_and_finalizers.md new file mode 100644 index 000000000..d30b4723d --- /dev/null +++ b/docs/design/lifecycle_hooks_and_finalizers.md @@ -0,0 +1,37 @@ +# Lifecycle hooks & Finalizers + +The ArangoDB operator expects full control of the `Pods` and `PersistentVolumeClaims` it creates. +Therefore it takes measures to prevent the removal of those resources +until it is safe to do so. + +To achieve this, the server containers in the `Pods` have +a `preStop` hook configured and finalizers are added to the `Pods` +and `PersistentVolumeClaims`. + +The `preStop` hook executes a binary that waits until all finalizers of +the current pod have been removed. +Until this `preStop` hook terminates, Kubernetes will not send a `TERM` signal +to the processes inside the container, which ensures that the server remains running +until it is safe to stop them. + +The operator performs all actions needed when a delete of a `Pod` or +`PersistentVolumeClaims` has been triggered. +E.g. for a dbserver it cleans out the server if the `Pod` and `PersistentVolumeClaim` are being deleted. + +## Lifecycle init-container + +Because the binary that is called in the `preStop` hook is not part of a standard +ArangoDB docker image, it has to be brought into the filesystem of a `Pod`. +This is done by an initial container that copies the binary to an `emptyDir` volume that +is shared between the init-container and the server container. + +## Finalizers + +The ArangoDB operators adds the following finalizers to `Pods`. + +- `dbserver.database.arangodb.com/drain`: Added to DBServers, removed only when the dbserver can be restarted or is completely drained +- `agent.database.arangodb.com/agency-serving`: Added to Agents, removed only when enough agents are left to keep the agency serving + +The ArangoDB operators adds the following finalizers to `PersistentVolumeClaims`. + +- `pvc.database.arangodb.com/member-exists`: removed only when its member exists no longer exists or can be safely rebuild diff --git a/lifecycle.go b/lifecycle.go new file mode 100644 index 000000000..7b5561388 --- /dev/null +++ b/lifecycle.go @@ -0,0 +1,150 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package main + +import ( + "io" + "os" + "path/filepath" + "time" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +var ( + cmdLifecycle = &cobra.Command{ + Use: "lifecycle", + Run: cmdUsage, + Hidden: true, + } + + cmdLifecyclePreStop = &cobra.Command{ + Use: "preStop", + Run: cmdLifecyclePreStopRun, + Hidden: true, + } + cmdLifecycleCopy = &cobra.Command{ + Use: "copy", + Run: cmdLifecycleCopyRun, + Hidden: true, + } + + lifecycleCopyOptions struct { + TargetDir string + } +) + +func init() { + cmdMain.AddCommand(cmdLifecycle) + cmdLifecycle.AddCommand(cmdLifecyclePreStop) + cmdLifecycle.AddCommand(cmdLifecycleCopy) + + cmdLifecycleCopy.Flags().StringVar(&lifecycleCopyOptions.TargetDir, "target", "", "Target directory to copy the executable to") +} + +// Wait until all finalizers of the current pod have been removed. +func cmdLifecyclePreStopRun(cmd *cobra.Command, args []string) { + cliLog.Info().Msgf("Starting arangodb-operator, lifecycle preStop, version %s build %s", projectVersion, projectBuild) + + // Get environment + namespace := os.Getenv(constants.EnvOperatorPodNamespace) + if len(namespace) == 0 { + cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodNamespace) + } + name := os.Getenv(constants.EnvOperatorPodName) + if len(name) == 0 { + cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodName) + } + + // Create kubernetes client + kubecli, err := k8sutil.NewKubeClient() + if err != nil { + cliLog.Fatal().Err(err).Msg("Failed to create Kubernetes client") + } + + pods := kubecli.CoreV1().Pods(namespace) + recentErrors := 0 + for { + p, err := pods.Get(name, metav1.GetOptions{}) + if k8sutil.IsNotFound(err) { + cliLog.Warn().Msg("Pod not found") + return + } else if err != nil { + recentErrors++ + cliLog.Error().Err(err).Msg("Failed to get pod") + if recentErrors > 20 { + cliLog.Fatal().Err(err).Msg("Too many recent errors") + return + } + } else { + // We got our pod + finalizerCount := len(p.GetFinalizers()) + if finalizerCount == 0 { + // No more finalizers, we're done + cliLog.Info().Msg("All finalizers gone, we can stop now") + return + } + cliLog.Info().Msgf("Waiting for %d more finalizers to be removed", finalizerCount) + } + // Wait a bit + time.Sleep(time.Second) + } +} + +// Copy the executable to a given place. +func cmdLifecycleCopyRun(cmd *cobra.Command, args []string) { + cliLog.Info().Msgf("Starting arangodb-operator, lifecycle copy, version %s build %s", projectVersion, projectBuild) + + exePath, err := os.Executable() + if err != nil { + cliLog.Fatal().Err(err).Msg("Failed to get executable path") + } + + // Open source + rd, err := os.Open(exePath) + if err != nil { + cliLog.Fatal().Err(err).Msg("Failed to open executable file") + } + defer rd.Close() + + // Open target + targetPath := filepath.Join(lifecycleCopyOptions.TargetDir, filepath.Base(exePath)) + wr, err := os.Create(targetPath) + if err != nil { + cliLog.Fatal().Err(err).Msg("Failed to create target file") + } + defer wr.Close() + + if _, err := io.Copy(wr, rd); err != nil { + cliLog.Fatal().Err(err).Msg("Failed to copy") + } + + // Set file mode + if err := os.Chmod(targetPath, 0755); err != nil { + cliLog.Fatal().Err(err).Msg("Failed to chmod") + } +} diff --git a/main.go b/main.go index 7a80c339f..5f83ac185 100644 --- a/main.go +++ b/main.go @@ -193,7 +193,7 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper return operator.Config{}, operator.Dependencies{}, maskAny(err) } - serviceAccount, err := getMyPodServiceAccount(kubecli, namespace, name) + image, serviceAccount, err := getMyPodInfo(kubecli, namespace, name) if err != nil { return operator.Config{}, operator.Dependencies{}, maskAny(fmt.Errorf("Failed to get my pod's service account: %s", err)) } @@ -213,6 +213,7 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper Namespace: namespace, PodName: name, ServiceAccount: serviceAccount, + LifecycleImage: image, EnableDeployment: operatorOptions.enableDeployment, EnableStorage: operatorOptions.enableStorage, AllowChaos: chaosOptions.allowed, @@ -231,9 +232,10 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper return cfg, deps, nil } -// getMyPodServiceAccount looks up the service account of the pod with given name in given namespace -func getMyPodServiceAccount(kubecli kubernetes.Interface, namespace, name string) (string, error) { - var sa string +// getMyPodInfo looks up the image & service account of the pod with given name in given namespace +// Returns image, serviceAccount, error. +func getMyPodInfo(kubecli kubernetes.Interface, namespace, name string) (string, string, error) { + var image, sa string op := func() error { pod, err := kubecli.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{}) if err != nil { @@ -244,12 +246,17 @@ func getMyPodServiceAccount(kubecli kubernetes.Interface, namespace, name string return maskAny(err) } sa = pod.Spec.ServiceAccountName + image = k8sutil.ConvertImageID2Image(pod.Status.ContainerStatuses[0].ImageID) + if image == "" { + // Fallback in case we don't know the id. + image = pod.Spec.Containers[0].Image + } return nil } if err := retry.Retry(op, time.Minute*5); err != nil { - return "", maskAny(err) + return "", "", maskAny(err) } - return sa, nil + return image, sa, nil } func createRecorder(log zerolog.Logger, kubecli kubernetes.Interface, name, namespace string) record.EventRecorder { diff --git a/pkg/apis/deployment/v1alpha/conditions.go b/pkg/apis/deployment/v1alpha/conditions.go index b4b023803..77a19d17e 100644 --- a/pkg/apis/deployment/v1alpha/conditions.go +++ b/pkg/apis/deployment/v1alpha/conditions.go @@ -37,6 +37,9 @@ const ( ConditionTypeTerminated ConditionType = "Terminated" // ConditionTypeAutoUpgrade indicates that the member has to be started with `--database.auto-upgrade` once. ConditionTypeAutoUpgrade ConditionType = "AutoUpgrade" + // ConditionTypeCleanedOut indicates that the member (dbserver) has been cleaned out. + // Always check in combination with ConditionTypeTerminated. + ConditionTypeCleanedOut ConditionType = "CleanedOut" // ConditionTypePodSchedulingFailure indicates that one or more pods belonging to the deployment cannot be schedule. ConditionTypePodSchedulingFailure ConditionType = "PodSchedulingFailure" // ConditionTypeSecretsChanged indicates that the value of one of more secrets used by diff --git a/pkg/apis/deployment/v1alpha/deployment.go b/pkg/apis/deployment/v1alpha/deployment.go index d5ccd5663..a47b63278 100644 --- a/pkg/apis/deployment/v1alpha/deployment.go +++ b/pkg/apis/deployment/v1alpha/deployment.go @@ -50,11 +50,14 @@ type ArangoDeployment struct { // AsOwner creates an OwnerReference for the given deployment func (d *ArangoDeployment) AsOwner() metav1.OwnerReference { + trueVar := true return metav1.OwnerReference{ - APIVersion: SchemeGroupVersion.String(), - Kind: ArangoDeploymentResourceKind, - Name: d.Name, - UID: d.UID, + APIVersion: SchemeGroupVersion.String(), + Kind: ArangoDeploymentResourceKind, + Name: d.Name, + UID: d.UID, + Controller: &trueVar, + BlockOwnerDeletion: &trueVar, } } diff --git a/pkg/apis/deployment/v1alpha/deployment_status_members.go b/pkg/apis/deployment/v1alpha/deployment_status_members.go index b13d9a932..0b8184075 100644 --- a/pkg/apis/deployment/v1alpha/deployment_status_members.go +++ b/pkg/apis/deployment/v1alpha/deployment_status_members.go @@ -121,6 +121,22 @@ func (ds DeploymentStatusMembers) MemberStatusByPodName(podName string) (MemberS return MemberStatus{}, 0, false } +// MemberStatusByPVCName returns a reference to the element in the given set of lists that has the given PVC name. +// If no such element exists, nil is returned. +func (ds DeploymentStatusMembers) MemberStatusByPVCName(pvcName string) (MemberStatus, ServerGroup, bool) { + if result, found := ds.Single.ElementByPVCName(pvcName); found { + return result, ServerGroupSingle, true + } + if result, found := ds.Agents.ElementByPVCName(pvcName); found { + return result, ServerGroupAgents, true + } + if result, found := ds.DBServers.ElementByPVCName(pvcName); found { + return result, ServerGroupDBServers, true + } + // Note: Other server groups do not have PVC's so we can skip them. + return MemberStatus{}, 0, false +} + // UpdateMemberStatus updates the given status in the given group. func (ds *DeploymentStatusMembers) UpdateMemberStatus(status MemberStatus, group ServerGroup) error { var err error diff --git a/pkg/apis/deployment/v1alpha/member_status_list.go b/pkg/apis/deployment/v1alpha/member_status_list.go index fff418e25..18b38165a 100644 --- a/pkg/apis/deployment/v1alpha/member_status_list.go +++ b/pkg/apis/deployment/v1alpha/member_status_list.go @@ -63,6 +63,17 @@ func (l MemberStatusList) ElementByPodName(podName string) (MemberStatus, bool) return MemberStatus{}, false } +// ElementByPVCName returns the element in the given list that has the given PVC name and true. +// If no such element exists, an empty element and false is returned. +func (l MemberStatusList) ElementByPVCName(pvcName string) (MemberStatus, bool) { + for i, x := range l { + if x.PersistentVolumeClaimName == pvcName { + return l[i], true + } + } + return MemberStatus{}, false +} + // Add a member to the list. // Returns an AlreadyExistsError if the ID of the given member already exists. func (l *MemberStatusList) Add(m MemberStatus) error { diff --git a/pkg/apis/deployment/v1alpha/server_group.go b/pkg/apis/deployment/v1alpha/server_group.go index 8d8725693..894ba0b1e 100644 --- a/pkg/apis/deployment/v1alpha/server_group.go +++ b/pkg/apis/deployment/v1alpha/server_group.go @@ -22,6 +22,8 @@ package v1alpha +import time "time" + type ServerGroup int const ( @@ -85,6 +87,20 @@ func (g ServerGroup) AsRoleAbbreviated() string { } } +// DefaultTerminationGracePeriod returns the default period between SIGTERM & SIGKILL for a server in the given group. +func (g ServerGroup) DefaultTerminationGracePeriod() time.Duration { + switch g { + case ServerGroupSingle: + return time.Minute + case ServerGroupAgents: + return time.Minute + case ServerGroupDBServers: + return time.Hour + default: + return time.Second * 30 + } +} + // IsArangod returns true when the groups runs servers of type `arangod`. func (g ServerGroup) IsArangod() bool { switch g { diff --git a/pkg/deployment/cleanup.go b/pkg/deployment/cleanup.go new file mode 100644 index 000000000..99ad9a497 --- /dev/null +++ b/pkg/deployment/cleanup.go @@ -0,0 +1,59 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package deployment + +import ( + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +// removePodFinalizers removes all finalizers from all pods owned by us. +func (d *Deployment) removePodFinalizers() error { + log := d.deps.Log + kubecli := d.GetKubeCli() + pods, err := d.GetOwnedPods() + if err != nil { + return maskAny(err) + } + for _, p := range pods { + if err := k8sutil.RemovePodFinalizers(log, kubecli, &p, p.GetFinalizers()); err != nil { + log.Warn().Err(err).Msg("Failed to remove pod finalizers") + } + } + return nil +} + +// removePVCFinalizers removes all finalizers from all PVCs owned by us. +func (d *Deployment) removePVCFinalizers() error { + log := d.deps.Log + kubecli := d.GetKubeCli() + pvcs, err := d.GetOwnedPVCs() + if err != nil { + return maskAny(err) + } + for _, p := range pvcs { + if err := k8sutil.RemovePVCFinalizers(log, kubecli, &p, p.GetFinalizers()); err != nil { + log.Warn().Err(err).Msg("Failed to remove PVC finalizers") + } + } + return nil +} diff --git a/pkg/deployment/context_impl.go b/pkg/deployment/context_impl.go index 5ebec2886..f3e56ad16 100644 --- a/pkg/deployment/context_impl.go +++ b/pkg/deployment/context_impl.go @@ -50,6 +50,11 @@ func (d *Deployment) GetKubeCli() kubernetes.Interface { return d.deps.KubeCli } +// GetLifecycleImage returns the image name containing the lifecycle helper (== name of operator image) +func (d *Deployment) GetLifecycleImage() string { + return d.config.LifecycleImage +} + // GetNamespace returns the kubernetes namespace that contains // this deployment. func (d *Deployment) GetNamespace() string { @@ -191,6 +196,24 @@ func (d *Deployment) GetOwnedPods() ([]v1.Pod, error) { return myPods, nil } +// GetOwnedPVCs returns a list of all PVCs owned by the deployment. +func (d *Deployment) GetOwnedPVCs() ([]v1.PersistentVolumeClaim, error) { + // Get all current PVCs + log := d.deps.Log + pvcs, err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(d.apiObject.GetNamespace()).List(k8sutil.DeploymentListOpt(d.apiObject.GetName())) + if err != nil { + log.Debug().Err(err).Msg("Failed to list PVCs") + return nil, maskAny(err) + } + myPVCs := make([]v1.PersistentVolumeClaim, 0, len(pvcs.Items)) + for _, p := range pvcs.Items { + if d.isOwnerOf(&p) { + myPVCs = append(myPVCs, p) + } + } + return myPVCs, nil +} + // GetTLSKeyfile returns the keyfile encoded TLS certificate+key for // the given member. func (d *Deployment) GetTLSKeyfile(group api.ServerGroup, member api.MemberStatus) (string, error) { diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index 902dc60df..4dbe57a1e 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -50,6 +50,7 @@ import ( type Config struct { ServiceAccount string AllowChaos bool + LifecycleImage string } // Dependencies holds dependent services for a Deployment @@ -219,6 +220,14 @@ func (d *Deployment) run() { for { select { case <-d.stopCh: + // Remove finalizers from created resources + log.Info().Msg("Deployment removed, removing finalizers to prevent orphaned resources") + if err := d.removePodFinalizers(); err != nil { + log.Warn().Err(err).Msg("Failed to remove Pod finalizers") + } + if err := d.removePVCFinalizers(); err != nil { + log.Warn().Err(err).Msg("Failed to remove PVC finalizers") + } // We're being stopped. return diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index b7fe8acb2..d7a1c1403 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -80,10 +80,14 @@ func (d *Deployment) inspectDeployment(lastInterval time.Duration) time.Duration } // Inspection of generated resources needed - if err := d.resources.InspectPods(); err != nil { + if err := d.resources.InspectPods(ctx); err != nil { hasError = true d.CreateEvent(k8sutil.NewErrorEvent("Pod inspection failed", err, d.apiObject)) } + if err := d.resources.InspectPVCs(ctx); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("PVC inspection failed", err, d.apiObject)) + } // Check members for resilience if err := d.resilience.CheckMemberFailure(); err != nil { diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index daaa56836..4fac455f8 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -27,6 +27,7 @@ import ( "crypto/sha1" "fmt" "strings" + "time" "github.com/rs/zerolog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -38,7 +39,7 @@ import ( ) const ( - dockerPullableImageIDPrefix = "docker-pullable://" + dockerPullableImageIDPrefix_ = "docker-pullable://" ) type imagesBuilder struct { @@ -117,10 +118,8 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima log.Warn().Msg("Empty list of ContainerStatuses") return true, nil } - imageID := pod.Status.ContainerStatuses[0].ImageID - if strings.HasPrefix(imageID, dockerPullableImageIDPrefix) { - imageID = imageID[len(dockerPullableImageIDPrefix):] - } else if imageID == "" { + imageID := k8sutil.ConvertImageID2Image(pod.Status.ContainerStatuses[0].ImageID) + if imageID == "" { // Fall back to specified image imageID = image } @@ -168,7 +167,8 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima "--server.authentication=false", fmt.Sprintf("--server.endpoint=tcp://[::]:%d", k8sutil.ArangoPort), } - if err := k8sutil.CreateArangodPod(ib.KubeCli, true, ib.APIObject, role, id, podName, "", image, ib.Spec.GetImagePullPolicy(), "", false, args, nil, nil, nil, "", ""); err != nil { + terminationGracePeriod := time.Second * 30 + if err := k8sutil.CreateArangodPod(ib.KubeCli, true, ib.APIObject, role, id, podName, "", image, "", ib.Spec.GetImagePullPolicy(), "", false, terminationGracePeriod, args, nil, nil, nil, nil, "", ""); err != nil { log.Debug().Err(err).Msg("Failed to create image ID pod") return true, maskAny(err) } diff --git a/pkg/deployment/reconcile/action_cleanout_member.go b/pkg/deployment/reconcile/action_cleanout_member.go index 6292a5c52..eb9d9fbc6 100644 --- a/pkg/deployment/reconcile/action_cleanout_member.go +++ b/pkg/deployment/reconcile/action_cleanout_member.go @@ -82,6 +82,11 @@ func (a *actionCleanoutMember) Start(ctx context.Context) (bool, error) { // Returns true if the action is completely finished, false otherwise. func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, error) { log := a.log + m, ok := a.actionCtx.GetMemberStatusByID(a.action.MemberID) + if !ok { + // We wanted to remove and it is already gone. All ok + return true, nil + } c, err := a.actionCtx.GetDatabaseClient(ctx) if err != nil { log.Debug().Err(err).Msg("Failed to create member client") @@ -101,5 +106,11 @@ func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, error) return false, nil } // Cleanout completed + if m.Conditions.Update(api.ConditionTypeCleanedOut, true, "CleanedOut", "") { + if a.actionCtx.UpdateMember(m); err != nil { + return false, maskAny(err) + } + } + // Cleanout completed return true, nil } diff --git a/pkg/deployment/reconcile/plan_builder.go b/pkg/deployment/reconcile/plan_builder.go index 54dfb3028..7bc8b3a6f 100644 --- a/pkg/deployment/reconcile/plan_builder.go +++ b/pkg/deployment/reconcile/plan_builder.go @@ -33,6 +33,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) // upgradeDecision is the result of an upgrade check. @@ -108,6 +109,16 @@ func createPlan(log zerolog.Logger, apiObject metav1.Object, return nil }) + // Check for cleaned out dbserver in created state + for _, m := range status.Members.DBServers { + if len(plan) == 0 && m.Phase == api.MemberPhaseCreated && m.Conditions.IsTrue(api.ConditionTypeCleanedOut) { + plan = append(plan, + api.NewAction(api.ActionTypeRemoveMember, api.ServerGroupDBServers, m.ID), + api.NewAction(api.ActionTypeAddMember, api.ServerGroupDBServers, ""), + ) + } + } + // Check for scale up/down if len(plan) == 0 { switch spec.GetMode() { @@ -208,8 +219,7 @@ func createPlan(log zerolog.Logger, apiObject metav1.Object, // podNeedsUpgrading decides if an upgrade of the pod is needed (to comply with // the given spec) and if that is allowed. func podNeedsUpgrading(p v1.Pod, spec api.DeploymentSpec, images api.ImageInfoList) upgradeDecision { - if len(p.Spec.Containers) == 1 { - c := p.Spec.Containers[0] + if c, found := k8sutil.GetContainerByName(&p, k8sutil.ServerContainerName); found { specImageInfo, found := images.GetByImage(spec.GetImage()) if !found { return upgradeDecision{UpgradeNeeded: false} @@ -253,14 +263,13 @@ func podNeedsUpgrading(p v1.Pod, spec api.DeploymentSpec, images api.ImageInfoLi // When true is returned, a reason for the rotation is already returned. func podNeedsRotation(p v1.Pod, apiObject metav1.Object, spec api.DeploymentSpec, group api.ServerGroup, agents api.MemberStatusList, id string) (bool, string) { - // Check number of containers - if len(p.Spec.Containers) != 1 { - return true, "Number of containers changed" - } // Check image pull policy - c := p.Spec.Containers[0] - if c.ImagePullPolicy != spec.GetImagePullPolicy() { - return true, "Image pull policy changed" + if c, found := k8sutil.GetContainerByName(&p, k8sutil.ServerContainerName); found { + if c.ImagePullPolicy != spec.GetImagePullPolicy() { + return true, "Image pull policy changed" + } + } else { + return true, "Server container not found" } // Check arguments /*expectedArgs := createArangodArgs(apiObject, spec, group, agents, id) diff --git a/pkg/deployment/resources/context.go b/pkg/deployment/resources/context.go index e7a5d6683..571754caa 100644 --- a/pkg/deployment/resources/context.go +++ b/pkg/deployment/resources/context.go @@ -23,6 +23,9 @@ package resources import ( + "context" + + driver "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" "k8s.io/api/core/v1" @@ -55,6 +58,8 @@ type Context interface { UpdateStatus(status api.DeploymentStatus, force ...bool) error // GetKubeCli returns the kubernetes client GetKubeCli() kubernetes.Interface + // GetLifecycleImage returns the image name containing the lifecycle helper (== name of operator image) + GetLifecycleImage() string // GetNamespace returns the namespace that contains the deployment GetNamespace() string // createEvent creates a given event. @@ -62,7 +67,14 @@ type Context interface { CreateEvent(evt *v1.Event) // GetOwnedPods returns a list of all pods owned by the deployment. GetOwnedPods() ([]v1.Pod, error) + // GetOwnedPVCs returns a list of all PVCs owned by the deployment. + GetOwnedPVCs() ([]v1.PersistentVolumeClaim, error) // CleanupPod deletes a given pod with force and explicit UID. // If the pod does not exist, the error is ignored. CleanupPod(p v1.Pod) error + // GetAgencyClients returns a client connection for every agency member. + GetAgencyClients(ctx context.Context, predicate func(memberID string) bool) ([]driver.Connection, error) + // GetDatabaseClient returns a cached client for the entire database (cluster coordinators or single server), + // creating one if needed. + GetDatabaseClient(ctx context.Context) (driver.Client, error) } diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 3a265df3c..670951c3e 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -377,6 +377,18 @@ func (r *Resources) createReadinessProbe(spec api.DeploymentSpec, group api.Serv }, nil } +// createPodFinalizers creates a list of finalizers for a pod created for the given group. +func (r *Resources) createPodFinalizers(group api.ServerGroup) []string { + switch group { + case api.ServerGroupAgents: + return []string{constants.FinalizerPodAgencyServing} + case api.ServerGroupDBServers: + return []string{constants.FinalizerPodDrainDBServer} + default: + return nil + } +} + // createPodForMember creates all Pods listed in member status func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.ServerGroup, groupSpec api.ServerGroupSpec, m api.MemberStatus, memberStatusList *api.MemberStatusList) error { @@ -385,6 +397,8 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server apiObject := r.context.GetAPIObject() ns := r.context.GetNamespace() status := r.context.GetStatus() + lifecycleImage := r.context.GetLifecycleImage() + terminationGracePeriod := group.DefaultTerminationGracePeriod() // Update pod name role := group.AsRole() @@ -445,8 +459,9 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server } engine := spec.GetStorageEngine().AsArangoArgument() requireUUID := group == api.ServerGroupDBServers && m.IsInitialized - if err := k8sutil.CreateArangodPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, m.PersistentVolumeClaimName, info.ImageID, spec.GetImagePullPolicy(), - engine, requireUUID, args, env, livenessProbe, readinessProbe, tlsKeyfileSecretName, rocksdbEncryptionSecretName); err != nil { + finalizers := r.createPodFinalizers(group) + if err := k8sutil.CreateArangodPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, m.PersistentVolumeClaimName, info.ImageID, lifecycleImage, spec.GetImagePullPolicy(), + engine, requireUUID, terminationGracePeriod, args, env, finalizers, livenessProbe, readinessProbe, tlsKeyfileSecretName, rocksdbEncryptionSecretName); err != nil { return maskAny(err) } log.Debug().Str("pod-name", m.PodName).Msg("Created pod") @@ -516,7 +531,7 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server if group == api.ServerGroupSyncWorkers { affinityWithRole = api.ServerGroupDBServers.AsRole() } - if err := k8sutil.CreateArangoSyncPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, info.ImageID, spec.Sync.GetImagePullPolicy(), args, env, + if err := k8sutil.CreateArangoSyncPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, info.ImageID, lifecycleImage, spec.Sync.GetImagePullPolicy(), terminationGracePeriod, args, env, livenessProbe, tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName, affinityWithRole); err != nil { return maskAny(err) } @@ -548,6 +563,9 @@ func (r *Resources) EnsurePods() error { if m.Phase != api.MemberPhaseNone { continue } + if m.Conditions.IsTrue(api.ConditionTypeCleanedOut) { + continue + } spec := r.context.GetSpec() if err := r.createPodForMember(spec, group, groupSpec, m, status); err != nil { return maskAny(err) diff --git a/pkg/deployment/resources/pod_finalizers.go b/pkg/deployment/resources/pod_finalizers.go new file mode 100644 index 000000000..5626d6caa --- /dev/null +++ b/pkg/deployment/resources/pod_finalizers.go @@ -0,0 +1,179 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package resources + +import ( + "context" + "fmt" + + "github.com/rs/zerolog" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/arangodb/go-driver/agency" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +// runPodFinalizers goes through the list of pod finalizers to see if they can be removed. +func (r *Resources) runPodFinalizers(ctx context.Context, p *v1.Pod, memberStatus api.MemberStatus, updateMember func(api.MemberStatus) error) error { + log := r.log.With().Str("pod-name", p.GetName()).Logger() + var removalList []string + for _, f := range p.ObjectMeta.GetFinalizers() { + switch f { + case constants.FinalizerPodAgencyServing: + log.Debug().Msg("Inspecting agency-serving finalizer") + if err := r.inspectFinalizerPodAgencyServing(ctx, log, p, memberStatus); err == nil { + removalList = append(removalList, f) + } else { + log.Debug().Err(err).Str("finalizer", f).Msg("Cannot remove finalizer yet") + } + case constants.FinalizerPodDrainDBServer: + log.Debug().Msg("Inspecting drain dbserver finalizer") + if err := r.inspectFinalizerPodDrainDBServer(ctx, log, p, memberStatus, updateMember); err == nil { + removalList = append(removalList, f) + } else { + log.Debug().Err(err).Str("finalizer", f).Msg("Cannot remove finalizer yet") + } + } + } + // Remove finalizers (if needed) + if len(removalList) > 0 { + kubecli := r.context.GetKubeCli() + if err := k8sutil.RemovePodFinalizers(log, kubecli, p, removalList); err != nil { + log.Debug().Err(err).Msg("Failed to update pod (to remove finalizers)") + return maskAny(err) + } + } + return nil +} + +// inspectFinalizerPodAgencyServing checks the finalizer condition for agency-serving. +// It returns nil if the finalizer can be removed. +func (r *Resources) inspectFinalizerPodAgencyServing(ctx context.Context, log zerolog.Logger, p *v1.Pod, memberStatus api.MemberStatus) error { + // Inspect member phase + if memberStatus.Phase.IsFailed() { + log.Debug().Msg("Pod is already failed, safe to remove agency serving finalizer") + return nil + } + // Inspect deployment deletion state + apiObject := r.context.GetAPIObject() + if apiObject.GetDeletionTimestamp() != nil { + log.Debug().Msg("Entire deployment is being deleted, safe to remove agency serving finalizer") + return nil + } + // Check PVC + pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(apiObject.GetNamespace()) + pvc, err := pvcs.Get(memberStatus.PersistentVolumeClaimName, metav1.GetOptions{}) + if err != nil { + log.Warn().Err(err).Msg("Failed to get PVC for member") + return maskAny(err) + } + if !k8sutil.IsPersistentVolumeClaimMarkedForDeletion(pvc) { + log.Debug().Msg("PVC is not being deleted, so it is safe to remove agency serving finalizer") + return nil + } + log.Debug().Msg("PVC is being deleted, so we will check agency serving status first") + // Inspect agency state + agencyConns, err := r.context.GetAgencyClients(ctx, func(id string) bool { return id != memberStatus.ID }) + if err != nil { + log.Debug().Err(err).Msg("Failed to create member client") + return maskAny(err) + } + if len(agencyConns) == 0 { + log.Debug().Err(err).Msg("No more remaining agents, we cannot delete this one") + return maskAny(fmt.Errorf("No more remaining agents")) + } + if err := agency.AreAgentsHealthy(ctx, agencyConns); err != nil { + log.Debug().Err(err).Msg("Remaining agents are not health") + return maskAny(err) + } + // Remaining agents are health, we can remove this one + return nil +} + +// inspectFinalizerPodDrainDBServer checks the finalizer condition for drain-dbserver. +// It returns nil if the finalizer can be removed. +func (r *Resources) inspectFinalizerPodDrainDBServer(ctx context.Context, log zerolog.Logger, p *v1.Pod, memberStatus api.MemberStatus, updateMember func(api.MemberStatus) error) error { + // Inspect member phase + if memberStatus.Phase.IsFailed() { + log.Debug().Msg("Pod is already failed, safe to remove drain dbserver finalizer") + return nil + } + // Inspect deployment deletion state + apiObject := r.context.GetAPIObject() + if apiObject.GetDeletionTimestamp() != nil { + log.Debug().Msg("Entire deployment is being deleted, safe to remove drain dbserver finalizer") + return nil + } + // Check PVC + pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(apiObject.GetNamespace()) + pvc, err := pvcs.Get(memberStatus.PersistentVolumeClaimName, metav1.GetOptions{}) + if err != nil { + log.Warn().Err(err).Msg("Failed to get PVC for member") + return maskAny(err) + } + if !k8sutil.IsPersistentVolumeClaimMarkedForDeletion(pvc) { + log.Debug().Msg("PVC is not being deleted, so it is safe to remove drain dbserver finalizer") + return nil + } + log.Debug().Msg("PVC is being deleted, so we will cleanout the dbserver first") + // Inspect cleaned out state + c, err := r.context.GetDatabaseClient(ctx) + if err != nil { + log.Debug().Err(err).Msg("Failed to create member client") + return maskAny(err) + } + cluster, err := c.Cluster(ctx) + if err != nil { + log.Debug().Err(err).Msg("Failed to access cluster") + return maskAny(err) + } + cleanedOut, err := cluster.IsCleanedOut(ctx, memberStatus.ID) + if err != nil { + return maskAny(err) + } + if cleanedOut { + // Cleanout completed + if memberStatus.Conditions.Update(api.ConditionTypeCleanedOut, true, "CleanedOut", "") { + if err := updateMember(memberStatus); err != nil { + return maskAny(err) + } + } + log.Debug().Msg("Server is cleaned out. Save to remove drain dbserver finalizer") + return nil + } + // Not cleaned out yet, check member status + if memberStatus.Conditions.IsTrue(api.ConditionTypeTerminated) { + log.Warn().Msg("Member is already terminated before it could be cleaned out. Not good, but removing drain dbserver finalizer because we cannot do anything further") + return nil + } + // Ensure the cleanout is triggered + log.Debug().Msg("Server is not yet clean out. Triggering a clean out now") + if err := cluster.CleanOutServer(ctx, memberStatus.ID); err != nil { + log.Debug().Err(err).Msg("Failed to clean out server") + return maskAny(err) + } + return maskAny(fmt.Errorf("Server is not yet cleaned out")) +} diff --git a/pkg/deployment/resources/pod_inspector.go b/pkg/deployment/resources/pod_inspector.go index 5594878e0..f5d3c1548 100644 --- a/pkg/deployment/resources/pod_inspector.go +++ b/pkg/deployment/resources/pod_inspector.go @@ -23,6 +23,7 @@ package resources import ( + "context" "fmt" "time" @@ -44,7 +45,7 @@ const ( // InspectPods lists all pods that belong to the given deployment and updates // the member status of the deployment accordingly. -func (r *Resources) InspectPods() error { +func (r *Resources) InspectPods(ctx context.Context) error { log := r.log var events []*v1.Event @@ -72,6 +73,16 @@ func (r *Resources) InspectPods() error { memberStatus, group, found := status.Members.MemberStatusByPodName(p.GetName()) if !found { log.Debug().Str("pod", p.GetName()).Msg("no memberstatus found for pod") + if k8sutil.IsPodMarkedForDeletion(&p) && len(p.GetFinalizers()) > 0 { + // Strange, pod belongs to us, but we have no member for it. + // Remove all finalizers, so it can be removed. + log.Warn().Msg("Pod belongs to this deployment, but we don't know the member. Removing all finalizers") + kubecli := r.context.GetKubeCli() + if err := k8sutil.RemovePodFinalizers(log, kubecli, &p, p.GetFinalizers()); err != nil { + log.Debug().Err(err).Msg("Failed to update pod (to remove all finalizers)") + return maskAny(err) + } + } continue } @@ -123,6 +134,17 @@ func (r *Resources) InspectPods() error { } else if !k8sutil.IsPodScheduled(&p) { unscheduledPodNames = append(unscheduledPodNames, p.GetName()) } + if k8sutil.IsPodMarkedForDeletion(&p) { + // Process finalizers + if err := r.runPodFinalizers(ctx, &p, memberStatus, func(m api.MemberStatus) error { + updateMemberStatusNeeded = true + memberStatus = m + return nil + }); err != nil { + // Only log here, since we'll be called to try again. + log.Warn().Err(err).Msg("Failed to run pod finalizers") + } + } if updateMemberStatusNeeded { if err := status.Members.UpdateMemberStatus(memberStatus, group); err != nil { return maskAny(err) diff --git a/pkg/deployment/resources/pvc_finalizers.go b/pkg/deployment/resources/pvc_finalizers.go new file mode 100644 index 000000000..a964a4090 --- /dev/null +++ b/pkg/deployment/resources/pvc_finalizers.go @@ -0,0 +1,91 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package resources + +import ( + "context" + "fmt" + + "github.com/rs/zerolog" + "k8s.io/api/core/v1" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +// runPVCFinalizers goes through the list of PVC finalizers to see if they can be removed. +func (r *Resources) runPVCFinalizers(ctx context.Context, p *v1.PersistentVolumeClaim, group api.ServerGroup, memberStatus api.MemberStatus) error { + log := r.log.With().Str("pvc-name", p.GetName()).Logger() + var removalList []string + for _, f := range p.ObjectMeta.GetFinalizers() { + switch f { + case constants.FinalizerPVCMemberExists: + log.Debug().Msg("Inspecting member exists finalizer") + if err := r.inspectFinalizerPVCMemberExists(ctx, log, p, group, memberStatus); err == nil { + removalList = append(removalList, f) + } else { + log.Debug().Err(err).Str("finalizer", f).Msg("Cannot remove finalizer yet") + } + } + } + // Remove finalizers (if needed) + if len(removalList) > 0 { + kubecli := r.context.GetKubeCli() + if err := k8sutil.RemovePVCFinalizers(log, kubecli, p, removalList); err != nil { + log.Debug().Err(err).Msg("Failed to update PVC (to remove finalizers)") + return maskAny(err) + } + } + return nil +} + +// inspectFinalizerPVCMemberExists checks the finalizer condition for member-exists. +// It returns nil if the finalizer can be removed. +func (r *Resources) inspectFinalizerPVCMemberExists(ctx context.Context, log zerolog.Logger, p *v1.PersistentVolumeClaim, group api.ServerGroup, memberStatus api.MemberStatus) error { + // Inspect member phase + if memberStatus.Phase.IsFailed() { + log.Debug().Msg("Member is already failed, safe to remove member-exists finalizer") + return nil + } + // Inspect deployment deletion state + apiObject := r.context.GetAPIObject() + if apiObject.GetDeletionTimestamp() != nil { + log.Debug().Msg("Entire deployment is being deleted, safe to remove member-exists finalizer") + return nil + } + // We do allow to rebuild agents & dbservers + switch group { + case api.ServerGroupAgents: + if memberStatus.Conditions.IsTrue(api.ConditionTypeTerminated) { + log.Debug().Msg("Rebuilding terminated agents is allowed, safe to remove member-exists finalizer") + return nil + } + case api.ServerGroupDBServers: + if memberStatus.Conditions.IsTrue(api.ConditionTypeCleanedOut) { + log.Debug().Msg("Removing cleanedout dbservers is allowed, safe to remove member-exists finalizer") + return nil + } + } + return maskAny(fmt.Errorf("Member still exists")) +} diff --git a/pkg/deployment/resources/pvc_inspector.go b/pkg/deployment/resources/pvc_inspector.go new file mode 100644 index 000000000..e04d49022 --- /dev/null +++ b/pkg/deployment/resources/pvc_inspector.go @@ -0,0 +1,80 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package resources + +import ( + "context" + + "github.com/arangodb/kube-arangodb/pkg/metrics" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +var ( + inspectedPVCCounter = metrics.MustRegisterCounter("deployment", "inspected_ppvcs", "Number of PVCs inspections") +) + +// InspectPVCs lists all PVCs that belong to the given deployment and updates +// the member status of the deployment accordingly. +func (r *Resources) InspectPVCs(ctx context.Context) error { + log := r.log + + pvcs, err := r.context.GetOwnedPVCs() + if err != nil { + log.Debug().Err(err).Msg("Failed to get owned PVCs") + return maskAny(err) + } + + // Update member status from all pods found + status := r.context.GetStatus() + for _, p := range pvcs { + // PVC belongs to this deployment, update metric + inspectedPVCCounter.Inc() + + // Find member status + memberStatus, group, found := status.Members.MemberStatusByPVCName(p.GetName()) + if !found { + log.Debug().Str("pvc", p.GetName()).Msg("no memberstatus found for PVC") + if k8sutil.IsPersistentVolumeClaimMarkedForDeletion(&p) && len(p.GetFinalizers()) > 0 { + // Strange, pvc belongs to us, but we have no member for it. + // Remove all finalizers, so it can be removed. + log.Warn().Msg("PVC belongs to this deployment, but we don't know the member. Removing all finalizers") + kubecli := r.context.GetKubeCli() + if err := k8sutil.RemovePVCFinalizers(log, kubecli, &p, p.GetFinalizers()); err != nil { + log.Debug().Err(err).Msg("Failed to update PVC (to remove all finalizers)") + return maskAny(err) + } + } + continue + } + + if k8sutil.IsPersistentVolumeClaimMarkedForDeletion(&p) { + // Process finalizers + if err := r.runPVCFinalizers(ctx, &p, group, memberStatus); err != nil { + // Only log here, since we'll be called to try again. + log.Warn().Err(err).Msg("Failed to run PVC finalizers") + } + } + } + + return nil +} diff --git a/pkg/deployment/resources/pvcs.go b/pkg/deployment/resources/pvcs.go index abb2b135b..72ddf243e 100644 --- a/pkg/deployment/resources/pvcs.go +++ b/pkg/deployment/resources/pvcs.go @@ -25,9 +25,15 @@ package resources import ( api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/util/constants" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) +// createPVCFinalizers creates a list of finalizers for a PVC created for the given group. +func (r *Resources) createPVCFinalizers(group api.ServerGroup) []string { + return []string{constants.FinalizerPVCMemberExists} +} + // EnsurePVCs creates all PVC's listed in member status func (r *Resources) EnsurePVCs() error { kubecli := r.context.GetKubeCli() @@ -44,7 +50,8 @@ func (r *Resources) EnsurePVCs() error { storageClassName := spec.GetStorageClassName() role := group.AsRole() resources := spec.Resources - if err := k8sutil.CreatePersistentVolumeClaim(kubecli, m.PersistentVolumeClaimName, deploymentName, ns, storageClassName, role, resources, owner); err != nil { + finalizers := r.createPVCFinalizers(group) + if err := k8sutil.CreatePersistentVolumeClaim(kubecli, m.PersistentVolumeClaimName, deploymentName, ns, storageClassName, role, resources, finalizers, owner); err != nil { return maskAny(err) } } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 5fdeb85a9..fa0d75bd4 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -66,6 +66,7 @@ type Config struct { Namespace string PodName string ServiceAccount string + LifecycleImage string EnableDeployment bool EnableStorage bool AllowChaos bool diff --git a/pkg/operator/operator_deployment.go b/pkg/operator/operator_deployment.go index d62d32090..62cd54182 100644 --- a/pkg/operator/operator_deployment.go +++ b/pkg/operator/operator_deployment.go @@ -203,6 +203,7 @@ func (o *Operator) handleDeploymentEvent(event *Event) error { func (o *Operator) makeDeploymentConfigAndDeps(apiObject *api.ArangoDeployment) (deployment.Config, deployment.Dependencies) { cfg := deployment.Config{ ServiceAccount: o.Config.ServiceAccount, + LifecycleImage: o.Config.LifecycleImage, AllowChaos: o.Config.AllowChaos, } deps := deployment.Dependencies{ diff --git a/pkg/util/constants/constants.go b/pkg/util/constants/constants.go index 99fba7220..aeead5659 100644 --- a/pkg/util/constants/constants.go +++ b/pkg/util/constants/constants.go @@ -39,4 +39,8 @@ const ( SecretCAKey = "ca.key" // Key in Secret.data used to store a PEM encoded CA private key SecretTLSKeyfile = "tls.keyfile" // Key in Secret.data used to store a PEM encoded TLS certificate in the format used by ArangoDB (`--ssl.keyfile`) + + FinalizerPodDrainDBServer = "dbserver.database.arangodb.com/drain" // Finalizer added to DBServers, indicating the need for draining that dbserver + FinalizerPodAgencyServing = "agent.database.arangodb.com/agency-serving" // Finalizer added to Agents, indicating the need for keeping enough agents alive + FinalizerPVCMemberExists = "pvc.database.arangodb.com/member-exists" // Finalizer added to PVCs, indicating the need to keep is as long as its member exists ) diff --git a/pkg/util/k8sutil/container.go b/pkg/util/k8sutil/container.go new file mode 100644 index 000000000..b018eff97 --- /dev/null +++ b/pkg/util/k8sutil/container.go @@ -0,0 +1,38 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package k8sutil + +import ( + "k8s.io/api/core/v1" +) + +// GetContainerByName returns the container in the given pod with the given name. +// Returns false if not found. +func GetContainerByName(p *v1.Pod, name string) (v1.Container, bool) { + for _, c := range p.Spec.Containers { + if c.Name == name { + return c, true + } + } + return v1.Container{}, false +} diff --git a/pkg/util/k8sutil/finalizers.go b/pkg/util/k8sutil/finalizers.go new file mode 100644 index 000000000..ec4ea3b6d --- /dev/null +++ b/pkg/util/k8sutil/finalizers.go @@ -0,0 +1,137 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package k8sutil + +import ( + "github.com/rs/zerolog" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + maxRemoveFinalizersAttempts = 50 +) + +// RemovePodFinalizers removes the given finalizers from the given pod. +func RemovePodFinalizers(log zerolog.Logger, kubecli kubernetes.Interface, p *v1.Pod, finalizers []string) error { + pods := kubecli.CoreV1().Pods(p.GetNamespace()) + getFunc := func() (metav1.Object, error) { + result, err := pods.Get(p.GetName(), metav1.GetOptions{}) + if err != nil { + return nil, maskAny(err) + } + return result, nil + } + updateFunc := func(updated metav1.Object) error { + updatedPod := updated.(*v1.Pod) + result, err := pods.Update(updatedPod) + if err != nil { + return maskAny(err) + } + *p = *result + return nil + } + if err := removeFinalizers(log, finalizers, getFunc, updateFunc); err != nil { + return maskAny(err) + } + return nil +} + +// RemovePVCFinalizers removes the given finalizers from the given PVC. +func RemovePVCFinalizers(log zerolog.Logger, kubecli kubernetes.Interface, p *v1.PersistentVolumeClaim, finalizers []string) error { + pvcs := kubecli.CoreV1().PersistentVolumeClaims(p.GetNamespace()) + getFunc := func() (metav1.Object, error) { + result, err := pvcs.Get(p.GetName(), metav1.GetOptions{}) + if err != nil { + return nil, maskAny(err) + } + return result, nil + } + updateFunc := func(updated metav1.Object) error { + updatedPVC := updated.(*v1.PersistentVolumeClaim) + result, err := pvcs.Update(updatedPVC) + if err != nil { + return maskAny(err) + } + *p = *result + return nil + } + if err := removeFinalizers(log, finalizers, getFunc, updateFunc); err != nil { + return maskAny(err) + } + return nil +} + +// removeFinalizers is a helper used to remove finalizers from an object. +// The functions tries to get the object using the provided get function, +// then remove the given finalizers and update the update using the given update function. +// In case of an update conflict, the functions tries again. +func removeFinalizers(log zerolog.Logger, finalizers []string, getFunc func() (metav1.Object, error), updateFunc func(metav1.Object) error) error { + attempts := 0 + for { + attempts++ + obj, err := getFunc() + if err != nil { + log.Warn().Err(err).Msg("Failed to get resource") + return maskAny(err) + } + original := obj.GetFinalizers() + if len(original) == 0 { + // We're done + return nil + } + newList := make([]string, 0, len(original)) + shouldRemove := func(f string) bool { + for _, x := range finalizers { + if x == f { + return true + } + } + return false + } + for _, f := range original { + if !shouldRemove(f) { + newList = append(newList, f) + } + } + if len(newList) < len(original) { + obj.SetFinalizers(newList) + if err := updateFunc(obj); IsConflict(err) { + if attempts > maxRemoveFinalizersAttempts { + log.Warn().Err(err).Msg("Failed to update resource with fewer finalizers after many attempts") + return maskAny(err) + } else { + // Try again + continue + } + } else if err != nil { + log.Warn().Err(err).Msg("Failed to update resource with fewer finalizers") + return maskAny(err) + } + } else { + log.Debug().Msg("No finalizers needed removal. Resource unchanged") + } + return nil + } +} diff --git a/pkg/util/k8sutil/images.go b/pkg/util/k8sutil/images.go new file mode 100644 index 000000000..f5f2327b6 --- /dev/null +++ b/pkg/util/k8sutil/images.go @@ -0,0 +1,38 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package k8sutil + +import "strings" + +const ( + dockerPullableImageIDPrefix = "docker-pullable://" +) + +// ConvertImageID2Image converts a ImageID from a ContainerStatus to an Image that can be used +// in a Container specification. +func ConvertImageID2Image(imageID string) string { + if strings.HasPrefix(imageID, dockerPullableImageIDPrefix) { + return imageID[len(dockerPullableImageIDPrefix):] + } + return imageID +} diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 417edba52..2eb645a53 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -24,19 +24,26 @@ package k8sutil import ( "fmt" + "math" + "os" "path/filepath" "strings" "time" + "github.com/arangodb/kube-arangodb/pkg/util/constants" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) const ( + InitDataContainerName = "init-data" + InitLifecycleContainerName = "init-lifecycle" + ServerContainerName = "server" alpineImage = "alpine" arangodVolumeName = "arangod-data" tlsKeyfileVolumeName = "tls-keyfile" + lifecycleVolumeName = "lifecycle" clientAuthCAVolumeName = "client-auth-ca" clusterJWTSecretVolumeName = "cluster-jwt" masterJWTSecretVolumeName = "master-jwt" @@ -44,6 +51,7 @@ const ( ArangodVolumeMountDir = "/data" RocksDBEncryptionVolumeMountDir = "/secrets/rocksdb/encryption" TLSKeyfileVolumeMountDir = "/secrets/tls" + LifecycleVolumeMountDir = "/lifecycle/tools" ClientAuthCAVolumeMountDir = "/secrets/client-auth/ca" ClusterJWTSecretVolumeMountDir = "/secrets/cluster/jwt" MasterJWTSecretVolumeMountDir = "/secrets/master/jwt" @@ -110,6 +118,11 @@ func IsPodNotScheduledFor(pod *v1.Pod, timeout time.Duration) bool { condition.LastTransitionTime.Time.Add(timeout).Before(time.Now()) } +// IsPodMarkedForDeletion returns true if the pod has been marked for deletion. +func IsPodMarkedForDeletion(pod *v1.Pod) bool { + return pod.DeletionTimestamp != nil +} + // IsArangoDBImageIDAndVersionPod returns true if the given pod is used for fetching image ID and ArangoDB version of an image func IsArangoDBImageIDAndVersionPod(p v1.Pod) bool { role, found := p.GetLabels()[LabelKeyRole] @@ -148,6 +161,13 @@ func CreateTLSKeyfileSecretName(deploymentName, role, id string) string { return CreatePodName(deploymentName, role, id, "-tls-keyfile") } +// lifecycleVolumeMounts creates a volume mount structure for shared lifecycle emptyDir. +func lifecycleVolumeMounts() []v1.VolumeMount { + return []v1.VolumeMount{ + {Name: lifecycleVolumeName, MountPath: LifecycleVolumeMountDir}, + } +} + // arangodVolumeMounts creates a volume mount structure for arangod. func arangodVolumeMounts() []v1.VolumeMount { return []v1.VolumeMount{ @@ -238,12 +258,14 @@ func arangodInitContainer(name, id, engine string, requireUUID bool) v1.Containe } // arangodContainer creates a container configured to run `arangod`. -func arangodContainer(name string, image string, imagePullPolicy v1.PullPolicy, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig) v1.Container { +func arangodContainer(image string, imagePullPolicy v1.PullPolicy, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig, + lifecycle *v1.Lifecycle, lifecycleEnvVars []v1.EnvVar) v1.Container { c := v1.Container{ Command: append([]string{"/usr/sbin/arangod"}, args...), - Name: name, + Name: ServerContainerName, Image: image, ImagePullPolicy: imagePullPolicy, + Lifecycle: lifecycle, Ports: []v1.ContainerPort{ { Name: "server", @@ -262,17 +284,23 @@ func arangodContainer(name string, image string, imagePullPolicy v1.PullPolicy, if readinessProbe != nil { c.ReadinessProbe = readinessProbe.Create() } + if lifecycle != nil { + c.Env = append(c.Env, lifecycleEnvVars...) + c.VolumeMounts = append(c.VolumeMounts, lifecycleVolumeMounts()...) + } return c } // arangosyncContainer creates a container configured to run `arangosync`. -func arangosyncContainer(name string, image string, imagePullPolicy v1.PullPolicy, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig) v1.Container { +func arangosyncContainer(image string, imagePullPolicy v1.PullPolicy, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, + lifecycle *v1.Lifecycle, lifecycleEnvVars []v1.EnvVar) v1.Container { c := v1.Container{ Command: append([]string{"/usr/sbin/arangosync"}, args...), - Name: name, + Name: ServerContainerName, Image: image, ImagePullPolicy: imagePullPolicy, + Lifecycle: lifecycle, Ports: []v1.ContainerPort{ { Name: "server", @@ -287,17 +315,82 @@ func arangosyncContainer(name string, image string, imagePullPolicy v1.PullPolic if livenessProbe != nil { c.LivenessProbe = livenessProbe.Create() } + if lifecycle != nil { + c.Env = append(c.Env, lifecycleEnvVars...) + c.VolumeMounts = append(c.VolumeMounts, lifecycleVolumeMounts()...) + } return c } +// newLifecycle creates a lifecycle structure with preStop handler. +func newLifecycle() (*v1.Lifecycle, []v1.EnvVar, []v1.Volume, error) { + binaryPath, err := os.Executable() + if err != nil { + return nil, nil, nil, maskAny(err) + } + exePath := filepath.Join(LifecycleVolumeMountDir, filepath.Base(binaryPath)) + lifecycle := &v1.Lifecycle{ + PreStop: &v1.Handler{ + Exec: &v1.ExecAction{ + Command: append([]string{exePath}, "lifecycle", "preStop"), + }, + }, + } + envVars := []v1.EnvVar{ + v1.EnvVar{ + Name: constants.EnvOperatorPodName, + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + v1.EnvVar{ + Name: constants.EnvOperatorPodNamespace, + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + } + vols := []v1.Volume{ + v1.Volume{ + Name: lifecycleVolumeName, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + } + return lifecycle, envVars, vols, nil +} + +// initLifecycleContainer creates an init-container to copy the lifecycle binary +// to a shared volume. +func initLifecycleContainer(image string) (v1.Container, error) { + binaryPath, err := os.Executable() + if err != nil { + return v1.Container{}, maskAny(err) + } + c := v1.Container{ + Command: append([]string{binaryPath}, "lifecycle", "copy", "--target", LifecycleVolumeMountDir), + Name: InitLifecycleContainerName, + Image: image, + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: lifecycleVolumeMounts(), + } + return c, nil +} + // newPod creates a basic Pod for given settings. -func newPod(deploymentName, ns, role, id, podName string) v1.Pod { +func newPod(deploymentName, ns, role, id, podName string, finalizers []string) v1.Pod { hostname := CreatePodHostName(deploymentName, role, id) p := v1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Labels: LabelsForDeployment(deploymentName, role), + Name: podName, + Labels: LabelsForDeployment(deploymentName, role), + Finalizers: finalizers, }, Spec: v1.PodSpec{ Hostname: hostname, @@ -312,16 +405,34 @@ func newPod(deploymentName, ns, role, id, podName string) v1.Pod { // If the pod already exists, nil is returned. // If another error occurs, that error is returned. func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deployment APIObject, - role, id, podName, pvcName, image string, imagePullPolicy v1.PullPolicy, - engine string, requireUUID bool, - args []string, env map[string]EnvValue, + role, id, podName, pvcName, image, lifecycleImage string, imagePullPolicy v1.PullPolicy, + engine string, requireUUID bool, terminationGracePeriod time.Duration, + args []string, env map[string]EnvValue, finalizers []string, livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig, tlsKeyfileSecretName, rocksdbEncryptionSecretName string) error { // Prepare basic pod - p := newPod(deployment.GetName(), deployment.GetNamespace(), role, id, podName) + p := newPod(deployment.GetName(), deployment.GetNamespace(), role, id, podName, finalizers) + terminationGracePeriodSeconds := int64(math.Ceil(terminationGracePeriod.Seconds())) + p.Spec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds + + // Add lifecycle container + var lifecycle *v1.Lifecycle + var lifecycleEnvVars []v1.EnvVar + var lifecycleVolumes []v1.Volume + if lifecycleImage != "" { + c, err := initLifecycleContainer(lifecycleImage) + if err != nil { + return maskAny(err) + } + p.Spec.InitContainers = append(p.Spec.InitContainers, c) + lifecycle, lifecycleEnvVars, lifecycleVolumes, err = newLifecycle() + if err != nil { + return maskAny(err) + } + } // Add arangod container - c := arangodContainer("arangod", image, imagePullPolicy, args, env, livenessProbe, readinessProbe) + c := arangodContainer(image, imagePullPolicy, args, env, livenessProbe, readinessProbe, lifecycle, lifecycleEnvVars) if tlsKeyfileSecretName != "" { c.VolumeMounts = append(c.VolumeMounts, tlsKeyfileVolumeMounts()...) } @@ -382,6 +493,9 @@ func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deploy p.Spec.Volumes = append(p.Spec.Volumes, vol) } + // Lifecycle volumes (if any) + p.Spec.Volumes = append(p.Spec.Volumes, lifecycleVolumes...) + // Add (anti-)affinity p.Spec.Affinity = createAffinity(deployment.GetName(), role, !developmentMode, "") @@ -394,13 +508,35 @@ func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deploy // CreateArangoSyncPod creates a Pod that runs `arangosync`. // If the pod already exists, nil is returned. // If another error occurs, that error is returned. -func CreateArangoSyncPod(kubecli kubernetes.Interface, developmentMode bool, deployment APIObject, role, id, podName, image string, imagePullPolicy v1.PullPolicy, - args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName, affinityWithRole string) error { +func CreateArangoSyncPod(kubecli kubernetes.Interface, developmentMode bool, deployment APIObject, role, id, podName, image, lifecycleImage string, imagePullPolicy v1.PullPolicy, + terminationGracePeriod time.Duration, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, + tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName, affinityWithRole string) error { // Prepare basic pod - p := newPod(deployment.GetName(), deployment.GetNamespace(), role, id, podName) + p := newPod(deployment.GetName(), deployment.GetNamespace(), role, id, podName, nil) + terminationGracePeriodSeconds := int64(math.Ceil(terminationGracePeriod.Seconds())) + p.Spec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds + + // Add lifecycle container + var lifecycle *v1.Lifecycle + var lifecycleEnvVars []v1.EnvVar + var lifecycleVolumes []v1.Volume + if lifecycleImage != "" { + c, err := initLifecycleContainer(lifecycleImage) + if err != nil { + return maskAny(err) + } + p.Spec.InitContainers = append(p.Spec.InitContainers, c) + lifecycle, lifecycleEnvVars, lifecycleVolumes, err = newLifecycle() + if err != nil { + return maskAny(err) + } + } + + // Lifecycle volumes (if any) + p.Spec.Volumes = append(p.Spec.Volumes, lifecycleVolumes...) // Add arangosync container - c := arangosyncContainer("arangosync", image, imagePullPolicy, args, env, livenessProbe) + c := arangosyncContainer(image, imagePullPolicy, args, env, livenessProbe, lifecycle, lifecycleEnvVars) if tlsKeyfileSecretName != "" { c.VolumeMounts = append(c.VolumeMounts, tlsKeyfileVolumeMounts()...) } diff --git a/pkg/util/k8sutil/pvc.go b/pkg/util/k8sutil/pvc.go index cfed48981..d366ee5ae 100644 --- a/pkg/util/k8sutil/pvc.go +++ b/pkg/util/k8sutil/pvc.go @@ -28,6 +28,11 @@ import ( "k8s.io/client-go/kubernetes" ) +// IsPersistentVolumeClaimMarkedForDeletion returns true if the pod has been marked for deletion. +func IsPersistentVolumeClaimMarkedForDeletion(pvc *v1.PersistentVolumeClaim) bool { + return pvc.DeletionTimestamp != nil +} + // CreatePersistentVolumeClaimName returns the name of the persistent volume claim for a member with // a given id in a deployment with a given name. func CreatePersistentVolumeClaimName(deploymentName, role, id string) string { @@ -37,13 +42,14 @@ func CreatePersistentVolumeClaimName(deploymentName, role, id string) string { // CreatePersistentVolumeClaim creates a persistent volume claim with given name and configuration. // If the pvc already exists, nil is returned. // If another error occurs, that error is returned. -func CreatePersistentVolumeClaim(kubecli kubernetes.Interface, pvcName, deploymentName, ns, storageClassName, role string, resources v1.ResourceRequirements, owner metav1.OwnerReference) error { +func CreatePersistentVolumeClaim(kubecli kubernetes.Interface, pvcName, deploymentName, ns, storageClassName, role string, resources v1.ResourceRequirements, finalizers []string, owner metav1.OwnerReference) error { labels := LabelsForDeployment(deploymentName, role) volumeMode := v1.PersistentVolumeFilesystem pvc := &v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: pvcName, - Labels: labels, + Name: pvcName, + Labels: labels, + Finalizers: finalizers, }, Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{ diff --git a/tests/resilience_test.go b/tests/resilience_test.go index 5144d7250..7958e25a8 100644 --- a/tests/resilience_test.go +++ b/tests/resilience_test.go @@ -166,8 +166,9 @@ func TestResiliencePVC(t *testing.T) { // Delete one pvc after the other apiObject.ForeachServerGroup(func(group api.ServerGroup, spec api.ServerGroupSpec, status *api.MemberStatusList) error { - if group == api.ServerGroupCoordinators { + if group != api.ServerGroupAgents { // Coordinators have no PVC + // DBServers will be cleaned out and create a new member return nil } for _, m := range *status { @@ -179,6 +180,10 @@ func TestResiliencePVC(t *testing.T) { if err := kubecli.CoreV1().PersistentVolumeClaims(ns).Delete(m.PersistentVolumeClaimName, &metav1.DeleteOptions{}); err != nil { t.Fatalf("Failed to delete pvc %s: %v", m.PersistentVolumeClaimName, err) } + // Now delete the pod as well, otherwise the PVC will only have a deletion timestamp but its finalizers will stay on. + if err := kubecli.CoreV1().Pods(ns).Delete(m.PodName, &metav1.DeleteOptions{}); err != nil { + t.Fatalf("Failed to delete pod %s: %v", m.PodName, err) + } // Wait for pvc to return with different UID op := func() error { pvc, err := kubecli.CoreV1().PersistentVolumeClaims(ns).Get(m.PersistentVolumeClaimName, metav1.GetOptions{})