Skip to content

Commit

Permalink
Propagate VR conditions .Message
Browse files Browse the repository at this point in the history 
When condition is not met, we set the protected PVC condition message
using the error message returned from isVRConditionMet(). When using
csi-addons > 0.10.0, we use now the message from the condition instead
of the default message.

Since the Validated condition is not reported by older version of
csi-addons, and ramen may run on a system with older csi-addons,
isVRConditionMet() return now a boolean value telling if the check was
successful; we found the condition, its observed generation matches the
object generation, and its value is not Unknown.

Since we want to propagate the message from the Validated condition, if
Validate.Status was False, we set the DataReady and DataProtected using
the Validated.Message, and skip checking the next conditions.

If we could not check the Validated condition we continue to check the
other conditions.

Signed-off-by: Nir Soffer <[email protected]>
  • Loading branch information
@nirs
nirs committed Nov 5, 2024
1 parent c496fe9 commit f84aa7f
Showing 1 changed file with 47 additions and 24 deletions.
71 changes: 47 additions & 24 deletions internal/controller/vrg_volrep.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1412,19 +1412,21 @@ func (v *VRGInstance) checkVRStatus(pvc *corev1.PersistentVolumeClaim, volRep *v
// We handle 3 cases:
// - Primary deleted VRG: If Validated condition exists and false, the VR will never complete and can be
// deleted safely. Otherwise Completed condition is checked.
// - Primary VRG: Completed condition is checked.
// - Primary VRG: Validated condition is checked, and if successful the Completed conditions is also checked.
// - Secondary VRG: Completed, Degraded and Resyncing conditions are checked and ensured healthy.
func (v *VRGInstance) validateVRStatus(pvc *corev1.PersistentVolumeClaim, volRep *volrep.VolumeReplication,
state ramendrv1alpha1.ReplicationState,
) bool {
// Check validated for primary during VRG deletion.
if state == ramendrv1alpha1.Primary && rmnutil.ResourceIsDeleted(v.instance) {
validated, ok := v.validateVRValidatedStatus(volRep)
// If primary, check the validated condition.
if state == ramendrv1alpha1.Primary {
validated, ok := v.validateVRValidatedStatus(pvc, volRep)
if !validated && ok {
// This VR will never complete since it failed initial validation. If we are in deleting flow the VR has reached the
// desired state and we can delete it. Otherwise there is no needd to check the rest of the conditions.
v.log.Info(fmt.Sprintf("VolumeReplication %s/%s failed validation and can be deleted",
volRep.GetName(), volRep.GetNamespace()))

return true
return rmnutil.ResourceIsDeleted(v.instance)
}
}

Expand All @@ -1448,19 +1450,26 @@ func (v *VRGInstance) validateVRStatus(pvc *corev1.PersistentVolumeClaim, volRep
return true
}

// validateVRValidatedStatus validates that VolumeReplicaion resource was validated.
// Return 2 booleans
// validateVRValidatedStatus validates that VolumeReplication resource was validated.
// Returns 2 booleans:
// - validated: true if the condition is true, otherwise false
// - ok: true if the check was succeesfull, false if the condition is missing, stale, or unknown.
// - ok: true if the check was successful, otherwise false
func (v *VRGInstance) validateVRValidatedStatus(
pvc *corev1.PersistentVolumeClaim,
volRep *volrep.VolumeReplication,
) (bool, bool) {
conditionMet, errorMsg := isVRConditionMet(volRep, volrep.ConditionValidated, metav1.ConditionTrue)
if errorMsg != "" {
v.log.Info(fmt.Sprintf("%s (VolRep: %s/%s)", errorMsg, volRep.GetName(), volRep.GetNamespace()))
// Check the condition, but update the pvc conditions only if the check was successful.
conditionMet, ok, errorMsg := isVRConditionMet(volRep, volrep.ConditionValidated, metav1.ConditionTrue)
if !conditionMet && ok {
defaultMsg := "VolumeReplication resource failed validation"
v.updatePVCDataReadyConditionHelper(pvc.Namespace, pvc.Name, VRGConditionReasonError, errorMsg,
defaultMsg)
v.updatePVCDataProtectedConditionHelper(pvc.Namespace, pvc.Name, VRGConditionReasonError, errorMsg,
defaultMsg)
v.log.Info(fmt.Sprintf("%s (VolRep: %s/%s)", defaultMsg, volRep.GetName(), volRep.GetNamespace()))
}

return conditionMet, errorMsg == ""
return conditionMet, ok
}

// validateVRCompletedStatus validates if the VolumeReplication resource Completed condition is met and update
Expand All @@ -1483,7 +1492,7 @@ func (v *VRGInstance) validateVRCompletedStatus(pvc *corev1.PersistentVolumeClai
action = "demoted"
}

conditionMet, msg := isVRConditionMet(volRep, volrep.ConditionCompleted, metav1.ConditionTrue)
conditionMet, _, msg := isVRConditionMet(volRep, volrep.ConditionCompleted, metav1.ConditionTrue)
if !conditionMet {
defaultMsg := fmt.Sprintf("VolumeReplication resource for pvc not %s to %s", action, stateString)
v.updatePVCDataReadyConditionHelper(pvc.Namespace, pvc.Name, VRGConditionReasonError, msg,
Expand Down Expand Up @@ -1520,12 +1529,12 @@ func (v *VRGInstance) validateAdditionalVRStatusForSecondary(pvc *corev1.Persist
) bool {
v.updatePVCLastSyncCounters(pvc.Namespace, pvc.Name, nil)

conditionMet, _ := isVRConditionMet(volRep, volrep.ConditionResyncing, metav1.ConditionTrue)
conditionMet, _, _ := isVRConditionMet(volRep, volrep.ConditionResyncing, metav1.ConditionTrue)
if !conditionMet {
return v.checkResyncCompletionAsSecondary(pvc, volRep)
}

conditionMet, msg := isVRConditionMet(volRep, volrep.ConditionDegraded, metav1.ConditionTrue)
conditionMet, _, msg := isVRConditionMet(volRep, volrep.ConditionDegraded, metav1.ConditionTrue)
if !conditionMet {
defaultMsg := "VolumeReplication resource for pvc is not in Degraded condition while resyncing"
v.updatePVCDataProtectedConditionHelper(pvc.Namespace, pvc.Name, VRGConditionReasonError, msg,
Expand Down Expand Up @@ -1554,7 +1563,7 @@ func (v *VRGInstance) validateAdditionalVRStatusForSecondary(pvc *corev1.Persist
func (v *VRGInstance) checkResyncCompletionAsSecondary(pvc *corev1.PersistentVolumeClaim,
volRep *volrep.VolumeReplication,
) bool {
conditionMet, msg := isVRConditionMet(volRep, volrep.ConditionResyncing, metav1.ConditionFalse)
conditionMet, _, msg := isVRConditionMet(volRep, volrep.ConditionResyncing, metav1.ConditionFalse)
if !conditionMet {
defaultMsg := "VolumeReplication resource for pvc not syncing as Secondary"
v.updatePVCDataReadyConditionHelper(pvc.Namespace, pvc.Name, VRGConditionReasonError, msg,
Expand All @@ -1568,7 +1577,7 @@ func (v *VRGInstance) checkResyncCompletionAsSecondary(pvc *corev1.PersistentVol
return false
}

conditionMet, msg = isVRConditionMet(volRep, volrep.ConditionDegraded, metav1.ConditionFalse)
conditionMet, _, msg = isVRConditionMet(volRep, volrep.ConditionDegraded, metav1.ConditionFalse)
if !conditionMet {
defaultMsg := "VolumeReplication resource for pvc is not syncing and is degraded as Secondary"
v.updatePVCDataReadyConditionHelper(pvc.Namespace, pvc.Name, VRGConditionReasonError, msg,
Expand All @@ -1592,35 +1601,49 @@ func (v *VRGInstance) checkResyncCompletionAsSecondary(pvc *corev1.PersistentVol
return true
}

// isVRConditionMet returns true if the condition is met, and an error mesage if we could not get the
// condition value.
// isVRConditionMet check if condition is met.
// Returns 3 values:
// - met: true if the condition status matches the desired status, otherwise false
// - ok: true if the check was successful; the condition was found, its observedGeneration matches the object
// generation, and its value is not unknown.
// - errorMsg: error message describing why the condition is not met
func isVRConditionMet(volRep *volrep.VolumeReplication,
conditionType string,
desiredStatus metav1.ConditionStatus,
) (bool, string) {
) (bool, bool, string) {
met := true
ok := true

volRepCondition := findCondition(volRep.Status.Conditions, conditionType)
if volRepCondition == nil {
errorMsg := fmt.Sprintf("Failed to get the %s condition from status of VolumeReplication resource.",
conditionType)

return false, errorMsg
return !met, !ok, errorMsg
}

if volRep.GetGeneration() != volRepCondition.ObservedGeneration {
errorMsg := fmt.Sprintf("Stale generation for condition %s from status of VolumeReplication resource.",
conditionType)

return false, errorMsg
return !met, !ok, errorMsg
}

if volRepCondition.Status == metav1.ConditionUnknown {
errorMsg := fmt.Sprintf("Unknown status for condition %s from status of VolumeReplication resource.",
conditionType)

return false, errorMsg
return !met, !ok, errorMsg
}

// We can compare the condition status.

if volRepCondition.Status != desiredStatus {
// csi-addons > 0.10.0 returns defailed error message
return !met, ok, volRepCondition.Message
}

return volRepCondition.Status == desiredStatus, ""
return met, ok, ""
}

// Disabling unparam linter as currently every invokation of this
Expand Down

0 comments on commit f84aa7f

Please sign in to comment.