From 0fd937eaee95869d1c649a4206b60db382757b15 Mon Sep 17 00:00:00 2001 From: Sean Eagan Date: Wed, 9 Sep 2020 14:50:28 -0500 Subject: [PATCH] Refactor status API This removes: - Installed, Upgraded, RolledBack, and Uninstalled status conditions since they did not represent current state, but rather actions taken, which are already recorded by events. - status.observedStateReconciled since it solved the problem of remembering past release (install/upgrade/test) success, but not past release failures, after other subsequent failures such as dependency failures, k8s API failures, etc. This adds: - Remediated status condition which records whether the release is currently in a remediated state. It is used to prevent release retries after remediation failures. We were previously not doing this for rollback failures. - Released status condition which records whether the current state has been successfully released (install/upgrade/test). This is used to remember the last release attempt status, regardless of any subsequent other failures such as dependency failures, k8s API failures, etc. This renames: - Tested > TestsSuccess status condition, for forward compatibility with interval based helm tests. --- .github/workflows/e2e.yaml | 32 +++--- api/v2alpha1/condition_types.go | 24 ++-- api/v2alpha1/helmrelease_types.go | 28 +++-- .../helm.toolkit.fluxcd.io_helmreleases.yaml | 4 - controllers/helmrelease_controller.go | 108 ++++++++++-------- docs/api/helmrelease.md | 12 -- docs/spec/v2alpha1/helmreleases.md | 95 ++++++++++----- 7 files changed, 174 insertions(+), 129 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index f5f7e88ac..56c12e71c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -84,7 +84,7 @@ jobs: kubectl -n helm-system apply -f config/testdata/$test_name echo -n ">>> Waiting for expected conditions" count=0 - until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="False" and .Ready=="False"' )" ]; do + until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False"' )" ]; do echo -n '.' sleep 5 count=$((count + 1)) @@ -107,7 +107,7 @@ jobs: kubectl -n helm-system apply -f config/testdata/$test_name echo -n ">>> Waiting for expected conditions" count=0 - until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Tested=="False" and .Ready=="False"' )" ]; do + until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .TestSuccess=="False" and .Ready=="False"' )" ]; do echo -n '.' sleep 5 count=$((count + 1)) @@ -130,7 +130,7 @@ jobs: kubectl -n helm-system apply -f config/testdata/$test_name echo -n ">>> Waiting for expected conditions" count=0 - until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Tested=="False" and .Ready=="True"' )" ]; do + until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .TestSuccess=="False" and .Ready=="True"' )" ]; do echo -n '.' sleep 5 count=$((count + 1)) @@ -153,7 +153,7 @@ jobs: kubectl -n helm-system apply -f config/testdata/$test_name echo -n ">>> Waiting for expected conditions" count=0 - until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="False" and .Ready=="False" and .Uninstalled=="True"' )" ]; do + until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" and .Remediated=="True"' )" ]; do echo -n '.' sleep 5 count=$((count + 1)) @@ -176,7 +176,7 @@ jobs: kubectl -n helm-system apply -f config/testdata/$test_name echo -n ">>> Waiting for expected conditions" count=0 - until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.installFailures == 2 and ( .status.conditions | map( { (.type): .status } ) | add | .Installed=="False" and .Ready=="False" )' )" ]; do + until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.installFailures == 2 and ( .status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" )' )" ]; do echo -n '.' sleep 5 count=$((count + 1)) @@ -199,7 +199,7 @@ jobs: kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml echo -n ">>> Waiting for expected conditions" count=0 - until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Ready=="True"' )" ]; do + until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do echo -n '.' sleep 5 count=$((count + 1)) @@ -220,7 +220,7 @@ jobs: kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml echo -n ">>> Waiting for expected conditions" count=0 - until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Upgraded=="False" and .Ready=="False"' )" ]; do + until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False"' )" ]; do echo -n '.' sleep 5 count=$((count + 1)) @@ -243,7 +243,7 @@ jobs: kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml echo -n ">>> Waiting for expected conditions" count=0 - until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Ready=="True"' )" ]; do + until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do echo -n '.' sleep 5 count=$((count + 1)) @@ -264,7 +264,7 @@ jobs: kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml echo -n ">>> Waiting for expected conditions" count=0 - until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Upgraded=="True" and .Tested=="False" and .Ready=="False"' )" ]; do + until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .TestSuccess=="False" and .Ready=="False"' )" ]; do echo -n '.' sleep 5 count=$((count + 1)) @@ -287,7 +287,7 @@ jobs: kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml echo -n ">>> Waiting for expected conditions" count=0 - until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Ready=="True"' )" ]; do + until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do echo -n '.' sleep 5 count=$((count + 1)) @@ -308,7 +308,7 @@ jobs: kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml echo -n ">>> Waiting for expected conditions" count=0 - until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Upgraded=="False" and .Ready=="False" and .RolledBack=="True"' )" ]; do + until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" and .Remediated=="True"' )" ]; do echo -n '.' sleep 5 count=$((count + 1)) @@ -336,7 +336,7 @@ jobs: kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml echo -n ">>> Waiting for expected conditions" count=0 - until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Ready=="True"' )" ]; do + until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do echo -n '.' sleep 5 count=$((count + 1)) @@ -357,7 +357,7 @@ jobs: kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml echo -n ">>> Waiting for expected conditions" count=0 - until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.upgradeFailures == 2 and ( .status.conditions | map( { (.type): .status } ) | add | .Upgraded=="False" and .Ready=="False" and .RolledBack=="True" )' )" ]; do + until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.upgradeFailures == 2 and ( .status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" )' )" ]; do echo -n '.' sleep 5 count=$((count + 1)) @@ -385,7 +385,7 @@ jobs: kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml echo -n ">>> Waiting for expected conditions" count=0 - until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Ready=="True"' )" ]; do + until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do echo -n '.' sleep 5 count=$((count + 1)) @@ -412,7 +412,7 @@ jobs: kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml echo -n ">>> Waiting for expected conditions" count=0 - until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.upgradeFailures == 1 and .status.installFailures == 1 and ( .status.conditions | map( { (.type): .status } ) | add | .Upgraded=="False" and .Uninstalled=="True" and .Installed=="False" and .Ready=="False" )' )" ]; do + until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.upgradeFailures == 1 and .status.installFailures == 1 and ( .status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" )' )" ]; do echo -n '.' sleep 5 count=$((count + 1)) @@ -449,6 +449,6 @@ jobs: kubectl -n helm-system get helmcharts -oyaml kubectl -n helm-system get helmreleases -oyaml kubectl -n helm-system get all - helm ls -n helm-system + helm ls -n helm-system --all kubectl -n helm-system logs deploy/source-controller kubectl -n helm-system logs deploy/helm-controller diff --git a/api/v2alpha1/condition_types.go b/api/v2alpha1/condition_types.go index d5a97c54d..899e04f27 100644 --- a/api/v2alpha1/condition_types.go +++ b/api/v2alpha1/condition_types.go @@ -51,20 +51,14 @@ const ( // ReadyCondition represents the fact that the HelmRelease has been successfully reconciled. ReadyCondition string = "Ready" - // InstalledCondition represents the fact that the HelmRelease has been successfully installed. - InstalledCondition string = "Installed" + // ReleasedCondition represents the fact that the HelmRelease has been successfully released. + ReleasedCondition string = "Released" - // UpgradedCondition represents the fact that the HelmRelease has been successfully upgraded. - UpgradedCondition string = "Upgraded" + // TestSuccessCondition represents the fact that the tests for the HelmRelease are succeeding. + TestSuccessCondition string = "TestSuccess" - // TestedCondition represents the fact that the HelmRelease has been successfully tested. - TestedCondition string = "Tested" - - // RolledBackCondition represents the fact that the HelmRelease has been successfully rolled back. - RolledBackCondition string = "RolledBack" - - // UninstalledCondition represents the fact that the HelmRelease has been successfully uninstalled. - UninstalledCondition string = "Uninstalled" + // RemediatedCondition represents the fact that the HelmRelease has been successfully remediated. + RemediatedCondition string = "Remediated" ) const ( @@ -86,11 +80,11 @@ const ( // UpgradeFailedReason represents the fact that the Helm upgrade for the HelmRelease failed. UpgradeFailedReason string = "UpgradeFailed" - // TestSucceededReason represents the fact that the Helm test for the HelmRelease succeeded. + // TestSucceededReason represents the fact that the Helm tests for the HelmRelease succeeded. TestSucceededReason string = "TestSucceeded" - // TestFailedReason represents the fact that the Helm test for the HelmRelease failed. - TestFailedReason string = "TestFailed" + // TestFailedReason represents the fact that the Helm tests for the HelmRelease failed. + TestFailedReason string = "TestsFailed" // RollbackSucceededReason represents the fact that the Helm rollback for the HelmRelease succeeded. RollbackSucceededReason string = "RollbackSucceeded" diff --git a/api/v2alpha1/helmrelease_types.go b/api/v2alpha1/helmrelease_types.go index a923a0224..8cdcf7ceb 100644 --- a/api/v2alpha1/helmrelease_types.go +++ b/api/v2alpha1/helmrelease_types.go @@ -584,10 +584,6 @@ type HelmReleaseStatus struct { // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // ObservedStateReconciled represents whether the observed state has been successfully reconciled. - // +optional - ObservedStateReconciled bool `json:"observedStateReconciled,omitempty"` - // LastObservedTime is the last time at which the HelmRelease was observed. // +optional LastObservedTime metav1.Time `json:"lastObservedTime,omitempty"` @@ -646,23 +642,21 @@ func (in HelmReleaseStatus) GetHelmChart() (string, string) { // by setting the ReadyCondition to ConditionUnknown for ProgressingReason. func HelmReleaseProgressing(hr HelmRelease) HelmRelease { resetFailureCounts(&hr) - hr.Status.ObservedStateReconciled = false hr.Status.Conditions = []Condition{} SetHelmReleaseCondition(&hr, ReadyCondition, corev1.ConditionUnknown, ProgressingReason, "reconciliation in progress") return hr } -// HelmReleaseNotReady registers a failed release attempt of the given HelmRelease. +// HelmReleaseNotReady registers a failed reconciliation of the given HelmRelease. func HelmReleaseNotReady(hr HelmRelease, reason, message string) HelmRelease { SetHelmReleaseCondition(&hr, ReadyCondition, corev1.ConditionFalse, reason, message) hr.Status.Failures++ return hr } -// HelmReleaseReady registers a successful release attempt of the given HelmRelease. +// HelmReleaseReady registers a successful reconciliation of the given HelmRelease. func HelmReleaseReady(hr HelmRelease) HelmRelease { resetFailureCounts(&hr) - hr.Status.ObservedStateReconciled = true hr.Status.LastAppliedRevision = hr.Status.LastAttemptedRevision SetHelmReleaseCondition(&hr, ReadyCondition, corev1.ConditionTrue, ReconciliationSucceededReason, "release reconciliation succeeded") return hr @@ -687,6 +681,18 @@ func resetFailureCounts(hr *HelmRelease) { hr.Status.UpgradeFailures = 0 } +// GetHelmReleaseCondition return the given condition of the given HelmRelease +// or nil if not present. +func GetHelmReleaseCondition(hr HelmRelease, condition string) *Condition { + for _, c := range hr.Status.Conditions { + if c.Type == condition { + return &c + } + } + + return nil +} + // SetHelmReleaseCondition sets the given condition with the given status, reason and message // on the HelmRelease. func SetHelmReleaseCondition(hr *HelmRelease, condition string, status corev1.ConditionStatus, reason, message string) { @@ -700,6 +706,12 @@ func SetHelmReleaseCondition(hr *HelmRelease, condition string, status corev1.Co }) } +// DeleteHelmReleaseCondition deletes the given condition of the given HelmRelease +// if present. +func DeleteHelmReleaseCondition(hr *HelmRelease, condition string) { + hr.Status.Conditions = filterOutCondition(hr.Status.Conditions, condition) +} + const ( // SourceIndexKey is the key used for indexing HelmReleases based on // their sources. diff --git a/config/crd/bases/helm.toolkit.fluxcd.io_helmreleases.yaml b/config/crd/bases/helm.toolkit.fluxcd.io_helmreleases.yaml index c5343b34a..2355b006d 100644 --- a/config/crd/bases/helm.toolkit.fluxcd.io_helmreleases.yaml +++ b/config/crd/bases/helm.toolkit.fluxcd.io_helmreleases.yaml @@ -436,10 +436,6 @@ spec: description: ObservedGeneration is the last observed generation. format: int64 type: integer - observedStateReconciled: - description: ObservedStateReconciled represents whether the observed - state has been successfully reconciled. - type: boolean upgradeFailures: description: UpgradeFailures is the upgrade failure count against the latest observed state. It is reset after a successful reconciliation. diff --git a/controllers/helmrelease_controller.go b/controllers/helmrelease_controller.go index f8b44ce87..ffc0973f3 100644 --- a/controllers/helmrelease_controller.go +++ b/controllers/helmrelease_controller.go @@ -330,48 +330,65 @@ func (r *HelmReleaseReconciler) release(ctx context.Context, log logr.Logger, hr } } - // Determine release deployment action. - var deployAction v2.DeploymentAction - switch { - // Install if there is no release. - case rel == nil: - deployAction = hr.Spec.GetInstall() - // Fail if the release was due to a failed install (which was not uninstalled). - // The uninstall may have failed, or was not needed due to retries being exhausted - // and remediateLastFailure being false. - case hr.Spec.GetInstall().GetRemediation().GetFailureCount(hr) > 0: - return hr, fmt.Errorf("last install failed but was not uninstalled") - // Skip and mark ready if the observed state was already reconciled. - case hr.Status.ObservedStateReconciled: - return v2.HelmReleaseReady(hr), nil - // Otherwise upgrade. - default: - deployAction = hr.Spec.GetUpgrade() + // Check status of any previous release attempt. + released := v2.GetHelmReleaseCondition(hr, v2.ReleasedCondition) + if released != nil { + switch released.Status { + // Succeed if the previous release attempt succeeded. + case corev1.ConditionTrue: + return v2.HelmReleaseReady(hr), nil + case corev1.ConditionFalse: + // Fail if the previous release attempt remediation failed. + remediated := v2.GetHelmReleaseCondition(hr, v2.RemediatedCondition) + if remediated != nil && remediated.Status == corev1.ConditionFalse { + err = fmt.Errorf("previous release attempt remediation failed") + return v2.HelmReleaseNotReady(hr, remediated.Reason, remediated.Message), err + } + } } - // Check if retries exhausted. - remediation := deployAction.GetRemediation() - if remediation.RetriesExhausted(hr) { - return hr, fmt.Errorf("%s retries exhausted", deployAction.GetDescription()) + // Fail if install retries are exhausted. + if hr.Spec.GetInstall().GetRemediation().RetriesExhausted(hr) { + err = fmt.Errorf("install retries exhausted") + return v2.HelmReleaseNotReady(hr, released.Reason, released.Message), err + } + + // Fail if there is a release and upgrade retries are exhausted. + // This avoids failing after an upgrade uninstall remediation strategy. + if rel != nil && hr.Spec.GetUpgrade().GetRemediation().RetriesExhausted(hr) { + err = fmt.Errorf("upgrade retries exhausted") + return v2.HelmReleaseNotReady(hr, released.Reason, released.Message), err } // Deploy the release. - switch a := deployAction.(type) { - case v2.Install: + var deployAction v2.DeploymentAction + if rel == nil { + deployAction = hr.Spec.GetInstall() rel, err = install(cfg, loadedChart, hr, values) - err = r.handleHelmActionResult(&hr, revision, err, a.GetDescription(), v2.InstalledCondition, v2.InstallSucceededReason, v2.InstallFailedReason) - case v2.Upgrade: + err = r.handleHelmActionResult(&hr, revision, err, deployAction.GetDescription(), v2.ReleasedCondition, v2.InstallSucceededReason, v2.InstallFailedReason) + } else { + deployAction = hr.Spec.GetUpgrade() rel, err = upgrade(cfg, loadedChart, hr, values) - err = r.handleHelmActionResult(&hr, revision, err, a.GetDescription(), v2.UpgradedCondition, v2.UpgradeSucceededReason, v2.UpgradeFailedReason) + err = r.handleHelmActionResult(&hr, revision, err, deployAction.GetDescription(), v2.ReleasedCondition, v2.UpgradeSucceededReason, v2.UpgradeFailedReason) } + remediation := deployAction.GetRemediation() - // Run tests if enabled and there is a successful new release revision. - if getReleaseRevision(rel) > releaseRevision && err == nil && hr.Spec.GetTest().Enable { - _, testErr := test(cfg, hr) - testErr = r.handleHelmActionResult(&hr, revision, testErr, "test", v2.TestedCondition, v2.TestSucceededReason, v2.TestFailedReason) - // Propagate any test error if not marked ignored. - if testErr != nil && !remediation.MustIgnoreTestFailures(hr.Spec.GetTest().IgnoreFailures) { - err = testErr + // If there is a new release revision... + if getReleaseRevision(rel) > releaseRevision { + // Ensure release is not marked remediated. + v2.DeleteHelmReleaseCondition(&hr, v2.RemediatedCondition) + + // If new release revision is successful and tests are enabled, run them. + if err == nil && hr.Spec.GetTest().Enable { + _, testErr := test(cfg, hr) + testErr = r.handleHelmActionResult(&hr, revision, testErr, "test", v2.TestSuccessCondition, v2.TestSucceededReason, v2.TestFailedReason) + + // Propagate any test error if not marked ignored. + if testErr != nil && !remediation.MustIgnoreTestFailures(hr.Spec.GetTest().IgnoreFailures) { + testsPassing := v2.GetHelmReleaseCondition(hr, v2.TestSuccessCondition) + v2.SetHelmReleaseCondition(&hr, v2.ReleasedCondition, corev1.ConditionFalse, testsPassing.Reason, testsPassing.Message) + err = testErr + } } } @@ -380,20 +397,21 @@ func (r *HelmReleaseReconciler) release(ctx context.Context, log logr.Logger, hr remediation.IncrementFailureCount(&hr) // Remediate deployment failure if necessary. if !remediation.RetriesExhausted(hr) || remediation.MustRemediateLastFailure() { - switch { - case getReleaseRevision(rel) <= releaseRevision: + if getReleaseRevision(rel) <= releaseRevision { log.Info(fmt.Sprintf("skipping remediation, no new release revision created")) - case remediation.GetStrategy() == v2.RollbackRemediationStrategy: - rollbackErr := rollback(cfg, hr) - rollbackConditionErr := r.handleHelmActionResult(&hr, revision, rollbackErr, "rollback", v2.RolledBackCondition, v2.RollbackSucceededReason, v2.RollbackFailedReason) - if rollbackConditionErr != nil { - err = rollbackConditionErr + } else { + var remediationErr error + switch remediation.GetStrategy() { + case v2.RollbackRemediationStrategy: + rollbackErr := rollback(cfg, hr) + remediationErr = r.handleHelmActionResult(&hr, revision, rollbackErr, "rollback", v2.RemediatedCondition, v2.RollbackSucceededReason, v2.RollbackFailedReason) + case v2.UninstallRemediationStrategy: + uninstallErr := uninstall(cfg, hr) + remediationErr = r.handleHelmActionResult(&hr, revision, uninstallErr, "uninstall", v2.RemediatedCondition, v2.UninstallSucceededReason, v2.UninstallFailedReason) } - case remediation.GetStrategy() == v2.UninstallRemediationStrategy: - uninstallErr := uninstall(cfg, hr) - uninstallConditionErr := r.handleHelmActionResult(&hr, revision, uninstallErr, "uninstall", v2.UninstalledCondition, v2.UninstallSucceededReason, v2.UninstallFailedReason) - if uninstallConditionErr != nil { - err = uninstallConditionErr + + if remediationErr != nil { + err = remediationErr } } diff --git a/docs/api/helmrelease.md b/docs/api/helmrelease.md index a528a03f7..d9c860af9 100644 --- a/docs/api/helmrelease.md +++ b/docs/api/helmrelease.md @@ -895,18 +895,6 @@ int64 -observedStateReconciled
- -bool - - - -(Optional) -

ObservedStateReconciled represents whether the observed state has been successfully reconciled.

- - - - lastObservedTime
diff --git a/docs/spec/v2alpha1/helmreleases.md b/docs/spec/v2alpha1/helmreleases.md index 05723b27b..7b9b4b0e4 100644 --- a/docs/spec/v2alpha1/helmreleases.md +++ b/docs/spec/v2alpha1/helmreleases.md @@ -368,20 +368,14 @@ const ( // ReadyCondition represents the fact that the HelmRelease has been successfully reconciled. ReadyCondition string = "Ready" - // InstalledCondition represents the fact that the HelmRelease has been successfully installed. - InstalledCondition string = "Installed" + // ReleasedCondition represents the fact that the HelmRelease has been successfully released. + ReleasedCondition string = "Released" - // UpgradedCondition represents the fact that the HelmRelease has been successfully upgraded. - UpgradedCondition string = "Upgraded" + // TestSuccessCondition represents the fact that the tests for the HelmRelease are succeeding. + TestSuccessCondition string = "TestSuccess" - // TestedCondition represents the fact that the HelmRelease has been successfully tested. - TestedCondition string = "Tested" - - // RolledBackCondition represents the fact that the HelmRelease has been successfully rolled back. - RolledBackCondition string = "RolledBack" - - // UninstalledCondition represents the fact that the HelmRelease has been successfully uninstalled. - UninstalledCondition string = "Uninstalled" + // RemediatedCondition represents the fact that the HelmRelease has been successfully remediated. + RemediatedCondition string = "Remediated" ) ``` @@ -407,11 +401,11 @@ const ( // UpgradeFailedReason represents the fact that the Helm upgrade for the HelmRelease failed. UpgradeFailedReason string = "UpgradeFailed" - // TestSucceededReason represents the fact that the Helm test for the HelmRelease succeeded. + // TestSucceededReason represents the fact that the Helm tests for the HelmRelease succeeded. TestSucceededReason string = "TestSucceeded" - // TestFailedReason represents the fact that the Helm test for the HelmRelease failed. - TestFailedReason string = "TestFailed" + // TestFailedReason represents the fact that the Helm tests for the HelmRelease failed. + TestFailedReason string = "TestsFailed" // RollbackSucceededReason represents the fact that the Helm rollback for the HelmRelease succeeded. RollbackSucceededReason string = "RollbackSucceeded" @@ -675,22 +669,42 @@ spec: When the controller completes a reconciliation, it reports the result in the status sub-resource. -A successful reconciliation sets the `Ready` condition to `true`: +The following `status.conditions` are supported: + +* `Ready` - status of the last reconciliation attempt +* `Released` - status of the last release attempt (install/upgrade/test) against the current state +* `TestSuccess` - status of the last test attempt against the current state +* `Remediated` - status of the last remediation attempt (uninstall/rollback) due to a failure of the + last release attempt against the current state + +You can wait for the helm-controller to complete a reconciliation with: + +```sh +kubectl wait helmrelease/podinfo --for=condition=ready +``` + +### Examples + +Install Success: ```yaml status: conditions: - lastTransitionTime: "2020-07-13T13:13:40Z" - message: Helm installation succeeded + message: Helm install succeeded reason: InstallSucceeded status: "True" - type: Install + type: Released + - lastTransitionTime: "2020-07-13T13:13:40Z" + message: Helm test succeeded + reason: TestSucceeded + status: "True" + type: TestSuccess - lastTransitionTime: "2020-07-13T13:13:42Z" message: release reconciliation succeeded reason: ReconciliationSucceeded status: "True" type: Ready - observedStateReconciled: true lastAppliedRevision: 4.0.6 lastAttemptedRevision: 4.0.6 lastObservedTime: "2020-07-13T13:18:42Z" @@ -698,13 +712,7 @@ status: observedGeneration: 2 ``` -You can wait for the helm-controller to complete a reconciliation with: - -```sh -kubectl wait helmrelease/podinfo --for=condition=ready -``` - -A failed reconciliation sets the `Ready` condition to `false`: +Upgrade Failure: ```yaml status: @@ -715,10 +723,12 @@ status: expected "integer"' reason: UpgradeFailed status: "False" - type: Upgrade + type: Released - lastTransitionTime: "2020-07-13T13:17:28Z" - message: release reconciliation failed - reason: ReconciliationFailed + message: 'error validating "": error validating data: ValidationError(Deployment.spec.replicas): + invalid type for io.k8s.api.apps.v1.DeploymentSpec.replicas: got "string", + expected "integer"' + reason: UpgradeFailed status: "False" type: Ready failures: 1 @@ -728,3 +738,30 @@ status: lastReleaseRevision: 1 observedGeneration: 3 ``` + +Ignored Test Failure: + +```yaml +status: + conditions: + - lastTransitionTime: "2020-07-13T13:13:40Z" + message: Helm install succeeded + reason: InstallSucceeded + status: "True" + type: Released + - lastTransitionTime: "2020-07-13T13:13:40Z" + message: Helm test failed + reason: TestsFailed + status: "False" + type: TestSuccess + - lastTransitionTime: "2020-07-13T13:13:42Z" + message: release reconciliation succeeded + reason: ReconciliationSucceeded + status: "True" + type: Ready + lastAppliedRevision: 4.0.6 + lastAttemptedRevision: 4.0.6 + lastObservedTime: "2020-07-13T13:18:42Z" + lastReleaseRevision: 1 + observedGeneration: 2 +```