Skip to content

Commit

Permalink
Refactor status API
Browse files Browse the repository at this point in the history
This removes:

- Installed, Upgraded, RolledBack, and Uninstalled status conditions
  since they did not represent current state, but rather actions
  taken, which are already recorded by events.
- status.observedStateReconciled since it solved the problem of
  remembering past release (install/upgrade/test) success, but not
  past release failures, after other subsequent failures such as
  dependency failures, k8s API failures, etc.

This adds:

- Remediated status condition which records whether the release is
  currently in a remediated state. It is used to prevent release retries
  after remediation failures. We were previously not doing this for
  rollback failures.
- Released status condition which records whether the current state
  has been successfully released (install/upgrade/test). This is used to
  remember the last release attempt status, regardless of any subsequent
  other failures such as dependency failures, k8s API failures, etc.

This renames:

- Tested > TestsSuccess status condition, for forward compatibility
  with interval based helm tests.
  • Loading branch information
seaneagan committed Sep 11, 2020
1 parent d77695a commit 2b59511
Show file tree
Hide file tree
Showing 7 changed files with 174 additions and 129 deletions.
32 changes: 16 additions & 16 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ jobs:
kubectl -n helm-system apply -f config/testdata/$test_name
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="False" and .Ready=="False"' )" ]; do
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
Expand All @@ -107,7 +107,7 @@ jobs:
kubectl -n helm-system apply -f config/testdata/$test_name
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Tested=="False" and .Ready=="False"' )" ]; do
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .TestSuccess=="False" and .Ready=="False"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
Expand All @@ -130,7 +130,7 @@ jobs:
kubectl -n helm-system apply -f config/testdata/$test_name
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Tested=="False" and .Ready=="True"' )" ]; do
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .TestSuccess=="False" and .Ready=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
Expand All @@ -153,7 +153,7 @@ jobs:
kubectl -n helm-system apply -f config/testdata/$test_name
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="False" and .Ready=="False" and .Uninstalled=="True"' )" ]; do
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" and .Remediated=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
Expand All @@ -176,7 +176,7 @@ jobs:
kubectl -n helm-system apply -f config/testdata/$test_name
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.installFailures == 2 and ( .status.conditions | map( { (.type): .status } ) | add | .Installed=="False" and .Ready=="False" )' )" ]; do
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.installFailures == 2 and ( .status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" )' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
Expand All @@ -199,7 +199,7 @@ jobs:
kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Ready=="True"' )" ]; do
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
Expand All @@ -220,7 +220,7 @@ jobs:
kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Upgraded=="False" and .Ready=="False"' )" ]; do
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
Expand All @@ -243,7 +243,7 @@ jobs:
kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Ready=="True"' )" ]; do
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
Expand All @@ -264,7 +264,7 @@ jobs:
kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Upgraded=="True" and .Tested=="False" and .Ready=="False"' )" ]; do
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .TestSuccess=="False" and .Ready=="False"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
Expand All @@ -287,7 +287,7 @@ jobs:
kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Ready=="True"' )" ]; do
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
Expand All @@ -308,7 +308,7 @@ jobs:
kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Upgraded=="False" and .Ready=="False" and .RolledBack=="True"' )" ]; do
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" and .Remediated=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
Expand Down Expand Up @@ -336,7 +336,7 @@ jobs:
kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Ready=="True"' )" ]; do
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
Expand All @@ -357,7 +357,7 @@ jobs:
kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.upgradeFailures == 2 and ( .status.conditions | map( { (.type): .status } ) | add | .Upgraded=="False" and .Ready=="False" and .RolledBack=="True" )' )" ]; do
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.upgradeFailures == 2 and ( .status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" )' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
Expand Down Expand Up @@ -385,7 +385,7 @@ jobs:
kubectl -n helm-system apply -f config/testdata/$test_name/install.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Installed=="True" and .Ready=="True"' )" ]; do
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .Ready=="True"' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
Expand All @@ -412,7 +412,7 @@ jobs:
kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
echo -n ">>> Waiting for expected conditions"
count=0
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.upgradeFailures == 1 and .status.installFailures == 1 and ( .status.conditions | map( { (.type): .status } ) | add | .Upgraded=="False" and .Uninstalled=="True" and .Installed=="False" and .Ready=="False" )' )" ]; do
until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.upgradeFailures == 1 and .status.installFailures == 1 and ( .status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" )' )" ]; do
echo -n '.'
sleep 5
count=$((count + 1))
Expand Down Expand Up @@ -449,6 +449,6 @@ jobs:
kubectl -n helm-system get helmcharts -oyaml
kubectl -n helm-system get helmreleases -oyaml
kubectl -n helm-system get all
helm ls -n helm-system
helm ls -n helm-system --all
kubectl -n helm-system logs deploy/source-controller
kubectl -n helm-system logs deploy/helm-controller
24 changes: 9 additions & 15 deletions api/v2alpha1/condition_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,20 +51,14 @@ const (
// ReadyCondition represents the fact that the HelmRelease has been successfully reconciled.
ReadyCondition string = "Ready"

// InstalledCondition represents the fact that the HelmRelease has been successfully installed.
InstalledCondition string = "Installed"
// ReleasedCondition represents the fact that the HelmRelease has been successfully released.
ReleasedCondition string = "Released"

// UpgradedCondition represents the fact that the HelmRelease has been successfully upgraded.
UpgradedCondition string = "Upgraded"
// TestSuccessCondition represents the fact that the tests for the HelmRelease are succeeding.
TestSuccessCondition string = "TestSuccess"

// TestedCondition represents the fact that the HelmRelease has been successfully tested.
TestedCondition string = "Tested"

// RolledBackCondition represents the fact that the HelmRelease has been successfully rolled back.
RolledBackCondition string = "RolledBack"

// UninstalledCondition represents the fact that the HelmRelease has been successfully uninstalled.
UninstalledCondition string = "Uninstalled"
// RemediatedCondition represents the fact that the HelmRelease has been successfully remediated.
RemediatedCondition string = "Remediated"
)

const (
Expand All @@ -86,11 +80,11 @@ const (
// UpgradeFailedReason represents the fact that the Helm upgrade for the HelmRelease failed.
UpgradeFailedReason string = "UpgradeFailed"

// TestSucceededReason represents the fact that the Helm test for the HelmRelease succeeded.
// TestSucceededReason represents the fact that the Helm tests for the HelmRelease succeeded.
TestSucceededReason string = "TestSucceeded"

// TestFailedReason represents the fact that the Helm test for the HelmRelease failed.
TestFailedReason string = "TestFailed"
// TestsFailedReason represents the fact that the Helm tests for the HelmRelease failed.
TestsFailedReason string = "TestsFailed"

// RollbackSucceededReason represents the fact that the Helm rollback for the HelmRelease succeeded.
RollbackSucceededReason string = "RollbackSucceeded"
Expand Down
28 changes: 20 additions & 8 deletions api/v2alpha1/helmrelease_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -584,10 +584,6 @@ type HelmReleaseStatus struct {
// +optional
ObservedGeneration int64 `json:"observedGeneration,omitempty"`

// ObservedStateReconciled represents whether the observed state has been successfully reconciled.
// +optional
ObservedStateReconciled bool `json:"observedStateReconciled,omitempty"`

// LastObservedTime is the last time at which the HelmRelease was observed.
// +optional
LastObservedTime metav1.Time `json:"lastObservedTime,omitempty"`
Expand Down Expand Up @@ -646,23 +642,21 @@ func (in HelmReleaseStatus) GetHelmChart() (string, string) {
// by setting the ReadyCondition to ConditionUnknown for ProgressingReason.
func HelmReleaseProgressing(hr HelmRelease) HelmRelease {
resetFailureCounts(&hr)
hr.Status.ObservedStateReconciled = false
hr.Status.Conditions = []Condition{}
SetHelmReleaseCondition(&hr, ReadyCondition, corev1.ConditionUnknown, ProgressingReason, "reconciliation in progress")
return hr
}

// HelmReleaseNotReady registers a failed release attempt of the given HelmRelease.
// HelmReleaseNotReady registers a failed reconciliation of the given HelmRelease.
func HelmReleaseNotReady(hr HelmRelease, reason, message string) HelmRelease {
SetHelmReleaseCondition(&hr, ReadyCondition, corev1.ConditionFalse, reason, message)
hr.Status.Failures++
return hr
}

// HelmReleaseReady registers a successful release attempt of the given HelmRelease.
// HelmReleaseReady registers a successful reconciliation of the given HelmRelease.
func HelmReleaseReady(hr HelmRelease) HelmRelease {
resetFailureCounts(&hr)
hr.Status.ObservedStateReconciled = true
hr.Status.LastAppliedRevision = hr.Status.LastAttemptedRevision
SetHelmReleaseCondition(&hr, ReadyCondition, corev1.ConditionTrue, ReconciliationSucceededReason, "release reconciliation succeeded")
return hr
Expand All @@ -687,6 +681,18 @@ func resetFailureCounts(hr *HelmRelease) {
hr.Status.UpgradeFailures = 0
}

// GetHelmReleaseCondition return the given condition of the given HelmRelease
// or nil if not present.
func GetHelmReleaseCondition(hr HelmRelease, condition string) *Condition {
for _, c := range hr.Status.Conditions {
if c.Type == condition {
return &c
}
}

return nil
}

// SetHelmReleaseCondition sets the given condition with the given status, reason and message
// on the HelmRelease.
func SetHelmReleaseCondition(hr *HelmRelease, condition string, status corev1.ConditionStatus, reason, message string) {
Expand All @@ -700,6 +706,12 @@ func SetHelmReleaseCondition(hr *HelmRelease, condition string, status corev1.Co
})
}

// DeleteHelmReleaseCondition deletes the given condition of the given HelmRelease
// if present.
func DeleteHelmReleaseCondition(hr *HelmRelease, condition string) {
hr.Status.Conditions = filterOutCondition(hr.Status.Conditions, condition)
}

const (
// SourceIndexKey is the key used for indexing HelmReleases based on
// their sources.
Expand Down
4 changes: 0 additions & 4 deletions config/crd/bases/helm.toolkit.fluxcd.io_helmreleases.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -436,10 +436,6 @@ spec:
description: ObservedGeneration is the last observed generation.
format: int64
type: integer
observedStateReconciled:
description: ObservedStateReconciled represents whether the observed
state has been successfully reconciled.
type: boolean
upgradeFailures:
description: UpgradeFailures is the upgrade failure count against
the latest observed state. It is reset after a successful reconciliation.
Expand Down
Loading

0 comments on commit 2b59511

Please sign in to comment.