Skip to content

Commit

Permalink
Merge pull request #205 from fluxcd/filter-apply-errors
Browse files Browse the repository at this point in the history
Refactor apply error reporting
  • Loading branch information
stefanprodan authored Dec 14, 2020
2 parents 4da53d1 + d65ea71 commit 45c0809
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 37 deletions.
10 changes: 5 additions & 5 deletions api/v1beta1/kustomization_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
const (
KustomizationKind = "Kustomization"
KustomizationFinalizer = "finalizers.fluxcd.io"
MaxConditionMessageLength = 4000
MaxConditionMessageLength = 20000
)

// KustomizationSpec defines the desired state of a kustomization.
Expand Down Expand Up @@ -181,14 +181,14 @@ func KustomizationProgressing(k Kustomization) Kustomization {
// SetKustomizeReadiness sets the ReadyCondition, ObservedGeneration, and LastAttemptedRevision,
// on the Kustomization.
func SetKustomizationReadiness(k *Kustomization, status metav1.ConditionStatus, reason, message string, revision string) {
meta.SetResourceCondition(k, meta.ReadyCondition, status, reason, message)
meta.SetResourceCondition(k, meta.ReadyCondition, status, reason, trimString(message, MaxConditionMessageLength))
k.Status.ObservedGeneration = k.Generation
k.Status.LastAttemptedRevision = revision
}

// KustomizationNotReady registers a failed apply attempt of the given Kustomization.
func KustomizationNotReady(k Kustomization, revision, reason, message string) Kustomization {
SetKustomizationReadiness(&k, metav1.ConditionFalse, reason, message, revision)
SetKustomizationReadiness(&k, metav1.ConditionFalse, reason, trimString(message, MaxConditionMessageLength), revision)
if revision != "" {
k.Status.LastAttemptedRevision = revision
}
Expand All @@ -198,15 +198,15 @@ func KustomizationNotReady(k Kustomization, revision, reason, message string) Ku
// KustomizationNotReady registers a failed apply attempt of the given Kustomization,
// including a Snapshot.
func KustomizationNotReadySnapshot(k Kustomization, snapshot *Snapshot, revision, reason, message string) Kustomization {
SetKustomizationReadiness(&k, metav1.ConditionFalse, reason, message, revision)
SetKustomizationReadiness(&k, metav1.ConditionFalse, reason, trimString(message, MaxConditionMessageLength), revision)
k.Status.Snapshot = snapshot
k.Status.LastAttemptedRevision = revision
return k
}

// KustomizationReady registers a successful apply attempt of the given Kustomization.
func KustomizationReady(k Kustomization, snapshot *Snapshot, revision, reason, message string) Kustomization {
SetKustomizationReadiness(&k, metav1.ConditionTrue, reason, message, revision)
SetKustomizationReadiness(&k, metav1.ConditionTrue, reason, trimString(message, MaxConditionMessageLength), revision)
k.Status.Snapshot = snapshot
k.Status.LastAppliedRevision = revision
return k
Expand Down
39 changes: 7 additions & 32 deletions controllers/kustomization_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -711,10 +711,15 @@ func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization,
if errors.Is(err, context.DeadlineExceeded) {
return "", fmt.Errorf("apply timeout: %w", err)
}
return "", fmt.Errorf("apply failed: %s", string(output))

if string(output) == "" {
return "", fmt.Errorf("apply failed: %w, kubectl process was killed, probably due to OOM", err)
}

return "", fmt.Errorf("apply failed: %s", parseApplyError(output))
}

resources := r.parseApplyOutput(output)
resources := parseApplyOutput(output)
r.Log.WithValues(
strings.ToLower(kustomization.Kind),
fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
Expand Down Expand Up @@ -805,27 +810,6 @@ func (r *KustomizationReconciler) checkHealth(statusPoller *polling.StatusPoller
return nil
}

func (r *KustomizationReconciler) parseApplyOutput(in []byte) map[string]string {
result := make(map[string]string)
input := strings.Split(string(in), "\n")
if len(input) == 0 {
return result
}
var parts []string
for _, str := range input {
if str != "" {
parts = append(parts, str)
}
}
for _, str := range parts {
kv := strings.Split(str, " ")
if len(kv) > 1 {
result[kv[0]] = kv[1]
}
}
return result
}

func (r *KustomizationReconciler) checkDependencies(kustomization kustomizev1.Kustomization) error {
for _, d := range kustomization.Spec.DependsOn {
if d.Namespace == "" {
Expand Down Expand Up @@ -1041,12 +1025,3 @@ func (r *KustomizationReconciler) updateStatus(ctx context.Context, req ctrl.Req

return r.Status().Patch(ctx, &kustomization, patch)
}

func containsString(slice []string, s string) bool {
for _, item := range slice {
if item == s {
return true
}
}
return false
}
71 changes: 71 additions & 0 deletions controllers/utils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
/*
Copyright 2020 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import "strings"

// parseApplyOutput extracts the objects and the action
// performed by kubectl e.g.:
// service/backend created
// service/frontend configured
// service/database unchanged
func parseApplyOutput(in []byte) map[string]string {
result := make(map[string]string)
input := strings.Split(string(in), "\n")
if len(input) == 0 {
return result
}
var parts []string
for _, str := range input {
if str != "" {
parts = append(parts, str)
}
}
for _, str := range parts {
kv := strings.Split(str, " ")
if len(kv) > 1 {
result[kv[0]] = kv[1]
}
}
return result
}

// parseApplyError extracts the errors from the kubectl
// apply output by removing the successfully applied objects
func parseApplyError(in []byte) string {
errors := ""
lines := strings.Split(string(in), "\n")
for _, line := range lines {
if line != "" &&
!strings.HasSuffix(line, "created") &&
!strings.HasSuffix(line, "configured") &&
!strings.HasSuffix(line, "unchanged") {
errors += line + "\n"
}
}

return errors
}

func containsString(slice []string, s string) bool {
for _, item := range slice {
if item == s {
return true
}
}
return false
}

0 comments on commit 45c0809

Please sign in to comment.