Skip to content

Commit

Permalink
Refactor apply error reporting
Browse files Browse the repository at this point in the history
- filter kubectl apply output and extract errors
- limit apply output to 20K charts (avoid reaching max etcd size)
- log kubectl exit code when the process is killed

Signed-off-by: Stefan Prodan <[email protected]>
  • Loading branch information
stefanprodan committed Dec 14, 2020
1 parent 4da53d1 commit a95ddaf
Show file tree
Hide file tree
Showing 3 changed files with 80 additions and 37 deletions.
10 changes: 5 additions & 5 deletions api/v1beta1/kustomization_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
const (
KustomizationKind = "Kustomization"
KustomizationFinalizer = "finalizers.fluxcd.io"
MaxConditionMessageLength = 4000
MaxConditionMessageLength = 20000
)

// KustomizationSpec defines the desired state of a kustomization.
Expand Down Expand Up @@ -181,14 +181,14 @@ func KustomizationProgressing(k Kustomization) Kustomization {
// SetKustomizeReadiness sets the ReadyCondition, ObservedGeneration, and LastAttemptedRevision,
// on the Kustomization.
func SetKustomizationReadiness(k *Kustomization, status metav1.ConditionStatus, reason, message string, revision string) {
meta.SetResourceCondition(k, meta.ReadyCondition, status, reason, message)
meta.SetResourceCondition(k, meta.ReadyCondition, status, reason, trimString(message, MaxConditionMessageLength))
k.Status.ObservedGeneration = k.Generation
k.Status.LastAttemptedRevision = revision
}

// KustomizationNotReady registers a failed apply attempt of the given Kustomization.
func KustomizationNotReady(k Kustomization, revision, reason, message string) Kustomization {
SetKustomizationReadiness(&k, metav1.ConditionFalse, reason, message, revision)
SetKustomizationReadiness(&k, metav1.ConditionFalse, reason, trimString(message, MaxConditionMessageLength), revision)
if revision != "" {
k.Status.LastAttemptedRevision = revision
}
Expand All @@ -198,15 +198,15 @@ func KustomizationNotReady(k Kustomization, revision, reason, message string) Ku
// KustomizationNotReady registers a failed apply attempt of the given Kustomization,
// including a Snapshot.
func KustomizationNotReadySnapshot(k Kustomization, snapshot *Snapshot, revision, reason, message string) Kustomization {
SetKustomizationReadiness(&k, metav1.ConditionFalse, reason, message, revision)
SetKustomizationReadiness(&k, metav1.ConditionFalse, reason, trimString(message, MaxConditionMessageLength), revision)
k.Status.Snapshot = snapshot
k.Status.LastAttemptedRevision = revision
return k
}

// KustomizationReady registers a successful apply attempt of the given Kustomization.
func KustomizationReady(k Kustomization, snapshot *Snapshot, revision, reason, message string) Kustomization {
SetKustomizationReadiness(&k, metav1.ConditionTrue, reason, message, revision)
SetKustomizationReadiness(&k, metav1.ConditionTrue, reason, trimString(message, MaxConditionMessageLength), revision)
k.Status.Snapshot = snapshot
k.Status.LastAppliedRevision = revision
return k
Expand Down
39 changes: 7 additions & 32 deletions controllers/kustomization_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -711,10 +711,15 @@ func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization,
if errors.Is(err, context.DeadlineExceeded) {
return "", fmt.Errorf("apply timeout: %w", err)
}
return "", fmt.Errorf("apply failed: %s", string(output))

if string(output) == "" {
return "", fmt.Errorf("apply failed: %w, kubectl process was killed, probably due to OOM", err)
}

return "", fmt.Errorf("apply failed: %s", parseApplyError(output))
}

resources := r.parseApplyOutput(output)
resources := parseApplyOutput(output)
r.Log.WithValues(
strings.ToLower(kustomization.Kind),
fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
Expand Down Expand Up @@ -805,27 +810,6 @@ func (r *KustomizationReconciler) checkHealth(statusPoller *polling.StatusPoller
return nil
}

func (r *KustomizationReconciler) parseApplyOutput(in []byte) map[string]string {
result := make(map[string]string)
input := strings.Split(string(in), "\n")
if len(input) == 0 {
return result
}
var parts []string
for _, str := range input {
if str != "" {
parts = append(parts, str)
}
}
for _, str := range parts {
kv := strings.Split(str, " ")
if len(kv) > 1 {
result[kv[0]] = kv[1]
}
}
return result
}

func (r *KustomizationReconciler) checkDependencies(kustomization kustomizev1.Kustomization) error {
for _, d := range kustomization.Spec.DependsOn {
if d.Namespace == "" {
Expand Down Expand Up @@ -1041,12 +1025,3 @@ func (r *KustomizationReconciler) updateStatus(ctx context.Context, req ctrl.Req

return r.Status().Patch(ctx, &kustomization, patch)
}

func containsString(slice []string, s string) bool {
for _, item := range slice {
if item == s {
return true
}
}
return false
}
68 changes: 68 additions & 0 deletions controllers/utils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/*
Copyright 2020 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import "strings"

// parseApplyOutput extracts the objects and the action
// performed by kubectl (created, configured, unchanged)
func parseApplyOutput(in []byte) map[string]string {
result := make(map[string]string)
input := strings.Split(string(in), "\n")
if len(input) == 0 {
return result
}
var parts []string
for _, str := range input {
if str != "" {
parts = append(parts, str)
}
}
for _, str := range parts {
kv := strings.Split(str, " ")
if len(kv) > 1 {
result[kv[0]] = kv[1]
}
}
return result
}

// parseApplyError extracts the errors from the kubectl
// apply output by removing the successfully applied objects
func parseApplyError(in []byte) string {
errors := ""
lines := strings.Split(string(in), "\n")
for _, line := range lines {
if line != "" &&
!strings.HasSuffix(line, "created") &&
!strings.HasSuffix(line, "configured") &&
!strings.HasSuffix(line, "unchanged") {
errors += line + "\n"
}
}

return errors
}

func containsString(slice []string, s string) bool {
for _, item := range slice {
if item == s {
return true
}
}
return false
}

0 comments on commit a95ddaf

Please sign in to comment.