diff --git a/api/v1beta1/kustomization_types.go b/api/v1beta1/kustomization_types.go index 6710dfbd..4d955ee7 100644 --- a/api/v1beta1/kustomization_types.go +++ b/api/v1beta1/kustomization_types.go @@ -30,7 +30,7 @@ import ( const ( KustomizationKind = "Kustomization" KustomizationFinalizer = "finalizers.fluxcd.io" - MaxConditionMessageLength = 4000 + MaxConditionMessageLength = 20000 ) // KustomizationSpec defines the desired state of a kustomization. @@ -181,14 +181,14 @@ func KustomizationProgressing(k Kustomization) Kustomization { // SetKustomizeReadiness sets the ReadyCondition, ObservedGeneration, and LastAttemptedRevision, // on the Kustomization. func SetKustomizationReadiness(k *Kustomization, status metav1.ConditionStatus, reason, message string, revision string) { - meta.SetResourceCondition(k, meta.ReadyCondition, status, reason, message) + meta.SetResourceCondition(k, meta.ReadyCondition, status, reason, trimString(message, MaxConditionMessageLength)) k.Status.ObservedGeneration = k.Generation k.Status.LastAttemptedRevision = revision } // KustomizationNotReady registers a failed apply attempt of the given Kustomization. func KustomizationNotReady(k Kustomization, revision, reason, message string) Kustomization { - SetKustomizationReadiness(&k, metav1.ConditionFalse, reason, message, revision) + SetKustomizationReadiness(&k, metav1.ConditionFalse, reason, trimString(message, MaxConditionMessageLength), revision) if revision != "" { k.Status.LastAttemptedRevision = revision } @@ -198,7 +198,7 @@ func KustomizationNotReady(k Kustomization, revision, reason, message string) Ku // KustomizationNotReady registers a failed apply attempt of the given Kustomization, // including a Snapshot. func KustomizationNotReadySnapshot(k Kustomization, snapshot *Snapshot, revision, reason, message string) Kustomization { - SetKustomizationReadiness(&k, metav1.ConditionFalse, reason, message, revision) + SetKustomizationReadiness(&k, metav1.ConditionFalse, reason, trimString(message, MaxConditionMessageLength), revision) k.Status.Snapshot = snapshot k.Status.LastAttemptedRevision = revision return k @@ -206,7 +206,7 @@ func KustomizationNotReadySnapshot(k Kustomization, snapshot *Snapshot, revision // KustomizationReady registers a successful apply attempt of the given Kustomization. func KustomizationReady(k Kustomization, snapshot *Snapshot, revision, reason, message string) Kustomization { - SetKustomizationReadiness(&k, metav1.ConditionTrue, reason, message, revision) + SetKustomizationReadiness(&k, metav1.ConditionTrue, reason, trimString(message, MaxConditionMessageLength), revision) k.Status.Snapshot = snapshot k.Status.LastAppliedRevision = revision return k diff --git a/controllers/kustomization_controller.go b/controllers/kustomization_controller.go index 05ce4ff7..a2c4ad05 100644 --- a/controllers/kustomization_controller.go +++ b/controllers/kustomization_controller.go @@ -711,10 +711,15 @@ func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization, if errors.Is(err, context.DeadlineExceeded) { return "", fmt.Errorf("apply timeout: %w", err) } - return "", fmt.Errorf("apply failed: %s", string(output)) + + if string(output) == "" { + return "", fmt.Errorf("apply failed: %w, kubectl process was killed, probably due to OOM", err) + } + + return "", fmt.Errorf("apply failed: %s", parseApplyError(output)) } - resources := r.parseApplyOutput(output) + resources := parseApplyOutput(output) r.Log.WithValues( strings.ToLower(kustomization.Kind), fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()), @@ -805,27 +810,6 @@ func (r *KustomizationReconciler) checkHealth(statusPoller *polling.StatusPoller return nil } -func (r *KustomizationReconciler) parseApplyOutput(in []byte) map[string]string { - result := make(map[string]string) - input := strings.Split(string(in), "\n") - if len(input) == 0 { - return result - } - var parts []string - for _, str := range input { - if str != "" { - parts = append(parts, str) - } - } - for _, str := range parts { - kv := strings.Split(str, " ") - if len(kv) > 1 { - result[kv[0]] = kv[1] - } - } - return result -} - func (r *KustomizationReconciler) checkDependencies(kustomization kustomizev1.Kustomization) error { for _, d := range kustomization.Spec.DependsOn { if d.Namespace == "" { @@ -1041,12 +1025,3 @@ func (r *KustomizationReconciler) updateStatus(ctx context.Context, req ctrl.Req return r.Status().Patch(ctx, &kustomization, patch) } - -func containsString(slice []string, s string) bool { - for _, item := range slice { - if item == s { - return true - } - } - return false -} diff --git a/controllers/utils.go b/controllers/utils.go new file mode 100644 index 00000000..fbcac1f2 --- /dev/null +++ b/controllers/utils.go @@ -0,0 +1,71 @@ +/* +Copyright 2020 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import "strings" + +// parseApplyOutput extracts the objects and the action +// performed by kubectl e.g.: +// service/backend created +// service/frontend configured +// service/database unchanged +func parseApplyOutput(in []byte) map[string]string { + result := make(map[string]string) + input := strings.Split(string(in), "\n") + if len(input) == 0 { + return result + } + var parts []string + for _, str := range input { + if str != "" { + parts = append(parts, str) + } + } + for _, str := range parts { + kv := strings.Split(str, " ") + if len(kv) > 1 { + result[kv[0]] = kv[1] + } + } + return result +} + +// parseApplyError extracts the errors from the kubectl +// apply output by removing the successfully applied objects +func parseApplyError(in []byte) string { + errors := "" + lines := strings.Split(string(in), "\n") + for _, line := range lines { + if line != "" && + !strings.HasSuffix(line, "created") && + !strings.HasSuffix(line, "configured") && + !strings.HasSuffix(line, "unchanged") { + errors += line + "\n" + } + } + + return errors +} + +func containsString(slice []string, s string) bool { + for _, item := range slice { + if item == s { + return true + } + } + return false +}