Refactor watcher and kyma reconcilers to improve load test performance

.gitignore

@@ -21,7 +21,8 @@
 atlassian-ide-plugin.xml
 
 ## VSCode
-.vscode
+**/*.vscode
+**/*.devcontainer*
 
 ### OSX template
 *.DS_Store
@@ -90,5 +91,8 @@ out.txt
 **/go.work
 **/go.work.sum
 
+# ignore .pem key files
+**/*.pem
+
 # rendered manifest
 **/manifest.yaml

operator/Dockerfile

@@ -23,7 +23,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
 COPY --from=builder /workspace/manager .
-COPY --from=builder /workspace/internal/charts/skr-webhook ./skr-webhook
+COPY charts/skr-webhook charts/skr-webhook
 USER 65532:65532
 
 ENTRYPOINT ["/manager"]

operator/api/v1alpha1/condition_builder.go

@@ -17,6 +17,15 @@ const (
 	MessageModuleNotInReadyState    = "not all modules are in ready state"
 	MessageModuleCatalogIsSynced    = "module catalog is synchronized"
 	MessageModuleCatalogIsOutOfSync = "module catalog is out of sync and needs to be resynchronized"
+	MessageSKRWebhookIsSynced       = "skrwebhook is synchronized"
+	MessageSKRWebhookIsOutOfSync    = "skrwebhook is out of sync and needs to be resynchronized"
+)
+
+// Extend this list by actual needs.
+const (
+	ConditionReasonModulesAreReady      KymaConditionReason = "ModulesAreReady"
+	ConditionReasonModuleCatalogIsReady KymaConditionReason = "ModuleCatalogIsReady"
+	ConditionReasonSKRWebhookIsReady    KymaConditionReason = "SKRWebhookIsReady"
 )
 
 func NewConditionBuilder() *ConditionBuilder {
@@ -69,6 +78,15 @@ func (cb *ConditionBuilder) generateMessage() string {
 		}
 
 		return MessageModuleCatalogIsOutOfSync
+	case ConditionReasonSKRWebhookIsReady:
+		switch cb.Status {
+		case metav1.ConditionTrue:
+			return MessageSKRWebhookIsSynced
+		case metav1.ConditionUnknown:
+		case metav1.ConditionFalse:
+		}
+
+		return MessageSKRWebhookIsOutOfSync
 	}
 
 	return "no detailed message available as reason is unknown to API"

operator/api/v1alpha1/kyma_types.go

@@ -249,12 +249,6 @@ const (
 // and the actual state of individual module can be found in related ModuleStatus.
 type KymaConditionReason string
 
-// Extend this list by actual needs.
-const (
-	ConditionReasonModulesAreReady      KymaConditionReason = "ModulesAreReady"
-	ConditionReasonModuleCatalogIsReady KymaConditionReason = "ModuleCatalogIsReady"
-)
-
 //+genclient
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status

operator/api/v1alpha1/watcher_types.go

@@ -17,8 +17,6 @@ limitations under the License.
 package v1alpha1
 
 import (
-	"time"
-
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -143,8 +141,6 @@ const (
 )
 
 //+kubebuilder:object:root=true
-//+kubebuilder:subresource:status
-//+kubebuilder:printcolumn:name="State",type=string,JSONPath=".status.state"
 //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
 
 // Watcher is the Schema for the watchers API.
@@ -156,14 +152,6 @@ type Watcher struct {
 
 	// +kubebuilder:validation:Optional
 	Spec WatcherSpec `json:"spec"`
-
-	// +kubebuilder:validation:Optional
-	Status WatcherStatus `json:"status"`
-}
-
-func (w *Watcher) SetObservedGeneration() *Watcher {
-	w.Status.ObservedGeneration = w.Generation
-	return w
 }
 
 func (w *Watcher) GetModuleName() string {
@@ -173,25 +161,6 @@ func (w *Watcher) GetModuleName() string {
 	return w.Labels[ManagedBylabel]
 }
 
-func (w *Watcher) AddOrUpdateReadyCondition(state WatcherConditionStatus, msg string) {
-	lastTransitionTime := &metav1.Time{Time: time.Now()}
-	if len(w.Status.Conditions) == 0 {
-		w.Status.Conditions = []WatcherCondition{{
-			Type:               WatcherConditionTypeReady,
-			Status:             state,
-			Message:            msg,
-			LastTransitionTime: lastTransitionTime,
-		}}
-	}
-	for i := range w.Status.Conditions {
-		condition := &w.Status.Conditions[i]
-		if condition.Type == WatcherConditionTypeReady {
-			condition.Status = state
-			condition.LastTransitionTime = lastTransitionTime
-		}
-	}
-}
-
 //+kubebuilder:object:root=true
 
 // WatcherList contains a list of Watcher.

operator/internal/charts/skr-webhook/templates/deployment.yaml → operator/charts/skr-webhook/templates/deployment.yaml

@@ -30,6 +30,7 @@ spec:
     metadata:
       labels:
         app: skr-webhook
+        operator.kyma-project.io/pod-restart-trigger: "{{.Values.triggerLabel}}"
     spec:
       serviceAccountName: {{.Release.Name}}-webhook-sa
       containers:

operator/internal/charts/skr-webhook/templates/secret.yaml → operator/charts/skr-webhook/templates/secret.yaml

@@ -6,19 +6,13 @@ type: Opaque
 data:
   {{- $caCert := "" }}
   {{- if .Values.tls.helmCertGen }}
-  {{- $data := (lookup "v1" "Secret" .Release.Namespace (printf "%s-webhook-tls" .Release.Name )).data }}
-  {{- if $data }}
-  {{ $data | toYaml | nindent 2 }}
-  {{- $caCert = index $data "ca.crt" }}
-  {{- else }}
   {{- $cn := printf "%s-webhook.%s.svc" .Release.Name .Release.Namespace }}
   {{- $ca := genCA (printf "%s-webhook-ca" .Release.Name  ) 36500 }}
   {{- $cert := genSignedCert $cn (list "127.0.0.1") (list $cn "localhost") 36500 $ca }}
   ca.crt: {{ $ca.Cert | b64enc }}
   tls.crt: {{ $cert.Cert | b64enc }}
   tls.key: {{ $cert.Key | b64enc }}
   {{- $caCert = $ca.Cert | b64enc }}
-  {{- end }}
   {{- else }}
   ca.crt: {{.Values.tls.caCert}}
   tls.crt: {{.Values.tls.clientCert }}

operator/internal/charts/skr-webhook/values.yaml → operator/charts/skr-webhook/values.yaml

@@ -2,6 +2,8 @@
 sidecar: true
 # -- Replicas
 replicas: 1
+# -- Trigger pod restart
+triggerLabel: ""
 image: 
   # -- Image repository
   repository: "eu.gcr.io/kyma-project"

operator/config/crd/bases/operator.kyma-project.io_watchers.yaml

@@ -16,9 +16,6 @@ spec:
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
-    - jsonPath: .status.state
-      name: State
-      type: string
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
@@ -78,67 +75,7 @@ spec:
             - labelsToWatch
             - serviceInfo
             type: object
-          status:
-            description: WatcherStatus defines the observed state of Watcher.
-            properties:
-              conditions:
-                description: List of status conditions to indicate the status of a
-                  Watcher.
-                items:
-                  description: WatcherCondition describes condition information for
-                    Watcher.
-                  properties:
-                    lastTransitionTime:
-                      description: Timestamp for when Watcher last transitioned from
-                        one status to another.
-                      format: date-time
-                      type: string
-                    message:
-                      description: Human-readable message indicating details about
-                        the last status transition.
-                      type: string
-                    reason:
-                      description: Machine-readable text indicating the reason for
-                        the condition's last transition.
-                      type: string
-                    status:
-                      description: Status of the Watcher Condition. Value can be one
-                        of ("True", "False", "Unknown").
-                      enum:
-                      - "True"
-                      - "False"
-                      - Unknown
-                      type: string
-                    type:
-                      description: Type is used to reflect what type of condition
-                        we are dealing with. Most commonly WatcherConditionTypeReady
-                        it is used as extension marker in the future.
-                      enum:
-                      - Ready
-                      type: string
-                  required:
-                  - status
-                  - type
-                  type: object
-                type: array
-              observedGeneration:
-                description: ObservedGeneration
-                format: int64
-                type: integer
-              state:
-                description: State signifies current state of a Watcher. Value can
-                  be one of ("Ready", "Processing", "Error", "Deleting")
-                enum:
-                - Processing
-                - Deleting
-                - Ready
-                - Error
-                type: string
-            required:
-            - state
-            type: object
         type: object
     served: true
     storage: true
-    subresources:
-      status: {}
+    subresources: {}

operator/config/watcher/kustomization.yaml

@@ -11,7 +11,7 @@ patches:
   - patch: |-
       - op: add
         path: /spec/template/spec/containers/0/args/-
-        value: --enable-kcp-watcher=false
+        value: --enable-kcp-watcher
       - op: add
         path: /spec/template/spec/containers/0/args/-
         value: --skr-watcher-path=/skr-webhook      

operator/controllers/kyma_controller.go

@@ -18,9 +18,16 @@ package controllers
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
+	"github.com/kyma-project/lifecycle-manager/operator/internal/deploy"
+	"k8s.io/client-go/rest"
+
+	manifestV1alpha1 "github.com/kyma-project/module-manager/operator/api/v1alpha1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	"github.com/kyma-project/lifecycle-manager/operator/api/v1alpha1"
 	"github.com/kyma-project/lifecycle-manager/operator/pkg/adapter"
 	"github.com/kyma-project/lifecycle-manager/operator/pkg/catalog"
@@ -31,8 +38,6 @@ import (
 	"github.com/kyma-project/lifecycle-manager/operator/pkg/remote"
 	"github.com/kyma-project/lifecycle-manager/operator/pkg/signature"
 	"github.com/kyma-project/lifecycle-manager/operator/pkg/status"
-	manifestV1alpha1 "github.com/kyma-project/module-manager/operator/api/v1alpha1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -45,6 +50,8 @@ type EventErrorType string
 
 const ModuleReconciliationError EventErrorType = "ModuleReconciliationError"
 
+var ErrSkrChartConfigNotSet = errors.New("skr chart config is not set")
+
 type RequeueIntervals struct {
 	Success time.Duration
 }
@@ -56,6 +63,8 @@ type KymaReconciler struct {
 	RequeueIntervals
 	signature.VerificationSettings
 	RemoteClientCache *remote.ClientCache
+	deploy.SKRWebhookChartManager
+	KcpRestConfig *rest.Config
 }
 
 //nolint:lll
@@ -248,7 +257,14 @@ func (r *KymaReconciler) HandleProcessingState(ctx context.Context, kyma *v1alph
 		return r.UpdateStatusWithEventFromErr(ctx, kyma, v1alpha1.StateError,
 			fmt.Errorf("error while syncing conditions during deleting non exists modules: %w", err))
 	}
-
+	statusUpdateRequiredFromSKRWebhookSync := false
+	if kyma.Spec.Sync.Enabled {
+		if statusUpdateRequiredFromSKRWebhookSync, err = r.InstallWebhookChart(ctx, kyma,
+			r.RemoteClientCache, r.Client); err != nil {
+			kyma.UpdateCondition(v1alpha1.ConditionReasonSKRWebhookIsReady, metav1.ConditionFalse)
+			return err
+		}
+	}
 	kyma.SyncConditionsWithModuleStates()
 	// set ready condition if applicable
 	if kyma.AreAllConditionsReadyForKyma() && kyma.Status.State != v1alpha1.StateReady {
@@ -258,10 +274,10 @@ func (r *KymaReconciler) HandleProcessingState(ctx context.Context, kyma *v1alph
 		return r.UpdateStatusWithEvent(ctx, kyma, v1alpha1.StateReady, message)
 	}
 
+	isStatusUpdateRequired := statusUpdateRequiredFromModuleSync || statusUpdateRequiredFromModuleStatusSync ||
+		statusUpdateRequiredFromDeletion || statusUpdateRequiredFromSKRWebhookSync
 	// if the ready condition is not applicable, but we changed the conditions, we still need to issue an update
-	if statusUpdateRequiredFromModuleSync ||
-		statusUpdateRequiredFromModuleStatusSync ||
-		statusUpdateRequiredFromDeletion {
+	if isStatusUpdateRequired {
 		if err := r.UpdateStatusWithEvent(ctx, kyma, v1alpha1.StateProcessing, "updating component conditions"); err != nil {
 			return fmt.Errorf("error while updating status for condition change: %w", err)
 		}
@@ -279,6 +295,9 @@ func (r *KymaReconciler) HandleDeletingState(ctx context.Context, kyma *v1alpha1
 		if err != nil {
 			return false, fmt.Errorf("remote sync initialization failed: %w", err)
 		}
+		if err = r.RemoveWebhookChart(ctx, kyma, syncContext); err != nil {
+			return true, nil
+		}
 		force := true
 		if err := catalog.NewRemoteCatalog(
 			syncContext, catalog.Settings{