add operator e2e suites

Signed-off-by: zhzhuang-zju <[email protected]>

.github/workflows/ci.yml

@@ -161,3 +161,59 @@ jobs:
         with:
           name: karmada_kind_log_${{ matrix.k8s }}
           path: /tmp/karmada/
+
+  e2e-operator:
+    name: operator e2e test
+    needs: build
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        # Here support the latest three minor releases of Kubernetes, this can be considered to be roughly
+        # the same as the End of Life of the Kubernetes release: https://kubernetes.io/releases/
+        # Please remember to update the CI Schedule Workflow when we add a new version.
+        k8s: [ v1.29.0, v1.30.0, v1.31.0 ]
+    steps:
+      # Free up disk space on Ubuntu
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed, if set to "true" but frees about 6 GB
+          tool-cache: false
+          # all of these default to true, but feel free to set to "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          docker-images: false
+          swap-storage: false
+      - name: checkout code
+        uses: actions/checkout@v4
+        with:
+          # Number of commits to fetch. 0 indicates all history for all branches and tags.
+          # We need to guess version via git tags.
+          fetch-depth: 0
+      - name: install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: setup e2e test environment
+        run: |
+          export CLUSTER_VERSION=kindest/node:${{ matrix.k8s }}
+          hack/local-up-karmada-by-operator.sh
+      - name: run e2e
+        run: |
+          export ARTIFACTS_PATH=${{ github.workspace }}/karmada-operator-e2e-logs/${{ matrix.k8s }}/
+          hack/run-e2e-operator.sh
+      - name: upload logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: karmada-operator-e2e-log_${{ matrix.k8s }}
+          path: ${{ github.workspace }}/karmada-operator-e2e-logs/${{ matrix.k8s }}/
+      - name: upload kind logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: karmada_operator_kind_log_${{ matrix.k8s }}
+          path: /tmp/karmada/

hack/run-e2e-operator.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Copyright 2025 The Karmada Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# This script runs e2e test against on karmada control plane.
+# You should prepare your environment in advance and following environment may be you need to set or use default one.
+# - CONTROL_PLANE_KUBECONFIG: absolute path of control plane KUBECONFIG file.
+#
+# Usage: hack/run-e2e.sh
+# Example 1: hack/run-e2e.sh (run e2e with default config)
+# Example 2: export KARMADA_APISERVER_KUBECONFIG=<KUBECONFIG PATH> hack/run-e2e.sh (run e2e with your KUBECONFIG)
+
+KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"}
+KARMADA_APISERVER_KUBECONFIG=${KARMADA_APISERVER_KUBECONFIG:-"$KUBECONFIG_PATH/karmada.config"}
+
+# KARMADA_RUNNING_ON_KIND indicates if current testing against on karmada that installed on a kind cluster.
+# Defaults to true.
+# For kind cluster, the kind related logs will be collected after the testing.
+KARMADA_RUNNING_ON_KIND=${KARMADA_RUNNING_ON_KIND:-true}
+
+KARMADA_HOST_CLUSTER_NAME=${KARMADA_HOST_CLUSTER_NAME:-"karmada-host"}
+
+ARTIFACTS_PATH=${ARTIFACTS_PATH:-"${HOME}/karmada-operator-e2e-logs"}
+mkdir -p "$ARTIFACTS_PATH"
+
+# Install ginkgo
+GO111MODULE=on go install github.com/onsi/ginkgo/v2/ginkgo
+
+# Run e2e
+export KUBECONFIG=${KARMADA_APISERVER_KUBECONFIG}
+
+set +e
+ginkgo -v --race --trace --fail-fast -p --randomize-all ./test/e2e/suites/operator
+TESTING_RESULT=$?
+
+# Collect logs
+echo "Collect logs to $ARTIFACTS_PATH..."
+cp "$KARMADA_APISERVER_KUBECONFIG" "$ARTIFACTS_PATH"
+
+if [ "$KARMADA_RUNNING_ON_KIND" = true ]; then
+  echo "Collecting $KARMADA_HOST_CLUSTER_NAME logs..."
+  mkdir -p "$ARTIFACTS_PATH/$KARMADA_HOST_CLUSTER_NAME"
+  kind export logs --name="$KARMADA_HOST_CLUSTER_NAME" "$ARTIFACTS_PATH/$KARMADA_HOST_CLUSTER_NAME"
+fi
+
+echo "Collected logs at $ARTIFACTS_PATH:"
+ls -al "$ARTIFACTS_PATH"
+
+exit $TESTING_RESULT

test/e2e/framework/karmada.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2025 The Karmada Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog/v2"
+
+	operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
+	operator "github.com/karmada-io/karmada/operator/pkg/generated/clientset/versioned"
+)
+
+// WaitKarmadaReady wait karmada instance ready until timeout.
+// Since the karmada-operator updates the `karmada.spec` first and then the `karmada.status`, in order to ensure that the `ready` condition indicates
+// that the `karmada.spec` has been applied correctly, it will check the `lastTransitionTime` of the `ready` condition.
+func WaitKarmadaReady(client operator.Interface, namespace, name string, lastTransitionTime time.Time) {
+	klog.Infof("Waiting for karmada instance %s/%s ready", namespace, name)
+	gomega.Eventually(func(g gomega.Gomega) bool {
+		karmada, err := client.OperatorV1alpha1().Karmadas(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+		g.Expect(err).NotTo(gomega.HaveOccurred())
+		for _, condition := range karmada.Status.Conditions {
+			if condition.Type == "Ready" && condition.Status == "True" && condition.LastTransitionTime.After(lastTransitionTime) {
+				return true
+			}
+		}
+		return false
+	}, pollTimeout, pollInterval).Should(gomega.Equal(true))
+}
+
+// CreateKarmadaInstance creates a karmada instance.
+func CreateKarmadaInstance(operatorClient operator.Interface, karmada *operatorv1alpha1.Karmada) error {
+	_, err := operatorClient.OperatorV1alpha1().Karmadas(karmada.GetNamespace()).Create(context.TODO(), karmada, metav1.CreateOptions{})
+	if err != nil {
+		if apierrors.IsAlreadyExists(err) {
+			return nil
+		}
+
+		return err
+	}
+	return nil
+}
+
+// UpdateKarmadaInstanceWithSpec updates karmada instance with spec.
+func UpdateKarmadaInstanceWithSpec(client operator.Interface, namespace, name string, karmadaSpec operatorv1alpha1.KarmadaSpec) {
+	ginkgo.By(fmt.Sprintf("Updating Karmada(%s/%s) spec", namespace, name), func() {
+		karmada, err := client.OperatorV1alpha1().Karmadas(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+		gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+
+		karmada.Spec = karmadaSpec
+		_, err = client.OperatorV1alpha1().Karmadas(namespace).Update(context.TODO(), karmada, metav1.UpdateOptions{})
+		gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+	})
+}
+
+// GetLastTransitionTime gets the last transition time of the condition, return time.Now() if not found.
+func GetLastTransitionTime(karmada *operatorv1alpha1.Karmada, conditionType operatorv1alpha1.ConditionType) time.Time {
+	for _, condition := range karmada.Status.Conditions {
+		if condition.Type == string(conditionType) {
+			return condition.LastTransitionTime.Time
+		}
+	}
+	return time.Now()
+}

test/e2e/suites/operator/priorityclass_test.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2025 The Karmada Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/rand"
+
+	operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
+	"github.com/karmada-io/karmada/test/e2e/framework"
+)
+
+var _ = ginkgo.Describe("PriorityClass configuration testing", func() {
+	var karmadaName string
+	var karmadaObject *operatorv1alpha1.Karmada
+	var err error
+
+	ginkgo.Context("PriorityClass configuration testing", func() {
+		ginkgo.BeforeEach(func() {
+			karmadaName = KarmadaInstanceNamePrefix + rand.String(RandomStrLength)
+			InitializeKarmadaInstance(operatorClient, testNamespace, karmadaName)
+		})
+
+		ginkgo.AfterEach(func() {
+			err = operatorClient.OperatorV1alpha1().Karmadas(testNamespace).Delete(context.TODO(), karmadaName, metav1.DeleteOptions{})
+			gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+		})
+
+		ginkgo.It("Custom priorityClass configuration", func() {
+			ginkgo.By("Check if default value is system-node-critical", func() {
+				// take etcd as a representative of StatefulSet.
+				etcd, err := kubeClient.AppsV1().StatefulSets(testNamespace).Get(context.TODO(), karmadaName+"-etcd", metav1.GetOptions{})
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				gomega.Expect(etcd.Spec.Template.Spec.PriorityClassName).Should(gomega.Equal("system-node-critical"))
+
+				// take karmada-apiserver as a representative of Deployment.
+				karmadaApiserver, err := kubeClient.AppsV1().Deployments(testNamespace).Get(context.TODO(), karmadaName+"-apiserver", metav1.GetOptions{})
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				gomega.Expect(karmadaApiserver.Spec.Template.Spec.PriorityClassName).Should(gomega.Equal("system-node-critical"))
+			})
+
+			ginkgo.By("Set priorityClass to system-cluster-critical", func() {
+				karmadaObject, err = operatorClient.OperatorV1alpha1().Karmadas(testNamespace).Get(context.TODO(), karmadaName, metav1.GetOptions{})
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				karmadaObject.Spec.Components.Etcd.Local.PriorityClassName = "system-cluster-critical"
+				karmadaObject.Spec.Components.KarmadaAPIServer.PriorityClassName = "system-cluster-critical"
+				framework.UpdateKarmadaInstanceWithSpec(operatorClient, testNamespace, karmadaName, karmadaObject.Spec)
+				framework.WaitKarmadaReady(operatorClient, testNamespace, karmadaName, framework.GetLastTransitionTime(karmadaObject, operatorv1alpha1.Ready))
+			})
+
+			ginkgo.By("Check if the PriorityClass is applied correctly", func() {
+				// take etcd as a representative of StatefulSet.
+				etcd, err := kubeClient.AppsV1().StatefulSets(testNamespace).Get(context.TODO(), karmadaName+"-etcd", metav1.GetOptions{ResourceVersion: "0"})
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				gomega.Expect(etcd.Spec.Template.Spec.PriorityClassName).Should(gomega.Equal("system-cluster-critical"))
+
+				// take karmada-apiserver as a representative of Deployment.
+				karmadaApiserver, err := kubeClient.AppsV1().Deployments(testNamespace).Get(context.TODO(), karmadaName+"-apiserver", metav1.GetOptions{ResourceVersion: "0"})
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				gomega.Expect(karmadaApiserver.Spec.Template.Spec.PriorityClassName).Should(gomega.Equal("system-cluster-critical"))
+			})
+		})
+	})
+})

test/e2e/suites/operator/status_test.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2025 The Karmada Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/rand"
+
+	operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
+	operatorutil "github.com/karmada-io/karmada/operator/pkg/util"
+)
+
+var _ = ginkgo.Describe("Status testing", func() {
+	var karmadaName string
+	var karmadaObject *operatorv1alpha1.Karmada
+	var err error
+
+	ginkgo.Context("Karmada instance status testing", func() {
+		ginkgo.BeforeEach(func() {
+			karmadaName = KarmadaInstanceNamePrefix + rand.String(RandomStrLength)
+			InitializeKarmadaInstance(operatorClient, testNamespace, karmadaName)
+		})
+
+		ginkgo.AfterEach(func() {
+			err = operatorClient.OperatorV1alpha1().Karmadas(testNamespace).Delete(context.TODO(), karmadaName, metav1.DeleteOptions{})
+			gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+		})
+
+		ginkgo.It("Check if the karmada status meets the expectations", func() {
+			ginkgo.By("Get the latest karmada instance", func() {
+				karmadaObject, err = operatorClient.OperatorV1alpha1().Karmadas(testNamespace).Get(context.TODO(), karmadaName, metav1.GetOptions{})
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+			})
+
+			ginkgo.By("Check if status.conditions meets the expectations", func() {
+				conditions := karmadaObject.Status.Conditions
+				gomega.Expect(len(conditions)).Should(gomega.BeNumerically(">", 0))
+				// check if the Ready condition is true
+				hasReadyCondition := false
+				for i := range karmadaObject.Status.Conditions {
+					switch karmadaObject.Status.Conditions[i].Type {
+					case string(operatorv1alpha1.Ready):
+						gomega.Expect(karmadaObject.Status.Conditions[i].Status).Should(gomega.Equal(metav1.ConditionTrue))
+						hasReadyCondition = true
+					}
+				}
+				gomega.Expect(hasReadyCondition).Should(gomega.BeTrue())
+			})
+
+			ginkgo.By("Check if the status.SecretRef can ref to the right secret", func() {
+				secretRef := karmadaObject.Status.SecretRef
+				gomega.Expect(secretRef).ShouldNot(gomega.BeNil())
+				gomega.Expect(secretRef.Namespace).Should(gomega.Equal(karmadaObject.GetNamespace()))
+				gomega.Expect(secretRef.Name).Should(gomega.Equal(operatorutil.AdminKarmadaConfigSecretName(karmadaObject.GetName())))
+				_, err := kubeClient.CoreV1().Secrets(secretRef.Namespace).Get(context.TODO(), secretRef.Name, metav1.GetOptions{})
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+			})
+
+			ginkgo.By("Check if the status.apiServerService can ref to the right service", func() {
+				apiServerService := karmadaObject.Status.APIServerService
+				gomega.Expect(apiServerService).ShouldNot(gomega.BeNil())
+				gomega.Expect(apiServerService.Name).Should(gomega.Equal(operatorutil.KarmadaAPIServerName(karmadaObject.GetName())))
+				_, err := kubeClient.CoreV1().Services(karmadaObject.GetNamespace()).Get(context.TODO(), apiServerService.Name, metav1.GetOptions{})
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+			})
+		})
+	})
+})