OperationJob support "Restart" action

.github/workflows/e2e.yaml

@@ -13,6 +13,7 @@ env:
   GO_VERSION: '1.19'
   KIND_VERSION: 'v0.14.0'
   KIND_IMAGE: 'kindest/node:v1.22.2'
+  KIND_IMAGE_KRUISE:  'kindest/node:v1.24.6'
   KIND_CLUSTER_NAME: 'e2e-test'
 
 jobs:
@@ -82,3 +83,146 @@ jobs:
           kubectl get pod -n kusionstack-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kusionstack-system
           exit 1
         fi
+
+  OperationJob:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Cache Go Dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-go-
+      - name: Setup Kind Cluster
+        uses: helm/[email protected]
+        with:
+          node_image: ${{ env.KIND_IMAGE_KRUISE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/e2e/scripts/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Setup Kube-config
+        run: |
+          mkdir -p /tmp/kind
+          make kind-kube-config
+      - name: Clone Kruise Repo
+        uses: GuillaumeFalourd/[email protected]
+        with:
+          branch: release-1.6
+          owner: openkruise
+          repository: kruise
+      - name: Install Kruise
+        run: |
+          set -ex
+          cd kruise
+          export KRUISE_IMG="openkruise/kruise-manager:v1.6"
+          docker build -t ${KRUISE_IMG} .
+          kind load docker-image --name=${KIND_CLUSTER_NAME} ${KRUISE_IMG}
+          make deploy
+          NODES=$(kubectl get node | wc -l)
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+            set -e
+            if [ "$PODS" -eq "$NODES" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+          kubectl get node -o yaml
+          kubectl get all -n kruise-system -o yaml
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
+          set -e
+          if [ "$PODS" -eq "$NODES" ]; then
+            echo "Wait for kruise-manager and kruise-daemon ready successfully"
+          else
+            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
+            exit 1
+          fi
+      - name: Install Operating
+        run: |
+          set -ex
+          kubectl cluster-info
+          make docker-build
+          make sync-kind-image
+          make deploy
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kusionstack-system | grep -c '1/1')
+            set -e
+            if [ "$PODS" -eq 1 ]; then
+              break
+            fi
+            sleep 3
+          done
+          set -e
+          PODS=$(kubectl get pod -n kusionstack-system | grep -c '1/1')
+          if [ "$PODS" -eq 1 ]; then
+            echo "Wait for Kusionstack-manager ready successfully"
+          else 
+            echo "Timeout to wait for Kusionstack-manager ready"
+          fi
+      - name: Run e2e Tests
+        run: |
+          make ginkgo
+          set -e
+          KUBECONFIG=/tmp/kind/kubeconfig.yaml ./bin/ginkgo -timeout 10m -v --focus='\[apps\] OperationJob' test/e2e
+      - name: Check Operating Manager
+        run: |
+          restartCount=$(kubectl get pod -n kusionstack-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+            echo "Kusionstack-manager has not restarted"
+          else
+            kubectl get pod -n kusionstack-system -l control-plane=controller-manager --no-headers
+            echo "Kusionstack-manager has restarted, abort!!!"
+            kubectl get pod -n kusionstack-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kusionstack-system
+            exit 1
+          fi
+      - name: Check Kruise Manager
+        run: |
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "$out"
+              echo "Kruise-manager has not restarted"
+          else
+              echo "$out"
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
+
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')            
+          fi
+          exit $retVal

apis/apps/v1alpha1/operationjob_types.go

@@ -0,0 +1,168 @@
+/*
+Copyright 2024 The KusionStack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	OpsActionRestart = "Restart"
+	OpsActionReplace = "Replace"
+)
+
+const (
+	ReasonPodNotFound       = "PodNotFound"
+	ReasonContainerNotFound = "ContainerNotFound"
+	ReasonReplacedByNewPod  = "ReplacedByNewPod"
+)
+
+// OperationProgress indicates operation progress of pod
+type OperationProgress string
+
+const (
+	OperationProgressPending    OperationProgress = "Pending"
+	OperationProgressProcessing OperationProgress = "Processing"
+	OperationProgressFailed     OperationProgress = "Failed"
+	OperationProgressSucceeded  OperationProgress = "Succeeded"
+)
+
+// OperationJobSpec defines the desired state of OperationJob
+type OperationJobSpec struct {
+	// Specify the operation actions including: Restart, Replace
+	// +optional
+	Action string `json:"action,omitempty"`
+
+	// Define the operation target pods
+	// +optional
+	Targets []PodOpsTarget `json:"targets,omitempty"`
+
+	// Partition controls the operation progress by indicating how many pods should be operated.
+	// Defaults to nil (all pods will be updated)
+	// +optional
+	Partition *int32 `json:"partition,omitempty"`
+
+	// OperationDelaySeconds indicates how many seconds it should delay before operating update.
+	// +optional
+	OperationDelaySeconds *int32 `json:"operationDelaySeconds,omitempty"`
+
+	// Specify the duration in seconds relative to the startTime
+	// that the job may be active before the system tries to terminate it
+	// +optional
+	ActiveDeadlineSeconds *int32 `json:"activeDeadlineSeconds,omitempty"`
+
+	// Limit the lifetime of an operation that has finished execution (either Complete or Failed)
+	// +optional
+	TTLSecondsAfterFinished *int32 `json:"TTLSecondsAfterFinished,omitempty"`
+}
+
+// PodOpsTarget defines the target pods of the OperationJob
+type PodOpsTarget struct {
+	// Specify the operation target pods
+	// +optional
+	Name string `json:"name,omitempty"`
+
+	// Specify the containers to restart
+	// +optional
+	Containers []string `json:"containers,omitempty"`
+}
+
+// OperationJobStatus defines the observed state of OperationJob
+type OperationJobStatus struct {
+	// ObservedGeneration is the most recent generation observed for this OperationJob. It corresponds to the
+	// OperationJob's generation, which is updated on mutation by the API Server.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+
+	// Phase indicates the of the OperationJob
+	// +optional
+	Progress OperationProgress `json:"progress,omitempty"`
+
+	// Operation start time
+	// +optional
+	StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"`
+
+	// Operation end time
+	// +optional
+	EndTimestamp *metav1.Time `json:"endTimestamp,omitempty"`
+
+	// Replicas of the pods involved in the OperationJob
+	// +optional
+	TotalPodCount int32 `json:"totalPodCount,omitempty"`
+
+	// Succeeded replicas of the pods involved in the OperationJob
+	// +optional
+	SucceededPodCount int32 `json:"succeededPodCount,omitempty"`
+
+	// failed pod count of the pods involved in the OperationJob
+	// +optional
+	FailedPodCount int32 `json:"failedPodCount,omitempty"`
+
+	// Operation details of the target pods
+	// +optional
+	TargetDetails []OpsStatus `json:"targetDetails,omitempty"`
+}
+
+type OpsStatus struct {
+	// name of the target pod
+	// +optional
+	Name string `json:"name,omitempty"`
+
+	// operation progress of target pod
+	// +optional
+	Progress OperationProgress `json:"progress,omitempty"`
+
+	// reason for current operation progress
+	// +optional
+	Reason string `json:"reason,omitempty"`
+
+	// message displays detail of reason
+	// +optional
+	Message string `json:"message,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+
+// +k8s:openapi-gen=true
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +kubebuilder:resource:shortName=oj
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="PROGRESS",type="string",JSONPath=".status.progress"
+// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp"
+
+// OperationJob is the Schema for the operationjobs API
+type OperationJob struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   OperationJobSpec   `json:"spec,omitempty"`
+	Status OperationJobStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// OperationJobList contains a list of OperationJob
+type OperationJobList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []OperationJob `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&OperationJob{}, &OperationJobList{})
+}