From 3bef8a8ac68bdbe00d5f45859ce62961b9636709 Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Fri, 23 Sep 2022 18:44:21 -0700
Subject: [PATCH 01/18] when Spec.selector changed, since it is immutable, the
 old deployment/statefulset/daemonset will be deleted and a new one with
 updated selector will be created.

---
 pkg/collector/reconcile/daemonset.go   | 45 +++++++++++++++++++++++---
 pkg/collector/reconcile/deployment.go  | 44 +++++++++++++++++++++++--
 pkg/collector/reconcile/statefulset.go | 44 +++++++++++++++++++++++--
 3 files changed, 123 insertions(+), 10 deletions(-)

diff --git a/pkg/collector/reconcile/daemonset.go b/pkg/collector/reconcile/daemonset.go
index be597cd7e3..be905df62a 100644
--- a/pkg/collector/reconcile/daemonset.go
+++ b/pkg/collector/reconcile/daemonset.go
@@ -17,16 +17,20 @@ package reconcile
 import (
 	"context"
 	"fmt"
-
 	appsv1 "k8s.io/api/apps/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"time"
 
 	"github.com/open-telemetry/opentelemetry-operator/pkg/collector"
 )
 
+const DeleteDaemonSetWaitTimeOutInSeconds = 300
+
 // +kubebuilder:rbac:groups="apps",resources=daemonsets,verbs=get;list;watch;create;update;patch;delete
 
 // DaemonSets reconciles the daemon set(s) required for the instance in the current context.
@@ -70,6 +74,24 @@ func expectedDaemonSets(ctx context.Context, params Params, expected []appsv1.Da
 			return fmt.Errorf("failed to get: %w", err)
 		}
 
+		// Selector is an immutable field, if set, we cannot modify it otherwise we will face reconciliation error.
+		if !apiequality.Semantic.DeepEqual(desired.Spec.Selector, existing.Spec.Selector) {
+			params.Log.V(2).Info("Spec.Selector change detected, trying to delete and re-create", "daemonset.name", existing.Name, "daemonset.namespace", existing.Namespace)
+
+			do := &client.DeleteOptions{}
+			client.PropagationPolicy(metav1.DeletePropagationForeground).ApplyToDelete(do)
+			if err := params.Client.Delete(ctx, existing, do); err != nil {
+				return fmt.Errorf("failed to request deleting daemonset: %w", err)
+			}
+			if err := waitDaemonSetDeleted(ctx, params, nns); err != nil {
+				return fmt.Errorf("failed to delete daemonset: %w", err)
+			}
+			if err := params.Client.Create(ctx, &desired); err != nil {
+				return fmt.Errorf("failed to create new daemonset: %w", err)
+			}
+			continue
+		}
+
 		// it exists already, merge the two if the end result isn't identical to the existing one
 		updated := existing.DeepCopy()
 		if updated.Annotations == nil {
@@ -89,9 +111,6 @@ func expectedDaemonSets(ctx context.Context, params Params, expected []appsv1.Da
 			updated.ObjectMeta.Labels[k] = v
 		}
 
-		// Selector is an immutable field, if set, we cannot modify it otherwise we will face reconciliation error.
-		updated.Spec.Selector = existing.Spec.Selector.DeepCopy()
-
 		patch := client.MergeFrom(existing)
 		if err := params.Client.Patch(ctx, updated, patch); err != nil {
 			return fmt.Errorf("failed to apply changes: %w", err)
@@ -136,3 +155,21 @@ func deleteDaemonSets(ctx context.Context, params Params, expected []appsv1.Daem
 
 	return nil
 }
+
+func waitDaemonSetDeleted(ctx context.Context, params Params, nns types.NamespacedName) error {
+	for i := 0; i < DeleteDaemonSetWaitTimeOutInSeconds; i++ {
+		existing := &appsv1.DaemonSet{}
+		err := params.Client.Get(ctx, nns, existing)
+		if err != nil && k8serrors.IsNotFound(err) {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("wait Interrupted for updating daemonset.name %s, daemonset.namespace %s", nns.Name, nns.Namespace)
+		case <-time.After(time.Second):
+		}
+	}
+	return fmt.Errorf("wait timeout for updating daemonset.name %s, daemonset.namespace %s", nns.Name, nns.Namespace)
+}
diff --git a/pkg/collector/reconcile/deployment.go b/pkg/collector/reconcile/deployment.go
index 3cf567ce27..0664b8af6b 100644
--- a/pkg/collector/reconcile/deployment.go
+++ b/pkg/collector/reconcile/deployment.go
@@ -17,6 +17,9 @@ package reconcile
 import (
 	"context"
 	"fmt"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -31,6 +34,8 @@ import (
 
 // +kubebuilder:rbac:groups="apps",resources=deployments,verbs=get;list;watch;create;update;patch;delete
 
+const DeleteDeploymentWaitTimeOutInSeconds = 300
+
 // Deployments reconciles the deployment(s) required for the instance in the current context.
 func Deployments(ctx context.Context, params Params) error {
 	desired := []appsv1.Deployment{}
@@ -76,6 +81,24 @@ func expectedDeployments(ctx context.Context, params Params, expected []appsv1.D
 			return fmt.Errorf("failed to get: %w", err)
 		}
 
+		// Selector is an immutable field, if set, we cannot modify it otherwise we will face reconciliation error.
+		if !apiequality.Semantic.DeepEqual(desired.Spec.Selector, existing.Spec.Selector) {
+			params.Log.V(2).Info("Spec.Selector change detected, trying to delete and re-create", "deployment.name", existing.Name, "deployment.namespace", existing.Namespace)
+
+			do := &client.DeleteOptions{}
+			client.PropagationPolicy(metav1.DeletePropagationForeground).ApplyToDelete(do)
+			if err := params.Client.Delete(ctx, existing); err != nil {
+				return fmt.Errorf("failed to delete deployment: %w", err)
+			}
+			if err := waitDeploymentDeleted(ctx, params, nns); err != nil {
+				return fmt.Errorf("failed to delete deployment: %w", err)
+			}
+			if err := params.Client.Create(ctx, &desired); err != nil {
+				return fmt.Errorf("failed to create new deployment: %w", err)
+			}
+			continue
+		}
+
 		// it exists already, merge the two if the end result isn't identical to the existing one
 		updated := existing.DeepCopy()
 		if updated.Annotations == nil {
@@ -95,9 +118,6 @@ func expectedDeployments(ctx context.Context, params Params, expected []appsv1.D
 			updated.ObjectMeta.Labels[k] = v
 		}
 
-		// Selector is an immutable field, if set, we cannot modify it otherwise we will face reconciliation error.
-		updated.Spec.Selector = existing.Spec.Selector.DeepCopy()
-
 		patch := client.MergeFrom(existing)
 
 		if err := params.Client.Patch(ctx, updated, patch); err != nil {
@@ -156,3 +176,21 @@ func currentReplicasWithHPA(spec v1alpha1.OpenTelemetryCollectorSpec, curr int32
 
 	return curr
 }
+
+func waitDeploymentDeleted(ctx context.Context, params Params, nns types.NamespacedName) error {
+	for i := 0; i < DeleteDeploymentWaitTimeOutInSeconds; i++ {
+		existing := &appsv1.Deployment{}
+		err := params.Client.Get(ctx, nns, existing)
+		if err != nil && k8serrors.IsNotFound(err) {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("wait Interrupted for updating deployment.name %s, deployment.namespace %s", nns.Name, nns.Namespace)
+		case <-time.After(time.Second):
+		}
+	}
+	return fmt.Errorf("wait timeout for updating deployment.name %s, deployment.namespace %s", nns.Name, nns.Namespace)
+}
diff --git a/pkg/collector/reconcile/statefulset.go b/pkg/collector/reconcile/statefulset.go
index 372b28d3b0..610b10aca8 100644
--- a/pkg/collector/reconcile/statefulset.go
+++ b/pkg/collector/reconcile/statefulset.go
@@ -17,6 +17,9 @@ package reconcile
 import (
 	"context"
 	"fmt"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -29,6 +32,8 @@ import (
 
 // +kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list;watch;create;update;patch;delete
 
+const DeleteStatefulSetWaitTimeOutInSeconds = 300
+
 // StatefulSets reconciles the stateful set(s) required for the instance in the current context.
 func StatefulSets(ctx context.Context, params Params) error {
 
@@ -71,6 +76,24 @@ func expectedStatefulSets(ctx context.Context, params Params, expected []appsv1.
 			return fmt.Errorf("failed to get: %w", err)
 		}
 
+		// Selector is an immutable field, if set, we cannot modify it otherwise we will face reconciliation error.
+		if !apiequality.Semantic.DeepEqual(desired.Spec.Selector, existing.Spec.Selector) {
+			params.Log.V(2).Info("Spec.Selector change detected, trying to delete and re-create", "statefulset.name", existing.Name, "statefulset.namespace", existing.Namespace)
+
+			do := &client.DeleteOptions{}
+			client.PropagationPolicy(metav1.DeletePropagationForeground).ApplyToDelete(do)
+			if err := params.Client.Delete(ctx, existing); err != nil {
+				return fmt.Errorf("failed to delete statefulset: %w", err)
+			}
+			if err := waitStatefulSetDeleted(ctx, params, nns); err != nil {
+				return fmt.Errorf("failed to delete statefulset: %w", err)
+			}
+			if err := params.Client.Create(ctx, &desired); err != nil {
+				return fmt.Errorf("failed to create new statefulset: %w", err)
+			}
+			continue
+		}
+
 		// it exists already, merge the two if the end result isn't identical to the existing one
 		updated := existing.DeepCopy()
 		if updated.Annotations == nil {
@@ -90,9 +113,6 @@ func expectedStatefulSets(ctx context.Context, params Params, expected []appsv1.
 			updated.ObjectMeta.Labels[k] = v
 		}
 
-		// Selector is an immutable field, if set, we cannot modify it otherwise we will face reconciliation error.
-		updated.Spec.Selector = existing.Spec.Selector.DeepCopy()
-
 		patch := client.MergeFrom(existing)
 		if err := params.Client.Patch(ctx, updated, patch); err != nil {
 			return fmt.Errorf("failed to apply changes: %w", err)
@@ -137,3 +157,21 @@ func deleteStatefulSets(ctx context.Context, params Params, expected []appsv1.St
 
 	return nil
 }
+
+func waitStatefulSetDeleted(ctx context.Context, params Params, nns types.NamespacedName) error {
+	for i := 0; i < DeleteStatefulSetWaitTimeOutInSeconds; i++ {
+		existing := &appsv1.StatefulSet{}
+		err := params.Client.Get(ctx, nns, existing)
+		if err != nil && k8serrors.IsNotFound(err) {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("wait Interrupted for updating statefulset.name %s, statefulset.namespace %s", nns.Name, nns.Namespace)
+		case <-time.After(time.Second):
+		}
+	}
+	return fmt.Errorf("wait timeout for updating statefulset.name %s, statefulset.namespace: %s", nns.Name, nns.Namespace)
+}

From a3f8809df150b0985bd7f6a29b5390aa60885ddc Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Sat, 24 Sep 2022 02:55:06 -0700
Subject: [PATCH 02/18] add e2e test.

---
 Makefile                                      |  5 ++
 kuttl-test-upgrade.yaml                       | 13 ++++
 tests/e2e-upgrade/upgrade-test/00-assert.yaml | 69 +++++++++++++++++++
 .../e2e-upgrade/upgrade-test/00-install.yaml  | 28 ++++++++
 .../upgrade-test/01-upgrade-operator.yaml     |  5 ++
 tests/e2e-upgrade/upgrade-test/02-assert.yaml |  8 +++
 .../upgrade-test/02-upgrade-collector.yaml    | 28 ++++++++
 7 files changed, 156 insertions(+)
 create mode 100644 kuttl-test-upgrade.yaml
 create mode 100644 tests/e2e-upgrade/upgrade-test/00-assert.yaml
 create mode 100644 tests/e2e-upgrade/upgrade-test/00-install.yaml
 create mode 100644 tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml
 create mode 100644 tests/e2e-upgrade/upgrade-test/02-assert.yaml
 create mode 100644 tests/e2e-upgrade/upgrade-test/02-upgrade-collector.yaml

diff --git a/Makefile b/Makefile
index 09ce4b790a..3c41427df5 100644
--- a/Makefile
+++ b/Makefile
@@ -157,6 +157,11 @@ generate: controller-gen api-docs
 e2e:
 	$(KUTTL) test
 
+# end-to-end-test for testing upgrading
+.PHONY: e2e-upgrade
+e2e-upgrade:
+	$(KUTTL) test --config kuttl-test-upgrade.yaml
+
 .PHONY: prepare-e2e
 prepare-e2e: kuttl set-test-image-vars set-image-controller container container-target-allocator start-kind install-metrics-server load-image-all
 	mkdir -p tests/_build/crds tests/_build/manifests
diff --git a/kuttl-test-upgrade.yaml b/kuttl-test-upgrade.yaml
new file mode 100644
index 0000000000..49500b9441
--- /dev/null
+++ b/kuttl-test-upgrade.yaml
@@ -0,0 +1,13 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+artifactsDir: ./tests/_build/artifacts/
+kindContainers:
+  - local/opentelemetry-operator:e2e
+  - ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator:v0.49.0
+commands:
+  - command: make cert-manager
+  - command: kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.49.0/opentelemetry-operator.yaml
+  - command: kubectl rollout status -w deployment/opentelemetry-operator-controller-manager -n opentelemetry-operator-system
+testDirs:
+  - ./tests/e2e-upgrade/
+timeout: 150
\ No newline at end of file
diff --git a/tests/e2e-upgrade/upgrade-test/00-assert.yaml b/tests/e2e-upgrade/upgrade-test/00-assert.yaml
new file mode 100644
index 0000000000..f6462600fd
--- /dev/null
+++ b/tests/e2e-upgrade/upgrade-test/00-assert.yaml
@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: simplest-collector
+  annotations:
+    operatorVersion: "v0.49.0"
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/version: latest
+status:
+  readyReplicas: 1
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: simplest-collector-headless
+spec:
+  ports:
+    - appProtocol: grpc
+      name: jaeger-grpc
+      port: 14250
+      protocol: TCP
+      targetPort: 14250
+    - appProtocol: grpc
+      name: otlp-grpc
+      port: 4317
+      protocol: TCP
+      targetPort: 4317
+    - appProtocol: http
+      name: otlp-http
+      port: 4318
+      protocol: TCP
+      targetPort: 4318
+    - appProtocol: http
+      name: otlp-http-legacy
+      port: 55681
+      protocol: TCP
+      targetPort: 4318
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: simplest-collector
+spec:
+  ports:
+    - appProtocol: grpc
+      name: jaeger-grpc
+      port: 14250
+      protocol: TCP
+      targetPort: 14250
+    - appProtocol: grpc
+      name: otlp-grpc
+      port: 4317
+      protocol: TCP
+      targetPort: 4317
+    - appProtocol: http
+      name: otlp-http
+      port: 4318
+      protocol: TCP
+      targetPort: 4318
+    - appProtocol: http
+      name: otlp-http-legacy
+      port: 55681
+      protocol: TCP
+      targetPort: 4318
diff --git a/tests/e2e-upgrade/upgrade-test/00-install.yaml b/tests/e2e-upgrade/upgrade-test/00-install.yaml
new file mode 100644
index 0000000000..653e806fbc
--- /dev/null
+++ b/tests/e2e-upgrade/upgrade-test/00-install.yaml
@@ -0,0 +1,28 @@
+apiVersion: opentelemetry.io/v1alpha1
+kind: OpenTelemetryCollector
+metadata:
+  name: simplest
+  annotations:
+    operatorVersion: "v0.49.0"
+spec:
+  replicas: 1
+  config: |
+    receivers:
+      jaeger:
+        protocols:
+          grpc:
+      otlp:
+        protocols:
+          grpc:
+          http:
+    processors:
+
+    exporters:
+      logging:
+
+    service:
+      pipelines:
+        traces:
+          receivers: [jaeger,otlp]
+          processors: []
+          exporters: [logging]
\ No newline at end of file
diff --git a/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml b/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml
new file mode 100644
index 0000000000..3eed483284
--- /dev/null
+++ b/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml
@@ -0,0 +1,5 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: kubectl apply -f ../../_build/manifests/01-opentelemetry-operator.yaml
+  - command: kubectl rollout status -w deployment/opentelemetry-operator-controller-manager -n opentelemetry-operator-system
\ No newline at end of file
diff --git a/tests/e2e-upgrade/upgrade-test/02-assert.yaml b/tests/e2e-upgrade/upgrade-test/02-assert.yaml
new file mode 100644
index 0000000000..1cd0b3aed8
--- /dev/null
+++ b/tests/e2e-upgrade/upgrade-test/02-assert.yaml
@@ -0,0 +1,8 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: simplest-collector
+  annotations:
+    operatorVersion: "latest"
+status:
+  readyReplicas: 2
\ No newline at end of file
diff --git a/tests/e2e-upgrade/upgrade-test/02-upgrade-collector.yaml b/tests/e2e-upgrade/upgrade-test/02-upgrade-collector.yaml
new file mode 100644
index 0000000000..ae1064183e
--- /dev/null
+++ b/tests/e2e-upgrade/upgrade-test/02-upgrade-collector.yaml
@@ -0,0 +1,28 @@
+apiVersion: opentelemetry.io/v1alpha1
+kind: OpenTelemetryCollector
+metadata:
+  name: simplest
+  annotations:
+    operatorVersion: "latest"
+spec:
+  replicas: 2
+  config: |
+    receivers:
+      jaeger:
+        protocols:
+          grpc:
+      otlp:
+        protocols:
+          grpc:
+          http:
+    processors:
+
+    exporters:
+      logging:
+
+    service:
+      pipelines:
+        traces:
+          receivers: [jaeger,otlp]
+          processors: []
+          exporters: [logging]
\ No newline at end of file

From 86b7ebbab700f882b420dd9f861476ef2fcd3039 Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Sat, 24 Sep 2022 15:44:39 -0700
Subject: [PATCH 03/18] address lint.

---
 pkg/collector/reconcile/daemonset.go   | 3 ++-
 pkg/collector/reconcile/deployment.go  | 3 ++-
 pkg/collector/reconcile/statefulset.go | 3 ++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/pkg/collector/reconcile/daemonset.go b/pkg/collector/reconcile/daemonset.go
index be905df62a..4f9909fd65 100644
--- a/pkg/collector/reconcile/daemonset.go
+++ b/pkg/collector/reconcile/daemonset.go
@@ -17,6 +17,8 @@ package reconcile
 import (
 	"context"
 	"fmt"
+	"time"
+
 	appsv1 "k8s.io/api/apps/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -24,7 +26,6 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-	"time"
 
 	"github.com/open-telemetry/opentelemetry-operator/pkg/collector"
 )
diff --git a/pkg/collector/reconcile/deployment.go b/pkg/collector/reconcile/deployment.go
index 0664b8af6b..bb946e6a30 100644
--- a/pkg/collector/reconcile/deployment.go
+++ b/pkg/collector/reconcile/deployment.go
@@ -17,9 +17,10 @@ package reconcile
 import (
 	"context"
 	"fmt"
+	"time"
+
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
diff --git a/pkg/collector/reconcile/statefulset.go b/pkg/collector/reconcile/statefulset.go
index 610b10aca8..7d258fd984 100644
--- a/pkg/collector/reconcile/statefulset.go
+++ b/pkg/collector/reconcile/statefulset.go
@@ -17,9 +17,10 @@ package reconcile
 import (
 	"context"
 	"fmt"
+	"time"
+
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"

From 842f72003ccc89b477aae3d9190a56ba95830bab Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Sat, 24 Sep 2022 15:48:06 -0700
Subject: [PATCH 04/18] add newline to end.

---
 tests/e2e-upgrade/upgrade-test/00-install.yaml           | 2 +-
 tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml  | 2 +-
 tests/e2e-upgrade/upgrade-test/02-assert.yaml            | 2 +-
 tests/e2e-upgrade/upgrade-test/02-upgrade-collector.yaml | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/e2e-upgrade/upgrade-test/00-install.yaml b/tests/e2e-upgrade/upgrade-test/00-install.yaml
index 653e806fbc..b0e18de3c6 100644
--- a/tests/e2e-upgrade/upgrade-test/00-install.yaml
+++ b/tests/e2e-upgrade/upgrade-test/00-install.yaml
@@ -25,4 +25,4 @@ spec:
         traces:
           receivers: [jaeger,otlp]
           processors: []
-          exporters: [logging]
\ No newline at end of file
+          exporters: [logging]
diff --git a/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml b/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml
index 3eed483284..33adeb09ed 100644
--- a/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml
+++ b/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml
@@ -2,4 +2,4 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
   - command: kubectl apply -f ../../_build/manifests/01-opentelemetry-operator.yaml
-  - command: kubectl rollout status -w deployment/opentelemetry-operator-controller-manager -n opentelemetry-operator-system
\ No newline at end of file
+  - command: kubectl rollout status -w deployment/opentelemetry-operator-controller-manager -n opentelemetry-operator-system
diff --git a/tests/e2e-upgrade/upgrade-test/02-assert.yaml b/tests/e2e-upgrade/upgrade-test/02-assert.yaml
index 1cd0b3aed8..b40c5a7bb0 100644
--- a/tests/e2e-upgrade/upgrade-test/02-assert.yaml
+++ b/tests/e2e-upgrade/upgrade-test/02-assert.yaml
@@ -5,4 +5,4 @@ metadata:
   annotations:
     operatorVersion: "latest"
 status:
-  readyReplicas: 2
\ No newline at end of file
+  readyReplicas: 2
diff --git a/tests/e2e-upgrade/upgrade-test/02-upgrade-collector.yaml b/tests/e2e-upgrade/upgrade-test/02-upgrade-collector.yaml
index ae1064183e..dceb4999cd 100644
--- a/tests/e2e-upgrade/upgrade-test/02-upgrade-collector.yaml
+++ b/tests/e2e-upgrade/upgrade-test/02-upgrade-collector.yaml
@@ -25,4 +25,4 @@ spec:
         traces:
           receivers: [jaeger,otlp]
           processors: []
-          exporters: [logging]
\ No newline at end of file
+          exporters: [logging]

From b13f969defe4e1505203104b1514238aced9a290 Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Sat, 24 Sep 2022 15:48:48 -0700
Subject: [PATCH 05/18] add newline to end.

---
 kuttl-test-upgrade.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kuttl-test-upgrade.yaml b/kuttl-test-upgrade.yaml
index 49500b9441..043b455a7e 100644
--- a/kuttl-test-upgrade.yaml
+++ b/kuttl-test-upgrade.yaml
@@ -10,4 +10,4 @@ commands:
   - command: kubectl rollout status -w deployment/opentelemetry-operator-controller-manager -n opentelemetry-operator-system
 testDirs:
   - ./tests/e2e-upgrade/
-timeout: 150
\ No newline at end of file
+timeout: 150

From 302ef4557e2082c736265e9841e80f84d284632e Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Wed, 28 Sep 2022 16:14:43 -0700
Subject: [PATCH 06/18] add delete options to deployment and statefulset.

---
 pkg/collector/reconcile/deployment.go  | 2 +-
 pkg/collector/reconcile/statefulset.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/collector/reconcile/deployment.go b/pkg/collector/reconcile/deployment.go
index bb946e6a30..e3c08428a7 100644
--- a/pkg/collector/reconcile/deployment.go
+++ b/pkg/collector/reconcile/deployment.go
@@ -88,7 +88,7 @@ func expectedDeployments(ctx context.Context, params Params, expected []appsv1.D
 
 			do := &client.DeleteOptions{}
 			client.PropagationPolicy(metav1.DeletePropagationForeground).ApplyToDelete(do)
-			if err := params.Client.Delete(ctx, existing); err != nil {
+			if err := params.Client.Delete(ctx, existing, do); err != nil {
 				return fmt.Errorf("failed to delete deployment: %w", err)
 			}
 			if err := waitDeploymentDeleted(ctx, params, nns); err != nil {
diff --git a/pkg/collector/reconcile/statefulset.go b/pkg/collector/reconcile/statefulset.go
index 7d258fd984..7356dbf0f6 100644
--- a/pkg/collector/reconcile/statefulset.go
+++ b/pkg/collector/reconcile/statefulset.go
@@ -83,7 +83,7 @@ func expectedStatefulSets(ctx context.Context, params Params, expected []appsv1.
 
 			do := &client.DeleteOptions{}
 			client.PropagationPolicy(metav1.DeletePropagationForeground).ApplyToDelete(do)
-			if err := params.Client.Delete(ctx, existing); err != nil {
+			if err := params.Client.Delete(ctx, existing, do); err != nil {
 				return fmt.Errorf("failed to delete statefulset: %w", err)
 			}
 			if err := waitStatefulSetDeleted(ctx, params, nns); err != nil {

From 6b520ce8a11bd231b37ab9167a30803c4133ce18 Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Fri, 30 Sep 2022 17:29:30 -0700
Subject: [PATCH 07/18] remove unecessary propagation policy, the default
 background is working fine.

---
 config/manager/kustomization.yaml                       | 2 +-
 kuttl-test-upgrade.yaml                                 | 2 ++
 pkg/collector/reconcile/daemonset.go                    | 5 +----
 pkg/collector/reconcile/deployment.go                   | 8 ++------
 pkg/collector/reconcile/statefulset.go                  | 8 ++------
 tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml | 1 +
 6 files changed, 9 insertions(+), 17 deletions(-)

diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
index 2e6cc79764..5c5f0b84cb 100644
--- a/config/manager/kustomization.yaml
+++ b/config/manager/kustomization.yaml
@@ -1,2 +1,2 @@
 resources:
-- manager.yaml
\ No newline at end of file
+- manager.yaml
diff --git a/kuttl-test-upgrade.yaml b/kuttl-test-upgrade.yaml
index 043b455a7e..fb9c4f6781 100644
--- a/kuttl-test-upgrade.yaml
+++ b/kuttl-test-upgrade.yaml
@@ -1,5 +1,6 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestSuite
+crdDir: ./tests/_build/crds/
 artifactsDir: ./tests/_build/artifacts/
 kindContainers:
   - local/opentelemetry-operator:e2e
@@ -8,6 +9,7 @@ commands:
   - command: make cert-manager
   - command: kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.49.0/opentelemetry-operator.yaml
   - command: kubectl rollout status -w deployment/opentelemetry-operator-controller-manager -n opentelemetry-operator-system
+  - command: sleep 60
 testDirs:
   - ./tests/e2e-upgrade/
 timeout: 150
diff --git a/pkg/collector/reconcile/daemonset.go b/pkg/collector/reconcile/daemonset.go
index 4f9909fd65..31a5b6cf22 100644
--- a/pkg/collector/reconcile/daemonset.go
+++ b/pkg/collector/reconcile/daemonset.go
@@ -22,7 +22,6 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -79,9 +78,7 @@ func expectedDaemonSets(ctx context.Context, params Params, expected []appsv1.Da
 		if !apiequality.Semantic.DeepEqual(desired.Spec.Selector, existing.Spec.Selector) {
 			params.Log.V(2).Info("Spec.Selector change detected, trying to delete and re-create", "daemonset.name", existing.Name, "daemonset.namespace", existing.Namespace)
 
-			do := &client.DeleteOptions{}
-			client.PropagationPolicy(metav1.DeletePropagationForeground).ApplyToDelete(do)
-			if err := params.Client.Delete(ctx, existing, do); err != nil {
+			if err := params.Client.Delete(ctx, existing); err != nil {
 				return fmt.Errorf("failed to request deleting daemonset: %w", err)
 			}
 			if err := waitDaemonSetDeleted(ctx, params, nns); err != nil {
diff --git a/pkg/collector/reconcile/deployment.go b/pkg/collector/reconcile/deployment.go
index e3c08428a7..3971cf7d0c 100644
--- a/pkg/collector/reconcile/deployment.go
+++ b/pkg/collector/reconcile/deployment.go
@@ -19,10 +19,8 @@ import (
 	"fmt"
 	"time"
 
-	apiequality "k8s.io/apimachinery/pkg/api/equality"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	appsv1 "k8s.io/api/apps/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -86,9 +84,7 @@ func expectedDeployments(ctx context.Context, params Params, expected []appsv1.D
 		if !apiequality.Semantic.DeepEqual(desired.Spec.Selector, existing.Spec.Selector) {
 			params.Log.V(2).Info("Spec.Selector change detected, trying to delete and re-create", "deployment.name", existing.Name, "deployment.namespace", existing.Namespace)
 
-			do := &client.DeleteOptions{}
-			client.PropagationPolicy(metav1.DeletePropagationForeground).ApplyToDelete(do)
-			if err := params.Client.Delete(ctx, existing, do); err != nil {
+			if err := params.Client.Delete(ctx, existing); err != nil {
 				return fmt.Errorf("failed to delete deployment: %w", err)
 			}
 			if err := waitDeploymentDeleted(ctx, params, nns); err != nil {
diff --git a/pkg/collector/reconcile/statefulset.go b/pkg/collector/reconcile/statefulset.go
index 7356dbf0f6..0ec58a4214 100644
--- a/pkg/collector/reconcile/statefulset.go
+++ b/pkg/collector/reconcile/statefulset.go
@@ -19,10 +19,8 @@ import (
 	"fmt"
 	"time"
 
-	apiequality "k8s.io/apimachinery/pkg/api/equality"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	appsv1 "k8s.io/api/apps/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -81,9 +79,7 @@ func expectedStatefulSets(ctx context.Context, params Params, expected []appsv1.
 		if !apiequality.Semantic.DeepEqual(desired.Spec.Selector, existing.Spec.Selector) {
 			params.Log.V(2).Info("Spec.Selector change detected, trying to delete and re-create", "statefulset.name", existing.Name, "statefulset.namespace", existing.Namespace)
 
-			do := &client.DeleteOptions{}
-			client.PropagationPolicy(metav1.DeletePropagationForeground).ApplyToDelete(do)
-			if err := params.Client.Delete(ctx, existing, do); err != nil {
+			if err := params.Client.Delete(ctx, existing); err != nil {
 				return fmt.Errorf("failed to delete statefulset: %w", err)
 			}
 			if err := waitStatefulSetDeleted(ctx, params, nns); err != nil {
diff --git a/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml b/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml
index 33adeb09ed..cdd592303d 100644
--- a/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml
+++ b/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml
@@ -3,3 +3,4 @@ kind: TestStep
 commands:
   - command: kubectl apply -f ../../_build/manifests/01-opentelemetry-operator.yaml
   - command: kubectl rollout status -w deployment/opentelemetry-operator-controller-manager -n opentelemetry-operator-system
+  - command: sleep 60

From 2a6c7300c7fc732d58e8f35deaf6c8cb517a1293 Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Fri, 30 Sep 2022 17:44:21 -0700
Subject: [PATCH 08/18] Add unit test.

---
 pkg/collector/reconcile/daemonset_test.go   | 19 +++++++++++++++++++
 pkg/collector/reconcile/deployment_test.go  | 19 +++++++++++++++++++
 pkg/collector/reconcile/statefulset_test.go | 19 +++++++++++++++++++
 3 files changed, 57 insertions(+)

diff --git a/pkg/collector/reconcile/daemonset_test.go b/pkg/collector/reconcile/daemonset_test.go
index 8a4751e8b6..1117358698 100644
--- a/pkg/collector/reconcile/daemonset_test.go
+++ b/pkg/collector/reconcile/daemonset_test.go
@@ -130,4 +130,23 @@ func TestExpectedDaemonsets(t *testing.T) {
 		assert.True(t, exists)
 
 	})
+
+	t.Run("change Spec.Selector should recreate daemonset", func(t *testing.T) {
+
+		oldDs := collector.DaemonSet(param.Config, logger, param.Instance)
+		oldDs.Spec.Selector.MatchLabels["app.kubernetes.io/version"] = "latest"
+		oldDs.Spec.Template.Labels["app.kubernetes.io/version"] = "latest"
+		err := expectedDaemonSets(context.Background(), param, []v1.DaemonSet{expectedDs})
+		assert.NoError(t, err)
+
+		err = expectedDaemonSets(context.Background(), param, []v1.DaemonSet{expectedDs})
+		assert.NoError(t, err)
+
+		actual := v1.DaemonSet{}
+		exists, err := populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: "test-collector"})
+
+		assert.NoError(t, err)
+		assert.True(t, exists)
+		assert.Equal(t, expectedDs.Spec.Selector.MatchLabels, actual.Spec.Selector.MatchLabels)
+	})
 }
diff --git a/pkg/collector/reconcile/deployment_test.go b/pkg/collector/reconcile/deployment_test.go
index d5d794d889..2c0f034240 100644
--- a/pkg/collector/reconcile/deployment_test.go
+++ b/pkg/collector/reconcile/deployment_test.go
@@ -342,6 +342,25 @@ func TestExpectedDeployments(t *testing.T) {
 		assert.True(t, exists)
 
 	})
+
+	t.Run("change Spec.Selector should recreate deployment", func(t *testing.T) {
+
+		oldDeploy := collector.Deployment(param.Config, logger, param.Instance)
+		oldDeploy.Spec.Selector.MatchLabels["app.kubernetes.io/version"] = "latest"
+		oldDeploy.Spec.Template.Labels["app.kubernetes.io/version"] = "latest"
+		err := expectedDeployments(context.Background(), param, []v1.Deployment{oldDeploy})
+		assert.NoError(t, err)
+
+		err = expectedDeployments(context.Background(), param, []v1.Deployment{expectedDeploy})
+		assert.NoError(t, err)
+
+		actual := v1.Deployment{}
+		exists, err := populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: "test-collector"})
+
+		assert.NoError(t, err)
+		assert.True(t, exists)
+		assert.Equal(t, expectedDeploy.Spec.Selector.MatchLabels, actual.Spec.Selector.MatchLabels)
+	})
 }
 
 func TestCurrentReplicasWithHPA(t *testing.T) {
diff --git a/pkg/collector/reconcile/statefulset_test.go b/pkg/collector/reconcile/statefulset_test.go
index 4a0de27f27..d87139552b 100644
--- a/pkg/collector/reconcile/statefulset_test.go
+++ b/pkg/collector/reconcile/statefulset_test.go
@@ -132,4 +132,23 @@ func TestExpectedStatefulsets(t *testing.T) {
 		assert.True(t, exists)
 
 	})
+
+	t.Run("change Spec.Selector should recreate statefulset", func(t *testing.T) {
+
+		oldSs := collector.StatefulSet(param.Config, logger, param.Instance)
+		oldSs.Spec.Selector.MatchLabels["app.kubernetes.io/version"] = "latest"
+		oldSs.Spec.Template.Labels["app.kubernetes.io/version"] = "latest"
+		err := expectedStatefulSets(context.Background(), param, []v1.StatefulSet{oldSs})
+		assert.NoError(t, err)
+
+		err = expectedStatefulSets(context.Background(), param, []v1.StatefulSet{expectedSs})
+		assert.NoError(t, err)
+
+		actual := v1.StatefulSet{}
+		exists, err := populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: "test-collector"})
+
+		assert.NoError(t, err)
+		assert.True(t, exists)
+		assert.Equal(t, expectedSs.Spec.Selector.MatchLabels, actual.Spec.Selector.MatchLabels)
+	})
 }

From 30fa35a265a52f793258b749e63154233fcaf963 Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Fri, 30 Sep 2022 17:53:09 -0700
Subject: [PATCH 09/18] revert accident change.

---
 config/manager/kustomization.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
index 5c5f0b84cb..2e6cc79764 100644
--- a/config/manager/kustomization.yaml
+++ b/config/manager/kustomization.yaml
@@ -1,2 +1,2 @@
 resources:
-- manager.yaml
+- manager.yaml
\ No newline at end of file

From dbc4074ebbc44179e08edf00220c84fc9575ea31 Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Sun, 2 Oct 2022 23:24:44 -0700
Subject: [PATCH 10/18] remove timeout wait, don't 2 changes in one reconcile
 cycle.

---
 config/manager/kustomization.yaml           |  2 +-
 pkg/collector/reconcile/daemonset.go        |  8 +-----
 pkg/collector/reconcile/daemonset_test.go   |  9 +++++--
 pkg/collector/reconcile/deployment.go       | 28 +--------------------
 pkg/collector/reconcile/deployment_test.go  |  8 +++++-
 pkg/collector/reconcile/statefulset.go      |  8 +-----
 pkg/collector/reconcile/statefulset_test.go |  7 +++++-
 7 files changed, 24 insertions(+), 46 deletions(-)

diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
index 2e6cc79764..5c5f0b84cb 100644
--- a/config/manager/kustomization.yaml
+++ b/config/manager/kustomization.yaml
@@ -1,2 +1,2 @@
 resources:
-- manager.yaml
\ No newline at end of file
+- manager.yaml
diff --git a/pkg/collector/reconcile/daemonset.go b/pkg/collector/reconcile/daemonset.go
index 31a5b6cf22..4765f35caf 100644
--- a/pkg/collector/reconcile/daemonset.go
+++ b/pkg/collector/reconcile/daemonset.go
@@ -76,17 +76,11 @@ func expectedDaemonSets(ctx context.Context, params Params, expected []appsv1.Da
 
 		// Selector is an immutable field, if set, we cannot modify it otherwise we will face reconciliation error.
 		if !apiequality.Semantic.DeepEqual(desired.Spec.Selector, existing.Spec.Selector) {
-			params.Log.V(2).Info("Spec.Selector change detected, trying to delete and re-create", "daemonset.name", existing.Name, "daemonset.namespace", existing.Namespace)
+			params.Log.V(2).Info("Spec.Selector change detected, trying to delete, the new collector daemonset will be created in the next reconcile cycle", "daemonset.name", existing.Name, "daemonset.namespace", existing.Namespace)
 
 			if err := params.Client.Delete(ctx, existing); err != nil {
 				return fmt.Errorf("failed to request deleting daemonset: %w", err)
 			}
-			if err := waitDaemonSetDeleted(ctx, params, nns); err != nil {
-				return fmt.Errorf("failed to delete daemonset: %w", err)
-			}
-			if err := params.Client.Create(ctx, &desired); err != nil {
-				return fmt.Errorf("failed to create new daemonset: %w", err)
-			}
 			continue
 		}
 
diff --git a/pkg/collector/reconcile/daemonset_test.go b/pkg/collector/reconcile/daemonset_test.go
index 1117358698..15ab16762c 100644
--- a/pkg/collector/reconcile/daemonset_test.go
+++ b/pkg/collector/reconcile/daemonset_test.go
@@ -136,14 +136,19 @@ func TestExpectedDaemonsets(t *testing.T) {
 		oldDs := collector.DaemonSet(param.Config, logger, param.Instance)
 		oldDs.Spec.Selector.MatchLabels["app.kubernetes.io/version"] = "latest"
 		oldDs.Spec.Template.Labels["app.kubernetes.io/version"] = "latest"
-		err := expectedDaemonSets(context.Background(), param, []v1.DaemonSet{expectedDs})
+		err := expectedDaemonSets(context.Background(), param, []v1.DaemonSet{oldDs})
 		assert.NoError(t, err)
 
+		err = expectedDaemonSets(context.Background(), param, []v1.DaemonSet{expectedDs})
+		exists, err := populateObjectIfExists(t, &v1.DaemonSet{}, types.NamespacedName{Namespace: "default", Name: "test-collector"})
+		assert.NoError(t, err)
+		assert.False(t, exists)
+
 		err = expectedDaemonSets(context.Background(), param, []v1.DaemonSet{expectedDs})
 		assert.NoError(t, err)
 
 		actual := v1.DaemonSet{}
-		exists, err := populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: "test-collector"})
+		exists, err = populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: "test-collector"})
 
 		assert.NoError(t, err)
 		assert.True(t, exists)
diff --git a/pkg/collector/reconcile/deployment.go b/pkg/collector/reconcile/deployment.go
index 3971cf7d0c..5f54ae3b83 100644
--- a/pkg/collector/reconcile/deployment.go
+++ b/pkg/collector/reconcile/deployment.go
@@ -17,8 +17,6 @@ package reconcile
 import (
 	"context"
 	"fmt"
-	"time"
-
 	appsv1 "k8s.io/api/apps/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -82,17 +80,11 @@ func expectedDeployments(ctx context.Context, params Params, expected []appsv1.D
 
 		// Selector is an immutable field, if set, we cannot modify it otherwise we will face reconciliation error.
 		if !apiequality.Semantic.DeepEqual(desired.Spec.Selector, existing.Spec.Selector) {
-			params.Log.V(2).Info("Spec.Selector change detected, trying to delete and re-create", "deployment.name", existing.Name, "deployment.namespace", existing.Namespace)
+			params.Log.V(2).Info("Spec.Selector change detected, trying to delete, the new collector deployment will be created in the next reconcile cycle ", "deployment.name", existing.Name, "deployment.namespace", existing.Namespace)
 
 			if err := params.Client.Delete(ctx, existing); err != nil {
 				return fmt.Errorf("failed to delete deployment: %w", err)
 			}
-			if err := waitDeploymentDeleted(ctx, params, nns); err != nil {
-				return fmt.Errorf("failed to delete deployment: %w", err)
-			}
-			if err := params.Client.Create(ctx, &desired); err != nil {
-				return fmt.Errorf("failed to create new deployment: %w", err)
-			}
 			continue
 		}
 
@@ -173,21 +165,3 @@ func currentReplicasWithHPA(spec v1alpha1.OpenTelemetryCollectorSpec, curr int32
 
 	return curr
 }
-
-func waitDeploymentDeleted(ctx context.Context, params Params, nns types.NamespacedName) error {
-	for i := 0; i < DeleteDeploymentWaitTimeOutInSeconds; i++ {
-		existing := &appsv1.Deployment{}
-		err := params.Client.Get(ctx, nns, existing)
-		if err != nil && k8serrors.IsNotFound(err) {
-			return nil
-		} else if err != nil {
-			return err
-		}
-		select {
-		case <-ctx.Done():
-			return fmt.Errorf("wait Interrupted for updating deployment.name %s, deployment.namespace %s", nns.Name, nns.Namespace)
-		case <-time.After(time.Second):
-		}
-	}
-	return fmt.Errorf("wait timeout for updating deployment.name %s, deployment.namespace %s", nns.Name, nns.Namespace)
-}
diff --git a/pkg/collector/reconcile/deployment_test.go b/pkg/collector/reconcile/deployment_test.go
index 2c0f034240..ca562a6b34 100644
--- a/pkg/collector/reconcile/deployment_test.go
+++ b/pkg/collector/reconcile/deployment_test.go
@@ -351,11 +351,17 @@ func TestExpectedDeployments(t *testing.T) {
 		err := expectedDeployments(context.Background(), param, []v1.Deployment{oldDeploy})
 		assert.NoError(t, err)
 
+		err = expectedDeployments(context.Background(), param, []v1.Deployment{expectedDeploy})
+		assert.NoError(t, err)
+		exists, err := populateObjectIfExists(t, &v1.Deployment{}, types.NamespacedName{Namespace: "default", Name: "test-collector"})
+		assert.NoError(t, err)
+		assert.False(t, exists)
+
 		err = expectedDeployments(context.Background(), param, []v1.Deployment{expectedDeploy})
 		assert.NoError(t, err)
 
 		actual := v1.Deployment{}
-		exists, err := populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: "test-collector"})
+		exists, err = populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: "test-collector"})
 
 		assert.NoError(t, err)
 		assert.True(t, exists)
diff --git a/pkg/collector/reconcile/statefulset.go b/pkg/collector/reconcile/statefulset.go
index 0ec58a4214..e053c980c6 100644
--- a/pkg/collector/reconcile/statefulset.go
+++ b/pkg/collector/reconcile/statefulset.go
@@ -77,17 +77,11 @@ func expectedStatefulSets(ctx context.Context, params Params, expected []appsv1.
 
 		// Selector is an immutable field, if set, we cannot modify it otherwise we will face reconciliation error.
 		if !apiequality.Semantic.DeepEqual(desired.Spec.Selector, existing.Spec.Selector) {
-			params.Log.V(2).Info("Spec.Selector change detected, trying to delete and re-create", "statefulset.name", existing.Name, "statefulset.namespace", existing.Namespace)
+			params.Log.V(2).Info("Spec.Selector change detected, trying to delete, the new collector statfulset will be created in the next reconcile cycle", "statefulset.name", existing.Name, "statefulset.namespace", existing.Namespace)
 
 			if err := params.Client.Delete(ctx, existing); err != nil {
 				return fmt.Errorf("failed to delete statefulset: %w", err)
 			}
-			if err := waitStatefulSetDeleted(ctx, params, nns); err != nil {
-				return fmt.Errorf("failed to delete statefulset: %w", err)
-			}
-			if err := params.Client.Create(ctx, &desired); err != nil {
-				return fmt.Errorf("failed to create new statefulset: %w", err)
-			}
 			continue
 		}
 
diff --git a/pkg/collector/reconcile/statefulset_test.go b/pkg/collector/reconcile/statefulset_test.go
index d87139552b..2cfc871c84 100644
--- a/pkg/collector/reconcile/statefulset_test.go
+++ b/pkg/collector/reconcile/statefulset_test.go
@@ -141,11 +141,16 @@ func TestExpectedStatefulsets(t *testing.T) {
 		err := expectedStatefulSets(context.Background(), param, []v1.StatefulSet{oldSs})
 		assert.NoError(t, err)
 
+		err = expectedStatefulSets(context.Background(), param, []v1.StatefulSet{expectedSs})
+		exists, err := populateObjectIfExists(t, &v1.StatefulSet{}, types.NamespacedName{Namespace: "default", Name: "test-collector"})
+		assert.NoError(t, err)
+		assert.False(t, exists)
+
 		err = expectedStatefulSets(context.Background(), param, []v1.StatefulSet{expectedSs})
 		assert.NoError(t, err)
 
 		actual := v1.StatefulSet{}
-		exists, err := populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: "test-collector"})
+		exists, err = populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: "test-collector"})
 
 		assert.NoError(t, err)
 		assert.True(t, exists)

From 3c2c6e98fb1b977b9faa30700d126a83b5eb372d Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Mon, 3 Oct 2022 02:41:08 -0700
Subject: [PATCH 11/18] fix lint

---
 pkg/collector/reconcile/daemonset.go        | 19 -------------------
 pkg/collector/reconcile/daemonset_test.go   |  2 ++
 pkg/collector/reconcile/deployment.go       |  1 +
 pkg/collector/reconcile/statefulset.go      | 19 -------------------
 pkg/collector/reconcile/statefulset_test.go |  2 ++
 5 files changed, 5 insertions(+), 38 deletions(-)

diff --git a/pkg/collector/reconcile/daemonset.go b/pkg/collector/reconcile/daemonset.go
index 4765f35caf..001629c09f 100644
--- a/pkg/collector/reconcile/daemonset.go
+++ b/pkg/collector/reconcile/daemonset.go
@@ -17,7 +17,6 @@ package reconcile
 import (
 	"context"
 	"fmt"
-	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
@@ -147,21 +146,3 @@ func deleteDaemonSets(ctx context.Context, params Params, expected []appsv1.Daem
 
 	return nil
 }
-
-func waitDaemonSetDeleted(ctx context.Context, params Params, nns types.NamespacedName) error {
-	for i := 0; i < DeleteDaemonSetWaitTimeOutInSeconds; i++ {
-		existing := &appsv1.DaemonSet{}
-		err := params.Client.Get(ctx, nns, existing)
-		if err != nil && k8serrors.IsNotFound(err) {
-			return nil
-		} else if err != nil {
-			return err
-		}
-		select {
-		case <-ctx.Done():
-			return fmt.Errorf("wait Interrupted for updating daemonset.name %s, daemonset.namespace %s", nns.Name, nns.Namespace)
-		case <-time.After(time.Second):
-		}
-	}
-	return fmt.Errorf("wait timeout for updating daemonset.name %s, daemonset.namespace %s", nns.Name, nns.Namespace)
-}
diff --git a/pkg/collector/reconcile/daemonset_test.go b/pkg/collector/reconcile/daemonset_test.go
index 15ab16762c..d81c05f488 100644
--- a/pkg/collector/reconcile/daemonset_test.go
+++ b/pkg/collector/reconcile/daemonset_test.go
@@ -140,6 +140,8 @@ func TestExpectedDaemonsets(t *testing.T) {
 		assert.NoError(t, err)
 
 		err = expectedDaemonSets(context.Background(), param, []v1.DaemonSet{expectedDs})
+		assert.NoError(t, err)
+
 		exists, err := populateObjectIfExists(t, &v1.DaemonSet{}, types.NamespacedName{Namespace: "default", Name: "test-collector"})
 		assert.NoError(t, err)
 		assert.False(t, exists)
diff --git a/pkg/collector/reconcile/deployment.go b/pkg/collector/reconcile/deployment.go
index 5f54ae3b83..cc4386f09f 100644
--- a/pkg/collector/reconcile/deployment.go
+++ b/pkg/collector/reconcile/deployment.go
@@ -17,6 +17,7 @@ package reconcile
 import (
 	"context"
 	"fmt"
+
 	appsv1 "k8s.io/api/apps/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
diff --git a/pkg/collector/reconcile/statefulset.go b/pkg/collector/reconcile/statefulset.go
index e053c980c6..e86083e3c1 100644
--- a/pkg/collector/reconcile/statefulset.go
+++ b/pkg/collector/reconcile/statefulset.go
@@ -17,7 +17,6 @@ package reconcile
 import (
 	"context"
 	"fmt"
-	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
@@ -148,21 +147,3 @@ func deleteStatefulSets(ctx context.Context, params Params, expected []appsv1.St
 
 	return nil
 }
-
-func waitStatefulSetDeleted(ctx context.Context, params Params, nns types.NamespacedName) error {
-	for i := 0; i < DeleteStatefulSetWaitTimeOutInSeconds; i++ {
-		existing := &appsv1.StatefulSet{}
-		err := params.Client.Get(ctx, nns, existing)
-		if err != nil && k8serrors.IsNotFound(err) {
-			return nil
-		} else if err != nil {
-			return err
-		}
-		select {
-		case <-ctx.Done():
-			return fmt.Errorf("wait Interrupted for updating statefulset.name %s, statefulset.namespace %s", nns.Name, nns.Namespace)
-		case <-time.After(time.Second):
-		}
-	}
-	return fmt.Errorf("wait timeout for updating statefulset.name %s, statefulset.namespace: %s", nns.Name, nns.Namespace)
-}
diff --git a/pkg/collector/reconcile/statefulset_test.go b/pkg/collector/reconcile/statefulset_test.go
index 2cfc871c84..e6f0e30d0a 100644
--- a/pkg/collector/reconcile/statefulset_test.go
+++ b/pkg/collector/reconcile/statefulset_test.go
@@ -142,6 +142,8 @@ func TestExpectedStatefulsets(t *testing.T) {
 		assert.NoError(t, err)
 
 		err = expectedStatefulSets(context.Background(), param, []v1.StatefulSet{expectedSs})
+		assert.NoError(t, err)
+
 		exists, err := populateObjectIfExists(t, &v1.StatefulSet{}, types.NamespacedName{Namespace: "default", Name: "test-collector"})
 		assert.NoError(t, err)
 		assert.False(t, exists)

From ce1b940a8f74c65a1300edcb517ddf977242597d Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Mon, 3 Oct 2022 03:37:01 -0700
Subject: [PATCH 12/18] fix test, change name to isolate it from other tests.

---
 pkg/collector/reconcile/daemonset_test.go   | 20 ++++++++++++--------
 pkg/collector/reconcile/deployment_test.go  | 19 ++++++++++++-------
 pkg/collector/reconcile/statefulset_test.go | 20 ++++++++++++--------
 3 files changed, 36 insertions(+), 23 deletions(-)

diff --git a/pkg/collector/reconcile/daemonset_test.go b/pkg/collector/reconcile/daemonset_test.go
index d81c05f488..02d8d0c8aa 100644
--- a/pkg/collector/reconcile/daemonset_test.go
+++ b/pkg/collector/reconcile/daemonset_test.go
@@ -136,24 +136,28 @@ func TestExpectedDaemonsets(t *testing.T) {
 		oldDs := collector.DaemonSet(param.Config, logger, param.Instance)
 		oldDs.Spec.Selector.MatchLabels["app.kubernetes.io/version"] = "latest"
 		oldDs.Spec.Template.Labels["app.kubernetes.io/version"] = "latest"
+		oldDs.Name = "update-ds"
+
 		err := expectedDaemonSets(context.Background(), param, []v1.DaemonSet{oldDs})
 		assert.NoError(t, err)
-
-		err = expectedDaemonSets(context.Background(), param, []v1.DaemonSet{expectedDs})
+		exists, err := populateObjectIfExists(t, &v1.DaemonSet{}, types.NamespacedName{Namespace: "default", Name: oldDs.Name})
 		assert.NoError(t, err)
+		assert.True(t, exists)
 
-		exists, err := populateObjectIfExists(t, &v1.DaemonSet{}, types.NamespacedName{Namespace: "default", Name: "test-collector"})
+		newDs := collector.DaemonSet(param.Config, logger, param.Instance)
+		newDs.Name = oldDs.Name
+		err = expectedDaemonSets(context.Background(), param, []v1.DaemonSet{newDs})
+		assert.NoError(t, err)
+		exists, err = populateObjectIfExists(t, &v1.DaemonSet{}, types.NamespacedName{Namespace: "default", Name: oldDs.Name})
 		assert.NoError(t, err)
 		assert.False(t, exists)
 
-		err = expectedDaemonSets(context.Background(), param, []v1.DaemonSet{expectedDs})
+		err = expectedDaemonSets(context.Background(), param, []v1.DaemonSet{newDs})
 		assert.NoError(t, err)
-
 		actual := v1.DaemonSet{}
-		exists, err = populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: "test-collector"})
-
+		exists, err = populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: oldDs.Name})
 		assert.NoError(t, err)
 		assert.True(t, exists)
-		assert.Equal(t, expectedDs.Spec.Selector.MatchLabels, actual.Spec.Selector.MatchLabels)
+		assert.Equal(t, newDs.Spec.Selector.MatchLabels, actual.Spec.Selector.MatchLabels)
 	})
 }
diff --git a/pkg/collector/reconcile/deployment_test.go b/pkg/collector/reconcile/deployment_test.go
index ca562a6b34..ab146b469a 100644
--- a/pkg/collector/reconcile/deployment_test.go
+++ b/pkg/collector/reconcile/deployment_test.go
@@ -348,24 +348,29 @@ func TestExpectedDeployments(t *testing.T) {
 		oldDeploy := collector.Deployment(param.Config, logger, param.Instance)
 		oldDeploy.Spec.Selector.MatchLabels["app.kubernetes.io/version"] = "latest"
 		oldDeploy.Spec.Template.Labels["app.kubernetes.io/version"] = "latest"
+		oldDeploy.Name = "update-deploy"
+
 		err := expectedDeployments(context.Background(), param, []v1.Deployment{oldDeploy})
 		assert.NoError(t, err)
+		exists, err := populateObjectIfExists(t, &v1.Deployment{}, types.NamespacedName{Namespace: "default", Name: oldDeploy.Name})
+		assert.NoError(t, err)
+		assert.True(t, exists)
 
-		err = expectedDeployments(context.Background(), param, []v1.Deployment{expectedDeploy})
+		newDeploy := collector.Deployment(param.Config, logger, param.Instance)
+		newDeploy.Name = oldDeploy.Name
+		err = expectedDeployments(context.Background(), param, []v1.Deployment{newDeploy})
 		assert.NoError(t, err)
-		exists, err := populateObjectIfExists(t, &v1.Deployment{}, types.NamespacedName{Namespace: "default", Name: "test-collector"})
+		exists, err = populateObjectIfExists(t, &v1.Deployment{}, types.NamespacedName{Namespace: "default", Name: oldDeploy.Name})
 		assert.NoError(t, err)
 		assert.False(t, exists)
 
-		err = expectedDeployments(context.Background(), param, []v1.Deployment{expectedDeploy})
+		err = expectedDeployments(context.Background(), param, []v1.Deployment{newDeploy})
 		assert.NoError(t, err)
-
 		actual := v1.Deployment{}
-		exists, err = populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: "test-collector"})
-
+		exists, err = populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: oldDeploy.Name})
 		assert.NoError(t, err)
 		assert.True(t, exists)
-		assert.Equal(t, expectedDeploy.Spec.Selector.MatchLabels, actual.Spec.Selector.MatchLabels)
+		assert.Equal(t, newDeploy.Spec.Selector.MatchLabels, actual.Spec.Selector.MatchLabels)
 	})
 }
 
diff --git a/pkg/collector/reconcile/statefulset_test.go b/pkg/collector/reconcile/statefulset_test.go
index e6f0e30d0a..bf83534295 100644
--- a/pkg/collector/reconcile/statefulset_test.go
+++ b/pkg/collector/reconcile/statefulset_test.go
@@ -138,24 +138,28 @@ func TestExpectedStatefulsets(t *testing.T) {
 		oldSs := collector.StatefulSet(param.Config, logger, param.Instance)
 		oldSs.Spec.Selector.MatchLabels["app.kubernetes.io/version"] = "latest"
 		oldSs.Spec.Template.Labels["app.kubernetes.io/version"] = "latest"
+		oldSs.Name = "update-ss"
+
 		err := expectedStatefulSets(context.Background(), param, []v1.StatefulSet{oldSs})
 		assert.NoError(t, err)
-
-		err = expectedStatefulSets(context.Background(), param, []v1.StatefulSet{expectedSs})
+		exists, err := populateObjectIfExists(t, &v1.StatefulSet{}, types.NamespacedName{Namespace: "default", Name: oldSs.Name})
 		assert.NoError(t, err)
+		assert.True(t, exists)
 
-		exists, err := populateObjectIfExists(t, &v1.StatefulSet{}, types.NamespacedName{Namespace: "default", Name: "test-collector"})
+		newSs := collector.StatefulSet(param.Config, logger, param.Instance)
+		newSs.Name = oldSs.Name
+		err = expectedStatefulSets(context.Background(), param, []v1.StatefulSet{newSs})
+		assert.NoError(t, err)
+		exists, err = populateObjectIfExists(t, &v1.StatefulSet{}, types.NamespacedName{Namespace: "default", Name: oldSs.Name})
 		assert.NoError(t, err)
 		assert.False(t, exists)
 
-		err = expectedStatefulSets(context.Background(), param, []v1.StatefulSet{expectedSs})
+		err = expectedStatefulSets(context.Background(), param, []v1.StatefulSet{newSs})
 		assert.NoError(t, err)
-
 		actual := v1.StatefulSet{}
-		exists, err = populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: "test-collector"})
-
+		exists, err = populateObjectIfExists(t, &actual, types.NamespacedName{Namespace: "default", Name: oldSs.Name})
 		assert.NoError(t, err)
 		assert.True(t, exists)
-		assert.Equal(t, expectedSs.Spec.Selector.MatchLabels, actual.Spec.Selector.MatchLabels)
+		assert.Equal(t, newSs.Spec.Selector.MatchLabels, actual.Spec.Selector.MatchLabels)
 	})
 }

From db8da58a8596bd8d47dbc2b5e40198326c87d30a Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Mon, 3 Oct 2022 09:19:30 -0700
Subject: [PATCH 13/18] revert accident change.

---
 config/manager/kustomization.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
index 5c5f0b84cb..2e6cc79764 100644
--- a/config/manager/kustomization.yaml
+++ b/config/manager/kustomization.yaml
@@ -1,2 +1,2 @@
 resources:
-- manager.yaml
+- manager.yaml
\ No newline at end of file

From 0b4d703c28e5e331b9cb1d075a6a3d0123b8b298 Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Mon, 3 Oct 2022 09:27:14 -0700
Subject: [PATCH 14/18] remove unsed constant.

---
 pkg/collector/reconcile/daemonset.go   | 2 --
 pkg/collector/reconcile/deployment.go  | 2 --
 pkg/collector/reconcile/statefulset.go | 2 --
 3 files changed, 6 deletions(-)

diff --git a/pkg/collector/reconcile/daemonset.go b/pkg/collector/reconcile/daemonset.go
index 001629c09f..a14649c59c 100644
--- a/pkg/collector/reconcile/daemonset.go
+++ b/pkg/collector/reconcile/daemonset.go
@@ -28,8 +28,6 @@ import (
 	"github.com/open-telemetry/opentelemetry-operator/pkg/collector"
 )
 
-const DeleteDaemonSetWaitTimeOutInSeconds = 300
-
 // +kubebuilder:rbac:groups="apps",resources=daemonsets,verbs=get;list;watch;create;update;patch;delete
 
 // DaemonSets reconciles the daemon set(s) required for the instance in the current context.
diff --git a/pkg/collector/reconcile/deployment.go b/pkg/collector/reconcile/deployment.go
index cc4386f09f..5e4b4bd18d 100644
--- a/pkg/collector/reconcile/deployment.go
+++ b/pkg/collector/reconcile/deployment.go
@@ -32,8 +32,6 @@ import (
 
 // +kubebuilder:rbac:groups="apps",resources=deployments,verbs=get;list;watch;create;update;patch;delete
 
-const DeleteDeploymentWaitTimeOutInSeconds = 300
-
 // Deployments reconciles the deployment(s) required for the instance in the current context.
 func Deployments(ctx context.Context, params Params) error {
 	desired := []appsv1.Deployment{}
diff --git a/pkg/collector/reconcile/statefulset.go b/pkg/collector/reconcile/statefulset.go
index e86083e3c1..e663734329 100644
--- a/pkg/collector/reconcile/statefulset.go
+++ b/pkg/collector/reconcile/statefulset.go
@@ -30,8 +30,6 @@ import (
 
 // +kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list;watch;create;update;patch;delete
 
-const DeleteStatefulSetWaitTimeOutInSeconds = 300
-
 // StatefulSets reconciles the stateful set(s) required for the instance in the current context.
 func StatefulSets(ctx context.Context, params Params) error {
 

From 620f48606dee21550f6a4a3934baf5fc51eefe23 Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Mon, 10 Oct 2022 09:27:04 -0700
Subject: [PATCH 15/18] Add comment for explaining the upgrading e2e test.

---
 kuttl-test-upgrade.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/kuttl-test-upgrade.yaml b/kuttl-test-upgrade.yaml
index fb9c4f6781..133066fe57 100644
--- a/kuttl-test-upgrade.yaml
+++ b/kuttl-test-upgrade.yaml
@@ -1,3 +1,5 @@
+# Make sure that the OT operator after upgrading itself, can upgrade the OT collectors without error.
+# See issues 840 and 1117 for the bugs on upgrading OT operator.
 apiVersion: kuttl.dev/v1beta1
 kind: TestSuite
 crdDir: ./tests/_build/crds/

From 50db127e51417e6315b5a39f0b3184972f45ddad Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Mon, 10 Oct 2022 10:52:39 -0700
Subject: [PATCH 16/18] add e2e upgrade to workflow.

---
 .github/workflows/e2e.yaml                              | 2 +-
 kuttl-test-upgrade.yaml                                 | 2 +-
 tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 15b61ce402..e5af64f91b 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -35,4 +35,4 @@ jobs:
     - name: "run tests"
       env:
         KUBE_VERSION: ${{ matrix.kube-version }}
-      run: make prepare-e2e e2e KUBE_VERSION=$KUBE_VERSION
+      run: make prepare-e2e e2e e2e-upgrade KUBE_VERSION=$KUBE_VERSION
diff --git a/kuttl-test-upgrade.yaml b/kuttl-test-upgrade.yaml
index 133066fe57..38d84d79dd 100644
--- a/kuttl-test-upgrade.yaml
+++ b/kuttl-test-upgrade.yaml
@@ -11,7 +11,7 @@ commands:
   - command: make cert-manager
   - command: kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.49.0/opentelemetry-operator.yaml
   - command: kubectl rollout status -w deployment/opentelemetry-operator-controller-manager -n opentelemetry-operator-system
-  - command: sleep 60
+  - command: sleep 60s
 testDirs:
   - ./tests/e2e-upgrade/
 timeout: 150
diff --git a/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml b/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml
index cdd592303d..346d4facde 100644
--- a/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml
+++ b/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml
@@ -3,4 +3,4 @@ kind: TestStep
 commands:
   - command: kubectl apply -f ../../_build/manifests/01-opentelemetry-operator.yaml
   - command: kubectl rollout status -w deployment/opentelemetry-operator-controller-manager -n opentelemetry-operator-system
-  - command: sleep 60
+  - command: sleep 60s

From e2889f70a9ca8b62878e72632640915c9869b018 Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Tue, 11 Oct 2022 03:31:03 -0700
Subject: [PATCH 17/18] Address comment, adding detail comments for e2e upgrade
 test.

---
 apis/v1alpha1/zz_generated.deepcopy.go |  7 ++++++-
 kuttl-test-upgrade.yaml                | 11 ++++++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go
index f2b8397930..655948e0ea 100644
--- a/apis/v1alpha1/zz_generated.deepcopy.go
+++ b/apis/v1alpha1/zz_generated.deepcopy.go
@@ -33,6 +33,11 @@ func (in *AutoscalerSpec) DeepCopyInto(out *AutoscalerSpec) {
 		*out = new(v2.HorizontalPodAutoscalerBehavior)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.TargetCPUUtilization != nil {
+		in, out := &in.TargetCPUUtilization, &out.TargetCPUUtilization
+		*out = new(int32)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalerSpec.
@@ -434,12 +439,12 @@ func (in *OpenTelemetryCollectorStatus) DeepCopy() *OpenTelemetryCollectorStatus
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *OpenTelemetryTargetAllocator) DeepCopyInto(out *OpenTelemetryTargetAllocator) {
 	*out = *in
-	out.PrometheusCR = in.PrometheusCR
 	if in.Replicas != nil {
 		in, out := &in.Replicas, &out.Replicas
 		*out = new(int32)
 		**out = **in
 	}
+	out.PrometheusCR = in.PrometheusCR
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenTelemetryTargetAllocator.
diff --git a/kuttl-test-upgrade.yaml b/kuttl-test-upgrade.yaml
index 38d84d79dd..e77a2e2fae 100644
--- a/kuttl-test-upgrade.yaml
+++ b/kuttl-test-upgrade.yaml
@@ -1,5 +1,14 @@
 # Make sure that the OT operator after upgrading itself, can upgrade the OT collectors without error.
-# See issues 840 and 1117 for the bugs on upgrading OT operator.
+# The test is based on the version v0.49.0, a breaking change was introduced from PR
+# https://github.com/open-telemetry/opentelemetry-operator/pull/797, which added a version label "app.kubernetes.io/version",
+# The version label would change between OT operator upgrade, and since at the time, the collector pod selector was the same
+# as this labels, resulted in selector being modified during reconciliation which caused error due to the selector is immutable.
+# Please be aware of that the collector labels are changeable in various ways, so this issue may happen in any operator < v0.52.0
+# which changed the selector to be a static set of labels.
+# The fix for this issue including:
+# https://github.com/open-telemetry/opentelemetry-operator/issues/840, make the selector be a static set of labels;
+# https://github.com/open-telemetry/opentelemetry-operator/issues/1117, delete the old collector to let the operator
+# create a new one when the selector changed.
 apiVersion: kuttl.dev/v1beta1
 kind: TestSuite
 crdDir: ./tests/_build/crds/

From a10fa977be20c15a0bed9d2146c3ad104acbea22 Mon Sep 17 00:00:00 2001
From: "yuanfan.peng" <yuanpeng@cisco.com>
Date: Tue, 11 Oct 2022 04:50:15 -0700
Subject: [PATCH 18/18] rever accident change.

---
 apis/v1alpha1/zz_generated.deepcopy.go | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go
index 655948e0ea..f2b8397930 100644
--- a/apis/v1alpha1/zz_generated.deepcopy.go
+++ b/apis/v1alpha1/zz_generated.deepcopy.go
@@ -33,11 +33,6 @@ func (in *AutoscalerSpec) DeepCopyInto(out *AutoscalerSpec) {
 		*out = new(v2.HorizontalPodAutoscalerBehavior)
 		(*in).DeepCopyInto(*out)
 	}
-	if in.TargetCPUUtilization != nil {
-		in, out := &in.TargetCPUUtilization, &out.TargetCPUUtilization
-		*out = new(int32)
-		**out = **in
-	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalerSpec.
@@ -439,12 +434,12 @@ func (in *OpenTelemetryCollectorStatus) DeepCopy() *OpenTelemetryCollectorStatus
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *OpenTelemetryTargetAllocator) DeepCopyInto(out *OpenTelemetryTargetAllocator) {
 	*out = *in
+	out.PrometheusCR = in.PrometheusCR
 	if in.Replicas != nil {
 		in, out := &in.Replicas, &out.Replicas
 		*out = new(int32)
 		**out = **in
 	}
-	out.PrometheusCR = in.PrometheusCR
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenTelemetryTargetAllocator.