diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 55fa0e87..6862e135 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -19,8 +19,6 @@ jobs:
       - name: Setup Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v1
-        with:
-          buildkitd-flags: "--debug"
       - name: Restore Go cache
         uses: actions/cache@v1
         with:
@@ -92,6 +90,27 @@ jobs:
           make dev-deploy IMG=test/kustomize-controller:latest
           kubectl -n kustomize-system rollout status deploy/source-controller --timeout=1m
           kubectl -n kustomize-system rollout status deploy/kustomize-controller --timeout=1m
+      - name: Run tests for removing kubectl managed fields
+        run: |
+          kubectl create ns managed-fields
+          kustomize build github.com/stefanprodan/podinfo//kustomize?ref=6.0.0 > /tmp/podinfo.yaml
+          kubectl -n managed-fields apply -f /tmp/podinfo.yaml
+          kubectl -n managed-fields apply -f ./config/testdata/managed-fields
+          kubectl -n managed-fields wait kustomization/podinfo --for=condition=ready --timeout=4m
+          OUTDATA=$(kubectl -n managed-fields get deploy podinfo --show-managed-fields -oyaml)
+          if echo "$OUTDATA" | grep -q "kubectl";then
+            echo "kubectl client-side manager not removed"
+            exit 1
+          fi
+          kubectl -n managed-fields apply --server-side --force-conflicts -f /tmp/podinfo.yaml
+          kubectl -n managed-fields annotate --overwrite kustomization/podinfo reconcile.fluxcd.io/requestedAt="$(date +%s)"
+          kubectl -n managed-fields wait kustomization/podinfo --for=condition=ready --timeout=4m
+          OUTDATA=$(kubectl -n managed-fields get deploy podinfo --show-managed-fields -oyaml)
+          if echo "$OUTDATA" | grep -q "kubectl";then
+            echo "kubectl server-side manager not removed"
+            exit 1
+          fi
+          kubectl delete ns managed-fields
       - name: Run overlays tests
         run: |
           kubectl -n kustomize-system apply -k ./config/testdata/overlays
diff --git a/config/testdata/managed-fields/podinfo.yaml b/config/testdata/managed-fields/podinfo.yaml
new file mode 100644
index 00000000..1f065fc0
--- /dev/null
+++ b/config/testdata/managed-fields/podinfo.yaml
@@ -0,0 +1,23 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: podinfo
+spec:
+  interval: 15m
+  path: "./kustomize/"
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: podinfo
+  timeout: 1m
+  targetNamespace: managed-fields
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: GitRepository
+metadata:
+  name: podinfo
+spec:
+  interval: 5m
+  url: https://github.com/stefanprodan/podinfo
+  ref:
+    semver: "6.0.0"
diff --git a/controllers/kustomization_controller.go b/controllers/kustomization_controller.go
index 7f728c71..23bcbb2f 100644
--- a/controllers/kustomization_controller.go
+++ b/controllers/kustomization_controller.go
@@ -32,6 +32,7 @@ import (
 
 	securejoin "github.com/cyphar/filepath-securejoin"
 	"github.com/hashicorp/go-retryablehttp"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -682,6 +683,36 @@ func (r *KustomizationReconciler) apply(ctx context.Context, manager *ssa.Resour
 	applyOpts.Exclusions = map[string]string{
 		fmt.Sprintf("%s/reconcile", kustomizev1.GroupVersion.Group): kustomizev1.DisabledValue,
 	}
+	applyOpts.Cleanup = ssa.ApplyCleanupOptions{
+		Annotations: []string{
+			// remove the kubectl annotation
+			corev1.LastAppliedConfigAnnotation,
+			// remove deprecated fluxcd.io annotations
+			"kustomize.toolkit.fluxcd.io/checksum",
+			"fluxcd.io/sync-checksum",
+		},
+		Labels: []string{
+			// remove deprecated fluxcd.io labels
+			"fluxcd.io/sync-gc-mark",
+		},
+		FieldManagers: []ssa.FiledManager{
+			{
+				// to undo changes made with 'kubectl apply --server-side --force-conflicts'
+				Name:          "kubectl",
+				OperationType: metav1.ManagedFieldsOperationApply,
+			},
+			{
+				// to undo changes made with 'kubectl apply'
+				Name:          "kubectl",
+				OperationType: metav1.ManagedFieldsOperationUpdate,
+			},
+			{
+				// to undo changes made with 'kubectl apply'
+				Name:          "before-first-apply",
+				OperationType: metav1.ManagedFieldsOperationUpdate,
+			},
+		},
+	}
 
 	// contains only CRDs and Namespaces
 	var stageOne []*unstructured.Unstructured
@@ -897,7 +928,7 @@ func (r *KustomizationReconciler) finalize(ctx context.Context, kustomization ku
 
 	// Remove our finalizer from the list and update it
 	controllerutil.RemoveFinalizer(&kustomization, kustomizev1.KustomizationFinalizer)
-	if err := r.Update(ctx, &kustomization); err != nil {
+	if err := r.Update(ctx, &kustomization, client.FieldOwner(r.ControllerName)); err != nil {
 		return ctrl.Result{}, err
 	}
 
@@ -995,5 +1026,5 @@ func (r *KustomizationReconciler) patchStatus(ctx context.Context, req ctrl.Requ
 	patch := client.MergeFrom(kustomization.DeepCopy())
 	kustomization.Status = newStatus
 
-	return r.Status().Patch(ctx, &kustomization, patch)
+	return r.Status().Patch(ctx, &kustomization, patch, client.FieldOwner(r.ControllerName))
 }