diff --git a/config/rbac/imagejob_pods_cluster_role_binding.yaml b/config/rbac/cluster_role_binding.yaml
similarity index 66%
rename from config/rbac/imagejob_pods_cluster_role_binding.yaml
rename to config/rbac/cluster_role_binding.yaml
index d27033b304..2070ede446 100644
--- a/config/rbac/imagejob_pods_cluster_role_binding.yaml
+++ b/config/rbac/cluster_role_binding.yaml
@@ -1,12 +1,12 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: imagejob-pods-cluster-rolebinding
+  name: manager-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: imagejob-pods-cluster-role
+  name: manager-role
 subjects:
 - kind: ServiceAccount
-  name: imagejob-pods
+  name: controller-manager
   namespace: system
diff --git a/config/rbac/eraserconfig_editor_role.yaml b/config/rbac/eraserconfig_editor_role.yaml
deleted file mode 100644
index f4e162009c..0000000000
--- a/config/rbac/eraserconfig_editor_role.yaml
+++ /dev/null
@@ -1,31 +0,0 @@
-# permissions for end users to edit eraserconfigs.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    app.kubernetes.io/name: clusterrole
-    app.kubernetes.io/instance: eraserconfig-editor-role
-    app.kubernetes.io/component: rbac
-    app.kubernetes.io/created-by: eraser
-    app.kubernetes.io/part-of: eraser
-    app.kubernetes.io/managed-by: kustomize
-  name: eraserconfig-editor-role
-rules:
-- apiGroups:
-  - eraser.sh
-  resources:
-  - eraserconfigs
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - eraser.sh
-  resources:
-  - eraserconfigs/status
-  verbs:
-  - get
diff --git a/config/rbac/eraserconfig_viewer_role.yaml b/config/rbac/eraserconfig_viewer_role.yaml
deleted file mode 100644
index b3798179ed..0000000000
--- a/config/rbac/eraserconfig_viewer_role.yaml
+++ /dev/null
@@ -1,27 +0,0 @@
-# permissions for end users to view eraserconfigs.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    app.kubernetes.io/name: clusterrole
-    app.kubernetes.io/instance: eraserconfig-viewer-role
-    app.kubernetes.io/component: rbac
-    app.kubernetes.io/created-by: eraser
-    app.kubernetes.io/part-of: eraser
-    app.kubernetes.io/managed-by: kustomize
-  name: eraserconfig-viewer-role
-rules:
-- apiGroups:
-  - eraser.sh
-  resources:
-  - eraserconfigs
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - eraser.sh
-  resources:
-  - eraserconfigs/status
-  verbs:
-  - get
diff --git a/config/rbac/imagejob_pods_cluster_role.yaml b/config/rbac/imagejob_pods_cluster_role.yaml
deleted file mode 100644
index 1fdc5f7ef6..0000000000
--- a/config/rbac/imagejob_pods_cluster_role.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  name: imagejob-pods-cluster-role
diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml
index 408f075d94..5661f261ca 100644
--- a/config/rbac/kustomization.yaml
+++ b/config/rbac/kustomization.yaml
@@ -7,9 +7,8 @@ resources:
 - service_account.yaml
 - role.yaml
 - role_binding.yaml
-- imagejob_pods_cluster_role.yaml
 - imagejob_pods_service.yaml
-- imagejob_pods_cluster_role_binding.yaml
+- cluster_role_binding.yaml
 # Comment the following 4 lines if you want to disable
 # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
 # which protects your /metrics endpoint.
diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
index 31ac9ca678..dddba82ed2 100644
--- a/config/rbac/role.yaml
+++ b/config/rbac/role.yaml
@@ -5,18 +5,6 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
 - apiGroups:
   - ""
   resources:
@@ -26,52 +14,51 @@ rules:
   - list
   - watch
 - apiGroups:
-  - ""
+  - eraser.sh
   resources:
-  - pods
+  - imagejobs
   verbs:
   - create
   - delete
   - get
   - list
-  - update
   - watch
 - apiGroups:
-  - ""
+  - eraser.sh
   resources:
-  - podtemplates
+  - imagejobs/status
   verbs:
-  - create
-  - delete
   - get
-  - list
   - patch
   - update
-  - watch
 - apiGroups:
   - eraser.sh
   resources:
-  - imagejobs
+  - imagelists
   verbs:
-  - create
-  - delete
   - get
   - list
-  - patch
-  - update
   - watch
 - apiGroups:
   - eraser.sh
   resources:
-  - imagejobs/status
+  - imagelists/status
   verbs:
   - get
   - patch
   - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  creationTimestamp: null
+  name: manager-role
+  namespace: system
+rules:
 - apiGroups:
-  - eraser.sh
+  - ""
   resources:
-  - imagelists
+  - configmaps
   verbs:
   - create
   - delete
@@ -81,10 +68,25 @@ rules:
   - update
   - watch
 - apiGroups:
-  - eraser.sh
+  - ""
   resources:
-  - imagelists/status
+  - pods
   verbs:
+  - create
+  - delete
   - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - podtemplates
+  verbs:
+  - create
+  - delete
+  - get
+  - list
   - patch
   - update
+  - watch
diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml
index 2070ede446..0f67d06101 100644
--- a/config/rbac/role_binding.yaml
+++ b/config/rbac/role_binding.yaml
@@ -1,10 +1,11 @@
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
+kind: RoleBinding
 metadata:
   name: manager-rolebinding
+  namespace: system
 roleRef:
   apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
+  kind: Role
   name: manager-role
 subjects:
 - kind: ServiceAccount
diff --git a/controllers/imagecollector/imagecollector_controller.go b/controllers/imagecollector/imagecollector_controller.go
index 9201add8ed..400ba36f46 100644
--- a/controllers/imagecollector/imagecollector_controller.go
+++ b/controllers/imagecollector/imagecollector_controller.go
@@ -198,7 +198,11 @@ func add(mgr manager.Manager, r *Reconciler) error {
 	return nil
 }
 
-//+kubebuilder:rbac:groups="",resources=podtemplates,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=eraser.sh,resources=imagelists,verbs=get;list;watch
+//+kubebuilder:rbac:groups="",namespace="system",resources=podtemplates,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=eraser.sh,resources=imagelists/status,verbs=get;update;patch
+//+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch
+//+kubebuilder:rbac:groups="",namespace="system",resources=pods,verbs=get;list;watch;update;create;delete
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
diff --git a/controllers/imagejob/imagejob_controller.go b/controllers/imagejob/imagejob_controller.go
index 2a83867fb8..b4e3555e43 100644
--- a/controllers/imagejob/imagejob_controller.go
+++ b/controllers/imagejob/imagejob_controller.go
@@ -188,10 +188,10 @@ func checkNodeFitness(pod *corev1.Pod, node *corev1.Node) bool {
 	return true
 }
 
-//+kubebuilder:rbac:groups=eraser.sh,resources=imagejobs,verbs=get;list;watch;create;update;patch;delete
-//+kubebuilder:rbac:groups="",resources=podtemplates,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=eraser.sh,resources=imagejobs,verbs=get;list;watch;create;delete
+//+kubebuilder:rbac:groups="",namespace="system",resources=podtemplates,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=eraser.sh,resources=imagejobs/status,verbs=get;update;patch
-//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups="",namespace="system",resources=configmaps,verbs=get;list;watch;create;update;patch;delete
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
diff --git a/controllers/imagelist/imagelist_controller.go b/controllers/imagelist/imagelist_controller.go
index 96d1b5ed04..940d8efc79 100644
--- a/controllers/imagelist/imagelist_controller.go
+++ b/controllers/imagelist/imagelist_controller.go
@@ -121,11 +121,11 @@ type Reconciler struct {
 	eraserConfig *config.Manager
 }
 
-//+kubebuilder:rbac:groups=eraser.sh,resources=imagelists,verbs=get;list;watch;create;update;patch;delete
-//+kubebuilder:rbac:groups="",resources=podtemplates,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=eraser.sh,resources=imagelists,verbs=get;list;watch
+//+kubebuilder:rbac:groups="",namespace="system",resources=podtemplates,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=eraser.sh,resources=imagelists/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch
-//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;update;create;delete
+//+kubebuilder:rbac:groups="",namespace="system",resources=pods,verbs=get;list;watch;update;create;delete
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
diff --git a/main.go b/main.go
index 489ee43dbd..7b79a40fd5 100644
--- a/main.go
+++ b/main.go
@@ -32,11 +32,14 @@ import (
 	"k8s.io/utils/inotify"
 	"sigs.k8s.io/yaml"
 
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/conversion"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 
 	"github.com/eraser-dev/eraser/api/unversioned"
@@ -50,6 +53,7 @@ import (
 	v1alpha3Config "github.com/eraser-dev/eraser/api/v1alpha3/config"
 	"github.com/eraser-dev/eraser/controllers"
 	"github.com/eraser-dev/eraser/pkg/logger"
+	"github.com/eraser-dev/eraser/pkg/utils"
 	"github.com/eraser-dev/eraser/version"
 	//+kubebuilder:scaffold:imports
 )
@@ -104,6 +108,26 @@ func main() {
 		Port:                   9443,
 		HealthProbeBindAddress: ":8081",
 		LeaderElection:         false,
+		NewCache: cache.BuilderWithOptions(cache.Options{
+			SelectorsByObject: cache.SelectorsByObject{
+				// to watch eraser pods
+				&corev1.Pod{}: {
+					Field: fields.OneTermEqualSelector("metadata.namespace", utils.GetNamespace()),
+				},
+				// to watch eraser podTemplates
+				&corev1.PodTemplate{}: {
+					Field: fields.OneTermEqualSelector("metadata.namespace", utils.GetNamespace()),
+				},
+				// to watch eraser-manager-configs
+				&corev1.ConfigMap{}: {
+					Field: fields.OneTermEqualSelector("metadata.namespace", utils.GetNamespace()),
+				},
+				// to watch ImageJobs
+				&eraserv1.ImageJob{}: {},
+				// to watch ImageLists
+				&eraserv1.ImageList{}: {},
+			},
+		}),
 	}
 
 	if configFile == "" {
diff --git a/manifest_staging/charts/eraser/templates/eraser-imagejob-pods-cluster-role-clusterrole.yaml b/manifest_staging/charts/eraser/templates/eraser-imagejob-pods-cluster-role-clusterrole.yaml
deleted file mode 100644
index b6612978cd..0000000000
--- a/manifest_staging/charts/eraser/templates/eraser-imagejob-pods-cluster-role-clusterrole.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    app.kubernetes.io/instance: '{{ .Release.Name }}'
-    app.kubernetes.io/managed-by: '{{ .Release.Service }}'
-    app.kubernetes.io/name: '{{ template "eraser.name" . }}'
-    helm.sh/chart: '{{ template "eraser.name" . }}'
-  name: eraser-imagejob-pods-cluster-role
diff --git a/manifest_staging/charts/eraser/templates/eraser-manager-role-clusterrole.yaml b/manifest_staging/charts/eraser/templates/eraser-manager-role-clusterrole.yaml
index 7364f359e7..44f24f097e 100644
--- a/manifest_staging/charts/eraser/templates/eraser-manager-role-clusterrole.yaml
+++ b/manifest_staging/charts/eraser/templates/eraser-manager-role-clusterrole.yaml
@@ -9,18 +9,6 @@ metadata:
     helm.sh/chart: '{{ template "eraser.name" . }}'
   name: eraser-manager-role
 rules:
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
 - apiGroups:
   - ""
   resources:
@@ -29,29 +17,6 @@ rules:
   - get
   - list
   - watch
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - update
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - podtemplates
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
 - apiGroups:
   - eraser.sh
   resources:
@@ -61,8 +26,6 @@ rules:
   - delete
   - get
   - list
-  - patch
-  - update
   - watch
 - apiGroups:
   - eraser.sh
@@ -77,12 +40,8 @@ rules:
   resources:
   - imagelists
   verbs:
-  - create
-  - delete
   - get
   - list
-  - patch
-  - update
   - watch
 - apiGroups:
   - eraser.sh
diff --git a/manifest_staging/charts/eraser/templates/eraser-manager-role-role.yaml b/manifest_staging/charts/eraser/templates/eraser-manager-role-role.yaml
new file mode 100644
index 0000000000..0711761a63
--- /dev/null
+++ b/manifest_staging/charts/eraser/templates/eraser-manager-role-role.yaml
@@ -0,0 +1,47 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/instance: '{{ .Release.Name }}'
+    app.kubernetes.io/managed-by: '{{ .Release.Service }}'
+    app.kubernetes.io/name: '{{ template "eraser.name" . }}'
+    helm.sh/chart: '{{ template "eraser.name" . }}'
+  name: eraser-manager-role
+  namespace: '{{ .Release.Namespace }}'
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - podtemplates
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
diff --git a/manifest_staging/charts/eraser/templates/eraser-imagejob-pods-cluster-rolebinding-clusterrolebinding.yaml b/manifest_staging/charts/eraser/templates/eraser-manager-rolebinding-rolebinding.yaml
similarity index 70%
rename from manifest_staging/charts/eraser/templates/eraser-imagejob-pods-cluster-rolebinding-clusterrolebinding.yaml
rename to manifest_staging/charts/eraser/templates/eraser-manager-rolebinding-rolebinding.yaml
index 272c4a7dc7..94262d3948 100644
--- a/manifest_staging/charts/eraser/templates/eraser-imagejob-pods-cluster-rolebinding-clusterrolebinding.yaml
+++ b/manifest_staging/charts/eraser/templates/eraser-manager-rolebinding-rolebinding.yaml
@@ -1,17 +1,18 @@
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
+kind: RoleBinding
 metadata:
   labels:
     app.kubernetes.io/instance: '{{ .Release.Name }}'
     app.kubernetes.io/managed-by: '{{ .Release.Service }}'
     app.kubernetes.io/name: '{{ template "eraser.name" . }}'
     helm.sh/chart: '{{ template "eraser.name" . }}'
-  name: eraser-imagejob-pods-cluster-rolebinding
+  name: eraser-manager-rolebinding
+  namespace: '{{ .Release.Namespace }}'
 roleRef:
   apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: eraser-imagejob-pods-cluster-role
+  kind: Role
+  name: eraser-manager-role
 subjects:
 - kind: ServiceAccount
-  name: eraser-imagejob-pods
+  name: eraser-controller-manager
   namespace: '{{ .Release.Namespace }}'
diff --git a/manifest_staging/deploy/eraser.yaml b/manifest_staging/deploy/eraser.yaml
index 88f987a4b6..fe8f9425ec 100644
--- a/manifest_staging/deploy/eraser.yaml
+++ b/manifest_staging/deploy/eraser.yaml
@@ -258,16 +258,11 @@ metadata:
   namespace: eraser-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  name: eraser-imagejob-pods-cluster-role
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: Role
 metadata:
   creationTimestamp: null
   name: eraser-manager-role
+  namespace: eraser-system
 rules:
 - apiGroups:
   - ""
@@ -284,33 +279,40 @@ rules:
 - apiGroups:
   - ""
   resources:
-  - nodes
+  - pods
   verbs:
+  - create
+  - delete
   - get
   - list
+  - update
   - watch
 - apiGroups:
   - ""
   resources:
-  - pods
+  - podtemplates
   verbs:
   - create
   - delete
   - get
   - list
+  - patch
   - update
   - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: eraser-manager-role
+rules:
 - apiGroups:
   - ""
   resources:
-  - podtemplates
+  - nodes
   verbs:
-  - create
-  - delete
   - get
   - list
-  - patch
-  - update
   - watch
 - apiGroups:
   - eraser.sh
@@ -321,8 +323,6 @@ rules:
   - delete
   - get
   - list
-  - patch
-  - update
   - watch
 - apiGroups:
   - eraser.sh
@@ -337,12 +337,8 @@ rules:
   resources:
   - imagelists
   verbs:
-  - create
-  - delete
   - get
   - list
-  - patch
-  - update
   - watch
 - apiGroups:
   - eraser.sh
@@ -354,16 +350,17 @@ rules:
   - update
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
+kind: RoleBinding
 metadata:
-  name: eraser-imagejob-pods-cluster-rolebinding
+  name: eraser-manager-rolebinding
+  namespace: eraser-system
 roleRef:
   apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: eraser-imagejob-pods-cluster-role
+  kind: Role
+  name: eraser-manager-role
 subjects:
 - kind: ServiceAccount
-  name: eraser-imagejob-pods
+  name: eraser-controller-manager
   namespace: eraser-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1