diff --git a/go.mod b/go.mod
index 3e4438d6..779efde8 100644
--- a/go.mod
+++ b/go.mod
@@ -32,6 +32,7 @@ require (
 	github.com/emicklei/go-restful/v3 v3.11.2 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-openapi/jsonpointer v0.20.2 // indirect
@@ -39,6 +40,7 @@ require (
 	github.com/go-openapi/swag v0.22.9 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
@@ -81,6 +83,7 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/api v0.31.2
+	k8s.io/apiextensions-apiserver v0.31.0 // indirect
 	k8s.io/component-base v0.31.2 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
diff --git a/go.sum b/go.sum
index 3916f82c..26a807fe 100644
--- a/go.sum
+++ b/go.sum
@@ -152,6 +152,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea h1:CyhwejzVGvZ3Q2PSbQ4NRRYn+ZWv5eS1vlaEusT+bAI=
 github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea/go.mod h1:eNr558nEUjP8acGw8FFjTeWvSgU1stO7FAO6eknhHe4=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
diff --git a/pkg/policy/provider.go b/pkg/policy/provider.go
index 3aaa9e3f..34289f50 100644
--- a/pkg/policy/provider.go
+++ b/pkg/policy/provider.go
@@ -2,11 +2,15 @@ package policy
 
 import (
 	"context"
+	"fmt"
+	"sync"
 
 	"github.com/kyverno/kyverno-envoy-plugin/apis/v1alpha1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/rest"
-	"sigs.k8s.io/controller-runtime/pkg/cache"
+	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -19,55 +23,67 @@ func NewKubeProvider(ctx context.Context, config *rest.Config, compiler Compiler
 	if err := v1alpha1.Install(scheme); err != nil {
 		return nil, err
 	}
-	// create kubernetes client
-	// TODO: do we want to use a cache ?
-	cache, err := cache.New(config, cache.Options{
+	mgr, err := ctrl.NewManager(config, ctrl.Options{
 		Scheme: scheme,
 	})
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to construct manager: %w", err)
 	}
-	client, err := client.New(config, client.Options{
-		Cache: &client.CacheOptions{
-			Reader: cache,
-		},
-		Scheme: scheme,
-	})
-	if err != nil {
+	r := newPolicyReconciler(mgr.GetClient(), compiler)
+	if err = ctrl.NewControllerManagedBy(mgr).For(&v1alpha1.AuthorizationPolicy{}).Complete(r); err != nil {
+		return nil, fmt.Errorf("failed to construct manager: %w", err)
+	}
+	if err := mgr.Start(ctx); err != nil {
 		return nil, err
 	}
-	go func() {
-		if err := cache.Start(ctx); err != nil {
-			// TODO: better error handling
-			panic(err)
-		}
-	}()
-	// TODO: use the result of the wait
-	cache.WaitForCacheSync(ctx)
-	return &kubeProvider{
-		client:   client,
-		compiler: compiler,
-	}, nil
+	return r, nil
 }
 
-type kubeProvider struct {
+type policyReconciler struct {
 	client   client.Client
 	compiler Compiler
+	lock     *sync.RWMutex
+	policies map[types.NamespacedName]PolicyFunc
 }
 
-func (p *kubeProvider) CompiledPolicies(ctx context.Context) ([]PolicyFunc, error) {
-	// fetch policies
-	var policies v1alpha1.AuthorizationPolicyList
-	if err := p.client.List(ctx, &policies, &client.ListOptions{}); err != nil {
-		return nil, err
+func newPolicyReconciler(client client.Client, compiler Compiler) *policyReconciler {
+	return &policyReconciler{
+		client:   client,
+		compiler: compiler,
+		lock:     &sync.RWMutex{},
+		policies: map[types.NamespacedName]PolicyFunc{},
 	}
-	var out []PolicyFunc
-	for _, policy := range policies.Items {
-		compiled, err := p.compiler.Compile(policy)
-		if err != nil {
-			return nil, err
-		}
-		out = append(out, compiled)
+}
+
+func (r *policyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	var policy v1alpha1.AuthorizationPolicy
+	err := r.client.Get(ctx, req.NamespacedName, &policy)
+	if errors.IsNotFound(err) {
+		r.lock.Lock()
+		defer r.lock.Unlock()
+		delete(r.policies, req.NamespacedName)
+		return ctrl.Result{}, nil
+	}
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	compiled, err := r.compiler.Compile(policy)
+	if err != nil {
+		// TODO: not sure we should retry it
+		return ctrl.Result{}, err
+	}
+	r.lock.Lock()
+	defer r.lock.Unlock()
+	r.policies[req.NamespacedName] = compiled
+	return ctrl.Result{}, nil
+}
+
+func (r *policyReconciler) CompiledPolicies(ctx context.Context) ([]PolicyFunc, error) {
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+	out := make([]PolicyFunc, 0, len(r.policies))
+	for _, policy := range r.policies {
+		out = append(out, policy)
 	}
 	return out, nil
 }