Skip to content

Commit

Permalink
Merge branch 'release-1.12' into release-1.12
Browse files Browse the repository at this point in the history
  • Loading branch information
realshuting authored Jul 9, 2024
2 parents 8b161ac + 2b94698 commit 235b2fc
Show file tree
Hide file tree
Showing 13 changed files with 631 additions and 8 deletions.
2 changes: 2 additions & 0 deletions charts/kyverno/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ annotations:
artifacthub.io/changes: |
- kind: added
description: Add a key to preserve configmap settings during upgrade
- kind: added
description: Make admission reports breaker threshold configurable
dependencies:
- name: grafana
version: 3.2.5
Expand Down
3 changes: 2 additions & 1 deletion charts/kyverno/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -355,7 +355,8 @@ The chart values are organised per component.

| Key | Type | Default | Description |
|-----|------|---------|-------------|
| admissionController.featuresOverride | object | `{}` | Overrides features defined at the root level |
| admissionController.featuresOverride | object | `{"admissionReports":{"backPressureThreshold":1000}}` | Overrides features defined at the root level |
| admissionController.featuresOverride.admissionReports.backPressureThreshold | int | `1000` | Max number of admission reports allowed in flight until the admission controller stops creating new ones |
| admissionController.rbac.create | bool | `true` | Create RBAC resources |
| admissionController.rbac.serviceAccount.name | string | `nil` | The ServiceAccount name |
| admissionController.rbac.serviceAccount.annotations | object | `{}` | Annotations for the ServiceAccount |
Expand Down
3 changes: 3 additions & 0 deletions charts/kyverno/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
{{- $flags := list -}}
{{- with .admissionReports -}}
{{- $flags = append $flags (print "--admissionReports=" .enabled) -}}
{{- with .backPressureThreshold -}}
{{- $flags = append $flags (print "--maxAdmissionReports=" .) -}}
{{- end -}}
{{- end -}}
{{- with .aggregateReports -}}
{{- $flags = append $flags (print "--aggregateReports=" .enabled) -}}
Expand Down
5 changes: 4 additions & 1 deletion charts/kyverno/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1096,7 +1096,10 @@ cleanupJobs:
admissionController:

# -- Overrides features defined at the root level
featuresOverride: {}
featuresOverride:
admissionReports:
# -- Max number of admission reports allowed in flight until the admission controller stops creating new ones
backPressureThreshold: 1000

rbac:
# -- Create RBAC resources
Expand Down
125 changes: 125 additions & 0 deletions cmd/kyverno/breaker.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
package main

import (
"context"
"sync"

reportsv1 "github.com/kyverno/kyverno/api/reports/v1"
watchtools "github.com/kyverno/kyverno/cmd/kyverno/watch"
"github.com/kyverno/kyverno/pkg/client/informers/externalversions/internalinterfaces"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/watch"
metadataclient "k8s.io/client-go/metadata"
"k8s.io/client-go/tools/cache"
)

type resourceUIDGetter interface {
GetUID() types.UID
}

type Counter interface {
Count() (int, bool)
}

type counter struct {
lock sync.RWMutex
entries sets.Set[types.UID]
retryWatcher *watchtools.RetryWatcher
}

func (c *counter) Record(uid types.UID) {
c.lock.Lock()
defer c.lock.Unlock()
c.entries.Insert(uid)
}

func (c *counter) Forget(uid types.UID) {
c.lock.Lock()
defer c.lock.Unlock()
c.entries.Delete(uid)
}

func (c *counter) Count() (int, bool) {
c.lock.RLock()
defer c.lock.RUnlock()
return c.entries.Len(), c.retryWatcher.IsRunning()
}

func StartResourceCounter(ctx context.Context, client metadataclient.Interface, gvr schema.GroupVersionResource, tweakListOptions internalinterfaces.TweakListOptionsFunc) (*counter, error) {
objs, err := client.Resource(gvr).List(ctx, metav1.ListOptions{})
if err != nil {
return nil, err
}
watcher := &cache.ListWatch{
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
if tweakListOptions != nil {
tweakListOptions(&options)
}
return client.Resource(gvr).Watch(ctx, options)
},
}
watchInterface, err := watchtools.NewRetryWatcher(objs.GetResourceVersion(), watcher)
if err != nil {
return nil, err
}
entries := sets.New[types.UID]()
for _, entry := range objs.Items {
entries.Insert(entry.GetUID())
}
w := &counter{
entries: entries,
retryWatcher: watchInterface,
}
go func() {
for event := range watchInterface.ResultChan() {
getter, ok := event.Object.(resourceUIDGetter)
if ok {
switch event.Type {
case watch.Added:
w.Record(getter.GetUID())
case watch.Modified:
w.Record(getter.GetUID())
case watch.Deleted:
w.Forget(getter.GetUID())
}
}
}
}()
return w, nil
}

func StartAdmissionReportsCounter(ctx context.Context, client metadataclient.Interface) (Counter, error) {
tweakListOptions := func(lo *metav1.ListOptions) {
lo.LabelSelector = "audit.kyverno.io/source==admission"
}
ephrs, err := StartResourceCounter(ctx, client, reportsv1.SchemeGroupVersion.WithResource("ephemeralreports"), tweakListOptions)
if err != nil {
return nil, err
}
cephrs, err := StartResourceCounter(ctx, client, reportsv1.SchemeGroupVersion.WithResource("clusterephemeralreports"), tweakListOptions)
if err != nil {
return nil, err
}
return composite{
inner: []Counter{ephrs, cephrs},
}, nil
}

type composite struct {
inner []Counter
}

func (c composite) Count() (int, bool) {
sum := 0
for _, counter := range c.inner {
count, isRunning := counter.Count()
if !isRunning {
return 0, false
}
sum += count
}
return sum, true
}
17 changes: 16 additions & 1 deletion cmd/kyverno/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
policycachecontroller "github.com/kyverno/kyverno/pkg/controllers/policycache"
vapcontroller "github.com/kyverno/kyverno/pkg/controllers/validatingadmissionpolicy-generate"
webhookcontroller "github.com/kyverno/kyverno/pkg/controllers/webhook"
"github.com/kyverno/kyverno/pkg/d4f"
"github.com/kyverno/kyverno/pkg/engine/apicall"
"github.com/kyverno/kyverno/pkg/event"
"github.com/kyverno/kyverno/pkg/globalcontext/store"
Expand Down Expand Up @@ -122,7 +123,6 @@ func createrLeaderControllers(
eventGenerator event.Interface,
) ([]internal.Controller, func(context.Context) error, error) {
var leaderControllers []internal.Controller

certManager := certmanager.NewController(
caInformer,
tlsInformer,
Expand Down Expand Up @@ -251,6 +251,7 @@ func main() {
renewBefore time.Duration
maxAuditWorkers int
maxAuditCapacity int
maxAdmissionReports int
)
flagset := flag.NewFlagSet("kyverno", flag.ExitOnError)
flagset.BoolVar(&dumpPayload, "dumpPayload", false, "Set this flag to activate/deactivate debug mode.")
Expand All @@ -273,6 +274,7 @@ func main() {
flagset.DurationVar(&renewBefore, "renewBefore", 15*24*time.Hour, "The certificate renewal time before expiration")
flagset.IntVar(&maxAuditWorkers, "maxAuditWorkers", 8, "Maximum number of workers for audit policy processing")
flagset.IntVar(&maxAuditCapacity, "maxAuditCapacity", 1000, "Maximum capacity of the audit policy task queue")
flagset.IntVar(&maxAdmissionReports, "maxAdmissionReports", 10000, "Maximum number of admission reports before we stop creating new ones")
// config
appConfig := internal.NewConfiguration(
internal.WithProfiling(),
Expand Down Expand Up @@ -513,6 +515,18 @@ func main() {
setup.KyvernoClient,
backgroundServiceAccountName,
)
ephrs, err := StartAdmissionReportsCounter(signalCtx, setup.MetadataClient)
if err != nil {
setup.Logger.Error(errors.New("failed to start admission reports watcher"), "failed to start admission reports watcher")
os.Exit(1)
}
reportsBreaker := d4f.NewBreaker("admission reports", func(context.Context) bool {
count, isRunning := ephrs.Count()
if !isRunning {
return true
}
return count > maxAdmissionReports
})
resourceHandlers := webhooksresource.NewHandlers(
engine,
setup.KyvernoDynamicClient,
Expand All @@ -531,6 +545,7 @@ func main() {
setup.Jp,
maxAuditWorkers,
maxAuditCapacity,
reportsBreaker,
)
exceptionHandlers := webhooksexception.NewHandlers(exception.ValidationOptions{
Enabled: internal.PolicyExceptionEnabled(),
Expand Down
Loading

0 comments on commit 235b2fc

Please sign in to comment.