From 25995d57b64f7cd68c281c401bd9468dc321913c Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Mon, 18 Mar 2019 15:57:00 +0100 Subject: [PATCH] Silence access errors logged internally by kubernetes libraries The Kubernetes Go libraries sometimes handle errors internally (using `runtime.HandleError`) which doesn't allow for custom error handling. This is really unfortunate since it doesn't give us the opportunity to selectively silence tolerable errors. In particular, fluxd cannot know in advance whether it has access to certain cluster resources. An, admitedly not ideal, solution is silencing access errors when logged by `runtime.HandleError`, since it can lead to the logs being flooded. Alternatives like forking and maintaining parts of the kubernetes libraries to add proper error handling are even worse. This change is was triggered by the use of [`cache.NewInformer`](https://godoc.org/k8s.io/client-go/tools/cache#NewInformer) to track CRD changes, which lead to the logs being flooded by (tolerable) access errors when flux doesn't have permission to list CRDs. --- cmd/fluxd/main.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cmd/fluxd/main.go b/cmd/fluxd/main.go index 1ebd95bef..541c1e8e5 100644 --- a/cmd/fluxd/main.go +++ b/cmd/fluxd/main.go @@ -17,11 +17,12 @@ import ( "github.com/go-kit/kit/log" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/pflag" - integrations "github.com/weaveworks/flux/integrations/client/clientset/versioned" crd "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + k8sruntime "k8s.io/apimachinery/pkg/util/runtime" k8sclientdynamic "k8s.io/client-go/dynamic" k8sclient "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + k8serrors "k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/api/errors" "github.com/weaveworks/flux/checkpoint" "github.com/weaveworks/flux/cluster" @@ -33,6 +34,7 @@ import ( "github.com/weaveworks/flux/http/client" daemonhttp "github.com/weaveworks/flux/http/daemon" "github.com/weaveworks/flux/image" + integrations "github.com/weaveworks/flux/integrations/client/clientset/versioned" "github.com/weaveworks/flux/job" "github.com/weaveworks/flux/registry" "github.com/weaveworks/flux/registry/cache" @@ -164,6 +166,16 @@ func main() { } logger.Log("version", version) + // Silence access errors logged internally by client-go + klog := log.With(logger, "type", "internal kubernetes error") + logErrorUnlessAccessRelated := func(err error) { + if k8serrors.IsForbidden(err) || k8serrors.IsNotFound(err) { + return + } + klog.Log("err", err) + } + k8sruntime.ErrorHandlers = []func(error){logErrorUnlessAccessRelated} + // Argument validation // Sort out values for the git tag and notes ref. There are