diff --git a/pkg/autoclean/main.go b/pkg/autoclean/main.go new file mode 100644 index 000000000..c35ce66e9 --- /dev/null +++ b/pkg/autoclean/main.go @@ -0,0 +1,158 @@ +package autoclean + +import ( + "context" + "strings" + "time" + + "github.com/spf13/viper" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/client-go/discovery" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + + v1 "github.com/jaegertracing/jaeger-operator/apis/v1" + "github.com/jaegertracing/jaeger-operator/pkg/inject" +) + +type Background struct { + cl client.Client + clReader client.Reader + dcl discovery.DiscoveryInterface + ticker *time.Ticker +} + +// New creates a new auto-clean runner +func New(mgr manager.Manager) (*Background, error) { + dcl, err := discovery.NewDiscoveryClientForConfig(mgr.GetConfig()) + if err != nil { + return nil, err + } + + return WithClients(mgr.GetClient(), dcl, mgr.GetAPIReader()), nil +} + +// WithClients builds a new Background with the provided clients +func WithClients(cl client.Client, dcl discovery.DiscoveryInterface, clr client.Reader) *Background { + return &Background{ + cl: cl, + dcl: dcl, + clReader: clr, + } +} + +// Start initializes the auto-clean process that runs in the background +func (b *Background) Start() { + b.ticker = time.NewTicker(5 * time.Second) + b.autoClean() + + go func() { + for { + <-b.ticker.C + b.autoClean() + } + }() +} + +// Stop causes the background process to stop auto clean capabilities +func (b *Background) Stop() { + b.ticker.Stop() +} + +func (b *Background) autoClean() { + ctx := context.Background() + b.cleanDeployments(ctx) +} + +func (b *Background) cleanDeployments(ctx context.Context) { + log.Log.V(-1).Info("cleaning orphaned deployments.") + + instancesMap := make(map[string]*v1.Jaeger) + deployments := &appsv1.DeploymentList{} + deployOpts := []client.ListOption{ + matchingLabelKeys(map[string]string{inject.Label: ""}), + } + + // if we are not watching all namespaces, we have to get items from each namespace being watched + if namespaces := viper.GetString(v1.ConfigWatchNamespace); namespaces != v1.WatchAllNamespaces { + for _, ns := range strings.Split(namespaces, ",") { + nsDeps := &appsv1.DeploymentList{} + if err := b.clReader.List(ctx, nsDeps, append(deployOpts, client.InNamespace(ns))...); err != nil { + log.Log.Error( + err, + "error getting a list of deployments to analyze in namespace", + "namespace", ns, + ) + } + deployments.Items = append(deployments.Items, nsDeps.Items...) + + instances := &v1.JaegerList{} + if err := b.clReader.List(ctx, instances, client.InNamespace(ns)); err != nil { + log.Log.Error( + err, + "error getting a list of existing jaeger instances in namespace", + "namespace", ns, + ) + } + for i := range instances.Items { + instancesMap[instances.Items[i].Name] = &instances.Items[i] + } + } + } else { + if err := b.clReader.List(ctx, deployments, deployOpts...); err != nil { + log.Log.Error( + err, + "error getting a list of deployments to analyze", + ) + } + + instances := &v1.JaegerList{} + if err := b.clReader.List(ctx, instances); err != nil { + log.Log.Error( + err, + "error getting a list of existing jaeger instances", + ) + } + for i := range instances.Items { + instancesMap[instances.Items[i].Name] = &instances.Items[i] + } + } + + // check deployments to see which one needs to be cleaned. + for i := range deployments.Items { + dep := deployments.Items[i] + if instanceName, ok := dep.Labels[inject.Label]; ok { + _, instanceExists := instancesMap[instanceName] + if !instanceExists { // Jaeger instance not exist anymore, we need to clean this up. + inject.CleanSidecar(instanceName, &dep) + if err := b.cl.Update(ctx, &dep); err != nil { + log.Log.Error( + err, + "error cleaning orphaned deployment", + "deploymentName", dep.Name, + "deploymentNamespace", dep.Namespace, + ) + } + } + } + } +} + +type matchingLabelKeys map[string]string + +func (m matchingLabelKeys) ApplyToList(opts *client.ListOptions) { + sel := labels.NewSelector() + for k := range map[string]string(m) { + req, err := labels.NewRequirement(k, selection.Exists, []string{}) + if err != nil { + log.Log.Error(err, "failed to build label selector") + return + } + sel.Add(*req) + } + opts.LabelSelector = sel +} diff --git a/pkg/autoclean/main_test.go b/pkg/autoclean/main_test.go new file mode 100644 index 000000000..d7e714795 --- /dev/null +++ b/pkg/autoclean/main_test.go @@ -0,0 +1,168 @@ +package autoclean + +import ( + "context" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/version" + "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + v1 "github.com/jaegertracing/jaeger-operator/apis/v1" + "github.com/jaegertracing/jaeger-operator/pkg/inject" +) + +func TestCleanDeployments(t *testing.T) { + for _, tt := range []struct { + cap string // caption for the test + watchNamespace string // the value for WATCH_NAMESPACE + jaegerNamespace string // in which namespace the jaeger exists, empty for non existing + deleted bool // whether the sidecar should have been deleted + }{ + { + cap: "existing-same-namespace", + watchNamespace: "observability", + jaegerNamespace: "observability", + deleted: false, + }, + { + cap: "not-existing-same-namespace", + watchNamespace: "observability", + jaegerNamespace: "", + deleted: true, + }, + { + cap: "existing-watched-namespace", + watchNamespace: "observability,other-observability", + jaegerNamespace: "other-observability", + deleted: false, + }, + { + cap: "existing-non-watched-namespace", + watchNamespace: "observability", + jaegerNamespace: "other-observability", + deleted: true, + }, + { + cap: "existing-watching-all-namespaces", + watchNamespace: v1.WatchAllNamespaces, + jaegerNamespace: "other-observability", + deleted: false, + }, + } { + t.Run(tt.cap, func(t *testing.T) { + // prepare the test data + viper.Set(v1.ConfigWatchNamespace, tt.watchNamespace) + defer viper.Reset() + + jaeger := v1.NewJaeger(types.NamespacedName{ + Name: "my-instance", + Namespace: "observability", // at first, it exists in the same namespace as the deployment + }) + + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mydep", + Namespace: "observability", + Annotations: map[string]string{inject.Annotation: jaeger.Name}, + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "C1", + Image: "image1", + }, + }, + }, + }, + }, + } + dep = inject.Sidecar(jaeger, dep) + + // sanity check + require.Equal(t, 2, len(dep.Spec.Template.Spec.Containers)) + + // prepare the list of existing objects + objs := []runtime.Object{dep} + if len(tt.jaegerNamespace) > 0 { + jaeger.Namespace = tt.jaegerNamespace // now, it exists only in this namespace + objs = append(objs, jaeger) + } + + // prepare the client + s := scheme.Scheme + s.AddKnownTypes(v1.GroupVersion, &v1.Jaeger{}) + s.AddKnownTypes(v1.GroupVersion, &v1.JaegerList{}) + cl := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() + b := WithClients(cl, &fakeDiscoveryClient{}, cl) + + // test + b.cleanDeployments(context.Background()) + + // verify + persisted := &appsv1.Deployment{} + err := cl.Get(context.Background(), types.NamespacedName{ + Namespace: dep.Namespace, + Name: dep.Name, + }, persisted) + require.NoError(t, err) + + // should the sidecar have been deleted? + if tt.deleted { + assert.Equal(t, 1, len(persisted.Spec.Template.Spec.Containers)) + assert.NotContains(t, persisted.Labels, inject.Label) + } else { + assert.Equal(t, 2, len(persisted.Spec.Template.Spec.Containers)) + assert.Contains(t, persisted.Labels, inject.Label) + } + }) + } +} + +type fakeDiscoveryClient struct { + discovery.DiscoveryInterface + ServerGroupsFunc func() (apiGroupList *metav1.APIGroupList, err error) + ServerResourcesForGroupVersionFunc func(groupVersion string) (resources *metav1.APIResourceList, err error) +} + +func (d *fakeDiscoveryClient) ServerGroups() (apiGroupList *metav1.APIGroupList, err error) { + if d.ServerGroupsFunc == nil { + return &metav1.APIGroupList{}, nil + } + return d.ServerGroupsFunc() +} + +func (d *fakeDiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (resources *metav1.APIResourceList, err error) { + if d.ServerGroupsFunc == nil { + return &metav1.APIResourceList{}, nil + } + return d.ServerResourcesForGroupVersionFunc(groupVersion) +} + +func (d *fakeDiscoveryClient) ServerResources() ([]*metav1.APIResourceList, error) { + return []*metav1.APIResourceList{}, nil +} + +func (d *fakeDiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, error) { + return []*metav1.APIResourceList{}, nil +} + +func (d *fakeDiscoveryClient) ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) { + return []*metav1.APIResourceList{}, nil +} + +func (d *fakeDiscoveryClient) ServerVersion() (*version.Info, error) { + return &version.Info{}, nil +} diff --git a/pkg/autodetect/main.go b/pkg/autodetect/main.go index b1167d7ab..fe2c4dc79 100644 --- a/pkg/autodetect/main.go +++ b/pkg/autodetect/main.go @@ -10,11 +10,8 @@ import ( osimagev1 "github.com/openshift/api/image/v1" "github.com/spf13/viper" "go.opentelemetry.io/otel" - appsv1 "k8s.io/api/apps/v1" authenticationapi "k8s.io/api/authentication/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/discovery" "sigs.k8s.io/controller-runtime/pkg/client" @@ -22,7 +19,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" v1 "github.com/jaegertracing/jaeger-operator/apis/v1" - "github.com/jaegertracing/jaeger-operator/pkg/inject" "github.com/jaegertracing/jaeger-operator/pkg/tracing" ) @@ -110,7 +106,6 @@ func (b *Background) autoDetectCapabilities() { b.detectKafka(ctx, apiList) } b.detectClusterRoles(ctx) - b.cleanDeployments(ctx) } func (b *Background) detectCronjobsVersion(ctx context.Context) { @@ -384,85 +379,12 @@ func (b *Background) detectClusterRoles(ctx context.Context) { } newAuthDelegator = true } + if currentAuthDelegator != newAuthDelegator || !viper.IsSet("auth-delegator-available") { viper.Set("auth-delegator-available", newAuthDelegator) } } -func (b *Background) cleanDeployments(ctx context.Context) { - log.Log.V(-1).Info("detecting orphaned deployments.") - - instancesMap := make(map[string]*v1.Jaeger) - deployments := &appsv1.DeploymentList{} - deployOpts := []client.ListOption{ - matchingLabelKeys(map[string]string{inject.Label: ""}), - } - - // if we are not watching all namespaces, we have to get items from each namespace being watched - if namespaces := viper.GetString(v1.ConfigWatchNamespace); namespaces != v1.WatchAllNamespaces { - for _, ns := range strings.Split(namespaces, ",") { - nsDeps := &appsv1.DeploymentList{} - if err := b.clReader.List(ctx, nsDeps, append(deployOpts, client.InNamespace(ns))...); err != nil { - log.Log.Error( - err, - "error getting a list of deployments to analyze in namespace", - "namespace", ns, - ) - } - deployments.Items = append(deployments.Items, nsDeps.Items...) - - instances := &v1.JaegerList{} - if err := b.clReader.List(ctx, instances, client.InNamespace(ns)); err != nil { - log.Log.Error( - err, - "error getting a list of existing jaeger instances in namespace", - "namespace", ns, - ) - } - for i := range instances.Items { - instancesMap[instances.Items[i].Name] = &instances.Items[i] - } - } - } else { - if err := b.clReader.List(ctx, deployments, deployOpts...); err != nil { - log.Log.Error( - err, - "error getting a list of deployments to analyze", - ) - } - - instances := &v1.JaegerList{} - if err := b.clReader.List(ctx, instances); err != nil { - log.Log.Error( - err, - "error getting a list of existing jaeger instances", - ) - } - for i := range instances.Items { - instancesMap[instances.Items[i].Name] = &instances.Items[i] - } - } - - // check deployments to see which one needs to be cleaned. - for i := range deployments.Items { - dep := deployments.Items[i] - if instanceName, ok := dep.Labels[inject.Label]; ok { - _, instanceExists := instancesMap[instanceName] - if !instanceExists { // Jaeger instance not exist anymore, we need to clean this up. - inject.CleanSidecar(instanceName, &dep) - if err := b.cl.Update(ctx, &dep); err != nil { - log.Log.Error( - err, - "error cleaning orphaned deployment", - "deploymentName", dep.Name, - "deploymentNamespace", dep.Namespace, - ) - } - } - } - } -} - func isOpenShift(apiList []*metav1.APIResourceList) bool { for _, r := range apiList { if strings.HasPrefix(r.GroupVersion, "route.openshift.io") { @@ -494,18 +416,3 @@ func isKafkaOperatorAvailable(apiList []*metav1.APIResourceList) bool { } return false } - -type matchingLabelKeys map[string]string - -func (m matchingLabelKeys) ApplyToList(opts *client.ListOptions) { - sel := labels.NewSelector() - for k := range map[string]string(m) { - req, err := labels.NewRequirement(k, selection.Exists, []string{}) - if err != nil { - log.Log.Error(err, "failed to build label selector") - return - } - sel.Add(*req) - } - opts.LabelSelector = sel -} diff --git a/pkg/autodetect/main_test.go b/pkg/autodetect/main_test.go index c3ea1752d..6376877cf 100644 --- a/pkg/autodetect/main_test.go +++ b/pkg/autodetect/main_test.go @@ -9,22 +9,15 @@ import ( openapi_v2 "github.com/google/gnostic/openapiv2" "github.com/spf13/viper" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/discovery" - "k8s.io/client-go/kubernetes/scheme" restclient "k8s.io/client-go/rest" fakeRest "k8s.io/client-go/rest/fake" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" v1 "github.com/jaegertracing/jaeger-operator/apis/v1" - "github.com/jaegertracing/jaeger-operator/pkg/inject" ) func TestStart(t *testing.T) { @@ -580,116 +573,6 @@ func TestAuthDelegatorBecomesUnavailable(t *testing.T) { assert.False(t, viper.GetBool("auth-delegator-available")) } -func TestCleanDeployments(t *testing.T) { - for _, tt := range []struct { - cap string // caption for the test - watchNamespace string // the value for WATCH_NAMESPACE - jaegerNamespace string // in which namespace the jaeger exists, empty for non existing - deleted bool // whether the sidecar should have been deleted - }{ - { - cap: "existing-same-namespace", - watchNamespace: "observability", - jaegerNamespace: "observability", - deleted: false, - }, - { - cap: "not-existing-same-namespace", - watchNamespace: "observability", - jaegerNamespace: "", - deleted: true, - }, - { - cap: "existing-watched-namespace", - watchNamespace: "observability,other-observability", - jaegerNamespace: "other-observability", - deleted: false, - }, - { - cap: "existing-non-watched-namespace", - watchNamespace: "observability", - jaegerNamespace: "other-observability", - deleted: true, - }, - { - cap: "existing-watching-all-namespaces", - watchNamespace: v1.WatchAllNamespaces, - jaegerNamespace: "other-observability", - deleted: false, - }, - } { - t.Run(tt.cap, func(t *testing.T) { - // prepare the test data - viper.Set(v1.ConfigWatchNamespace, tt.watchNamespace) - defer viper.Reset() - - jaeger := v1.NewJaeger(types.NamespacedName{ - Name: "my-instance", - Namespace: "observability", // at first, it exists in the same namespace as the deployment - }) - - dep := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mydep", - Namespace: "observability", - Annotations: map[string]string{inject.Annotation: jaeger.Name}, - }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{}, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "C1", - Image: "image1", - }, - }, - }, - }, - }, - } - dep = inject.Sidecar(jaeger, dep) - - // sanity check - require.Equal(t, 2, len(dep.Spec.Template.Spec.Containers)) - - // prepare the list of existing objects - objs := []runtime.Object{dep} - if len(tt.jaegerNamespace) > 0 { - jaeger.Namespace = tt.jaegerNamespace // now, it exists only in this namespace - objs = append(objs, jaeger) - } - - // prepare the client - s := scheme.Scheme - s.AddKnownTypes(v1.GroupVersion, &v1.Jaeger{}) - s.AddKnownTypes(v1.GroupVersion, &v1.JaegerList{}) - cl := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() - b := WithClients(cl, &fakeDiscoveryClient{}, cl) - - // test - b.cleanDeployments(context.Background()) - - // verify - persisted := &appsv1.Deployment{} - err := cl.Get(context.Background(), types.NamespacedName{ - Namespace: dep.Namespace, - Name: dep.Name, - }, persisted) - require.NoError(t, err) - - // should the sidecar have been deleted? - if tt.deleted { - assert.Equal(t, 1, len(persisted.Spec.Template.Spec.Containers)) - assert.NotContains(t, persisted.Labels, inject.Label) - } else { - assert.Equal(t, 2, len(persisted.Spec.Template.Spec.Containers)) - assert.Contains(t, persisted.Labels, inject.Label) - } - }) - } -} - type fakeClient struct { client.Client CreateFunc func(ctx context.Context, obj client.Object, opts ...client.CreateOption) error diff --git a/pkg/cmd/start/bootstrap.go b/pkg/cmd/start/bootstrap.go index a6461f54c..649308e63 100644 --- a/pkg/cmd/start/bootstrap.go +++ b/pkg/cmd/start/bootstrap.go @@ -41,6 +41,7 @@ import ( appsv1controllers "github.com/jaegertracing/jaeger-operator/controllers/appsv1" esv1controllers "github.com/jaegertracing/jaeger-operator/controllers/elasticsearch" jaegertracingcontrollers "github.com/jaegertracing/jaeger-operator/controllers/jaegertracing" + "github.com/jaegertracing/jaeger-operator/pkg/autoclean" "github.com/jaegertracing/jaeger-operator/pkg/autodetect" kafkav1beta2 "github.com/jaegertracing/jaeger-operator/pkg/kafka/v1beta2" opmetrics "github.com/jaegertracing/jaeger-operator/pkg/metrics" @@ -124,6 +125,15 @@ func bootstrap(ctx context.Context) manager.Manager { d.Start() } + if c, err := autoclean.New(mgr); err != nil { + log.Log.Error( + err, + "failed to start the background process to auto-clean the operator objects", + ) + } else { + c.Start() + } + detectNamespacePermissions(ctx, mgr) performUpgrades(ctx, mgr) setupControllers(ctx, mgr)