From 1860f22c1019642b0e7d5c2a006fe1cb7264fbd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juraci=20Paix=C3=A3o=20Kr=C3=B6hling?= Date: Tue, 24 Sep 2019 11:39:49 +0200 Subject: [PATCH] Recognize when a resource has been deleted while the operator waits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juraci Paixão Kröhling --- pkg/controller/jaeger/dependencies.go | 18 ++++++++++++++++++ pkg/controller/jaeger/deployment.go | 19 +++++++++++++++++++ pkg/controller/jaeger/elasticsearch.go | 19 +++++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/pkg/controller/jaeger/dependencies.go b/pkg/controller/jaeger/dependencies.go index 1d738ad0f..1324aeea3 100644 --- a/pkg/controller/jaeger/dependencies.go +++ b/pkg/controller/jaeger/dependencies.go @@ -2,6 +2,7 @@ package jaeger import ( "context" + "errors" "time" log "github.com/sirupsen/logrus" @@ -14,6 +15,11 @@ import ( "github.com/jaegertracing/jaeger-operator/pkg/strategy" ) +var ( + // ErrDependencyRemoved is returned when a dependency existed but has been removed + ErrDependencyRemoved = errors.New("dependency has been removed") +) + func (r *ReconcileJaeger) handleDependencies(str strategy.S) error { for _, dep := range str.Dependencies() { err := r.client.Create(context.Background(), &dep) @@ -28,10 +34,21 @@ func (r *ReconcileJaeger) handleDependencies(str strategy.S) error { deadline = time.Duration(int64(*dep.Spec.ActiveDeadlineSeconds)) } + seen := false return wait.PollImmediate(time.Second, deadline*time.Second, func() (done bool, err error) { batch := &batchv1.Job{} if err = r.client.Get(context.Background(), types.NamespacedName{Name: dep.Name, Namespace: dep.Namespace}, batch); err != nil { if k8serrors.IsNotFound(err) { + if seen { + // we have seen this object before, but it doesn't exist anymore! + // we don't have anything else to do here, break the poll + log.WithFields(log.Fields{ + "namespace": dep.Namespace, + "name": dep.Name, + }).Warn("Dependency has been removed.") + return true, ErrDependencyRemoved + } + // the object might have not been created yet log.WithFields(log.Fields{ "namespace": dep.Namespace, @@ -42,6 +59,7 @@ func (r *ReconcileJaeger) handleDependencies(str strategy.S) error { return false, err } + seen = true // for now, we just assume each batch job has one pod if batch.Status.Succeeded != 1 { log.WithFields(log.Fields{ diff --git a/pkg/controller/jaeger/deployment.go b/pkg/controller/jaeger/deployment.go index 913a204e8..095c914b3 100644 --- a/pkg/controller/jaeger/deployment.go +++ b/pkg/controller/jaeger/deployment.go @@ -2,6 +2,7 @@ package jaeger import ( "context" + "errors" "time" log "github.com/sirupsen/logrus" @@ -15,6 +16,11 @@ import ( "github.com/jaegertracing/jaeger-operator/pkg/inventory" ) +var ( + // ErrDeploymentRemoved is returned when a deployment existed but has been removed + ErrDeploymentRemoved = errors.New("deployment has been removed") +) + func (r *ReconcileJaeger) applyDeployments(jaeger v1.Jaeger, desired []appsv1.Deployment) error { opts := client.InNamespace(jaeger.Namespace).MatchingLabels(map[string]string{ "app.kubernetes.io/instance": jaeger.Name, @@ -79,10 +85,22 @@ func (r *ReconcileJaeger) applyDeployments(jaeger v1.Jaeger, desired []appsv1.De func (r *ReconcileJaeger) waitForStability(dep appsv1.Deployment) error { // TODO: decide what's a good timeout... the first cold run might take a while to download // the images, subsequent runs should take only a few seconds + + seen := false return wait.PollImmediate(time.Second, 5*time.Minute, func() (done bool, err error) { d := &appsv1.Deployment{} if err := r.client.Get(context.Background(), types.NamespacedName{Name: dep.Name, Namespace: dep.Namespace}, d); err != nil { if k8serrors.IsNotFound(err) { + if seen { + // we have seen this object before, but it doesn't exist anymore! + // we don't have anything else to do here, break the poll + log.WithFields(log.Fields{ + "namespace": dep.Namespace, + "name": dep.Name, + }).Warn("Deployment has been removed.") + return true, ErrDeploymentRemoved + } + // the object might have not been created yet log.WithFields(log.Fields{ "namespace": dep.Namespace, @@ -93,6 +111,7 @@ func (r *ReconcileJaeger) waitForStability(dep appsv1.Deployment) error { return false, err } + seen = true if d.Status.ReadyReplicas != d.Status.Replicas { log.WithFields(log.Fields{ "namespace": dep.Namespace, diff --git a/pkg/controller/jaeger/elasticsearch.go b/pkg/controller/jaeger/elasticsearch.go index 5562b5362..a73d2c895 100644 --- a/pkg/controller/jaeger/elasticsearch.go +++ b/pkg/controller/jaeger/elasticsearch.go @@ -17,6 +17,11 @@ import ( esv1 "github.com/jaegertracing/jaeger-operator/pkg/storage/elasticsearch/v1" ) +var ( + // ErrElasticsearchRemoved is returned when an ES cluster existed but has been removed + ErrElasticsearchRemoved = errors.New("Elasticsearch cluster has been removed") +) + func (r *ReconcileJaeger) applyElasticsearches(jaeger v1.Jaeger, desired []esv1.Elasticsearch) error { opts := client.InNamespace(jaeger.Namespace).MatchingLabels(map[string]string{ "app.kubernetes.io/instance": jaeger.Name, @@ -69,6 +74,8 @@ func waitForAvailableElastic(c client.Client, es esv1.Elasticsearch) error { for _, n := range es.Spec.Nodes { expectedSize += n.NodeCount } + + seen := false return wait.PollImmediate(time.Second, 2*time.Minute, func() (done bool, err error) { depList := corev1.DeploymentList{} labels := map[string]string{ @@ -77,6 +84,16 @@ func waitForAvailableElastic(c client.Client, es esv1.Elasticsearch) error { } if err = c.List(context.Background(), client.MatchingLabels(labels).InNamespace(es.Namespace), &depList); err != nil { if k8serrors.IsNotFound(err) { + if seen { + // we have seen this object before, but it doesn't exist anymore! + // we don't have anything else to do here, break the poll + log.WithFields(log.Fields{ + "namespace": es.Namespace, + "name": es.Name, + }).Warn("Elasticsearch cluster has been removed.") + return true, ErrElasticsearchRemoved + } + // the object might have not been created yet log.WithFields(log.Fields{ "namespace": es.Namespace, @@ -86,6 +103,8 @@ func waitForAvailableElastic(c client.Client, es esv1.Elasticsearch) error { } return false, err } + + seen = true availableDep := int32(0) for _, d := range depList.Items { if d.Status.Replicas == d.Status.AvailableReplicas {