Recognize when a resource has been deleted while the operator waits

pkg/controller/jaeger/dependencies.go

@@ -2,6 +2,7 @@ package jaeger
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -14,6 +15,11 @@ import (
 	"github.com/jaegertracing/jaeger-operator/pkg/strategy"
 )
 
+var (
+	// ErrDependencyRemoved is returned when a dependency existed but has been removed
+	ErrDependencyRemoved = errors.New("dependency has been removed")
+)
+
 func (r *ReconcileJaeger) handleDependencies(str strategy.S) error {
 	for _, dep := range str.Dependencies() {
 		err := r.client.Create(context.Background(), &dep)
@@ -28,10 +34,21 @@ func (r *ReconcileJaeger) handleDependencies(str strategy.S) error {
 			deadline = time.Duration(int64(*dep.Spec.ActiveDeadlineSeconds))
 		}
 
+		seen := false
 		return wait.PollImmediate(time.Second, deadline*time.Second, func() (done bool, err error) {
 			batch := &batchv1.Job{}
 			if err = r.client.Get(context.Background(), types.NamespacedName{Name: dep.Name, Namespace: dep.Namespace}, batch); err != nil {
 				if k8serrors.IsNotFound(err) {
+					if seen {
+						// we have seen this object before, but it doesn't exist anymore!
+						// we don't have anything else to do here, break the poll
+						log.WithFields(log.Fields{
+							"namespace": dep.Namespace,
+							"name":      dep.Name,
+						}).Warn("Dependency has been removed.")
+						return true, ErrDependencyRemoved
+					}
+
 					// the object might have not been created yet
 					log.WithFields(log.Fields{
 						"namespace": dep.Namespace,
@@ -42,6 +59,7 @@ func (r *ReconcileJaeger) handleDependencies(str strategy.S) error {
 				return false, err
 			}
 
+			seen = true
 			// for now, we just assume each batch job has one pod
 			if batch.Status.Succeeded != 1 {
 				log.WithFields(log.Fields{

pkg/controller/jaeger/deployment.go

@@ -2,6 +2,7 @@ package jaeger
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -15,6 +16,11 @@ import (
 	"github.com/jaegertracing/jaeger-operator/pkg/inventory"
 )
 
+var (
+	// ErrDeploymentRemoved is returned when a deployment existed but has been removed
+	ErrDeploymentRemoved = errors.New("deployment has been removed")
+)
+
 func (r *ReconcileJaeger) applyDeployments(jaeger v1.Jaeger, desired []appsv1.Deployment) error {
 	opts := client.InNamespace(jaeger.Namespace).MatchingLabels(map[string]string{
 		"app.kubernetes.io/instance":   jaeger.Name,
@@ -79,10 +85,22 @@ func (r *ReconcileJaeger) applyDeployments(jaeger v1.Jaeger, desired []appsv1.De
 func (r *ReconcileJaeger) waitForStability(dep appsv1.Deployment) error {
 	// TODO: decide what's a good timeout... the first cold run might take a while to download
 	// the images, subsequent runs should take only a few seconds
+
+	seen := false
 	return wait.PollImmediate(time.Second, 5*time.Minute, func() (done bool, err error) {
 		d := &appsv1.Deployment{}
 		if err := r.client.Get(context.Background(), types.NamespacedName{Name: dep.Name, Namespace: dep.Namespace}, d); err != nil {
 			if k8serrors.IsNotFound(err) {
+				if seen {
+					// we have seen this object before, but it doesn't exist anymore!
+					// we don't have anything else to do here, break the poll
+					log.WithFields(log.Fields{
+						"namespace": dep.Namespace,
+						"name":      dep.Name,
+					}).Warn("Deployment has been removed.")
+					return true, ErrDeploymentRemoved
+				}
+
 				// the object might have not been created yet
 				log.WithFields(log.Fields{
 					"namespace": dep.Namespace,
@@ -93,6 +111,7 @@ func (r *ReconcileJaeger) waitForStability(dep appsv1.Deployment) error {
 			return false, err
 		}
 
+		seen = true
 		if d.Status.ReadyReplicas != d.Status.Replicas {
 			log.WithFields(log.Fields{
 				"namespace": dep.Namespace,

pkg/controller/jaeger/elasticsearch.go

@@ -17,6 +17,11 @@ import (
 	esv1 "github.com/jaegertracing/jaeger-operator/pkg/storage/elasticsearch/v1"
 )
 
+var (
+	// ErrElasticsearchRemoved is returned when an ES cluster existed but has been removed
+	ErrElasticsearchRemoved = errors.New("Elasticsearch cluster has been removed")
+)
+
 func (r *ReconcileJaeger) applyElasticsearches(jaeger v1.Jaeger, desired []esv1.Elasticsearch) error {
 	opts := client.InNamespace(jaeger.Namespace).MatchingLabels(map[string]string{
 		"app.kubernetes.io/instance": jaeger.Name,
@@ -69,6 +74,8 @@ func waitForAvailableElastic(c client.Client, es esv1.Elasticsearch) error {
 	for _, n := range es.Spec.Nodes {
 		expectedSize += n.NodeCount
 	}
+
+	seen := false
 	return wait.PollImmediate(time.Second, 2*time.Minute, func() (done bool, err error) {
 		depList := corev1.DeploymentList{}
 		labels := map[string]string{
@@ -77,6 +84,16 @@ func waitForAvailableElastic(c client.Client, es esv1.Elasticsearch) error {
 		}
 		if err = c.List(context.Background(), client.MatchingLabels(labels).InNamespace(es.Namespace), &depList); err != nil {
 			if k8serrors.IsNotFound(err) {
+				if seen {
+					// we have seen this object before, but it doesn't exist anymore!
+					// we don't have anything else to do here, break the poll
+					log.WithFields(log.Fields{
+						"namespace": es.Namespace,
+						"name":      es.Name,
+					}).Warn("Elasticsearch cluster has been removed.")
+					return true, ErrElasticsearchRemoved
+				}
+
 				// the object might have not been created yet
 				log.WithFields(log.Fields{
 					"namespace": es.Namespace,
@@ -86,6 +103,8 @@ func waitForAvailableElastic(c client.Client, es esv1.Elasticsearch) error {
 			}
 			return false, err
 		}
+
+		seen = true
 		availableDep := int32(0)
 		for _, d := range depList.Items {
 			if d.Status.Replicas == d.Status.AvailableReplicas {