Wait for elasticsearch cluster to be up

pkg/controller/jaeger/elasticsearch.go

@@ -2,7 +2,12 @@ package jaeger
 
 import (
 	"context"
+	"time"
 
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/jaegertracing/jaeger-operator/pkg/apis/io/v1alpha1"
@@ -12,8 +17,8 @@ import (
 
 func (r *ReconcileJaeger) applyElasticsearches(jaeger v1alpha1.Jaeger, desired []esv1alpha1.Elasticsearch) error {
 	opts := client.MatchingLabels(map[string]string{
-		"app.kubernetes.io/instance":   jaeger.Name,
-		"app.kubernetes.io/managed-by": "jaeger-operator",
+		"app.kubernetes.io/instance": jaeger.Name,
+		"app.kubernetes.io/part-of":  "jaeger",
 	})
 	list := &esv1alpha1.ElasticsearchList{}
 	if err := r.client.List(context.Background(), opts, list); err != nil {
@@ -26,6 +31,9 @@ func (r *ReconcileJaeger) applyElasticsearches(jaeger v1alpha1.Jaeger, desired [
 		if err := r.client.Create(context.Background(), &d); err != nil {
 			return err
 		}
+		if err := waitForAvailableElastic(r.client, d); err != nil {
+			return errors.Wrap(err, "elasticsearch cluster didn't get to ready state")
+		}
 	}
 
 	for _, d := range inv.Update {
@@ -44,3 +52,30 @@ func (r *ReconcileJaeger) applyElasticsearches(jaeger v1alpha1.Jaeger, desired [
 
 	return nil
 }
+
+func waitForAvailableElastic(c client.Client, es esv1alpha1.Elasticsearch) error {
+	var expectedSize int32
+	for _, n := range es.Spec.Nodes {
+		expectedSize += n.NodeCount
+	}
+	return wait.PollImmediate(time.Second, 2*time.Minute, func() (done bool, err error) {
+		depList := v1.DeploymentList{}
+		if err = c.List(context.Background(), client.MatchingLabels(es.Labels).InNamespace(es.Namespace), &depList); err != nil {
+			return false, err
+		}
+		available := int32(0)
+		for _, d := range depList.Items {
+			if d.Status.Replicas == d.Status.AvailableReplicas {
+				available++
+			}
+		}
+		logrus.WithFields(logrus.Fields{
+			"namespace":      es.Namespace,
+			"name":           es.Name,
+			"desiredNodes":   expectedSize,
+			"availableNodes": available,
+		}).Debug("Waiting for Elasticsearch to be available")
+		return available == expectedSize, nil
+	})
+	return nil
+}

pkg/controller/jaeger/jaeger_controller.go

@@ -136,6 +136,11 @@ func defaultStrategyChooser(instance *v1alpha1.Jaeger) strategy.S {
 }
 
 func (r *ReconcileJaeger) apply(jaeger v1alpha1.Jaeger, str strategy.S) error {
+	// secrets have to be created before ES - they are mounted to the ES pod
+	if err := r.applySecrets(jaeger, str.Secrets()); err != nil {
+		return err
+	}
+
 	elasticsearches := str.Elasticsearches()
 	if strings.EqualFold(viper.GetString("es-provision"), v1alpha1.FlagProvisionElasticsearchTrue) {
 		if err := r.applyElasticsearches(jaeger, elasticsearches); err != nil {
@@ -164,10 +169,6 @@ func (r *ReconcileJaeger) apply(jaeger v1alpha1.Jaeger, str strategy.S) error {
 		return err
 	}
 
-	if err := r.applySecrets(jaeger, str.Secrets()); err != nil {
-		return err
-	}
-
 	if err := r.applyCronJobs(jaeger, str.CronJobs()); err != nil {
 		return err
 	}

pkg/storage/elasticsearch.go

@@ -105,12 +105,14 @@ func (ed *ElasticsearchDeployment) Elasticsearch() *esv1alpha1.Elasticsearch {
 			Namespace: ed.Jaeger.Namespace,
 			Name:      esSecret.name,
 			Labels: map[string]string{
-				"app":                          "jaeger",
-				"app.kubernetes.io/name":       esSecret.name,
-				"app.kubernetes.io/instance":   ed.Jaeger.Name,
-				"app.kubernetes.io/component":  "elasticsearch",
-				"app.kubernetes.io/part-of":    "jaeger",
-				"app.kubernetes.io/managed-by": "jaeger-operator",
+				"app":                         "jaeger",
+				"app.kubernetes.io/name":      esSecret.name,
+				"app.kubernetes.io/instance":  ed.Jaeger.Name,
+				"app.kubernetes.io/component": "elasticsearch",
+				"app.kubernetes.io/part-of":   "jaeger",
+				// We cannot use jaeger-operator label because our controllers would try
+				// to manipulate with objects created by ES operator.
+				//"app.kubernetes.io/managed-by": "jaeger-operator",
 			},
 			OwnerReferences: []metav1.OwnerReference{asOwner(ed.Jaeger)},
 		},

scripts/cert_generation.sh

@@ -1,6 +1,6 @@
 #! /bin/bash
 
-# The script it taken from https://github.com/openshift/cluster-logging-operator
+# The script is taken from https://github.com/openshift/cluster-logging-operator
 
 WORKING_DIR=${WORKING_DIR:-/tmp/_working_dir}
 NAMESPACE=${NAMESPACE:-openshift-logging}