From 630b0bf7de2cb41f37d94caccd248c06153c163e Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Thu, 28 Feb 2019 12:48:34 +0100 Subject: [PATCH 1/3] Wait for elasticsearch cluster to be up Signed-off-by: Pavol Loffay --- pkg/controller/jaeger/elasticsearch.go | 28 ++++++++++++++++++++++ pkg/controller/jaeger/jaeger_controller.go | 9 +++---- scripts/cert_generation.sh | 2 +- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/pkg/controller/jaeger/elasticsearch.go b/pkg/controller/jaeger/elasticsearch.go index 2832901d0..47bf55330 100644 --- a/pkg/controller/jaeger/elasticsearch.go +++ b/pkg/controller/jaeger/elasticsearch.go @@ -2,7 +2,11 @@ package jaeger import ( "context" + "time" + "github.com/pkg/errors" + "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/jaegertracing/jaeger-operator/pkg/apis/io/v1alpha1" @@ -26,6 +30,9 @@ func (r *ReconcileJaeger) applyElasticsearches(jaeger v1alpha1.Jaeger, desired [ if err := r.client.Create(context.Background(), &d); err != nil { return err } + if err := waitForAvailableElastic(r.client, d); err != nil { + return errors.Wrap(err, "elasticsearch cluster didn't get to ready state") + } } for _, d := range inv.Update { @@ -44,3 +51,24 @@ func (r *ReconcileJaeger) applyElasticsearches(jaeger v1alpha1.Jaeger, desired [ return nil } + +func waitForAvailableElastic(c client.Client, es esv1alpha1.Elasticsearch) error { + var expectedSize int32 + for _, n := range es.Spec.Nodes { + expectedSize += n.NodeCount + } + return wait.PollImmediate(time.Second, 2*time.Minute, func() (done bool, err error) { + depList := v1.DeploymentList{} + if err = c.List(context.Background(), client.MatchingLabels(es.Labels).InNamespace(es.Namespace), &depList); err != nil { + return false, err + } + available := int32(0) + for _, d := range depList.Items { + if d.Status.Replicas == d.Status.AvailableReplicas { + available++ + } + } + return available == expectedSize, nil + }) + return nil +} diff --git a/pkg/controller/jaeger/jaeger_controller.go b/pkg/controller/jaeger/jaeger_controller.go index 534d6129d..5a3cdc4fa 100644 --- a/pkg/controller/jaeger/jaeger_controller.go +++ b/pkg/controller/jaeger/jaeger_controller.go @@ -136,6 +136,11 @@ func defaultStrategyChooser(instance *v1alpha1.Jaeger) strategy.S { } func (r *ReconcileJaeger) apply(jaeger v1alpha1.Jaeger, str strategy.S) error { + // secrets have to be created before ES - they are mounted to the ES pod + if err := r.applySecrets(jaeger, str.Secrets()); err != nil { + return err + } + elasticsearches := str.Elasticsearches() if strings.EqualFold(viper.GetString("es-provision"), v1alpha1.FlagProvisionElasticsearchTrue) { if err := r.applyElasticsearches(jaeger, elasticsearches); err != nil { @@ -164,10 +169,6 @@ func (r *ReconcileJaeger) apply(jaeger v1alpha1.Jaeger, str strategy.S) error { return err } - if err := r.applySecrets(jaeger, str.Secrets()); err != nil { - return err - } - if err := r.applyCronJobs(jaeger, str.CronJobs()); err != nil { return err } diff --git a/scripts/cert_generation.sh b/scripts/cert_generation.sh index 0a60bb08d..1b2c0d4d9 100644 --- a/scripts/cert_generation.sh +++ b/scripts/cert_generation.sh @@ -1,6 +1,6 @@ #! /bin/bash -# The script it taken from https://github.com/openshift/cluster-logging-operator +# The script is taken from https://github.com/openshift/cluster-logging-operator WORKING_DIR=${WORKING_DIR:-/tmp/_working_dir} NAMESPACE=${NAMESPACE:-openshift-logging} From 92252f645c1e5d1e19a69dfcf87fe2b6e570c0ac Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Thu, 28 Feb 2019 14:53:53 +0100 Subject: [PATCH 2/3] Fix removing of ES objects by our operator Signed-off-by: Pavol Loffay --- pkg/controller/jaeger/elasticsearch.go | 4 ++-- pkg/storage/elasticsearch.go | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/controller/jaeger/elasticsearch.go b/pkg/controller/jaeger/elasticsearch.go index 47bf55330..b39a1a13d 100644 --- a/pkg/controller/jaeger/elasticsearch.go +++ b/pkg/controller/jaeger/elasticsearch.go @@ -16,8 +16,8 @@ import ( func (r *ReconcileJaeger) applyElasticsearches(jaeger v1alpha1.Jaeger, desired []esv1alpha1.Elasticsearch) error { opts := client.MatchingLabels(map[string]string{ - "app.kubernetes.io/instance": jaeger.Name, - "app.kubernetes.io/managed-by": "jaeger-operator", + "app.kubernetes.io/instance": jaeger.Name, + "app.kubernetes.io/part-of": "jaeger", }) list := &esv1alpha1.ElasticsearchList{} if err := r.client.List(context.Background(), opts, list); err != nil { diff --git a/pkg/storage/elasticsearch.go b/pkg/storage/elasticsearch.go index fc034c5ed..9d3388457 100644 --- a/pkg/storage/elasticsearch.go +++ b/pkg/storage/elasticsearch.go @@ -105,12 +105,14 @@ func (ed *ElasticsearchDeployment) Elasticsearch() *esv1alpha1.Elasticsearch { Namespace: ed.Jaeger.Namespace, Name: esSecret.name, Labels: map[string]string{ - "app": "jaeger", - "app.kubernetes.io/name": esSecret.name, - "app.kubernetes.io/instance": ed.Jaeger.Name, - "app.kubernetes.io/component": "elasticsearch", - "app.kubernetes.io/part-of": "jaeger", - "app.kubernetes.io/managed-by": "jaeger-operator", + "app": "jaeger", + "app.kubernetes.io/name": esSecret.name, + "app.kubernetes.io/instance": ed.Jaeger.Name, + "app.kubernetes.io/component": "elasticsearch", + "app.kubernetes.io/part-of": "jaeger", + // we cannot use jaeger-operator because our component controllers removes all objects + // created by ES operator + //"app.kubernetes.io/managed-by": "jaeger-operator", }, OwnerReferences: []metav1.OwnerReference{asOwner(ed.Jaeger)}, }, From 2906cb683b2683a01303d518ab9b04a9b51016c0 Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Thu, 28 Feb 2019 22:06:37 +0100 Subject: [PATCH 3/3] Fix review comments Signed-off-by: Pavol Loffay --- pkg/controller/jaeger/elasticsearch.go | 7 +++++++ pkg/storage/elasticsearch.go | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/controller/jaeger/elasticsearch.go b/pkg/controller/jaeger/elasticsearch.go index b39a1a13d..5b1112646 100644 --- a/pkg/controller/jaeger/elasticsearch.go +++ b/pkg/controller/jaeger/elasticsearch.go @@ -5,6 +5,7 @@ import ( "time" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" @@ -68,6 +69,12 @@ func waitForAvailableElastic(c client.Client, es esv1alpha1.Elasticsearch) error available++ } } + logrus.WithFields(logrus.Fields{ + "namespace": es.Namespace, + "name": es.Name, + "desiredNodes": expectedSize, + "availableNodes": available, + }).Debug("Waiting for Elasticsearch to be available") return available == expectedSize, nil }) return nil diff --git a/pkg/storage/elasticsearch.go b/pkg/storage/elasticsearch.go index 9d3388457..ba96404ba 100644 --- a/pkg/storage/elasticsearch.go +++ b/pkg/storage/elasticsearch.go @@ -110,8 +110,8 @@ func (ed *ElasticsearchDeployment) Elasticsearch() *esv1alpha1.Elasticsearch { "app.kubernetes.io/instance": ed.Jaeger.Name, "app.kubernetes.io/component": "elasticsearch", "app.kubernetes.io/part-of": "jaeger", - // we cannot use jaeger-operator because our component controllers removes all objects - // created by ES operator + // We cannot use jaeger-operator label because our controllers would try + // to manipulate with objects created by ES operator. //"app.kubernetes.io/managed-by": "jaeger-operator", }, OwnerReferences: []metav1.OwnerReference{asOwner(ed.Jaeger)},