Make RHOL Elasticsearch cert-management feature optional (#1824)

* Make RHOL Elasticsearch cert-management feature optional The reason is that the cert-management is not supported on OCP 4.6. The feature was added in RHOL 5.2 shipped on OCP 4.7. Signed-off-by: Pavol Loffay <[email protected]> * Fix Signed-off-by: Pavol Loffay <[email protected]> * Fix Signed-off-by: Pavol Loffay <[email protected]> * Rename Signed-off-by: Pavol Loffay <[email protected]>
jaegertracing · Mar 24, 2022 · bf3b076 · bf3b076
1 parent a530a02
commit bf3b076
Show file tree

Hide file tree

Showing 13 changed files with 1,114 additions and 108 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -32,9 +32,22 @@ ARG TARGETARCH
 # Build
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on go build -ldflags="-X ${VERSION_PKG}.version=${VERSION} -X ${VERSION_PKG}.buildDate=${VERSION_DATE} -X ${VERSION_PKG}.defaultJaeger=${JAEGER_VERSION}" -a -o jaeger-operator main.go
 
-FROM gcr.io/distroless/static:nonroot
+FROM registry.access.redhat.com/ubi8/ubi
+
+ENV USER_UID=1001 \
+    USER_NAME=jaeger-operator
+
+RUN INSTALL_PKGS="openssl" && \
+    yum install -y $INSTALL_PKGS && \
+    rpm -V $INSTALL_PKGS && \
+    yum clean all && \
+    mkdir /tmp/_working_dir && \
+    chmod og+w /tmp/_working_dir
+
 WORKDIR /
 COPY --from=builder /workspace/jaeger-operator .
-USER 65532:65532
+COPY scripts/cert_generation.sh scripts/cert_generation.sh
+
+USER ${USER_UID}:${USER_UID}
 
 ENTRYPOINT ["/jaeger-operator"]
diff --git a/apis/v1/jaeger_types.go b/apis/v1/jaeger_types.go
@@ -534,6 +534,12 @@ type ElasticsearchSpec struct {
 	// +optional
 	DoNotProvision bool `json:"doNotProvision,omitempty"`
 
+	// Whether Elasticsearch cert management feature should be used.
+	// This is a preferred setting for new Jaeger deployments on OCP versions newer than 4.6.
+	// The cert management feature was added to Red Hat Openshift logging 5.2 in OCP 4.7.
+	// +optional
+	UseCertManagement *bool `json:"useCertManagement,omitempty"`
+
 	// +optional
 	Image string `json:"image,omitempty"`
 

diff --git a/apis/v1/zz_generated.deepcopy.go b/apis/v1/zz_generated.deepcopy.go
diff --git a/bundle/manifests/jaegertracing.io_jaegers.yaml b/bundle/manifests/jaegertracing.io_jaegers.yaml
@@ -9642,6 +9642,8 @@ spec:
                           type: object
                         type: array
                         x-kubernetes-list-type: atomic
+                      useCertManagement:
+                        type: boolean
                     type: object
                   esIndexCleaner:
                     properties:

diff --git a/config/crd/bases/jaegertracing.io_jaegers.yaml b/config/crd/bases/jaegertracing.io_jaegers.yaml
@@ -9640,6 +9640,8 @@ spec:
                           type: object
                         type: array
                         x-kubernetes-list-type: atomic
+                      useCertManagement:
+                        type: boolean
                     type: object
                   esIndexCleaner:
                     properties:

diff --git a/docs/api.md b/docs/api.md
@@ -37503,6 +37503,13 @@ Resource Types:
           <br/>
         </td>
         <td>false</td>
+      </tr><tr>
+        <td><b>useCertManagement</b></td>
+        <td>boolean</td>
+        <td>
+          <br/>
+        </td>
+        <td>false</td>
       </tr></tbody>
 </table>
 

diff --git a/pkg/controller/jaeger/jaeger_controller.go b/pkg/controller/jaeger/jaeger_controller.go
@@ -21,6 +21,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	v1 "github.com/jaegertracing/jaeger-operator/apis/v1"
+	"github.com/jaegertracing/jaeger-operator/pkg/storage"
 	"github.com/jaegertracing/jaeger-operator/pkg/strategy"
 	"github.com/jaegertracing/jaeger-operator/pkg/tracing"
 )
@@ -29,19 +30,21 @@ import (
 type ReconcileJaeger struct {
 	// This client, initialized using mgr.Client() above, is a split client
 	// that reads objects from the cache and writes to the apiserver
-	client          client.Client
-	rClient         client.Reader
-	scheme          *runtime.Scheme
-	strategyChooser func(context.Context, *v1.Jaeger) strategy.S
+	client               client.Client
+	rClient              client.Reader
+	scheme               *runtime.Scheme
+	strategyChooser      func(context.Context, *v1.Jaeger) strategy.S
+	certGenerationScript string
 }
 
 // New creates new jaeger controller
 func New(client client.Client, clientReader client.Reader, scheme *runtime.Scheme) *ReconcileJaeger {
 	return &ReconcileJaeger{
-		client:          client,
-		rClient:         clientReader,
-		scheme:          scheme,
-		strategyChooser: defaultStrategyChooser,
+		client:               client,
+		rClient:              clientReader,
+		scheme:               scheme,
+		strategyChooser:      defaultStrategyChooser,
+		certGenerationScript: "./scripts/cert_generation.sh",
 	}
 }
 
@@ -210,7 +213,37 @@ func (r *ReconcileJaeger) apply(ctx context.Context, jaeger v1.Jaeger, str strat
 		return jaeger, tracing.HandleError(err, span)
 	}
 
-	// TODO this can be removed after previously released version is using cert management from EO e.g. in 1.32.0
+	// ES cert handling requires secrets from environment
+	// therefore running this here and not in the strategy
+	if v1.ShouldInjectOpenShiftElasticsearchConfiguration(jaeger.Spec.Storage) &&
+		// generate the certs only if cert management is disabled
+		(jaeger.Spec.Storage.Elasticsearch.UseCertManagement == nil ||
+			*jaeger.Spec.Storage.Elasticsearch.UseCertManagement == false) {
+
+		opts := client.MatchingLabels(map[string]string{
+			"app.kubernetes.io/instance":   jaeger.Name,
+			"app.kubernetes.io/managed-by": "jaeger-operator",
+		})
+		secrets := &corev1.SecretList{}
+		if err := r.rClient.List(ctx, secrets, opts); err != nil {
+			jaeger.Status.Phase = v1.JaegerPhaseFailed
+			if err := r.client.Status().Update(ctx, &jaeger); err != nil {
+				// we let it return the real error later
+				jaeger.Logger().WithError(err).Error("failed to store the failed status into the current CustomResource after preconditions")
+			}
+			return jaeger, tracing.HandleError(err, span)
+		}
+		secretsForNamespace := r.getSecretsForNamespace(secrets.Items, jaeger.Namespace)
+
+		es := &storage.ElasticsearchDeployment{Jaeger: &jaeger, CertScript: r.certGenerationScript, Secrets: secretsForNamespace}
+		err = es.CreateCerts()
+		if err != nil {
+			es.Jaeger.Logger().WithError(err).Error("failed to create Elasticsearch certificates, Elasticsearch won't be deployed")
+			return jaeger, err
+		}
+		str = str.WithSecrets(append(str.Secrets(), es.ExtractSecrets()...))
+	}
+
 	if err := r.applySecrets(ctx, jaeger, str.Secrets()); err != nil {
 		return jaeger, tracing.HandleError(err, span)
 	}

diff --git a/pkg/controller/jaeger/jaeger_controller_test.go b/pkg/controller/jaeger/jaeger_controller_test.go
@@ -9,6 +9,7 @@ import (
 	esv1 "github.com/openshift/elasticsearch-operator/apis/logging/v1"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -199,6 +200,39 @@ func TestGetSecretsForNamespace(t *testing.T) {
 	assert.Contains(t, filteredSecrets, secretThree)
 }
 
+func TestElasticsearchProvisioning(t *testing.T) {
+	namespacedName := types.NamespacedName{Name: "prod", Namespace: "jaeger"}
+	j := v1.NewJaeger(namespacedName)
+	j.Spec.Storage.Type = "elasticsearch"
+	j.Spec.Storage.Elasticsearch.Name = "elasticserach"
+	j.Spec.Storage.Elasticsearch.NodeCount = 1
+
+	reconciler, cl := getReconciler([]runtime.Object{j})
+
+	req := reconcile.Request{NamespacedName: namespacedName}
+	result, err := reconciler.Reconcile(req)
+	require.NoError(t, err)
+	assert.Equal(t, reconcile.Result{}, result)
+
+	secrets := &corev1.SecretList{}
+	err = cl.List(context.Background(), secrets, client.InNamespace("jaeger"))
+	require.NoError(t, err)
+	assert.Equal(t, 4, len(secrets.Items))
+	assert.NotNil(t, getSecret("prod-jaeger-elasticsearch", *secrets))
+	assert.NotNil(t, getSecret("prod-master-certs", *secrets))
+	assert.NotNil(t, getSecret("prod-curator", *secrets))
+	assert.NotNil(t, getSecret("elasticsearch", *secrets))
+}
+
+func getSecret(name string, secrets corev1.SecretList) *corev1.Secret {
+	for _, s := range secrets.Items {
+		if s.Name == name {
+			return &s
+		}
+	}
+	return nil
+}
+
 func createSecret(secretNamespace, secretName string) corev1.Secret {
 	return corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
@@ -228,5 +262,7 @@ func getReconciler(objs []runtime.Object) (*ReconcileJaeger, client.Client) {
 	s.AddKnownTypes(v1beta2.GroupVersion, &v1beta2.Kafka{}, &v1beta2.KafkaList{}, &v1beta2.KafkaUser{}, &v1beta2.KafkaUserList{})
 
 	cl := fake.NewFakeClient(objs...)
-	return &ReconcileJaeger{client: cl, scheme: s, rClient: cl}, cl
+	r := New(cl, cl, s)
+	r.certGenerationScript = "../../../scripts/cert_generation.sh"
+	return r, cl
 }
diff --git a/pkg/storage/elasticsearch.go b/pkg/storage/elasticsearch.go
@@ -5,23 +5,46 @@ import (
 	"strconv"
 	"strings"
 
+	esv1 "github.com/openshift/elasticsearch-operator/apis/logging/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	esv1 "github.com/openshift/elasticsearch-operator/apis/logging/v1"
-
 	v1 "github.com/jaegertracing/jaeger-operator/apis/v1"
 	"github.com/jaegertracing/jaeger-operator/pkg/util"
 )
 
 const (
-	volumeName      = "certs"
-	volumeMountPath = "/certs"
-	caPath          = volumeMountPath + "/ca-bundle.crt"
-	keyPath         = volumeMountPath + "/tls.key"
-	certPath        = volumeMountPath + "/tls.crt"
+	volumeName               = "certs"
+	volumeMountPath          = "/certs"
+	caPathESCerManagement    = volumeMountPath + "/ca-bundle.crt"
+	keyPathESCertManagement  = volumeMountPath + "/tls.key"
+	certPathESCertManagement = volumeMountPath + "/tls.crt"
+	caPath                   = volumeMountPath + "/ca"
+	keyPath                  = volumeMountPath + "/key"
+	certPath                 = volumeMountPath + "/cert"
 )
 
+func (ed *ElasticsearchDeployment) getCertPath() string {
+	if ed.Jaeger.Spec.Storage.Elasticsearch.UseCertManagement != nil && *ed.Jaeger.Spec.Storage.Elasticsearch.UseCertManagement {
+		return certPathESCertManagement
+	}
+	return certPath
+}
+
+func (ed *ElasticsearchDeployment) getCertKeyPath() string {
+	if ed.Jaeger.Spec.Storage.Elasticsearch.UseCertManagement != nil && *ed.Jaeger.Spec.Storage.Elasticsearch.UseCertManagement {
+		return keyPathESCertManagement
+	}
+	return keyPath
+}
+
+func (ed *ElasticsearchDeployment) getCertCaPath() string {
+	if ed.Jaeger.Spec.Storage.Elasticsearch.UseCertManagement != nil && *ed.Jaeger.Spec.Storage.Elasticsearch.UseCertManagement {
+		return caPathESCerManagement
+	}
+	return caPath
+}
+
 // ElasticsearchDeployment represents an ES deployment for Jaeger
 type ElasticsearchDeployment struct {
 	Jaeger     *v1.Jaeger
@@ -35,9 +58,9 @@ func (ed *ElasticsearchDeployment) injectArguments(container *corev1.Container)
 		container.Args = append(container.Args, "--es.tls.enabled=true")
 	}
 	container.Args = append(container.Args,
-		"--es.tls.ca="+caPath,
-		"--es.tls.cert="+certPath,
-		"--es.tls.key="+keyPath)
+		"--es.tls.ca="+ed.getCertCaPath(),
+		"--es.tls.cert="+ed.getCertPath(),
+		"--es.tls.key="+ed.getCertKeyPath())
 
 	if util.FindItem("--es.timeout", container.Args) == "" {
 		container.Args = append(container.Args, "--es.timeout=15s")
@@ -57,9 +80,9 @@ func (ed *ElasticsearchDeployment) injectArguments(container *corev1.Container)
 			container.Args = append(container.Args, "--es-archive.tls.enabled=true")
 		}
 		container.Args = append(container.Args,
-			"--es-archive.tls.ca="+caPath,
-			"--es-archive.tls.cert="+certPath,
-			"--es-archive.tls.key="+keyPath,
+			"--es-archive.tls.ca="+ed.getCertCaPath(),
+			"--es-archive.tls.cert="+ed.getCertPath(),
+			"--es-archive.tls.key="+ed.getCertKeyPath(),
 		)
 		if util.FindItem("--es-archive.timeout", container.Args) == "" {
 			container.Args = append(container.Args, "--es-archive.timeout=15s")
@@ -82,7 +105,7 @@ func (ed *ElasticsearchDeployment) InjectStorageConfiguration(p *corev1.PodSpec)
 		Name: volumeName,
 		VolumeSource: corev1.VolumeSource{
 			Secret: &corev1.SecretVolumeSource{
-				SecretName: jaegerESSecretName(ed.Jaeger.Spec.Storage.Elasticsearch.Name),
+				SecretName: jaegerESSecretName(*ed.Jaeger),
 			},
 		},
 	})
@@ -103,7 +126,7 @@ func (ed *ElasticsearchDeployment) InjectSecretsConfiguration(p *corev1.PodSpec)
 		Name: volumeName,
 		VolumeSource: corev1.VolumeSource{
 			Secret: &corev1.SecretVolumeSource{
-				SecretName: jaegerESSecretName(ed.Jaeger.Spec.Storage.Elasticsearch.Name),
+				SecretName: jaegerESSecretName(*ed.Jaeger),
 			},
 		},
 	})
@@ -113,9 +136,9 @@ func (ed *ElasticsearchDeployment) InjectSecretsConfiguration(p *corev1.PodSpec)
 		p.Containers[0].Args[1] = fmt.Sprintf("https://%s:9200", ed.Jaeger.Spec.Storage.Elasticsearch.Name)
 		p.Containers[0].Env = append(p.Containers[0].Env,
 			corev1.EnvVar{Name: "ES_TLS", Value: "true"},
-			corev1.EnvVar{Name: "ES_TLS_CA", Value: caPath},
-			corev1.EnvVar{Name: "ES_TLS_KEY", Value: keyPath},
-			corev1.EnvVar{Name: "ES_TLS_CERT", Value: certPath},
+			corev1.EnvVar{Name: "ES_TLS_CA", Value: ed.getCertCaPath()},
+			corev1.EnvVar{Name: "ES_TLS_KEY", Value: ed.getCertKeyPath()},
+			corev1.EnvVar{Name: "ES_TLS_CERT", Value: ed.getCertPath()},
 			corev1.EnvVar{Name: "SHARDS", Value: strconv.Itoa(int(ed.Jaeger.Spec.Storage.Elasticsearch.NodeCount))},
 			corev1.EnvVar{Name: "REPLICAS", Value: strconv.Itoa(calculateReplicaShards(ed.Jaeger.Spec.Storage.Elasticsearch.RedundancyPolicy, int(ed.Jaeger.Spec.Storage.Elasticsearch.NodeCount)))},
 		)
@@ -143,6 +166,14 @@ func (ed *ElasticsearchDeployment) Elasticsearch() *esv1.Elasticsearch {
 	if ed.Jaeger.Spec.Storage.Elasticsearch.Resources != nil {
 		res = *ed.Jaeger.Spec.Storage.Elasticsearch.Resources
 	}
+
+	annotations := map[string]string{}
+	if ed.Jaeger.Spec.Storage.Elasticsearch.UseCertManagement != nil && *ed.Jaeger.Spec.Storage.Elasticsearch.UseCertManagement == true {
+		annotations["logging.openshift.io/elasticsearch-cert-management"] = "true"
+		// The value has to match searchguard configuration
+		// https://github.com/openshift/origin-aggregated-logging/blob/50126fb8e0c602e9c623d6a8599857aaf98f80f8/elasticsearch/sgconfig/roles_mapping.yml#L34
+		annotations[fmt.Sprintf("logging.openshift.io/elasticsearch-cert.%s", jaegerESSecretName(*ed.Jaeger))] = "user.jaeger"
+	}
 	return &esv1.Elasticsearch{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: ed.Jaeger.Namespace,
@@ -157,12 +188,7 @@ func (ed *ElasticsearchDeployment) Elasticsearch() *esv1.Elasticsearch {
 				// to manipulate with objects created by ES operator.
 				//"app.kubernetes.io/managed-by": "jaeger-operator",
 			},
-			Annotations: map[string]string{
-				"logging.openshift.io/elasticsearch-cert-management": "true",
-				// The value has to match searchguard configuration
-				// https://github.com/openshift/origin-aggregated-logging/blob/50126fb8e0c602e9c623d6a8599857aaf98f80f8/elasticsearch/sgconfig/roles_mapping.yml#L34
-				fmt.Sprintf("logging.openshift.io/elasticsearch-cert.%s", jaegerESSecretName(ed.Jaeger.Spec.Storage.Elasticsearch.Name)): "user.jaeger",
-			},
+			Annotations:     annotations,
 			OwnerReferences: []metav1.OwnerReference{util.AsOwner(ed.Jaeger)},
 		},
 		Spec: esv1.ElasticsearchSpec{
@@ -225,6 +251,12 @@ func calculateReplicaShards(policyType esv1.RedundancyPolicyType, dataNodes int)
 	}
 }
 
-func jaegerESSecretName(elasticsearch string) string {
-	return fmt.Sprintf("jaeger-%s", elasticsearch)
+func jaegerESSecretName(jaeger v1.Jaeger) string {
+	prefix := ""
+	// ES cert management creates cert named jaeger-<elasticsearch-name>
+	// Cert management in Jaeger creates cert named <jaeger-name>-jaeger-elasticsearch
+	if jaeger.Spec.Storage.Elasticsearch.UseCertManagement == nil || !*jaeger.Spec.Storage.Elasticsearch.UseCertManagement {
+		prefix = jaeger.Name + "-"
+	}
+	return fmt.Sprintf("%sjaeger-%s", prefix, jaeger.Spec.Storage.Elasticsearch.Name)
 }