Skip to content

Commit

Permalink
Fix secret creation when using self provisioned elasticsearch instanc…
Browse files Browse the repository at this point in the history
…es (#1288)

Signed-off-by: Kevin Earls <[email protected]>
  • Loading branch information
kevinearls authored Nov 5, 2020
1 parent 6e724f3 commit b7d3cfd
Show file tree
Hide file tree
Showing 6 changed files with 221 additions and 64 deletions.
5 changes: 5 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,11 @@ e2e-tests-autoscale: prepare-e2e-tests es kafka
@echo Running Autoscale end-to-end tests...
@STORAGE_NAMESPACE=$(STORAGE_NAMESPACE) KAFKA_NAMESPACE=$(KAFKA_NAMESPACE) go test -tags=autoscale ./test/e2e/... $(TEST_OPTIONS)

.PHONY: e2e-tests-multi-instance
e2e-tests-multi-instance: prepare-e2e-tests es kafka
@echo Running Multiple Instance end-to-end tests...
@STORAGE_NAMESPACE=$(STORAGE_NAMESPACE) KAFKA_NAMESPACE=$(KAFKA_NAMESPACE) go test -tags=multiple ./test/e2e/... $(TEST_OPTIONS)

.PHONY: e2e-tests-upgrade
e2e-tests-upgrade: prepare-e2e-tests
@echo Prepare next version image...
Expand Down
14 changes: 13 additions & 1 deletion pkg/controller/jaeger/jaeger_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,9 @@ func (r *ReconcileJaeger) apply(ctx context.Context, jaeger v1.Jaeger, str strat
}
return jaeger, tracing.HandleError(err, span)
}
es := &storage.ElasticsearchDeployment{Jaeger: &jaeger, CertScript: "./scripts/cert_generation.sh", Secrets: secrets.Items}
secretsForNamespace := r.getSecretsForNamespace(secrets.Items, jaeger.Namespace)

es := &storage.ElasticsearchDeployment{Jaeger: &jaeger, CertScript: "./scripts/cert_generation.sh", Secrets: secretsForNamespace}
err = es.CreateCerts()
if err != nil {
es.Jaeger.Logger().WithError(err).Error("failed to create Elasticsearch certificates, Elasticsearch won't be deployed")
Expand Down Expand Up @@ -363,3 +365,13 @@ func (r *ReconcileJaeger) apply(ctx context.Context, jaeger v1.Jaeger, str strat

return jaeger, nil
}

func (r ReconcileJaeger) getSecretsForNamespace(secrets []corev1.Secret, namespace string) []corev1.Secret {
var secretsForNamespace []corev1.Secret
for _, secret := range secrets {
if secret.Namespace == namespace {
secretsForNamespace = append(secretsForNamespace, secret)
}
}
return secretsForNamespace
}
29 changes: 29 additions & 0 deletions pkg/controller/jaeger/jaeger_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ import (
osv1 "github.com/openshift/api/route/v1"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes/scheme"
Expand Down Expand Up @@ -180,6 +182,33 @@ func TestGetResourceFromNonCachedClient(t *testing.T) {
assert.True(t, errors.IsNotFound(err))
}

func TestGetSecretsForNamespace(t *testing.T) {
r := &ReconcileJaeger{}

secretOne := createSecret("foo", "secretOne")
secretTwo := createSecret("foo", "secretTwo")

secrets := []corev1.Secret{secretOne, secretTwo}
filteredSecrets := r.getSecretsForNamespace(secrets, "foo")
assert.Equal(t, 2, len(filteredSecrets))

secretThree := createSecret("bar", "secretThree")
secrets = append(secrets, secretThree)
filteredSecrets = r.getSecretsForNamespace(secrets, "bar")
assert.Equal(t, 1, len(filteredSecrets))
assert.Contains(t, filteredSecrets, secretThree)
}

func createSecret(secretNamespace, secretName string) corev1.Secret {
return corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: secretNamespace,
},
Type: corev1.SecretTypeOpaque,
}
}

func getReconciler(objs []runtime.Object) (*ReconcileJaeger, client.Client) {
s := scheme.Scheme

Expand Down
94 changes: 94 additions & 0 deletions test/e2e/multiple_instances_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// +build multiple

package e2e

import (
"context"
"testing"

framework "github.com/operator-framework/operator-sdk/pkg/test"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type MultipleInstanceTestSuite struct {
suite.Suite
}

func (suite *MultipleInstanceTestSuite) SetupSuite() {
t = suite.T()
var err error
ctx, err = prepare(t)
if err != nil {
if ctx != nil {
ctx.Cleanup()
}
require.FailNow(t, "Failed in prepare")
}
fw = framework.Global
namespace = ctx.GetID()
require.NotNil(t, namespace, "GetID failed")

addToFrameworkSchemeForSmokeTests(t)
}

func (suite *MultipleInstanceTestSuite) TearDownSuite() {
handleSuiteTearDown()
}

func TestMultipleInstanceSuite(t *testing.T) {
suite.Run(t, new(MultipleInstanceTestSuite))
}

func (suite *MultipleInstanceTestSuite) SetupTest() {
t = suite.T()
}

func (suite *MultipleInstanceTestSuite) AfterTest(suiteName, testName string) {
handleTestFailure()
}

/*
* This test verifies that we create the elasticsearch secrets correctly if someone creates production Jaeger
* instances with the same name in different namespaces
*/
func (suite *MultipleInstanceTestSuite) TestVerifySecrets() {
if !isOpenShift(t) {
t.Skip("This test is currently only supported on OpenShift")
}

jaegerInstanceName := "simple-prod"
// In production we'd use 3 nodes but 1 is sufficient for this test.
jaegerInstance := getJaegerSelfProvSimpleProd(jaegerInstanceName, namespace, 1)
createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace)
defer undeployJaegerInstance(jaegerInstance)

// Create a second instance with the same name but in a different namespace
secondContext, err := createNewTestContext()
defer secondContext.Cleanup()
secondNamespace := secondContext.GetID()
secondJaegerInstance := getJaegerSelfProvSimpleProd(jaegerInstanceName, secondNamespace, 1)
createESSelfProvDeployment(secondJaegerInstance, jaegerInstanceName, secondNamespace)
defer undeployJaegerInstance(secondJaegerInstance)

// Get the secrets from both and verify that the logging-es.crt values differ
secretOne, err := fw.KubeClient.CoreV1().Secrets(namespace).Get(context.Background(), "elasticsearch", metav1.GetOptions{})
require.NoError(t, err)
loggingEsCrtOne := secretOne.Data["logging-es.crt"]
require.NotNil(t, loggingEsCrtOne)

secretTwo, err := fw.KubeClient.CoreV1().Secrets(secondNamespace).Get(context.Background(), "elasticsearch", metav1.GetOptions{})
require.NoError(t, err)
loggingEsCrtTwo := secretTwo.Data["logging-es.crt"]
require.NotNil(t, loggingEsCrtTwo)

require.NotEqual(t, string(loggingEsCrtOne), string(loggingEsCrtTwo))
}

func createNewTestContext() (*framework.Context, error) {
secondContext, err := prepare(t)
require.NoError(t, err, "Failed trying to create a new test context")

return secondContext, err
}
70 changes: 7 additions & 63 deletions test/e2e/self_provisioned_elasticsearch_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ package e2e

import (
"context"
goctx "context"
"fmt"
"os"
"strings"
Expand All @@ -16,8 +15,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
Expand Down Expand Up @@ -83,16 +80,9 @@ func (suite *SelfProvisionedTestSuite) AfterTest(suiteName, testName string) {
func (suite *SelfProvisionedTestSuite) TestSelfProvisionedESSmokeTest() {
// create jaeger custom resource
jaegerInstanceName := "simple-prod"
exampleJaeger := getJaegerSimpleProd(jaegerInstanceName)
err := fw.Client.Create(goctx.TODO(), exampleJaeger, &framework.CleanupOptions{TestContext: ctx, Timeout: timeout, RetryInterval: retryInterval})
require.NoError(t, err, "Error deploying example Jaeger")
defer undeployJaegerInstance(exampleJaeger)

err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName+"-collector", 1, retryInterval, timeout)
require.NoError(t, err, "Error waiting for collector deployment")

err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName+"-query", 1, retryInterval, timeout)
require.NoError(t, err, "Error waiting for query deployment")
jaegerInstance := getJaegerSelfProvSimpleProd(jaegerInstanceName, namespace, 1)
createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace)
defer undeployJaegerInstance(jaegerInstance)

ProductionSmokeTest(jaegerInstanceName)

Expand All @@ -102,16 +92,9 @@ func (suite *SelfProvisionedTestSuite) TestSelfProvisionedESSmokeTest() {

func (suite *SelfProvisionedTestSuite) TestIncreasingReplicas() {
jaegerInstanceName := "simple-prod2"
exampleJaeger := getJaegerSimpleProd(jaegerInstanceName)
err := fw.Client.Create(goctx.TODO(), exampleJaeger, &framework.CleanupOptions{TestContext: ctx, Timeout: timeout, RetryInterval: retryInterval})
require.NoError(t, err, "Error deploying example Jaeger")
defer undeployJaegerInstance(exampleJaeger)

err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName+"-collector", 1, retryInterval, timeout)
require.NoError(t, err, "Error waiting for collector deployment")

err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName+"-query", 1, retryInterval, timeout)
require.NoError(t, err, "Error waiting for query deployment")
jaegerInstance := getJaegerSelfProvSimpleProd(jaegerInstanceName, namespace, 1)
createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace)
defer undeployJaegerInstance(jaegerInstance)

ProductionSmokeTest(jaegerInstanceName)

Expand All @@ -125,7 +108,7 @@ func (suite *SelfProvisionedTestSuite) TestIncreasingReplicas() {
require.EqualValues(t, updateCollectorCount, *updatedJaegerInstance.Spec.Collector.Replicas)
require.EqualValues(t, updateQueryCount, *updatedJaegerInstance.Spec.Query.Replicas)

err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName+"-collector", int(updateCollectorCount), retryInterval, timeout)
err := e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName+"-collector", int(updateCollectorCount), retryInterval, timeout)
require.NoError(t, err, "Error waiting for collector deployment")

err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName+"-query", int(updateQueryCount), retryInterval, timeout)
Expand Down Expand Up @@ -215,45 +198,6 @@ func (suite *SelfProvisionedTestSuite) TestValidateEsOperatorImage() {
require.Equal(t, expectedEsOperatorImage, imageName)
}

func getJaegerSimpleProd(instanceName string) *v1.Jaeger {
ingressEnabled := true
exampleJaeger := &v1.Jaeger{
TypeMeta: metav1.TypeMeta{
Kind: "Jaeger",
APIVersion: "jaegertracing.io/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: instanceName,
Namespace: namespace,
},
Spec: v1.JaegerSpec{
Ingress: v1.JaegerIngressSpec{
Enabled: &ingressEnabled,
Security: v1.IngressSecurityNoneExplicit,
},
Strategy: v1.DeploymentStrategyProduction,
Storage: v1.JaegerStorageSpec{
Type: v1.JaegerESStorage,
Elasticsearch: v1.ElasticsearchSpec{
NodeCount: 1,
Resources: &corev1.ResourceRequirements{
Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")},
Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")},
},
},
},
},
}

if specifyOtelImages {
logrus.Infof("Using OTEL collector for %s", instanceName)
exampleJaeger.Spec.Collector.Image = otelCollectorImage
exampleJaeger.Spec.Collector.Config = v1.NewFreeForm(getOtelConfigForHealthCheckPort("14269"))
}

return exampleJaeger
}

func getElasticSearchOperatorImage(kubeclient kubernetes.Interface, namespace string) string {
deployment, err := kubeclient.AppsV1().Deployments(namespace).Get(context.Background(), "elasticsearch-operator", metav1.GetOptions{})
require.NoErrorf(t, err, "Did not find elasticsearch-operator in namespace %s\n", namespace)
Expand Down
73 changes: 73 additions & 0 deletions test/e2e/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,10 @@ import (
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
Expand Down Expand Up @@ -729,3 +731,74 @@ func waitForElasticSearch() {
err := WaitForStatefulset(t, fw.KubeClient, storageNamespace, string(v1.JaegerESStorage), retryInterval, timeout)
require.NoError(t, err, "Error waiting for elasticsearch")
}

func getJaegerSelfProvSimpleProd(instanceName, namespace string, nodeCount int32) *v1.Jaeger {
ingressEnabled := true
exampleJaeger := &v1.Jaeger{
TypeMeta: metav1.TypeMeta{
Kind: "Jaeger",
APIVersion: "jaegertracing.io/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: instanceName,
Namespace: namespace,
},
Spec: v1.JaegerSpec{
Ingress: v1.JaegerIngressSpec{
Enabled: &ingressEnabled,
Security: v1.IngressSecurityNoneExplicit,
},
Strategy: v1.DeploymentStrategyProduction,
Storage: v1.JaegerStorageSpec{
Type: v1.JaegerESStorage,
Elasticsearch: v1.ElasticsearchSpec{
NodeCount: nodeCount,
Resources: &corev1.ResourceRequirements{
Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")},
Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")},
},
},
},
},
}

if specifyOtelImages {
logrus.Infof("Using OTEL collector for %s", instanceName)
exampleJaeger.Spec.Collector.Image = otelCollectorImage
exampleJaeger.Spec.Collector.Config = v1.NewFreeForm(getOtelConfigForHealthCheckPort("14269"))
}

return exampleJaeger
}

func createESSelfProvDeployment(jaegerInstance *v1.Jaeger, jaegerInstanceName, jaegerNamespace string) {
err := fw.Client.Create(context.TODO(), jaegerInstance, &framework.CleanupOptions{TestContext: ctx, Timeout: timeout, RetryInterval: retryInterval})
require.NoError(t, err, "Error deploying example Jaeger")

// Wait for all elasticsearch instances to appear
listOptions := &metav1.ListOptions{LabelSelector: "component=elasticsearch"}
var deployments []appsv1.Deployment
err = wait.Poll(retryInterval, timeout, func() (done bool, err error) {
esDeployments, err := fw.KubeClient.AppsV1().Deployments(jaegerNamespace).List(context.Background(), *listOptions)
if int32(len(esDeployments.Items)) == jaegerInstance.Spec.Storage.Elasticsearch.NodeCount {
deployments = esDeployments.Items
return true, nil
}
return false, nil
})
require.NoError(t, err, "Failed waiting for elasticsearch deployments to be available")

// And then wait for them to finish deploying
for _, deployment := range deployments {
logrus.Infof("Waiting for deployment of %s", deployment.Name)
err = e2eutil.WaitForDeployment(t, fw.KubeClient, jaegerNamespace, deployment.Name, 1, retryInterval, timeout)
require.NoError(t, err, "Failed waiting for elasticsearch deployment(s) %s to start", deployment.Name)
}

err = e2eutil.WaitForDeployment(t, fw.KubeClient, jaegerNamespace, jaegerInstanceName+"-collector", 1, retryInterval, timeout)
require.NoError(t, err, "Error waiting for collector deployment")

err = e2eutil.WaitForDeployment(t, fw.KubeClient, jaegerNamespace, jaegerInstanceName+"-query", 1, retryInterval, timeout)
require.NoError(t, err, "Error waiting for query deployment")
logrus.Infof("Jaeger instance %s finished deploying in %s", jaegerInstanceName, jaegerNamespace)
}

0 comments on commit b7d3cfd

Please sign in to comment.