diff --git a/Makefile b/Makefile index 68fd0f41e..238481ecc 100644 --- a/Makefile +++ b/Makefile @@ -181,6 +181,11 @@ e2e-tests-autoscale: prepare-e2e-tests es kafka @echo Running Autoscale end-to-end tests... @STORAGE_NAMESPACE=$(STORAGE_NAMESPACE) KAFKA_NAMESPACE=$(KAFKA_NAMESPACE) go test -tags=autoscale ./test/e2e/... $(TEST_OPTIONS) +.PHONY: e2e-tests-multi-instance +e2e-tests-multi-instance: prepare-e2e-tests es kafka + @echo Running Multiple Instance end-to-end tests... + @STORAGE_NAMESPACE=$(STORAGE_NAMESPACE) KAFKA_NAMESPACE=$(KAFKA_NAMESPACE) go test -tags=multiple ./test/e2e/... $(TEST_OPTIONS) + .PHONY: e2e-tests-upgrade e2e-tests-upgrade: prepare-e2e-tests @echo Prepare next version image... diff --git a/pkg/controller/jaeger/jaeger_controller.go b/pkg/controller/jaeger/jaeger_controller.go index feeabff67..091b276b2 100644 --- a/pkg/controller/jaeger/jaeger_controller.go +++ b/pkg/controller/jaeger/jaeger_controller.go @@ -255,7 +255,9 @@ func (r *ReconcileJaeger) apply(ctx context.Context, jaeger v1.Jaeger, str strat } return jaeger, tracing.HandleError(err, span) } - es := &storage.ElasticsearchDeployment{Jaeger: &jaeger, CertScript: "./scripts/cert_generation.sh", Secrets: secrets.Items} + secretsForNamespace := r.getSecretsForNamespace(secrets.Items, jaeger.Namespace) + + es := &storage.ElasticsearchDeployment{Jaeger: &jaeger, CertScript: "./scripts/cert_generation.sh", Secrets: secretsForNamespace} err = es.CreateCerts() if err != nil { es.Jaeger.Logger().WithError(err).Error("failed to create Elasticsearch certificates, Elasticsearch won't be deployed") @@ -363,3 +365,13 @@ func (r *ReconcileJaeger) apply(ctx context.Context, jaeger v1.Jaeger, str strat return jaeger, nil } + +func (r ReconcileJaeger) getSecretsForNamespace(secrets []corev1.Secret, namespace string) []corev1.Secret { + var secretsForNamespace []corev1.Secret + for _, secret := range secrets { + if secret.Namespace == namespace { + secretsForNamespace = append(secretsForNamespace, secret) + } + } + return secretsForNamespace +} diff --git a/pkg/controller/jaeger/jaeger_controller_test.go b/pkg/controller/jaeger/jaeger_controller_test.go index 24317f4a3..ae3d7b606 100644 --- a/pkg/controller/jaeger/jaeger_controller_test.go +++ b/pkg/controller/jaeger/jaeger_controller_test.go @@ -8,7 +8,9 @@ import ( osv1 "github.com/openshift/api/route/v1" "github.com/spf13/viper" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" @@ -180,6 +182,33 @@ func TestGetResourceFromNonCachedClient(t *testing.T) { assert.True(t, errors.IsNotFound(err)) } +func TestGetSecretsForNamespace(t *testing.T) { + r := &ReconcileJaeger{} + + secretOne := createSecret("foo", "secretOne") + secretTwo := createSecret("foo", "secretTwo") + + secrets := []corev1.Secret{secretOne, secretTwo} + filteredSecrets := r.getSecretsForNamespace(secrets, "foo") + assert.Equal(t, 2, len(filteredSecrets)) + + secretThree := createSecret("bar", "secretThree") + secrets = append(secrets, secretThree) + filteredSecrets = r.getSecretsForNamespace(secrets, "bar") + assert.Equal(t, 1, len(filteredSecrets)) + assert.Contains(t, filteredSecrets, secretThree) +} + +func createSecret(secretNamespace, secretName string) corev1.Secret { + return corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: secretNamespace, + }, + Type: corev1.SecretTypeOpaque, + } +} + func getReconciler(objs []runtime.Object) (*ReconcileJaeger, client.Client) { s := scheme.Scheme diff --git a/test/e2e/multiple_instances_test.go b/test/e2e/multiple_instances_test.go new file mode 100644 index 000000000..20ec117cf --- /dev/null +++ b/test/e2e/multiple_instances_test.go @@ -0,0 +1,94 @@ +// +build multiple + +package e2e + +import ( + "context" + "testing" + + framework "github.com/operator-framework/operator-sdk/pkg/test" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type MultipleInstanceTestSuite struct { + suite.Suite +} + +func (suite *MultipleInstanceTestSuite) SetupSuite() { + t = suite.T() + var err error + ctx, err = prepare(t) + if err != nil { + if ctx != nil { + ctx.Cleanup() + } + require.FailNow(t, "Failed in prepare") + } + fw = framework.Global + namespace = ctx.GetID() + require.NotNil(t, namespace, "GetID failed") + + addToFrameworkSchemeForSmokeTests(t) +} + +func (suite *MultipleInstanceTestSuite) TearDownSuite() { + handleSuiteTearDown() +} + +func TestMultipleInstanceSuite(t *testing.T) { + suite.Run(t, new(MultipleInstanceTestSuite)) +} + +func (suite *MultipleInstanceTestSuite) SetupTest() { + t = suite.T() +} + +func (suite *MultipleInstanceTestSuite) AfterTest(suiteName, testName string) { + handleTestFailure() +} + +/* + * This test verifies that we create the elasticsearch secrets correctly if someone creates production Jaeger + * instances with the same name in different namespaces + */ +func (suite *MultipleInstanceTestSuite) TestVerifySecrets() { + if !isOpenShift(t) { + t.Skip("This test is currently only supported on OpenShift") + } + + jaegerInstanceName := "simple-prod" + // In production we'd use 3 nodes but 1 is sufficient for this test. + jaegerInstance := getJaegerSelfProvSimpleProd(jaegerInstanceName, namespace, 1) + createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace) + defer undeployJaegerInstance(jaegerInstance) + + // Create a second instance with the same name but in a different namespace + secondContext, err := createNewTestContext() + defer secondContext.Cleanup() + secondNamespace := secondContext.GetID() + secondJaegerInstance := getJaegerSelfProvSimpleProd(jaegerInstanceName, secondNamespace, 1) + createESSelfProvDeployment(secondJaegerInstance, jaegerInstanceName, secondNamespace) + defer undeployJaegerInstance(secondJaegerInstance) + + // Get the secrets from both and verify that the logging-es.crt values differ + secretOne, err := fw.KubeClient.CoreV1().Secrets(namespace).Get(context.Background(), "elasticsearch", metav1.GetOptions{}) + require.NoError(t, err) + loggingEsCrtOne := secretOne.Data["logging-es.crt"] + require.NotNil(t, loggingEsCrtOne) + + secretTwo, err := fw.KubeClient.CoreV1().Secrets(secondNamespace).Get(context.Background(), "elasticsearch", metav1.GetOptions{}) + require.NoError(t, err) + loggingEsCrtTwo := secretTwo.Data["logging-es.crt"] + require.NotNil(t, loggingEsCrtTwo) + + require.NotEqual(t, string(loggingEsCrtOne), string(loggingEsCrtTwo)) +} + +func createNewTestContext() (*framework.Context, error) { + secondContext, err := prepare(t) + require.NoError(t, err, "Failed trying to create a new test context") + + return secondContext, err +} diff --git a/test/e2e/self_provisioned_elasticsearch_test.go b/test/e2e/self_provisioned_elasticsearch_test.go index 55a7f090d..c7f86f41d 100644 --- a/test/e2e/self_provisioned_elasticsearch_test.go +++ b/test/e2e/self_provisioned_elasticsearch_test.go @@ -4,7 +4,6 @@ package e2e import ( "context" - goctx "context" "fmt" "os" "strings" @@ -16,8 +15,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" @@ -83,16 +80,9 @@ func (suite *SelfProvisionedTestSuite) AfterTest(suiteName, testName string) { func (suite *SelfProvisionedTestSuite) TestSelfProvisionedESSmokeTest() { // create jaeger custom resource jaegerInstanceName := "simple-prod" - exampleJaeger := getJaegerSimpleProd(jaegerInstanceName) - err := fw.Client.Create(goctx.TODO(), exampleJaeger, &framework.CleanupOptions{TestContext: ctx, Timeout: timeout, RetryInterval: retryInterval}) - require.NoError(t, err, "Error deploying example Jaeger") - defer undeployJaegerInstance(exampleJaeger) - - err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName+"-collector", 1, retryInterval, timeout) - require.NoError(t, err, "Error waiting for collector deployment") - - err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName+"-query", 1, retryInterval, timeout) - require.NoError(t, err, "Error waiting for query deployment") + jaegerInstance := getJaegerSelfProvSimpleProd(jaegerInstanceName, namespace, 1) + createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace) + defer undeployJaegerInstance(jaegerInstance) ProductionSmokeTest(jaegerInstanceName) @@ -102,16 +92,9 @@ func (suite *SelfProvisionedTestSuite) TestSelfProvisionedESSmokeTest() { func (suite *SelfProvisionedTestSuite) TestIncreasingReplicas() { jaegerInstanceName := "simple-prod2" - exampleJaeger := getJaegerSimpleProd(jaegerInstanceName) - err := fw.Client.Create(goctx.TODO(), exampleJaeger, &framework.CleanupOptions{TestContext: ctx, Timeout: timeout, RetryInterval: retryInterval}) - require.NoError(t, err, "Error deploying example Jaeger") - defer undeployJaegerInstance(exampleJaeger) - - err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName+"-collector", 1, retryInterval, timeout) - require.NoError(t, err, "Error waiting for collector deployment") - - err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName+"-query", 1, retryInterval, timeout) - require.NoError(t, err, "Error waiting for query deployment") + jaegerInstance := getJaegerSelfProvSimpleProd(jaegerInstanceName, namespace, 1) + createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace) + defer undeployJaegerInstance(jaegerInstance) ProductionSmokeTest(jaegerInstanceName) @@ -125,7 +108,7 @@ func (suite *SelfProvisionedTestSuite) TestIncreasingReplicas() { require.EqualValues(t, updateCollectorCount, *updatedJaegerInstance.Spec.Collector.Replicas) require.EqualValues(t, updateQueryCount, *updatedJaegerInstance.Spec.Query.Replicas) - err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName+"-collector", int(updateCollectorCount), retryInterval, timeout) + err := e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName+"-collector", int(updateCollectorCount), retryInterval, timeout) require.NoError(t, err, "Error waiting for collector deployment") err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName+"-query", int(updateQueryCount), retryInterval, timeout) @@ -215,45 +198,6 @@ func (suite *SelfProvisionedTestSuite) TestValidateEsOperatorImage() { require.Equal(t, expectedEsOperatorImage, imageName) } -func getJaegerSimpleProd(instanceName string) *v1.Jaeger { - ingressEnabled := true - exampleJaeger := &v1.Jaeger{ - TypeMeta: metav1.TypeMeta{ - Kind: "Jaeger", - APIVersion: "jaegertracing.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: instanceName, - Namespace: namespace, - }, - Spec: v1.JaegerSpec{ - Ingress: v1.JaegerIngressSpec{ - Enabled: &ingressEnabled, - Security: v1.IngressSecurityNoneExplicit, - }, - Strategy: v1.DeploymentStrategyProduction, - Storage: v1.JaegerStorageSpec{ - Type: v1.JaegerESStorage, - Elasticsearch: v1.ElasticsearchSpec{ - NodeCount: 1, - Resources: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}, - Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}, - }, - }, - }, - }, - } - - if specifyOtelImages { - logrus.Infof("Using OTEL collector for %s", instanceName) - exampleJaeger.Spec.Collector.Image = otelCollectorImage - exampleJaeger.Spec.Collector.Config = v1.NewFreeForm(getOtelConfigForHealthCheckPort("14269")) - } - - return exampleJaeger -} - func getElasticSearchOperatorImage(kubeclient kubernetes.Interface, namespace string) string { deployment, err := kubeclient.AppsV1().Deployments(namespace).Get(context.Background(), "elasticsearch-operator", metav1.GetOptions{}) require.NoErrorf(t, err, "Did not find elasticsearch-operator in namespace %s\n", namespace) diff --git a/test/e2e/utils.go b/test/e2e/utils.go index 1cdf2cc5d..f4abdc927 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -23,8 +23,10 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -729,3 +731,74 @@ func waitForElasticSearch() { err := WaitForStatefulset(t, fw.KubeClient, storageNamespace, string(v1.JaegerESStorage), retryInterval, timeout) require.NoError(t, err, "Error waiting for elasticsearch") } + +func getJaegerSelfProvSimpleProd(instanceName, namespace string, nodeCount int32) *v1.Jaeger { + ingressEnabled := true + exampleJaeger := &v1.Jaeger{ + TypeMeta: metav1.TypeMeta{ + Kind: "Jaeger", + APIVersion: "jaegertracing.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: instanceName, + Namespace: namespace, + }, + Spec: v1.JaegerSpec{ + Ingress: v1.JaegerIngressSpec{ + Enabled: &ingressEnabled, + Security: v1.IngressSecurityNoneExplicit, + }, + Strategy: v1.DeploymentStrategyProduction, + Storage: v1.JaegerStorageSpec{ + Type: v1.JaegerESStorage, + Elasticsearch: v1.ElasticsearchSpec{ + NodeCount: nodeCount, + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + }, + }, + } + + if specifyOtelImages { + logrus.Infof("Using OTEL collector for %s", instanceName) + exampleJaeger.Spec.Collector.Image = otelCollectorImage + exampleJaeger.Spec.Collector.Config = v1.NewFreeForm(getOtelConfigForHealthCheckPort("14269")) + } + + return exampleJaeger +} + +func createESSelfProvDeployment(jaegerInstance *v1.Jaeger, jaegerInstanceName, jaegerNamespace string) { + err := fw.Client.Create(context.TODO(), jaegerInstance, &framework.CleanupOptions{TestContext: ctx, Timeout: timeout, RetryInterval: retryInterval}) + require.NoError(t, err, "Error deploying example Jaeger") + + // Wait for all elasticsearch instances to appear + listOptions := &metav1.ListOptions{LabelSelector: "component=elasticsearch"} + var deployments []appsv1.Deployment + err = wait.Poll(retryInterval, timeout, func() (done bool, err error) { + esDeployments, err := fw.KubeClient.AppsV1().Deployments(jaegerNamespace).List(context.Background(), *listOptions) + if int32(len(esDeployments.Items)) == jaegerInstance.Spec.Storage.Elasticsearch.NodeCount { + deployments = esDeployments.Items + return true, nil + } + return false, nil + }) + require.NoError(t, err, "Failed waiting for elasticsearch deployments to be available") + + // And then wait for them to finish deploying + for _, deployment := range deployments { + logrus.Infof("Waiting for deployment of %s", deployment.Name) + err = e2eutil.WaitForDeployment(t, fw.KubeClient, jaegerNamespace, deployment.Name, 1, retryInterval, timeout) + require.NoError(t, err, "Failed waiting for elasticsearch deployment(s) %s to start", deployment.Name) + } + + err = e2eutil.WaitForDeployment(t, fw.KubeClient, jaegerNamespace, jaegerInstanceName+"-collector", 1, retryInterval, timeout) + require.NoError(t, err, "Error waiting for collector deployment") + + err = e2eutil.WaitForDeployment(t, fw.KubeClient, jaegerNamespace, jaegerInstanceName+"-query", 1, retryInterval, timeout) + require.NoError(t, err, "Error waiting for query deployment") + logrus.Infof("Jaeger instance %s finished deploying in %s", jaegerInstanceName, jaegerNamespace) +}