diff --git a/.ci/run-e2e-tests.sh b/.ci/run-e2e-tests.sh index 96d34353da..0e95f42f72 100755 --- a/.ci/run-e2e-tests.sh +++ b/.ci/run-e2e-tests.sh @@ -42,12 +42,6 @@ then elif [ "${TEST_GROUP}" = "es-token-propagation" ] then echo "Running token propagation tests" - oc create user user-test-token - oc adm policy add-cluster-role-to-user cluster-admin user-test-token - # for ocp 4.2 - htpasswd -c -B -b users.htpasswd user-test-token any - oc delete secret htpass-secret -n openshift-config - oc create secret generic htpass-secret --from-file=htpasswd=./users.htpasswd -n openshift-config make e2e-tests-token-propagation-es else echo "Unknown TEST_GROUP [${TEST_GROUP}]"; exit 1 diff --git a/deploy/crds/jaegertracing.io_jaegers_crd.yaml b/deploy/crds/jaegertracing.io_jaegers_crd.yaml index 0615e6b65c..cf8cec3359 100644 --- a/deploy/crds/jaegertracing.io_jaegers_crd.yaml +++ b/deploy/crds/jaegertracing.io_jaegers_crd.yaml @@ -9434,6 +9434,10 @@ spec: sar: type: string type: object + options: + description: Options defines a common options parameter to the different + structs + type: object resources: description: ResourceRequirements describes the compute resource requirements. diff --git a/pkg/apis/jaegertracing/v1/zz_generated.openapi.go b/pkg/apis/jaegertracing/v1/zz_generated.openapi.go index bbf1da80c5..bbce3cdc20 100644 --- a/pkg/apis/jaegertracing/v1/zz_generated.openapi.go +++ b/pkg/apis/jaegertracing/v1/zz_generated.openapi.go @@ -1361,11 +1361,16 @@ func schema_pkg_apis_jaegertracing_v1_JaegerIngressSpec(ref common.ReferenceCall Format: "", }, }, + "options": { + SchemaProps: spec.SchemaProps{ + Ref: ref("./pkg/apis/jaegertracing/v1.Options"), + }, + }, }, }, }, Dependencies: []string{ - "./pkg/apis/jaegertracing/v1.JaegerIngressOpenShiftSpec", "./pkg/apis/jaegertracing/v1.JaegerIngressTLSSpec", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.PodSecurityContext", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.Toleration", "k8s.io/api/core/v1.Volume", "k8s.io/api/core/v1.VolumeMount"}, + "./pkg/apis/jaegertracing/v1.JaegerIngressOpenShiftSpec", "./pkg/apis/jaegertracing/v1.JaegerIngressTLSSpec", "./pkg/apis/jaegertracing/v1.Options", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.PodSecurityContext", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.Toleration", "k8s.io/api/core/v1.Volume", "k8s.io/api/core/v1.VolumeMount"}, } } diff --git a/pkg/strategy/production.go b/pkg/strategy/production.go index f91f2161f2..6af0938d53 100644 --- a/pkg/strategy/production.go +++ b/pkg/strategy/production.go @@ -24,7 +24,7 @@ import ( "github.com/jaegertracing/jaeger-operator/pkg/storage" ) -func newProductionStrategy(ctx context.Context, jaeger *v1.Jaeger) { +func newProductionStrategy(ctx context.Context, jaeger *v1.Jaeger) S { tracer := global.TraceProvider().GetTracer(v1.ReconciliationTracer) ctx, span := tracer.Start(ctx, "newProductionStrategy") defer span.End() @@ -115,25 +115,6 @@ func newProductionStrategy(ctx context.Context, jaeger *v1.Jaeger) { } for i := range esRollover { jobs = append(jobs, &esRollover[i].Spec.JobTemplate.Spec.Template.Spec) - - err := es.CreateCerts() - if err != nil { - jaeger.Logger().WithError(err).Error("failed to create Elasticsearch certificates, Elasticsearch won't be deployed") - } else { - c.secrets = es.ExtractSecrets() - c.elasticsearches = append(c.elasticsearches, *es.Elasticsearch()) - - es.InjectStorageConfiguration(&queryDep.Spec.Template.Spec) - es.InjectStorageConfiguration(&cDep.Spec.Template.Spec) - if indexCleaner != nil { - es.InjectSecretsConfiguration(&indexCleaner.Spec.JobTemplate.Spec.Template.Spec) - } - for i := range esRollover { - es.InjectSecretsConfiguration(&esRollover[i].Spec.JobTemplate.Spec.Template.Spec) - } - for i := range c.dependencies { - es.InjectSecretsConfiguration(&c.dependencies[i].Spec.Template.Spec) - } } autoProvisionElasticsearch(&c, jaeger, jobs, []*appsv1.Deployment{queryDep, cDep}) } diff --git a/test/e2e/elasticsearch_token_propagation_test.go b/test/e2e/elasticsearch_token_propagation_test.go index f425dabba7..12c55f7650 100644 --- a/test/e2e/elasticsearch_token_propagation_test.go +++ b/test/e2e/elasticsearch_token_propagation_test.go @@ -3,15 +3,12 @@ package e2e import ( - "bytes" goctx "context" "crypto/tls" "errors" "fmt" "io/ioutil" "net/http" - "net/http/cookiejar" - "net/url" "strings" "testing" "time" @@ -22,12 +19,14 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/uber/jaeger-client-go/config" - "golang.org/x/net/html" corev1 "k8s.io/api/core/v1" + rbac "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/types" + "github.com/jaegertracing/jaeger-operator/pkg/apis" v1 "github.com/jaegertracing/jaeger-operator/pkg/apis/jaegertracing/v1" esv1 "github.com/jaegertracing/jaeger-operator/pkg/storage/elasticsearch/v1" @@ -39,17 +38,19 @@ const username = "user-test-token" const password = "any" const collectorPodImageName = "jaeger-collector" const testServiceName = "token-propagation" +const test_account = "token-test-user" -type TokenPropagationTestSuite struct { +type TokenTestSuite struct { suite.Suite exampleJaeger *v1.Jaeger queryName string collectorName string queryServiceEndPoint string host string + token string } -func (suite *TokenPropagationTestSuite) SetupSuite() { +func (suite *TokenTestSuite) SetupSuite() { t = suite.T() if !isOpenShift(t) { t.Skipf("Test %s is currently supported only on OpenShift because es-operator runs only on OpenShift\n", t.Name()) @@ -78,24 +79,20 @@ func (suite *TokenPropagationTestSuite) SetupSuite() { namespace, _ = ctx.GetNamespace() require.NotNil(t, namespace, "GetNamespace failed") addToFrameworkSchemeForSmokeTests(t) - suite.deployJaegerWithPropagationEnabled() - } -func (suite *TokenPropagationTestSuite) TearDownSuite() { - // undeployJaegerInstance(suite.exampleJaeger) - // handleSuiteTearDown() +func (suite *TokenTestSuite) TearDownSuite() { + undeployJaegerInstance(suite.exampleJaeger) + handleSuiteTearDown() } -func (suite *TokenPropagationTestSuite) TestTokenPropagationNoToken() { - +func (suite *TokenTestSuite) TestTokenPropagationNoToken() { client := &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, } - err := wait.Poll(retryInterval, timeout, func() (done bool, err error) { req, err := http.NewRequest(http.MethodGet, suite.queryServiceEndPoint, nil) resp, err := client.Do(req) @@ -104,10 +101,9 @@ func (suite *TokenPropagationTestSuite) TestTokenPropagationNoToken() { return true, nil }) require.NoError(t, err, "Token propagation test failed") - } -func (suite *TokenPropagationTestSuite) TestTokenPropagationValidToken() { +func (suite *TokenTestSuite) TestTokenPropagationValidToken() { /* Create an span */ collectorPort := randomPortNumber() collectorPorts := []string{collectorPort + ":14268"} @@ -115,7 +111,6 @@ func (suite *TokenPropagationTestSuite) TestTokenPropagationValidToken() { defer portForwColl.Close() defer close(closeChanColl) collectorEndpoint := fmt.Sprintf("http://localhost:%s/api/traces", collectorPort) - cfg := config.Configuration{ Reporter: &config.ReporterConfig{CollectorEndpoint: collectorEndpoint}, Sampler: &config.SamplerConfig{Type: "const", Param: 1}, @@ -123,22 +118,18 @@ func (suite *TokenPropagationTestSuite) TestTokenPropagationValidToken() { } tracer, closer, err := cfg.NewTracer() require.NoError(t, err, "Failed to create tracer in token propagation test") - tStr := time.Now().Format(time.RFC3339Nano) tracer.StartSpan("TokenTest"). SetTag("time-RFC3339Nano", tStr). Finish() closer.Close() - - /* Get token using oauth OpenShift authorization */ - client, err := oAuthAuthorization(suite.host, username, password) - + client := newHTTPSClient() /* Try to reach query endpoint */ err = wait.Poll(retryInterval, timeout, func() (done bool, err error) { req, err := http.NewRequest(http.MethodGet, suite.queryServiceEndPoint, nil) + req.Header.Add("Authorization", "Bearer "+suite.token) resp, err := client.Do(req) defer resp.Body.Close() - require.Equal(t, http.StatusOK, resp.StatusCode) if resp.StatusCode != http.StatusOK { return false, errors.New("Query service returns http code: " + string(resp.StatusCode)) @@ -148,15 +139,145 @@ func (suite *TokenPropagationTestSuite) TestTokenPropagationValidToken() { if !strings.Contains(bodyString, "errors\":null") { return false, errors.New("query service returns errors: " + bodyString) } - - return strings.Contains(bodyString, tStr), nil return true, nil }) - require.NoError(t, err, "Token propagation test failed") } -func getESJaegerInstance() *v1.Jaeger { +func (suite *TokenTestSuite) deployJaegerWithPropagationEnabled() { + queryName := fmt.Sprintf("%s-query", name) + collectorName := fmt.Sprintf("%s-collector", name) + bindOperatorWithAuthDelegator() + createTestServiceAccount() + suite.token = testAccountToken() + + suite.exampleJaeger = jaegerInstance() + err := fw.Client.Create(goctx.Background(), + suite.exampleJaeger, + &framework.CleanupOptions{ + TestContext: ctx, + Timeout: timeout, + RetryInterval: retryInterval, + }) + require.NoError(t, err, "Error deploying example Jaeger") + + err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, collectorName, 1, retryInterval, timeout) + require.NoError(t, err, "Error waiting for collector deployment") + + err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, queryName, 1, retryInterval, timeout) + require.NoError(t, err, "Error waiting for query deployment") + + route := findRoute(t, fw, name) + + suite.host = route.Spec.Host + suite.queryServiceEndPoint = fmt.Sprintf("https://%s/api/traces?service=%s", suite.host, testServiceName) +} + +func TestTokenSuite(t *testing.T) { + suite.Run(t, new(TokenTestSuite)) +} + +func bindOperatorWithAuthDelegator() { + roleBinding := rbac.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "jaeger-operator:system:auth-delegator", + Namespace: namespace, + }, + Subjects: []rbac.Subject{{ + Kind: "ServiceAccount", + Name: "jaeger-operator", + Namespace: namespace, + }}, + RoleRef: rbac.RoleRef{ + Kind: "ClusterRole", + Name: "system:auth-delegator", + }, + } + err := fw.Client.Create(goctx.Background(), + &roleBinding, + &framework.CleanupOptions{ + TestContext: ctx, + Timeout: timeout, + RetryInterval: retryInterval, + }) + require.NoError(t, err, "Error binding operator service account with auth-delegator") +} + +func createTestServiceAccount() { + + serviceAccount := corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: test_account, + Namespace: namespace, + }, + } + err := fw.Client.Create(goctx.Background(), + &serviceAccount, + &framework.CleanupOptions{ + TestContext: ctx, + Timeout: timeout, + RetryInterval: retryInterval, + }) + require.NoError(t, err, "Error deploying example Jaeger") + + roleBinding := rbac.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: test_account + "-bind", + Namespace: namespace, + }, + Subjects: []rbac.Subject{{ + Kind: "ServiceAccount", + Name: serviceAccount.Name, + Namespace: namespace, + }}, + RoleRef: rbac.RoleRef{ + Kind: "ClusterRole", + Name: "cluster-admin", + }, + } + + err = fw.Client.Create(goctx.Background(), + &roleBinding, + &framework.CleanupOptions{ + TestContext: ctx, + Timeout: timeout, + RetryInterval: retryInterval, + }) + require.NoError(t, err, "Error deploying example Jaeger") + +} + +func testAccountToken() string { + var secretName string + err := wait.Poll(retryInterval, timeout, func() (done bool, err error) { + serviceAccount := corev1.ServiceAccount{} + e := fw.Client.Get(goctx.Background(), types.NamespacedName{ + Namespace: namespace, + Name: test_account, + }, &serviceAccount) + if e != nil { + return false, e + } + for _, s := range serviceAccount.Secrets { + if strings.HasPrefix(s.Name, test_account+"-token") { + secretName = s.Name + return true, nil + } + } + return false, nil + }) + require.NoError(t, err, "Error getting service account token") + require.NotEmpty(t, secretName, "secret with token not found") + secret := corev1.Secret{} + err = fw.Client.Get(goctx.Background(), types.NamespacedName{ + Namespace: namespace, + Name: secretName, + }, &secret) + require.NoError(t, err, "Error deploying example Jaeger") + return string(secret.Data["token"]) +} + +func jaegerInstance() *v1.Jaeger { exampleJaeger := &v1.Jaeger{ TypeMeta: metav1.TypeMeta{ Kind: "Jaeger", @@ -186,6 +307,10 @@ func getESJaegerInstance() *v1.Jaeger { }), }, Ingress: v1.JaegerIngressSpec{ + Openshift: v1.JaegerIngressOpenShiftSpec{ + SAR: "{\"namespace\": \"default\", \"resource\": \"pods\", \"verb\": \"get\"}", + DelegateUrls: `{"/":{"namespace": "default", "resource": "pods", "verb": "get"}}`, + }, Options: v1.NewOptions(map[string]interface{}{ "pass-access-token": "true", "pass-user-bearer-token": "true", @@ -198,38 +323,6 @@ func getESJaegerInstance() *v1.Jaeger { return exampleJaeger } -func (suite *TokenPropagationTestSuite) deployJaegerWithPropagationEnabled() { - queryName := fmt.Sprintf("%s-query", name) - collectorName := fmt.Sprintf("%s-collector", name) - - // create jaeger custom resource - suite.exampleJaeger = getESJaegerInstance() - err := fw.Client.Create(goctx.TODO(), - suite.exampleJaeger, - &framework.CleanupOptions{ - TestContext: ctx, - Timeout: timeout, - RetryInterval: retryInterval, - }) - require.NoError(t, err, "Error deploying example Jaeger") - - err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, collectorName, 1, retryInterval, timeout) - require.NoError(t, err, "Error waiting for collector deployment") - - err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, queryName, 1, retryInterval, timeout) - require.NoError(t, err, "Error waiting for query deployment") - - route := findRoute(t, fw, name) - - suite.host = route.Spec.Host - suite.queryServiceEndPoint = fmt.Sprintf("https://%s/api/traces?service=%s", suite.host, testServiceName) - -} - -func TestTokenPropagationSuite(t *testing.T) { - suite.Run(t, new(TokenPropagationTestSuite)) -} - func newHTTPSClient() *http.Client { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -242,156 +335,3 @@ func newHTTPSClient() *http.Client { } return client } - -func oAuthAuthorization(host, user, pass string) (*http.Client, error) { - /* Setup client*/ - cookieJar, _ := cookiejar.New(nil) - client := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, - Jar: cookieJar, - } - /* Start oauth */ - resp, err := client.Get("https://" + host + "/oauth/start") - defer resp.Body.Close() - if err != nil { - return nil, err - } - responseBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - var req *http.Request - /* Submit form */ - if hasForm(responseBytes) { - req = getLoginFormRequest(responseBytes, resp.Request.URL, user, pass) - } else { - // OCP 4.2 or newer. - // Choose idp - link := getLinkToHtpassIDP(responseBytes) - resp, err := client.Get("https://" + resp.Request.URL.Host + link) - if err != nil { - return nil, err - } - - responseBytes, err := ioutil.ReadAll(resp.Body) - defer resp.Body.Close() - - if err != nil { - return nil, err - } - req = getLoginFormRequest(responseBytes, resp.Request.URL, user, pass) - } - resp, err = client.Do(req) - defer resp.Body.Close() - if resp.Request.URL.Path == "/oauth/authorize/approve" { - req = submitGrantForm(resp) - resp, err = client.Do(req) - defer resp.Body.Close() - } - return client, nil -} - -func hasForm(responseBytes []byte) bool { - root, _ := html.Parse(bytes.NewBuffer(responseBytes)) - form := false - visit(root, func(n *html.Node) { - if n.Type == html.ElementNode && n.Data == "form" { - form = true - } - }) - return form -} - -func getLinkToHtpassIDP(responseBytes []byte) string { - root, _ := html.Parse(bytes.NewBuffer(responseBytes)) - link := "" - visit(root, func(n *html.Node) { - if n.Type == html.ElementNode && n.Data == "a" { - href := getAttr(n, "href") - url, _ := url.Parse(href) - if url.Query().Get("idp") == "htpasswd_provider" { - link = href - } - } - }) - return link -} - -func getLoginFormRequest(responseBytes []byte, currentURL *url.URL, username, password string) *http.Request { - reqHeader := http.Header{} - action := "" - formData := url.Values{} - root, _ := html.Parse(bytes.NewBuffer(responseBytes)) - visit(root, func(n *html.Node) { - if n.Type == html.ElementNode && n.Data == "input" { - inputType := getAttr(n, "type") - if inputType == "hidden" { - name := getAttr(n, "name") - value := getAttr(n, "value") - formData.Add(name, value) - } - } - if n.Type == html.ElementNode && n.Data == "form" { - action = getAttr(n, "action") - } - }) - - formData.Add("username", username) - formData.Add("password", password) - reqHeader.Set("Content-Type", "application/x-www-form-urlencoded") - reqBody := strings.NewReader(formData.Encode()) - reqURL, _ := currentURL.Parse(action) - req, _ := http.NewRequest("POST", reqURL.String(), reqBody) - req.Header = reqHeader - return req -} - -func submitGrantForm(response *http.Response) *http.Request { - reqHeader := http.Header{} - action := "" - formData := url.Values{} - currentURL := response.Request.URL - responseBytes, err := ioutil.ReadAll(response.Body) - if err != nil { - return nil - } - root, err := html.Parse(bytes.NewBuffer(responseBytes)) - visit(root, func(n *html.Node) { - if n.Type == html.ElementNode && n.Data == "input" { - inputType := getAttr(n, "type") - if inputType == "hidden" || inputType == "checkbox" || inputType == "radio" { - name := getAttr(n, "name") - value := getAttr(n, "value") - formData.Add(name, value) - } - } - if n.Type == html.ElementNode && n.Data == "form" { - action = getAttr(n, "action") - } - }) - formData.Add("approve", "Allow selected permissions") - reqHeader.Set("Content-Type", "application/x-www-form-urlencoded") - reqBody := strings.NewReader(formData.Encode()) - reqURL, _ := currentURL.Parse(action) - req, _ := http.NewRequest("POST", reqURL.String(), reqBody) - req.Header = reqHeader - return req -} - -func getAttr(element *html.Node, attrName string) string { - for _, attr := range element.Attr { - if attr.Key == attrName { - return attr.Val - } - } - return "" -} - -func visit(n *html.Node, visitor func(*html.Node)) { - visitor(n) - for c := n.FirstChild; c != nil; c = c.NextSibling { - visit(c, visitor) - } -}