diff --git a/apis/ossm.plugins.kubeflow.org/v1alpha1/ossm_types.go b/apis/ossm.plugins.kubeflow.org/v1alpha1/ossm_types.go index 2c2f1bcb938..32cb7abef20 100644 --- a/apis/ossm.plugins.kubeflow.org/v1alpha1/ossm_types.go +++ b/apis/ossm.plugins.kubeflow.org/v1alpha1/ossm_types.go @@ -78,6 +78,15 @@ type OssmResourceTracker struct { Status OssmResourceTrackerStatus `json:"status,omitempty"` } +func (o *OssmResourceTracker) ToOwnerReference() metav1.OwnerReference { + return metav1.OwnerReference{ + APIVersion: o.APIVersion, + Kind: o.Kind, + Name: o.Name, + UID: o.UID, + } +} + // OssmResourceTrackerSpec defines the desired state of OssmResourceTracker type OssmResourceTrackerSpec struct { } diff --git a/config/crd/bases/ossmplugin.internal.kubeflow.org_kfossmplugins.yaml b/config/crd/bases/ossmplugin.internal.kubeflow.org_kfossmplugins.yaml index 66607f88e0d..832c0ce65a3 100644 --- a/config/crd/bases/ossmplugin.internal.kubeflow.org_kfossmplugins.yaml +++ b/config/crd/bases/ossmplugin.internal.kubeflow.org_kfossmplugins.yaml @@ -36,6 +36,10 @@ spec: description: OssmPluginSpec defines the extra data provided by the Openshift Service Mesh Plugin in KfDef spec. properties: + appNamespace: + description: Additional non-user facing fields (should not be copied + to the CRD) + type: string auth: properties: authorino: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 1f162507228..aafae9e7194 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -12,5 +12,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: quay.io/opendatahub/opendatahub-operator - newTag: dev-0.0.1 + newName: quay.io/maistra-dev/opendatahub-operator + newTag: dev-0.0.6 diff --git a/pkg/kfapp/coordinator/coordinator.go b/pkg/kfapp/coordinator/coordinator.go index e7f9fd9a658..cb617866ae6 100644 --- a/pkg/kfapp/coordinator/coordinator.go +++ b/pkg/kfapp/coordinator/coordinator.go @@ -483,7 +483,7 @@ func (kfapp *coordinator) Delete(resources kftypesv3.ResourceEnum) error { } } else { ossmInstaller := p.(*ossm.OssmInstaller) - return ossmInstaller.CleanupOwnedResources() + return ossmInstaller.CleanupResources() } } diff --git a/pkg/kfapp/ossm/cleanup.go b/pkg/kfapp/ossm/cleanup.go deleted file mode 100644 index e3c74d61c4a..00000000000 --- a/pkg/kfapp/ossm/cleanup.go +++ /dev/null @@ -1,250 +0,0 @@ -package ossm - -import ( - "context" - "fmt" - "github.com/hashicorp/go-multierror" - "github.com/opendatahub-io/opendatahub-operator/apis/ossm.plugins.kubeflow.org/v1alpha1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/dynamic" -) - -type cleanup func() error - -func (o *OssmInstaller) CleanupOwnedResources() error { - var cleanupErrors *multierror.Error - for _, cleanupFunc := range o.cleanupFuncs { - cleanupErrors = multierror.Append(cleanupErrors, cleanupFunc()) - } - - return cleanupErrors.ErrorOrNil() -} - -func (o *OssmInstaller) onCleanup(cleanupFunc ...cleanup) { - o.cleanupFuncs = append(o.cleanupFuncs, cleanupFunc...) -} - -// createResourceTracker instantiates OssmResourceTracker for given KfDef application in a namespace. -// This cluster-scoped resource is used as OwnerReference in all objects OssmInstaller is created across the cluster. -// Once created, there's a cleanup function added which will be invoked on deletion of the KfDef. -func (o *OssmInstaller) createResourceTracker() error { - tracker := &v1alpha1.OssmResourceTracker{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "ossm.plugins.kubeflow.org/v1alpha1", - Kind: "OssmResourceTracker", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: o.KfConfig.Name + "." + o.KfConfig.Namespace, - }, - } - - c, err := dynamic.NewForConfig(o.config) - if err != nil { - return err - } - - gvr := schema.GroupVersionResource{ - Group: "ossm.plugins.kubeflow.org", - Version: "v1alpha1", - Resource: "ossmresourcetrackers", - } - - foundTracker, err := c.Resource(gvr).Get(context.Background(), tracker.Name, metav1.GetOptions{}) - if k8serrors.IsNotFound(err) { - unstructuredTracker, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tracker) - if err != nil { - return err - } - - u := unstructured.Unstructured{Object: unstructuredTracker} - - foundTracker, err = c.Resource(gvr).Create(context.Background(), &u, metav1.CreateOptions{}) - if err != nil { - return err - } - } else if err != nil { - return err - } - - o.tracker = &v1alpha1.OssmResourceTracker{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(foundTracker.Object, o.tracker); err != nil { - return err - } - - o.onCleanup(func() error { - err := c.Resource(gvr).Delete(context.Background(), o.tracker.Name, metav1.DeleteOptions{}) - if k8serrors.IsNotFound(err) { - return nil - } - return err - }) - - return nil -} - -func (o *OssmInstaller) ingressVolumesRemoval() cleanup { - - return func() error { - spec, err := o.GetPluginSpec() - if err != nil { - return err - } - - tokenVolume := fmt.Sprintf("%s-oauth2-tokens", o.KfConfig.Namespace) - - dynamicClient, err := dynamic.NewForConfig(o.config) - if err != nil { - return err - } - - gvr := schema.GroupVersionResource{ - Group: "maistra.io", - Version: "v2", - Resource: "servicemeshcontrolplanes", - } - - smcp, err := dynamicClient.Resource(gvr).Namespace(spec.Mesh.Namespace).Get(context.Background(), spec.Mesh.Name, metav1.GetOptions{}) - if err != nil { - return err - } - volumes, found, err := unstructured.NestedSlice(smcp.Object, "spec", "gateways", "ingress", "volumes") - if err != nil { - return err - } - if !found { - log.Info("no volumes found", "smcp", spec.Mesh.Name, "istio-ns", spec.Mesh.Namespace) - return nil - } - - for i, v := range volumes { - volume, ok := v.(map[string]interface{}) - if !ok { - fmt.Println("Unexpected type for volume") - continue - } - - volumeMount, found, err := unstructured.NestedMap(volume, "volumeMount") - if err != nil { - return err - } - if !found { - fmt.Println("No volumeMount found in the volume") - continue - } - - if volumeMount["name"] == tokenVolume { - volumes = append(volumes[:i], volumes[i+1:]...) - err = unstructured.SetNestedSlice(smcp.Object, volumes, "spec", "gateways", "ingress", "volumes") - if err != nil { - return err - } - break - } - } - - _, err = dynamicClient.Resource(gvr).Namespace(spec.Mesh.Namespace).Update(context.Background(), smcp, metav1.UpdateOptions{}) - if err != nil { - return err - } - - return nil - } - -} - -func (o *OssmInstaller) oauthClientRemoval() func() error { - - return func() error { - c, err := dynamic.NewForConfig(o.config) - if err != nil { - return err - } - - oauthClientName := fmt.Sprintf("%s-oauth2-client", o.KfConfig.Namespace) - gvr := schema.GroupVersionResource{ - Group: "oauth.openshift.io", - Version: "v1", - Resource: "oauthclients", - } - - if _, err := c.Resource(gvr).Get(context.Background(), oauthClientName, metav1.GetOptions{}); err != nil { - if k8serrors.IsNotFound(err) { - return nil - } - - return err - } - - if err := c.Resource(gvr).Delete(context.Background(), oauthClientName, metav1.DeleteOptions{}); err != nil { - log.Error(err, "failed deleting OAuthClient", "name", oauthClientName) - return err - } - - return nil - } -} - -func (o *OssmInstaller) externalAuthzProviderRemoval() cleanup { - - return func() error { - spec, err := o.GetPluginSpec() - if err != nil { - return err - } - - ossmAuthzProvider := fmt.Sprintf("%s-odh-auth-provider", o.KfConfig.Namespace) - - dynamicClient, err := dynamic.NewForConfig(o.config) - if err != nil { - return err - } - - gvr := schema.GroupVersionResource{ - Group: "maistra.io", - Version: "v2", - Resource: "servicemeshcontrolplanes", - } - - smcp, err := dynamicClient.Resource(gvr).Namespace(spec.Mesh.Namespace).Get(context.Background(), spec.Mesh.Name, metav1.GetOptions{}) - if err != nil { - return err - } - - extensionProviders, found, err := unstructured.NestedSlice(smcp.Object, "spec", "techPreview", "meshConfig", "extensionProviders") - if err != nil { - return err - } - if !found { - log.Info("no extension providers found", "smcp", spec.Mesh.Name, "istio-ns", spec.Mesh.Namespace) - return nil - } - - for i, v := range extensionProviders { - extensionProvider, ok := v.(map[string]interface{}) - if !ok { - fmt.Println("Unexpected type for extensionProvider") - continue - } - - if extensionProvider["name"] == ossmAuthzProvider { - extensionProviders = append(extensionProviders[:i], extensionProviders[i+1:]...) - err = unstructured.SetNestedSlice(smcp.Object, extensionProviders, "spec", "techPreview", "meshConfig", "extensionProviders") - if err != nil { - return err - } - break - } - } - - _, err = dynamicClient.Resource(gvr).Namespace(spec.Mesh.Namespace).Update(context.Background(), smcp, metav1.UpdateOptions{}) - if err != nil { - return err - } - - return nil - } -} diff --git a/pkg/kfapp/ossm/feature/builder.go b/pkg/kfapp/ossm/feature/builder.go new file mode 100644 index 00000000000..2218b02a76a --- /dev/null +++ b/pkg/kfapp/ossm/feature/builder.go @@ -0,0 +1,138 @@ +package feature + +import ( + "github.com/opendatahub-io/opendatahub-operator/pkg/kfconfig/ossmplugin" + "github.com/pkg/errors" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type partialBuilder func(f *Feature) error + +type featureBuilder struct { + name string + builders []partialBuilder +} + +func CreateFeature(name string) *featureBuilder { + return &featureBuilder{name: name} +} + +func (fb *featureBuilder) For(spec *ossmplugin.OssmPluginSpec) *featureBuilder { + createSpec := func(f *Feature) error { + f.Spec = &Spec{ + OssmPluginSpec: spec, + } + + return nil + } + + // Ensures creation of .Spec object is always invoked first + fb.builders = append([]partialBuilder{createSpec}, fb.builders...) + + return fb +} + +func (fb *featureBuilder) UsingConfig(config *rest.Config) *featureBuilder { + fb.builders = append(fb.builders, func(f *Feature) error { + var err error + f.clientset, err = kubernetes.NewForConfig(config) + if err != nil { + return err + } + + f.dynamicClient, err = dynamic.NewForConfig(config) + if err != nil { + return err + } + + f.client, err = client.New(config, client.Options{}) + if err != nil { + return errors.WithStack(err) + } + + return nil + }) + + return fb +} + +func (fb *featureBuilder) Manifests(paths ...string) *featureBuilder { + fb.builders = append(fb.builders, func(f *Feature) error { + var err error + var manifests []manifest + + for _, path := range paths { + manifests, err = loadManifestsFrom(path) + if err != nil { + return errors.WithStack(err) + } + + f.manifests = append(f.manifests, manifests...) + } + + return nil + }) + + return fb +} + +func (fb *featureBuilder) WithData(loader ...action) *featureBuilder { + fb.builders = append(fb.builders, func(f *Feature) error { + f.loaders = append(f.loaders, loader...) + + return nil + }) + + return fb +} + +func (fb *featureBuilder) Preconditions(preconditions ...action) *featureBuilder { + fb.builders = append(fb.builders, func(f *Feature) error { + f.preconditions = append(f.preconditions, preconditions...) + + return nil + }) + + return fb +} + +func (fb *featureBuilder) OnDelete(cleanups ...action) *featureBuilder { + fb.builders = append(fb.builders, func(f *Feature) error { + f.addCleanup(cleanups...) + + return nil + }) + + return fb +} + +func (fb *featureBuilder) WithResources(resources ...action) *featureBuilder { + fb.builders = append(fb.builders, func(f *Feature) error { + f.resources = resources + + return nil + }) + + return fb +} + +func (fb *featureBuilder) Load() (*Feature, error) { + feature := &Feature{ + Name: fb.name, + } + + for i := range fb.builders { + if err := fb.builders[i](feature); err != nil { + return nil, err + } + } + + if err := feature.createResourceTracker(); err != nil { + return nil, err + } + + return feature, nil +} diff --git a/pkg/kfapp/ossm/cert.go b/pkg/kfapp/ossm/feature/cert.go similarity index 73% rename from pkg/kfapp/ossm/cert.go rename to pkg/kfapp/ossm/feature/cert.go index 1e039918139..aaa7e65bfc8 100644 --- a/pkg/kfapp/ossm/cert.go +++ b/pkg/kfapp/ossm/feature/cert.go @@ -1,8 +1,7 @@ -package ossm +package feature import ( "bytes" - "context" cryptorand "crypto/rand" "crypto/rsa" "crypto/x509" @@ -10,9 +9,7 @@ import ( "encoding/pem" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" "math/big" "math/rand" "net" @@ -21,43 +18,20 @@ import ( var seededRand = rand.New(rand.NewSource(time.Now().UnixNano())) -func (o *OssmInstaller) createSelfSignedCerts(addr string, objectMeta metav1.ObjectMeta) error { +func generateSelfSignedCertificateAsSecret(addr string, objectMeta metav1.ObjectMeta) (*corev1.Secret, error) { cert, key, err := generateCertificate(addr) if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } - objectMeta.SetOwnerReferences([]metav1.OwnerReference{ - { - APIVersion: o.tracker.APIVersion, - Kind: o.tracker.Kind, - Name: o.tracker.Name, - UID: o.tracker.UID, - }, - }) - - secret := &corev1.Secret{ + return &corev1.Secret{ ObjectMeta: objectMeta, Data: map[string][]byte{ corev1.TLSCertKey: cert, corev1.TLSPrivateKeyKey: key, }, - } - - clientset, err := kubernetes.NewForConfig(o.config) - if err != nil { - return errors.WithStack(err) - } - - _, err = clientset.CoreV1(). - Secrets(objectMeta.Namespace). - Create(context.TODO(), secret, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { - return errors.WithStack(err) - } - - return nil + }, nil } func generateCertificate(addr string) ([]byte, []byte, error) { diff --git a/pkg/kfapp/ossm/feature/cleanup.go b/pkg/kfapp/ossm/feature/cleanup.go new file mode 100644 index 00000000000..1928c13afc7 --- /dev/null +++ b/pkg/kfapp/ossm/feature/cleanup.go @@ -0,0 +1,145 @@ +package feature + +import ( + "context" + "fmt" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func RemoveTokenVolumes(feature *Feature) error { + tokenVolume := fmt.Sprintf("%s-oauth2-tokens", feature.Spec.AppNamespace) + + gvr := schema.GroupVersionResource{ + Group: "maistra.io", + Version: "v2", + Resource: "servicemeshcontrolplanes", + } + + meshNs := feature.Spec.Mesh.Namespace + meshName := feature.Spec.Mesh.Name + + smcp, err := feature.dynamicClient.Resource(gvr).Namespace(meshNs).Get(context.Background(), meshName, metav1.GetOptions{}) + if err != nil { + return err + } + volumes, found, err := unstructured.NestedSlice(smcp.Object, "spec", "gateways", "ingress", "volumes") + if err != nil { + return err + } + if !found { + log.Info("no volumes found", "smcp", meshName, "istio-ns", meshNs) + return nil + } + + for i, v := range volumes { + volume, ok := v.(map[string]interface{}) + if !ok { + fmt.Println("Unexpected type for volume") + continue + } + + volumeMount, found, err := unstructured.NestedMap(volume, "volumeMount") + if err != nil { + return err + } + if !found { + fmt.Println("No volumeMount found in the volume") + continue + } + + if volumeMount["name"] == tokenVolume { + volumes = append(volumes[:i], volumes[i+1:]...) + err = unstructured.SetNestedSlice(smcp.Object, volumes, "spec", "gateways", "ingress", "volumes") + if err != nil { + return err + } + break + } + } + + _, err = feature.dynamicClient.Resource(gvr).Namespace(meshNs).Update(context.Background(), smcp, metav1.UpdateOptions{}) + if err != nil { + return err + } + + return nil +} + +func RemoveOAuthClient(feature *Feature) error { + oauthClientName := fmt.Sprintf("%s-oauth2-client", feature.Spec.AppNamespace) + gvr := schema.GroupVersionResource{ + Group: "oauth.openshift.io", + Version: "v1", + Resource: "oauthclients", + } + + if _, err := feature.dynamicClient.Resource(gvr).Get(context.Background(), oauthClientName, metav1.GetOptions{}); err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + + return err + } + + if err := feature.dynamicClient.Resource(gvr).Delete(context.Background(), oauthClientName, metav1.DeleteOptions{}); err != nil { + log.Error(err, "failed deleting OAuthClient", "name", oauthClientName) + return err + } + + return nil +} + +func RemoveExtensionProvider(feature *Feature) error { + ossmAuthzProvider := fmt.Sprintf("%s-odh-auth-provider", feature.Spec.AppNamespace) + + gvr := schema.GroupVersionResource{ + Group: "maistra.io", + Version: "v2", + Resource: "servicemeshcontrolplanes", + } + + mesh := feature.Spec.Mesh + + smcp, err := feature.dynamicClient.Resource(gvr). + Namespace(mesh.Namespace). + Get(context.Background(), mesh.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + extensionProviders, found, err := unstructured.NestedSlice(smcp.Object, "spec", "techPreview", "meshConfig", "extensionProviders") + if err != nil { + return err + } + if !found { + log.Info("no extension providers found", "smcp", mesh.Name, "istio-ns", mesh.Namespace) + return nil + } + + for i, v := range extensionProviders { + extensionProvider, ok := v.(map[string]interface{}) + if !ok { + fmt.Println("Unexpected type for extensionProvider") + continue + } + + if extensionProvider["name"] == ossmAuthzProvider { + extensionProviders = append(extensionProviders[:i], extensionProviders[i+1:]...) + err = unstructured.SetNestedSlice(smcp.Object, extensionProviders, "spec", "techPreview", "meshConfig", "extensionProviders") + if err != nil { + return err + } + break + } + } + + _, err = feature.dynamicClient.Resource(gvr). + Namespace(mesh.Namespace). + Update(context.Background(), smcp, metav1.UpdateOptions{}) + + return err + +} diff --git a/pkg/kfapp/ossm/cluster_config.go b/pkg/kfapp/ossm/feature/cluster_config.go similarity index 74% rename from pkg/kfapp/ossm/cluster_config.go rename to pkg/kfapp/ossm/feature/cluster_config.go index 74806f07733..f0329b7e115 100644 --- a/pkg/kfapp/ossm/cluster_config.go +++ b/pkg/kfapp/ossm/feature/cluster_config.go @@ -1,4 +1,4 @@ -package ossm +package feature import ( "context" @@ -12,19 +12,14 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" - "k8s.io/client-go/rest" "net/http" "net/url" "os" + "strconv" "strings" ) -func GetDomain(config *rest.Config) (string, error) { - dynamicClient, err := dynamic.NewForConfig(config) - if err != nil { - return "", nil - } - +func GetDomain(dynamicClient dynamic.Interface) (string, error) { cluster, err := dynamicClient.Resource( schema.GroupVersionResource{ Group: "config.openshift.io", @@ -119,3 +114,36 @@ func getKubeAPIURLWithPath(path string) *url.URL { Path: path, } } + +// ExtractHostNameAndPort strips given URL in string from http(s):// prefix and subsequent path, +// returning host name and port if defined (otherwise defaults to 443). +// +// This is useful when getting value from http headers (such as origin). +// If given string does not start with http(s) prefix it will be returned as is. +func ExtractHostNameAndPort(s string) (string, string, error) { + u, err := url.Parse(s) + if err != nil { + return "", "", err + } + + if u.Scheme != "http" && u.Scheme != "https" { + return s, "", nil + } + + hostname := u.Hostname() + + port := "443" // default for https + if u.Scheme == "http" { + port = "80" + } + + if u.Port() != "" { + port = u.Port() + _, err := strconv.Atoi(port) + if err != nil { + return "", "", errors.New("invalid port number: " + port) + } + } + + return hostname, port, nil +} diff --git a/pkg/kfapp/ossm/envoy_secrets.go b/pkg/kfapp/ossm/feature/envoy_secrets.go similarity index 62% rename from pkg/kfapp/ossm/envoy_secrets.go rename to pkg/kfapp/ossm/feature/envoy_secrets.go index dc1cd604531..3945fececc7 100644 --- a/pkg/kfapp/ossm/envoy_secrets.go +++ b/pkg/kfapp/ossm/feature/envoy_secrets.go @@ -1,14 +1,11 @@ -package ossm +package feature import ( "bytes" - "context" "fmt" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" "text/template" ) @@ -30,48 +27,26 @@ resources: inline_bytes: "{{ .Secret }}" ` -func (o *OssmInstaller) createEnvoySecret(oAuth oAuth, objectMeta metav1.ObjectMeta) error { +func createEnvoySecret(oAuth OAuth, objectMeta metav1.ObjectMeta) (*corev1.Secret, error) { clientSecret, err := processInlineTemplate(tokenSecret, struct{ Secret string }{Secret: oAuth.ClientSecret}) if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } hmacSecret, err := processInlineTemplate(hmacSecret, struct{ Secret string }{Secret: oAuth.Hmac}) if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } - objectMeta.SetOwnerReferences([]metav1.OwnerReference{ - { - APIVersion: o.tracker.APIVersion, - Kind: o.tracker.Kind, - Name: o.tracker.Name, - UID: o.tracker.UID, - }, - }) - - secret := &corev1.Secret{ + return &corev1.Secret{ ObjectMeta: objectMeta, Data: map[string][]byte{ "token-secret.yaml": clientSecret, "hmac-secret.yaml": hmacSecret, }, - } - - clientset, err := kubernetes.NewForConfig(o.config) - if err != nil { - return errors.WithStack(err) - } - - _, err = clientset.CoreV1(). - Secrets(objectMeta.Namespace). - Create(context.TODO(), secret, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { - return errors.WithStack(err) - } + }, nil - return nil } func processInlineTemplate(templateString string, data interface{}) ([]byte, error) { diff --git a/pkg/kfapp/ossm/feature/feature.go b/pkg/kfapp/ossm/feature/feature.go new file mode 100644 index 00000000000..b7419221244 --- /dev/null +++ b/pkg/kfapp/ossm/feature/feature.go @@ -0,0 +1,243 @@ +package feature + +import ( + "context" + "github.com/hashicorp/go-multierror" + "github.com/opendatahub-io/opendatahub-operator/apis/ossm.plugins.kubeflow.org/v1alpha1" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "net/url" + "regexp" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlLog "sigs.k8s.io/controller-runtime/pkg/log" + "strings" +) + +var log = ctrlLog.Log.WithName("ossm-features") + +type Feature struct { + Name string + Spec *Spec + + clientset *kubernetes.Clientset + dynamicClient dynamic.Interface + client client.Client + + manifests []manifest + cleanups []action + resources []action + preconditions []action + postconditions []action + loaders []action +} + +// action is a func type which can be used for different purposes while having access to Feature struct +type action func(feature *Feature) error + +func (f *Feature) Apply() error { + // Verify all precondition and collect errors + var multiErr *multierror.Error + for _, precondition := range f.preconditions { + multiErr = multierror.Append(multiErr, precondition(f)) + } + + if multiErr.ErrorOrNil() != nil { + return multiErr.ErrorOrNil() + } + + // Load necessary data + for _, loader := range f.loaders { + multiErr = multierror.Append(multiErr, loader(f)) + } + if multiErr.ErrorOrNil() != nil { + return multiErr.ErrorOrNil() + } + + // create or update resources + for _, resource := range f.resources { + if err := resource(f); err != nil { + return err + } + } + + // Process and apply manifests + for _, m := range f.manifests { + if err := m.processTemplate(f.Spec); err != nil { + return errors.WithStack(err) + } + + log.Info("applying manifest", "path", m.targetPath()) + } + + if err := f.applyManifests(); err != nil { + return err + } + + // TODO postconditions + + return nil +} + +func (f *Feature) Cleanup() error { + var cleanupErrors *multierror.Error + for _, cleanupFunc := range f.cleanups { + cleanupErrors = multierror.Append(cleanupErrors, cleanupFunc(f)) + } + + return cleanupErrors.ErrorOrNil() +} + +func (f *Feature) applyManifests() error { + var applyErrors *multierror.Error + for _, m := range f.manifests { + err := f.apply(m) + applyErrors = multierror.Append(applyErrors, err) + } + + return applyErrors.ErrorOrNil() +} + +func (f *Feature) createConfigMap(cfgMapName string, data map[string]string) error { + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cfgMapName, + Namespace: f.Spec.AppNamespace, + OwnerReferences: []metav1.OwnerReference{ + f.OwnerReference(), + }, + }, + Data: data, + } + + configMaps := f.clientset.CoreV1().ConfigMaps(configMap.Namespace) + + _, err := configMaps.Get(context.TODO(), configMap.Name, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + _, err = configMaps.Create(context.TODO(), configMap, metav1.CreateOptions{}) + if err != nil { + return err + } + + } else if k8serrors.IsAlreadyExists(err) { + _, err = configMaps.Update(context.TODO(), configMap, metav1.UpdateOptions{}) + if err != nil { + return err + } + } else { + return err + } + + return nil +} + +func (f *Feature) addCleanup(cleanupFuncs ...action) { + f.cleanups = append(f.cleanups, cleanupFuncs...) +} + +type apply func(filename string) error + +func (f *Feature) apply(m manifest) error { + var applier apply + targetPath := m.targetPath() + + if m.patch { + applier = func(filename string) error { + log.Info("patching using manifest", "name", m.name, "path", targetPath) + + return f.patchResourceFromFile(filename) + } + } else { + applier = func(filename string) error { + log.Info("applying manifest", "name", m.name, "path", targetPath) + + return f.createResourceFromFile(filename) + } + } + + if err := applier(targetPath); err != nil { + log.Error(err, "failed to create resource", "name", m.name, "path", targetPath) + + return err + } + + return nil +} + +func (f *Feature) OwnerReference() metav1.OwnerReference { + return f.Spec.Tracker.ToOwnerReference() +} + +// createResourceTracker instantiates OssmResourceTracker for given a Feature. All resources created when applying +// it will have this object attached as OwnerReference. It's a cluster-scoped resource. +// Once created, there's a cleanup hook added which will be invoked on deletion. +func (f *Feature) createResourceTracker() error { + tracker := &v1alpha1.OssmResourceTracker{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "ossm.plugins.kubeflow.org/v1alpha1", + Kind: "OssmResourceTracker", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: f.Spec.AppNamespace + "-" + convertToRFC1123Subdomain(f.Name), + }, + } + + gvr := schema.GroupVersionResource{ + Group: "ossm.plugins.kubeflow.org", + Version: "v1alpha1", + Resource: "ossmresourcetrackers", + } + + foundTracker, err := f.dynamicClient.Resource(gvr).Get(context.Background(), tracker.Name, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + unstructuredTracker, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tracker) + if err != nil { + return err + } + + u := unstructured.Unstructured{Object: unstructuredTracker} + + foundTracker, err = f.dynamicClient.Resource(gvr).Create(context.Background(), &u, metav1.CreateOptions{}) + if err != nil { + return err + } + } else if err != nil { + return err + } + + f.Spec.Tracker = &v1alpha1.OssmResourceTracker{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(foundTracker.Object, f.Spec.Tracker); err != nil { + return err + } + + // Register its own cleanup + f.addCleanup(func(feature *Feature) error { + if err := f.dynamicClient.Resource(gvr).Delete(context.Background(), f.Spec.Tracker.Name, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { + return err + } + + return nil + }) + + return nil +} + +func convertToRFC1123Subdomain(input string) string { + escaped := url.PathEscape(input) + + // Define a regular expression to match characters that need to be replaced + regex := regexp.MustCompile(`[^A-Za-z0-9.\-_]+`) + + // Replace non-alphanumeric characters with a hyphen + replaced := regex.ReplaceAllString(escaped, "-") + + // Convert the result to lowercase + return strings.ToLower(replaced) +} diff --git a/pkg/kfapp/ossm/ossm_suite_test.go b/pkg/kfapp/ossm/feature/feature_suite_test.go similarity index 58% rename from pkg/kfapp/ossm/ossm_suite_test.go rename to pkg/kfapp/ossm/feature/feature_suite_test.go index c02e801911f..725085e100a 100644 --- a/pkg/kfapp/ossm/ossm_suite_test.go +++ b/pkg/kfapp/ossm/feature/feature_suite_test.go @@ -1,4 +1,4 @@ -package ossm_test +package feature_test import ( "testing" @@ -7,8 +7,8 @@ import ( . "github.com/onsi/gomega" ) -func TestOssmInstaller(t *testing.T) { +func TestOssmFeatures(t *testing.T) { RegisterFailHandler(Fail) // for integration tests see tests/integration directory - RunSpecs(t, "Openshift Service Mesh installer unit tests") + RunSpecs(t, "Openshift Service Mesh features unit tests") } diff --git a/pkg/kfapp/ossm/hostname_and_port_extraction_unit_test.go b/pkg/kfapp/ossm/feature/hostname_and_port_extraction_unit_test.go similarity index 71% rename from pkg/kfapp/ossm/hostname_and_port_extraction_unit_test.go rename to pkg/kfapp/ossm/feature/hostname_and_port_extraction_unit_test.go index 115fd9e3485..c2170f4ac78 100644 --- a/pkg/kfapp/ossm/hostname_and_port_extraction_unit_test.go +++ b/pkg/kfapp/ossm/feature/hostname_and_port_extraction_unit_test.go @@ -1,15 +1,15 @@ -package ossm_test +package feature_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/opendatahub-io/opendatahub-operator/pkg/kfapp/ossm" + "github.com/opendatahub-io/opendatahub-operator/pkg/kfapp/ossm/feature" ) var _ = Describe("extracting hostname and port from URL", func() { It("should extract hostname and port for HTTP URL", func() { - hostname, port, err := ossm.ExtractHostNameAndPort("http://opendatahub.io:8080/path") + hostname, port, err := feature.ExtractHostNameAndPort("http://opendatahub.io:8080/path") Expect(err).ToNot(HaveOccurred()) Expect(hostname).To(Equal("opendatahub.io")) Expect(port).To(Equal("8080")) @@ -17,7 +17,7 @@ var _ = Describe("extracting hostname and port from URL", func() { It("should return original URL if it does not start with http(s) but with other valid protocol", func() { originalURL := "gopher://opendatahub.io" - hostname, port, err := ossm.ExtractHostNameAndPort(originalURL) + hostname, port, err := feature.ExtractHostNameAndPort(originalURL) Expect(err).ToNot(HaveOccurred()) Expect(hostname).To(Equal(originalURL)) Expect(port).To(Equal("")) @@ -25,20 +25,20 @@ var _ = Describe("extracting hostname and port from URL", func() { It("should handle invalid URLs by returning corresponding error", func() { invalidURL := ":opendatahub.io" - _, _, err := ossm.ExtractHostNameAndPort(invalidURL) + _, _, err := feature.ExtractHostNameAndPort(invalidURL) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(ContainSubstring("missing protocol scheme"))) }) It("should handle URLs without port and default to 443 for HTTPS", func() { - hostname, port, err := ossm.ExtractHostNameAndPort("https://opendatahub.io") + hostname, port, err := feature.ExtractHostNameAndPort("https://opendatahub.io") Expect(err).ToNot(HaveOccurred()) Expect(hostname).To(Equal("opendatahub.io")) Expect(port).To(Equal("443")) }) It("should handle URLs without port and default to 80 for HTTP", func() { - hostname, port, err := ossm.ExtractHostNameAndPort("http://opendatahub.io") + hostname, port, err := feature.ExtractHostNameAndPort("http://opendatahub.io") Expect(err).ToNot(HaveOccurred()) Expect(hostname).To(Equal("opendatahub.io")) Expect(port).To(Equal("80")) diff --git a/pkg/kfapp/ossm/feature/loaders.go b/pkg/kfapp/ossm/feature/loaders.go new file mode 100644 index 00000000000..7dd700adaad --- /dev/null +++ b/pkg/kfapp/ossm/feature/loaders.go @@ -0,0 +1,52 @@ +package feature + +import ( + "github.com/opendatahub-io/opendatahub-operator/pkg/secret" + "github.com/pkg/errors" +) + +func ClusterDetails(feature *Feature) error { + data := feature.Spec + + if domain, err := GetDomain(feature.dynamicClient); err == nil { + data.Domain = domain + } else { + return errors.WithStack(err) + } + + return nil +} + +func OAuthConfig(feature *Feature) error { + data := feature.Spec + + var err error + var clientSecret, hmac *secret.Secret + if clientSecret, err = secret.NewSecret("ossm-odh-oauth", "random", 32); err != nil { + return errors.WithStack(err) + } + + if hmac, err = secret.NewSecret("ossm-odh-hmac", "random", 32); err != nil { + return errors.WithStack(err) + } + + if oauthServerDetailsJson, err := GetOAuthServerDetails(); err == nil { + hostName, port, errUrlParsing := ExtractHostNameAndPort(oauthServerDetailsJson.Get("issuer").MustString("issuer")) + if errUrlParsing != nil { + return errUrlParsing + } + + data.OAuth = OAuth{ + AuthzEndpoint: oauthServerDetailsJson.Get("authorization_endpoint").MustString("authorization_endpoint"), + TokenEndpoint: oauthServerDetailsJson.Get("token_endpoint").MustString("token_endpoint"), + Route: hostName, + Port: port, + ClientSecret: clientSecret.Value, + Hmac: hmac.Value, + } + } else { + return errors.WithStack(err) + } + + return nil +} diff --git a/pkg/kfapp/ossm/feature/manifest.go b/pkg/kfapp/ossm/feature/manifest.go new file mode 100644 index 00000000000..3ac7b5362fe --- /dev/null +++ b/pkg/kfapp/ossm/feature/manifest.go @@ -0,0 +1,78 @@ +package feature + +import ( + "fmt" + "github.com/pkg/errors" + "html/template" + "os" + "path/filepath" + "strings" +) + +const ( + ControlPlaneDir = "templates/control-plane" + AuthDir = "templates/authorino" + BaseOutputDir = "/tmp/ossm-installer/" +) + +type manifest struct { + name, + path string + template, + patch, + processed bool +} + +func loadManifestsFrom(path string) ([]manifest, error) { + var manifests []manifest + if err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + basePath := filepath.Base(path) + manifests = append(manifests, manifest{ + name: basePath, + path: path, + patch: strings.Contains(basePath, ".patch"), + template: filepath.Ext(path) == ".tmpl", + }) + return nil + }); err != nil { + return nil, errors.WithStack(err) + } + + return manifests, nil +} + +func (m *manifest) targetPath() string { + return fmt.Sprintf("%s%s", m.path[:len(m.path)-len(filepath.Ext(m.path))], ".yaml") +} + +func (m *manifest) processTemplate(data interface{}) error { + if !m.template { + return nil + } + path := m.targetPath() + + f, err := os.Create(path) + if err != nil { + log.Error(err, "Failed to create file") + + return err + } + + tmpl := template.New(m.name).Funcs(template.FuncMap{"ReplaceChar": ReplaceChar}) + + tmpl, err = tmpl.ParseFiles(m.path) + if err != nil { + return err + } + + err = tmpl.Execute(f, data) + m.processed = err == nil + + return err +} diff --git a/pkg/kfapp/ossm/k8s_resources.go b/pkg/kfapp/ossm/feature/raw_resources.go similarity index 51% rename from pkg/kfapp/ossm/k8s_resources.go rename to pkg/kfapp/ossm/feature/raw_resources.go index 8fd0e56bdf4..fa8862d6b5a 100644 --- a/pkg/kfapp/ossm/k8s_resources.go +++ b/pkg/kfapp/ossm/feature/raw_resources.go @@ -11,7 +11,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ossm +package feature import ( "context" @@ -25,10 +25,8 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" k8stypes "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/dynamic" "os" "regexp" - "sigs.k8s.io/controller-runtime/pkg/client" "strings" ) @@ -36,15 +34,11 @@ const ( YamlSeparator = "(?m)^---[ \t]*$" ) -func (o *OssmInstaller) CreateResourceFromFile(filename string, elems ...configtypes.NameValue) error { +func (f *Feature) createResourceFromFile(filename string, elems ...configtypes.NameValue) error { elemsMap := make(map[string]configtypes.NameValue) for _, nv := range elems { elemsMap[nv.Name] = nv } - c, err := client.New(o.config, client.Options{}) - if err != nil { - return errors.WithStack(err) - } data, err := os.ReadFile(filename) if err != nil { @@ -72,17 +66,12 @@ func (o *OssmInstaller) CreateResourceFromFile(filename string, elems ...configt } u.SetOwnerReferences([]metav1.OwnerReference{ - { - APIVersion: o.tracker.APIVersion, - Kind: o.tracker.Kind, - Name: o.tracker.Name, - UID: o.tracker.UID, - }, + f.OwnerReference(), }) logrus.Infof("Creating %s", name) - err := c.Get(context.TODO(), k8stypes.NamespacedName{Name: name, Namespace: namespace}, u.DeepCopy()) + err := f.client.Get(context.TODO(), k8stypes.NamespacedName{Name: name, Namespace: namespace}, u.DeepCopy()) if err == nil { log.Info("Object already exists...") continue @@ -91,7 +80,7 @@ func (o *OssmInstaller) CreateResourceFromFile(filename string, elems ...configt return errors.WithStack(err) } - err = c.Create(context.TODO(), u) + err = f.client.Create(context.TODO(), u) if err != nil { return errors.WithStack(err) } @@ -99,17 +88,12 @@ func (o *OssmInstaller) CreateResourceFromFile(filename string, elems ...configt return nil } -func (o *OssmInstaller) PatchResourceFromFile(filename string, elems ...configtypes.NameValue) error { +func (f *Feature) patchResourceFromFile(filename string, elems ...configtypes.NameValue) error { elemsMap := make(map[string]configtypes.NameValue) for _, nv := range elems { elemsMap[nv.Name] = nv } - dynamicClient, err := dynamic.NewForConfig(o.config) - if err != nil { - return errors.WithStack(err) - } - data, err := os.ReadFile(filename) if err != nil { return errors.WithStack(err) @@ -151,7 +135,7 @@ func (o *OssmInstaller) PatchResourceFromFile(filename string, elems ...configty return errors.WithStack(err) } - _, err = dynamicClient.Resource(gvr). + _, err = f.dynamicClient.Resource(gvr). Namespace(p.GetNamespace()). Patch(context.Background(), p.GetName(), k8stypes.MergePatchType, patchAsJson, metav1.PatchOptions{}) if err != nil { @@ -168,96 +152,3 @@ func (o *OssmInstaller) PatchResourceFromFile(filename string, elems ...configty } return nil } - -func (o *OssmInstaller) VerifyCRDInstalled(group string, version string, resource string) error { - dynamicClient, err := dynamic.NewForConfig(o.config) - if err != nil { - log.Error(err, "Failed to initialize dynamic client") - return err - } - - crdGVR := schema.GroupVersionResource{ - Group: group, - Version: version, - Resource: resource, - } - - _, err = dynamicClient.Resource(crdGVR).List(context.Background(), metav1.ListOptions{}) - return err -} - -func (o *OssmInstaller) CheckSMCPStatus(name, namespace string) (string, error) { - dynamicClient, err := dynamic.NewForConfig(o.config) - if err != nil { - log.Info("Failed to initialize dynamic client") - return "", err - } - - gvr := schema.GroupVersionResource{ - Group: "maistra.io", - Version: "v1", - Resource: "servicemeshcontrolplanes", - } - - unstructObj, err := dynamicClient.Resource(gvr).Namespace(namespace).Get(context.TODO(), name, metav1.GetOptions{}) - if err != nil { - log.Info("Failed to find SMCP") - return "", err - } - - conditions, found, err := unstructured.NestedSlice(unstructObj.Object, "status", "conditions") - if err != nil || !found { - log.Info("status conditions not found or error in parsing of SMCP") - return "", err - } - lastCondition := conditions[len(conditions)-1].(map[string]interface{}) - status := lastCondition["type"].(string) - - return status, nil -} - -func (o *OssmInstaller) PatchODHDashboardConfig(namespace string) error { - dynamicClient, err := dynamic.NewForConfig(o.config) - if err != nil { - log.Info("Failed to initialize dynamic client") - return err - } - - gvr := schema.GroupVersionResource{ - Group: "opendatahub.io", - Version: "v1alpha", - Resource: "odhdashboardconfigs", - } - - configs, err := dynamicClient.Resource(gvr).List(context.Background(), metav1.ListOptions{}) - if err != nil { - return err - } - - if len(configs.Items) == 0 { - log.Info("No odhdashboardconfig found in namespace, doing nothing") - return nil - } - - // Assuming there is only one odhdashboardconfig in the namespace, patching the first one - config := configs.Items[0] - if config.Object["spec"] == nil { - config.Object["spec"] = map[string]interface{}{} - } - spec := config.Object["spec"].(map[string]interface{}) - if spec["dashboardConfig"] == nil { - spec["dashboardConfig"] = map[string]interface{}{} - } - dashboardConfig := spec["dashboardConfig"].(map[string]interface{}) - dashboardConfig["disableServiceMesh"] = false - - _, err = dynamicClient.Resource(gvr).Namespace(namespace).Update(context.Background(), &config, metav1.UpdateOptions{}) - if err != nil { - log.Error(err, "Failed to update odhdashboardconfig") - return err - } - - log.Info("Successfully patched odhdashboardconfig") - return nil - -} diff --git a/pkg/kfapp/ossm/feature/resources.go b/pkg/kfapp/ossm/feature/resources.go new file mode 100644 index 00000000000..439b8a78b7e --- /dev/null +++ b/pkg/kfapp/ossm/feature/resources.go @@ -0,0 +1,154 @@ +package feature + +import ( + "context" + "fmt" + "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func SelfSignedCertificate(feature *Feature) error { + if feature.Spec.Mesh.Certificate.Generate { + meta := metav1.ObjectMeta{ + Name: feature.Spec.Mesh.Certificate.Name, + Namespace: feature.Spec.Mesh.Namespace, + OwnerReferences: []metav1.OwnerReference{ + feature.OwnerReference(), + }, + } + + cert, err := generateSelfSignedCertificateAsSecret(feature.Spec.Domain, meta) + if err != nil { + return errors.WithStack(err) + } + + if err != nil { + return errors.WithStack(err) + } + + _, err = feature.clientset.CoreV1(). + Secrets(feature.Spec.Mesh.Namespace). + Create(context.TODO(), cert, metav1.CreateOptions{}) + if err != nil && !k8serrors.IsAlreadyExists(err) { + return errors.WithStack(err) + } + } + + return nil +} + +func EnvoyOAuthSecrets(feature *Feature) error { + objectMeta := metav1.ObjectMeta{ + Name: feature.Spec.AppNamespace + "-oauth2-tokens", + Namespace: feature.Spec.Mesh.Namespace, + OwnerReferences: []metav1.OwnerReference{ + feature.OwnerReference(), + }, + } + + envoySecret, err := createEnvoySecret(feature.Spec.OAuth, objectMeta) + if err != nil { + return errors.WithStack(err) + } + + _, err = feature.clientset.CoreV1(). + Secrets(objectMeta.Namespace). + Create(context.TODO(), envoySecret, metav1.CreateOptions{}) + if err != nil && !k8serrors.IsAlreadyExists(err) { + return errors.WithStack(err) + } + + return nil +} + +func ConfigMaps(feature *Feature) error { + if err := feature.createConfigMap("service-mesh-refs", + map[string]string{ + "CONTROL_PLANE_NAME": feature.Spec.Mesh.Name, + "MESH_NAMESPACE": feature.Spec.Mesh.Namespace, + }); err != nil { + return errors.WithStack(err) + } + + if err := feature.createConfigMap("auth-refs", + map[string]string{ + "AUTHORINO_LABEL": feature.Spec.Auth.Authorino.Label, + }); err != nil { + return errors.WithStack(err) + } + + return nil +} + +func ServiceMeshEnabledInDashboard(feature *Feature) error { + gvr := schema.GroupVersionResource{ + Group: "opendatahub.io", + Version: "v1alpha", + Resource: "odhdashboardconfigs", + } + + configs, err := feature.dynamicClient.Resource(gvr).List(context.Background(), metav1.ListOptions{}) + if err != nil { + return err + } + + if len(configs.Items) == 0 { + log.Info("No odhdashboardconfig found in namespace, doing nothing") + return nil + } + + // Assuming there is only one odhdashboardconfig in the namespace, patching the first one + config := configs.Items[0] + if config.Object["spec"] == nil { + config.Object["spec"] = map[string]interface{}{} + } + spec := config.Object["spec"].(map[string]interface{}) + if spec["dashboardConfig"] == nil { + spec["dashboardConfig"] = map[string]interface{}{} + } + dashboardConfig := spec["dashboardConfig"].(map[string]interface{}) + dashboardConfig["disableServiceMesh"] = false + + _, err = feature.dynamicClient.Resource(gvr). + Namespace(feature.Spec.AppNamespace). + Update(context.Background(), &config, metav1.UpdateOptions{}) + if err != nil { + log.Error(err, "Failed to update odhdashboardconfig") + return err + } + + log.Info("Successfully patched odhdashboardconfig") + return nil +} + +func MigratedDataScienceProjects(feature *Feature) error { + selector := labels.SelectorFromSet(labels.Set{"opendatahub.io/dashboard": "true"}) + + namespaceClient := feature.clientset.CoreV1().Namespaces() + + namespaces, err := namespaceClient.List(context.TODO(), metav1.ListOptions{LabelSelector: selector.String()}) + if err != nil { + return fmt.Errorf("failed to get namespaces: %v", err) + } + + var result *multierror.Error + + for _, namespace := range namespaces.Items { + annotations := namespace.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations["opendatahub.io/service-mesh"] = "true" + namespace.SetAnnotations(annotations) + + if _, err := namespaceClient.Update(context.TODO(), &namespace, metav1.UpdateOptions{}); err != nil { + result = multierror.Append(result, err) + } + } + + return result.ErrorOrNil() +} diff --git a/pkg/kfapp/ossm/feature/types.go b/pkg/kfapp/ossm/feature/types.go new file mode 100644 index 00000000000..c4ad414b3e4 --- /dev/null +++ b/pkg/kfapp/ossm/feature/types.go @@ -0,0 +1,27 @@ +package feature + +import ( + "github.com/opendatahub-io/opendatahub-operator/apis/ossm.plugins.kubeflow.org/v1alpha1" + "github.com/opendatahub-io/opendatahub-operator/pkg/kfconfig/ossmplugin" + "strings" +) + +type Spec struct { + *ossmplugin.OssmPluginSpec + OAuth OAuth + Domain string + Tracker *v1alpha1.OssmResourceTracker +} + +type OAuth struct { + AuthzEndpoint, + TokenEndpoint, + Route, + Port, + ClientSecret, + Hmac string +} + +func ReplaceChar(s string, oldChar, newChar string) string { + return strings.ReplaceAll(s, oldChar, newChar) +} diff --git a/pkg/kfapp/ossm/feature/verification.go b/pkg/kfapp/ossm/feature/verification.go new file mode 100644 index 00000000000..752de668569 --- /dev/null +++ b/pkg/kfapp/ossm/feature/verification.go @@ -0,0 +1,70 @@ +package feature + +import ( + "context" + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" +) + +func EnsureCRDIsInstalled(group string, version string, resource string) action { + return func(f *Feature) error { + crdGVR := schema.GroupVersionResource{ + Group: group, + Version: version, + Resource: resource, + } + + _, err := f.dynamicClient.Resource(crdGVR).List(context.Background(), metav1.ListOptions{}) + + return err + } +} + +func EnsureServiceMeshInstalled(feature *Feature) error { + if err := EnsureCRDIsInstalled("maistra.io", "v2", "servicemeshcontrolplanes")(feature); err != nil { + log.Info("Failed to find the pre-requisite SMCP CRD, please ensure OSSM operator is installed.") + return err + } + + smcp := feature.Spec.Mesh.Name + smcpNs := feature.Spec.Mesh.Namespace + + status, err := checkSMCPStatus(feature.dynamicClient, smcp, smcpNs) + if err != nil { + log.Info("An error occurred while checking SMCP status - ensure the SMCP referenced exists.") + return err + } + if status != "Ready" { + log.Info("The referenced SMCP is not ready.", "name", smcp, "namespace", smcpNs) + return errors.New("SMCP status is not ready") + } + return nil + +} + +func checkSMCPStatus(dynamicClient dynamic.Interface, name, namespace string) (string, error) { + gvr := schema.GroupVersionResource{ + Group: "maistra.io", + Version: "v1", + Resource: "servicemeshcontrolplanes", + } + + unstructObj, err := dynamicClient.Resource(gvr).Namespace(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + log.Info("Failed to find SMCP") + return "", err + } + + conditions, found, err := unstructured.NestedSlice(unstructObj.Object, "status", "conditions") + if err != nil || !found { + log.Info("status conditions not found or error in parsing of SMCP") + return "", err + } + lastCondition := conditions[len(conditions)-1].(map[string]interface{}) + status := lastCondition["type"].(string) + + return status, nil +} diff --git a/pkg/kfapp/ossm/ossm_installer.go b/pkg/kfapp/ossm/ossm_installer.go index 1a8715f4cfe..4acfe52e2f5 100644 --- a/pkg/kfapp/ossm/ossm_installer.go +++ b/pkg/kfapp/ossm/ossm_installer.go @@ -1,24 +1,18 @@ package ossm import ( - "context" "fmt" "github.com/hashicorp/go-multierror" kfapisv3 "github.com/opendatahub-io/opendatahub-operator/apis" kftypesv3 "github.com/opendatahub-io/opendatahub-operator/apis/apps" - "github.com/opendatahub-io/opendatahub-operator/apis/ossm.plugins.kubeflow.org/v1alpha1" + "github.com/opendatahub-io/opendatahub-operator/pkg/kfapp/ossm/feature" "github.com/opendatahub-io/opendatahub-operator/pkg/kfconfig" "github.com/opendatahub-io/opendatahub-operator/pkg/kfconfig/ossmplugin" "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "net/url" + "path" + "path/filepath" ctrlLog "sigs.k8s.io/controller-runtime/pkg/log" - "strconv" ) const ( @@ -29,11 +23,9 @@ var log = ctrlLog.Log.WithName(PluginName) type OssmInstaller struct { *kfconfig.KfConfig - pluginSpec *ossmplugin.OssmPluginSpec - config *rest.Config - manifests []manifest - tracker *v1alpha1.OssmResourceTracker - cleanupFuncs []cleanup + PluginSpec *ossmplugin.OssmPluginSpec + config *rest.Config + features []*feature.Feature } func NewOssmInstaller(kfConfig *kfconfig.KfConfig, restConfig *rest.Config) *OssmInstaller { @@ -51,14 +43,19 @@ func GetPlatform(kfConfig *kfconfig.KfConfig) (kftypesv3.Platform, error) { // GetPluginSpec gets the plugin spec. func (o *OssmInstaller) GetPluginSpec() (*ossmplugin.OssmPluginSpec, error) { - if o.pluginSpec != nil { - return o.pluginSpec, nil + if o.PluginSpec != nil { + return o.PluginSpec, nil } - o.pluginSpec = &ossmplugin.OssmPluginSpec{} - err := o.KfConfig.GetPluginSpec(PluginName, o.pluginSpec) + o.PluginSpec = &ossmplugin.OssmPluginSpec{} + if err := o.KfConfig.GetPluginSpec(PluginName, o.PluginSpec); err != nil { + return nil, err + } + + // Populate target Kubeflow namespace to have it in one struct instead + o.PluginSpec.AppNamespace = o.KfConfig.Namespace - return o.pluginSpec, err + return o.PluginSpec, nil } func (o *OssmInstaller) Init(_ kftypesv3.ResourceEnum) error { @@ -78,187 +75,129 @@ func (o *OssmInstaller) Init(_ kftypesv3.ResourceEnum) error { return internalError(errors.New(reason)) } - if err := o.VerifyCRDInstalled("operator.authorino.kuadrant.io", "v1beta1", "authorinos"); err != nil { - log.Info("Failed to find the pre-requisite authorinos CRD, please ensure Authorino operator is installed.") - return internalError(err) - } - if err := o.ensureServiceMeshInstalled(pluginSpec); err != nil { - return internalError(err) - } - - if err := o.createResourceTracker(); err != nil { - return internalError(err) - } - - if err := o.createConfigMap("service-mesh-refs", - map[string]string{ - "CONTROL_PLANE_NAME": pluginSpec.Mesh.Name, - "MESH_NAMESPACE": pluginSpec.Mesh.Namespace, - }); err != nil { - return internalError(err) - } - - if err := o.createConfigMap("auth-refs", - map[string]string{ - "AUTHORINO_LABEL": pluginSpec.Auth.Authorino.Label, - }); err != nil { - return internalError(err) - } + return o.enableFeatures() +} - if err := o.MigrateDataScienceProjects(); err != nil { - log.Error(err, "failed migrating Data Science Projects") - } +func (o *OssmInstaller) enableFeatures() error { - if err := o.processManifests(); err != nil { + if err := o.SyncCache(); err != nil { return internalError(err) } - return nil -} - -func (o *OssmInstaller) Generate(resources kftypesv3.ResourceEnum) error { - // TODO sort by Kind as .Apply does - if err := o.applyManifests(); err != nil { + var rootDir = filepath.Join(feature.BaseOutputDir, o.Namespace, o.Name) + if err := copyEmbeddedFiles("templates", rootDir); err != nil { return internalError(errors.WithStack(err)) } - o.PatchODHDashboardConfig(o.Namespace) - - o.onCleanup( - o.oauthClientRemoval(), - o.ingressVolumesRemoval(), - o.externalAuthzProviderRemoval(), - ) - - return nil -} - -// ExtractHostNameAndPort strips given URL in string from http(s):// prefix and subsequent path, -// returning host name and port if defined (otherwise defaults to 443). -// -// This is useful when getting value from http headers (such as origin). -// If given string does not start with http(s) prefix it will be returned as is. -func ExtractHostNameAndPort(s string) (string, string, error) { - u, err := url.Parse(s) - if err != nil { - return "", "", err - } - - if u.Scheme != "http" && u.Scheme != "https" { - return s, "", nil - } - - hostname := u.Hostname() - - port := "443" // default for https - if u.Scheme == "http" { - port = "80" - } - - if u.Port() != "" { - port = u.Port() - _, err := strconv.Atoi(port) - if err != nil { - return "", "", errors.New("invalid port number: " + port) - } + if oauth, err := feature.CreateFeature("control-plane-configure-oauth"). + For(o.PluginSpec). + UsingConfig(o.config). + Manifests( + path.Join(rootDir, feature.ControlPlaneDir, "base"), + path.Join(rootDir, feature.ControlPlaneDir, "oauth"), + path.Join(rootDir, feature.ControlPlaneDir, "filters"), + ). + WithResources( + feature.SelfSignedCertificate, + feature.EnvoyOAuthSecrets, + ). + WithData(feature.ClusterDetails, feature.OAuthConfig). + Preconditions( + feature.EnsureCRDIsInstalled("operator.authorino.kuadrant.io", "v1beta1", "authorinos"), + feature.EnsureServiceMeshInstalled, + ). + OnDelete( + feature.RemoveOAuthClient, + feature.RemoveTokenVolumes, + ).Load(); err != nil { + return nil + } else { + o.features = append(o.features, oauth) } - return hostname, port, nil -} - -func (o *OssmInstaller) createConfigMap(cfgMapName string, data map[string]string) error { - - configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: cfgMapName, - Namespace: o.KfConfig.Namespace, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: o.tracker.APIVersion, - Kind: o.tracker.Kind, - Name: o.tracker.Name, - UID: o.tracker.UID, - }, - }, - }, - Data: data, + if cfMaps, err := feature.CreateFeature("shared-config-maps"). + For(o.PluginSpec). + UsingConfig(o.config). + WithResources(feature.ConfigMaps). + Load(); err != nil { + return err + } else { + o.features = append(o.features, cfMaps) + } + + if serviceMesh, err := feature.CreateFeature("app-add-namespace-to-service-mesh"). + For(o.PluginSpec). + UsingConfig(o.config). + Manifests( + path.Join(rootDir, feature.ControlPlaneDir, "smm.tmpl"), + path.Join(rootDir, feature.ControlPlaneDir, "namespace.patch.tmpl"), + ). + WithData(feature.ClusterDetails). + Load(); err != nil { + return err + } else { + o.features = append(o.features, serviceMesh) } - client, err := clientset.NewForConfig(o.config) - if err != nil { + if dashboard, err := feature.CreateFeature("app-enable-service-mesh-in-dashboard"). + For(o.PluginSpec). + UsingConfig(o.config). + WithResources(feature.ServiceMeshEnabledInDashboard). + Load(); err != nil { return err + } else { + o.features = append(o.features, dashboard) } - configMaps := client.CoreV1().ConfigMaps(configMap.Namespace) - _, err = configMaps.Get(context.TODO(), configMap.Name, metav1.GetOptions{}) - if k8serrors.IsNotFound(err) { - _, err = configMaps.Create(context.TODO(), configMap, metav1.CreateOptions{}) - if err != nil { - return err - } - - } else if k8serrors.IsAlreadyExists(err) { - _, err = configMaps.Update(context.TODO(), configMap, metav1.UpdateOptions{}) - if err != nil { - return err - } + if dataScienceProjects, err := feature.CreateFeature("app-migrate-data-science-projects"). + For(o.PluginSpec). + UsingConfig(o.config). + WithResources(feature.MigratedDataScienceProjects). + Load(); err != nil { + return err } else { + o.features = append(o.features, dataScienceProjects) + } + + if extAuthz, err := feature.CreateFeature("control-plane-setup-external-authorization"). + For(o.PluginSpec). + UsingConfig(o.config). + Manifests( + path.Join(rootDir, feature.AuthDir, "namespace.tmpl"), + path.Join(rootDir, feature.AuthDir, "auth-smm.tmpl"), + path.Join(rootDir, feature.AuthDir, "base"), + path.Join(rootDir, feature.AuthDir, "rbac"), + path.Join(rootDir, feature.AuthDir, "mesh-authz-ext-provider.patch.tmpl"), + ). + WithData(feature.ClusterDetails). + OnDelete(feature.RemoveExtensionProvider). + Load(); err != nil { return err + } else { + o.features = append(o.features, extAuthz) } return nil } -func (o *OssmInstaller) MigrateDataScienceProjects() error { - - client, err := clientset.NewForConfig(o.config) - if err != nil { - return err - } - - selector := labels.SelectorFromSet(labels.Set{"opendatahub.io/dashboard": "true"}) +func (o *OssmInstaller) Generate(_ kftypesv3.ResourceEnum) error { + var applyErrors *multierror.Error - namespaces, err := client.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{LabelSelector: selector.String()}) - if err != nil { - return fmt.Errorf("failed to get namespaces: %v", err) + for _, f := range o.features { + err := f.Apply() + applyErrors = multierror.Append(applyErrors, err) } - var result *multierror.Error - - for _, namespace := range namespaces.Items { - annotations := namespace.GetAnnotations() - if annotations == nil { - annotations = map[string]string{} - } - annotations["opendatahub.io/service-mesh"] = "true" - namespace.SetAnnotations(annotations) - - if _, err := client.CoreV1().Namespaces().Update(context.TODO(), &namespace, metav1.UpdateOptions{}); err != nil { - result = multierror.Append(result, err) - } - } - - return result.ErrorOrNil() + return applyErrors.ErrorOrNil() } -func (o *OssmInstaller) ensureServiceMeshInstalled(pluginSpec *ossmplugin.OssmPluginSpec) error { - if err := o.VerifyCRDInstalled("maistra.io", "v2", "servicemeshcontrolplanes"); err != nil { - log.Info("Failed to find the pre-requisite SMCP CRD, please ensure OSSM operator is installed.") - return internalError(err) +func (o *OssmInstaller) CleanupResources() error { + var cleanupErrors *multierror.Error + for _, f := range o.features { + cleanupErrors = multierror.Append(cleanupErrors, f.Cleanup()) } - smcp := pluginSpec.Mesh.Name - smcpNs := pluginSpec.Mesh.Namespace - status, err := o.CheckSMCPStatus(smcp, smcpNs) - if err != nil { - log.Info("An error occurred while checking SMCP status - ensure the SMCP referenced exists.") - return internalError(err) - } - if status != "Ready" { - log.Info("The referenced SMCP is not ready.", "SMCP name", smcp, "SMCP NS", smcpNs) - return internalError(errors.New("SMCP status is not ready")) - } - return nil + return cleanupErrors.ErrorOrNil() } func internalError(err error) error { diff --git a/pkg/kfapp/ossm/ossm_manifests.go b/pkg/kfapp/ossm/ossm_manifests.go deleted file mode 100644 index 76b5038a54d..00000000000 --- a/pkg/kfapp/ossm/ossm_manifests.go +++ /dev/null @@ -1,208 +0,0 @@ -package ossm - -import ( - "embed" - "fmt" - configtypes "github.com/opendatahub-io/opendatahub-operator/apis/config" - "github.com/opendatahub-io/opendatahub-operator/pkg/kfconfig/ossmplugin" - "github.com/opendatahub-io/opendatahub-operator/pkg/secret" - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/rest" - "os" - "path" - "path/filepath" - "strings" -) - -//go:embed templates -var embeddedFiles embed.FS - -type applier func(config *rest.Config, filename string, elems ...configtypes.NameValue) error - -func (o *OssmInstaller) applyManifests() error { - var apply applier - - for _, m := range o.manifests { - targetPath := m.targetPath() - if m.patch { - apply = func(config *rest.Config, filename string, elems ...configtypes.NameValue) error { - log.Info("patching using manifest", "name", m.name, "path", targetPath) - return o.PatchResourceFromFile(filename, elems...) - } - } else { - apply = func(config *rest.Config, filename string, elems ...configtypes.NameValue) error { - log.Info("applying manifest", "name", m.name, "path", targetPath) - return o.CreateResourceFromFile(filename, elems...) - } - } - - err := apply( - o.config, - targetPath, - ) - - if err != nil { - log.Error(err, "failed to create resource", "name", m.name, "path", targetPath) - return err - } - } - - return nil -} - -func (o *OssmInstaller) processManifests() error { - if err := o.SyncCache(); err != nil { - return internalError(err) - } - - var rootDir = filepath.Join(baseOutputDir, o.Namespace, o.Name) - // We copy the embedded template files into /tmp/ - // As embedded files are read-only, and we need write to templates - if copyFsErr := copyEmbeddedFS(embeddedFiles, "templates", rootDir); copyFsErr != nil { - return internalError(errors.WithStack(copyFsErr)) - } - - // IMPORTANT: Order of locations from where we load manifests/templates to process is significant - err := o.loadManifestsFrom( - path.Join(rootDir, ControlPlaneDir, "base"), - path.Join(rootDir, ControlPlaneDir, "filters"), - path.Join(rootDir, ControlPlaneDir, "oauth"), - path.Join(rootDir, ControlPlaneDir, "smm.tmpl"), - path.Join(rootDir, ControlPlaneDir, "namespace.patch.tmpl"), - - path.Join(rootDir, AuthDir, "namespace.tmpl"), - path.Join(rootDir, AuthDir, "auth-smm.tmpl"), - path.Join(rootDir, AuthDir, "base"), - path.Join(rootDir, AuthDir, "rbac"), - path.Join(rootDir, AuthDir, "mesh-authz-ext-provider.patch.tmpl"), - ) - if err != nil { - return internalError(errors.WithStack(err)) - } - - data, err := o.prepareTemplateData() - if err != nil { - return internalError(errors.WithStack(err)) - } - - for i, m := range o.manifests { - if err := m.processTemplate(data); err != nil { - return internalError(errors.WithStack(err)) - } - - fmt.Printf("%d: %+v\n", i, m) - } - - return nil -} - -func (o *OssmInstaller) loadManifestsFrom(paths ...string) error { - var err error - var manifests []manifest - - for _, p := range paths { - manifests, err = loadManifestsFrom(manifests, p) - if err != nil { - return internalError(errors.WithStack(err)) - } - } - - o.manifests = manifests - - return nil -} - -func loadManifestsFrom(manifests []manifest, path string) ([]manifest, error) { - if err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - basePath := filepath.Base(path) - manifests = append(manifests, manifest{ - name: basePath, - path: path, - patch: strings.Contains(basePath, ".patch"), - template: filepath.Ext(path) == ".tmpl", - }) - return nil - }); err != nil { - return nil, internalError(errors.WithStack(err)) - } - - return manifests, nil -} - -// TODO(smell) this is now holding two responsibilities: -// - creates data structure to be fed to templates -// - creates secrets using k8s API calls -func (o *OssmInstaller) prepareTemplateData() (interface{}, error) { - data := struct { - *ossmplugin.OssmPluginSpec - OAuth oAuth - Domain, - AppNamespace string - }{ - AppNamespace: o.KfConfig.Namespace, - } - - spec, err := o.GetPluginSpec() - if err != nil { - return nil, internalError(errors.WithStack(err)) - } - data.OssmPluginSpec = spec - - if domain, err := GetDomain(o.config); err == nil { - data.Domain = domain - } else { - return nil, internalError(errors.WithStack(err)) - } - - var clientSecret, hmac *secret.Secret - if clientSecret, err = secret.NewSecret("ossm-odh-oauth", "random", 32); err != nil { - return nil, internalError(errors.WithStack(err)) - } - - if hmac, err = secret.NewSecret("ossm-odh-hmac", "random", 32); err != nil { - return nil, internalError(errors.WithStack(err)) - } - - if oauthServerDetailsJson, err := GetOAuthServerDetails(); err == nil { - hostName, port, errUrlParsing := ExtractHostNameAndPort(oauthServerDetailsJson.Get("issuer").MustString("issuer")) - if errUrlParsing != nil { - return nil, internalError(errUrlParsing) - } - - data.OAuth = oAuth{ - AuthzEndpoint: oauthServerDetailsJson.Get("authorization_endpoint").MustString("authorization_endpoint"), - TokenEndpoint: oauthServerDetailsJson.Get("token_endpoint").MustString("token_endpoint"), - Route: hostName, - Port: port, - ClientSecret: clientSecret.Value, - Hmac: hmac.Value, - } - } else { - return nil, internalError(errors.WithStack(err)) - } - - if spec.Mesh.Certificate.Generate { - if err := o.createSelfSignedCerts(data.Domain, metav1.ObjectMeta{ - Name: spec.Mesh.Certificate.Name, - Namespace: spec.Mesh.Namespace, - }); err != nil { - return nil, internalError(err) - } - } - - if err := o.createEnvoySecret(data.OAuth, metav1.ObjectMeta{ - Name: data.AppNamespace + "-oauth2-tokens", - Namespace: data.Mesh.Namespace, - }); err != nil { - return nil, internalError(err) - } - - return data, nil -} diff --git a/pkg/kfapp/ossm/template_loader.go b/pkg/kfapp/ossm/template_loader.go new file mode 100644 index 00000000000..3ee114a335d --- /dev/null +++ b/pkg/kfapp/ossm/template_loader.go @@ -0,0 +1,40 @@ +package ossm + +import ( + "embed" + "io/fs" + "os" + "path/filepath" +) + +//go:embed templates +var embeddedFiles embed.FS + +// In order to process the templates, we need to create a tmp directory +// to store the files. This is because embedded files are read only. +// copyEmbeddedFS ensures that files embedded using go:embed are populated +// to dest directory +func copyEmbeddedFiles(src, dest string) error { + return fs.WalkDir(embeddedFiles, src, func(path string, dir fs.DirEntry, err error) error { + if err != nil { + return err + } + + destPath := filepath.Join(dest, path) + if dir.IsDir() { + if err := os.MkdirAll(destPath, 0755); err != nil { + return err + } + } else { + data, err := fs.ReadFile(embeddedFiles, path) + if err != nil { + return err + } + if err := os.WriteFile(destPath, data, 0644); err != nil { + return err + } + } + + return nil + }) +} diff --git a/pkg/kfapp/ossm/types.go b/pkg/kfapp/ossm/types.go deleted file mode 100644 index d1690b105b7..00000000000 --- a/pkg/kfapp/ossm/types.go +++ /dev/null @@ -1,96 +0,0 @@ -package ossm - -import ( - "fmt" - "html/template" - "io/fs" - "os" - "path/filepath" - "strings" -) - -type oAuth struct { - AuthzEndpoint, - TokenEndpoint, - Route, - Port, - ClientSecret, - Hmac string -} - -type manifest struct { - name, - path string - template, - patch, - processed bool -} - -const ( - ControlPlaneDir = "templates/control-plane" - AuthDir = "templates/authorino" - baseOutputDir = "/tmp/ossm-installer/" -) - -func (m *manifest) targetPath() string { - return fmt.Sprintf("%s%s", m.path[:len(m.path)-len(filepath.Ext(m.path))], ".yaml") -} - -func (m *manifest) processTemplate(data interface{}) error { - if !m.template { - return nil - } - path := m.targetPath() - - f, err := os.Create(path) - if err != nil { - log.Error(err, "Failed to create file") - return err - } - - tmpl := template.New(m.name). - Funcs(template.FuncMap{"ReplaceChar": ReplaceChar}) - - tmpl, err = tmpl.ParseFiles(m.path) - if err != nil { - return err - } - - err = tmpl.Execute(f, data) - m.processed = err == nil - - return err -} - -func ReplaceChar(s string, oldChar, newChar string) string { - return strings.ReplaceAll(s, oldChar, newChar) -} - -// In order to process the templates, we need to create a tmp directory -// to store the files. This is because embedded files are read only. -// copyEmbeddedFS ensures that files embedded using go:embed are populated -// to dest directory -func copyEmbeddedFS(fsys fs.FS, root, dest string) error { - return fs.WalkDir(fsys, root, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - - destPath := filepath.Join(dest, path) - if d.IsDir() { - if err := os.MkdirAll(destPath, 0755); err != nil { - return err - } - } else { - data, err := fs.ReadFile(fsys, path) - if err != nil { - return err - } - if err := os.WriteFile(destPath, data, 0644); err != nil { - return err - } - } - - return nil - }) -} diff --git a/pkg/kfconfig/ossmplugin/types.go b/pkg/kfconfig/ossmplugin/types.go index 6a74b3b1502..d4d4f9ceed3 100644 --- a/pkg/kfconfig/ossmplugin/types.go +++ b/pkg/kfconfig/ossmplugin/types.go @@ -20,6 +20,9 @@ type KfOssmPlugin struct { type OssmPluginSpec struct { Mesh MeshSpec `json:"mesh,omitempty"` Auth AuthSpec `json:"auth,omitempty"` + + // Additional non-user facing fields (should not be copied to the CRD) + AppNamespace string `json:"appNamespace,omitempty"` } type MeshSpec struct { diff --git a/tests/data/test-data.tar.gz b/tests/data/test-data.tar.gz index 445e441d306..5e3383f05d5 100644 Binary files a/tests/data/test-data.tar.gz and b/tests/data/test-data.tar.gz differ diff --git a/pkg/kfapp/ossm/test/crd/servicemeshcontrolplanes.crd.yaml b/tests/integration/crd/servicemeshcontrolplanes.crd.yaml similarity index 100% rename from pkg/kfapp/ossm/test/crd/servicemeshcontrolplanes.crd.yaml rename to tests/integration/crd/servicemeshcontrolplanes.crd.yaml diff --git a/pkg/kfapp/ossm/test/crd/test-resource.yaml b/tests/integration/crd/test-resource.yaml similarity index 100% rename from pkg/kfapp/ossm/test/crd/test-resource.yaml rename to tests/integration/crd/test-resource.yaml diff --git a/tests/integration/ossm_installer_int_test.go b/tests/integration/ossm_installer_int_test.go index ac3ddaae05f..0c19099d51e 100644 --- a/tests/integration/ossm_installer_int_test.go +++ b/tests/integration/ossm_installer_int_test.go @@ -2,11 +2,16 @@ package ossm_test import ( "context" + "crypto/rand" + "encoding/hex" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/opendatahub-io/opendatahub-operator/pkg/kfapp/ossm" - "github.com/opendatahub-io/opendatahub-operator/pkg/kfapp/ossm/test/testenv" + "github.com/opendatahub-io/opendatahub-operator/pkg/kfapp/ossm/feature" "github.com/opendatahub-io/opendatahub-operator/pkg/kfconfig" + "github.com/opendatahub-io/opendatahub-operator/pkg/kfconfig/ossmplugin" + "github.com/opendatahub-io/opendatahub-operator/tests/integration/testenv" + "io" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -14,6 +19,9 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" + "os" + "path" + "path/filepath" "time" ) @@ -22,28 +30,151 @@ const ( interval = 250 * time.Millisecond ) -var _ = When("Migrating Data Science Projects", func() { +var _ = Describe("CRD presence verification", func() { + var ( + ossmInstaller *ossm.OssmInstaller + ossmPluginSpec *ossmplugin.OssmPluginSpec + verificationFeature *feature.Feature + ) + + BeforeEach(func() { + ossmInstaller = newOssmInstaller("default") + var err error + ossmPluginSpec, err = ossmInstaller.GetPluginSpec() + Expect(err).ToNot(HaveOccurred()) + }) + + It("should successfully check existing CRD", func() { + // given example CRD installed into env from /ossm/test/crd/ + crdGroup := "ossm.plugins.kubeflow.org" + crdVersion := "test-version" + crdResource := "test-resources" + + var err error + verificationFeature, err = feature.CreateFeature("CRD verification"). + For(ossmPluginSpec). + UsingConfig(envTest.Config). + Preconditions(feature.EnsureCRDIsInstalled(crdGroup, crdVersion, crdResource)). + Load() + Expect(err).ToNot(HaveOccurred()) + + // when + err = verificationFeature.Apply() + + // then + Expect(err).ToNot(HaveOccurred()) + }) + + It("should fail to check non-existing CRD", func() { + // given + crdGroup := "non-existing-group" + crdVersion := "non-existing-version" + crdResource := "non-existing-resource" + + var err error + verificationFeature, err = feature.CreateFeature("CRD verification"). + For(ossmPluginSpec). + UsingConfig(envTest.Config). + Preconditions(feature.EnsureCRDIsInstalled(crdGroup, crdVersion, crdResource)). + Load() + Expect(err).ToNot(HaveOccurred()) + + // when + err = verificationFeature.Apply() + + // then + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("server could not find the requested resource")) + }) +}) + +var _ = Describe("Ensuring service mesh is set up correctly", func() { + + var ( + objectCleaner *testenv.Cleaner + ossmInstaller *ossm.OssmInstaller + ossmPluginSpec *ossmplugin.OssmPluginSpec + serviceMeshCheck *feature.Feature + name = "test-name" + namespace = "test-namespace" + ) + + BeforeEach(func() { + ossmInstaller = newOssmInstaller(namespace) + var err error + ossmPluginSpec, err = ossmInstaller.GetPluginSpec() + Expect(err).ToNot(HaveOccurred()) + + ossmPluginSpec.Mesh.Name = name + ossmPluginSpec.Mesh.Namespace = namespace + + serviceMeshCheck, err = feature.CreateFeature("datascience-project-migration"). + For(ossmPluginSpec). + UsingConfig(envTest.Config). + Preconditions(feature.EnsureServiceMeshInstalled).Load() + + Expect(err).ToNot(HaveOccurred()) + + objectCleaner = testenv.CreateCleaner(envTestClient, envTest.Config, timeout, interval) + }) + + It("should find installed SMCP", func() { + ns := createNamespace(namespace) + Expect(envTestClient.Create(context.Background(), ns)).To(Succeed()) + defer objectCleaner.DeleteAll(ns) + + createServiceMeshControlPlane(name, namespace) + + // when + err := serviceMeshCheck.Apply() + + // then + Expect(err).ToNot(HaveOccurred()) + }) + + It("should fail to find SMCP if not present", func() { + Expect(serviceMeshCheck.Apply()).To(HaveOccurred()) + }) + +}) + +var _ = Describe("Data Science Project Migration", func() { var ( - objectCleaner *testenv.Cleaner - ossmInstaller *ossm.OssmInstaller + objectCleaner *testenv.Cleaner + ossmInstaller *ossm.OssmInstaller + ossmPluginSpec *ossmplugin.OssmPluginSpec + migrationFeature *feature.Feature ) BeforeEach(func() { - ossmInstaller = ossm.NewOssmInstaller(&kfconfig.KfConfig{}, envTest.Config) - objectCleaner = testenv.CreateCleaner(cli, envTest.Config, timeout, interval) + objectCleaner = testenv.CreateCleaner(envTestClient, envTest.Config, timeout, interval) + + ossmInstaller = newOssmInstaller("default") + + var err error + ossmPluginSpec, err = ossmInstaller.GetPluginSpec() + Expect(err).ToNot(HaveOccurred()) + + migrationFeature, err = feature.CreateFeature("datascience-project-migration"). + For(ossmPluginSpec). + UsingConfig(envTest.Config). + WithResources(feature.MigratedDataScienceProjects).Load() + + Expect(err).ToNot(HaveOccurred()) + }) It("should migrate single namespace", func() { // given dataScienceNs := createDataScienceProject("dsp-01") regularNs := createNamespace("non-dsp") - Expect(cli.Create(context.Background(), dataScienceNs)).To(Succeed()) - Expect(cli.Create(context.Background(), regularNs)).To(Succeed()) + Expect(envTestClient.Create(context.Background(), dataScienceNs)).To(Succeed()) + Expect(envTestClient.Create(context.Background(), regularNs)).To(Succeed()) defer objectCleaner.DeleteAll(dataScienceNs, regularNs) // when - Expect(ossmInstaller.MigrateDataScienceProjects()).ToNot(HaveOccurred()) + Expect(migrationFeature.Apply()).ToNot(HaveOccurred()) // then Eventually(findMigratedNamespaces, timeout, interval).Should( @@ -57,11 +188,11 @@ var _ = When("Migrating Data Science Projects", func() { It("should not migrate any non-datascience namespace", func() { // given regularNs := createNamespace("non-dsp") - Expect(cli.Create(context.Background(), regularNs)).To(Succeed()) + Expect(envTestClient.Create(context.Background(), regularNs)).To(Succeed()) defer objectCleaner.DeleteAll(regularNs) // when - Expect(ossmInstaller.MigrateDataScienceProjects()).ToNot(HaveOccurred()) + Expect(migrationFeature.Apply()).ToNot(HaveOccurred()) // then Consistently(findMigratedNamespaces, timeout, interval).Should(BeEmpty()) // we can't wait forever, but this should be good enough @@ -73,14 +204,14 @@ var _ = When("Migrating Data Science Projects", func() { dataScienceNs02 := createDataScienceProject("dsp-02") dataScienceNs03 := createDataScienceProject("dsp-03") regularNs := createNamespace("non-dsp") - Expect(cli.Create(context.Background(), dataScienceNs01)).To(Succeed()) - Expect(cli.Create(context.Background(), dataScienceNs02)).To(Succeed()) - Expect(cli.Create(context.Background(), dataScienceNs03)).To(Succeed()) - Expect(cli.Create(context.Background(), regularNs)).To(Succeed()) + Expect(envTestClient.Create(context.Background(), dataScienceNs01)).To(Succeed()) + Expect(envTestClient.Create(context.Background(), dataScienceNs02)).To(Succeed()) + Expect(envTestClient.Create(context.Background(), dataScienceNs03)).To(Succeed()) + Expect(envTestClient.Create(context.Background(), regularNs)).To(Succeed()) defer objectCleaner.DeleteAll(dataScienceNs01, dataScienceNs02, dataScienceNs03, regularNs) // when - Expect(ossmInstaller.MigrateDataScienceProjects()).ToNot(HaveOccurred()) + Expect(migrationFeature.Apply()).ToNot(HaveOccurred()) // then Eventually(findMigratedNamespaces, timeout, interval).Should( @@ -93,96 +224,130 @@ var _ = When("Migrating Data Science Projects", func() { }) -var _ = When("Checking for CRD", func() { - var ( - ossmInstaller *ossm.OssmInstaller - ) +var _ = Describe("Cleanup operations", func() { - BeforeEach(func() { - ossmInstaller = ossm.NewOssmInstaller(&kfconfig.KfConfig{}, envTest.Config) - }) + Context("configuring control plane for auth(z)", func() { - It("should successfully check existing CRD", func() { - // given example CRD installed into env from /ossm/test/crd/ - crdGroup := "ossm.plugins.kubeflow.org" - crdVersion := "test-version" - crdResource := "test-resources" + var ( + objectCleaner *testenv.Cleaner + ossmInstaller *ossm.OssmInstaller + ossmPluginSpec *ossmplugin.OssmPluginSpec + namespace = "test" + name = "minimal" + ) - // when - err := ossmInstaller.VerifyCRDInstalled(crdGroup, crdVersion, crdResource) + BeforeEach(func() { + objectCleaner = testenv.CreateCleaner(envTestClient, envTest.Config, timeout, interval) - // then - Expect(err).ToNot(HaveOccurred()) - }) + ossmInstaller = newOssmInstaller(namespace) - It("should fail to check non-existing CRD", func() { - // given - crdGroup := "non-existing-group" - crdVersion := "non-existing-version" - crdResource := "non-existing-resource" + var err error + ossmPluginSpec, err = ossmInstaller.GetPluginSpec() + Expect(err).ToNot(HaveOccurred()) - // when - err := ossmInstaller.VerifyCRDInstalled(crdGroup, crdVersion, crdResource) + ossmPluginSpec.Mesh.Name = name + ossmPluginSpec.Mesh.Namespace = namespace + }) - // then - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("server could not find the requested resource")) - }) -}) + It("should be able to remove mounted secret volumes on cleanup", func() { + // given + ns := createNamespace(namespace) + Expect(envTestClient.Create(context.Background(), ns)).To(Succeed()) + defer objectCleaner.DeleteAll(ns) -var _ = When("Ensuring environment is set up correctly", func() { + createServiceMeshControlPlane(name, namespace) - var ( - objectCleaner *testenv.Cleaner - ossmInstaller *ossm.OssmInstaller - name = "test-name" - namespace = "test-namespace" - ) + controlPlaneWithSecretVolumes, err := feature.CreateFeature("control-plane-with-secret-volumes"). + For(ossmPluginSpec). + Manifests(inTestTmpDir(path.Join(feature.ControlPlaneDir, "base/control-plane-ingress.patch.tmpl"))). + UsingConfig(envTest.Config). + Load() - BeforeEach(func() { - ossmInstaller = ossm.NewOssmInstaller(&kfconfig.KfConfig{}, envTest.Config) - objectCleaner = testenv.CreateCleaner(cli, envTest.Config, timeout, interval) - }) + Expect(err).ToNot(HaveOccurred()) - It("should return status if SMCP is found and status is available", func() { - ns := createNamespace(namespace) - Expect(cli.Create(context.Background(), ns)).To(Succeed()) - smcpObj := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "maistra.io/v1", - "kind": "ServiceMeshControlPlane", - "metadata": map[string]interface{}{ - "name": name, - "namespace": namespace, - }, - "spec": map[string]interface{}{}, - }, - } - createErr := CreateSMCP(namespace, smcpObj, envTest.Config) - Expect(createErr).To(BeNil()) - defer objectCleaner.DeleteAll(ns) + // when + Expect(controlPlaneWithSecretVolumes.Apply()).ToNot(HaveOccurred()) + // Testing removal function on its own relying on feature setup + err = feature.RemoveTokenVolumes(controlPlaneWithSecretVolumes) - // when - status, err := ossmInstaller.CheckSMCPStatus(name, namespace) + // then + serviceMeshControlPlane, err := getServiceMeshControlPlane(envTest.Config, namespace, name) + Expect(err).ToNot(HaveOccurred()) - // then - Expect(err).To(BeNil()) - Expect(status).To(Equal("Ready")) - }) + volumes, found, err := unstructured.NestedSlice(serviceMeshControlPlane.Object, "spec", "gateways", "ingress", "volumes") + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(volumes).To(BeEmpty()) + }) - It("should return error if failed to find SMCP", func() { - // Don't create namespace or SMCP. + It("should be able to remove external provider on cleanup", func() { + // given + ns := createNamespace(namespace) + Expect(envTestClient.Create(context.Background(), ns)).To(Succeed()) + defer objectCleaner.DeleteAll(ns) - // when - status, err := ossmInstaller.CheckSMCPStatus(name, namespace) + ossmPluginSpec.Auth.Namespace = "test-provider" + ossmPluginSpec.Auth.Authorino.Name = "authorino" + + createServiceMeshControlPlane(name, namespace) + + controlPlaneWithExtAuthzProvider, err := feature.CreateFeature("control-plane-with-external-authz-provider"). + For(ossmPluginSpec). + Manifests(inTestTmpDir(path.Join(feature.AuthDir, "mesh-authz-ext-provider.patch.tmpl"))). + UsingConfig(envTest.Config). + Load() + + Expect(err).ToNot(HaveOccurred()) + + // when + By("verifying extension provider has been added after applying feature", func() { + Expect(controlPlaneWithExtAuthzProvider.Apply()).ToNot(HaveOccurred()) + serviceMeshControlPlane, err := getServiceMeshControlPlane(envTest.Config, namespace, name) + Expect(err).ToNot(HaveOccurred()) + + extensionProviders, found, err := unstructured.NestedSlice(serviceMeshControlPlane.Object, "spec", "techPreview", "meshConfig", "extensionProviders") + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + + extensionProvider := extensionProviders[0].(map[string]interface{}) + Expect(extensionProvider["name"]).To(Equal("test-odh-auth-provider")) + Expect(extensionProvider["envoyExtAuthzGrpc"].(map[string]interface{})["service"]).To(Equal("authorino-authorino-authorization.test-provider.svc.cluster.local")) + }) + + // then + By("verifying that extension provider has been removed", func() { + err = feature.RemoveExtensionProvider(controlPlaneWithExtAuthzProvider) + serviceMeshControlPlane, err := getServiceMeshControlPlane(envTest.Config, namespace, name) + Expect(err).ToNot(HaveOccurred()) + + extensionProviders, found, err := unstructured.NestedSlice(serviceMeshControlPlane.Object, "spec", "techPreview", "meshConfig", "extensionProviders") + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(extensionProviders).To(BeEmpty()) + }) + + }) - // then - Expect(err).To(HaveOccurred()) - Expect(status).To(BeEmpty()) }) }) +func createServiceMeshControlPlane(name, namespace string) { + serviceMeshControlPlane := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "maistra.io/v2", + "kind": "ServiceMeshControlPlane", + "metadata": map[string]interface{}{ + "name": name, + "namespace": namespace, + }, + "spec": map[string]interface{}{}, + }, + } + createErr := createSMCPInCluster(envTest.Config, serviceMeshControlPlane, namespace) + Expect(createErr).ToNot(HaveOccurred()) +} + func createDataScienceProject(name string) *v1.Namespace { namespace := createNamespace(name) namespace.Labels = map[string]string{ @@ -202,7 +367,7 @@ func createNamespace(name string) *v1.Namespace { func findMigratedNamespaces() []string { namespaces := &v1.NamespaceList{} var ns []string - if err := cli.List(context.Background(), namespaces); err != nil && !errors.IsNotFound(err) { + if err := envTestClient.List(context.Background(), namespaces); err != nil && !errors.IsNotFound(err) { Fail(err.Error()) } for _, namespace := range namespaces.Items { @@ -213,8 +378,18 @@ func findMigratedNamespaces() []string { return ns } -// CreateSMCP uses dynamic client to create a dummy SMCP for testing -func CreateSMCP(namespace string, smcpObj *unstructured.Unstructured, cfg *rest.Config) error { +func newOssmInstaller(ns string) *ossm.OssmInstaller { + config := kfconfig.KfConfig{} + config.SetNamespace(ns) + config.Spec.Plugins = append(config.Spec.Plugins, kfconfig.Plugin{ + Name: "KfOssmPlugin", + Kind: "KfOssmPlugin", + }) + return ossm.NewOssmInstaller(&config, envTest.Config) +} + +// createSMCPInCluster uses dynamic client to create a dummy SMCP resource for testing +func createSMCPInCluster(cfg *rest.Config, smcpObj *unstructured.Unstructured, namespace string) error { dynamicClient, err := dynamic.NewForConfig(cfg) if err != nil { return err @@ -222,7 +397,7 @@ func CreateSMCP(namespace string, smcpObj *unstructured.Unstructured, cfg *rest. gvr := schema.GroupVersionResource{ Group: "maistra.io", - Version: "v1", + Version: "v2", Resource: "servicemeshcontrolplanes", } @@ -231,7 +406,6 @@ func CreateSMCP(namespace string, smcpObj *unstructured.Unstructured, cfg *rest. return err } - // Since we don't have maistra operator, we simulate the status statusConditions := []interface{}{ map[string]interface{}{ "type": "Ready", @@ -239,8 +413,16 @@ func CreateSMCP(namespace string, smcpObj *unstructured.Unstructured, cfg *rest. }, } + // Since we don't have actual service mesh operator deployed, we simulate the status status := map[string]interface{}{ "conditions": statusConditions, + "readiness": map[string]interface{}{ + "components": map[string]interface{}{ + "pending": []interface{}{}, + "ready": []interface{}{}, + "unready": []interface{}{}, + }, + }, } if err := unstructured.SetNestedField(result.Object, status, "status"); err != nil { @@ -254,3 +436,68 @@ func CreateSMCP(namespace string, smcpObj *unstructured.Unstructured, cfg *rest. return nil } + +func getServiceMeshControlPlane(cfg *rest.Config, namespace, name string) (*unstructured.Unstructured, error) { + dynamicClient, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, err + } + + gvr := schema.GroupVersionResource{ + Group: "maistra.io", + Version: "v2", + Resource: "servicemeshcontrolplanes", + } + + smcp, err := dynamicClient.Resource(gvr).Namespace(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + return smcp, nil +} + +func inTestTmpDir(fileName string) string { + root, err := findProjectRoot() + Expect(err).ToNot(HaveOccurred()) + + tmpDir := filepath.Join(os.TempDir(), randomUUIDName(16)) + if err := os.Mkdir(tmpDir, os.ModePerm); err != nil { + Fail(err.Error()) + } + + src := path.Join(root, "pkg/kfapp/ossm", fileName) + dest := path.Join(tmpDir, fileName) + if err := copyFile(src, dest); err != nil { + Fail(err.Error()) + } + + return dest +} + +func copyFile(src, dst string) error { + source, err := os.Open(src) + if err != nil { + return err + } + defer source.Close() + + if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil { + return err + } + + destination, err := os.Create(dst) + if err != nil { + return err + } + defer destination.Close() + + _, err = io.Copy(destination, source) + return err +} + +func randomUUIDName(len int) string { + uuidBytes := make([]byte, len) + _, _ = rand.Read(uuidBytes) + return hex.EncodeToString(uuidBytes)[:len] +} diff --git a/tests/integration/ossm_suite_int_test.go b/tests/integration/ossm_suite_int_test.go index eeb73f185de..8fc23a0b8d6 100644 --- a/tests/integration/ossm_suite_int_test.go +++ b/tests/integration/ossm_suite_int_test.go @@ -21,10 +21,10 @@ import ( ) var ( - cli client.Client - envTest *envtest.Environment - ctx context.Context - cancel context.CancelFunc + envTestClient client.Client + envTest *envtest.Environment + ctx context.Context + cancel context.CancelFunc ) var testScheme = runtime.NewScheme() @@ -55,20 +55,20 @@ var _ = BeforeSuite(func() { Paths: []string{ filepath.Join(projectDir, "config", "crd", "bases"), filepath.Join(projectDir, "config", "crd", "dashboard-crds"), - filepath.Join(projectDir, "pkg", "kfapp", "ossm", "test", "crd"), + filepath.Join(projectDir, "tests", "integration", "crd"), }, ErrorIfPathMissing: true, CleanUpAfterUse: false, }, } - cfg, err := envTest.Start() + config, err := envTest.Start() Expect(err).NotTo(HaveOccurred()) - Expect(cfg).NotTo(BeNil()) + Expect(config).NotTo(BeNil()) - cli, err = client.New(cfg, client.Options{Scheme: testScheme}) + envTestClient, err = client.New(config, client.Options{Scheme: testScheme}) Expect(err).NotTo(HaveOccurred()) - Expect(cli).NotTo(BeNil()) + Expect(envTestClient).NotTo(BeNil()) }) var _ = AfterSuite(func() { diff --git a/pkg/kfapp/ossm/test/testenv/cleaner.go b/tests/integration/testenv/cleaner.go similarity index 100% rename from pkg/kfapp/ossm/test/testenv/cleaner.go rename to tests/integration/testenv/cleaner.go