From 0ec07de6d02eafd2bd93c1a2cfb7260bdfda502d Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Mon, 1 Apr 2024 09:40:46 +0300 Subject: [PATCH] e2e:control and data planes clients On Hypershift we should be able to access both the management and hosted clusters. For that, we should now have two clients, each communicates with a different cluster. ControlPlaneClient - defines the API client to run CRUD operations on the control plane that will be used for testing. DataPlaneClient - defines the API client to run CRUD operations on the data plane that will be used for testing. On non-hypershift both client would point to the same the cluster. We also added Ginkgo assertions that prevent the user from using the wrong clients for the wrong objects. For example using the `DataPlaneClient` for querying the `PerformanceProfile` object is completley fine on non-hypershift cluster, but will fail badly on hypershift (since `PerformanceProfile` objects, are not presented on the hosted clusters. Signed-off-by: Talor Itzhak --- .../functests/utils/client/clients.go | 51 ++++- .../functests/utils/client/dataplane.go | 70 +++++++ .../functests/utils/hypershift/hypershift.go | 185 ++++++++++++++++++ 3 files changed, 298 insertions(+), 8 deletions(-) create mode 100644 test/e2e/performanceprofile/functests/utils/client/dataplane.go create mode 100644 test/e2e/performanceprofile/functests/utils/hypershift/hypershift.go diff --git a/test/e2e/performanceprofile/functests/utils/client/clients.go b/test/e2e/performanceprofile/functests/utils/client/clients.go index 28a80da393..0e7cd3e82d 100644 --- a/test/e2e/performanceprofile/functests/utils/client/clients.go +++ b/test/e2e/performanceprofile/functests/utils/client/clients.go @@ -6,6 +6,7 @@ import ( . "github.com/onsi/gomega" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/klog" @@ -15,18 +16,24 @@ import ( configv1 "github.com/openshift/api/config/v1" mcov1 "github.com/openshift/api/machineconfiguration/v1" - tunedv1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/tuned/v1" - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" performancev1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v1" performancev1alpha1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v1alpha1" performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2" + tunedv1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/tuned/v1" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/hypershift" testlog "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/log" ) var ( - // Client defines the API client to run CRUD operations, that will be used for testing + // Client kept for backport compatibility until all tests will be converted Client client.Client + // ControlPlaneClient defines the API client to run CRUD operations on the control plane cluster, + //that will be used for testing + ControlPlaneClient client.Client + // DataPlaneClient defines the API client to run CRUD operations on the data plane cluster, + //that will be used for testing + DataPlaneClient client.Client // K8sClient defines k8s client to run subresource operations, for example you should use it to get pod logs K8sClient *kubernetes.Clientset // ClientsEnabled tells if the client from the package can be used @@ -67,27 +74,55 @@ func init() { Client, err = New() if err != nil { testlog.Info("Failed to initialize client, check the KUBECONFIG env variable", err.Error()) - ClientsEnabled = false + return + } + + ControlPlaneClient, err = NewControlPlane() + if err != nil { + testlog.Info("Failed to initialize ControlPlaneClient client, check the KUBECONFIG env variable", err.Error()) + return + } + DataPlaneClient, err = NewDataPlane() + if err != nil { + testlog.Info("Failed to initialize DataPlaneClient client, check the KUBECONFIG env variable", err.Error()) return } K8sClient, err = NewK8s() if err != nil { testlog.Info("Failed to initialize k8s client, check the KUBECONFIG env variable", err.Error()) - ClientsEnabled = false return } ClientsEnabled = true } -// New returns a controller-runtime client. func New() (client.Client, error) { cfg, err := config.GetConfig() if err != nil { return nil, err } + return client.New(cfg, client.Options{}) +} + +// NewControlPlane returns a new controller-runtime client for ControlPlaneClient cluster. +func NewControlPlane() (client.Client, error) { + if hypershift.IsHypershiftCluster() { + testlog.Info("creating ControlPlaneClient client for hypershift cluster") + return hypershift.BuildControlPlaneClient() + } + return New() +} - c, err := client.New(cfg, client.Options{}) - return c, err +// NewDataPlane returns a new controller-runtime client for DataPlaneClient cluster. +func NewDataPlane() (client.Client, error) { + var c client.Client + var err error + if hypershift.IsHypershiftCluster() { + testlog.Info("creating DataPlaneClient client for hypershift cluster") + c, err = hypershift.BuildDataPlaneClient() + } else { + c, err = New() + } + return &dataPlaneImpl{c}, err } // NewK8s returns a kubernetes clientset diff --git a/test/e2e/performanceprofile/functests/utils/client/dataplane.go b/test/e2e/performanceprofile/functests/utils/client/dataplane.go new file mode 100644 index 0000000000..c1c4758963 --- /dev/null +++ b/test/e2e/performanceprofile/functests/utils/client/dataplane.go @@ -0,0 +1,70 @@ +package client + +import ( + "context" + "fmt" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/hypershift" +) + +type dataPlaneImpl struct { + client.Client +} + +func (dpi *dataPlaneImpl) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if hypershift.ObjIsEncapsulatedInConfigMap(obj) { + return fmt.Errorf("the provided object %s/%s might not be presented on hypershift cluster while using this client."+ + "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) + } + return dpi.Client.Get(ctx, key, obj, opts...) +} + +func (dpi *dataPlaneImpl) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if hypershift.ObjIsEncapsulatedInConfigMap(list) { + return fmt.Errorf("the provided list of %s objects might not be presented on hypershift cluster while using this client."+ + "please use ControlPlaneClient client instead", list.GetObjectKind()) + } + return dpi.Client.List(ctx, list, opts...) +} + +func (dpi *dataPlaneImpl) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if hypershift.ObjIsEncapsulatedInConfigMap(obj) { + return fmt.Errorf("the provided object %s/%s might not get created on hypershift cluster while using this client."+ + "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) + } + return dpi.Client.Create(ctx, obj, opts...) +} + +func (dpi *dataPlaneImpl) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + if hypershift.ObjIsEncapsulatedInConfigMap(obj) { + return fmt.Errorf("the provided object %s/%s might not get deleted on hypershift cluster while using this client."+ + "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) + } + return dpi.Client.Delete(ctx, obj, opts...) +} + +func (dpi *dataPlaneImpl) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if hypershift.ObjIsEncapsulatedInConfigMap(obj) { + return fmt.Errorf("the provided object %s/%s might not get updated on hypershift cluster while using this client."+ + "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) + } + return dpi.Client.Update(ctx, obj, opts...) +} + +func (dpi *dataPlaneImpl) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if hypershift.ObjIsEncapsulatedInConfigMap(obj) { + return fmt.Errorf("the provided object %s/%s might not get patched on hypershift cluster while using this client."+ + "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) + } + return dpi.Client.Patch(ctx, obj, patch, opts...) +} + +func (dpi *dataPlaneImpl) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + if hypershift.ObjIsEncapsulatedInConfigMap(obj) { + return fmt.Errorf("the provided object %s/%s might not get deleted on hypershift cluster while using this client."+ + "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) + } + return dpi.Client.DeleteAllOf(ctx, obj, opts...) +} diff --git a/test/e2e/performanceprofile/functests/utils/hypershift/hypershift.go b/test/e2e/performanceprofile/functests/utils/hypershift/hypershift.go new file mode 100644 index 0000000000..646ad86fde --- /dev/null +++ b/test/e2e/performanceprofile/functests/utils/hypershift/hypershift.go @@ -0,0 +1,185 @@ +package hypershift + +import ( + "bytes" + "context" + "fmt" + "os" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + serializer "k8s.io/apimachinery/pkg/runtime/serializer/json" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + + mcov1 "github.com/openshift/api/machineconfiguration/v1" + performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2" + tunedv1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/tuned/v1" +) + +var isHypershiftCluster bool + +func init() { + if v, ok := os.LookupEnv("CLUSTER_TYPE"); ok && v == "hypershift" { + isHypershiftCluster = true + } +} + +const ( + ManagementClusterKubeConfigEnv = "HYPERSHIFT_MANAGEMENT_CLUSTER_KUBECONFIG" + ManagementClusterNamespaceEnv = "HYPERSHIFT_MANAGEMENT_CLUSTER_NAMESPACE" + HostedClusterKubeConfigEnv = "HYPERSHIFT_HOSTED_CLUSTER_KUBECONFIG" + HostedClusterNameEnv = "CLUSTER_NAME" + NodePoolNamespace = "clusters" +) + +// a set of keys which used to classify the encapsulated objects in the ConfigMap +const ( + TuningKey = "tuning" + ConfigKey = "config" +) + +type ControlPlaneClientImpl struct { + // A client with access to the management cluster + client.Client + + // managementClusterNamespaceName is the namespace name on the management cluster + // on which the control-plane objects reside + managementClusterNamespaceName string +} + +func (ci *ControlPlaneClientImpl) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if ObjIsEncapsulatedInConfigMap(obj) { + return ci.getFromConfigMap(ctx, key, obj, opts...) + } + return ci.Client.Get(ctx, key, obj, opts...) +} + +func (ci *ControlPlaneClientImpl) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if ObjIsEncapsulatedInConfigMap(obj) { + return ci.createInConfigMap(ctx, obj, opts...) + } + return ci.Client.Create(ctx, obj, opts...) +} + +func buildClient(kubeConfigPath string) (client.Client, error) { + restConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) + if err != nil { + return nil, err + } + c, err := client.New(restConfig, client.Options{}) + if err != nil { + return nil, err + } + return c, nil +} + +func ObjIsEncapsulatedInConfigMap(obj runtime.Object) bool { + switch obj.(type) { + case *performancev2.PerformanceProfile, *performancev2.PerformanceProfileList, + *mcov1.KubeletConfig, *mcov1.KubeletConfigList, + *mcov1.MachineConfig, *mcov1.MachineConfigList, + *tunedv1.Profile, *tunedv1.ProfileList: + return true + default: + return false + } +} + +func BuildControlPlaneClient() (client.Client, error) { + kcPath, ok := os.LookupEnv(ManagementClusterKubeConfigEnv) + if !ok { + return nil, fmt.Errorf("failed to build management-cluster client for hypershift, environment variable %q is not defined", ManagementClusterKubeConfigEnv) + } + c, err := buildClient(kcPath) + if err != nil { + return nil, err + } + ns, err := GetManagementClusterNamespace() + if err != nil { + return nil, fmt.Errorf("failed to build management-cluster client for hypershift; err %v", err) + } + return &ControlPlaneClientImpl{ + Client: c, + managementClusterNamespaceName: ns, + }, nil +} + +func BuildDataPlaneClient() (client.Client, error) { + kcPath, ok := os.LookupEnv(HostedClusterKubeConfigEnv) + if !ok { + return nil, fmt.Errorf("failed to build hosted-cluster client for hypershift, environment variable %q is not defined", HostedClusterKubeConfigEnv) + } + return buildClient(kcPath) +} + +func (ci *ControlPlaneClientImpl) getFromConfigMap(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + cm := &corev1.ConfigMap{} + key.Namespace = ci.managementClusterNamespaceName + err := ci.Client.Get(ctx, key, cm, opts...) + if err != nil { + return err + } + var objAsYAML string + // can't have both + if s, ok := cm.Data[TuningKey]; ok { + objAsYAML = s + } + if s, ok := cm.Data[ConfigKey]; ok { + objAsYAML = s + } + return decodeManifest([]byte(objAsYAML), scheme.Scheme, obj) +} + +func (ci *ControlPlaneClientImpl) createInConfigMap(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: obj.GetName(), Namespace: ci.managementClusterNamespaceName}} + b, err := encodeManifest(obj, scheme.Scheme) + if err != nil { + return err + } + cm.Data[TuningKey] = string(b) + return ci.Client.Create(ctx, cm, opts...) +} + +func GetHostedClusterName() (string, error) { + v, ok := os.LookupEnv(HostedClusterNameEnv) + if !ok { + return "", fmt.Errorf("failed to retrieve hosted cluster name; %q environment var is not set", HostedClusterNameEnv) + } + return v, nil +} + +func GetManagementClusterNamespace() (string, error) { + ns, ok := os.LookupEnv(ManagementClusterNamespaceEnv) + if !ok { + return "", fmt.Errorf("failed to retrieve management cluster namespace; %q environment var is not set", ManagementClusterNamespaceEnv) + } + return ns, nil +} + +func IsHypershiftCluster() bool { + return isHypershiftCluster +} + +func encodeManifest(obj runtime.Object, scheme *runtime.Scheme) ([]byte, error) { + yamlSerializer := serializer.NewSerializerWithOptions( + serializer.DefaultMetaFactory, scheme, scheme, + serializer.SerializerOptions{Yaml: true, Pretty: true, Strict: true}, + ) + buff := bytes.Buffer{} + err := yamlSerializer.Encode(obj, &buff) + return buff.Bytes(), err +} + +func decodeManifest(b []byte, scheme *runtime.Scheme, obj runtime.Object) error { + yamlSerializer := serializer.NewSerializerWithOptions( + serializer.DefaultMetaFactory, scheme, scheme, + serializer.SerializerOptions{Yaml: true, Pretty: true, Strict: true}, + ) + // Get the GroupVersionKind of the object + gvk := obj.GetObjectKind().GroupVersionKind() + _, _, err := yamlSerializer.Decode(b, &gvk, obj) + return err +}