CNF-11817: e2e: control and data planes clients

pkg/performanceprofile/controller/performanceprofile/hypershift/hypershift.go

@@ -0,0 +1,115 @@
+package hypershift
+
+import (
+	"bytes"
+	"context"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	serializer "k8s.io/apimachinery/pkg/runtime/serializer/json"
+	"k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	machineconfigv1 "github.com/openshift/api/machineconfiguration/v1"
+	performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2"
+	tunedv1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/tuned/v1"
+)
+
+// a set of keys which used to classify the encapsulated objects in the ConfigMap
+const (
+	TuningKey = "tuning"
+	ConfigKey = "config"
+)
+
+type ControlPlaneClientImpl struct {
+	// A client with access to the management cluster
+	client.Client
+
+	// managementClusterNamespaceName is the namespace name on the management cluster
+	// on which the control-plane objects reside
+	managementClusterNamespaceName string
+}
+
+func (ci *ControlPlaneClientImpl) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
+	if ObjIsEncapsulatedInConfigMap(obj) {
+		return ci.getFromConfigMap(ctx, key, obj, opts...)
+	}
+	return ci.Client.Get(ctx, key, obj, opts...)
+}
+
+func (ci *ControlPlaneClientImpl) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
+	if ObjIsEncapsulatedInConfigMap(obj) {
+		return ci.createInConfigMap(ctx, obj, opts...)
+	}
+	return ci.Client.Create(ctx, obj, opts...)
+}
+
+func ObjIsEncapsulatedInConfigMap(obj runtime.Object) bool {
+	switch obj.(type) {
+	case *performancev2.PerformanceProfile, *performancev2.PerformanceProfileList,
+		*machineconfigv1.KubeletConfig, *machineconfigv1.KubeletConfigList,
+		*machineconfigv1.MachineConfig, *machineconfigv1.MachineConfigList,
+		*tunedv1.Tuned, *tunedv1.TunedList:
+		return true
+	default:
+		return false
+	}
+}
+
+func NewControlPlaneClient(c client.Client, ns string) *ControlPlaneClientImpl {
+	return &ControlPlaneClientImpl{
+		Client:                         c,
+		managementClusterNamespaceName: ns,
+	}
+}
+
+func (ci *ControlPlaneClientImpl) getFromConfigMap(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
+	cm := &corev1.ConfigMap{}
+	if key.Namespace == metav1.NamespaceNone {
+		key.Namespace = ci.managementClusterNamespaceName
+	}
+	err := ci.Client.Get(ctx, key, cm, opts...)
+	if err != nil {
+		return err
+	}
+	var objAsYAML string
+	// can't have both
+	if s, ok := cm.Data[TuningKey]; ok {
+		objAsYAML = s
+	}
+	if s, ok := cm.Data[ConfigKey]; ok {
+		objAsYAML = s
+	}
+	return DecodeManifest([]byte(objAsYAML), scheme.Scheme, obj)
+}
+
+func (ci *ControlPlaneClientImpl) createInConfigMap(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
+	cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: obj.GetName(), Namespace: ci.managementClusterNamespaceName}}
+	b, err := EncodeManifest(obj, scheme.Scheme)
+	if err != nil {
+		return err
+	}
+	cm.Data[TuningKey] = string(b)
+	return ci.Client.Create(ctx, cm, opts...)
+}
+
+func EncodeManifest(obj runtime.Object, scheme *runtime.Scheme) ([]byte, error) {
+	yamlSerializer := serializer.NewSerializerWithOptions(
+		serializer.DefaultMetaFactory, scheme, scheme,
+		serializer.SerializerOptions{Yaml: true, Pretty: true, Strict: true},
+	)
+	buff := bytes.Buffer{}
+	err := yamlSerializer.Encode(obj, &buff)
+	return buff.Bytes(), err
+}
+
+func DecodeManifest(b []byte, scheme *runtime.Scheme, obj runtime.Object) error {
+	yamlSerializer := serializer.NewSerializerWithOptions(
+		serializer.DefaultMetaFactory, scheme, scheme,
+		serializer.SerializerOptions{Yaml: true, Pretty: true, Strict: true},
+	)
+	// Get the GroupVersionKind of the object
+	gvk := obj.GetObjectKind().GroupVersionKind()
+	_, _, err := yamlSerializer.Decode(b, &gvk, obj)
+	return err
+}

test/e2e/performanceprofile/functests/utils/client/clients.go

@@ -21,12 +21,19 @@ import (
 	performancev1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v1"
 	performancev1alpha1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v1alpha1"
 	performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2"
+	hypershiftutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/hypershift"
 	testlog "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/log"
 )
 
 var (
-	// Client defines the API client to run CRUD operations, that will be used for testing
+	// Client kept for backport compatibility until all tests will be converted
 	Client client.Client
+	// ControlPlaneClient defines the API client to run CRUD operations on the control plane cluster,
+	//that will be used for testing
+	ControlPlaneClient client.Client
+	// DataPlaneClient defines the API client to run CRUD operations on the data plane cluster,
+	//that will be used for testing
+	DataPlaneClient client.Client
 	// K8sClient defines k8s client to run subresource operations, for example you should use it to get pod logs
 	K8sClient *kubernetes.Clientset
 	// ClientsEnabled tells if the client from the package can be used
@@ -64,30 +71,58 @@ func init() {
 	}
 
 	var err error
-	Client, err = New()
+	Client, err = newClient()
 	if err != nil {
 		testlog.Info("Failed to initialize client, check the KUBECONFIG env variable", err.Error())
-		ClientsEnabled = false
+		return
+	}
+
+	ControlPlaneClient, err = NewControlPlane()
+	if err != nil {
+		testlog.Info("Failed to initialize ControlPlaneClient client, check the KUBECONFIG env variable", err.Error())
+		return
+	}
+	DataPlaneClient, err = NewDataPlane()
+	if err != nil {
+		testlog.Info("Failed to initialize DataPlaneClient client, check the KUBECONFIG env variable", err.Error())
 		return
 	}
 	K8sClient, err = NewK8s()
 	if err != nil {
 		testlog.Info("Failed to initialize k8s client, check the KUBECONFIG env variable", err.Error())
-		ClientsEnabled = false
 		return
 	}
 	ClientsEnabled = true
 }
 
-// New returns a controller-runtime client.
-func New() (client.Client, error) {
+func newClient() (client.Client, error) {
 	cfg, err := config.GetConfig()
 	if err != nil {
 		return nil, err
 	}
+	return client.New(cfg, client.Options{})
+}
+
+// NewControlPlane returns a new controller-runtime client for ControlPlaneClient cluster.
+func NewControlPlane() (client.Client, error) {
+	if hypershiftutils.IsHypershiftCluster() {
+		testlog.Info("creating ControlPlaneClient client for hypershift cluster")
+		return hypershiftutils.BuildControlPlaneClient()
+	}
+	return newClient()
+}
 
-	c, err := client.New(cfg, client.Options{})
-	return c, err
+// NewDataPlane returns a new controller-runtime client for DataPlaneClient cluster.
+func NewDataPlane() (client.Client, error) {
+	var c client.Client
+	var err error
+	if hypershiftutils.IsHypershiftCluster() {
+		testlog.Info("creating DataPlaneClient client for hypershift cluster")
+		c, err = hypershiftutils.BuildDataPlaneClient()
+	} else {
+		c, err = newClient()
+	}
+	return &dataPlaneImpl{c}, err
 }
 
 // NewK8s returns a kubernetes clientset

test/e2e/performanceprofile/functests/utils/client/dataplane.go

@@ -0,0 +1,70 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/hypershift"
+)
+
+type dataPlaneImpl struct {
+	client.Client
+}
+
+func (dpi *dataPlaneImpl) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
+	if hypershift.ObjIsEncapsulatedInConfigMap(obj) {
+		return fmt.Errorf("the provided object %s/%s might not be presented on hypershift cluster while using this client."+
+			"please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName())
+	}
+	return dpi.Client.Get(ctx, key, obj, opts...)
+}
+
+func (dpi *dataPlaneImpl) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
+	if hypershift.ObjIsEncapsulatedInConfigMap(list) {
+		return fmt.Errorf("the provided list of %s objects might not be presented on hypershift cluster while using this client."+
+			"please use ControlPlaneClient client instead", list.GetObjectKind())
+	}
+	return dpi.Client.List(ctx, list, opts...)
+}
+
+func (dpi *dataPlaneImpl) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
+	if hypershift.ObjIsEncapsulatedInConfigMap(obj) {
+		return fmt.Errorf("the provided object %s/%s might not get created on hypershift cluster while using this client."+
+			"please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName())
+	}
+	return dpi.Client.Create(ctx, obj, opts...)
+}
+
+func (dpi *dataPlaneImpl) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error {
+	if hypershift.ObjIsEncapsulatedInConfigMap(obj) {
+		return fmt.Errorf("the provided object %s/%s might not get deleted on hypershift cluster while using this client."+
+			"please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName())
+	}
+	return dpi.Client.Delete(ctx, obj, opts...)
+}
+
+func (dpi *dataPlaneImpl) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
+	if hypershift.ObjIsEncapsulatedInConfigMap(obj) {
+		return fmt.Errorf("the provided object %s/%s might not get updated on hypershift cluster while using this client."+
+			"please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName())
+	}
+	return dpi.Client.Update(ctx, obj, opts...)
+}
+
+func (dpi *dataPlaneImpl) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
+	if hypershift.ObjIsEncapsulatedInConfigMap(obj) {
+		return fmt.Errorf("the provided object %s/%s might not get patched on hypershift cluster while using this client."+
+			"please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName())
+	}
+	return dpi.Client.Patch(ctx, obj, patch, opts...)
+}
+
+func (dpi *dataPlaneImpl) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error {
+	if hypershift.ObjIsEncapsulatedInConfigMap(obj) {
+		return fmt.Errorf("the provided object %s/%s might not get deleted on hypershift cluster while using this client."+
+			"please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName())
+	}
+	return dpi.Client.DeleteAllOf(ctx, obj, opts...)
+}

test/e2e/performanceprofile/functests/utils/hypershift/hypershift.go

@@ -0,0 +1,84 @@
+package hypershift
+
+import (
+	"fmt"
+	"os"
+
+	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/hypershift"
+)
+
+var isHypershiftCluster bool
+
+func init() {
+	if v, ok := os.LookupEnv("CLUSTER_TYPE"); ok && v == "hypershift" {
+		isHypershiftCluster = true
+	}
+}
+
+const (
+	ManagementClusterKubeConfigEnv = "HYPERSHIFT_MANAGEMENT_CLUSTER_KUBECONFIG"
+	ManagementClusterNamespaceEnv  = "HYPERSHIFT_MANAGEMENT_CLUSTER_NAMESPACE"
+	HostedClusterKubeConfigEnv     = "HYPERSHIFT_HOSTED_CLUSTER_KUBECONFIG"
+	HostedClusterNameEnv           = "CLUSTER_NAME"
+	NodePoolNamespace              = "clusters"
+)
+
+func BuildControlPlaneClient() (client.Client, error) {
+	kcPath, ok := os.LookupEnv(ManagementClusterKubeConfigEnv)
+	if !ok {
+		return nil, fmt.Errorf("failed to build management-cluster client for hypershift, environment variable %q is not defined", ManagementClusterKubeConfigEnv)
+	}
+	c, err := buildClient(kcPath)
+	if err != nil {
+		return nil, err
+	}
+	ns, err := GetManagementClusterNamespace()
+	if err != nil {
+		return nil, fmt.Errorf("failed to build management-cluster client for hypershift; err %v", err)
+	}
+	return hypershift.NewControlPlaneClient(c, ns), nil
+}
+
+func BuildDataPlaneClient() (client.Client, error) {
+	kcPath, ok := os.LookupEnv(HostedClusterKubeConfigEnv)
+	if !ok {
+		return nil, fmt.Errorf("failed to build hosted-cluster client for hypershift, environment variable %q is not defined", HostedClusterKubeConfigEnv)
+	}
+	return buildClient(kcPath)
+}
+
+func GetHostedClusterName() (string, error) {
+	v, ok := os.LookupEnv(HostedClusterNameEnv)
+	if !ok {
+		return "", fmt.Errorf("failed to retrieve hosted cluster name; %q environment var is not set", HostedClusterNameEnv)
+	}
+	return v, nil
+}
+
+func GetManagementClusterNamespace() (string, error) {
+	ns, ok := os.LookupEnv(ManagementClusterNamespaceEnv)
+	if !ok {
+		return "", fmt.Errorf("failed to retrieve management cluster namespace; %q environment var is not set", ManagementClusterNamespaceEnv)
+	}
+	return ns, nil
+}
+
+// IsHypershiftCluster should be used only on CI environment
+func IsHypershiftCluster() bool {
+	return isHypershiftCluster
+}
+
+func buildClient(kubeConfigPath string) (client.Client, error) {
+	restConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
+	if err != nil {
+		return nil, err
+	}
+	c, err := client.New(restConfig, client.Options{})
+	if err != nil {
+		return nil, err
+	}
+	return c, nil
+}