diff --git a/pkg/performanceprofile/controller/performanceprofile/hypershift/hypershift.go b/pkg/performanceprofile/controller/performanceprofile/hypershift/hypershift.go new file mode 100644 index 0000000000..fff2d0a39a --- /dev/null +++ b/pkg/performanceprofile/controller/performanceprofile/hypershift/hypershift.go @@ -0,0 +1,115 @@ +package hypershift + +import ( + "bytes" + "context" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + serializer "k8s.io/apimachinery/pkg/runtime/serializer/json" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + + machineconfigv1 "github.com/openshift/api/machineconfiguration/v1" + performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2" + tunedv1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/tuned/v1" +) + +// a set of keys which used to classify the encapsulated objects in the ConfigMap +const ( + TuningKey = "tuning" + ConfigKey = "config" +) + +type ControlPlaneClientImpl struct { + // A client with access to the management cluster + client.Client + + // managementClusterNamespaceName is the namespace name on the management cluster + // on which the control-plane objects reside + managementClusterNamespaceName string +} + +func (ci *ControlPlaneClientImpl) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if ObjIsEncapsulatedInConfigMap(obj) { + return ci.getFromConfigMap(ctx, key, obj, opts...) + } + return ci.Client.Get(ctx, key, obj, opts...) +} + +func (ci *ControlPlaneClientImpl) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if ObjIsEncapsulatedInConfigMap(obj) { + return ci.createInConfigMap(ctx, obj, opts...) + } + return ci.Client.Create(ctx, obj, opts...) +} + +func ObjIsEncapsulatedInConfigMap(obj runtime.Object) bool { + switch obj.(type) { + case *performancev2.PerformanceProfile, *performancev2.PerformanceProfileList, + *machineconfigv1.KubeletConfig, *machineconfigv1.KubeletConfigList, + *machineconfigv1.MachineConfig, *machineconfigv1.MachineConfigList, + *tunedv1.Tuned, *tunedv1.TunedList: + return true + default: + return false + } +} + +func NewControlPlaneClient(c client.Client, ns string) *ControlPlaneClientImpl { + return &ControlPlaneClientImpl{ + Client: c, + managementClusterNamespaceName: ns, + } +} + +func (ci *ControlPlaneClientImpl) getFromConfigMap(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + cm := &corev1.ConfigMap{} + if key.Namespace == metav1.NamespaceNone { + key.Namespace = ci.managementClusterNamespaceName + } + err := ci.Client.Get(ctx, key, cm, opts...) + if err != nil { + return err + } + var objAsYAML string + // can't have both + if s, ok := cm.Data[TuningKey]; ok { + objAsYAML = s + } + if s, ok := cm.Data[ConfigKey]; ok { + objAsYAML = s + } + return DecodeManifest([]byte(objAsYAML), scheme.Scheme, obj) +} + +func (ci *ControlPlaneClientImpl) createInConfigMap(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: obj.GetName(), Namespace: ci.managementClusterNamespaceName}} + b, err := EncodeManifest(obj, scheme.Scheme) + if err != nil { + return err + } + cm.Data[TuningKey] = string(b) + return ci.Client.Create(ctx, cm, opts...) +} + +func EncodeManifest(obj runtime.Object, scheme *runtime.Scheme) ([]byte, error) { + yamlSerializer := serializer.NewSerializerWithOptions( + serializer.DefaultMetaFactory, scheme, scheme, + serializer.SerializerOptions{Yaml: true, Pretty: true, Strict: true}, + ) + buff := bytes.Buffer{} + err := yamlSerializer.Encode(obj, &buff) + return buff.Bytes(), err +} + +func DecodeManifest(b []byte, scheme *runtime.Scheme, obj runtime.Object) error { + yamlSerializer := serializer.NewSerializerWithOptions( + serializer.DefaultMetaFactory, scheme, scheme, + serializer.SerializerOptions{Yaml: true, Pretty: true, Strict: true}, + ) + // Get the GroupVersionKind of the object + gvk := obj.GetObjectKind().GroupVersionKind() + _, _, err := yamlSerializer.Decode(b, &gvk, obj) + return err +} diff --git a/test/e2e/performanceprofile/functests/utils/client/clients.go b/test/e2e/performanceprofile/functests/utils/client/clients.go index 28a80da393..48451897a6 100644 --- a/test/e2e/performanceprofile/functests/utils/client/clients.go +++ b/test/e2e/performanceprofile/functests/utils/client/clients.go @@ -21,12 +21,19 @@ import ( performancev1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v1" performancev1alpha1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v1alpha1" performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2" + hypershiftutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/hypershift" testlog "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/log" ) var ( - // Client defines the API client to run CRUD operations, that will be used for testing + // Client kept for backport compatibility until all tests will be converted Client client.Client + // ControlPlaneClient defines the API client to run CRUD operations on the control plane cluster, + //that will be used for testing + ControlPlaneClient client.Client + // DataPlaneClient defines the API client to run CRUD operations on the data plane cluster, + //that will be used for testing + DataPlaneClient client.Client // K8sClient defines k8s client to run subresource operations, for example you should use it to get pod logs K8sClient *kubernetes.Clientset // ClientsEnabled tells if the client from the package can be used @@ -64,30 +71,58 @@ func init() { } var err error - Client, err = New() + Client, err = newClient() if err != nil { testlog.Info("Failed to initialize client, check the KUBECONFIG env variable", err.Error()) - ClientsEnabled = false + return + } + + ControlPlaneClient, err = NewControlPlane() + if err != nil { + testlog.Info("Failed to initialize ControlPlaneClient client, check the KUBECONFIG env variable", err.Error()) + return + } + DataPlaneClient, err = NewDataPlane() + if err != nil { + testlog.Info("Failed to initialize DataPlaneClient client, check the KUBECONFIG env variable", err.Error()) return } K8sClient, err = NewK8s() if err != nil { testlog.Info("Failed to initialize k8s client, check the KUBECONFIG env variable", err.Error()) - ClientsEnabled = false return } ClientsEnabled = true } -// New returns a controller-runtime client. -func New() (client.Client, error) { +func newClient() (client.Client, error) { cfg, err := config.GetConfig() if err != nil { return nil, err } + return client.New(cfg, client.Options{}) +} + +// NewControlPlane returns a new controller-runtime client for ControlPlaneClient cluster. +func NewControlPlane() (client.Client, error) { + if hypershiftutils.IsHypershiftCluster() { + testlog.Info("creating ControlPlaneClient client for hypershift cluster") + return hypershiftutils.BuildControlPlaneClient() + } + return newClient() +} - c, err := client.New(cfg, client.Options{}) - return c, err +// NewDataPlane returns a new controller-runtime client for DataPlaneClient cluster. +func NewDataPlane() (client.Client, error) { + var c client.Client + var err error + if hypershiftutils.IsHypershiftCluster() { + testlog.Info("creating DataPlaneClient client for hypershift cluster") + c, err = hypershiftutils.BuildDataPlaneClient() + } else { + c, err = newClient() + } + return &dataPlaneImpl{c}, err } // NewK8s returns a kubernetes clientset diff --git a/test/e2e/performanceprofile/functests/utils/client/dataplane.go b/test/e2e/performanceprofile/functests/utils/client/dataplane.go new file mode 100644 index 0000000000..4c29931ac7 --- /dev/null +++ b/test/e2e/performanceprofile/functests/utils/client/dataplane.go @@ -0,0 +1,70 @@ +package client + +import ( + "context" + "fmt" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/hypershift" +) + +type dataPlaneImpl struct { + client.Client +} + +func (dpi *dataPlaneImpl) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if hypershift.ObjIsEncapsulatedInConfigMap(obj) { + return fmt.Errorf("the provided object %s/%s might not be presented on hypershift cluster while using this client."+ + "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) + } + return dpi.Client.Get(ctx, key, obj, opts...) +} + +func (dpi *dataPlaneImpl) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if hypershift.ObjIsEncapsulatedInConfigMap(list) { + return fmt.Errorf("the provided list of %s objects might not be presented on hypershift cluster while using this client."+ + "please use ControlPlaneClient client instead", list.GetObjectKind()) + } + return dpi.Client.List(ctx, list, opts...) +} + +func (dpi *dataPlaneImpl) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if hypershift.ObjIsEncapsulatedInConfigMap(obj) { + return fmt.Errorf("the provided object %s/%s might not get created on hypershift cluster while using this client."+ + "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) + } + return dpi.Client.Create(ctx, obj, opts...) +} + +func (dpi *dataPlaneImpl) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + if hypershift.ObjIsEncapsulatedInConfigMap(obj) { + return fmt.Errorf("the provided object %s/%s might not get deleted on hypershift cluster while using this client."+ + "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) + } + return dpi.Client.Delete(ctx, obj, opts...) +} + +func (dpi *dataPlaneImpl) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if hypershift.ObjIsEncapsulatedInConfigMap(obj) { + return fmt.Errorf("the provided object %s/%s might not get updated on hypershift cluster while using this client."+ + "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) + } + return dpi.Client.Update(ctx, obj, opts...) +} + +func (dpi *dataPlaneImpl) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if hypershift.ObjIsEncapsulatedInConfigMap(obj) { + return fmt.Errorf("the provided object %s/%s might not get patched on hypershift cluster while using this client."+ + "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) + } + return dpi.Client.Patch(ctx, obj, patch, opts...) +} + +func (dpi *dataPlaneImpl) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + if hypershift.ObjIsEncapsulatedInConfigMap(obj) { + return fmt.Errorf("the provided object %s/%s might not get deleted on hypershift cluster while using this client."+ + "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) + } + return dpi.Client.DeleteAllOf(ctx, obj, opts...) +} diff --git a/test/e2e/performanceprofile/functests/utils/hypershift/hypershift.go b/test/e2e/performanceprofile/functests/utils/hypershift/hypershift.go new file mode 100644 index 0000000000..5905883e66 --- /dev/null +++ b/test/e2e/performanceprofile/functests/utils/hypershift/hypershift.go @@ -0,0 +1,84 @@ +package hypershift + +import ( + "fmt" + "os" + + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/hypershift" +) + +var isHypershiftCluster bool + +func init() { + if v, ok := os.LookupEnv("CLUSTER_TYPE"); ok && v == "hypershift" { + isHypershiftCluster = true + } +} + +const ( + ManagementClusterKubeConfigEnv = "HYPERSHIFT_MANAGEMENT_CLUSTER_KUBECONFIG" + ManagementClusterNamespaceEnv = "HYPERSHIFT_MANAGEMENT_CLUSTER_NAMESPACE" + HostedClusterKubeConfigEnv = "HYPERSHIFT_HOSTED_CLUSTER_KUBECONFIG" + HostedClusterNameEnv = "CLUSTER_NAME" + NodePoolNamespace = "clusters" +) + +func BuildControlPlaneClient() (client.Client, error) { + kcPath, ok := os.LookupEnv(ManagementClusterKubeConfigEnv) + if !ok { + return nil, fmt.Errorf("failed to build management-cluster client for hypershift, environment variable %q is not defined", ManagementClusterKubeConfigEnv) + } + c, err := buildClient(kcPath) + if err != nil { + return nil, err + } + ns, err := GetManagementClusterNamespace() + if err != nil { + return nil, fmt.Errorf("failed to build management-cluster client for hypershift; err %v", err) + } + return hypershift.NewControlPlaneClient(c, ns), nil +} + +func BuildDataPlaneClient() (client.Client, error) { + kcPath, ok := os.LookupEnv(HostedClusterKubeConfigEnv) + if !ok { + return nil, fmt.Errorf("failed to build hosted-cluster client for hypershift, environment variable %q is not defined", HostedClusterKubeConfigEnv) + } + return buildClient(kcPath) +} + +func GetHostedClusterName() (string, error) { + v, ok := os.LookupEnv(HostedClusterNameEnv) + if !ok { + return "", fmt.Errorf("failed to retrieve hosted cluster name; %q environment var is not set", HostedClusterNameEnv) + } + return v, nil +} + +func GetManagementClusterNamespace() (string, error) { + ns, ok := os.LookupEnv(ManagementClusterNamespaceEnv) + if !ok { + return "", fmt.Errorf("failed to retrieve management cluster namespace; %q environment var is not set", ManagementClusterNamespaceEnv) + } + return ns, nil +} + +// IsHypershiftCluster should be used only on CI environment +func IsHypershiftCluster() bool { + return isHypershiftCluster +} + +func buildClient(kubeConfigPath string) (client.Client, error) { + restConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) + if err != nil { + return nil, err + } + c, err := client.New(restConfig, client.Options{}) + if err != nil { + return nil, err + } + return c, nil +}