-
Notifications
You must be signed in to change notification settings - Fork 105
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
On Hypershift we should be able to access both the management and hosted clusters. For that, we should now have two clients, each communicates with a different cluster. ControlPlaneClient - defines the API client to run CRUD operations on the control plane that will be used for testing. DataPlaneClient - defines the API client to run CRUD operations on the data plane that will be used for testing. On non-hypershift both client would point to the same the cluster. We also added Ginkgo assertions that prevent the user from using the wrong clients for the wrong objects. For example using the `DataPlaneClient` for querying the `PerformanceProfile` object is completley fine on non-hypershift cluster, but will fail badly on hypershift (since `PerformanceProfile` objects, are not presented on the hosted clusters. Signed-off-by: Talor Itzhak <[email protected]>
- Loading branch information
Showing
3 changed files
with
298 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
70 changes: 70 additions & 0 deletions
70
test/e2e/performanceprofile/functests/utils/client/dataplane.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| package client | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
|
|
||
| "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/hypershift" | ||
| ) | ||
|
|
||
| type dataPlaneImpl struct { | ||
| client.Client | ||
| } | ||
|
|
||
| func (dpi *dataPlaneImpl) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { | ||
| if hypershift.ObjIsEncapsulatedInConfigMap(obj) { | ||
| return fmt.Errorf("the provided object %s/%s might not be presented on hypershift cluster while using this client."+ | ||
| "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) | ||
| } | ||
| return dpi.Client.Get(ctx, key, obj, opts...) | ||
| } | ||
|
|
||
| func (dpi *dataPlaneImpl) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { | ||
| if hypershift.ObjIsEncapsulatedInConfigMap(list) { | ||
| return fmt.Errorf("the provided list of %s objects might not be presented on hypershift cluster while using this client."+ | ||
| "please use ControlPlaneClient client instead", list.GetObjectKind()) | ||
| } | ||
| return dpi.Client.List(ctx, list, opts...) | ||
| } | ||
|
|
||
| func (dpi *dataPlaneImpl) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { | ||
| if hypershift.ObjIsEncapsulatedInConfigMap(obj) { | ||
| return fmt.Errorf("the provided object %s/%s might not get created on hypershift cluster while using this client."+ | ||
| "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) | ||
| } | ||
| return dpi.Client.Create(ctx, obj, opts...) | ||
| } | ||
|
|
||
| func (dpi *dataPlaneImpl) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { | ||
| if hypershift.ObjIsEncapsulatedInConfigMap(obj) { | ||
| return fmt.Errorf("the provided object %s/%s might not get deleted on hypershift cluster while using this client."+ | ||
| "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) | ||
| } | ||
| return dpi.Client.Delete(ctx, obj, opts...) | ||
| } | ||
|
|
||
| func (dpi *dataPlaneImpl) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { | ||
| if hypershift.ObjIsEncapsulatedInConfigMap(obj) { | ||
| return fmt.Errorf("the provided object %s/%s might not get updated on hypershift cluster while using this client."+ | ||
| "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) | ||
| } | ||
| return dpi.Client.Update(ctx, obj, opts...) | ||
| } | ||
|
|
||
| func (dpi *dataPlaneImpl) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { | ||
| if hypershift.ObjIsEncapsulatedInConfigMap(obj) { | ||
| return fmt.Errorf("the provided object %s/%s might not get patched on hypershift cluster while using this client."+ | ||
| "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) | ||
| } | ||
| return dpi.Client.Patch(ctx, obj, patch, opts...) | ||
| } | ||
|
|
||
| func (dpi *dataPlaneImpl) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { | ||
| if hypershift.ObjIsEncapsulatedInConfigMap(obj) { | ||
| return fmt.Errorf("the provided object %s/%s might not get deleted on hypershift cluster while using this client."+ | ||
| "please use ControlPlaneClient client instead", obj.GetObjectKind(), obj.GetName()) | ||
| } | ||
| return dpi.Client.DeleteAllOf(ctx, obj, opts...) | ||
| } |
185 changes: 185 additions & 0 deletions
185
test/e2e/performanceprofile/functests/utils/hypershift/hypershift.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,185 @@ | ||
| package hypershift | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "fmt" | ||
| "os" | ||
|
|
||
| corev1 "k8s.io/api/core/v1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| serializer "k8s.io/apimachinery/pkg/runtime/serializer/json" | ||
| "k8s.io/client-go/kubernetes/scheme" | ||
| "k8s.io/client-go/tools/clientcmd" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
|
|
||
| mcov1 "github.com/openshift/api/machineconfiguration/v1" | ||
| performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2" | ||
| tunedv1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/tuned/v1" | ||
| ) | ||
|
|
||
| var isHypershiftCluster bool | ||
|
|
||
| func init() { | ||
| if v, ok := os.LookupEnv("CLUSTER_TYPE"); ok && v == "hypershift" { | ||
| isHypershiftCluster = true | ||
| } | ||
| } | ||
|
|
||
| const ( | ||
| ManagementClusterKubeConfigEnv = "HYPERSHIFT_MANAGEMENT_CLUSTER_KUBECONFIG" | ||
| ManagementClusterNamespaceEnv = "HYPERSHIFT_MANAGEMENT_CLUSTER_NAMESPACE" | ||
| HostedClusterKubeConfigEnv = "HYPERSHIFT_HOSTED_CLUSTER_KUBECONFIG" | ||
| HostedClusterNameEnv = "CLUSTER_NAME" | ||
| NodePoolNamespace = "clusters" | ||
| ) | ||
|
|
||
| // a set of keys which used to classify the encapsulated objects in the ConfigMap | ||
| const ( | ||
| TuningKey = "tuning" | ||
| ConfigKey = "config" | ||
| ) | ||
|
|
||
| type ControlPlaneClientImpl struct { | ||
| // A client with access to the management cluster | ||
| client.Client | ||
|
|
||
| // managementClusterNamespaceName is the namespace name on the management cluster | ||
| // on which the control-plane objects reside | ||
| managementClusterNamespaceName string | ||
| } | ||
|
|
||
| func (ci *ControlPlaneClientImpl) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { | ||
| if ObjIsEncapsulatedInConfigMap(obj) { | ||
| return ci.getFromConfigMap(ctx, key, obj, opts...) | ||
| } | ||
| return ci.Client.Get(ctx, key, obj, opts...) | ||
| } | ||
|
|
||
| func (ci *ControlPlaneClientImpl) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { | ||
| if ObjIsEncapsulatedInConfigMap(obj) { | ||
| return ci.createInConfigMap(ctx, obj, opts...) | ||
| } | ||
| return ci.Client.Create(ctx, obj, opts...) | ||
| } | ||
|
|
||
| func buildClient(kubeConfigPath string) (client.Client, error) { | ||
| restConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| c, err := client.New(restConfig, client.Options{}) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return c, nil | ||
| } | ||
|
|
||
| func ObjIsEncapsulatedInConfigMap(obj runtime.Object) bool { | ||
| switch obj.(type) { | ||
| case *performancev2.PerformanceProfile, *performancev2.PerformanceProfileList, | ||
| *mcov1.KubeletConfig, *mcov1.KubeletConfigList, | ||
| *mcov1.MachineConfig, *mcov1.MachineConfigList, | ||
| *tunedv1.Profile, *tunedv1.ProfileList: | ||
| return true | ||
| default: | ||
| return false | ||
| } | ||
| } | ||
|
|
||
| func BuildControlPlaneClient() (client.Client, error) { | ||
| kcPath, ok := os.LookupEnv(ManagementClusterKubeConfigEnv) | ||
| if !ok { | ||
| return nil, fmt.Errorf("failed to build management-cluster client for hypershift, environment variable %q is not defined", ManagementClusterKubeConfigEnv) | ||
| } | ||
| c, err := buildClient(kcPath) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| ns, err := GetManagementClusterNamespace() | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to build management-cluster client for hypershift; err %v", err) | ||
| } | ||
| return &ControlPlaneClientImpl{ | ||
| Client: c, | ||
| managementClusterNamespaceName: ns, | ||
| }, nil | ||
| } | ||
|
|
||
| func BuildDataPlaneClient() (client.Client, error) { | ||
| kcPath, ok := os.LookupEnv(HostedClusterKubeConfigEnv) | ||
| if !ok { | ||
| return nil, fmt.Errorf("failed to build hosted-cluster client for hypershift, environment variable %q is not defined", HostedClusterKubeConfigEnv) | ||
| } | ||
| return buildClient(kcPath) | ||
| } | ||
|
|
||
| func (ci *ControlPlaneClientImpl) getFromConfigMap(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { | ||
| cm := &corev1.ConfigMap{} | ||
| key.Namespace = ci.managementClusterNamespaceName | ||
| err := ci.Client.Get(ctx, key, cm, opts...) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| var objAsYAML string | ||
| // can't have both | ||
| if s, ok := cm.Data[TuningKey]; ok { | ||
| objAsYAML = s | ||
| } | ||
| if s, ok := cm.Data[ConfigKey]; ok { | ||
| objAsYAML = s | ||
| } | ||
| return decodeManifest([]byte(objAsYAML), scheme.Scheme, obj) | ||
| } | ||
|
|
||
| func (ci *ControlPlaneClientImpl) createInConfigMap(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { | ||
| cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: obj.GetName(), Namespace: ci.managementClusterNamespaceName}} | ||
| b, err := encodeManifest(obj, scheme.Scheme) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| cm.Data[TuningKey] = string(b) | ||
| return ci.Client.Create(ctx, cm, opts...) | ||
| } | ||
|
|
||
| func GetHostedClusterName() (string, error) { | ||
| v, ok := os.LookupEnv(HostedClusterNameEnv) | ||
| if !ok { | ||
| return "", fmt.Errorf("failed to retrieve hosted cluster name; %q environment var is not set", HostedClusterNameEnv) | ||
| } | ||
| return v, nil | ||
| } | ||
|
|
||
| func GetManagementClusterNamespace() (string, error) { | ||
| ns, ok := os.LookupEnv(ManagementClusterNamespaceEnv) | ||
| if !ok { | ||
| return "", fmt.Errorf("failed to retrieve management cluster namespace; %q environment var is not set", ManagementClusterNamespaceEnv) | ||
| } | ||
| return ns, nil | ||
| } | ||
|
|
||
| func IsHypershiftCluster() bool { | ||
| return isHypershiftCluster | ||
| } | ||
|
|
||
| func encodeManifest(obj runtime.Object, scheme *runtime.Scheme) ([]byte, error) { | ||
| yamlSerializer := serializer.NewSerializerWithOptions( | ||
| serializer.DefaultMetaFactory, scheme, scheme, | ||
| serializer.SerializerOptions{Yaml: true, Pretty: true, Strict: true}, | ||
| ) | ||
| buff := bytes.Buffer{} | ||
| err := yamlSerializer.Encode(obj, &buff) | ||
| return buff.Bytes(), err | ||
| } | ||
|
|
||
| func decodeManifest(b []byte, scheme *runtime.Scheme, obj runtime.Object) error { | ||
| yamlSerializer := serializer.NewSerializerWithOptions( | ||
| serializer.DefaultMetaFactory, scheme, scheme, | ||
| serializer.SerializerOptions{Yaml: true, Pretty: true, Strict: true}, | ||
| ) | ||
| // Get the GroupVersionKind of the object | ||
| gvk := obj.GetObjectKind().GroupVersionKind() | ||
| _, _, err := yamlSerializer.Decode(b, &gvk, obj) | ||
| return err | ||
| } |