Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify k8s client creation #179

Merged
merged 1 commit into from
Mar 22, 2022
Merged

Simplify k8s client creation #179

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 8 additions & 53 deletions apiserver/pkg/client/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,13 @@
package client

import (
"path/filepath"
"time"

"github.com/pkg/errors"
"github.com/ray-project/kuberay/apiserver/pkg/util"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"

"github.com/cenkalti/backoff"
"github.com/golang/glog"
"github.com/ray-project/kuberay/apiserver/pkg/util"
rayclusterclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned"
rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/typed/raycluster/v1alpha1"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/client/config"
)

type ClusterClientInterface interface {
Expand All @@ -30,51 +23,13 @@ func (cc RayClusterClient) RayClusterClient(namespace string) rayiov1alpha1.RayC
}

func NewRayClusterClientOrFatal(initConnectionTimeout time.Duration, options util.ClientOptions) ClusterClientInterface {
var rayClusterClient rayiov1alpha1.RayV1alpha1Interface
var err error
var operation = func() error {
// in cluster
restConfig, err := rest.InClusterConfig()
if err != nil {
return errors.Wrap(err, "Failed to initialize RestConfig.")
}
restConfig.QPS = options.QPS
restConfig.Burst = options.Burst

rayClusterClient = rayclusterclient.NewForConfigOrDie(restConfig).RayV1alpha1()
return nil
}

// out of cluster
rayClusterClient, err = newOutOfClusterRayClusterClient()
if err == nil {
return &RayClusterClient{client: rayClusterClient}
}
klog.Infof("(Expected when in cluster) Failed to create RayCluster client by out of cluster kubeconfig. Error: %v", err)

klog.Infof("Starting to create RayCluster client by in cluster config.")
b := backoff.NewExponentialBackOff()
b.MaxElapsedTime = initConnectionTimeout
err = backoff.Retry(operation, b)

cfg, err := config.GetConfig()
if err != nil {
glog.Fatalf("Failed to create TokenReview client. Error: %v", err)
}
return &RayClusterClient{client: rayClusterClient}
}

func newOutOfClusterRayClusterClient() (rayiov1alpha1.RayV1alpha1Interface, error) {
home := homedir.HomeDir()
if home == "" {
return nil, errors.New("Cannot get home dir")
glog.Fatalf("Failed to create RayCluster client. Error: %v", err)
}
cfg.QPS = options.QPS
cfg.Burst = options.Burst

defaultKubeConfigPath := filepath.Join(home, ".kube", "config")
// use the current context in kubeconfig
config, err := clientcmd.BuildConfigFromFlags("", defaultKubeConfigPath)
if err != nil {
return nil, err
}
rayClusterClient := rayclusterclient.NewForConfigOrDie(config).RayV1alpha1()
return rayClusterClient, nil
rayClusterClient := rayclusterclient.NewForConfigOrDie(cfg).RayV1alpha1()
return &RayClusterClient{client: rayClusterClient}
}
72 changes: 9 additions & 63 deletions apiserver/pkg/client/kubernetes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,15 @@
package client

import (
"path/filepath"
"time"

"k8s.io/client-go/rest"
"github.com/golang/glog"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/client/config"

"github.com/cenkalti/backoff"
"github.com/pkg/errors"
"github.com/ray-project/kuberay/apiserver/pkg/util"
"k8s.io/client-go/kubernetes"
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
)

type KubernetesClientInterface interface {
Expand All @@ -35,66 +31,16 @@ func (c *KubernetesClient) ConfigMapClient(namespace string) v1.ConfigMapInterfa

// CreateKubernetesCoreOrFatal creates a new client for the Kubernetes pod.
func CreateKubernetesCoreOrFatal(initConnectionTimeout time.Duration, options util.ClientOptions) KubernetesClientInterface {
var client KubernetesClientInterface
var err error
var operation = func() error {
// In cluster
kubernetesClient, err := newInClusterKubernetesClient(options)
if err != nil {
return err
}
client = kubernetesClient
return nil
}

// Out of cluster
kubernetesClient, err := newOutOfClusterKubernetesClient()
if err == nil {
return kubernetesClient
}
klog.Infof("(Expected when in cluster) Failed to create Kubernetes client by out of cluster kubeconfig. Error: %v", err)

klog.Infof("Starting to create Kubernetes client by in cluster config.")
b := backoff.NewExponentialBackOff()
b.MaxElapsedTime = initConnectionTimeout
if err := backoff.Retry(operation, b); err != nil {
klog.Fatalf("Failed to create pod client. Error: %v", err)
}

return client
}

func newInClusterKubernetesClient(options util.ClientOptions) (*KubernetesClient, error) {
restConfig, err := rest.InClusterConfig()
cfg, err := config.GetConfig()
if err != nil {
return nil, errors.Wrap(err, "Failed to initialize kubernetes client.")
glog.Fatalf("Failed to create TokenReview client. Error: %v", err)
Copy link
Collaborator

@Jeffwan Jeffwan Mar 22, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The precedence changes a little bit. config.GetConfig() checks --kubeconfig first and then in-cluster config. I can not remember why I choose to check in-cluster first and then external. The change should work for us.

Config precedence after the change. https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/client/config#GetConfig

* --kubeconfig flag pointing at a file
* KUBECONFIG environment variable pointing at a file
* In-cluster config if running in cluster
* $HOME/.kube/config if exists.

}
restConfig.QPS = options.QPS
restConfig.Burst = options.Burst
cfg.QPS = options.QPS
cfg.Burst = options.Burst

clientSet, err := kubernetes.NewForConfig(restConfig)
clientSet, err := kubernetes.NewForConfig(cfg)
if err != nil {
return nil, errors.Wrap(err, "Failed to initialize kubernetes client set.")
}
return &KubernetesClient{clientSet.CoreV1()}, nil
}

func newOutOfClusterKubernetesClient() (*KubernetesClient, error) {
home := homedir.HomeDir()
if home == "" {
return nil, errors.New("Cannot get home dir")
}

defaultKubeConfigPath := filepath.Join(home, ".kube", "config")
// use the current context in kubeconfig
config, err := clientcmd.BuildConfigFromFlags("", defaultKubeConfigPath)
if err != nil {
return nil, err
}

clientSet, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, errors.Wrap(err, "Failed to initialize kubernetes clientSet set.")
klog.Fatalf("Failed to create pod client. Error: %v", err)
}
return &KubernetesClient{clientSet.CoreV1()}, nil
return &KubernetesClient{clientSet.CoreV1()}
}