Skip to content

Commit

Permalink
chore: do not rely on ENV variables to configure CAPI client
Browse files Browse the repository at this point in the history
Implement a custom config reader which is using Viper as in CAPI client,
but creates separate `Viper` objects for each manager.

That allows running >1 cluster managers in parallel.

Signed-off-by: Artem Chernyshev <[email protected]>
  • Loading branch information
Unix4ever committed Sep 21, 2021
1 parent 9587089 commit f2a34fd
Show file tree
Hide file tree
Showing 7 changed files with 199 additions and 45 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.16

require (
github.com/spf13/cobra v1.1.3
github.com/spf13/viper v1.7.0
github.com/talos-systems/go-debug v0.2.1
github.com/talos-systems/go-retry v0.3.1
github.com/talos-systems/talos/pkg/machinery v0.11.5
Expand Down
24 changes: 21 additions & 3 deletions pkg/capi/capi.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
"sigs.k8s.io/cluster-api/cmd/clusterctl/client"
"sigs.k8s.io/cluster-api/cmd/clusterctl/client/config"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"

"github.com/talos-systems/capi-utils/pkg/capi/infrastructure"
Expand All @@ -39,6 +40,7 @@ type Manager struct {
runtimeClient runtimeclient.Client
version string
providers []infrastructure.Provider
cfg *Config

options Options
}
Expand All @@ -58,11 +60,15 @@ type Options struct {
func NewManager(ctx context.Context, options Options) (*Manager, error) {
clusterAPI := &Manager{
options: options,
cfg: newConfig(),
}

var err error
configClient, err := config.New(options.ClusterctlConfigPath, config.InjectReader(clusterAPI.cfg))
if err != nil {
return nil, err
}

clusterAPI.client, err = client.New(options.ClusterctlConfigPath)
clusterAPI.client, err = client.New("", client.InjectConfig(configClient))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -200,6 +206,7 @@ func (clusterAPI *Manager) InstallCore(ctx context.Context, kubeconfig client.Ku
WatchingNamespace: "",
LogUsageInstructions: false,
}

if _, err = clusterAPI.client.Init(coreOpts); err != nil {
return err
}
Expand Down Expand Up @@ -229,10 +236,13 @@ func (clusterAPI *Manager) InstallProvider(ctx context.Context, kubeconfig clien
if !installed {
fmt.Printf("initializing infrastructure provider %s\n", providerString)

if err = provider.PreInstall(); err != nil {
vars, err := provider.ProviderVars()
if err != nil {
return err
}

clusterAPI.patchConfig(vars)

infraOpts := client.InitOptions{
Kubeconfig: kubeconfig,
CoreProvider: "",
Expand Down Expand Up @@ -339,6 +349,14 @@ func (clusterAPI *Manager) Version() string {
return clusterAPI.version
}

func (clusterAPI *Manager) patchConfig(vars infrastructure.Variables) {
for key, value := range vars {
if value != "" {
clusterAPI.cfg.Set(key, value)
}
}
}

type ref struct {
types.NamespacedName
gvk schema.GroupVersionKind
Expand Down
2 changes: 1 addition & 1 deletion pkg/capi/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ func (cluster *Cluster) TalosClient(ctx context.Context) (*talosclient.Client, e

// Health runs the healthcheck for the cluster.
func (cluster *Cluster) Health(ctx context.Context) error {
return retry.Constant(5*time.Minute, retry.WithUnits(10*time.Second)).Retry(func() error {
return retry.Constant(5*time.Minute, retry.WithUnits(10*time.Second)).RetryWithContext(ctx, func(ctx context.Context) error {
// retry health checks as sometimes bootstrap bootkube issues break the check
return retry.ExpectedError(cluster.health(ctx))
})
Expand Down
138 changes: 138 additions & 0 deletions pkg/capi/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Package capi manages CAPI installation, provides default client for CAPI CRDs.
package capi

import (
"context"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"

"github.com/spf13/viper"
"k8s.io/client-go/util/homedir"
"sigs.k8s.io/cluster-api/cmd/clusterctl/client/config"
)

// Config custom implementation of config reader for clusterctl.
type Config struct {
config *viper.Viper
configPaths []string
}

func newConfig() *Config {
c := viper.New()

replacer := strings.NewReplacer("-", "_")

c.SetEnvKeyReplacer(replacer)
c.AllowEmptyEnv(true)
c.AutomaticEnv()

return &Config{
config: c,
configPaths: []string{filepath.Join(homedir.HomeDir(), config.ConfigFolder)},
}
}

// Init initializes the config.
func (c *Config) Init(path string) error {
if path != "" {
url, err := url.Parse(path)
if err != nil {
return fmt.Errorf("failed to url parse the config path %w", err)
}

switch {
case url.Scheme == "https" || url.Scheme == "http":
client := &http.Client{
Timeout: 30 * time.Second,
}

request, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url.String(), nil)
if err != nil {
return err
}

resp, err := client.Do(request)
if err != nil {
return fmt.Errorf("failed to download the clusterctl config file from %s %w", url, err)
}

if resp.StatusCode != http.StatusOK {
return fmt.Errorf("failed to download the clusterctl config file from %s got %d", url, resp.StatusCode)
}

defer io.Copy(ioutil.Discard, resp.Body) //nolint:errcheck
defer resp.Body.Close() //nolint:errcheck

if err = c.config.ReadConfig(resp.Body); err != nil {
return err
}
default:
if _, err := os.Stat(path); err != nil {
return fmt.Errorf("failed to check if clusterctl config file exists %w", err)
}

c.config.SetConfigFile(path)
}
} else {
// Checks if there is a default .cluster-api/clusterctl{.extension} file in home directory
if !c.checkDefaultConfig() {
return nil
}

// Configure viper for reading .cluster-api/clusterctl{.extension} in home directory
c.config.SetConfigName(config.ConfigName)
for _, p := range c.configPaths {
c.config.AddConfigPath(p)
}
}

return c.config.ReadInConfig()
}

// Get implements config.Reader.
func (c *Config) Get(key string) (string, error) {
if c.config.Get(key) == nil {
return "", fmt.Errorf("failed to get value for variable %q. Please set the variable value using os env variables or using the .clusterctl config file", key)
}

return c.config.GetString(key), nil
}

// Set implements config.Reader.
func (c *Config) Set(key, value string) {
c.config.Set(key, value)
}

// UnmarshalKey implements config.Reader.
func (c *Config) UnmarshalKey(key string, rawval interface{}) error {
return c.config.UnmarshalKey(key, rawval)
}

// checkDefaultConfig checks the existence of the default config.
// Returns true if it finds a supported config file in the available config
// folders.
func (c *Config) checkDefaultConfig() bool {
for _, path := range c.configPaths {
for _, ext := range viper.SupportedExts {
f := filepath.Join(path, fmt.Sprintf("%s.%s", config.ConfigName, ext))

_, err := os.Stat(f)
if err == nil {
return true
}
}
}

return false
}
19 changes: 10 additions & 9 deletions pkg/capi/deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -202,19 +202,13 @@ func (clusterAPI *Manager) DeployCluster(ctx context.Context, clusterName string
}

// set up env variables common for all providers
vars := map[string]string{
clusterAPI.patchConfig(infrastructure.Variables{
"TALOS_VERSION": options.TalosVersion,
"KUBERNETES_VERSION": options.KubernetesVersion,
"CLUSTER_NAME": options.ClusterName,
"CONTROL_PLANE_MACHINE_COUNT": strconv.FormatInt(options.ControlPlaneNodes, 10),
"WORKER_MACHINE_COUNT": strconv.FormatInt(options.WorkerNodes, 10),
}

for key, value := range vars {
if err := os.Setenv(key, value); err != nil {
return nil, err
}
}
})

templateOptions := client.GetClusterTemplateOptions{
Kubeconfig: clusterAPI.kubeconfig,
Expand All @@ -240,7 +234,14 @@ func (clusterAPI *Manager) DeployCluster(ctx context.Context, clusterName string
defer file.Close() //nolint:errcheck
}

template, err := provider.GetClusterTemplate(clusterAPI.client, templateOptions, options.providerOptions)
vars, err := provider.ClusterVars(options.providerOptions)
if err != nil {
return nil, err
}

clusterAPI.patchConfig(vars)

template, err := provider.GetClusterTemplate(clusterAPI.client, templateOptions)
if err != nil {
return nil, err
}
Expand Down
52 changes: 22 additions & 30 deletions pkg/capi/infrastructure/aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"context"
_ "embed"
"fmt"
"os"
"strconv"
"time"

Expand Down Expand Up @@ -123,22 +122,12 @@ func (s *AWSProvider) Version() string {
return s.ProviderVersion
}

// PreInstall implements Provider interface.
func (s *AWSProvider) PreInstall() error {
vars := map[string]string{
"AWS_B64ENCODED_CREDENTIALS": s.B64EncodedCredentials,
}
// ProviderVars returns config overrides for the provider installation.
func (s *AWSProvider) ProviderVars() (Variables, error) {
vars := make(Variables)
vars["AWS_B64ENCODED_CREDENTIALS"] = s.B64EncodedCredentials

for key, value := range vars {
if value != "" {
err := os.Setenv(key, value)
if err != nil {
return err
}
}
}

return nil
return vars, nil
}

// IsInstalled implements Provider interface.
Expand All @@ -152,25 +141,32 @@ func (s *AWSProvider) IsInstalled(ctx context.Context, clientset *kubernetes.Cli
return false, err
}

if _, err := clientset.AppsV1().Deployments(s.Namespace()).Get("capa-controller-master", metav1.GetOptions{}); err != nil {
if errors.IsNotFound(err) {
return false, nil
}

return false, err
}

return true, nil
}

// GetClusterTemplate implements Provider interface.
func (s *AWSProvider) GetClusterTemplate(client client.Client, opts client.GetClusterTemplateOptions, providerOptions interface{}) (client.Template, error) {
// ClusterVars returns config overrides for template generation.
func (s *AWSProvider) ClusterVars(opts interface{}) (Variables, error) {
var (
deployOptions = NewAWSDeployOptions()
ok bool
)

if providerOptions != nil {
deployOptions, ok = providerOptions.(*AWSDeployOptions)
if opts != nil {
deployOptions, ok = opts.(*AWSDeployOptions)
if !ok {
return nil, fmt.Errorf("AWS deployment provider expects aws.DeployOptions as the deployment options")
}
}

// TODO: all these settings should probably go through the config instead of using env variables.
vars := map[string]string{
vars := Variables{
"AWS_REGION": deployOptions.Region,
"AWS_VPC_ID": deployOptions.VPCID,
"AWS_SUBNET": deployOptions.Subnet,
Expand All @@ -188,15 +184,11 @@ func (s *AWSProvider) GetClusterTemplate(client client.Client, opts client.GetCl
"AWS_CLOUD_PROVIDER_VERSION": deployOptions.CloudProviderVersion,
}

for key, value := range vars {
if value != "" {
err := os.Setenv(key, value)
if err != nil {
return nil, err
}
}
}
return vars, nil
}

// GetClusterTemplate implements Provider interface.
func (s *AWSProvider) GetClusterTemplate(client client.Client, opts client.GetClusterTemplateOptions) (client.Template, error) {
return client.GetClusterTemplate(opts)
}

Expand Down
8 changes: 6 additions & 2 deletions pkg/capi/infrastructure/infrastructure.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,20 @@ import (
"github.com/talos-systems/capi-utils/pkg/constants"
)

// Variables is a map of key value pairs of config parameters.
type Variables map[string]string

// Provider defines an interface for the infrastructure provider.
type Provider interface {
Name() string
Namespace() string
Version() string
WatchingNamespace() string
Configure(interface{}) error
PreInstall() error
ProviderVars() (Variables, error)
ClusterVars(interface{}) (Variables, error)
IsInstalled(ctx context.Context, clientset *kubernetes.Clientset) (bool, error)
GetClusterTemplate(client.Client, client.GetClusterTemplateOptions, interface{}) (client.Template, error)
GetClusterTemplate(client.Client, client.GetClusterTemplateOptions) (client.Template, error)
WaitReady(context.Context, *kubernetes.Clientset) error
}

Expand Down

0 comments on commit f2a34fd

Please sign in to comment.