Skip to content

Commit

Permalink
fix: update Talos machinery to 0.12, fix secrets persistence
Browse files Browse the repository at this point in the history 
There are several important changes:

* `talosVersion` now defaults to whatever current Talos version is vs.
`0.8` as it was before; we don't expect 0.8 being used to bootstrap new
clusters
* Talos machinery bumped to latest 0.12
* secrets for the configuration are now preserved directly as
machinery's `SecretsBundle` providing forward compatibility with future
machine configuration changes: fixes persistence for cluster.id/secret
for 0.12
* migration path from "legacy" secret format to new one
* general clean ups, less dependency on "init" node type, provider
should work fine in init-less flow.

Signed-off-by: Andrey Smirnov <[email protected]>
  • Loading branch information
@smira
smira committed Sep 20, 2021
1 parent f91b032 commit 755a2dd
Show file tree
Hide file tree
Showing 6 changed files with 221 additions and 174 deletions.
125 changes: 85 additions & 40 deletions controllers/secrets.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,27 +34,74 @@ func (r *TalosConfigReconciler) fetchSecret(ctx context.Context, config *bootstr
return retSecret, nil
}

func (r *TalosConfigReconciler) writeInputSecret(ctx context.Context, scope *TalosConfigScope, input *generate.Input) (*corev1.Secret, error) {
// getSecretsBundle either generates or loads existing secret.
func (r *TalosConfigReconciler) getSecretsBundle(ctx context.Context, scope *TalosConfigScope, secretName string, opts ...generate.GenOption) (*generate.SecretsBundle, error) {
var secretsBundle *generate.SecretsBundle

certMarshal, err := yaml.Marshal(input.Certs)
if err != nil {
return nil, err
}
retry:
secret, err := r.fetchSecret(ctx, scope.Config, secretName)

kubeSecretsMarshal, err := yaml.Marshal(input.Secrets)
if err != nil {
return nil, err
switch {
case err != nil && k8serrors.IsNotFound(err):
// no cluster secret yet, generate new one
secretsBundle, err = generate.NewSecretsBundle(generate.NewClock(), opts...)
if err != nil {
return nil, fmt.Errorf("error generating new secrets bundle: %w", err)
}

if err = r.writeSecretsBundleSecret(ctx, scope, secretName, secretsBundle); err != nil {
if k8serrors.IsAlreadyExists(err) {
// conflict on creation, retry loading
goto retry
}

return nil, fmt.Errorf("error writing secrets bundle: %w", err)
}
case err != nil:
return nil, fmt.Errorf("error reading secrets bundle: %w", err)
default:
// successfully loaded secret, initialize secretsBundle from it
secretsBundle = &generate.SecretsBundle{
Clock: generate.NewClock(),
}

if _, ok := secret.Data["bundle"]; ok {
// new format
if err = yaml.Unmarshal(secret.Data["bundle"], secretsBundle); err != nil {
return nil, fmt.Errorf("error unmarshaling secrets bundle: %w", err)
}
} else {
// legacy format
if err = yaml.Unmarshal(secret.Data["certs"], &secretsBundle.Certs); err != nil {
return nil, fmt.Errorf("error unmarshaling certs: %w", err)
}

if err = yaml.Unmarshal(secret.Data["kubeSecrets"], &secretsBundle.Secrets); err != nil {
return nil, fmt.Errorf("error unmarshaling secrets: %w", err)
}

if err = yaml.Unmarshal(secret.Data["trustdInfo"], &secretsBundle.TrustdInfo); err != nil {
return nil, fmt.Errorf("error unmarshaling trustd info: %w", err)
}

// not stored in legacy format, use empty values
secretsBundle.Cluster = &generate.Cluster{}
}
}

trustdInfoMarshal, err := yaml.Marshal(input.TrustdInfo)
return secretsBundle, nil
}

func (r *TalosConfigReconciler) writeSecretsBundleSecret(ctx context.Context, scope *TalosConfigScope, secretName string, secretsBundle *generate.SecretsBundle) error {
bundle, err := yaml.Marshal(secretsBundle)
if err != nil {
return nil, err
return fmt.Errorf("error marshaling secrets bundle: %w", err)
}

certSecret := &corev1.Secret{
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Namespace: scope.Config.Namespace,
Name: scope.Cluster.Name + "-talos",
Name: secretName,
Labels: map[string]string{
capiv1.ClusterLabelName: scope.Cluster.Name,
},
Expand All @@ -63,45 +110,43 @@ func (r *TalosConfigReconciler) writeInputSecret(ctx context.Context, scope *Tal
},
},
Data: map[string][]byte{
"certs": certMarshal,
"kubeSecrets": kubeSecretsMarshal,
"trustdInfo": trustdInfoMarshal,
"bundle": bundle,
},
}

err = r.Client.Create(ctx, certSecret)
if err != nil {
return nil, err
}
return certSecret, nil
return r.Client.Create(ctx, secret)
}

func (r *TalosConfigReconciler) writeK8sCASecret(ctx context.Context, scope *TalosConfigScope, certs *x509.PEMEncodedCertificateAndKey) error {
// Create ca secret only if it doesn't already exist
_, err := r.fetchSecret(ctx, scope.Config, scope.Cluster.Name+"-ca")
if k8serrors.IsNotFound(err) {
certSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Namespace: scope.Config.Namespace,
Name: scope.Cluster.Name + "-ca",
Labels: map[string]string{
capiv1.ClusterLabelName: scope.Cluster.Name,
},
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(scope.Cluster, capiv1.GroupVersion.WithKind("Cluster")),
},
if err == nil {
return nil
}

if !k8serrors.IsNotFound(err) {
return err
}

certSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Namespace: scope.Config.Namespace,
Name: scope.Cluster.Name + "-ca",
Labels: map[string]string{
capiv1.ClusterLabelName: scope.Cluster.Name,
},
Data: map[string][]byte{
"tls.crt": certs.Crt,
"tls.key": certs.Key,
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(scope.Cluster, capiv1.GroupVersion.WithKind("Cluster")),
},
}
},
Data: map[string][]byte{
"tls.crt": certs.Crt,
"tls.key": certs.Key,
},
}

err = r.Client.Create(ctx, certSecret)
if err != nil {
return err
}
} else if err != nil {
err = r.Client.Create(ctx, certSecret)
if err != nil && !k8serrors.IsAlreadyExists(err) {
return err
}

Expand Down
76 changes: 16 additions & 60 deletions controllers/talosconfig_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,10 @@ import (
"github.com/talos-systems/talos/pkg/machinery/config/configpatcher"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/generate"
configmachine "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
"github.com/talos-systems/talos/pkg/machinery/constants"
"gopkg.in/yaml.v2"
apierrors "k8s.io/apimachinery/pkg/api/errors"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
capiv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
bsutil "sigs.k8s.io/cluster-api/bootstrap/util"
Expand All @@ -48,7 +47,7 @@ const (
)

var (
defaultVersionContract = config.TalosVersion0_8
defaultVersionContract = config.TalosVersionCurrent
)

// TalosConfigReconciler reconciles a TalosConfig object
Expand Down Expand Up @@ -206,9 +205,11 @@ func (r *TalosConfigReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, rerr

var retData *TalosConfigBundle

switch config.Spec.GenerateType {
machineType, _ := machine.ParseType(config.Spec.GenerateType) //nolint:errcheck // handle errors later

switch {
// Slurp and use user-supplied configs
case "none":
case config.Spec.GenerateType == "none":
if config.Spec.Data == "" {
return ctrl.Result{}, errors.New("failed to specify config data with none generate type")
}
Expand All @@ -218,14 +219,14 @@ func (r *TalosConfigReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, rerr
}

// Generate configs on the fly
case "init", "controlplane", "join":
case machineType != machine.TypeUnknown:
retData, err = r.genConfigs(ctx, tcScope)
if err != nil {
return ctrl.Result{}, err
}

default:
return ctrl.Result{}, errors.New("unknown generate type specified")
return ctrl.Result{}, fmt.Errorf("unknown generate type specified: %q", config.Spec.GenerateType)
}

// Handle patches to the machine config if they were specified
Expand Down Expand Up @@ -319,11 +320,8 @@ func (r *TalosConfigReconciler) userConfigs(ctx context.Context, scope *TalosCon
}

// Create the secret with kubernetes certs so a kubeconfig can be generated
if userConfig.Machine().Type() == configmachine.TypeInit {
err = r.writeK8sCASecret(ctx, scope, userConfig.Cluster().CA())
if err != nil {
return retBundle, err
}
if err = r.writeK8sCASecret(ctx, scope, userConfig.Cluster().CA()); err != nil {
return retBundle, err
}

userConfigStr, err := userConfig.String()
Expand All @@ -341,12 +339,9 @@ func (r *TalosConfigReconciler) genConfigs(ctx context.Context, scope *TalosConf
retBundle := &TalosConfigBundle{}

// Determine what type of node this is
machineType := configmachine.TypeJoin
switch scope.Config.Spec.GenerateType {
case "init":
machineType = configmachine.TypeInit
case "controlplane":
machineType = configmachine.TypeControlPlane
machineType, err := machine.ParseType(scope.Config.Spec.GenerateType)
if err != nil {
machineType = machine.TypeWorker
}

// Allow user to override default kube version.
Expand Down Expand Up @@ -391,12 +386,13 @@ func (r *TalosConfigReconciler) genConfigs(ctx context.Context, scope *TalosConf

genOptions = append(genOptions, generate.WithVersionContract(versionContract))

secretBundle, err := generate.NewSecretsBundle(generate.NewClock(), genOptions...)
secretBundle, err := r.getSecretsBundle(ctx, scope, scope.Cluster.Name+"-talos", genOptions...)
if err != nil {
return retBundle, err
}

APIEndpointPort := strconv.Itoa(int(scope.Cluster.Spec.ControlPlaneEndpoint.Port))

input, err := generate.NewInput(
scope.Cluster.Name,
"https://"+scope.Cluster.Spec.ControlPlaneEndpoint.Host+":"+APIEndpointPort,
Expand All @@ -408,51 +404,11 @@ func (r *TalosConfigReconciler) genConfigs(ctx context.Context, scope *TalosConf
return retBundle, err
}

// Stash our generated input secrets so that we can reuse them for other nodes
inputSecret, err := r.fetchSecret(ctx, scope.Config, scope.Cluster.Name+"-talos")
if machineType == configmachine.TypeInit && k8serrors.IsNotFound(err) {
inputSecret, err = r.writeInputSecret(ctx, scope, input)
if err != nil {
return retBundle, err
}
} else if err != nil {
return retBundle, err
}

// Create the secret with kubernetes certs so a kubeconfig can be generated
_, err = r.fetchSecret(ctx, scope.Config, scope.Cluster.Name+"-ca")
if machineType == configmachine.TypeInit && k8serrors.IsNotFound(err) {
err = r.writeK8sCASecret(ctx, scope, input.Certs.K8s)
if err != nil {
return retBundle, err
}
} else if err != nil {
return retBundle, err
}

certs := &generate.Certs{}
kubeSecrets := &generate.Secrets{}
trustdInfo := &generate.TrustdInfo{}

err = yaml.Unmarshal(inputSecret.Data["certs"], certs)
if err != nil {
if err = r.writeK8sCASecret(ctx, scope, input.Certs.K8s); err != nil {
return retBundle, err
}

err = yaml.Unmarshal(inputSecret.Data["kubeSecrets"], kubeSecrets)
if err != nil {
return retBundle, err
}

err = yaml.Unmarshal(inputSecret.Data["trustdInfo"], trustdInfo)
if err != nil {
return retBundle, err
}

input.Certs = certs
input.Secrets = kubeSecrets
input.TrustdInfo = trustdInfo

tcString, err := genTalosConfigFile(input.ClusterName, input.Certs)
if err != nil {
return retBundle, err
Expand Down
22 changes: 11 additions & 11 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,14 @@ require (
github.com/go-logr/logr v0.1.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.7.0
github.com/talos-systems/crypto v0.3.1
github.com/talos-systems/talos/pkg/machinery v0.11.3
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71
github.com/talos-systems/crypto v0.3.2
github.com/talos-systems/talos/pkg/machinery v0.12.3-0.20210920195258-7e63e43eb399
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.17.9
k8s.io/apiextensions-apiserver v0.17.9
k8s.io/apimachinery v0.17.9
k8s.io/client-go v0.17.9
k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19
sigs.k8s.io/cluster-api v0.3.22
sigs.k8s.io/controller-runtime v0.5.14
)
Expand All @@ -28,7 +27,7 @@ require (
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/containerd/go-cni v1.0.2 // indirect
github.com/containernetworking/cni v0.8.1 // indirect
github.com/cosi-project/runtime v0.0.0-20210625174835-93ead370bf57 // indirect
github.com/cosi-project/runtime v0.0.0-20210707150857-25f235cd0682 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/drone/envsubst v1.0.3-0.20200709223903-efdb65b94e5a // indirect
Expand Down Expand Up @@ -79,21 +78,21 @@ require (
github.com/spf13/jwalterweatherman v1.0.0 // indirect
github.com/spf13/viper v1.6.2 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/talos-systems/go-blockdevice v0.2.1 // indirect
github.com/talos-systems/go-blockdevice v0.2.3 // indirect
github.com/talos-systems/net v0.3.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.16.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.18.1 // indirect
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
golang.org/x/net v0.0.0-20210525063256-abc453219eb5 // indirect
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
golang.org/x/text v0.3.6 // indirect
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
gomodules.xyz/jsonpatch/v2 v2.0.1 // indirect
google.golang.org/appengine v1.6.6 // indirect
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced // indirect
google.golang.org/grpc v1.38.0 // indirect
google.golang.org/protobuf v1.26.0 // indirect
google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f // indirect
google.golang.org/grpc v1.40.0 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/fsnotify.v1 v1.4.7 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.51.0 // indirect
Expand All @@ -104,5 +103,6 @@ require (
k8s.io/klog v1.0.0 // indirect
k8s.io/klog/v2 v2.0.0 // indirect
k8s.io/kube-openapi v0.0.0-20200410145947-bcb3869e6f29 // indirect
k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19 // indirect
sigs.k8s.io/yaml v1.2.0 // indirect
)
Loading

0 comments on commit 755a2dd

Please sign in to comment.