Skip to content

Commit

Permalink
Add client-side certificate generation support
Browse files Browse the repository at this point in the history 
Clients now generate keys client-side and send CSRs. If the server is down-level and sends a cert+key instead of just responding with a cert signed with the client's public key, we use the key from the server instead.

Signed-off-by: Brad Davidson <[email protected]>
  • Loading branch information
@brandond
brandond committed Dec 16, 2024
1 parent 194e60f commit 7595534
Showing 1 changed file with 105 additions and 41 deletions.
146 changes: 105 additions & 41 deletions pkg/agent/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,12 @@ package config

import (
"bufio"
"bytes"
"context"
"crypto/rand"
cryptorand "crypto/rand"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/pem"
"fmt"
Expand Down Expand Up @@ -32,6 +35,7 @@ import (
"github.com/k3s-io/k3s/pkg/version"
"github.com/k3s-io/k3s/pkg/vpn"
"github.com/pkg/errors"
certutil "github.com/rancher/dynamiclistener/cert"
"github.com/rancher/wharfie/pkg/registries"
"github.com/rancher/wrangler/v3/pkg/slice"
"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -133,9 +137,9 @@ func Request(path string, info *clientaccess.Info, requester HTTPRequester) ([]b
return requester(u.String(), clientaccess.GetHTTPClient(info.CACerts, info.CertFile, info.KeyFile), info.Username, info.Password, info.Token())
}

func getNodeNamedCrt(nodeName string, nodeIPs []net.IP, nodePasswordFile string) HTTPRequester {
func getNodeNamedCrt(nodeName string, nodeIPs []net.IP, nodePasswordFile string, csr []byte) HTTPRequester {
return func(u string, client *http.Client, username, password, token string) ([]byte, error) {
req, err := http.NewRequest(http.MethodGet, u, nil)
req, err := http.NewRequest(http.MethodPost, u, bytes.NewReader(csr))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -238,47 +242,93 @@ func upgradeOldNodePasswordPath(oldNodePasswordFile, newNodePasswordFile string)
}
}

func getServingCert(nodeName string, nodeIPs []net.IP, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) error {
body, err := Request("/v1-"+version.Program+"/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile))
// getKubeletServingCert fills the kubelet server certificate with content returned
// from the server. We attempt to POST a CSR to the server, in hopes that it will
// sign the cert using our locally generated key. If the server does not support CSR
// signing, the key generated by the server is used instead.
func getKubeletServingCert(nodeName string, nodeIPs []net.IP, certFile, keyFile, nodePasswordFile string, info *clientaccess.Info) error {
csr, err := getCSRBytes(keyFile)
if err != nil {
return err
return errors.Wrapf(err, "failed to create certificate request %s", certFile)
}

servingCert, servingKey := splitCertKeyPEM(body)

if err := os.WriteFile(servingCertFile, servingCert, 0600); err != nil {
return errors.Wrapf(err, "failed to write node cert")
basename := filepath.Base(certFile)
body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile, csr))
if err != nil {
return err
}

if err := os.WriteFile(servingKeyFile, servingKey, 0600); err != nil {
return errors.Wrapf(err, "failed to write node key")
// Always split the response, as down-level servers may send back a cert+key
// instead of signing a new cert with our key. If the response includes a key it
// must be used instead of the one we signed the CSR with.
certBytes, keyBytes := splitCertKeyPEM(body)
if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", certFile)
}
if len(keyBytes) > 0 {
if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write key %s", keyFile)
}
}

return nil
}

func getHostFile(filename, keyFile string, info *clientaccess.Info) error {
// getHostFile fills a file with content returned from the server.
func getHostFile(filename string, info *clientaccess.Info) error {
basename := filepath.Base(filename)
fileBytes, err := info.Get("/v1-" + version.Program + "/" + basename)
if err != nil {
return err
}
if keyFile == "" {
if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", filename)
}
} else {
fileBytes, keyBytes := splitCertKeyPEM(fileBytes)
if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", filename)
}
if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", filename)
}
return nil
}

// getClientCert fills a client certificate with content returned from the server.
// We attempt to POST a CSR to the server, in hopes that it will sign the cert using
// our locally generated key. If the server does not support CSR signing, the key
// generated by the server is used instead.
func getClientCert(certFile, keyFile string, info *clientaccess.Info) error {
csr, err := getCSRBytes(keyFile)
if err != nil {
return errors.Wrapf(err, "failed to create certificate request %s", certFile)
}

basename := filepath.Base(certFile)
fileBytes, err := info.Post("/v1-"+version.Program+"/"+basename, csr)
if err != nil {
return err
}

// Always split the response, as down-level servers may send back a cert+key
// instead of signing a new cert with our key. If the response includes a key it
// must be used instead of the one we signed the CSR with.
certBytes, keyBytes := splitCertKeyPEM(fileBytes)
if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", certFile)
}
if len(keyBytes) > 0 {
if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write key %s", filename)
return errors.Wrapf(err, "failed to write key %s", keyFile)
}
}
return nil
}

func getCSRBytes(keyFile string) ([]byte, error) {
keyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile, false)
if err != nil {
return nil, err
}
key, err := certutil.ParsePrivateKeyPEM(keyBytes)
if err != nil {
return nil, err
}
return x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{}, key)
}

func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) {
for {
b, rest := pem.Decode(bytes)
Expand All @@ -297,19 +347,33 @@ func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) {
return
}

func getNodeNamedHostFile(filename, keyFile, nodeName string, nodeIPs []net.IP, nodePasswordFile string, info *clientaccess.Info) error {
basename := filepath.Base(filename)
body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile))
// getKubeletClientCert fills the kubelet client certificate with content returned
// from the server. We attempt to POST a CSR to the server, in hopes that it will
// sign the cert using our locally generated key. If the server does not support CSR
// signing, the key generated by the server is used instead.
func getKubeletClientCert(certFile, keyFile, nodeName string, nodeIPs []net.IP, nodePasswordFile string, info *clientaccess.Info) error {
csr, err := getCSRBytes(keyFile)
if err != nil {
return errors.Wrapf(err, "failed to create certificate request %s", certFile)
}

basename := filepath.Base(certFile)
body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile, csr))
if err != nil {
return err
}
fileBytes, keyBytes := splitCertKeyPEM(body)

if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", filename)
// Always split the response, as down-level servers may send back a cert+key
// instead of signing a new cert with our key. If the response includes a key it
// must be used instead of the one we signed the CSR with.
certBytes, keyBytes := splitCertKeyPEM(body)
if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", certFile)
}
if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write key %s", filename)
if len(keyBytes) > 0 {
if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write key %s", keyFile)
}
}
return nil
}
Expand Down Expand Up @@ -395,12 +459,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
}

clientCAFile := filepath.Join(envInfo.DataDir, "agent", "client-ca.crt")
if err := getHostFile(clientCAFile, "", info); err != nil {
if err := getHostFile(clientCAFile, info); err != nil {
return nil, err
}

serverCAFile := filepath.Join(envInfo.DataDir, "agent", "server-ca.crt")
if err := getHostFile(serverCAFile, "", info); err != nil {
if err := getHostFile(serverCAFile, info); err != nil {
return nil, err
}

Expand Down Expand Up @@ -494,13 +558,13 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
// that the cert will not be valid for, as they are not present in the list collected here.
nodeExternalAndInternalIPs := append(nodeIPs, nodeExternalIPs...)

// Ask the server to generate a kubelet server cert+key. These files are unique to this node.
if err := getServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil {
// Ask the server to sign our kubelet server cert.
if err := getKubeletServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil {
return nil, errors.Wrap(err, servingKubeletCert)
}

// Ask the server to genrate a kubelet client cert+key. These files are unique to this node.
if err := getNodeNamedHostFile(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil {
// Ask the server to sign our kubelet client cert.
if err := getKubeletClientCert(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil {
return nil, errors.Wrap(err, clientKubeletCert)
}

Expand All @@ -513,8 +577,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
clientKubeProxyCert := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.crt")
clientKubeProxyKey := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.key")

// Ask the server to send us its kube-proxy client cert+key. These files are not unique to this node.
if err := getHostFile(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
// Ask the server to sign our kube-proxy client cert.
if err := getClientCert(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
return nil, errors.Wrap(err, clientKubeProxyCert)
}

Expand All @@ -527,8 +591,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
clientK3sControllerCert := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.crt")
clientK3sControllerKey := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.key")

// Ask the server to send us its agent controller client cert+key. These files are not unique to this node.
if err := getHostFile(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
// Ask the server to sign our agent controller client cert.
if err := getClientCert(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
return nil, errors.Wrap(err, clientK3sControllerCert)
}

Expand Down

0 comments on commit 7595534

Please sign in to comment.