From 3eec07bccecbb156b3decf1f1b89abaa1afb9e9c Mon Sep 17 00:00:00 2001 From: Diego Pontoriero Date: Thu, 13 Apr 2017 17:34:10 -0700 Subject: [PATCH] Add support for generating and using etcd TLS assets. This change adds rendering options to allow the apiserver to connect to etcd using TLS. This applies to both the temporary and self-hosted control plane. There are also some options (mostly intended for development) to help generate the etcd (client/server) certificates. Obviously this is only useful if etcd is not already up. Self-hosted etcd is not supported at this time. --- cmd/bootkube/render.go | 96 ++++++++++++++++++++++++++------- hack/quickstart/init-master.sh | 33 +++++++++--- pkg/asset/asset.go | 27 +++++++++- pkg/asset/internal/templates.go | 10 ++++ pkg/asset/k8s.go | 9 +++- pkg/asset/tls.go | 65 +++++++++++++++++++--- 6 files changed, 204 insertions(+), 36 deletions(-) diff --git a/cmd/bootkube/render.go b/cmd/bootkube/render.go index c449fe549..05912b1a7 100644 --- a/cmd/bootkube/render.go +++ b/cmd/bootkube/render.go @@ -18,11 +18,12 @@ import ( ) const ( - apiOffset = 1 - dnsOffset = 10 - etcdOffset = 15 - defaultServiceBaseIP = "10.3.0.0" - defaultEtcdServers = "http://127.0.0.1:2379" + apiOffset = 1 + dnsOffset = 10 + etcdOffset = 15 + defaultServiceBaseIP = "10.3.0.0" + defaultEtcdServers = "http://127.0.0.1:2379" + defaultEtcdTLSServers = "https://127.0.0.1:2379" ) var ( @@ -36,17 +37,21 @@ var ( } renderOpts struct { - assetDir string - caCertificatePath string - caPrivateKeyPath string - etcdServers string - apiServers string - altNames string - podCIDR string - serviceCIDR string - selfHostKubelet bool - cloudProvider string - selfHostedEtcd bool + assetDir string + caCertificatePath string + caPrivateKeyPath string + etcdCAPath string + etcdCertificatePath string + etcdPrivateKeyPath string + etcdServers string + etcdUseTLS bool + apiServers string + altNames string + podCIDR string + serviceCIDR string + selfHostKubelet bool + cloudProvider string + selfHostedEtcd bool } ) @@ -55,6 +60,9 @@ func init() { cmdRender.Flags().StringVar(&renderOpts.assetDir, "asset-dir", "", "Output path for rendered assets") cmdRender.Flags().StringVar(&renderOpts.caCertificatePath, "ca-certificate-path", "", "Path to an existing PEM encoded CA. If provided, TLS assets will be generated using this certificate authority.") cmdRender.Flags().StringVar(&renderOpts.caPrivateKeyPath, "ca-private-key-path", "", "Path to an existing Certificate Authority RSA private key. Required if --ca-certificate is set.") + cmdRender.Flags().StringVar(&renderOpts.etcdCAPath, "etcd-ca-path", "", "Path to an existing PEM encoded CA that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-certificate-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.") + cmdRender.Flags().StringVar(&renderOpts.etcdCertificatePath, "etcd-certificate-path", "", "Path to an existing server certificate that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.") + cmdRender.Flags().StringVar(&renderOpts.etcdPrivateKeyPath, "etcd-private-key-path", "", "Path to an existing server private key that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-certificate-path, and must have etcd configured to use TLS with matching secrets.") cmdRender.Flags().StringVar(&renderOpts.etcdServers, "etcd-servers", defaultEtcdServers, "List of etcd servers URLs including host:port, comma separated") cmdRender.Flags().StringVar(&renderOpts.apiServers, "api-servers", "https://127.0.0.1:443", "List of API server URLs including host:port, commma seprated") cmdRender.Flags().StringVar(&renderOpts.altNames, "api-server-alt-names", "", "List of SANs to use in api-server certificate. Example: 'IP=127.0.0.1,IP=127.0.0.2,DNS=localhost'. If empty, SANs will be extracted from the --api-servers flag.") @@ -63,6 +71,7 @@ func init() { cmdRender.Flags().BoolVar(&renderOpts.selfHostKubelet, "experimental-self-hosted-kubelet", false, "(Experimental) Create a self-hosted kubelet daemonset.") cmdRender.Flags().StringVar(&renderOpts.cloudProvider, "cloud-provider", "", "The provider for cloud services. Empty string for no provider") cmdRender.Flags().BoolVar(&renderOpts.selfHostedEtcd, "experimental-self-hosted-etcd", false, "(Experimental) Create self-hosted etcd assets.") + cmdRender.Flags().BoolVar(&renderOpts.etcdUseTLS, "etcd-use-tls", false, "If true, uses TLS for etcd. Implicitly true if --etcd-ca-path,--etcd-certificate-path,--etcd-private-key-path are set. If true but those flags are not set etcd TLS certificates will be generated. Not supported if --experimental-self-hosted-etcd=true.") } func runCmdRender(cmd *cobra.Command, args []string) error { @@ -86,6 +95,16 @@ func validateRenderOpts(cmd *cobra.Command, args []string) error { if renderOpts.caPrivateKeyPath != "" && renderOpts.caCertificatePath == "" { return errors.New("You must provide the --ca-certificate-path flag when --ca-private-key-path is provided.") } + if (renderOpts.etcdCAPath != "" || renderOpts.etcdCertificatePath != "" || renderOpts.etcdPrivateKeyPath != "") && (renderOpts.etcdCAPath == "" || renderOpts.etcdCertificatePath == "" || renderOpts.etcdPrivateKeyPath == "") { + return errors.New("You must specify either all or none of --etcd-ca-path, --etcd-certificate-path, and --etcd-private-key-path") + } + if renderOpts.etcdCAPath != "" && !renderOpts.etcdUseTLS { + bootkube.UserOutput("etcd TLS certificates specified. Overriding --etcd-use-tls=true\n") + renderOpts.etcdUseTLS = true + } + if renderOpts.etcdUseTLS && renderOpts.selfHostedEtcd { + return errors.New("Cannot use --etcd-use-tls with --experimental-self-hosted-etcd") + } if renderOpts.assetDir == "" { return errors.New("Missing required flag: --asset-dir") } @@ -163,19 +182,50 @@ func flagsToAssetConfig() (c *asset.Config, err error) { bootkube.UserOutput("--experimental-self-hosted-etcd and --service-cidr set. Overriding --etcd-servers setting with %s\n", etcdServers) } } else { + if renderOpts.etcdUseTLS && renderOpts.etcdServers == defaultEtcdServers { + renderOpts.etcdServers = defaultEtcdTLSServers + } etcdServers, err = parseURLs(renderOpts.etcdServers) if err != nil { return nil, err } } + if renderOpts.etcdUseTLS { + for _, url := range etcdServers { + if url.Scheme != "https" { + return nil, fmt.Errorf("--etcd-use-tls=true but insecure etcd server endpoint specified: %s\n", url) + } + } + } + + var etcdCACert *x509.Certificate + if renderOpts.etcdCAPath != "" { + etcdCACert, err = parseCertFromDisk(renderOpts.etcdCAPath) + if err != nil { + return nil, err + } + } + var etcdServerCert *x509.Certificate + var etcdServerKey *rsa.PrivateKey + if renderOpts.etcdCertificatePath != "" { + etcdServerKey, etcdServerCert, err = parseCertAndPrivateKeyFromDisk(renderOpts.etcdCertificatePath, renderOpts.etcdPrivateKeyPath) + if err != nil { + return nil, err + } + } + // TODO: Find better option than asking users to make manual changes if serviceNet.IP.String() != defaultServiceBaseIP { fmt.Printf("You have selected a non-default service CIDR %s - be sure your kubelet service file uses --cluster-dns=%s\n", serviceNet.String(), dnsServiceIP.String()) } return &asset.Config{ + EtcdCACert: etcdCACert, + EtcdServerCert: etcdServerCert, + EtcdServerKey: etcdServerKey, EtcdServers: etcdServers, + EtcdUseTLS: renderOpts.etcdUseTLS, CACert: caCert, CAPrivKey: caPrivKey, APIServers: apiServers, @@ -202,15 +252,23 @@ func parseCertAndPrivateKeyFromDisk(caCertPath, privKeyPath string) (*rsa.Privat return nil, nil, fmt.Errorf("unable to parse CA private key: %v", err) } // Parse CA Cert. + cert, err := parseCertFromDisk(caCertPath) + if err != nil { + return nil, nil, err + } + return key, cert, nil +} + +func parseCertFromDisk(caCertPath string) (*x509.Certificate, error) { capem, err := ioutil.ReadFile(caCertPath) if err != nil { - return nil, nil, fmt.Errorf("error reading ca cert file at %s: %v", caCertPath, err) + return nil, fmt.Errorf("error reading ca cert file at %s: %v", caCertPath, err) } cert, err := tlsutil.ParsePEMEncodedCACert(capem) if err != nil { - return nil, nil, fmt.Errorf("unable to parse CA Cert: %v", err) + return nil, fmt.Errorf("unable to parse CA Cert: %v", err) } - return key, cert, nil + return cert, nil } func parseURLs(s string) ([]*url.URL, error) { diff --git a/hack/quickstart/init-master.sh b/hack/quickstart/init-master.sh index 0fcd00039..ecca01d28 100755 --- a/hack/quickstart/init-master.sh +++ b/hack/quickstart/init-master.sh @@ -16,6 +16,10 @@ function usage() { function configure_etcd() { [ -f "/etc/systemd/system/etcd-member.service.d/10-etcd-member.conf" ] || { + mkdir -p /etc/etcd/tls + cp /home/core/assets/tls/etcd* /etc/etcd/tls + chown -R etcd:etcd /etc/etcd + chmod -R u=rX,g=,o= /etc/etcd mkdir -p /etc/systemd/system/etcd-member.service.d cat << EOF > /etc/systemd/system/etcd-member.service.d/10-etcd-member.conf [Service] @@ -23,10 +27,20 @@ Environment="ETCD_IMAGE_TAG=v3.1.0" Environment="ETCD_NAME=controller" Environment="ETCD_INITIAL_CLUSTER=controller=http://${COREOS_PRIVATE_IPV4}:2380" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=http://${COREOS_PRIVATE_IPV4}:2380" -Environment="ETCD_ADVERTISE_CLIENT_URLS=http://${COREOS_PRIVATE_IPV4}:2379" -Environment="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379" +Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${COREOS_PRIVATE_IPV4}:2379" +Environment="ETCD_SSL_DIR=/etc/etcd/tls" +Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379" Environment="ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380" +Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt" +Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd-server.crt" +Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd-server.key" +Environment="ETCD_CLIENT_CERT_AUTH=true" EOF + +#Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/ca.pem" +#Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/peers.pem" +#Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/peers-key.pem" +#Environment="ETCD_PEER_CLIENT_CERT_AUTH=true" } } @@ -35,19 +49,16 @@ function init_master_node() { systemctl daemon-reload systemctl stop update-engine; systemctl mask update-engine - etcd_render_flags="" - - # Start etcd. if [ "$SELF_HOST_ETCD" = true ] ; then echo "WARNING: THIS IS NOT YET FULLY WORKING - merely here to make ongoing testing easier" etcd_render_flags="--experimental-self-hosted-etcd" else - configure_etcd - systemctl enable etcd-member; sudo systemctl start etcd-member + etcd_render_flags="--etcd-use-tls --etcd-servers=https://${COREOS_PRIVATE_IPV4}:2379" fi # Render cluster assets - /home/core/bootkube render --asset-dir=/home/core/assets --api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443 ${etcd_render_flags} + /home/core/bootkube render --asset-dir=/home/core/assets ${etcd_render_flags} \ + --api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443 # Move the local kubeconfig into expected location chown -R core:core /home/core/assets @@ -55,6 +66,12 @@ function init_master_node() { cp /home/core/assets/auth/kubeconfig /etc/kubernetes/ cp /home/core/assets/tls/ca.crt /etc/kubernetes/ca.crt + # Start etcd. + if [ "$SELF_HOST_ETCD" = false ] ; then + configure_etcd + systemctl enable etcd-member; sudo systemctl start etcd-member + fi + # Start the kubelet systemctl enable kubelet; sudo systemctl start kubelet diff --git a/pkg/asset/asset.go b/pkg/asset/asset.go index 65575cb85..82a544535 100644 --- a/pkg/asset/asset.go +++ b/pkg/asset/asset.go @@ -19,6 +19,9 @@ const ( AssetPathCACert = "tls/ca.crt" AssetPathAPIServerKey = "tls/apiserver.key" AssetPathAPIServerCert = "tls/apiserver.crt" + AssetPathEtcdCA = "tls/etcd-ca.crt" + AssetPathEtcdServerCert = "tls/etcd-server.crt" + AssetPathEtcdServerKey = "tls/etcd-server.key" AssetPathServiceAccountPrivKey = "tls/service-account.key" AssetPathServiceAccountPubKey = "tls/service-account.pub" AssetPathKubeletKey = "tls/kubelet.key" @@ -55,7 +58,11 @@ const ( // AssetConfig holds all configuration needed when generating // the default set of assets. type Config struct { + EtcdCACert *x509.Certificate + EtcdServerCert *x509.Certificate + EtcdServerKey *rsa.PrivateKey EtcdServers []*url.URL + EtcdUseTLS bool APIServers []*url.URL CACert *x509.Certificate CAPrivKey *rsa.PrivateKey @@ -83,6 +90,15 @@ func NewDefaultAssets(conf Config) (Assets, error) { // Add kube-apiserver service IP conf.AltNames.IPs = append(conf.AltNames.IPs, conf.APIServiceIP) + // Create a CA if none was provided. + if conf.CACert == nil { + var err error + conf.CAPrivKey, conf.CACert, err = newCACert() + if err != nil { + return Assets{}, err + } + } + // TLS assets tlsAssets, err := newTLSAssets(conf.CACert, conf.CAPrivKey, *conf.AltNames) if err != nil { @@ -90,6 +106,15 @@ func NewDefaultAssets(conf Config) (Assets, error) { } as = append(as, tlsAssets...) + // etcd TLS assets. + if conf.EtcdUseTLS { + etcdTLSAssets, err := newEtcdTLSAssets(conf.EtcdCACert, conf.EtcdServerCert, conf.EtcdServerKey, conf.CACert, conf.CAPrivKey, conf.EtcdServers) + if err != nil { + return Assets{}, err + } + as = append(as, etcdTLSAssets...) + } + // K8S kubeconfig kubeConfig, err := newKubeConfigAsset(as, conf) if err != nil { @@ -98,7 +123,7 @@ func NewDefaultAssets(conf Config) (Assets, error) { as = append(as, kubeConfig) // K8S APIServer secret - apiSecret, err := newAPIServerSecretAsset(as) + apiSecret, err := newAPIServerSecretAsset(as, conf.EtcdUseTLS) if err != nil { return Assets{}, err } diff --git a/pkg/asset/internal/templates.go b/pkg/asset/internal/templates.go index 1dd0c3c55..764a94f72 100644 --- a/pkg/asset/internal/templates.go +++ b/pkg/asset/internal/templates.go @@ -167,6 +167,11 @@ spec: - --bind-address=0.0.0.0 - --client-ca-file=/etc/kubernetes/secrets/ca.crt - --cloud-provider={{ .CloudProvider }} +{{- if .EtcdUseTLS }} + - --etcd-cafile=/etc/kubernetes/secrets/etcd-ca.crt + - --etcd-certfile=/etc/kubernetes/secrets/etcd-server.crt + - --etcd-keyfile=/etc/kubernetes/secrets/etcd-server.key +{{- end }} - --etcd-servers={{ range $i, $e := .EtcdServers }}{{ if $i }},{{end}}{{ $e }}{{end}} - --insecure-port=8080 - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt @@ -227,6 +232,11 @@ spec: - --authorization-mode=RBAC - --bind-address=0.0.0.0 - --client-ca-file=/etc/kubernetes/secrets/ca.crt +{{- if .EtcdUseTLS }} + - --etcd-cafile=/etc/kubernetes/secrets/etcd-ca.crt + - --etcd-certfile=/etc/kubernetes/secrets/etcd-server.crt + - --etcd-keyfile=/etc/kubernetes/secrets/etcd-server.key +{{- end }} - --etcd-servers={{ range $i, $e := .EtcdServers }}{{ if $i }},{{end}}{{ $e }}{{end}}{{ if .SelfHostedEtcd }},http://127.0.0.1:12379{{end}} - --insecure-port=8080 - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt diff --git a/pkg/asset/k8s.go b/pkg/asset/k8s.go index 7605b276c..705fdfe4e 100644 --- a/pkg/asset/k8s.go +++ b/pkg/asset/k8s.go @@ -87,13 +87,20 @@ func newKubeConfigAsset(assets Assets, conf Config) (Asset, error) { }) } -func newAPIServerSecretAsset(assets Assets) (Asset, error) { +func newAPIServerSecretAsset(assets Assets, etcdUseTLS bool) (Asset, error) { secretAssets := []string{ AssetPathAPIServerKey, AssetPathAPIServerCert, AssetPathServiceAccountPubKey, AssetPathCACert, } + if etcdUseTLS { + secretAssets = append(secretAssets, []string{ + AssetPathEtcdCA, + AssetPathEtcdServerCert, + AssetPathEtcdServerKey, + }...) + } secretYAML, err := secretFromAssets(secretAPIServerName, secretNamespace, secretAssets, assets) if err != nil { diff --git a/pkg/asset/tls.go b/pkg/asset/tls.go index aa7aaf246..ebdadb179 100644 --- a/pkg/asset/tls.go +++ b/pkg/asset/tls.go @@ -3,6 +3,9 @@ package asset import ( "crypto/rsa" "crypto/x509" + "net" + "net/url" + "strings" "github.com/kubernetes-incubator/bootkube/pkg/tlsutil" ) @@ -13,13 +16,6 @@ func newTLSAssets(caCert *x509.Certificate, caPrivKey *rsa.PrivateKey, altNames err error ) - if caCert == nil { - caPrivKey, caCert, err = newCACert() - if err != nil { - return assets, err - } - } - apiKey, apiCert, err := newAPIKeyAndCert(caCert, caPrivKey, altNames) if err != nil { return assets, err @@ -118,3 +114,58 @@ func newKubeletKeyAndCert(caCert *x509.Certificate, caPrivKey *rsa.PrivateKey) ( } return key, cert, err } + +func newEtcdTLSAssets(etcdCACert, etcdServerCert *x509.Certificate, etcdServerKey *rsa.PrivateKey, caCert *x509.Certificate, caPrivKey *rsa.PrivateKey, etcdServers []*url.URL) ([]Asset, error) { + if etcdCACert == nil { + var err error + etcdServerKey, etcdServerCert, err = newEtcdServerKeyAndCert(caCert, caPrivKey, etcdServers) + if err != nil { + return nil, err + } + etcdCACert = caCert + } + + return []Asset{ + {Name: AssetPathEtcdCA, Data: tlsutil.EncodeCertificatePEM(etcdCACert)}, + {Name: AssetPathEtcdServerKey, Data: tlsutil.EncodePrivateKeyPEM(etcdServerKey)}, + {Name: AssetPathEtcdServerCert, Data: tlsutil.EncodeCertificatePEM(etcdServerCert)}, + }, nil +} + +func newEtcdServerKeyAndCert(caCert *x509.Certificate, caPrivKey *rsa.PrivateKey, etcdServers []*url.URL) (*rsa.PrivateKey, *x509.Certificate, error) { + key, err := tlsutil.NewPrivateKey() + if err != nil { + return nil, nil, err + } + var altNames tlsutil.AltNames + for _, etcdServer := range etcdServers { + hostname := stripPort(etcdServer.Host) + if ip := net.ParseIP(hostname); ip != nil { + altNames.IPs = append(altNames.IPs, ip) + } else { + altNames.DNSNames = append(altNames.DNSNames, hostname) + } + } + config := tlsutil.CertConfig{ + CommonName: "etcd-server", + Organization: []string{"etcd"}, + AltNames: altNames, + } + cert, err := tlsutil.NewSignedCertificate(config, key, caCert, caPrivKey) + if err != nil { + return nil, nil, err + } + return key, cert, err +} + +// TODO(diegs): remove this and switch to URL.Hostname() once bootkube uses Go 1.8. +func stripPort(hostport string) string { + colon := strings.IndexByte(hostport, ':') + if colon == -1 { + return hostport + } + if i := strings.IndexByte(hostport, ']'); i != -1 { + return strings.TrimPrefix(hostport[:i], "[") + } + return hostport[:colon] +}