Skip to content
This repository has been archived by the owner on Jul 30, 2021. It is now read-only.

Commit

Permalink
Merge pull request #433 from diegs/etcd-tls
Browse files Browse the repository at this point in the history 
Add support for generating and using etcd TLS assets.
  • Loading branch information
@diegs
diegs authored Apr 18, 2017
2 parents ada4927 + 1af5409 commit 45ef63b
Show file tree
Hide file tree
Showing 11 changed files with 268 additions and 51 deletions.
96 changes: 73 additions & 23 deletions cmd/bootkube/render.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@ import (
)

const (
apiOffset = 1
dnsOffset = 10
etcdOffset = 15
defaultServiceBaseIP = "10.3.0.0"
defaultEtcdServers = "http://127.0.0.1:2379"
apiOffset = 1
dnsOffset = 10
etcdOffset = 15
defaultServiceBaseIP = "10.3.0.0"
defaultEtcdServers = "https://127.0.0.1:2379"
defaultSelfHostedEtcdServers = "http://127.0.0.1:2379"
)

var (
Expand All @@ -36,17 +37,20 @@ var (
}

renderOpts struct {
assetDir string
caCertificatePath string
caPrivateKeyPath string
etcdServers string
apiServers string
altNames string
podCIDR string
serviceCIDR string
selfHostKubelet bool
cloudProvider string
selfHostedEtcd bool
assetDir string
caCertificatePath string
caPrivateKeyPath string
etcdCAPath string
etcdCertificatePath string
etcdPrivateKeyPath string
etcdServers string
apiServers string
altNames string
podCIDR string
serviceCIDR string
selfHostKubelet bool
cloudProvider string
selfHostedEtcd bool
}
)

Expand All @@ -55,6 +59,9 @@ func init() {
cmdRender.Flags().StringVar(&renderOpts.assetDir, "asset-dir", "", "Output path for rendered assets")
cmdRender.Flags().StringVar(&renderOpts.caCertificatePath, "ca-certificate-path", "", "Path to an existing PEM encoded CA. If provided, TLS assets will be generated using this certificate authority.")
cmdRender.Flags().StringVar(&renderOpts.caPrivateKeyPath, "ca-private-key-path", "", "Path to an existing Certificate Authority RSA private key. Required if --ca-certificate is set.")
cmdRender.Flags().StringVar(&renderOpts.etcdCAPath, "etcd-ca-path", "", "Path to an existing PEM encoded CA that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-certificate-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.")
cmdRender.Flags().StringVar(&renderOpts.etcdCertificatePath, "etcd-certificate-path", "", "Path to an existing certificate that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.")
cmdRender.Flags().StringVar(&renderOpts.etcdPrivateKeyPath, "etcd-private-key-path", "", "Path to an existing private key that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-certificate-path, and must have etcd configured to use TLS with matching secrets.")
cmdRender.Flags().StringVar(&renderOpts.etcdServers, "etcd-servers", defaultEtcdServers, "List of etcd servers URLs including host:port, comma separated")
cmdRender.Flags().StringVar(&renderOpts.apiServers, "api-servers", "https://127.0.0.1:443", "List of API server URLs including host:port, commma seprated")
cmdRender.Flags().StringVar(&renderOpts.altNames, "api-server-alt-names", "", "List of SANs to use in api-server certificate. Example: 'IP=127.0.0.1,IP=127.0.0.2,DNS=localhost'. If empty, SANs will be extracted from the --api-servers flag.")
Expand Down Expand Up @@ -86,6 +93,12 @@ func validateRenderOpts(cmd *cobra.Command, args []string) error {
if renderOpts.caPrivateKeyPath != "" && renderOpts.caCertificatePath == "" {
return errors.New("You must provide the --ca-certificate-path flag when --ca-private-key-path is provided.")
}
if (renderOpts.etcdCAPath != "" || renderOpts.etcdCertificatePath != "" || renderOpts.etcdPrivateKeyPath != "") && (renderOpts.etcdCAPath == "" || renderOpts.etcdCertificatePath == "" || renderOpts.etcdPrivateKeyPath == "") {
return errors.New("You must specify either all or none of --etcd-ca-path, --etcd-certificate-path, and --etcd-private-key-path")
}
if renderOpts.etcdCertificatePath != "" && renderOpts.selfHostedEtcd {
return errors.New("Cannot specify --etcd-certificate-path with --experimental-self-hosted-etcd")
}
if renderOpts.assetDir == "" {
return errors.New("Missing required flag: --asset-dir")
}
Expand Down Expand Up @@ -156,10 +169,8 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
if err != nil {
return nil, err
}

etcdServers = append(etcdServers, etcdServerUrl)

if renderOpts.etcdServers != defaultEtcdServers {
if renderOpts.etcdServers != defaultSelfHostedEtcdServers {
bootkube.UserOutput("--experimental-self-hosted-etcd and --service-cidr set. Overriding --etcd-servers setting with %s\n", etcdServers)
}
} else {
Expand All @@ -169,13 +180,44 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
}
}

etcdUseTLS := false
for _, url := range etcdServers {
if url.Scheme == "https" {
etcdUseTLS = true
}
}

var etcdCACert *x509.Certificate
if renderOpts.etcdCAPath != "" {
etcdCACert, err = parseCertFromDisk(renderOpts.etcdCAPath)
if err != nil {
return nil, err
}
}
var etcdClientCert *x509.Certificate
var etcdClientKey *rsa.PrivateKey
if renderOpts.etcdCertificatePath != "" {
etcdClientKey, etcdClientCert, err = parseCertAndPrivateKeyFromDisk(renderOpts.etcdCertificatePath, renderOpts.etcdPrivateKeyPath)
if err != nil {
return nil, err
}
}

if etcdUseTLS && etcdCACert == nil {
bootkube.UserOutput("NOTE: --etcd-servers=%s but -etcd-ca-path, --etcd-certificate-path, and --etcd-private-key-path were not set. Bootkube will create etcd certificates under '%s/tls'. You must configure etcd to use these certificates before invoking 'bootkube run'.\n", renderOpts.etcdServers, renderOpts.assetDir)
}

// TODO: Find better option than asking users to make manual changes
if serviceNet.IP.String() != defaultServiceBaseIP {
fmt.Printf("You have selected a non-default service CIDR %s - be sure your kubelet service file uses --cluster-dns=%s\n", serviceNet.String(), dnsServiceIP.String())
}

return &asset.Config{
EtcdCACert: etcdCACert,
EtcdClientCert: etcdClientCert,
EtcdClientKey: etcdClientKey,
EtcdServers: etcdServers,
EtcdUseTLS: etcdUseTLS,
CACert: caCert,
CAPrivKey: caPrivKey,
APIServers: apiServers,
Expand All @@ -184,7 +226,7 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
ServiceCIDR: serviceNet,
APIServiceIP: apiServiceIP,
DNSServiceIP: dnsServiceIP,
ETCDServiceIP: etcdServiceIP,
EtcdServiceIP: etcdServiceIP,
SelfHostKubelet: renderOpts.selfHostKubelet,
CloudProvider: renderOpts.cloudProvider,
SelfHostedEtcd: renderOpts.selfHostedEtcd,
Expand All @@ -202,15 +244,23 @@ func parseCertAndPrivateKeyFromDisk(caCertPath, privKeyPath string) (*rsa.Privat
return nil, nil, fmt.Errorf("unable to parse CA private key: %v", err)
}
// Parse CA Cert.
cert, err := parseCertFromDisk(caCertPath)
if err != nil {
return nil, nil, err
}
return key, cert, nil
}

func parseCertFromDisk(caCertPath string) (*x509.Certificate, error) {
capem, err := ioutil.ReadFile(caCertPath)
if err != nil {
return nil, nil, fmt.Errorf("error reading ca cert file at %s: %v", caCertPath, err)
return nil, fmt.Errorf("error reading ca cert file at %s: %v", caCertPath, err)
}
cert, err := tlsutil.ParsePEMEncodedCACert(capem)
if err != nil {
return nil, nil, fmt.Errorf("unable to parse CA Cert: %v", err)
return nil, fmt.Errorf("unable to parse CA Cert: %v", err)
}
return key, cert, nil
return cert, nil
}

func parseURLs(s string) ([]*url.URL, error) {
Expand Down
11 changes: 10 additions & 1 deletion hack/multi-node/Vagrantfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ CONTROLLER_USER_DATA_PATH = File.expand_path("./cluster/user-data-controller")
WORKER_USER_DATA_PATH = File.expand_path("./cluster/user-data-worker")
KUBECONFIG_PATH = File.expand_path("cluster/auth/kubeconfig")
CA_CERT_PATH = File.expand_path("cluster/tls/ca.crt")
ETCD_CERT_GLOB = File.expand_path("cluster/tls/etcd-*")

def etcdIP(num)
return "172.17.4.#{num+50}"
Expand All @@ -41,7 +42,7 @@ if !$self_host_etcd
$etcd_vm_memory = 512
ETCD_CLOUD_CONFIG_PATH = File.expand_path("./etcd-cloud-config.yaml")
etcdIPs = [*1..$etcd_count].map{ |i| etcdIP(i) }
initial_etcd_cluster = etcdIPs.map.with_index{ |ip, i| "e#{i+1}=http://#{ip}:2380" }.join(",")
initial_etcd_cluster = etcdIPs.map.with_index{ |ip, i| "e#{i+1}=https://#{ip}:2380" }.join(",")
end

Vagrant.configure("2") do |config|
Expand Down Expand Up @@ -109,6 +110,14 @@ Vagrant.configure("2") do |config|

etcd.vm.provision :file, source: etcd_config_file.path, destination: "/tmp/vagrantfile-user-data"
etcd.vm.provision :shell, inline: "mv /tmp/vagrantfile-user-data /var/lib/coreos-vagrant/", privileged: true

etcd.vm.provision :shell, :inline => "mkdir -p /etc/etcd/tls", :privileged => true
Dir.glob(ETCD_CERT_GLOB) do |etcd_cert_file|
etcd.vm.provision :file, :source => etcd_cert_file, :destination => "/tmp/#{File.basename(etcd_cert_file)}"
etcd.vm.provision :shell, :inline => "mv /tmp/#{File.basename(etcd_cert_file)} /etc/etcd/tls/", :privileged => true
end
etcd.vm.provision :shell, :inline => "chown -R etcd:etcd /etc/etcd", :privileged => true
etcd.vm.provision :shell, :inline => "chmod -R u=rX,g=,o= /etc/etcd", :privileged => true
end
end
end
Expand Down
3 changes: 2 additions & 1 deletion hack/multi-node/bootkube-up
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ if [ ${SELF_HOST_ETCD} = "true" ]; then
echo "WARNING: THIS IS NOT YET FULLY WORKING - merely here to make ongoing testing easier"
etcd_render_flags="--experimental-self-hosted-etcd"
else
etcd_render_flags="--etcd-servers=http://172.17.4.51:2379"
# Note: if you increase the number of etcd servers in the Vagrantfile you must also add them here.
etcd_render_flags="--etcd-servers=https://172.17.4.51:2379"
fi

# Render assets
Expand Down
16 changes: 12 additions & 4 deletions hack/multi-node/etcd-cloud-config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,16 @@ coreos:
[Service]
Environment="ETCD_IMAGE_TAG=v3.1.0"
Environment="ETCD_NAME={{ETCD_NODE_NAME}}"
Environment="ETCD_ADVERTISE_CLIENT_URLS=http://$private_ipv4:2379"
Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=http://$private_ipv4:2380"
Environment="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379"
Environment="ETCD_LISTEN_PEER_URLS=http://$private_ipv4:2380"
Environment="ETCD_ADVERTISE_CLIENT_URLS=https://$private_ipv4:2379"
Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://$private_ipv4:2380"
Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
Environment="ETCD_LISTEN_PEER_URLS=https://$private_ipv4:2380"
Environment="ETCD_INITIAL_CLUSTER={{ETCD_INITIAL_CLUSTER}}"
Environment="ETCD_SSL_DIR=/etc/etcd/tls"
Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt"
Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd-client.crt"
Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd-client.key"
Environment="ETCD_CLIENT_CERT_AUTH=true"
Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt"
Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd-peer.crt"
Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd-peer.key"
37 changes: 26 additions & 11 deletions hack/quickstart/init-master.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,28 @@ function usage() {

function configure_etcd() {
[ -f "/etc/systemd/system/etcd-member.service.d/10-etcd-member.conf" ] || {
mkdir -p /etc/etcd/tls
cp /home/core/assets/tls/etcd* /etc/etcd/tls
chown -R etcd:etcd /etc/etcd
chmod -R u=rX,g=,o= /etc/etcd
mkdir -p /etc/systemd/system/etcd-member.service.d
cat << EOF > /etc/systemd/system/etcd-member.service.d/10-etcd-member.conf
[Service]
Environment="ETCD_IMAGE_TAG=v3.1.0"
Environment="ETCD_NAME=controller"
Environment="ETCD_INITIAL_CLUSTER=controller=http://${COREOS_PRIVATE_IPV4}:2380"
Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=http://${COREOS_PRIVATE_IPV4}:2380"
Environment="ETCD_ADVERTISE_CLIENT_URLS=http://${COREOS_PRIVATE_IPV4}:2379"
Environment="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379"
Environment="ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380"
Environment="ETCD_INITIAL_CLUSTER=controller=https://${COREOS_PRIVATE_IPV4}:2380"
Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${COREOS_PRIVATE_IPV4}:2380"
Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${COREOS_PRIVATE_IPV4}:2379"
Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
Environment="ETCD_SSL_DIR=/etc/etcd/tls"
Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt"
Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd-client.crt"
Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd-client.key"
Environment="ETCD_CLIENT_CERT_AUTH=true"
Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt"
Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd-peer.crt"
Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd-peer.key"
EOF
}
}
Expand All @@ -35,26 +47,29 @@ function init_master_node() {
systemctl daemon-reload
systemctl stop update-engine; systemctl mask update-engine

etcd_render_flags=""

# Start etcd.
if [ "$SELF_HOST_ETCD" = true ] ; then
echo "WARNING: THIS IS NOT YET FULLY WORKING - merely here to make ongoing testing easier"
etcd_render_flags="--experimental-self-hosted-etcd"
else
configure_etcd
systemctl enable etcd-member; sudo systemctl start etcd-member
etcd_render_flags="--etcd-servers=https://${COREOS_PRIVATE_IPV4}:2379"
fi

# Render cluster assets
/home/core/bootkube render --asset-dir=/home/core/assets --api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443 ${etcd_render_flags}
/home/core/bootkube render --asset-dir=/home/core/assets ${etcd_render_flags} \
--api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443

# Move the local kubeconfig into expected location
chown -R core:core /home/core/assets
mkdir -p /etc/kubernetes
cp /home/core/assets/auth/kubeconfig /etc/kubernetes/
cp /home/core/assets/tls/ca.crt /etc/kubernetes/ca.crt

# Start etcd.
if [ "$SELF_HOST_ETCD" = false ] ; then
configure_etcd
systemctl enable etcd-member; sudo systemctl start etcd-member
fi

# Start the kubelet
systemctl enable kubelet; sudo systemctl start kubelet

Expand Down
9 changes: 9 additions & 0 deletions hack/single-node/Vagrantfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ NODE_IP = "172.17.4.100"
USER_DATA_PATH = File.expand_path("cluster/user-data")
KUBECONFIG_PATH = File.expand_path("cluster/auth/kubeconfig")
CA_CERT_PATH = File.expand_path("cluster/tls/ca.crt")
ETCD_CERT_GLOB = File.expand_path("cluster/tls/etcd-*")

Vagrant.configure("2") do |config|
# always use Vagrant's insecure key
Expand Down Expand Up @@ -61,4 +62,12 @@ Vagrant.configure("2") do |config|

config.vm.provision :file, :source => CA_CERT_PATH, :destination => "/tmp/ca.crt"
config.vm.provision :shell, :inline => "mv /tmp/ca.crt /etc/kubernetes/ca.crt", :privileged => true

config.vm.provision :shell, :inline => "mkdir -p /etc/etcd/tls", :privileged => true
Dir.glob(ETCD_CERT_GLOB) do |etcd_cert_file|
config.vm.provision :file, :source => etcd_cert_file, :destination => "/tmp/#{File.basename(etcd_cert_file)}"
config.vm.provision :shell, :inline => "mv /tmp/#{File.basename(etcd_cert_file)} /etc/etcd/tls/", :privileged => true
end
config.vm.provision :shell, :inline => "chown -R etcd:etcd /etc/etcd", :privileged => true
config.vm.provision :shell, :inline => "chmod -R u=rX,g=,o= /etc/etcd", :privileged => true
end
14 changes: 14 additions & 0 deletions hack/single-node/user-data-etcd.sample
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,18 @@
content: |
[Service]
Environment="ETCD_IMAGE_TAG=v3.1.0"
Environment="ETCD_NAME=default"
Environment="ETCD_INITIAL_CLUSTER=default=https://127.0.0.1:2380"
Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://127.0.0.1:2380"
Environment="ETCD_ADVERTISE_CLIENT_URLS=https://127.0.0.1:2379"
Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
Environment="ETCD_SSL_DIR=/etc/etcd/tls"
Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt"
Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd-client.crt"
Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd-client.key"
Environment="ETCD_CLIENT_CERT_AUTH=true"
Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt"
Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd-peer.crt"
Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd-peer.key"
command: start
Loading

0 comments on commit 45ef63b

Please sign in to comment.