Skip to content
This repository has been archived by the owner on Sep 4, 2021. It is now read-only.

Commit

Permalink
(WIP) HA control plane.
Browse files Browse the repository at this point in the history
  • Loading branch information
@colhom
colhom committed Sep 8, 2016
1 parent bbe383e commit f6619bf
Show file tree
Hide file tree
Showing 7 changed files with 182 additions and 169 deletions.
55 changes: 40 additions & 15 deletions multi-node/aws/pkg/cluster/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudformation"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/elb"
"github.com/aws/aws-sdk-go/service/route53"

"github.com/coreos/coreos-kubernetes/multi-node/aws/pkg/config"
Expand All @@ -24,8 +25,8 @@ import (
var VERSION = "UNKNOWN"

type Info struct {
Name string
ControllerIP string
Name string
ControllerHost string
}

func (c *Info) String() string {
Expand All @@ -34,7 +35,7 @@ func (c *Info) String() string {
w.Init(buf, 0, 8, 0, '\t', 0)

fmt.Fprintf(w, "Cluster Name:\t%s\n", c.Name)
fmt.Fprintf(w, "Controller IP:\t%s\n", c.ControllerIP)
fmt.Fprintf(w, "Controller DNS Name:\t%s\n", c.ControllerHost)

w.Flush()
return buf.String()
Expand Down Expand Up @@ -340,21 +341,45 @@ func (c *Cluster) Update(stackBody string) (string, error) {
}

func (c *Cluster) Info() (*Info, error) {
cfSvc := cloudformation.New(c.session)
resp, err := cfSvc.DescribeStackResource(
&cloudformation.DescribeStackResourceInput{
LogicalResourceId: aws.String("EIPController"),
StackName: aws.String(c.ClusterName),
},
)
if err != nil {
errmsg := "unable to get public IP of controller instance:\n" + err.Error()
return nil, fmt.Errorf(errmsg)
var elbName string
{
cfSvc := cloudformation.New(c.session)
resp, err := cfSvc.DescribeStackResource(
&cloudformation.DescribeStackResourceInput{
LogicalResourceId: aws.String("ElbAPIServer"),
StackName: aws.String(c.ClusterName),
},
)
if err != nil {
errmsg := "unable to get public IP of controller instance:\n" + err.Error()
return nil, fmt.Errorf(errmsg)
}
elbName = *resp.StackResourceDetail.PhysicalResourceId
}

elbSvc := elb.New(c.session)

var info Info
info.ControllerIP = *resp.StackResourceDetail.PhysicalResourceId
info.Name = c.ClusterName
{
resp, err := elbSvc.DescribeLoadBalancers(&elb.DescribeLoadBalancersInput{
LoadBalancerNames: []*string{
aws.String(elbName),
},
PageSize: aws.Int64(2),
})
if err != nil {
return nil, fmt.Errorf("error describing load balancer %s: %v", elbName, err)
}
if len(resp.LoadBalancerDescriptions) == 0 {
return nil, fmt.Errorf("could not find a load balancer with name %s", elbName)
}
if len(resp.LoadBalancerDescriptions) > 1 {
return nil, fmt.Errorf("found multiple load balancers with name %s: %v", elbName, resp)
}

info.Name = c.ClusterName
info.ControllerHost = *resp.LoadBalancerDescriptions[0].DNSName
}
return &info, nil
}

Expand Down
54 changes: 3 additions & 51 deletions multi-node/aws/pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,13 @@ func newDefaultCluster() *Cluster {
ClusterName: "kubernetes",
ReleaseChannel: "stable",
VPCCIDR: "10.0.0.0/16",
ControllerIP: "10.0.0.50",
PodCIDR: "10.2.0.0/16",
ServiceCIDR: "10.3.0.0/24",
DNSServiceIP: "10.3.0.10",
K8sVer: "v1.3.6_coreos.0",
HyperkubeImageRepo: "quay.io/coreos/hyperkube",
ContainerRuntime: "docker",
ControllerCount: 1,
ControllerInstanceType: "m3.medium",
ControllerRootVolumeType: "gp2",
ControllerRootVolumeIOPS: 0,
Expand Down Expand Up @@ -116,6 +116,7 @@ type Cluster struct {
Region string `yaml:"region,omitempty"`
AvailabilityZone string `yaml:"availabilityZone,omitempty"`
ReleaseChannel string `yaml:"releaseChannel,omitempty"`
ControllerCount int `yaml:"controllerCount,omitempty"`
ControllerInstanceType string `yaml:"controllerInstanceType,omitempty"`
ControllerRootVolumeType string `yaml:"controllerRootVolumeType,omitempty"`
ControllerRootVolumeIOPS int `yaml:"controllerRootVolumeIOPS,omitempty"`
Expand All @@ -135,7 +136,6 @@ type Cluster struct {
RouteTableID string `yaml:"routeTableId,omitempty"`
VPCCIDR string `yaml:"vpcCIDR,omitempty"`
InstanceCIDR string `yaml:"instanceCIDR,omitempty"`
ControllerIP string `yaml:"controllerIP,omitempty"`
PodCIDR string `yaml:"podCIDR,omitempty"`
ServiceCIDR string `yaml:"serviceCIDR,omitempty"`
DNSServiceIP string `yaml:"dnsServiceIP,omitempty"`
Expand Down Expand Up @@ -171,8 +171,6 @@ var supportedReleaseChannels = map[string]bool{
func (c Cluster) Config() (*Config, error) {
config := Config{Cluster: c}

config.APIServers = fmt.Sprintf("http://%s:8080", c.ControllerIP)
config.SecureAPIServers = fmt.Sprintf("https://%s:443", c.ControllerIP)
config.APIServerEndpoint = fmt.Sprintf("https://%s", c.ExternalDNSName)
config.K8sNetworkPlugin = "cni"

Expand Down Expand Up @@ -368,26 +366,6 @@ func (c Cluster) stackConfig(opts StackTemplateOptions, compressUserData bool) (

stackConfig.Config.TLSConfig = compactAssets

controllerIPAddr := net.ParseIP(stackConfig.ControllerIP)
if controllerIPAddr == nil {
return nil, fmt.Errorf("invalid controllerIP: %s", stackConfig.ControllerIP)
}

controllerSubnetFound := false
for i, subnet := range stackConfig.Subnets {
_, instanceCIDR, err := net.ParseCIDR(subnet.InstanceCIDR)
if err != nil {
return nil, fmt.Errorf("invalid instanceCIDR: %v", err)
}
if instanceCIDR.Contains(controllerIPAddr) {
stackConfig.ControllerSubnetIndex = i
controllerSubnetFound = true
}
}
if !controllerSubnetFound {
return nil, fmt.Errorf("Fail-fast occurred possibly because of a bug: ControllerSubnetIndex couldn't be determined for subnets (%v) and controllerIP (%v)", stackConfig.Subnets, stackConfig.ControllerIP)
}

if stackConfig.UserDataWorker, err = execute(opts.WorkerTmplFile, stackConfig.Config, compressUserData); err != nil {
return nil, fmt.Errorf("failed to render worker cloud config: %v", err)
}
Expand Down Expand Up @@ -516,8 +494,6 @@ type Config struct {
EtcdInitialCluster string
EtcdInstances []etcdInstance

APIServers string
SecureAPIServers string
APIServerEndpoint string
AMI string

Expand Down Expand Up @@ -587,11 +563,6 @@ func (c Cluster) valid() error {
return fmt.Errorf("invalid vpcCIDR: %v", err)
}

controllerIPAddr := net.ParseIP(c.ControllerIP)
if controllerIPAddr == nil {
return fmt.Errorf("invalid controllerIP: %s", c.ControllerIP)
}

if len(c.Subnets) == 0 {
if c.AvailabilityZone == "" {
return fmt.Errorf("availabilityZone must be set")
Expand All @@ -606,12 +577,6 @@ func (c Cluster) valid() error {
c.InstanceCIDR,
)
}
if !instanceCIDR.Contains(controllerIPAddr) {
return fmt.Errorf("instanceCIDR (%s) does not contain controllerIP (%s)",
c.InstanceCIDR,
c.ControllerIP,
)
}
} else {
if c.InstanceCIDR != "" {
return fmt.Errorf("The top-level instanceCIDR(%s) must be empty when subnets are specified", c.InstanceCIDR)
Expand Down Expand Up @@ -639,22 +604,9 @@ func (c Cluster) valid() error {
}
}

controllerInstanceCidrExists := false
for _, a := range instanceCIDRs {
if a.Contains(controllerIPAddr) {
controllerInstanceCidrExists = true
}
}
if !controllerInstanceCidrExists {
return fmt.Errorf("No instanceCIDRs in Subnets (%v) contain controllerIP (%s)",
instanceCIDRs,
c.ControllerIP,
)
}

for i, a := range instanceCIDRs {
for j, b := range instanceCIDRs[i+1:] {
if i > 0 && cidrOverlap(a, b) {
if cidrOverlap(a, b) {
return fmt.Errorf("CIDR of subnet %d (%s) overlaps with CIDR of subnet %d (%s)", i, a, j, b)
}
}
Expand Down
2 changes: 1 addition & 1 deletion multi-node/aws/pkg/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -456,7 +456,7 @@ subnets:
confBody := minimalConfigYaml + conf
_, err := ClusterFromBytes([]byte(confBody))
if err == nil {
t.Errorf("expected error parsing invalid config: %s", confBody)
t.Errorf("expected error parsing invalid config:\n%s", confBody)
}
}

Expand Down
6 changes: 3 additions & 3 deletions multi-node/aws/pkg/config/templates/cloud-config-worker
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ coreos:
--volume=stage,kind=host,source=/tmp \
--mount volume=stage,target=/tmp"
ExecStart=/usr/lib/coreos/kubelet-wrapper \
--api-servers={{.SecureAPIServers}} \
--api-servers={{.APIServerEndpoint}} \
--network-plugin-dir=/etc/kubernetes/cni/net.d \
--network-plugin={{.K8sNetworkPlugin}} \
--container-runtime={{.ContainerRuntime}} \
Expand Down Expand Up @@ -215,7 +215,7 @@ write_files:
command:
- /hyperkube
- proxy
- --master=https://{{.ControllerIP}}:443
- --master={{.APIServerEndpoint}}
- --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml
securityContext:
privileged: true
Expand Down Expand Up @@ -279,7 +279,7 @@ write_files:
"hostname": "$private_ipv4",
"policy": {
"type": "k8s",
"k8s_api_root": "https://{{.ControllerIP}}:443/api/v1/",
"k8s_api_root": "{{.APIServerEndpoint}}/api/v1/",
"k8s_client_key": "/etc/kubernetes/ssl/worker-key.pem",
"k8s_client_certificate": "/etc/kubernetes/ssl/worker.pem"
}
Expand Down
3 changes: 0 additions & 3 deletions multi-node/aws/pkg/config/templates/cluster.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,9 +113,6 @@ kmsKeyArn: "{{.KMSKeyARN}}"
# - availabilityZone: us-west-1b
# instanceCIDR: "10.0.1.0/24"

# IP Address for the controller in Kubernetes subnet. When we have 2 or more subnets, the controller is placed in the first subnet and controllerIP must be included in the instanceCIDR of the first subnet. This convention will change once we have H/A controllers
# controllerIP: 10.0.0.50

# CIDR for all service IP addresses
# serviceCIDR: "10.3.0.0/24"

Expand Down
Loading

0 comments on commit f6619bf

Please sign in to comment.