Create etcd and workers in private subnets, controllers in public subnet

cmd/nodepool/up.go

@@ -42,7 +42,7 @@ func runCmdUp(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("Failed to validate user data: %v", err)
 	}
 
-	data, err := conf.RenderStackTemplate(stackTemplateOptions(), upOpts.export)
+	data, err := conf.RenderStackTemplate(stackTemplateOptions(), upOpts.prettyPrint)
 	if err != nil {
 		return fmt.Errorf("Failed to render stack template: %v", err)
 	}

config/config.go

@@ -75,7 +75,8 @@ func NewDefaultCluster() *Cluster {
 			AWSCliImageRepo:    "quay.io/coreos/awscli",
 			AWSCliTag:          "master",
 			ContainerRuntime:   "docker",
-			Subnets:            []*Subnet{},
+			Subnets:            []*model.Subnet{},
+			EIPAllocationIDs:   []string{},
 			MapPublicIPs:       true,
 			Experimental:       experimental,
 		},
@@ -170,7 +171,7 @@ func ClusterFromBytes(data []byte) (*Cluster, error) {
 
 	// For backward-compatibility
 	if len(c.Subnets) == 0 {
-		c.Subnets = []*Subnet{
+		c.Subnets = []*model.Subnet{
 			{
 				AvailabilityZone: c.AvailabilityZone,
 				InstanceCIDR:     c.InstanceCIDR,
@@ -209,14 +210,15 @@ type ComputedDeploymentSettings struct {
 // Though it is highly configurable, it's basically users' responsibility to provide `correct` values if they're going beyond the defaults.
 type DeploymentSettings struct {
 	ComputedDeploymentSettings
-	ClusterName      string `yaml:"clusterName,omitempty"`
-	KeyName          string `yaml:"keyName,omitempty"`
-	Region           string `yaml:"region,omitempty"`
-	AvailabilityZone string `yaml:"availabilityZone,omitempty"`
-	ReleaseChannel   string `yaml:"releaseChannel,omitempty"`
-	AmiId            string `yaml:"amiId,omitempty"`
-	VPCID            string `yaml:"vpcId,omitempty"`
-	RouteTableID     string `yaml:"routeTableId,omitempty"`
+	ClusterName       string `yaml:"clusterName,omitempty"`
+	KeyName           string `yaml:"keyName,omitempty"`
+	Region            string `yaml:"region,omitempty"`
+	AvailabilityZone  string `yaml:"availabilityZone,omitempty"`
+	ReleaseChannel    string `yaml:"releaseChannel,omitempty"`
+	AmiId             string `yaml:"amiId,omitempty"`
+	VPCID             string `yaml:"vpcId,omitempty"`
+	InternetGatewayID string `yaml:"internetGatewayId,omitempty"`
+	RouteTableID      string `yaml:"routeTableId,omitempty"`
 	// Required for validations like e.g. if instance cidr is contained in vpc cidr
 	VPCCIDR             string            `yaml:"vpcCIDR,omitempty"`
 	InstanceCIDR        string            `yaml:"instanceCIDR,omitempty"`
@@ -227,7 +229,8 @@ type DeploymentSettings struct {
 	ContainerRuntime    string            `yaml:"containerRuntime,omitempty"`
 	KMSKeyARN           string            `yaml:"kmsKeyArn,omitempty"`
 	StackTags           map[string]string `yaml:"stackTags,omitempty"`
-	Subnets             []*Subnet         `yaml:"subnets,omitempty"`
+	Subnets             []*model.Subnet   `yaml:"subnets,omitempty"`
+	EIPAllocationIDs    []string          `yaml:"eipAllocationIDs,omitempty"`
 	MapPublicIPs        bool              `yaml:"mapPublicIPs,omitempty"`
 	ElasticFileSystemID string            `yaml:"elasticFileSystemId,omitempty"`
 	SSHAuthorizedKeys   []string          `yaml:"sshAuthorizedKeys,omitempty"`
@@ -246,22 +249,25 @@ type WorkerSettings struct {
 	WorkerSpotPrice        string   `yaml:"workerSpotPrice,omitempty"`
 	WorkerSecurityGroupIds []string `yaml:"workerSecurityGroupIds,omitempty"`
 	WorkerTenancy          string   `yaml:"workerTenancy,omitempty"`
+	WorkerTopologyPrivate  bool     `yaml:"workerTopologyPrivate,omitempty"`
 }
 
 // Part of configuration which is specific to controller nodes
 type ControllerSettings struct {
-	model.Controller         `yaml:"controller,omitempty"`
-	ControllerCount          int    `yaml:"controllerCount,omitempty"`
-	ControllerCreateTimeout  string `yaml:"controllerCreateTimeout,omitempty"`
-	ControllerInstanceType   string `yaml:"controllerInstanceType,omitempty"`
-	ControllerRootVolumeType string `yaml:"controllerRootVolumeType,omitempty"`
-	ControllerRootVolumeIOPS int    `yaml:"controllerRootVolumeIOPS,omitempty"`
-	ControllerRootVolumeSize int    `yaml:"controllerRootVolumeSize,omitempty"`
-	ControllerTenancy        string `yaml:"controllerTenancy,omitempty"`
+	model.Controller              `yaml:"controller,omitempty"`
+	ControllerCount               int    `yaml:"controllerCount,omitempty"`
+	ControllerCreateTimeout       string `yaml:"controllerCreateTimeout,omitempty"`
+	ControllerInstanceType        string `yaml:"controllerInstanceType,omitempty"`
+	ControllerLoadBalancerPrivate string `yaml:"controllerLoadBalancerPrivate,omitempty"`
+	ControllerRootVolumeType      string `yaml:"controllerRootVolumeType,omitempty"`
+	ControllerRootVolumeIOPS      int    `yaml:"controllerRootVolumeIOPS,omitempty"`
+	ControllerRootVolumeSize      int    `yaml:"controllerRootVolumeSize,omitempty"`
+	ControllerTenancy             string `yaml:"controllerTenancy,omitempty"`
 }
 
 // Part of configuration which is specific to etcd nodes
 type EtcdSettings struct {
+	model.Etcd              `yaml:"etcd,omitempty"`
 	EtcdCount               int    `yaml:"etcdCount"`
 	EtcdInstanceType        string `yaml:"etcdInstanceType,omitempty"`
 	EtcdRootVolumeSize      int    `yaml:"etcdRootVolumeSize,omitempty"`
@@ -296,12 +302,6 @@ type Cluster struct {
 	providedEncryptService EncryptService
 }
 
-type Subnet struct {
-	AvailabilityZone  string `yaml:"availabilityZone,omitempty"`
-	InstanceCIDR      string `yaml:"instanceCIDR,omitempty"`
-	lastAllocatedAddr *net.IP
-}
-
 type Experimental struct {
 	AuditLog              AuditLog              `yaml:"auditLog"`
 	AwsEnvironment        AwsEnvironment        `yaml:"awsEnvironment"`
@@ -385,7 +385,8 @@ type WaitSignal struct {
 }
 
 const (
-	vpcLogicalName = "VPC"
+	vpcLogicalName             = "VPC"
+	internetGatewayLogicalName = "InternetGateway"
 )
 
 var supportedReleaseChannels = map[string]bool{
@@ -474,52 +475,40 @@ func (c Cluster) Config() (*Config, error) {
 		config.AMI = c.AmiId
 	}
 
-	//Set logical name constants
-	config.VPCLogicalName = vpcLogicalName
-
-	//Set reference strings
-
-	//Assume VPC does not exist, reference by logical name
-	config.VPCRef = fmt.Sprintf(`{ "Ref" : %q }`, config.VPCLogicalName)
-	if config.VPCID != "" {
-		//This means this VPC already exists, and we can reference it directly by ID
-		config.VPCRef = fmt.Sprintf("%q", config.VPCID)
-	}
-
 	config.EtcdInstances = make([]etcdInstance, config.EtcdCount)
 	var etcdEndpoints, etcdInitialCluster bytes.Buffer
 
-	// Reset lastAllocatedAddr or we'll end up returning different cluster config w/ inconsistent static private ips
-	// for each time we call this function `cluster.Config()`
-	for _, subnet := range config.Subnets {
-		subnet.lastAllocatedAddr = nil
-	}
-
+	var lastAllocatedAddr = make(map[model.Subneter]*net.IP)
 	for etcdIndex := 0; etcdIndex < config.EtcdCount; etcdIndex++ {
 
 		//Round-robbin etcd instances across all available subnets
 		subnetIndex := etcdIndex % len(config.Subnets)
-		subnet := config.Subnets[subnetIndex]
+		subnet := model.Subneter(config.Subnets[subnetIndex])
+		subnetInstanceCIDR := config.Subnets[subnetIndex].InstanceCIDR
+		if config.Etcd.TopologyPrivate() {
+			subnet = model.Subneter(config.Etcd.PrivateSubnets[subnetIndex])
+			subnetInstanceCIDR = config.Etcd.PrivateSubnets[subnetIndex].InstanceCIDR
+		}
 
-		_, subnetCIDR, err := net.ParseCIDR(subnet.InstanceCIDR)
+		_, subnetCIDR, err := net.ParseCIDR(subnetInstanceCIDR)
 		if err != nil {
-			return nil, fmt.Errorf("error parsing subnet instance cidr %s: %v", subnet.InstanceCIDR, err)
+			return nil, fmt.Errorf("error parsing subnet instance cidr %s: %v", subnetInstanceCIDR, err)
 		}
 
-		if subnet.lastAllocatedAddr == nil {
+		if lastAllocatedAddr[subnet] == nil {
 			ip := subnetCIDR.IP
 			//TODO:(chom) this is sloppy, but "soon-ish" etcd with be self-hosted so we'll leave this be
 			for i := 0; i < 3; i++ {
 				ip = netutil.IncrementIP(ip)
 			}
-			subnet.lastAllocatedAddr = &ip
+			lastAllocatedAddr[subnet] = &ip
 		}
 
-		nextAddr := netutil.IncrementIP(*subnet.lastAllocatedAddr)
-		subnet.lastAllocatedAddr = &nextAddr
+		nextAddr := netutil.IncrementIP(*lastAllocatedAddr[subnet])
+		lastAllocatedAddr[subnet] = &nextAddr
 		instance := etcdInstance{
-			IPAddress:   *subnet.lastAllocatedAddr,
-			SubnetIndex: subnetIndex,
+			IPAddress: *lastAllocatedAddr[subnet],
+			Subnet:    subnet,
 		}
 
 		//TODO(chom): validate we're not overflowing the address space
@@ -670,8 +659,8 @@ func (c Cluster) RenderStackTemplate(opts StackTemplateOptions, prettyPrint bool
 }
 
 type etcdInstance struct {
-	IPAddress   net.IP
-	SubnetIndex int
+	IPAddress net.IP
+	Subnet    model.Subneter
 }
 
 type Config struct {
@@ -683,12 +672,6 @@ type Config struct {
 
 	// Encoded TLS assets
 	TLSConfig *CompactTLSAssets
-
-	//Logical names of dynamic resources
-	VPCLogicalName string
-
-	//Reference strings for dynamic resources
-	VPCRef string
 }
 
 // CloudFormation stack name which is unique in an AWS account.
@@ -697,6 +680,30 @@ func (c Config) StackName() string {
 	return c.ClusterName
 }
 
+func (c Config) VPCLogicalName() string {
+	return vpcLogicalName
+}
+
+func (c Config) VPCRef() string {
+	if c.VPCID != "" {
+		return fmt.Sprintf("%q", c.VPCID)
+	} else {
+		return fmt.Sprintf(`{ "Ref" : %q }`, c.VPCLogicalName())
+	}
+}
+
+func (c Config) InternetGatewayLogicalName() string {
+	return internetGatewayLogicalName
+}
+
+func (c Config) InternetGatewayRef() string {
+	if c.InternetGatewayID != "" {
+		return fmt.Sprintf("%q", c.InternetGatewayID)
+	} else {
+		return fmt.Sprintf(`{ "Ref" : %q }`, c.InternetGatewayLogicalName())
+	}
+}
+
 func (c Cluster) valid() error {
 	if c.CreateRecordSet {
 		if c.HostedZone == "" && c.HostedZoneID == "" {
@@ -835,8 +842,8 @@ func (c DeploymentSettings) Valid() (*DeploymentValidationResult, error) {
 		return nil, errors.New("kmsKeyArn must be set")
 	}
 
-	if c.VPCID == "" && c.RouteTableID != "" {
-		return nil, errors.New("vpcId must be specified if routeTableId is specified")
+	if c.VPCID == "" && (c.RouteTableID != "" || c.InternetGatewayID != "") {
+		return nil, errors.New("vpcId must be specified if routeTableId or internetGatewayId are specified")
 	}
 
 	if c.Region == "" {
@@ -981,12 +988,16 @@ func (c *Cluster) AvailabilityZones() []string {
 		return []string{c.AvailabilityZone}
 	}
 
-	azs := make([]string, len(c.Subnets))
-	for i := range azs {
-		azs[i] = c.Subnets[i].AvailabilityZone
+	result := []string{}
+	seen := map[string]bool{}
+	for _, s := range c.Subnets {
+		val := s.AvailabilityZone
+		if _, ok := seen[val]; !ok {
+			result = append(result, val)
+			seen[val] = true
+		}
 	}
-
-	return azs
+	return result
 }
 
 /*

config/config_test.go

@@ -11,6 +11,8 @@ import (
 	"strings"
 	"testing"
 	"text/template"
+
+	model "github.com/coreos/kube-aws/model"
 )
 
 const minimalConfigYaml = `externalDNSName: test.staging.core-os.net
@@ -314,7 +316,7 @@ func TestMultipleSubnets(t *testing.T) {
 
 	validConfigs := []struct {
 		conf    string
-		subnets []*Subnet
+		subnets []*model.Subnet
 	}{
 		{
 			conf: `
@@ -327,7 +329,7 @@ subnets:
   - availabilityZone: ap-northeast-1c
     instanceCIDR: 10.4.4.0/24
 `,
-			subnets: []*Subnet{
+			subnets: []*model.Subnet{
 				{
 					InstanceCIDR:     "10.4.3.0/24",
 					AvailabilityZone: "ap-northeast-1a",
@@ -346,7 +348,7 @@ controllerIP: 10.4.3.50
 availabilityZone: ap-northeast-1a
 instanceCIDR: 10.4.3.0/24
 `,
-			subnets: []*Subnet{
+			subnets: []*model.Subnet{
 				{
 					AvailabilityZone: "ap-northeast-1a",
 					InstanceCIDR:     "10.4.3.0/24",
@@ -362,7 +364,7 @@ availabilityZone: ap-northeast-1a
 instanceCIDR: 10.4.3.0/24
 subnets: []
 `,
-			subnets: []*Subnet{
+			subnets: []*model.Subnet{
 				{
 					AvailabilityZone: "ap-northeast-1a",
 					InstanceCIDR:     "10.4.3.0/24",
@@ -375,7 +377,7 @@ subnets: []
 availabilityZone: "ap-northeast-1a"
 subnets: []
 `,
-			subnets: []*Subnet{
+			subnets: []*model.Subnet{
 				{
 					AvailabilityZone: "ap-northeast-1a",
 					InstanceCIDR:     "10.0.0.0/24",
@@ -387,7 +389,7 @@ subnets: []
 # Missing subnets field fall-backs to the single subnet with the default az/cidr.
 availabilityZone: "ap-northeast-1a"
 `,
-			subnets: []*Subnet{
+			subnets: []*model.Subnet{
 				{
 					AvailabilityZone: "ap-northeast-1a",
 					InstanceCIDR:     "10.0.0.0/24",
@@ -730,12 +732,12 @@ func newMinimalConfig() (*Config, error) {
 	cluster := NewDefaultCluster()
 	cluster.ExternalDNSName = "k8s.example.com"
 	cluster.Region = "us-west-1"
-	cluster.Subnets = []*Subnet{
-		&Subnet{
+	cluster.Subnets = []*model.Subnet{
+		&model.Subnet{
 			AvailabilityZone: "us-west-1a",
 			InstanceCIDR:     "10.0.0.0/24",
 		},
-		&Subnet{
+		&model.Subnet{
 			AvailabilityZone: "us-west-1b",
 			InstanceCIDR:     "10.0.1.0/24",
 		},
@@ -873,9 +875,9 @@ func TestValidateExistingVPC(t *testing.T) {
 	cluster := NewDefaultCluster()
 
 	cluster.VPCCIDR = "10.0.0.0/16"
-	cluster.Subnets = []*Subnet{
-		{"ap-northeast-1a", "10.0.1.0/24", nil},
-		{"ap-northeast-1a", "10.0.2.0/24", nil},
+	cluster.Subnets = []*model.Subnet{
+		{"ap-northeast-1a", "10.0.1.0/24", "", model.NatGateway{}},
+		{"ap-northeast-1a", "10.0.2.0/24", "", model.NatGateway{}},
 	}
 
 	for _, testCase := range validCases {
@@ -899,9 +901,9 @@ func TestValidateUserData(t *testing.T) {
 	cluster := newDefaultClusterWithDeps(&dummyEncryptService{})
 
 	cluster.Region = "us-west-1"
-	cluster.Subnets = []*Subnet{
-		{"us-west-1a", "10.0.1.0/16", nil},
-		{"us-west-1b", "10.0.2.0/16", nil},
+	cluster.Subnets = []*model.Subnet{
+		{"us-west-1a", "10.0.1.0/16", "", model.NatGateway{}},
+		{"us-west-1b", "10.0.2.0/16", "", model.NatGateway{}},
 	}
 
 	helper.WithDummyCredentials(func(dir string) {
@@ -923,9 +925,9 @@ func TestRenderStackTemplate(t *testing.T) {
 	cluster := newDefaultClusterWithDeps(&dummyEncryptService{})
 
 	cluster.Region = "us-west-1"
-	cluster.Subnets = []*Subnet{
-		{"us-west-1a", "10.0.1.0/16", nil},
-		{"us-west-1b", "10.0.2.0/16", nil},
+	cluster.Subnets = []*model.Subnet{
+		{"us-west-1a", "10.0.1.0/16", "", model.NatGateway{}},
+		{"us-west-1b", "10.0.2.0/16", "", model.NatGateway{}},
 	}
 
 	helper.WithDummyCredentials(func(dir string) {