Merge pull request #438 from weaveworks/shared-nodegroup-sg

Shared nodegroup SG

pkg/apis/eksctl.io/v1alpha3/types.go

@@ -256,11 +256,12 @@ func (c *ClusterConfig) AppendAvailabilityZone(newAZ string) {
 // it returns pointer to the nodegroup for convenience
 func (c *ClusterConfig) NewNodeGroup() *NodeGroup {
 	ng := &NodeGroup{
-		PrivateNetworking: false,
-		DesiredCapacity:   DefaultNodeCount,
-		InstanceType:      DefaultNodeType,
-		VolumeSize:        0,
-		VolumeType:        DefaultNodeVolumeType,
+		PrivateNetworking:   false,
+		SharedSecurityGroup: true,
+		DesiredCapacity:     DefaultNodeCount,
+		InstanceType:        DefaultNodeType,
+		VolumeSize:          0,
+		VolumeType:          DefaultNodeVolumeType,
 	}
 
 	c.NodeGroups = append(c.NodeGroups, ng)
@@ -281,6 +282,8 @@ type NodeGroup struct {
 	// +optional
 	AvailabilityZones []string `json:"availabilityZones,omitempty"`
 	// +optional
+	SharedSecurityGroup bool `json:"sharedSecurityGroup,omitempty"`
+	// +optional
 	SecurityGroups []string `json:"securityGroups,omitempty"`
 	// +optional
 	Tags map[string]string `json:"tags,omitempty"`

pkg/apis/eksctl.io/v1alpha3/vpc.go

@@ -22,6 +22,8 @@ type (
 		// private subnets or any ad-hoc subnets
 		// +optional
 		ExtraCIDRs []*ipnet.IPNet `json:"extraCIDRs,omitempty"`
+		// for pre-defined shared node SG
+		SharedNodeSecurityGroup string `json:"sharedNodeSecurityGroup,omitempty"`
 	}
 	// SubnetTopology can be SubnetTopologyPrivate or SubnetTopologyPublic
 	SubnetTopology string

pkg/cfn/builder/api_test.go

@@ -24,7 +24,7 @@ import (
 )
 
 const (
-	totalNodeResources = 13
+	totalNodeResources = 10
 	clusterName        = "ferocious-mushroom-1532594698"
 	endpoint           = "https://DE37D8AFB23F7275D2361AD6B2599143.yl4.us-west-2.eks.amazonaws.com"
 	caCert             = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNE1EWXdOekExTlRBMU5Wb1hEVEk0TURZd05EQTFOVEExTlZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTWJoCnpvZElYR0drckNSZE1jUmVEN0YvMnB1NFZweTdvd3FEVDgrdk9zeGs2bXFMNWxQd3ZicFhmYkE3R0xzMDVHa0wKaDdqL0ZjcU91cnMwUFZSK3N5REtuQXltdDFORWxGNllGQktSV1dUQ1hNd2lwN1pweW9XMXdoYTlJYUlPUGxCTQpPTEVlckRabFVrVDFVV0dWeVdsMmxPeFgxa2JhV2gvakptWWdkeW5jMXhZZ3kxa2JybmVMSkkwLzVUVTRCajJxClB1emtrYW5Xd3lKbGdXQzhBSXlpWW82WFh2UVZmRzYrM3RISE5XM1F1b3ZoRng2MTFOYnl6RUI3QTdtZGNiNmgKR0ZpWjdOeThHZnFzdjJJSmI2Nk9FVzBSdW9oY1k3UDZPdnZmYnlKREhaU2hqTStRWFkxQXN5b3g4Ri9UelhHSgpQUWpoWUZWWEVhZU1wQmJqNmNFQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFCa2hKRVd4MHk1LzlMSklWdXJ1c1hZbjN6Z2EKRkZ6V0JsQU44WTlqUHB3S2t0Vy9JNFYyUGg3bWY2Z3ZwZ3Jhc2t1Slk1aHZPcDdBQmcxSTFhaHUxNUFpMUI0ZApuMllRaDlOaHdXM2pKMmhuRXk0VElpb0gza2JFdHRnUVB2bWhUQzNEYUJreEpkbmZJSEJCV1RFTTU1czRwRmxUClpzQVJ3aDc1Q3hYbjdScVU0akpKcWNPaTRjeU5qeFVpRDBqR1FaTmNiZWEyMkRCeTJXaEEzUWZnbGNScGtDVGUKRDVPS3NOWlF4MW9MZFAwci9TSmtPT1NPeUdnbVJURTIrODQxN21PRW02Z3RPMCszdWJkbXQ0aENsWEtFTTZYdwpuQWNlK0JxVUNYblVIN2ZNS3p2TDE5UExvMm5KbFU1TnlCbU1nL1pNVHVlUy80eFZmKy94WnpsQ0Q1WT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo="
@@ -106,7 +106,8 @@ func testVPC() *api.ClusterVPC {
 				},
 			},
 		},
-		SecurityGroup: "sg-0b44c48bcba5b7362",
+		SecurityGroup:           "sg-0b44c48bcba5b7362",
+		SharedNodeSecurityGroup: "sg-shared",
 		Subnets: map[api.SubnetTopology]map[string]api.Network{
 			"Public": map[string]api.Network{
 				"us-west-2b": {
@@ -286,14 +287,15 @@ var _ = Describe("CloudFormation template builder API", func() {
 			VPC:               testVPC(),
 			NodeGroups: []*api.NodeGroup{
 				{
-					AMI:               "",
-					AMIFamily:         "AmazonLinux2",
-					InstanceType:      "t2.medium",
-					Name:              "ng-abcd1234",
-					PrivateNetworking: false,
-					DesiredCapacity:   2,
-					VolumeSize:        2,
-					VolumeType:        api.NodeVolumeTypeIO1,
+					AMI:                 "",
+					AMIFamily:           "AmazonLinux2",
+					InstanceType:        "t2.medium",
+					Name:                "ng-abcd1234",
+					PrivateNetworking:   false,
+					SharedSecurityGroup: true,
+					DesiredCapacity:     2,
+					VolumeSize:          2,
+					VolumeType:          api.NodeVolumeTypeIO1,
 				},
 			},
 		}
@@ -328,6 +330,7 @@ var _ = Describe("CloudFormation template builder API", func() {
 			"CertificateAuthorityData": caCert,
 			"ARN":                      arn,
 			"ClusterStackName":         "",
+			"SharedNodeSecurityGroup":  "sg-shared",
 		}
 
 		sampleStack := newStackWithOutputs(sampleOutputs)
@@ -342,7 +345,6 @@ var _ = Describe("CloudFormation template builder API", func() {
 			Expect(err).ShouldNot(HaveOccurred())
 			expectedData, err := json.Marshal(expected)
 			Expect(err).ShouldNot(HaveOccurred())
-
 			Expect(cfgData).To(MatchJSON(expectedData))
 		})
 	})

pkg/cfn/builder/cluster.go

@@ -106,6 +106,7 @@ func (c *ClusterResourceSet) GetAllOutputs(stack cfn.Stack) error {
 
 	c.spec.VPC.ID = c.outputs[CfnOutputClusterVPC]
 	c.spec.VPC.SecurityGroup = c.outputs[CfnOutputClusterSecurityGroup]
+	c.spec.VPC.SharedNodeSecurityGroup = c.outputs[CfnOutputClusterSharedNodeSecurityGroup]
 
 	if err := vpc.UseSubnets(c.provider, c.spec, api.SubnetTopologyPrivate, strings.Split(c.outputs[CfnOutputClusterSubnetsPrivate], ",")); err != nil {
 		return err

pkg/cfn/builder/nodegroup.go

@@ -44,6 +44,9 @@ func (n *NodeGroupResourceSet) AddAllResources() error {
 		n.spec.AMIFamily, n.spec.AllowSSH, n.spec.SubnetTopology(),
 		templateDescriptionSuffix)
 
+	n.rs.newOutput(CfnOutputNodeGroupFeaturePrivateNetworking, n.spec.PrivateNetworking, false)
+	n.rs.newOutput(CfnOutputNodeGroupFeatureSharedSecurityGroup, n.spec.SharedSecurityGroup, false)
+
 	n.vpc = makeImportValue(n.clusterStackName, CfnOutputClusterVPC)
 
 	userData, err := nodebootstrap.NewUserData(n.clusterSpec, n.spec)

pkg/cfn/builder/outputs.go

@@ -23,9 +23,15 @@ const (
 	CfnOutputClusterEndpoint                 = "Endpoint"
 	CfnOutputClusterARN                      = "ARN"
 	CfnOutputClusterStackName                = "ClusterStackName"
+	CfnOutputClusterSharedNodeSecurityGroup  = "SharedNodeSecurityGroup"
 
 	// outputs from nodegroup stack
 	CfnOutputNodeGroupInstanceRoleARN = "InstanceRoleARN"
+	// outputs to indicate configuration attributes that may have critical effect
+	// on integration with the rest of the cluster and/or forward-compatibility
+	// with respect to e.g. upgrades
+	CfnOutputNodeGroupFeaturePrivateNetworking   = "FeaturePrivateNetworking"
+	CfnOutputNodeGroupFeatureSharedSecurityGroup = "FeatureSharedSecurityGroup"
 )
 
 // newOutput defines a new output and optionally exports it

pkg/cfn/builder/vpc.go

@@ -100,141 +100,147 @@ func (c *ClusterResourceSet) addOutputsForVPC() {
 	}
 }
 
+var (
+	sgProtoTCP           = gfn.NewString("tcp")
+	sgSourceAnywhereIPv4 = gfn.NewString("0.0.0.0/0")
+	sgSourceAnywhereIPv6 = gfn.NewString("::/0")
+
+	sgPortZero    = gfn.NewInteger(0)
+	sgMinNodePort = gfn.NewInteger(1025)
+	sgMaxNodePort = gfn.NewInteger(65535)
+
+	sgPortHTTPS = gfn.NewInteger(443)
+	sgPortSSH   = gfn.NewInteger(22)
+)
+
 func (c *ClusterResourceSet) addResourcesForSecurityGroups() {
-	refSG := c.newResource("ControlPlaneSecurityGroup", &gfn.AWSEC2SecurityGroup{
-		GroupDescription: gfn.NewString("Communication between the control plane and worker node groups"),
-		VpcId:            c.vpc,
-	})
-	c.securityGroups = []*gfn.Value{refSG}
-	c.rs.newJoinedOutput(CfnOutputClusterSecurityGroup, c.securityGroups, true)
+	var refControlPlaneSG, refClusterSharedNodeSG *gfn.Value
+
+	if c.spec.VPC.SecurityGroup == "" {
+		refControlPlaneSG = c.newResource("ControlPlaneSecurityGroup", &gfn.AWSEC2SecurityGroup{
+			GroupDescription: gfn.NewString("Communication between the control plane and worker nodegroups"),
+			VpcId:            c.vpc,
+		})
+	} else {
+		refControlPlaneSG = gfn.NewString(c.spec.VPC.SecurityGroup)
+	}
+	c.securityGroups = []*gfn.Value{refControlPlaneSG} // only this one SG is passed to EKS API, nodes are isolated
+
+	if c.spec.VPC.SharedNodeSecurityGroup == "" {
+		refClusterSharedNodeSG = c.newResource("ClusterSharedNodeSecurityGroup", &gfn.AWSEC2SecurityGroup{
+			GroupDescription: gfn.NewString("Communication between all nodes in the cluster"),
+			VpcId:            c.vpc,
+		})
+		c.newResource("IngressInterNodeGroupSG", &gfn.AWSEC2SecurityGroupIngress{
+			GroupId:               refClusterSharedNodeSG,
+			SourceSecurityGroupId: refClusterSharedNodeSG,
+			Description:           gfn.NewString("Allow nodes to communicate with each other (all ports)"),
+			IpProtocol:            gfn.NewString("-1"),
+			FromPort:              sgPortZero,
+			ToPort:                sgMaxNodePort,
+		})
+	} else {
+		refClusterSharedNodeSG = gfn.NewString(c.spec.VPC.SharedNodeSecurityGroup)
+	}
+
+	c.rs.newOutput(CfnOutputClusterSecurityGroup, refControlPlaneSG, true)
+	c.rs.newOutput(CfnOutputClusterSharedNodeSecurityGroup, refClusterSharedNodeSG, true)
 }
 
 func (n *NodeGroupResourceSet) addResourcesForSecurityGroups() {
 	desc := "worker nodes in group " + n.nodeGroupName
 
-	tcp := gfn.NewString("tcp")
-	udp := gfn.NewString("udp")
 	allInternalIPv4 := gfn.NewString(n.clusterSpec.VPC.CIDR.String())
-	anywhereIPv4 := gfn.NewString("0.0.0.0/0")
-	anywhereIPv6 := gfn.NewString("::/0")
-	var (
-		apiPort = gfn.NewInteger(443)
-		dnsPort = gfn.NewInteger(53)
-		sshPort = gfn.NewInteger(22)
-
-		portZero    = gfn.NewInteger(0)
-		nodeMinPort = gfn.NewInteger(1025)
-		nodeMaxPort = gfn.NewInteger(65535)
-	)
-
-	refCP := makeImportValue(n.clusterStackName, CfnOutputClusterSecurityGroup)
-	refSG := n.newResource("SG", &gfn.AWSEC2SecurityGroup{
+
+	refControlPlaneSG := makeImportValue(n.clusterStackName, CfnOutputClusterSecurityGroup)
+
+	refNodeGroupLocalSG := n.newResource("SG", &gfn.AWSEC2SecurityGroup{
 		VpcId:            makeImportValue(n.clusterStackName, CfnOutputClusterVPC),
 		GroupDescription: gfn.NewString("Communication between the control plane and " + desc),
 		Tags: []gfn.Tag{{
 			Key:   gfn.NewString("kubernetes.io/cluster/" + n.clusterSpec.Metadata.Name),
 			Value: gfn.NewString("owned"),
 		}},
 	})
-	n.securityGroups = []*gfn.Value{refSG}
 
-	if len(n.spec.SecurityGroups) > 0 {
-		for _, arn := range n.spec.SecurityGroups {
-			n.securityGroups = append(n.securityGroups, gfn.NewString(arn))
-		}
+	n.securityGroups = []*gfn.Value{refNodeGroupLocalSG}
+
+	if n.spec.SharedSecurityGroup {
+		refClusterSharedNodeSG := makeImportValue(n.clusterStackName, CfnOutputClusterSharedNodeSecurityGroup)
+		n.securityGroups = append(n.securityGroups, refClusterSharedNodeSG)
+	}
+
+	for _, id := range n.spec.SecurityGroups {
+		n.securityGroups = append(n.securityGroups, gfn.NewString(id))
 	}
 
-	n.newResource("IngressInterSG", &gfn.AWSEC2SecurityGroupIngress{
-		GroupId:               refSG,
-		SourceSecurityGroupId: refSG,
-		Description:           gfn.NewString("Allow " + desc + " to communicate with each other (all ports)"),
-		IpProtocol:            gfn.NewString("-1"),
-		FromPort:              portZero,
-		ToPort:                nodeMaxPort,
-	})
 	n.newResource("IngressInterCluster", &gfn.AWSEC2SecurityGroupIngress{
-		GroupId:               refSG,
-		SourceSecurityGroupId: refCP,
+		GroupId:               refNodeGroupLocalSG,
+		SourceSecurityGroupId: refControlPlaneSG,
 		Description:           gfn.NewString("Allow " + desc + " to communicate with control plane (kubelet and workload TCP ports)"),
-		IpProtocol:            tcp,
-		FromPort:              nodeMinPort,
-		ToPort:                nodeMaxPort,
+		IpProtocol:            sgProtoTCP,
+		FromPort:              sgMinNodePort,
+		ToPort:                sgMaxNodePort,
 	})
 	n.newResource("EgressInterCluster", &gfn.AWSEC2SecurityGroupEgress{
-		GroupId:                    refCP,
-		DestinationSecurityGroupId: refSG,
+		GroupId:                    refControlPlaneSG,
+		DestinationSecurityGroupId: refNodeGroupLocalSG,
 		Description:                gfn.NewString("Allow control plane to communicate with " + desc + " (kubelet and workload TCP ports)"),
-		IpProtocol:                 tcp,
-		FromPort:                   nodeMinPort,
-		ToPort:                     nodeMaxPort,
+		IpProtocol:                 sgProtoTCP,
+		FromPort:                   sgMinNodePort,
+		ToPort:                     sgMaxNodePort,
 	})
 	n.newResource("IngressInterClusterAPI", &gfn.AWSEC2SecurityGroupIngress{
-		GroupId:               refSG,
-		SourceSecurityGroupId: refCP,
+		GroupId:               refNodeGroupLocalSG,
+		SourceSecurityGroupId: refControlPlaneSG,
 		Description:           gfn.NewString("Allow " + desc + " to communicate with control plane (workloads using HTTPS port, commonly used with extension API servers)"),
-		IpProtocol:            tcp,
-		FromPort:              apiPort,
-		ToPort:                apiPort,
+		IpProtocol:            sgProtoTCP,
+		FromPort:              sgPortHTTPS,
+		ToPort:                sgPortHTTPS,
 	})
 	n.newResource("EgressInterClusterAPI", &gfn.AWSEC2SecurityGroupEgress{
-		GroupId:                    refCP,
-		DestinationSecurityGroupId: refSG,
+		GroupId:                    refControlPlaneSG,
+		DestinationSecurityGroupId: refNodeGroupLocalSG,
 		Description:                gfn.NewString("Allow control plane to communicate with " + desc + " (workloads using HTTPS port, commonly used with extension API servers)"),
-		IpProtocol:                 tcp,
-		FromPort:                   apiPort,
-		ToPort:                     apiPort,
+		IpProtocol:                 sgProtoTCP,
+		FromPort:                   sgPortHTTPS,
+		ToPort:                     sgPortHTTPS,
 	})
 	n.newResource("IngressInterClusterCP", &gfn.AWSEC2SecurityGroupIngress{
-		GroupId:               refCP,
-		SourceSecurityGroupId: refSG,
+		GroupId:               refControlPlaneSG,
+		SourceSecurityGroupId: refNodeGroupLocalSG,
 		Description:           gfn.NewString("Allow control plane to receive API requests from " + desc),
-		IpProtocol:            tcp,
-		FromPort:              apiPort,
-		ToPort:                apiPort,
+		IpProtocol:            sgProtoTCP,
+		FromPort:              sgPortHTTPS,
+		ToPort:                sgPortHTTPS,
 	})
 	if n.spec.AllowSSH {
 		if n.spec.PrivateNetworking {
 			n.newResource("SSHIPv4", &gfn.AWSEC2SecurityGroupIngress{
-				GroupId:     refSG,
+				GroupId:     refNodeGroupLocalSG,
 				CidrIp:      allInternalIPv4,
 				Description: gfn.NewString("Allow SSH access to " + desc + " (private, only inside VPC)"),
-				IpProtocol:  tcp,
-				FromPort:    sshPort,
-				ToPort:      sshPort,
+				IpProtocol:  sgProtoTCP,
+				FromPort:    sgPortSSH,
+				ToPort:      sgPortSSH,
 			})
 		} else {
 			n.newResource("SSHIPv4", &gfn.AWSEC2SecurityGroupIngress{
-				GroupId:     refSG,
-				CidrIp:      anywhereIPv4,
+				GroupId:     refNodeGroupLocalSG,
+				CidrIp:      sgSourceAnywhereIPv4,
 				Description: gfn.NewString("Allow SSH access to " + desc),
-				IpProtocol:  tcp,
-				FromPort:    sshPort,
-				ToPort:      sshPort,
+				IpProtocol:  sgProtoTCP,
+				FromPort:    sgPortSSH,
+				ToPort:      sgPortSSH,
 			})
 			n.newResource("SSHIPv6", &gfn.AWSEC2SecurityGroupIngress{
-				GroupId:     refSG,
-				CidrIpv6:    anywhereIPv6,
+				GroupId:     refNodeGroupLocalSG,
+				CidrIpv6:    sgSourceAnywhereIPv6,
 				Description: gfn.NewString("Allow SSH access to " + desc),
-				IpProtocol:  tcp,
-				FromPort:    sshPort,
-				ToPort:      sshPort,
+				IpProtocol:  sgProtoTCP,
+				FromPort:    sgPortSSH,
+				ToPort:      sgPortSSH,
 			})
 		}
 	}
-	n.newResource("DNSUDPIPv4", &gfn.AWSEC2SecurityGroupIngress{
-		GroupId:     refSG,
-		CidrIp:      allInternalIPv4,
-		Description: gfn.NewString("Allow DNS access to " + desc + " inside VPC"),
-		IpProtocol:  udp,
-		FromPort:    dnsPort,
-		ToPort:      dnsPort,
-	})
-	n.newResource("DNSTCPIPv4", &gfn.AWSEC2SecurityGroupIngress{
-		GroupId:     refSG,
-		CidrIp:      allInternalIPv4,
-		Description: gfn.NewString("Allow DNS access to " + desc + " inside VPC"),
-		IpProtocol:  tcp,
-		FromPort:    dnsPort,
-		ToPort:      dnsPort,
-	})
 }