Merge pull request #7178 from zetaab/feature/nonlb

Possibility to use OpenStack without lbaas (loadbalancer)

cmd/kops/create_cluster.go

@@ -912,20 +912,10 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 			if cluster.Spec.CloudConfig == nil {
 				cluster.Spec.CloudConfig = &api.CloudConfiguration{}
 			}
-			provider := "haproxy"
-			if c.OpenstackLBOctavia {
-				provider = "octavia"
-			}
 			cluster.Spec.CloudConfig.Openstack = &api.OpenstackConfiguration{
 				Router: &api.OpenstackRouter{
 					ExternalNetwork: fi.String(c.OpenstackExternalNet),
 				},
-				Loadbalancer: &api.OpenstackLoadbalancerConfig{
-					FloatingNetwork: fi.String(c.OpenstackExternalNet),
-					Method:          fi.String("ROUND_ROBIN"),
-					Provider:        fi.String(provider),
-					UseOctavia:      fi.Bool(c.OpenstackLBOctavia),
-				},
 				BlockStorage: &api.OpenstackBlockStorageConfig{
 					Version:  fi.String("v2"),
 					IgnoreAZ: fi.Bool(c.OpenstackStorageIgnoreAZ),
@@ -942,9 +932,6 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 			if c.OpenstackExternalSubnet != "" {
 				cluster.Spec.CloudConfig.Openstack.Router.ExternalSubnet = fi.String(c.OpenstackExternalSubnet)
 			}
-			if c.OpenstackLbSubnet != "" {
-				cluster.Spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet = fi.String(c.OpenstackLbSubnet)
-			}
 		}
 	}
 
@@ -1163,7 +1150,9 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 		cluster.Spec.API = &api.AccessSpec{}
 	}
 	if cluster.Spec.API.IsEmpty() {
-		if c.APILoadBalancerType != "" {
+		if c.Cloud == "openstack" {
+			initializeOpenstackAPI(c, cluster)
+		} else if c.APILoadBalancerType != "" {
 			cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{}
 		} else {
 			switch cluster.Spec.Topology.Masters {
@@ -1443,6 +1432,27 @@ func parseCloudLabels(s string) (map[string]string, error) {
 	return m, nil
 }
 
+func initializeOpenstackAPI(c *CreateClusterOptions, cluster *api.Cluster) {
+	if c.APILoadBalancerType != "" {
+		cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{}
+		provider := "haproxy"
+		if c.OpenstackLBOctavia {
+			provider = "octavia"
+		}
+
+		cluster.Spec.CloudConfig.Openstack.Loadbalancer = &api.OpenstackLoadbalancerConfig{
+			FloatingNetwork: fi.String(c.OpenstackExternalNet),
+			Method:          fi.String("ROUND_ROBIN"),
+			Provider:        fi.String(provider),
+			UseOctavia:      fi.Bool(c.OpenstackLBOctavia),
+		}
+
+		if c.OpenstackLbSubnet != "" {
+			cluster.Spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet = fi.String(c.OpenstackLbSubnet)
+		}
+	}
+}
+
 func getZoneToSubnetProviderID(VPCID string, region string, subnetIDs []string) (map[string]string, error) {
 	res := make(map[string]string)
 	cloudTags := map[string]string{}

docs/tutorial/openstack.md

@@ -139,3 +139,15 @@ Finally
 ```
 kops update cluster --name <cluster> --yes
 ```
+
+# Using OpenStack without lbaas
+Some OpenStack installations does not include installation of lbaas component. That is why we have added very-experimental support of installing OpenStack kops without lbaas. You can install it using:
+
+```
+kops create cluster \
+  --cloud openstack \
+  ... (like usually)
+  --api-loadbalancer-type=""
+```
+
+The biggest problem currently when installing without loadbalancer is that kubectl requests outside cluster is always going to first master. External loadbalancer is one option which can solve this issue.

pkg/model/openstackmodel/servergroup.go

@@ -60,6 +60,7 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
 	}
 	igMeta["k8s"] = b.ClusterName()
 	igMeta["KopsInstanceGroup"] = ig.Name
+	igMeta["KopsRole"] = fmt.Sprintf("%s", ig.Spec.Role)
 	igMeta[openstack.INSTANCE_GROUP_GENERATION] = fmt.Sprintf("%d", ig.GetGeneration())
 	igMeta[openstack.CLUSTER_GENERATION] = fmt.Sprintf("%d", b.Cluster.GetGeneration())
 
@@ -76,6 +77,14 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
 		igUserData = fi.String(startupStr)
 	}
 
+	var securityGroups []*openstacktasks.SecurityGroup
+	securityGroupName := b.SecurityGroupName(ig.Spec.Role)
+	securityGroups = append(securityGroups, b.LinkToSecurityGroup(securityGroupName))
+
+	if b.Cluster.Spec.CloudConfig.Openstack.Loadbalancer == nil && ig.Spec.Role == kops.InstanceGroupRoleMaster {
+		securityGroups = append(securityGroups, b.LinkToSecurityGroup(b.Cluster.Spec.MasterPublicName))
+	}
+
 	// In the future, OpenStack will use Machine API to manage groups,
 	// for now create d.InstanceGroups.Spec.MinSize amount of servers
 	for i := int32(0); i < *ig.Spec.MinSize; i++ {
@@ -87,8 +96,6 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
 		iName := strings.ToLower(fmt.Sprintf("%s-%d.%s", ig.Name, i+1, b.ClusterName()))
 		instanceName := fi.String(strings.Replace(iName, ".", "-", -1))
 
-		securityGroupName := b.SecurityGroupName(ig.Spec.Role)
-		securityGroup := b.LinkToSecurityGroup(securityGroupName)
 		var az *string
 		if len(ig.Spec.Subnets) > 0 {
 			// bastion subnet name is not actual zone name, it contains "utility-" prefix
@@ -102,7 +109,7 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
 		portTask := &openstacktasks.Port{
 			Name:           fi.String(fmt.Sprintf("%s-%s", "port", *instanceName)),
 			Network:        b.LinkToNetwork(),
-			SecurityGroups: append([]*openstacktasks.SecurityGroup{}, securityGroup),
+			SecurityGroups: securityGroups,
 			Lifecycle:      b.Lifecycle,
 		}
 		c.AddTask(portTask)
@@ -135,13 +142,14 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
 			}
 			c.AddTask(t)
 		case kops.InstanceGroupRoleMaster:
-			if !b.UseLoadBalancerForAPI() {
+			if b.Cluster.Spec.CloudConfig.Openstack.Loadbalancer == nil {
 				t := &openstacktasks.FloatingIP{
 					Name:      fi.String(fmt.Sprintf("%s-%s", "fip", *instanceTask.Name)),
 					Server:    instanceTask,
 					Lifecycle: b.Lifecycle,
 				}
 				c.AddTask(t)
+				b.associateFIPToKeypair(c, t)
 			}
 		default:
 			if !b.UsesSSHBastion() {
@@ -158,6 +166,19 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
 	return nil
 }
 
+func (b *ServerGroupModelBuilder) associateFIPToKeypair(c *fi.ModelBuilderContext, fipTask *openstacktasks.FloatingIP) error {
+	// Ensure the floating IP is included in the TLS certificate,
+	// if we're not going to use an alias for it
+	// TODO: I don't love this technique for finding the task by name & modifying it
+	masterKeypairTask, found := c.Tasks["Keypair/master"]
+	if !found {
+		return fmt.Errorf("keypair/master task not found")
+	}
+	masterKeypair := masterKeypairTask.(*fitasks.Keypair)
+	masterKeypair.AlternateNameTasks = append(masterKeypair.AlternateNameTasks, fipTask)
+	return nil
+}
+
 func (b *ServerGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 	clusterName := b.ClusterName()
 
@@ -184,7 +205,7 @@ func (b *ServerGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 	}
 
-	if b.UseLoadBalancerForAPI() {
+	if b.Cluster.Spec.CloudConfig.Openstack.Loadbalancer != nil {
 		lbSubnetName := b.MasterInstanceGroups()[0].Spec.Subnets[0]
 		lbTask := &openstacktasks.LB{
 			Name:          fi.String(b.Cluster.Spec.MasterPublicName),
@@ -202,15 +223,7 @@ func (b *ServerGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		c.AddTask(lbfipTask)
 
 		if dns.IsGossipHostname(b.Cluster.Name) || b.UsePrivateDNS() {
-			// Ensure the floating IP is included in the TLS certificate,
-			// if we're not going to use an alias for it
-			// TODO: I don't love this technique for finding the task by name & modifying it
-			masterKeypairTask, found := c.Tasks["Keypair/master"]
-			if !found {
-				return fmt.Errorf("keypair/master task not found")
-			}
-			masterKeypair := masterKeypairTask.(*fitasks.Keypair)
-			masterKeypair.AlternateNameTasks = append(masterKeypair.AlternateNameTasks, lbfipTask)
+			b.associateFIPToKeypair(c, lbfipTask)
 		}
 
 		poolTask := &openstacktasks.LBPool{

pkg/resources/openstack/securitygroups.go

@@ -17,6 +17,7 @@ limitations under the License.
 package openstack
 
 import (
+	"fmt"
 	"strings"
 
 	sg "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups"
@@ -39,7 +40,7 @@ func (os *clusterDiscoveryOS) ListSecurityGroups() ([]*resources.Resource, error
 	}
 
 	for _, sg := range sgs {
-		if strings.HasSuffix(sg.Name, os.clusterName) {
+		if strings.HasSuffix(sg.Name, fmt.Sprintf(".%s", os.clusterName)) {
 			resourceTracker := &resources.Resource{
 				Name: sg.Name,
 				ID:   sg.ID,

upup/pkg/fi/cloudup/openstack/BUILD.bazel

@@ -57,6 +57,7 @@ go_library(
         "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/ports:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/subnets:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud/pagination:go_default_library",
+        "//vendor/github.com/mitchellh/mapstructure:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",

upup/pkg/fi/cloudup/openstack/cloud.go

@@ -273,6 +273,8 @@ type OpenstackCloud interface {
 
 	AssociateFloatingIPToInstance(serverID string, opts floatingips.AssociateOpts) (err error)
 
+	ListServerFloatingIPs(id string) ([]*string, error)
+
 	ListFloatingIPs() (fips []floatingips.FloatingIP, err error)
 	ListL3FloatingIPs(opts l3floatingip.ListOpts) (fips []l3floatingip.FloatingIP, err error)
 	CreateFloatingIP(opts floatingips.CreateOpts) (*floatingips.FloatingIP, error)
@@ -407,30 +409,36 @@ func NewOpenstackCloud(tags map[string]string, spec *kops.ClusterSpec) (Openstac
 			}
 			spec.CloudConfig.Openstack.Loadbalancer.FloatingNetworkID = fi.String(lbNet[0].ID)
 		}
-		if spec.CloudConfig.Openstack.Loadbalancer.UseOctavia != nil {
-			octavia = fi.BoolValue(spec.CloudConfig.Openstack.Loadbalancer.UseOctavia)
-		}
-		if spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet != nil {
-			c.floatingSubnet = spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet
+		if spec.CloudConfig.Openstack.Loadbalancer != nil {
+			if spec.CloudConfig.Openstack.Loadbalancer.UseOctavia != nil {
+				octavia = fi.BoolValue(spec.CloudConfig.Openstack.Loadbalancer.UseOctavia)
+			}
+			if spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet != nil {
+				c.floatingSubnet = spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet
+			}
 		}
 	}
 	c.useOctavia = octavia
 	var lbClient *gophercloud.ServiceClient
-	if octavia {
-		klog.V(2).Infof("Openstack using Octavia lbaasv2 api")
-		lbClient, err = os.NewLoadBalancerV2(provider, gophercloud.EndpointOpts{
-			Region: region,
-		})
-		if err != nil {
-			return nil, fmt.Errorf("error building lb client: %v", err)
-		}
-	} else {
-		klog.V(2).Infof("Openstack using deprecated lbaasv2 api")
-		lbClient, err = os.NewNetworkV2(provider, gophercloud.EndpointOpts{
-			Region: region,
-		})
-		if err != nil {
-			return nil, fmt.Errorf("error building lb client: %v", err)
+	if spec != nil && spec.CloudConfig != nil && spec.CloudConfig.Openstack != nil {
+		if spec.CloudConfig.Openstack.Loadbalancer != nil && octavia {
+			klog.V(2).Infof("Openstack using Octavia lbaasv2 api")
+			lbClient, err = os.NewLoadBalancerV2(provider, gophercloud.EndpointOpts{
+				Region: region,
+			})
+			if err != nil {
+				return nil, fmt.Errorf("error building lb client: %v", err)
+			}
+		} else if spec.CloudConfig.Openstack.Loadbalancer != nil {
+			klog.V(2).Infof("Openstack using deprecated lbaasv2 api")
+			lbClient, err = os.NewNetworkV2(provider, gophercloud.EndpointOpts{
+				Region: region,
+			})
+			if err != nil {
+				return nil, fmt.Errorf("error building lb client: %v", err)
+			}
+		} else {
+			klog.V(2).Infof("Openstack disabled loadbalancer support")
 		}
 	}
 	c.lbClient = lbClient
@@ -547,30 +555,60 @@ func (c *openstackCloud) GetCloudTags() map[string]string {
 	return c.tags
 }
 
+type Address struct {
+	IPType string `mapstructure:"OS-EXT-IPS:type"`
+	Addr   string
+}
+
 func (c *openstackCloud) GetApiIngressStatus(cluster *kops.Cluster) ([]kops.ApiIngressStatus, error) {
 	var ingresses []kops.ApiIngressStatus
-	if cluster.Spec.MasterPublicName != "" {
-		// Note that this must match OpenstackModel lb name
-		klog.V(2).Infof("Querying Openstack to find Loadbalancers for API (%q)", cluster.Name)
-		lbList, err := c.ListLBs(loadbalancers.ListOpts{
-			Name: cluster.Spec.MasterPublicName,
-		})
-		if err != nil {
-			return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list openstack loadbalancers: %v", err)
+	if cluster.Spec.CloudConfig.Openstack.Loadbalancer != nil {
+		if cluster.Spec.MasterPublicName != "" {
+			// Note that this must match OpenstackModel lb name
+			klog.V(2).Infof("Querying Openstack to find Loadbalancers for API (%q)", cluster.Name)
+			lbList, err := c.ListLBs(loadbalancers.ListOpts{
+				Name: cluster.Spec.MasterPublicName,
+			})
+			if err != nil {
+				return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list openstack loadbalancers: %v", err)
+			}
+			// Must Find Floating IP related to this lb
+			fips, err := c.ListFloatingIPs()
+			if err != nil {
+				return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list floating IP's: %v", err)
+			}
+
+			for _, lb := range lbList {
+				for _, fip := range fips {
+					if fip.FixedIP == lb.VipAddress {
+						ingresses = append(ingresses, kops.ApiIngressStatus{
+							IP: fip.IP,
+						})
+					}
+				}
+			}
 		}
-		// Must Find Floating IP related to this lb
-		fips, err := c.ListFloatingIPs()
+	} else {
+		instances, err := c.ListInstances(servers.ListOpts{})
 		if err != nil {
-			return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list floating IP's: %v", err)
+			return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list master nodes: %v", err)
 		}
 
-		for _, lb := range lbList {
-			for _, fip := range fips {
-				if fip.FixedIP == lb.VipAddress {
-
-					ingresses = append(ingresses, kops.ApiIngressStatus{
-						IP: fip.IP,
-					})
+		for _, instance := range instances {
+			val, ok := instance.Metadata["k8s"]
+			val2, ok2 := instance.Metadata["KopsRole"]
+			if ok && val == cluster.Name && ok2 {
+				role, success := kops.ParseInstanceGroupRole(val2, false)
+				if success && role == kops.InstanceGroupRoleMaster {
+					ips, err := c.ListServerFloatingIPs(instance.ID)
+					if err != nil {
+						return ingresses, err
+					}
+					for _, ip := range ips {
+						ingresses = append(ingresses, kops.ApiIngressStatus{
+							IP: fi.StringValue(ip),
+						})
+					}
 				}
 			}
 		}

upup/pkg/fi/cloudup/openstack/floatingip.go

@@ -30,7 +30,7 @@ func (c *openstackCloud) GetFloatingIP(id string) (fip *floatingips.FloatingIP,
 
 		fip, err = floatingips.Get(c.ComputeClient(), id).Extract()
 		if err != nil {
-			return false, fmt.Errorf("GetFloatingIP: fetching floating IP failed: %v", err)
+			return false, fmt.Errorf("GetFloatingIP: fetching floating IP (%s) failed: %v", id, err)
 		}
 		return true, nil
 	})