Skip to content

Commit

Permalink
Possibility to use openstack without lbaas
Browse files Browse the repository at this point in the history
  • Loading branch information
@zetaab
zetaab committed Jun 23, 2019
1 parent 3b0af77 commit 9cf0b45
Show file tree
Hide file tree
Showing 10 changed files with 200 additions and 61 deletions.
38 changes: 24 additions & 14 deletions cmd/kops/create_cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -912,20 +912,10 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
if cluster.Spec.CloudConfig == nil {
cluster.Spec.CloudConfig = &api.CloudConfiguration{}
}
provider := "haproxy"
if c.OpenstackLBOctavia {
provider = "octavia"
}
cluster.Spec.CloudConfig.Openstack = &api.OpenstackConfiguration{
Router: &api.OpenstackRouter{
ExternalNetwork: fi.String(c.OpenstackExternalNet),
},
Loadbalancer: &api.OpenstackLoadbalancerConfig{
FloatingNetwork: fi.String(c.OpenstackExternalNet),
Method: fi.String("ROUND_ROBIN"),
Provider: fi.String(provider),
UseOctavia: fi.Bool(c.OpenstackLBOctavia),
},
BlockStorage: &api.OpenstackBlockStorageConfig{
Version: fi.String("v2"),
IgnoreAZ: fi.Bool(c.OpenstackStorageIgnoreAZ),
Expand All @@ -942,9 +932,6 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
if c.OpenstackExternalSubnet != "" {
cluster.Spec.CloudConfig.Openstack.Router.ExternalSubnet = fi.String(c.OpenstackExternalSubnet)
}
if c.OpenstackLbSubnet != "" {
cluster.Spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet = fi.String(c.OpenstackLbSubnet)
}
}
}

Expand Down Expand Up @@ -1163,7 +1150,9 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
cluster.Spec.API = &api.AccessSpec{}
}
if cluster.Spec.API.IsEmpty() {
if c.APILoadBalancerType != "" {
if c.Cloud == "openstack" {
initializeOpenstackAPI(c, cluster)
} else if c.APILoadBalancerType != "" {
cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{}
} else {
switch cluster.Spec.Topology.Masters {
Expand Down Expand Up @@ -1443,6 +1432,27 @@ func parseCloudLabels(s string) (map[string]string, error) {
return m, nil
}

func initializeOpenstackAPI(c *CreateClusterOptions, cluster *api.Cluster) {
if c.APILoadBalancerType != "" {
cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{}
provider := "haproxy"
if c.OpenstackLBOctavia {
provider = "octavia"
}

cluster.Spec.CloudConfig.Openstack.Loadbalancer = &api.OpenstackLoadbalancerConfig{
FloatingNetwork: fi.String(c.OpenstackExternalNet),
Method: fi.String("ROUND_ROBIN"),
Provider: fi.String(provider),
UseOctavia: fi.Bool(c.OpenstackLBOctavia),
}

if c.OpenstackLbSubnet != "" {
cluster.Spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet = fi.String(c.OpenstackLbSubnet)
}
}
}

func getZoneToSubnetProviderID(VPCID string, region string, subnetIDs []string) (map[string]string, error) {
res := make(map[string]string)
cloudTags := map[string]string{}
Expand Down
36 changes: 22 additions & 14 deletions pkg/model/openstackmodel/servergroup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,10 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
igUserData = fi.String(startupStr)
}

var securityGroups []*openstacktasks.SecurityGroup
securityGroupName := b.SecurityGroupName(ig.Spec.Role)
securityGroups = append(securityGroups, b.LinkToSecurityGroup(securityGroupName))

// In the future, OpenStack will use Machine API to manage groups,
// for now create d.InstanceGroups.Spec.MinSize amount of servers
for i := int32(0); i < *ig.Spec.MinSize; i++ {
Expand All @@ -87,8 +91,6 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
iName := strings.ToLower(fmt.Sprintf("%s-%d.%s", ig.Name, i+1, b.ClusterName()))
instanceName := fi.String(strings.Replace(iName, ".", "-", -1))

securityGroupName := b.SecurityGroupName(ig.Spec.Role)
securityGroup := b.LinkToSecurityGroup(securityGroupName)
var az *string
if len(ig.Spec.Subnets) > 0 {
// bastion subnet name is not actual zone name, it contains "utility-" prefix
Expand All @@ -102,7 +104,7 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
portTask := &openstacktasks.Port{
Name: fi.String(fmt.Sprintf("%s-%s", "port", *instanceName)),
Network: b.LinkToNetwork(),
SecurityGroups: append([]*openstacktasks.SecurityGroup{}, securityGroup),
SecurityGroups: securityGroups,
Lifecycle: b.Lifecycle,
}
c.AddTask(portTask)
Expand Down Expand Up @@ -135,13 +137,14 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
}
c.AddTask(t)
case kops.InstanceGroupRoleMaster:
if !b.UseLoadBalancerForAPI() {
if b.Cluster.Spec.CloudConfig.Openstack.Loadbalancer == nil {
t := &openstacktasks.FloatingIP{
Name: fi.String(fmt.Sprintf("%s-%s", "fip", *instanceTask.Name)),
Server: instanceTask,
Lifecycle: b.Lifecycle,
}
c.AddTask(t)
b.associateFIPToKeypair(c, t)
}
default:
if !b.UsesSSHBastion() {
Expand All @@ -158,6 +161,19 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *
return nil
}

func (b *ServerGroupModelBuilder) associateFIPToKeypair(c *fi.ModelBuilderContext, fipTask *openstacktasks.FloatingIP) error {
// Ensure the floating IP is included in the TLS certificate,
// if we're not going to use an alias for it
// TODO: I don't love this technique for finding the task by name & modifying it
masterKeypairTask, found := c.Tasks["Keypair/master"]
if !found {
return fmt.Errorf("keypair/master task not found")
}
masterKeypair := masterKeypairTask.(*fitasks.Keypair)
masterKeypair.AlternateNameTasks = append(masterKeypair.AlternateNameTasks, fipTask)
return nil
}

func (b *ServerGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
clusterName := b.ClusterName()

Expand All @@ -184,7 +200,7 @@ func (b *ServerGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
}
}

if b.UseLoadBalancerForAPI() {
if b.Cluster.Spec.CloudConfig.Openstack.Loadbalancer != nil {
lbSubnetName := b.MasterInstanceGroups()[0].Spec.Subnets[0]
lbTask := &openstacktasks.LB{
Name: fi.String(b.Cluster.Spec.MasterPublicName),
Expand All @@ -202,15 +218,7 @@ func (b *ServerGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
c.AddTask(lbfipTask)

if dns.IsGossipHostname(b.Cluster.Name) || b.UsePrivateDNS() {
// Ensure the floating IP is included in the TLS certificate,
// if we're not going to use an alias for it
// TODO: I don't love this technique for finding the task by name & modifying it
masterKeypairTask, found := c.Tasks["Keypair/master"]
if !found {
return fmt.Errorf("keypair/master task not found")
}
masterKeypair := masterKeypairTask.(*fitasks.Keypair)
masterKeypair.AlternateNameTasks = append(masterKeypair.AlternateNameTasks, lbfipTask)
b.associateFIPToKeypair(c, lbfipTask)
}

poolTask := &openstacktasks.LBPool{
Expand Down
3 changes: 2 additions & 1 deletion pkg/resources/openstack/securitygroups.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package openstack

import (
"fmt"
"strings"

sg "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups"
Expand All @@ -39,7 +40,7 @@ func (os *clusterDiscoveryOS) ListSecurityGroups() ([]*resources.Resource, error
}

for _, sg := range sgs {
if strings.HasSuffix(sg.Name, os.clusterName) {
if strings.HasSuffix(sg.Name, fmt.Sprintf(".%s", os.clusterName)) {
resourceTracker := &resources.Resource{
Name: sg.Name,
ID: sg.ID,
Expand Down
1 change: 1 addition & 0 deletions upup/pkg/fi/cloudup/openstack/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ go_library(
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/ports:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/subnets:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/pagination:go_default_library",
"//vendor/github.com/mitchellh/mapstructure:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
Expand Down
87 changes: 63 additions & 24 deletions upup/pkg/fi/cloudup/openstack/cloud.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ import (
"github.com/gophercloud/gophercloud/openstack/networking/v2/networks"
"github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
"github.com/gophercloud/gophercloud/openstack/networking/v2/subnets"
"github.com/mitchellh/mapstructure"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog"
Expand Down Expand Up @@ -407,31 +408,35 @@ func NewOpenstackCloud(tags map[string]string, spec *kops.ClusterSpec) (Openstac
}
spec.CloudConfig.Openstack.Loadbalancer.FloatingNetworkID = fi.String(lbNet[0].ID)
}
if spec.CloudConfig.Openstack.Loadbalancer.UseOctavia != nil {
octavia = fi.BoolValue(spec.CloudConfig.Openstack.Loadbalancer.UseOctavia)
}
if spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet != nil {
c.floatingSubnet = spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet
if spec.CloudConfig.Openstack.Loadbalancer != nil {
if spec.CloudConfig.Openstack.Loadbalancer.UseOctavia != nil {
octavia = fi.BoolValue(spec.CloudConfig.Openstack.Loadbalancer.UseOctavia)
}
if spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet != nil {
c.floatingSubnet = spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet
}
}
}
c.useOctavia = octavia
var lbClient *gophercloud.ServiceClient
if octavia {
if spec.CloudConfig.Openstack.Loadbalancer != nil && octavia {
klog.V(2).Infof("Openstack using Octavia lbaasv2 api")
lbClient, err = os.NewLoadBalancerV2(provider, gophercloud.EndpointOpts{
Region: region,
})
if err != nil {
return nil, fmt.Errorf("error building lb client: %v", err)
}
} else {
} else if spec.CloudConfig.Openstack.Loadbalancer != nil {
klog.V(2).Infof("Openstack using deprecated lbaasv2 api")
lbClient, err = os.NewNetworkV2(provider, gophercloud.EndpointOpts{
Region: region,
})
if err != nil {
return nil, fmt.Errorf("error building lb client: %v", err)
}
} else {
klog.V(2).Infof("Openstack disabled loadbalancer support")
}
c.lbClient = lbClient
return c, nil
Expand Down Expand Up @@ -547,30 +552,64 @@ func (c *openstackCloud) GetCloudTags() map[string]string {
return c.tags
}

type Address struct {
IPType string `mapstructure:"OS-EXT-IPS:type"`
Addr string
}

func (c *openstackCloud) GetApiIngressStatus(cluster *kops.Cluster) ([]kops.ApiIngressStatus, error) {
var ingresses []kops.ApiIngressStatus
if cluster.Spec.MasterPublicName != "" {
// Note that this must match OpenstackModel lb name
klog.V(2).Infof("Querying Openstack to find Loadbalancers for API (%q)", cluster.Name)
lbList, err := c.ListLBs(loadbalancers.ListOpts{
Name: cluster.Spec.MasterPublicName,
})
if err != nil {
return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list openstack loadbalancers: %v", err)
if cluster.Spec.CloudConfig.Openstack.Loadbalancer != nil {
if cluster.Spec.MasterPublicName != "" {
// Note that this must match OpenstackModel lb name
klog.V(2).Infof("Querying Openstack to find Loadbalancers for API (%q)", cluster.Name)
lbList, err := c.ListLBs(loadbalancers.ListOpts{
Name: cluster.Spec.MasterPublicName,
})
if err != nil {
return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list openstack loadbalancers: %v", err)
}
// Must Find Floating IP related to this lb
fips, err := c.ListFloatingIPs()
if err != nil {
return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list floating IP's: %v", err)
}

for _, lb := range lbList {
for _, fip := range fips {
if fip.FixedIP == lb.VipAddress {

ingresses = append(ingresses, kops.ApiIngressStatus{
IP: fip.IP,
})
}
}
}
}
// Must Find Floating IP related to this lb
fips, err := c.ListFloatingIPs()
} else {
instances, err := c.ListInstances(servers.ListOpts{})
if err != nil {
return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list floating IP's: %v", err)
return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list master nodes: %v", err)
}

for _, lb := range lbList {
for _, fip := range fips {
if fip.FixedIP == lb.VipAddress {
for _, instance := range instances {
val, ok := instance.Metadata["k8s"]
val2, ok2 := instance.Metadata["KopsInstanceGroup"]
if ok && val == cluster.Name && ok2 && strings.HasPrefix(val2, "master") {
var addresses map[string][]Address
err := mapstructure.Decode(instance.Addresses, &addresses)
if err != nil {
return nil, err
}

ingresses = append(ingresses, kops.ApiIngressStatus{
IP: fip.IP,
})
for _, addrList := range addresses {
for _, props := range addrList {
if props.IPType == "floating" {
ingresses = append(ingresses, kops.ApiIngressStatus{
IP: props.Addr,
})
}
}
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion upup/pkg/fi/cloudup/openstack/floatingip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func (c *openstackCloud) GetFloatingIP(id string) (fip *floatingips.FloatingIP,

fip, err = floatingips.Get(c.ComputeClient(), id).Extract()
if err != nil {
return false, fmt.Errorf("GetFloatingIP: fetching floating IP failed: %v", err)
return false, fmt.Errorf("GetFloatingIP: fetching floating IP (%s) failed: %v", id, err)
}
return true, nil
})
Expand Down
Loading

0 comments on commit 9cf0b45

Please sign in to comment.