Skip to content

Commit

Permalink
CCM: set predictable and proper server IP addresses order (#884)
Browse files Browse the repository at this point in the history 
* CCM: set predicted and proper server ip addresses order

* code review
  • Loading branch information
@kayrus @k8s-ci-robot
kayrus authored and k8s-ci-robot committed Jan 15, 2020
1 parent 1fa8b21 commit 46cab81
Show file tree
Hide file tree
Showing 5 changed files with 124 additions and 44 deletions.
1 change: 0 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ require (
github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect
github.com/emicklei/go-restful v2.9.6+incompatible // indirect
github.com/evanphx/json-patch v4.5.0+incompatible // indirect
github.com/go-logfmt/logfmt v0.4.0 // indirect
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
github.com/golang/protobuf v1.3.2
github.com/google/go-cmp v0.3.1 // indirect
Expand Down
88 changes: 58 additions & 30 deletions pkg/cloudprovider/providers/openstack/openstack.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -671,9 +671,60 @@ func getServerByName(client *gophercloud.ServiceClient, name types.NodeName) (*S
return &serverList[0], nil
}

func nodeAddresses(srv *servers.Server, networkingOpts NetworkingOpts) ([]v1.NodeAddress, error) {
// IP addresses order:
// * interfaces private IPs
// * access IPs
// * metadata hostname
// * server object Addresses (floating type)
func nodeAddresses(srv *servers.Server, interfaces []attachinterfaces.Interface, networkingOpts NetworkingOpts) ([]v1.NodeAddress, error) {
addrs := []v1.NodeAddress{}

// parse private IP addresses first in an ordered manner
for _, iface := range interfaces {
for _, fixedIP := range iface.FixedIPs {
if iface.PortState == "ACTIVE" {
isIPv6 := net.ParseIP(fixedIP.IPAddress).To4() == nil
if !(isIPv6 && networkingOpts.IPv6SupportDisabled) {
v1helper.AddToNodeAddresses(&addrs,
v1.NodeAddress{
Type: v1.NodeInternalIP,
Address: fixedIP.IPAddress,
},
)
}
}
}
}

// process public IP addresses
if srv.AccessIPv4 != "" {
v1helper.AddToNodeAddresses(&addrs,
v1.NodeAddress{
Type: v1.NodeExternalIP,
Address: srv.AccessIPv4,
},
)
}

if srv.AccessIPv6 != "" && !networkingOpts.IPv6SupportDisabled {
v1helper.AddToNodeAddresses(&addrs,
v1.NodeAddress{
Type: v1.NodeExternalIP,
Address: srv.AccessIPv6,
},
)
}

if srv.Metadata[TypeHostName] != "" {
v1helper.AddToNodeAddresses(&addrs,
v1.NodeAddress{
Type: v1.NodeHostName,
Address: srv.Metadata[TypeHostName],
},
)
}

// process the rest
type Address struct {
IPType string `mapstructure:"OS-EXT-IPS:type"`
Addr string
Expand Down Expand Up @@ -717,34 +768,6 @@ func nodeAddresses(srv *servers.Server, networkingOpts NetworkingOpts) ([]v1.Nod
}
}

// AccessIPs are usually duplicates of "public" addresses.
if srv.AccessIPv4 != "" {
v1helper.AddToNodeAddresses(&addrs,
v1.NodeAddress{
Type: v1.NodeExternalIP,
Address: srv.AccessIPv4,
},
)
}

if srv.AccessIPv6 != "" && !networkingOpts.IPv6SupportDisabled {
v1helper.AddToNodeAddresses(&addrs,
v1.NodeAddress{
Type: v1.NodeExternalIP,
Address: srv.AccessIPv6,
},
)
}

if srv.Metadata[TypeHostName] != "" {
v1helper.AddToNodeAddresses(&addrs,
v1.NodeAddress{
Type: v1.NodeHostName,
Address: srv.Metadata[TypeHostName],
},
)
}

return addrs, nil
}

Expand All @@ -754,7 +777,12 @@ func getAddressesByName(client *gophercloud.ServiceClient, name types.NodeName,
return nil, err
}

return nodeAddresses(&srv.Server, networkingOpts)
interfaces, err := getAttachedInterfacesByID(client, srv.ID)
if err != nil {
return nil, err
}

return nodeAddresses(&srv.Server, interfaces, networkingOpts)
}

func getAddressByName(client *gophercloud.ServiceClient, name types.NodeName, needIPv6 bool, networkingOpts NetworkingOpts) (string, error) {
Expand Down
7 changes: 6 additions & 1 deletion pkg/cloudprovider/providers/openstack/openstack_instances.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,12 @@ func (i *Instances) NodeAddressesByProviderID(ctx context.Context, providerID st
return []v1.NodeAddress{}, err
}

addresses, err := nodeAddresses(server, i.networkingOpts)
interfaces, err := getAttachedInterfacesByID(i.compute, server.ID)
if err != nil {
return []v1.NodeAddress{}, err
}

addresses, err := nodeAddresses(server, interfaces, i.networkingOpts)
if err != nil {
return []v1.NodeAddress{}, err
}
Expand Down
7 changes: 6 additions & 1 deletion pkg/cloudprovider/providers/openstack/openstack_routes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,12 @@ func (r *Routes) ListRoutes(ctx context.Context, clusterName string) ([]*cloudpr

nodeNamesByAddr := make(map[string]types.NodeName)
err := foreachServer(r.compute, servers.ListOpts{}, func(srv *servers.Server) (bool, error) {
addrs, err := nodeAddresses(srv, r.networkingOpts)
interfaces, err := getAttachedInterfacesByID(r.compute, srv.ID)
if err != nil {
return false, err
}

addrs, err := nodeAddresses(srv, interfaces, r.networkingOpts)
if err != nil {
return false, err
}
Expand Down
65 changes: 54 additions & 11 deletions pkg/cloudprovider/providers/openstack/openstack_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"time"

"github.com/gophercloud/gophercloud"
"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/attachinterfaces"
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
"github.com/spf13/pflag"

Expand Down Expand Up @@ -587,7 +588,21 @@ func TestNodeAddresses(t *testing.T) {
PublicNetworkName: "public",
}

addrs, err := nodeAddresses(&srv, networkingOpts)
interfaces := []attachinterfaces.Interface{
{
PortState: "ACTIVE",
FixedIPs: []attachinterfaces.FixedIP{
{
IPAddress: "10.0.0.32",
},
{
IPAddress: "10.0.0.31",
},
},
},
}

addrs, err := nodeAddresses(&srv, interfaces, networkingOpts)
if err != nil {
t.Fatalf("nodeAddresses returned error: %v", err)
}
Expand All @@ -596,13 +611,13 @@ func TestNodeAddresses(t *testing.T) {

want := []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "10.0.0.32"},
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
{Type: v1.NodeInternalIP, Address: "10.0.0.31"},
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
{Type: v1.NodeExternalIP, Address: "50.56.176.99"},
{Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"},
{Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.novalocal"},
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
}

if !reflect.DeepEqual(want, addrs) {
Expand Down Expand Up @@ -652,7 +667,21 @@ func TestNodeAddressesCustomPublicNetwork(t *testing.T) {
PublicNetworkName: "pub-net",
}

addrs, err := nodeAddresses(&srv, networkingOpts)
interfaces := []attachinterfaces.Interface{
{
PortState: "ACTIVE",
FixedIPs: []attachinterfaces.FixedIP{
{
IPAddress: "10.0.0.32",
},
{
IPAddress: "10.0.0.31",
},
},
},
}

addrs, err := nodeAddresses(&srv, interfaces, networkingOpts)
if err != nil {
t.Fatalf("nodeAddresses returned error: %v", err)
}
Expand All @@ -661,12 +690,12 @@ func TestNodeAddressesCustomPublicNetwork(t *testing.T) {

want := []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "10.0.0.32"},
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
{Type: v1.NodeInternalIP, Address: "10.0.0.31"},
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
{Type: v1.NodeExternalIP, Address: "50.56.176.99"},
{Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"},
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
}

if !reflect.DeepEqual(want, addrs) {
Expand Down Expand Up @@ -717,7 +746,21 @@ func TestNodeAddressesIPv6Disabled(t *testing.T) {
IPv6SupportDisabled: true,
}

addrs, err := nodeAddresses(&srv, networkingOpts)
interfaces := []attachinterfaces.Interface{
{
PortState: "ACTIVE",
FixedIPs: []attachinterfaces.FixedIP{
{
IPAddress: "10.0.0.32",
},
{
IPAddress: "10.0.0.31",
},
},
},
}

addrs, err := nodeAddresses(&srv, interfaces, networkingOpts)
if err != nil {
t.Fatalf("nodeAddresses returned error: %v", err)
}
Expand All @@ -726,10 +769,10 @@ func TestNodeAddressesIPv6Disabled(t *testing.T) {

want := []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "10.0.0.32"},
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
{Type: v1.NodeInternalIP, Address: "10.0.0.31"},
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
{Type: v1.NodeExternalIP, Address: "50.56.176.99"},
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
}

if !reflect.DeepEqual(want, addrs) {
Expand Down

0 comments on commit 46cab81

Please sign in to comment.