Skip to content

Commit

Permalink
Merge pull request #997 from giantswarm/kube-prefix
Browse files Browse the repository at this point in the history
Allow changing subnet/kube annotation prefix
  • Loading branch information
tomdee authored Oct 5, 2018
2 parents 2ca94f6 + bce8d23 commit f7c23a0
Show file tree
Hide file tree
Showing 4 changed files with 199 additions and 35 deletions.
4 changes: 3 additions & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ type CmdLineOpts struct {
version bool
kubeSubnetMgr bool
kubeApiUrl string
kubeAnnotationPrefix string
kubeConfigFile string
iface flagSlice
ifaceRegex flagSlice
Expand Down Expand Up @@ -122,6 +123,7 @@ func init() {
flannelFlags.BoolVar(&opts.ipMasq, "ip-masq", false, "setup IP masquerade rule for traffic destined outside of overlay network")
flannelFlags.BoolVar(&opts.kubeSubnetMgr, "kube-subnet-mgr", false, "contact the Kubernetes API for subnet assignment instead of etcd.")
flannelFlags.StringVar(&opts.kubeApiUrl, "kube-api-url", "", "Kubernetes API server URL. Does not need to be specified if flannel is running in a pod.")
flannelFlags.StringVar(&opts.kubeAnnotationPrefix, "kube-annotation-prefix", "flannel.alpha.coreos.com", `Kubernetes annotation prefix. Can contain single slash "/", otherwise it will be appended at the end.`)
flannelFlags.StringVar(&opts.kubeConfigFile, "kubeconfig-file", "", "kubeconfig file location. Does not need to be specified if flannel is running in a pod.")
flannelFlags.BoolVar(&opts.version, "version", false, "print version and exit")
flannelFlags.StringVar(&opts.healthzIP, "healthz-ip", "0.0.0.0", "the IP address for healthz server to listen")
Expand Down Expand Up @@ -157,7 +159,7 @@ func usage() {

func newSubnetManager() (subnet.Manager, error) {
if opts.kubeSubnetMgr {
return kube.NewSubnetManager(opts.kubeApiUrl, opts.kubeConfigFile)
return kube.NewSubnetManager(opts.kubeApiUrl, opts.kubeConfigFile, opts.kubeAnnotationPrefix)
}

cfg := &etcdv2.EtcdConfig{
Expand Down
66 changes: 66 additions & 0 deletions subnet/kube/annotations.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
// Copyright 2018 flannel authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kube

import (
"errors"
"regexp"
"strings"
)

type annotations struct {
SubnetKubeManaged string
BackendData string
BackendType string
BackendPublicIP string
BackendPublicIPOverwrite string
}

func newAnnotations(prefix string) (annotations, error) {
slashCnt := strings.Count(prefix, "/")
if slashCnt > 1 {
return annotations{}, errors.New("subnet/kube: prefix can contain at most single slash")
}
if slashCnt == 0 {
prefix += "/"
}
if !strings.HasSuffix(prefix, "/") && !strings.HasSuffix(prefix, "-") {
prefix += "-"
}

// matches is a regexp matching the format used by the kubernetes for
// annotations. Following rules apply:
//
// - must start with FQDN - must contain at most one slash "/"
// - must contain only lowercase letters, nubers, underscores,
// hyphens, dots and slash
matches, err := regexp.MatchString(`(?:[a-z0-9_-]+\.)+[a-z0-9_-]+/(?:[a-z0-9_-]+-)?$`, prefix)
if err != nil {
panic(err)
}
if !matches {
return annotations{}, errors.New("subnet/kube: prefix must be in a format: fqdn/[0-9a-z-_]*")
}

a := annotations{
SubnetKubeManaged: prefix + "kube-subnet-manager",
BackendData: prefix + "backend-data",
BackendType: prefix + "backend-type",
BackendPublicIP: prefix + "public-ip",
BackendPublicIPOverwrite: prefix + "public-ip-overwrite",
}

return a, nil
}
96 changes: 96 additions & 0 deletions subnet/kube/annotations_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
// Copyright 2018 flannel authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kube

import "testing"

func Test_newAnnotations(t *testing.T) {
testCases := []struct {
prefix string
expectedBackendType string
hasError bool
}{
{
prefix: "flannel.alpha.coreos.com",
expectedBackendType: "flannel.alpha.coreos.com/backend-type",
},
{
prefix: "flannel.alpha.coreos.com/",
expectedBackendType: "flannel.alpha.coreos.com/backend-type",
},
{
prefix: "flannel.alpha.coreos.com/prefix",
expectedBackendType: "flannel.alpha.coreos.com/prefix-backend-type",
},
{
prefix: "flannel.alpha.coreos.com/prefix-",
expectedBackendType: "flannel.alpha.coreos.com/prefix-backend-type",
},
{
prefix: "org.com",
expectedBackendType: "org.com/backend-type",
},
{
prefix: "org9.com",
expectedBackendType: "org9.com/backend-type",
},
{
prefix: "org.com/9",
expectedBackendType: "org.com/9-backend-type",
},
{
// Not a fqdn.
prefix: "org",
hasError: true,
},
{
// Not a fqdn before /.
prefix: "org/",
hasError: true,
},
{
// Not a fqdn before /.
prefix: "org/prefix",
hasError: true,
},
{
// Uppercase letters.
prefix: "org.COM",
hasError: true,
},
{
// Uppercase letters.
prefix: "org.com/PREFIX",
hasError: true,
},
}

for i, tc := range testCases {
as, err := newAnnotations(tc.prefix)

if err != nil && !tc.hasError {
t.Errorf("#%d: error = %s, want nil", i, err)
continue
}
if err == nil && tc.hasError {
t.Errorf("#%d: error = nil, want non-nil", i)
continue
}

if as.BackendType != tc.expectedBackendType {
t.Errorf("#%d: BackendType = %s, want %s", i, as.BackendType, tc.expectedBackendType)
}
}
}
68 changes: 34 additions & 34 deletions subnet/kube/kube.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,16 +51,11 @@ const (
resyncPeriod = 5 * time.Minute
nodeControllerSyncTimeout = 10 * time.Minute

subnetKubeManagedAnnotation = "flannel.alpha.coreos.com/kube-subnet-manager"
backendDataAnnotation = "flannel.alpha.coreos.com/backend-data"
backendTypeAnnotation = "flannel.alpha.coreos.com/backend-type"
backendPublicIPAnnotation = "flannel.alpha.coreos.com/public-ip"
backendPublicIPOverwriteAnnotation = "flannel.alpha.coreos.com/public-ip-overwrite"

netConfPath = "/etc/kube-flannel/net-conf.json"
)

type kubeSubnetManager struct {
annotations annotations
client clientset.Interface
nodeName string
nodeStore listers.NodeLister
Expand All @@ -69,7 +64,7 @@ type kubeSubnetManager struct {
events chan subnet.Event
}

func NewSubnetManager(apiUrl, kubeconfig string) (subnet.Manager, error) {
func NewSubnetManager(apiUrl, kubeconfig, prefix string) (subnet.Manager, error) {

var cfg *rest.Config
var err error
Expand Down Expand Up @@ -122,7 +117,7 @@ func NewSubnetManager(apiUrl, kubeconfig string) (subnet.Manager, error) {
return nil, fmt.Errorf("error parsing subnet config: %s", err)
}

sm, err := newKubeSubnetManager(c, sc, nodeName)
sm, err := newKubeSubnetManager(c, sc, nodeName, prefix)
if err != nil {
return nil, fmt.Errorf("error creating network manager: %s", err)
}
Expand All @@ -140,8 +135,13 @@ func NewSubnetManager(apiUrl, kubeconfig string) (subnet.Manager, error) {
return sm, nil
}

func newKubeSubnetManager(c clientset.Interface, sc *subnet.Config, nodeName string) (*kubeSubnetManager, error) {
func newKubeSubnetManager(c clientset.Interface, sc *subnet.Config, nodeName, prefix string) (*kubeSubnetManager, error) {
var err error
var ksm kubeSubnetManager
ksm.annotations, err = newAnnotations(prefix)
if err != nil {
return nil, err
}
ksm.client = c
ksm.nodeName = nodeName
ksm.subnetConf = sc
Expand Down Expand Up @@ -190,11 +190,11 @@ func newKubeSubnetManager(c clientset.Interface, sc *subnet.Config, nodeName str

func (ksm *kubeSubnetManager) handleAddLeaseEvent(et subnet.EventType, obj interface{}) {
n := obj.(*v1.Node)
if s, ok := n.Annotations[subnetKubeManagedAnnotation]; !ok || s != "true" {
if s, ok := n.Annotations[ksm.annotations.SubnetKubeManaged]; !ok || s != "true" {
return
}

l, err := nodeToLease(*n)
l, err := ksm.nodeToLease(*n)
if err != nil {
glog.Infof("Error turning node %q to lease: %v", n.ObjectMeta.Name, err)
return
Expand All @@ -205,16 +205,16 @@ func (ksm *kubeSubnetManager) handleAddLeaseEvent(et subnet.EventType, obj inter
func (ksm *kubeSubnetManager) handleUpdateLeaseEvent(oldObj, newObj interface{}) {
o := oldObj.(*v1.Node)
n := newObj.(*v1.Node)
if s, ok := n.Annotations[subnetKubeManagedAnnotation]; !ok || s != "true" {
if s, ok := n.Annotations[ksm.annotations.SubnetKubeManaged]; !ok || s != "true" {
return
}
if o.Annotations[backendDataAnnotation] == n.Annotations[backendDataAnnotation] &&
o.Annotations[backendTypeAnnotation] == n.Annotations[backendTypeAnnotation] &&
o.Annotations[backendPublicIPAnnotation] == n.Annotations[backendPublicIPAnnotation] {
if o.Annotations[ksm.annotations.BackendData] == n.Annotations[ksm.annotations.BackendData] &&
o.Annotations[ksm.annotations.BackendType] == n.Annotations[ksm.annotations.BackendType] &&
o.Annotations[ksm.annotations.BackendPublicIP] == n.Annotations[ksm.annotations.BackendPublicIP] {
return // No change to lease
}

l, err := nodeToLease(*n)
l, err := ksm.nodeToLease(*n)
if err != nil {
glog.Infof("Error turning node %q to lease: %v", n.ObjectMeta.Name, err)
return
Expand Down Expand Up @@ -248,24 +248,24 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
if err != nil {
return nil, err
}
if n.Annotations[backendDataAnnotation] != string(bd) ||
n.Annotations[backendTypeAnnotation] != attrs.BackendType ||
n.Annotations[backendPublicIPAnnotation] != attrs.PublicIP.String() ||
n.Annotations[subnetKubeManagedAnnotation] != "true" ||
(n.Annotations[backendPublicIPOverwriteAnnotation] != "" && n.Annotations[backendPublicIPOverwriteAnnotation] != attrs.PublicIP.String()) {
n.Annotations[backendTypeAnnotation] = attrs.BackendType
n.Annotations[backendDataAnnotation] = string(bd)
if n.Annotations[backendPublicIPOverwriteAnnotation] != "" {
if n.Annotations[backendPublicIPAnnotation] != n.Annotations[backendPublicIPOverwriteAnnotation] {
if n.Annotations[ksm.annotations.BackendData] != string(bd) ||
n.Annotations[ksm.annotations.BackendType] != attrs.BackendType ||
n.Annotations[ksm.annotations.BackendPublicIP] != attrs.PublicIP.String() ||
n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" ||
(n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != attrs.PublicIP.String()) {
n.Annotations[ksm.annotations.BackendType] = attrs.BackendType
n.Annotations[ksm.annotations.BackendData] = string(bd)
if n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" {
if n.Annotations[ksm.annotations.BackendPublicIP] != n.Annotations[ksm.annotations.BackendPublicIPOverwrite] {
glog.Infof("Overriding public ip with '%s' from node annotation '%s'",
n.Annotations[backendPublicIPOverwriteAnnotation],
backendPublicIPOverwriteAnnotation)
n.Annotations[backendPublicIPAnnotation] = n.Annotations[backendPublicIPOverwriteAnnotation]
n.Annotations[ksm.annotations.BackendPublicIPOverwrite],
ksm.annotations.BackendPublicIPOverwrite)
n.Annotations[ksm.annotations.BackendPublicIP] = n.Annotations[ksm.annotations.BackendPublicIPOverwrite]
}
} else {
n.Annotations[backendPublicIPAnnotation] = attrs.PublicIP.String()
n.Annotations[ksm.annotations.BackendPublicIP] = attrs.PublicIP.String()
}
n.Annotations[subnetKubeManagedAnnotation] = "true"
n.Annotations[ksm.annotations.SubnetKubeManaged] = "true"

oldData, err := json.Marshal(cachedNode)
if err != nil {
Expand Down Expand Up @@ -310,14 +310,14 @@ func (ksm *kubeSubnetManager) Run(ctx context.Context) {
ksm.nodeController.Run(ctx.Done())
}

func nodeToLease(n v1.Node) (l subnet.Lease, err error) {
l.Attrs.PublicIP, err = ip.ParseIP4(n.Annotations[backendPublicIPAnnotation])
func (ksm *kubeSubnetManager) nodeToLease(n v1.Node) (l subnet.Lease, err error) {
l.Attrs.PublicIP, err = ip.ParseIP4(n.Annotations[ksm.annotations.BackendPublicIP])
if err != nil {
return l, err
}

l.Attrs.BackendType = n.Annotations[backendTypeAnnotation]
l.Attrs.BackendData = json.RawMessage(n.Annotations[backendDataAnnotation])
l.Attrs.BackendType = n.Annotations[ksm.annotations.BackendType]
l.Attrs.BackendData = json.RawMessage(n.Annotations[ksm.annotations.BackendData])

_, cidr, err := net.ParseCIDR(n.Spec.PodCIDR)
if err != nil {
Expand Down

0 comments on commit f7c23a0

Please sign in to comment.