From e2026a93cd6bab6f4f2f28055233705f861f6ff6 Mon Sep 17 00:00:00 2001 From: dougbtv Date: Mon, 16 Jul 2018 11:06:54 -0400 Subject: [PATCH] Default network readiness enhancement --- README.md | 16 ++++++++++++ multus/multus.go | 33 +++++++++++++++++++++++ multus/multus_test.go | 61 +++++++++++++++++++++++++++++++++++++++++++ types/conf.go | 10 +++++++ types/conf_test.go | 40 ++++++++++++++++++++++++++++ types/types.go | 3 +++ 6 files changed, 163 insertions(+) diff --git a/README.md b/README.md index f146641ae..13c00cd9f 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ * [Configuring Pod to use the CRD network objects](#configuring-pod-to-use-the-crd-network-objects) * [Verifying Pod network interfaces](#verifying-pod-network-interfaces) * [Using with Multus conf file](#using-with-multus-conf-file) + * [Default Network Readiness Checks](#default-network-readiness-checks) * [Testing Multus CNI](#testing-multus-cni) * [Multiple flannel networks](#multiple-flannel-networks) * [Configure Kubernetes with CNI](#configure-kubernetes-with-cni) @@ -444,6 +445,21 @@ Given the following network configuration: EOF ``` +## Default Network Readiness Checks + +You may wish for your "default network" (that is, the CNI plugin & its configuration you specify as your default delegate) to become ready before you attach networks with Multus. This is disabled by default and not used unless you add the readiness check option(s) to your CNI configuration file. + +For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, hence, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists. + +In this manner, you may prevent pods from crash looping, and instead wait for that default network to be ready. + +Two options are available to configure this functionality: + +* `defaultnetworkfile`: The path to a file whose existance denotes that the default network is ready. +* `defaultnetworkwaitseconds`: The number of seconds to wait for that file to become available. Defaults to 120 seconds. + +*NOTE*: If `defaultnetworkfile` is unset, or is an empty string, this functionality will be disabled, and is disabled by default. + ## Testing Multus CNI ### Multiple flannel networks diff --git a/multus/multus.go b/multus/multus.go index ad9fcd599..dce664766 100644 --- a/multus/multus.go +++ b/multus/multus.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "os" "path/filepath" + "time" "github.com/containernetworking/cni/pkg/invoke" "github.com/containernetworking/cni/pkg/skel" @@ -148,6 +149,34 @@ func delPlugins(exec invoke.Exec, argIfname string, delegates []*types.DelegateN return nil } +// Sits in a wait loop until a file indicating the readiness of the "default network" +// is present (or until a user-defined timeout is reached) +func waitForDefaultNetwork(indicatorFile string, waitSeconds int) error { + // If there's no file to wait for, then, this is essentially disabled. + if len(indicatorFile) > 0 { + attempts := 0 + found := false + // Sleep in a loop until we find that file. + for attempts < waitSeconds { + attempts++ + if _, err := os.Stat(indicatorFile); err == nil { + found = true + attempts = waitSeconds + } else { + time.Sleep(1 * time.Second) + } + } + + if !found { + return fmt.Errorf("Multus: Timeout (%v seconds) finding default network file: %v", waitSeconds, indicatorFile) + } + return nil + + } + + return nil +} + func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cnitypes.Result, error) { var nopodnet bool n, err := types.LoadNetConf(args.StdinData) @@ -155,6 +184,10 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn return nil, fmt.Errorf("err in loading netconf: %v", err) } + if err = waitForDefaultNetwork(n.DefaultNetworkFile, n.DefaultNetworkWaitSeconds); err != nil { + return nil, err + } + if n.Kubeconfig != "" { delegates, err := k8s.GetK8sNetwork(args, n.Kubeconfig, kubeClient, n.ConfDir) if err != nil { diff --git a/multus/multus_test.go b/multus/multus_test.go index 96ea0e7a2..ec57a1bb5 100644 --- a/multus/multus_test.go +++ b/multus/multus_test.go @@ -173,6 +173,8 @@ var _ = Describe("multus operations", func() { StdinData: []byte(`{ "name": "node-cni-network", "type": "multus", + "defaultnetworkfile": "/tmp/foo.multus.conf", + "defaultnetworkwaitseconds": 3, "delegates": [{ "name": "weave1", "cniVersion": "0.2.0", @@ -185,6 +187,10 @@ var _ = Describe("multus operations", func() { }`), } + // Touch the default network file. + configPath := "/tmp/foo.multus.conf" + os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755) + fExec := &fakeExec{} expectedResult1 := &types020.Result{ CNIVersion: "0.2.0", @@ -226,6 +232,13 @@ var _ = Describe("multus operations", func() { err = cmdDel(args, fExec, nil) Expect(err).NotTo(HaveOccurred()) Expect(fExec.delIndex).To(Equal(len(fExec.plugins))) + + // Cleanup default network file. + if _, errStat := os.Stat(configPath); errStat == nil { + errRemove := os.Remove(configPath) + Expect(errRemove).NotTo(HaveOccurred()) + } + }) It("executes delegates and kubernetes networks", func() { @@ -306,4 +319,52 @@ var _ = Describe("multus operations", func() { // plugin 1 is the masterplugin Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue()) }) + + It("times out waiting for default networks", func() { + args := &skel.CmdArgs{ + ContainerID: "123456789", + Netns: testNS.Path(), + IfName: "eth0", + StdinData: []byte(`{ + "name": "defaultnetwork", + "type": "multus", + "defaultnetworkfile": "/tmp/foo.multus.conf", + "defaultnetworkwaitseconds": 1, + "delegates": [{ + "name": "weave1", + "cniVersion": "0.2.0", + "type": "weave-net" + }] +}`), + } + + // Always remove the defaultnetworkfile + configPath := "/tmp/foo.multus.conf" + if _, errStat := os.Stat(configPath); errStat == nil { + errRemove := os.Remove(configPath) + Expect(errRemove).NotTo(HaveOccurred()) + } + + fExec := &fakeExec{} + expectedResult1 := &types020.Result{ + CNIVersion: "0.2.0", + IP4: &types020.IPConfig{ + IP: *testhelpers.EnsureCIDR("1.1.1.2/24"), + }, + } + expectedConf1 := `{ + "name": "weave1", + "cniVersion": "0.2.0", + "type": "weave-net" +}` + fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + + os.Setenv("CNI_COMMAND", "ADD") + os.Setenv("CNI_IFNAME", "eth0") + _, err := cmdAdd(args, fExec, nil) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("Multus: Timeout (1 seconds) finding default network file: /tmp/foo.multus.conf")) + + }) }) diff --git a/types/conf.go b/types/conf.go index 18b8fc33f..5cf2e9b23 100644 --- a/types/conf.go +++ b/types/conf.go @@ -26,6 +26,8 @@ import ( const ( defaultCNIDir = "/var/lib/cni/multus" defaultConfDir = "/etc/cni/multus/net.d" + defaultDefaultNetworkFile = "" + defaultDefaultNetworkWaitSeconds = 120 ) // Convert raw CNI JSON into a DelegateNetConf structure @@ -84,6 +86,14 @@ func LoadNetConf(bytes []byte) (*NetConf, error) { netconf.ConfDir = defaultConfDir } + if netconf.DefaultNetworkFile == "" { + netconf.DefaultNetworkFile = defaultDefaultNetworkFile + } + if netconf.DefaultNetworkWaitSeconds < 1 { + netconf.DefaultNetworkWaitSeconds = defaultDefaultNetworkWaitSeconds + } + + for idx, rawConf := range netconf.RawDelegates { bytes, err := json.Marshal(rawConf) if err != nil { diff --git a/types/conf_test.go b/types/conf_test.go index d6916b8d4..009c09114 100644 --- a/types/conf_test.go +++ b/types/conf_test.go @@ -81,4 +81,44 @@ var _ = Describe("config operations", func() { _, err := LoadNetConf([]byte(conf)) Expect(err).To(HaveOccurred()) }) + + It("has defaults set for network readiness", func() { + conf := `{ + "name": "defaultnetwork", + "type": "multus", + "kubeconfig": "/etc/kubernetes/kubelet.conf", + "delegates": [{ + "cniVersion": "0.3.0", + "name": "defaultnetwork", + "type": "flannel", + "isDefaultGateway": true + }] +}` + netConf, err := LoadNetConf([]byte(conf)) + Expect(err).NotTo(HaveOccurred()) + Expect(netConf.DefaultNetworkFile).To(Equal("")) + Expect(netConf.DefaultNetworkWaitSeconds).To(Equal(120)) + }) + + It("honors overrides for network readiness", func() { + conf := `{ + "name": "defaultnetwork", + "type": "multus", + "defaultnetworkfile": "/etc/cni/net.d/foo", + "defaultnetworkwaitseconds": 30, + "kubeconfig": "/etc/kubernetes/kubelet.conf", + "delegates": [{ + "cniVersion": "0.3.0", + "name": "defaultnetwork", + "type": "flannel", + "isDefaultGateway": true + }] +}` + netConf, err := LoadNetConf([]byte(conf)) + Expect(err).NotTo(HaveOccurred()) + Expect(netConf.DefaultNetworkFile).To(Equal("/etc/cni/net.d/foo")) + Expect(netConf.DefaultNetworkWaitSeconds).To(Equal(30)) + }) + + }) diff --git a/types/types.go b/types/types.go index 45056a704..b873afc42 100644 --- a/types/types.go +++ b/types/types.go @@ -38,6 +38,9 @@ type NetConf struct { RawDelegates []map[string]interface{} `json:"delegates"` Delegates []*DelegateNetConf `json:"-"` Kubeconfig string `json:"kubeconfig"` + // Default network readiness options + DefaultNetworkFile string `json:defaultnetworkfile` + DefaultNetworkWaitSeconds int `json:defaultnetworkwaitseconds` } type DelegateNetConf struct {