diff --git a/README.md b/README.md index e9d773069..e00504a02 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ * [Verifying Pod network interfaces](#verifying-pod-network-interfaces) * [Using with Multus conf file](#using-with-multus-conf-file) * [Logging Options](#logging-options) + * [Default Network Readiness Checks](#default-network-readiness-checks) * [Testing Multus CNI](#testing-multus-cni) * [Multiple flannel networks](#multiple-flannel-networks) * [Configure Kubernetes with CNI](#configure-kubernetes-with-cni) @@ -490,6 +491,21 @@ You may configure the logging level by using the `LogLevel` option in your CNI c "LogLevel": "debug", ``` +## Default Network Readiness Checks + +You may wish for your "default network" (that is, the CNI plugin & its configuration you specify as your default delegate) to become ready before you attach networks with Multus. This is disabled by default and not used unless you add the readiness check option(s) to your CNI configuration file. + +For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, hence, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists. + +In this manner, you may prevent pods from crash looping, and instead wait for that default network to be ready. + +Two options are available to configure this functionality: + +* `defaultnetworkfile`: The path to a file whose existance denotes that the default network is ready. +* `defaultnetworkwaitseconds`: The number of seconds to wait for that file to become available. Defaults to 120 seconds. + +*NOTE*: If `defaultnetworkfile` is unset, or is an empty string, this functionality will be disabled, and is disabled by default. + ## Testing Multus CNI ### Multiple flannel networks diff --git a/multus/multus.go b/multus/multus.go index 591c9ffac..f1e141f9d 100644 --- a/multus/multus.go +++ b/multus/multus.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "os" "path/filepath" + "k8s.io/client-go/util/retry" "github.com/containernetworking/cni/libcni" "github.com/containernetworking/cni/pkg/invoke" @@ -204,6 +205,34 @@ func delPlugins(exec invoke.Exec, argIfname string, delegates []*types.DelegateN return nil } +// Sits in a wait loop until a file indicating the readiness of the "default network" +// is present (or until a user-defined timeout is reached) +func waitForDefaultNetwork(indicatorFile string, waitSeconds int) error { + // If there's no file to wait for, then, this is essentially disabled. + if len(indicatorFile) > 0 { + attempts := 0 + found := false + // Sleep in a loop until we find that file. + for attempts < waitSeconds { + attempts++ + if _, err := os.Stat(indicatorFile); err == nil { + found = true + attempts = waitSeconds + } else { + time.Sleep(1 * time.Second) + } + } + + if !found { + return fmt.Errorf("Multus: Timeout (%v seconds) finding default network file: %v", waitSeconds, indicatorFile) + } + return nil + + } + + return nil +} + func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cnitypes.Result, error) { n, err := types.LoadNetConf(args.StdinData) if err != nil { @@ -215,6 +244,10 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn return nil, fmt.Errorf("Multus: Err in getting k8s args: %v", err) } + if err = waitForDefaultNetwork(n.DefaultNetworkFile, n.DefaultNetworkWaitSeconds); err != nil { + return nil, err + } + numK8sDelegates, kc, err := k8s.TryLoadK8sDelegates(k8sArgs, n, kubeClient) if err != nil { return nil, fmt.Errorf("Multus: Err in loading K8s Delegates k8s args: %v", err) diff --git a/multus/multus_test.go b/multus/multus_test.go index 35cad0255..fb43978b8 100644 --- a/multus/multus_test.go +++ b/multus/multus_test.go @@ -173,6 +173,8 @@ var _ = Describe("multus operations", func() { StdinData: []byte(`{ "name": "node-cni-network", "type": "multus", + "defaultnetworkfile": "/tmp/foo.multus.conf", + "defaultnetworkwaitseconds": 3, "delegates": [{ "name": "weave1", "cniVersion": "0.2.0", @@ -185,6 +187,10 @@ var _ = Describe("multus operations", func() { }`), } + // Touch the default network file. + configPath := "/tmp/foo.multus.conf" + os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755) + fExec := &fakeExec{} expectedResult1 := &types020.Result{ CNIVersion: "0.2.0", @@ -226,6 +232,13 @@ var _ = Describe("multus operations", func() { err = cmdDel(args, fExec, nil) Expect(err).NotTo(HaveOccurred()) Expect(fExec.delIndex).To(Equal(len(fExec.plugins))) + + // Cleanup default network file. + if _, errStat := os.Stat(configPath); errStat == nil { + errRemove := os.Remove(configPath) + Expect(errRemove).NotTo(HaveOccurred()) + } + }) It("executes delegates and kubernetes networks", func() { @@ -306,4 +319,52 @@ var _ = Describe("multus operations", func() { // plugin 1 is the masterplugin Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue()) }) + + It("times out waiting for default networks", func() { + args := &skel.CmdArgs{ + ContainerID: "123456789", + Netns: testNS.Path(), + IfName: "eth0", + StdinData: []byte(`{ + "name": "defaultnetwork", + "type": "multus", + "defaultnetworkfile": "/tmp/foo.multus.conf", + "defaultnetworkwaitseconds": 1, + "delegates": [{ + "name": "weave1", + "cniVersion": "0.2.0", + "type": "weave-net" + }] +}`), + } + + // Always remove the defaultnetworkfile + configPath := "/tmp/foo.multus.conf" + if _, errStat := os.Stat(configPath); errStat == nil { + errRemove := os.Remove(configPath) + Expect(errRemove).NotTo(HaveOccurred()) + } + + fExec := &fakeExec{} + expectedResult1 := &types020.Result{ + CNIVersion: "0.2.0", + IP4: &types020.IPConfig{ + IP: *testhelpers.EnsureCIDR("1.1.1.2/24"), + }, + } + expectedConf1 := `{ + "name": "weave1", + "cniVersion": "0.2.0", + "type": "weave-net" +}` + fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + + os.Setenv("CNI_COMMAND", "ADD") + os.Setenv("CNI_IFNAME", "eth0") + _, err := cmdAdd(args, fExec, nil) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("Multus: Timeout (1 seconds) finding default network file: /tmp/foo.multus.conf")) + + }) }) diff --git a/types/conf.go b/types/conf.go index 6fcc92124..996c90d7e 100644 --- a/types/conf.go +++ b/types/conf.go @@ -31,6 +31,8 @@ const ( defaultCNIDir = "/var/lib/cni/multus" defaultConfDir = "/etc/cni/multus/net.d" defaultBinDir = "/opt/cni/bin" + defaultDefaultNetworkFile = "" + defaultDefaultNetworkWaitSeconds = 120 ) func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error { @@ -179,6 +181,14 @@ func LoadNetConf(bytes []byte) (*NetConf, error) { netconf.BinDir = defaultBinDir } + if netconf.DefaultNetworkFile == "" { + netconf.DefaultNetworkFile = defaultDefaultNetworkFile + } + + if netconf.DefaultNetworkWaitSeconds < 1 { + netconf.DefaultNetworkWaitSeconds = defaultDefaultNetworkWaitSeconds + } + for idx, rawConf := range netconf.RawDelegates { bytes, err := json.Marshal(rawConf) if err != nil { diff --git a/types/conf_test.go b/types/conf_test.go index fa64a3ddb..1260b4862 100644 --- a/types/conf_test.go +++ b/types/conf_test.go @@ -81,4 +81,44 @@ var _ = Describe("config operations", func() { _, err := LoadNetConf([]byte(conf)) Expect(err).To(HaveOccurred()) }) + + It("has defaults set for network readiness", func() { + conf := `{ + "name": "defaultnetwork", + "type": "multus", + "kubeconfig": "/etc/kubernetes/kubelet.conf", + "delegates": [{ + "cniVersion": "0.3.0", + "name": "defaultnetwork", + "type": "flannel", + "isDefaultGateway": true + }] +}` + netConf, err := LoadNetConf([]byte(conf)) + Expect(err).NotTo(HaveOccurred()) + Expect(netConf.DefaultNetworkFile).To(Equal("")) + Expect(netConf.DefaultNetworkWaitSeconds).To(Equal(120)) + }) + + It("honors overrides for network readiness", func() { + conf := `{ + "name": "defaultnetwork", + "type": "multus", + "defaultnetworkfile": "/etc/cni/net.d/foo", + "defaultnetworkwaitseconds": 30, + "kubeconfig": "/etc/kubernetes/kubelet.conf", + "delegates": [{ + "cniVersion": "0.3.0", + "name": "defaultnetwork", + "type": "flannel", + "isDefaultGateway": true + }] +}` + netConf, err := LoadNetConf([]byte(conf)) + Expect(err).NotTo(HaveOccurred()) + Expect(netConf.DefaultNetworkFile).To(Equal("/etc/cni/net.d/foo")) + Expect(netConf.DefaultNetworkWaitSeconds).To(Equal(30)) + }) + + }) diff --git a/types/types.go b/types/types.go index 20ed055ea..54ae40b36 100644 --- a/types/types.go +++ b/types/types.go @@ -42,6 +42,9 @@ type NetConf struct { Kubeconfig string `json:"kubeconfig"` LogFile string `json:"logFile"` LogLevel string `json:"logLevel"` + // Default network readiness options + DefaultNetworkFile string `json:defaultnetworkfile` + DefaultNetworkWaitSeconds int `json:defaultnetworkwaitseconds` } type NetworkStatus struct {