From b7a817da4cc73059d99e7c641e406a793e6aa75d Mon Sep 17 00:00:00 2001 From: MaiWJ <664647065@qq.com> Date: Thu, 23 Aug 2018 16:52:41 +0800 Subject: [PATCH] Windows Support Patch for https://github.com/containernetworking/plugins/pull/85 + Windows cni plugins are added (*) hostgw (l2bridge) (*) vxlan (overlay) + Windows netconf unit test + Fix appveyor config to run the test + Build Release support for windows plugins --- .appveyor.yml | 3 +- README.md | 3 + build.sh | 14 +- pkg/hns/endpoint_windows.go | 149 +++++++++++++ pkg/hns/netconf.go | 142 +++++++++++++ pkg/hns/netconf_suite_test.go | 26 +++ pkg/hns/netconf_test.go | 182 ++++++++++++++++ plugins/main/windows/build.sh | 14 ++ plugins/main/windows/l2bridge/README.md | 28 +++ .../main/windows/l2bridge/l2bridge_windows.go | 186 +++++++++++++++++ plugins/main/windows/l2bridge/sample.conf | 44 ++++ plugins/main/windows/overlay/README.md | 27 +++ .../main/windows/overlay/overlay_windows.go | 197 ++++++++++++++++++ plugins/main/windows/overlay/sample.conf | 37 ++++ plugins/meta/flannel/README.md | 47 +++++ plugins/meta/flannel/flannel.go | 32 ++- 16 files changed, 1126 insertions(+), 5 deletions(-) create mode 100644 pkg/hns/endpoint_windows.go create mode 100644 pkg/hns/netconf.go create mode 100644 pkg/hns/netconf_suite_test.go create mode 100644 pkg/hns/netconf_test.go create mode 100755 plugins/main/windows/build.sh create mode 100644 plugins/main/windows/l2bridge/README.md create mode 100644 plugins/main/windows/l2bridge/l2bridge_windows.go create mode 100755 plugins/main/windows/l2bridge/sample.conf create mode 100644 plugins/main/windows/overlay/README.md create mode 100644 plugins/main/windows/overlay/overlay_windows.go create mode 100755 plugins/main/windows/overlay/sample.conf diff --git a/.appveyor.yml b/.appveyor.yml index ea06455d3..1241f7aeb 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -6,9 +6,10 @@ environment: install: - echo %PATH% - echo %GOPATH% - - set PATH=%GOPATH%\bin;c:\go\bin;%PATH% - go version - go env + - ps: $webClient = New-Object System.Net.WebClient; $InstallPath="c:" ; $webClient.DownloadFile("https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/gcc.zip", "$InstallPath\gcc.zip"); Expand-Archive $InstallPath\gcc.zip -DestinationPath $InstallPath\gcc -Force; $webClient.DownloadFile("https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/runtime.zip", "$InstallPath\runtime.zip"); Expand-Archive $InstallPath\runtime.zip -DestinationPath $InstallPath\gcc -Force; $webClient.DownloadFile("https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/binutils.zip","$InstallPath\binutils.zip"); Expand-Archive $InstallPath\binutils.zip -DestinationPath $InstallPath\gcc -Force; + - set PATH=%GOPATH%\bin;c:\go\bin;c:\gcc\bin;%PATH% build: off diff --git a/README.md b/README.md index 864601763..0ddcae372 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,9 @@ Read [CONTRIBUTING](CONTRIBUTING.md) for build and test instructions. * `macvlan`: Creates a new MAC address, forwards all traffic to that to the container. * `ptp`: Creates a veth pair. * `vlan`: Allocates a vlan device. +#### Windows: windows specific +* `l2bridge`: Creates a bridge, adds the host and the container to it. +* `overlay`: Creates an overlay interface to the container ### IPAM: IP address allocation * `dhcp`: Runs a daemon on the host to make DHCP requests on behalf of the container diff --git a/build.sh b/build.sh index f55eb117a..6db861a5e 100755 --- a/build.sh +++ b/build.sh @@ -19,12 +19,20 @@ export GO="${GO:-go}" mkdir -p "${PWD}/bin" -echo "Building plugins" +echo "Building plugins ${GOOS}" PLUGINS="plugins/meta/* plugins/main/* plugins/ipam/* plugins/sample" for d in $PLUGINS; do if [ -d "$d" ]; then plugin="$(basename "$d")" - echo " $plugin" - $GO build -o "${PWD}/bin/$plugin" "$@" "$REPO_PATH"/$d + if [ $plugin == "windows" ] + then + if [ "$GOARCH" == "amd64" ] + then + GOOS=windows . $d/build.sh + fi + else + echo " $plugin" + $GO build -o "${PWD}/bin/$plugin" "$@" "$REPO_PATH"/$d + fi fi done diff --git a/pkg/hns/endpoint_windows.go b/pkg/hns/endpoint_windows.go new file mode 100644 index 000000000..0b098bb76 --- /dev/null +++ b/pkg/hns/endpoint_windows.go @@ -0,0 +1,149 @@ +// +build windows + +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hns + +import ( + "fmt" + "log" + "net" + "strings" + + "github.com/Microsoft/hcsshim" + "github.com/containernetworking/cni/pkg/types/current" +) + +// ConstructEndpointName constructs enpointId which is used to identify an endpoint from HNS +// There is a special consideration for netNs name here, which is required for Windows Server 1709 +// containerID is the Id of the container on which the endpoint is worked on +func ConstructEndpointName(containerID string, netNs string, networkName string) string { + if netNs != "" { + splits := strings.Split(netNs, ":") + if len(splits) == 2 { + containerID = splits[1] + } + } + epName := containerID + "_" + networkName + return epName +} + +// DeprovisionEndpoint removes an endpoint from the container by sending a Detach request to HNS +// For shared endpoint, ContainerDetach is used +// for removing the endpoint completely, HotDetachEndpoint is used +func DeprovisionEndpoint(epName string, netns string, containerID string) error { + hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epName) + if err != nil { + log.Printf("[win-cni] Failed to find endpoint %v, err:%v", epName, err) + return err + } + + if netns != "none" { + // Shared endpoint removal. Do not remove the endpoint. + err := hnsEndpoint.ContainerDetach(containerID) + if err != nil { + log.Printf("[win-cni] Failed to detach the container endpoint %v, err:%v", epName, err) + } + return nil + } + + err = hcsshim.HotDetachEndpoint(containerID, hnsEndpoint.Id) + if err != nil { + log.Printf("[win-cni] Failed to detach endpoint %v, err:%v", epName, err) + // Do not consider this as failure, else this would leak endpoints + } + + _, err = hnsEndpoint.Delete() + if err != nil { + log.Printf("[win-cni] Failed to delete endpoint %v, err:%v", epName, err) + // Do not return error + } + + return nil +} + +type EndpointMakerFunc func() (*hcsshim.HNSEndpoint, error) + +// ProvisionEndpoint provisions an endpoint to a container specified by containerID. +// If an endpoint already exists, the endpoint is reused. +// This call is idempotent +func ProvisionEndpoint(epName string, expectedNetworkId string, containerID string, makeEndpoint EndpointMakerFunc) (*hcsshim.HNSEndpoint, error) { + // check if endpoint already exists + createEndpoint := true + hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epName) + if hnsEndpoint != nil && hnsEndpoint.VirtualNetwork == expectedNetworkId { + log.Printf("[win-cni] Found existing endpoint %v", epName) + createEndpoint = false + } + + if createEndpoint { + if hnsEndpoint != nil { + _, err = hnsEndpoint.Delete() + if err != nil { + log.Printf("[win-cni] Failed to delete stale endpoint %v, err:%v", epName, err) + } + } + + if hnsEndpoint, err = makeEndpoint(); err != nil { + return nil, err + } + + if hnsEndpoint, err = hnsEndpoint.Create(); err != nil { + return nil, err + } + + } + + // hot attach + if err = hcsshim.HotAttachEndpoint(containerID, hnsEndpoint.Id); err != nil { + return nil, err + } + + return hnsEndpoint, nil +} + +// ConstructResult constructs the CNI result for the endpoint +func ConstructResult(hnsNetwork *hcsshim.HNSNetwork, hnsEndpoint *hcsshim.HNSEndpoint) (*current.Result, error) { + resultInterface := ¤t.Interface{ + Name: hnsEndpoint.Name, + Mac: hnsEndpoint.MacAddress, + } + _, ipSubnet, err := net.ParseCIDR(hnsNetwork.Subnets[0].AddressPrefix) + if err != nil { + return nil, err + } + + var ipVersion string + if ipv4 := hnsEndpoint.IPAddress.To4(); ipv4 != nil { + ipVersion = "4" + } else if ipv6 := hnsEndpoint.IPAddress.To16(); ipv6 != nil { + ipVersion = "6" + } else { + return nil, fmt.Errorf("[win-cni] The IPAddress of hnsEndpoint isn't a valid ipv4 or ipv6 Address.") + } + + resultIPConfig := ¤t.IPConfig{ + Version: ipVersion, + Address: net.IPNet{ + IP: hnsEndpoint.IPAddress, + Mask: ipSubnet.Mask}, + Gateway: net.ParseIP(hnsEndpoint.GatewayAddress), + } + result := ¤t.Result{} + result.Interfaces = []*current.Interface{resultInterface} + result.IPs = []*current.IPConfig{resultIPConfig} + + return result, nil +} diff --git a/pkg/hns/netconf.go b/pkg/hns/netconf.go new file mode 100644 index 000000000..47d7dc9d8 --- /dev/null +++ b/pkg/hns/netconf.go @@ -0,0 +1,142 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hns + +import ( + "encoding/json" + "github.com/containernetworking/cni/pkg/types" + "strings" +) + +// NetConf is the CNI spec +type NetConf struct { + types.NetConf + AdditionalArgs []policyArgument `json:"AdditionalArgs,omitempty"` +} + +type policyArgument struct { + Name string + Value map[string]interface{} +} + +// MarshalPolicies converts the Endpoint policies in AdditionalArgs +// to HNS specific policies as Json raw bytes +func (n *NetConf) MarshalPolicies() []json.RawMessage { + if n.AdditionalArgs == nil { + n.AdditionalArgs = []policyArgument{} + } + + var result []json.RawMessage + for _, policyArg := range n.AdditionalArgs { + if !strings.EqualFold(policyArg.Name, "EndpointPolicy") { + continue + } + if data, err := json.Marshal(policyArg.Value); err == nil { + result = append(result, data) + } + } + + return result +} + +// ApplyOutboundNatPolicy applies NAT Policy in VFP using HNS +// Simultaneously an exception is added for the network that has to be Nat'd +func (n *NetConf) ApplyOutboundNatPolicy(nwToNat string) { + if n.AdditionalArgs == nil { + n.AdditionalArgs = []policyArgument{} + } + + for _, policy := range n.AdditionalArgs { + if !strings.EqualFold(policy.Name, "EndpointPolicy") { + continue + } + + pv := policy.Value + if !hasKey(pv, "Type") { + continue + } + + if !strings.EqualFold(pv["Type"].(string), "OutBoundNAT") { + continue + } + + if !hasKey(pv, "ExceptionList") { + // add the exception since there weren't any + pv["ExceptionList"] = []interface{}{nwToNat} + return + } + + nets := pv["ExceptionList"].([]interface{}) + for _, net := range nets { + if net.(string) == nwToNat { + // found it - do nothing + return + } + } + + // its not in the list of exceptions, add it and we're done + pv["ExceptionList"] = append(nets, nwToNat) + return + } + + // didn't find the policy, add it + natEntry := policyArgument{ + Name: "EndpointPolicy", + Value: map[string]interface{}{ + "Type": "OutBoundNAT", + "ExceptionList": []interface{}{ + nwToNat, + }, + }, + } + + n.AdditionalArgs = append(n.AdditionalArgs, natEntry) +} + +// ApplyDefaultPAPolicy is used to configure a endpoint PA policy in HNS +func (n *NetConf) ApplyDefaultPAPolicy(paAddress string) { + if n.AdditionalArgs == nil { + n.AdditionalArgs = []policyArgument{} + } + + // if its already present, leave untouched + for _, policy := range n.AdditionalArgs { + if policy.Name == "EndpointPolicy" { + if hasKey(policy.Value, "PA") { + // found it, don't override + return + } + } + } + + // did not find, add it now + paPolicyData := map[string]interface{}{ + "Type": "PA", + "PA": paAddress, + } + paPolicy := &policyArgument{ + Name: "EndpointPolicy", + Value: paPolicyData, + } + + n.AdditionalArgs = append(n.AdditionalArgs, *paPolicy) + + return +} + +func hasKey(m map[string]interface{}, k string) bool { + _, ok := m[k] + return ok +} diff --git a/pkg/hns/netconf_suite_test.go b/pkg/hns/netconf_suite_test.go new file mode 100644 index 000000000..cc69e6fd5 --- /dev/null +++ b/pkg/hns/netconf_suite_test.go @@ -0,0 +1,26 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package hns + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" +) + +func TestHns(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "HNS NetConf Suite") +} diff --git a/pkg/hns/netconf_test.go b/pkg/hns/netconf_test.go new file mode 100644 index 000000000..69e7d3ea6 --- /dev/null +++ b/pkg/hns/netconf_test.go @@ -0,0 +1,182 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package hns + +import ( + "encoding/json" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("HNS NetConf", func() { + Describe("ApplyOutboundNatPolicy", func() { + Context("when not set by user", func() { + It("sets it by adding a policy", func() { + + // apply it + n := NetConf{} + n.ApplyOutboundNatPolicy("192.168.0.0/16") + + addlArgs := n.AdditionalArgs + Expect(addlArgs).Should(HaveLen(1)) + + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + + value := policy.Value + Expect(value).Should(HaveKey("Type")) + Expect(value).Should(HaveKey("ExceptionList")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + + exceptionList := value["ExceptionList"].([]interface{}) + Expect(exceptionList).Should(HaveLen(1)) + Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16")) + }) + }) + + Context("when set by user", func() { + It("appends exceptions to the existing policy", func() { + // first set it + n := NetConf{} + n.ApplyOutboundNatPolicy("192.168.0.0/16") + + // then attempt to update it + n.ApplyOutboundNatPolicy("10.244.0.0/16") + + // it should be unchanged! + addlArgs := n.AdditionalArgs + Expect(addlArgs).Should(HaveLen(1)) + + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + + value := policy.Value + Expect(value).Should(HaveKey("Type")) + Expect(value).Should(HaveKey("ExceptionList")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + + exceptionList := value["ExceptionList"].([]interface{}) + Expect(exceptionList).Should(HaveLen(2)) + Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16")) + Expect(exceptionList[1].(string)).Should(Equal("10.244.0.0/16")) + }) + }) + }) + + Describe("ApplyDefaultPAPolicy", func() { + Context("when not set by user", func() { + It("sets it by adding a policy", func() { + + n := NetConf{} + n.ApplyDefaultPAPolicy("192.168.0.1") + + addlArgs := n.AdditionalArgs + Expect(addlArgs).Should(HaveLen(1)) + + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + + value := policy.Value + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("PA")) + + paAddress := value["PA"].(string) + Expect(paAddress).Should(Equal("192.168.0.1")) + }) + }) + + Context("when set by user", func() { + It("does not override", func() { + n := NetConf{} + n.ApplyDefaultPAPolicy("192.168.0.1") + n.ApplyDefaultPAPolicy("192.168.0.2") + + addlArgs := n.AdditionalArgs + Expect(addlArgs).Should(HaveLen(1)) + + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + + value := policy.Value + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("PA")) + + paAddress := value["PA"].(string) + Expect(paAddress).Should(Equal("192.168.0.1")) + Expect(paAddress).ShouldNot(Equal("192.168.0.2")) + }) + }) + }) + + Describe("MarshalPolicies", func() { + Context("when not set by user", func() { + It("sets it by adding a policy", func() { + + n := NetConf{ + AdditionalArgs: []policyArgument{ + { + Name: "EndpointPolicy", + Value: map[string]interface{}{ + "someKey": "someValue", + }, + }, + { + Name: "someOtherType", + Value: map[string]interface{}{ + "someOtherKey": "someOtherValue", + }, + }, + }, + } + + result := n.MarshalPolicies() + Expect(len(result)).To(Equal(1)) + + var policy map[string]interface{} + err := json.Unmarshal(result[0], &policy) + Expect(err).ToNot(HaveOccurred()) + Expect(policy).Should(HaveKey("someKey")) + Expect(policy["someKey"]).To(Equal("someValue")) + }) + }) + + Context("when set by user", func() { + It("appends exceptions to the existing policy", func() { + // first set it + n := NetConf{} + n.ApplyOutboundNatPolicy("192.168.0.0/16") + + // then attempt to update it + n.ApplyOutboundNatPolicy("10.244.0.0/16") + + // it should be unchanged! + addlArgs := n.AdditionalArgs + Expect(addlArgs).Should(HaveLen(1)) + + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + + value := policy.Value + Expect(value).Should(HaveKey("Type")) + Expect(value).Should(HaveKey("ExceptionList")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + + exceptionList := value["ExceptionList"].([]interface{}) + Expect(exceptionList).Should(HaveLen(2)) + Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16")) + Expect(exceptionList[1].(string)).Should(Equal("10.244.0.0/16")) + }) + }) + }) +}) diff --git a/plugins/main/windows/build.sh b/plugins/main/windows/build.sh new file mode 100755 index 000000000..43cd9fcbf --- /dev/null +++ b/plugins/main/windows/build.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -e + +OUTDIR="${PWD}/bin/windows" +mkdir -p ${OUTDIR} + +PLUGINS="plugins/main/windows/*" +for d in $PLUGINS; do + if [ -d "$d" ]; then + plugin="$(basename "$d")" + echo " $plugin.exe" + $GO build -o "${PWD}/bin/$plugin" "$@" "$REPO_PATH"/$d + fi +done diff --git a/plugins/main/windows/l2bridge/README.md b/plugins/main/windows/l2bridge/README.md new file mode 100644 index 000000000..0e65fe371 --- /dev/null +++ b/plugins/main/windows/l2bridge/README.md @@ -0,0 +1,28 @@ +# L2Bridge plugin (Windows) + +## Overview + +With L2bridge plugin, all containers (on the same host) are plugged into an L2Bridge network that has one endpoint in the host namespace. + + +## Example configuration +``` +{ + "name": "mynet", + "type": "win-l2bridge", + "ipMasq": true, + "clusterNetworkPrefix": "10.244.0.0/16", + "ipam": { + "type": "host-local", + "subnet": "10.10.0.0/16" + } +} +``` + +## Network configuration reference + +* `name` (string, required): the name of the network. +* `type` (string, required): "win-l2bridge". +* `ipMasq` (string, optional): Set to true to setup NAT for the clusterNetworkPrefix. +* `clusterNetworkPrefix` (string, optional): Used to setup NAT if ipMasq is set to true. +* `ipam` (dictionary, required): IPAM configuration to be used for this network. diff --git a/plugins/main/windows/l2bridge/l2bridge_windows.go b/plugins/main/windows/l2bridge/l2bridge_windows.go new file mode 100644 index 000000000..b9ce873d2 --- /dev/null +++ b/plugins/main/windows/l2bridge/l2bridge_windows.go @@ -0,0 +1,186 @@ +// Copyright 2014 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "net" + "runtime" + + "github.com/Microsoft/hcsshim" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/containernetworking/cni/pkg/version" + "github.com/containernetworking/plugins/pkg/hns" + "github.com/containernetworking/plugins/pkg/ipam" + "strings" +) + +type NetConf struct { + hns.NetConf + + ipmasq bool `json:"ipmasq,omitempty"` + clusterNetworkPrefix net.IPNet `json:"clusterprefix,omitempty"` +} +type K8sCniEnvArgs struct { + types.CommonArgs + K8S_POD_NAMESPACE types.UnmarshallableString `json:"K8S_POD_NAMESPACE,omitempty"` + K8S_POD_NAME types.UnmarshallableString `json:"K8S_POD_NAME,omitempty"` + K8S_POD_INFRA_CONTAINER_ID types.UnmarshallableString `json:"K8S_POD_INFRA_CONTAINER_ID,omitempty"` +} + +func init() { + // this ensures that main runs only on main thread (thread group leader). + // since namespace ops (unshare, setns) are done for a single thread, we + // must ensure that the goroutine does not jump from OS thread to thread + runtime.LockOSThread() +} + +func parseCniArgs(args string) (*K8sCniEnvArgs, error) { + podConfig := K8sCniEnvArgs{} + err := types.LoadArgs(args, &podConfig) + if err != nil { + return nil, err + } + return &podConfig, nil +} + +func loadNetConf(bytes []byte) (*NetConf, string, error) { + n := &NetConf{} + if err := json.Unmarshal(bytes, n); err != nil { + return nil, "", fmt.Errorf("failed to load netconf: %v", err) + } + log.Printf("Loaded NetConf %v", n) + return n, n.CNIVersion, nil +} + +func cmdAdd(args *skel.CmdArgs) error { + log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", + args.ContainerID, args.Netns, args.IfName, args.Args, args.Path) + n, cniVersion, err := loadNetConf(args.StdinData) + if err != nil { + return err + } + cniargs, err := parseCniArgs(args.Args) + k8sNamespace := "default" + if err == nil { + k8sNamespace = string(cniargs.K8S_POD_NAMESPACE) + } + networkName := n.Name + hnsNetwork, err := hcsshim.GetHNSNetworkByName(networkName) + if err != nil { + return err + } + + if hnsNetwork == nil { + return fmt.Errorf("network %v not found", networkName) + } + + if !strings.EqualFold(hnsNetwork.Type, "L2Bridge") { + return fmt.Errorf("network %v is of an unexpected type: %v", networkName, hnsNetwork.Type) + } + + epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name) + + hnsEndpoint, err := hns.ProvisionEndpoint(epName, hnsNetwork.Id, args.ContainerID, func() (*hcsshim.HNSEndpoint, error) { + // run the IPAM plugin and get back the config to apply + r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData) + if err != nil { + return nil, err + } + + // Convert whatever the IPAM result was into the current Result type + result, err := current.NewResultFromResult(r) + if err != nil { + return nil, err + } + + if len(result.IPs) == 0 { + return nil, errors.New("IPAM plugin return is missing IP config") + } + + // Calculate gateway for bridge network (needs to be x.2) + gw := result.IPs[0].Address.IP.Mask(result.IPs[0].Address.Mask) + gw[len(gw)-1] += 2 + + // NAT based on the the configured cluster network + if n.ipmasq { + n.ApplyOutboundNatPolicy(n.clusterNetworkPrefix.String()) + } + + nameservers := strings.Join(n.DNS.Nameservers, ",") + if result.DNS.Nameservers != nil { + nameservers = strings.Join(result.DNS.Nameservers, ",") + } + + dnsSuffix := "" + if len(n.DNS.Search) > 0 { + dnsSuffix = k8sNamespace + "." + n.DNS.Search[0] + } + + hnsEndpoint := &hcsshim.HNSEndpoint{ + Name: epName, + VirtualNetwork: hnsNetwork.Id, + DNSServerList: nameservers, + DNSSuffix: dnsSuffix, + GatewayAddress: gw.String(), + IPAddress: result.IPs[0].Address.IP, + Policies: n.MarshalPolicies(), + } + + log.Printf("Adding Hns Endpoint %v", hnsEndpoint) + return hnsEndpoint, nil + }) + + if err != nil { + return err + } + + result, err := hns.ConstructResult(hnsNetwork, hnsEndpoint) + if err != nil { + return err + } + + return types.PrintResult(result, cniVersion) +} + +func cmdDel(args *skel.CmdArgs) error { + log.Printf("[cni-net] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", + args.ContainerID, args.Netns, args.IfName, args.Args, args.Path) + n, _, err := loadNetConf(args.StdinData) + if err != nil { + return err + } + + if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil { + return err + } + + if args.Netns == "" { + return nil + } + + epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name) + + return hns.DeprovisionEndpoint(epName, args.Netns, args.ContainerID) +} + +func main() { + skel.PluginMain(cmdAdd, cmdDel, version.All) +} diff --git a/plugins/main/windows/l2bridge/sample.conf b/plugins/main/windows/l2bridge/sample.conf new file mode 100755 index 000000000..89b83b355 --- /dev/null +++ b/plugins/main/windows/l2bridge/sample.conf @@ -0,0 +1,44 @@ +{ + "name": "cbr0", + "type": "flannel", + "delegate": { + "type": "l2bridge", + "dns": { + "Nameservers": [ + "11.0.0.10" + ], + "Search": [ + "svc.cluster.local" + ] + }, + "AdditionalArgs": [ + { + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "ExceptionList": [ + "192.168.0.0/16", + "11.0.0.0/8", + "10.137.196.0/23" + ] + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ROUTE", + "DestinationPrefix": "11.0.0.0/8", + "NeedEncap": true + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ROUTE", + "DestinationPrefix": "10.137.198.27/32", + "NeedEncap": true + } + } + ] + } +} diff --git a/plugins/main/windows/overlay/README.md b/plugins/main/windows/overlay/README.md new file mode 100644 index 000000000..2d70600d6 --- /dev/null +++ b/plugins/main/windows/overlay/README.md @@ -0,0 +1,27 @@ +# L2Bridge plugin (Windows) + +## Overview + +With Overlay plugin, all containers (on the same host) are plugged into an Overlay network based on VXLAN encapsulation. + + +## Example configuration +``` +{ + "name": "mynet", + "type": "win-overlay", + "ipMasq": true, + "endpointMacPrefix": "0E-2A", + "ipam": { + "type": "host-local", + "subnet": "10.10.0.0/16" + } +} +``` + +## Network configuration reference + +* `name` (string, required): the name of the network. +* `type` (string, required): "win-overlay". +* `endpointMacPrefix` (string, optional): required for vxlan mode, set to the MAC prefix configured for Flannel +* `ipam` (dictionary, required): IPAM configuration to be used for this network. diff --git a/plugins/main/windows/overlay/overlay_windows.go b/plugins/main/windows/overlay/overlay_windows.go new file mode 100644 index 000000000..56db9fa3c --- /dev/null +++ b/plugins/main/windows/overlay/overlay_windows.go @@ -0,0 +1,197 @@ +// Copyright 2014 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "runtime" + + "github.com/Microsoft/hcsshim" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/containernetworking/cni/pkg/version" + "github.com/containernetworking/plugins/pkg/hns" + "github.com/containernetworking/plugins/pkg/ipam" + "strings" +) + +type NetConf struct { + hns.NetConf + + IPMasq bool + endpointMacPrefix string `json:"endpointMacPrefix,omitempty"` +} + +type K8sCniEnvArgs struct { + types.CommonArgs + K8S_POD_NAMESPACE types.UnmarshallableString `json:"K8S_POD_NAMESPACE,omitempty"` + K8S_POD_NAME types.UnmarshallableString `json:"K8S_POD_NAME,omitempty"` + K8S_POD_INFRA_CONTAINER_ID types.UnmarshallableString `json:"K8S_POD_INFRA_CONTAINER_ID,omitempty"` +} + +func init() { + // this ensures that main runs only on main thread (thread group leader). + // since namespace ops (unshare, setns) are done for a single thread, we + // must ensure that the goroutine does not jump from OS thread to thread + runtime.LockOSThread() +} + +func parseCniArgs(args string) (*K8sCniEnvArgs, error) { + podConfig := K8sCniEnvArgs{} + err := types.LoadArgs(args, &podConfig) + if err != nil { + return nil, err + } + return &podConfig, nil +} + +func loadNetConf(bytes []byte) (*NetConf, string, error) { + n := &NetConf{} + if err := json.Unmarshal(bytes, n); err != nil { + return nil, "", fmt.Errorf("failed to load netconf: %v", err) + } + return n, n.CNIVersion, nil +} + +func cmdAdd(args *skel.CmdArgs) error { + log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", + args.ContainerID, args.Netns, args.IfName, args.Args, args.Path) + n, cniVersion, err := loadNetConf(args.StdinData) + if err != nil { + return err + } + + cniargs, err := parseCniArgs(args.Args) + k8sNamespace := "default" + if err == nil { + k8sNamespace = string(cniargs.K8S_POD_NAMESPACE) + } + + if n.endpointMacPrefix != "" { + if len(n.endpointMacPrefix) != 5 || n.endpointMacPrefix[2] != '-' { + return fmt.Errorf("endpointMacPrefix [%v] is invalid, value must be of the format xx-xx", n.endpointMacPrefix) + } + } else { + n.endpointMacPrefix = "0E-2A" + } + + networkName := n.Name + hnsNetwork, err := hcsshim.GetHNSNetworkByName(networkName) + if err != nil { + return fmt.Errorf("Error while GETHNSNewtorkByName(%v): %v", networkName, err) + } + + if hnsNetwork == nil { + return fmt.Errorf("network %v not found", networkName) + } + + if !strings.EqualFold(hnsNetwork.Type, "Overlay") { + return fmt.Errorf("network %v is of an unexpected type: %v", networkName, hnsNetwork.Type) + } + + epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name) + + hnsEndpoint, err := hns.ProvisionEndpoint(epName, hnsNetwork.Id, args.ContainerID, func() (*hcsshim.HNSEndpoint, error) { + // run the IPAM plugin and get back the config to apply + r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData) + if err != nil { + return nil, fmt.Errorf("Error while ipam.ExecAdd: %v", err) + } + + // Convert whatever the IPAM result was into the current Result type + result, err := current.NewResultFromResult(r) + if err != nil { + return nil, fmt.Errorf("Error while NewResultFromResult: %v", err) + } + + if len(result.IPs) == 0 { + return nil, errors.New("IPAM plugin return is missing IP config") + } + + ipAddr := result.IPs[0].Address.IP.To4() + // conjure a MAC based on the IP for Overlay + macAddr := fmt.Sprintf("%v-%02x-%02x-%02x-%02x", n.endpointMacPrefix, ipAddr[0], ipAddr[1], ipAddr[2], ipAddr[3]) + // use the HNS network gateway + gw := hnsNetwork.Subnets[0].GatewayAddress + n.ApplyDefaultPAPolicy(hnsNetwork.ManagementIP) + if n.IPMasq { + n.ApplyOutboundNatPolicy(hnsNetwork.Subnets[0].AddressPrefix) + } + + nameservers := strings.Join(n.DNS.Nameservers, ",") + if result.DNS.Nameservers != nil { + nameservers = strings.Join(result.DNS.Nameservers, ",") + } + + dnsSuffix := "" + if len(n.DNS.Search) > 0 { + dnsSuffix = k8sNamespace + "." + n.DNS.Search[0] + } + + hnsEndpoint := &hcsshim.HNSEndpoint{ + Name: epName, + VirtualNetwork: hnsNetwork.Id, + DNSServerList: nameservers, + DNSSuffix: dnsSuffix, + GatewayAddress: gw, + IPAddress: ipAddr, + MacAddress: macAddr, + Policies: n.MarshalPolicies(), + } + + log.Printf("Adding Hns Endpoint %v", hnsEndpoint) + return hnsEndpoint, nil + }) + + if err != nil { + return fmt.Errorf("Error while ProvisionEndpoint(%v,%v,%v) :%v", epName, hnsNetwork.Id, args.ContainerID, err) + } + + result, err := hns.ConstructResult(hnsNetwork, hnsEndpoint) + if err != nil { + return fmt.Errorf("Error while constructResult: %v", err) + } + + return types.PrintResult(result, cniVersion) +} + +func cmdDel(args *skel.CmdArgs) error { + log.Printf("[cni-net] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", + args.ContainerID, args.Netns, args.IfName, args.Args, args.Path) + n, _, err := loadNetConf(args.StdinData) + if err != nil { + return err + } + + if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil { + return err + } + + if args.Netns == "" { + return nil + } + + epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name) + + return hns.DeprovisionEndpoint(epName, args.Netns, args.ContainerID) +} + +func main() { + skel.PluginMain(cmdAdd, cmdDel, version.All) +} diff --git a/plugins/main/windows/overlay/sample.conf b/plugins/main/windows/overlay/sample.conf new file mode 100755 index 000000000..02aae089f --- /dev/null +++ b/plugins/main/windows/overlay/sample.conf @@ -0,0 +1,37 @@ +{ + "cniVersion": "0.2.0", + "name": "vxlan0", + "type": "flannel", + "delegate": { + "type": "overlay", + "dns": { + "Nameservers": [ + "11.0.0.10" + ], + "Search": [ + "svc.cluster.local" + ] + }, + "AdditionalArgs": [ + { + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "ExceptionList": [ + "192.168.0.0/16", + "11.0.0.0/8" + ] + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ROUTE", + "DestinationPrefix": "11.0.0.0/8", + "NeedEncap": true + } + } + ] + } +} + diff --git a/plugins/meta/flannel/README.md b/plugins/meta/flannel/README.md index 0efb69059..47a3ab1f9 100644 --- a/plugins/meta/flannel/README.md +++ b/plugins/meta/flannel/README.md @@ -86,3 +86,50 @@ flannel plugin will set the following fields in the delegated plugin configurati * `mtu`: `$FLANNEL_MTU` Additionally, for the bridge plugin, `isGateway` will be set to `true`, if not present. + +## Windows Support (Experimental) +This plugin supports delegating to the windows CNI plugins (overlay.exe, l2bridge.exe) to work in conjunction with [Flannel on Windows](https://github.com/coreos/flannel/issues/833). +Flannel sets up an [HNS Network](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-networking) in L2Bridge mode for host-gw and +in Overlay mode for vxlan. + +The following fields must be set in the delegated plugin configuration: +* `name` (string, required): the name of the network (must match the name in Flannel config / name of the HNS network) +* `type` (string, optional): set to win-l2bridge by default. Can be set to win-overlay or other custom windows CNI +* `endpointMacPrefix` (string, optional): required for vxlan mode, set to the MAC prefix configured for Flannel + +For host-gw, the Flannel CNI plugin will set: +* `ipam` (string, required): subnet to `$FLANNEL_SUBNET` and GW to the .2 address in the `$FLANNEL_SUBNET` (this is required by HNS). IPAM type is left empty to allow Windows HNS to do IPAM + +For vxlan, the Flannel CNI plugin will set: +* `ipam`: "host-local" type will be used with "subnet" set to `$FLANNEL_NETWORK` but limited to a range per `$FLANNEL_SUBNET` and gateway as the .1 address in `$FLANNEL_NETWORK` + +If IPMASQ is true, the Flannel CNI plugin will setup an OutBoundNAT policy and add FLANNEL_SUBNET to any existing exclusions. + +All other delegate config e.g. other HNS endpoint policis in AdditionalArgs will be passed to WINCNI as-is. + +Example VXLAN Flannel CNI config +``` +{ + "name": "vxlan0", + "type": "flannel", + "delegate": { + "type": "vxlan", + "endpointMacPrefix": "0E-2A" + } +} +``` + +For this example, Flannel CNI would generate the following config to delegate to the windows CNI when FLANNEL_NETWORK=10.244.0.0/16, FLANNEL_SUBNET=10.244.1.0/24 and IPMASQ=true +``` +{ + "name": "vxlan0", + "type": "win-overlay", + "endpointMacPrefix": "0E-2A", + "clusterNetworkPrefix": "10.244.0.0/16", + "IPMasq": true, + "ipam": { + "subnet": "10.244.1.0/24", + "type": "host-local" + } +} +``` \ No newline at end of file diff --git a/plugins/meta/flannel/flannel.go b/plugins/meta/flannel/flannel.go index 21190281e..46ac07da4 100644 --- a/plugins/meta/flannel/flannel.go +++ b/plugins/meta/flannel/flannel.go @@ -26,6 +26,7 @@ import ( "net" "os" "path/filepath" + "runtime" "strconv" "strings" @@ -202,6 +203,10 @@ func cmdAdd(args *skel.CmdArgs) error { } } + if runtime.GOOS == "windows" { + return cmdAddWindows(args.ContainerID, n, fenv) + } + n.Delegate["name"] = n.Name if !hasKey(n.Delegate, "type") { @@ -241,6 +246,32 @@ func cmdAdd(args *skel.CmdArgs) error { return delegateAdd(args.ContainerID, n.DataDir, n.Delegate) } +func cmdAddWindows(containerID string, n *NetConf, fenv *subnetEnv) error { + + n.Delegate["name"] = n.Name + + if !hasKey(n.Delegate, "type") { + n.Delegate["type"] = "l2bridge" + } + + // if flannel needs ipmasq - get the plugin to configure it + // (this is the opposite of how linux works - on linux the flannel daemon configure ipmasq) + n.Delegate["ipmasq"] = *fenv.ipmasq + n.Delegate["clusterNetworkPrefix"] = fenv.nw.String() + + n.Delegate["cniVersion"] = "0.2.0" + if n.CNIVersion != "" { + n.Delegate["cniVersion"] = n.CNIVersion + } + + n.Delegate["ipam"] = map[string]interface{}{ + "type": "host-local", + "subnet": fenv.sn.String(), + } + + return delegateAdd(containerID, n.DataDir, n.Delegate) +} + func cmdDel(args *skel.CmdArgs) error { nc, err := loadFlannelNetConf(args.StdinData) if err != nil { @@ -265,7 +296,6 @@ func cmdDel(args *skel.CmdArgs) error { } func main() { - // TODO: implement plugin version skel.PluginMain(cmdAdd, cmdGet, cmdDel, version.All, "TODO") }