From a6cb2b7b6d282f09bf3e7d2ed6b889e97a6ec0fd Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Sun, 22 Nov 2020 12:06:09 +0100 Subject: [PATCH 1/5] CI podman IPv6 multinode don't test podman HA clusters podman ci for stable and testing versions fix ci --- .github/workflows/podman.yml | 147 +++++++++++++++++++---------------- 1 file changed, 79 insertions(+), 68 deletions(-) diff --git a/.github/workflows/podman.yml b/.github/workflows/podman.yml index 67543db062..7768e0a941 100644 --- a/.github/workflows/podman.yml +++ b/.github/workflows/podman.yml @@ -7,7 +7,6 @@ on: - master jobs: - podman: name: Podman runs-on: ubuntu-latest @@ -17,85 +16,97 @@ jobs: matrix: ipFamily: [ipv4, ipv6] deployment: [singleNode, multiNode] - exclude: - # exclude IPv6 and multinode - - ipFamily: ipv6 - - deployment: multiNode env: JOB_NAME: "podman-${{ matrix.deployment }}-${{ matrix.ipFamily }}" KIND_EXPERIMENTAL_PROVIDER: "podman" IP_FAMILY: ${{ matrix.ipFamily }} + PODMAN_VERSION: "stable" steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Verify + run: make verify + + - name: Install kind + run: sudo make install INSTALL_DIR=/usr/local/bin - - name: Check out code into the Go module directory - uses: actions/checkout@v2 - with: - fetch-depth: 0 + - name: Install kubectl + run: | + curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl - - name: Verify - run: make verify + - name: Enable ipv4 and ipv6 forwarding + run: | + sudo sysctl -w net.ipv6.conf.all.forwarding=1 + sudo sysctl -w net.ipv4.ip_forward=1 - - name: Install kind - run: sudo make install INSTALL_DIR=/usr/local/bin + - name: Install podman + run: | + . /etc/os-release + echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/${PODMAN_VERSION}/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list + curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/${PODMAN_VERSION}/xUbuntu_${VERSION_ID}/Release.key | sudo apt-key add - + sudo apt-get update -qq + sudo apt-get -qq -y install podman + # Install network + sudo mkdir -p /etc/cni/net.d + curl -qsSL https://raw.githubusercontent.com/containers/libpod/master/cni/87-podman-bridge.conflist | sudo tee /etc/cni/net.d/87-podman-bridge.conf + curl -qsSL https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz --output /tmp/cni.tgz + sudo mkdir -p /usr/libexec/cni + sudo tar -C /usr/libexec/cni -xvzf /tmp/cni.tgz - - name: Install podman - run: | - . /etc/os-release - echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list - curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key | sudo apt-key add - - sudo apt-get update -qq - sudo apt-get -qq -y install podman - # Install network - sudo mkdir -p /etc/cni/net.d - curl -qsSL https://raw.githubusercontent.com/containers/libpod/master/cni/87-podman-bridge.conflist | sudo tee /etc/cni/net.d/87-podman-bridge.conf - curl -qsSL https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz --output /tmp/cni.tgz - sudo mkdir -p /usr/libexec/cni - sudo tar -C /usr/libexec/cni -xvzf /tmp/cni.tgz + - name: Create single node cluster + if: ${{ matrix.deployment == 'singleNode' }} + run: | + cat < Date: Sun, 22 Nov 2020 12:15:01 +0100 Subject: [PATCH 2/5] fix ipv6 log error to consider both providers --- pkg/cluster/internal/create/actions/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/internal/create/actions/config/config.go b/pkg/cluster/internal/create/actions/config/config.go index 98eba4c80c..1d273b1e85 100644 --- a/pkg/cluster/internal/create/actions/config/config.go +++ b/pkg/cluster/internal/create/actions/config/config.go @@ -201,7 +201,7 @@ func getKubeadmConfig(cfg *config.Cluster, data kubeadm.ConfigData, node nodes.N // configure the right protocol addresses if cfg.Networking.IPFamily == "ipv6" { if nodeAddressIPv6 == "" { - return "", errors.Errorf("failed to get IPV6 address; is the docker daemon configured to use IPV6 correctly?") + return "", errors.Errorf("failed to get IPV6 address; is the container provider (docker,podman) configured to use IPV6 correctly?") } data.NodeAddress = nodeAddressIPv6 } From e6078b4eed650a7adb89a70bdf1c03bee917fd6a Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Sun, 22 Nov 2020 12:54:46 +0100 Subject: [PATCH 3/5] podman user defined network implement user defined networks in podman --- .../internal/providers/podman/network.go | 135 ++++++++++++++++++ pkg/cluster/internal/providers/podman/node.go | 2 +- .../internal/providers/podman/provider.go | 23 +-- .../internal/providers/podman/provision.go | 11 +- 4 files changed, 156 insertions(+), 15 deletions(-) create mode 100644 pkg/cluster/internal/providers/podman/network.go diff --git a/pkg/cluster/internal/providers/podman/network.go b/pkg/cluster/internal/providers/podman/network.go new file mode 100644 index 0000000000..e08e6f01de --- /dev/null +++ b/pkg/cluster/internal/providers/podman/network.go @@ -0,0 +1,135 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podman + +import ( + "crypto/sha1" + "encoding/binary" + "net" + "regexp" + "strings" + + "sigs.k8s.io/kind/pkg/errors" + "sigs.k8s.io/kind/pkg/exec" +) + +// This may be overridden by KIND_EXPERIMENTAL_PODMAN_NETWORK env, +// experimentally... +// +// By default currently picking a single network is equivalent to the previous +// behavior *except* that we moved from the default bridge to a user defined +// network because the default bridge is actually special versus any other +// docker network and lacks the emebdded DNS +// +// For now this also makes it easier for apps to join the same network, and +// leaves users with complex networking desires to create and manage their own +// networks. +const fixedNetworkName = "kind" + +// ensureNetwork creates a new network +// podman only creates IPv6 networks for versions >= 2.2.0 +func ensureNetwork(name string) error { + // network already exists + if checkIfNetworkExists(name) { + return nil + } + + // generate unique subnet per network based on the name + // obtained from the ULA fc00::/8 range + // Make N attempts with "probing" in case we happen to collide + subnet := generateULASubnetFromName(name, 0) + err := createNetwork(name, subnet) + if err == nil { + // Success! + return nil + } + + if isUnknownIPv6FlagError(err) { + return createNetwork(name, "") + } + + // Only continue if the error is because of the subnet range + // is already allocated + if !isPoolOverlapError(err) { + return err + } + + // keep trying for ipv6 subnets + const maxAttempts = 5 + for attempt := int32(1); attempt < maxAttempts; attempt++ { + subnet := generateULASubnetFromName(name, attempt) + err = createNetwork(name, subnet) + if err == nil { + // success! + return nil + } else if !isPoolOverlapError(err) { + // unknown error ... + return err + } + } + return errors.New("exhausted attempts trying to find a non-overlapping subnet") + +} + +func createNetwork(name, ipv6Subnet string) error { + if ipv6Subnet == "" { + return exec.Command("podman", "network", "create", "-d=bridge", name).Run() + } + return exec.Command("podman", "network", "create", "-d=bridge", + "--ipv6", "--subnet", ipv6Subnet, name).Run() +} + +func checkIfNetworkExists(name string) bool { + _, err := exec.Output(exec.Command( + "podman", "network", "inspect", + regexp.QuoteMeta(name), + )) + return err == nil +} + +func isUnknownIPv6FlagError(err error) bool { + rerr := exec.RunErrorForError(err) + return rerr != nil && + strings.Contains(string(rerr.Output), "unknown flag: --ipv6") +} + +func isPoolOverlapError(err error) bool { + rerr := exec.RunErrorForError(err) + return rerr != nil && + (strings.Contains(string(rerr.Output), "is being used by a network interface") || + strings.Contains(string(rerr.Output), "is already being used by a cni configuration")) +} + +// generateULASubnetFromName generate an IPv6 subnet based on the +// name and Nth probing attempt +func generateULASubnetFromName(name string, attempt int32) string { + ip := make([]byte, 16) + ip[0] = 0xfc + ip[1] = 0x00 + h := sha1.New() + _, _ = h.Write([]byte(name)) + _ = binary.Write(h, binary.LittleEndian, attempt) + bs := h.Sum(nil) + for i := 2; i < 8; i++ { + ip[i] = bs[i] + } + subnet := &net.IPNet{ + IP: net.IP(ip), + Mask: net.CIDRMask(64, 128), + } + return subnet.String() +} diff --git a/pkg/cluster/internal/providers/podman/node.go b/pkg/cluster/internal/providers/podman/node.go index 4a530f08b2..5285dd2244 100644 --- a/pkg/cluster/internal/providers/podman/node.go +++ b/pkg/cluster/internal/providers/podman/node.go @@ -53,7 +53,7 @@ func (n *node) Role() (string, error) { func (n *node) IP() (ipv4 string, ipv6 string, err error) { // retrieve the IP address of the node using podman inspect cmd := exec.Command("podman", "inspect", - "-f", "{{.NetworkSettings.IPAddress}},{{.NetworkSettings.GlobalIPv6Address}}", + "-f", "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}", n.name, // ... against the "node" container ) lines, err := exec.OutputLines(cmd) diff --git a/pkg/cluster/internal/providers/podman/provider.go b/pkg/cluster/internal/providers/podman/provider.go index 65c35d7047..83d0e78765 100644 --- a/pkg/cluster/internal/providers/podman/provider.go +++ b/pkg/cluster/internal/providers/podman/provider.go @@ -79,13 +79,24 @@ func (p *provider) Provision(status *cli.Status, cfg *config.Cluster) (err error return err } + // ensure the pre-requesite network exists + networkName := fixedNetworkName + if n := os.Getenv("KIND_EXPERIMENTAL_PODMAN_NETWORK"); n != "" { + p.logger.Warn("WARNING: Overriding podman network due to KIND_EXPERIMENTAL_PODMAN_NETWORK") + p.logger.Warn("WARNING: Here be dragons! This is not supported currently.") + networkName = n + } + if err := ensureNetwork(networkName); err != nil { + return errors.Wrap(err, "failed to ensure podman network") + } + // actually provision the cluster icons := strings.Repeat("📦 ", len(cfg.Nodes)) status.Start(fmt.Sprintf("Preparing nodes %s", icons)) defer func() { status.End(err == nil) }() // plan creating the containers - createContainerFuncs, err := planCreation(cfg) + createContainerFuncs, err := planCreation(cfg, networkName) if err != nil { return err } @@ -247,14 +258,8 @@ func (p *provider) GetAPIServerInternalEndpoint(cluster string) (string, error) if err != nil { return "", errors.Wrap(err, "failed to get apiserver endpoint") } - // TODO: check cluster IP family and return the correct IP - // This means IPv6 singlestack is broken on podman - ipv4, _, err := n.IP() - if err != nil { - return "", errors.Wrap(err, "failed to get apiserver IP") - } - return net.JoinHostPort(ipv4, fmt.Sprintf("%d", common.APIServerInternalPort)), nil - + // NOTE: we're using the nodes's hostnames which are their names + return net.JoinHostPort(n.String(), fmt.Sprintf("%d", common.APIServerInternalPort)), nil } // node returns a new node handle for this provider diff --git a/pkg/cluster/internal/providers/podman/provision.go b/pkg/cluster/internal/providers/podman/provision.go index 8e4d75ffef..47577d3e56 100644 --- a/pkg/cluster/internal/providers/podman/provision.go +++ b/pkg/cluster/internal/providers/podman/provision.go @@ -32,10 +32,10 @@ import ( ) // planCreation creates a slice of funcs that will create the containers -func planCreation(cfg *config.Cluster) (createContainerFuncs []func() error, err error) { +func planCreation(cfg *config.Cluster, networkName string) (createContainerFuncs []func() error, err error) { // these apply to all container creation nodeNamer := common.MakeNodeNamer(cfg.Name) - genericArgs, err := commonArgs(cfg) + genericArgs, err := commonArgs(cfg, networkName) if err != nil { return nil, err } @@ -135,11 +135,12 @@ func clusterHasImplicitLoadBalancer(cfg *config.Cluster) bool { } // commonArgs computes static arguments that apply to all containers -func commonArgs(cfg *config.Cluster) ([]string, error) { +func commonArgs(cfg *config.Cluster, networkName string) ([]string, error) { // standard arguments all nodes containers need, computed once args := []string{ - "--detach", // run the container detached - "--tty", // allocate a tty for entrypoint logs + "--detach", // run the container detached + "--tty", // allocate a tty for entrypoint logs + "--net", networkName, // attach to its own network // label the node with the cluster ID "--label", fmt.Sprintf("%s=%s", clusterLabelKey, cfg.Name), } From 50d40e364e06cc633be3dcf5e0de17bcd0e9f784 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Tue, 1 Dec 2020 15:37:01 +0100 Subject: [PATCH 4/5] podman hostport info for 2.2.0+ --- .../internal/providers/podman/provider.go | 123 +++++++++++------- 1 file changed, 78 insertions(+), 45 deletions(-) diff --git a/pkg/cluster/internal/providers/podman/provider.go b/pkg/cluster/internal/providers/podman/provider.go index 83d0e78765..8cdd1519db 100644 --- a/pkg/cluster/internal/providers/podman/provider.go +++ b/pkg/cluster/internal/providers/podman/provider.go @@ -26,6 +26,7 @@ import ( "strings" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/version" "sigs.k8s.io/kind/pkg/cluster/nodes" "sigs.k8s.io/kind/pkg/cluster/nodeutils" @@ -185,13 +186,79 @@ func (p *provider) GetAPIServerEndpoint(cluster string) (string, error) { return "", errors.Wrap(err, "failed to get api server endpoint") } - // retrieve the specific port mapping using podman inspect + // TODO: get rid of this once podman settles on how to get the port mapping using podman inspect + // This is only used to get the Kubeconfig server field + v, err := getPodmanVersion() + if err != nil { + return "", errors.Wrap(err, "failed to check podman version") + } + if v.LessThan(version.MustParseSemantic("2.2.0")) { + cmd := exec.Command( + "podman", "inspect", + "--format", + "{{ json .NetworkSettings.Ports }}", + n.String(), + ) + lines, err := exec.OutputLines(cmd) + if err != nil { + return "", errors.Wrap(err, "failed to get api server port") + } + if len(lines) != 1 { + return "", errors.Errorf("network details should only be one line, got %d lines", len(lines)) + } + + // portMapping19 maps to the standard CNI portmapping capability used in podman 1.9 + // see: https://github.com/containernetworking/cni/blob/spec-v0.4.0/CONVENTIONS.md + type portMapping19 struct { + HostPort int32 `json:"hostPort"` + ContainerPort int32 `json:"containerPort"` + Protocol string `json:"protocol"` + HostIP string `json:"hostIP"` + } + // portMapping20 maps to the podman 2.0 portmap type + // see: https://github.com/containers/podman/blob/05988fc74fc25f2ad2256d6e011dfb7ad0b9a4eb/libpod/define/container_inspect.go#L134-L143 + type portMapping20 struct { + HostPort string `json:"HostPort"` + HostIP string `json:"HostIp"` + } + + portMappings20 := make(map[string][]portMapping20) + if err := json.Unmarshal([]byte(lines[0]), &portMappings20); err == nil { + for k, v := range portMappings20 { + protocol := "tcp" + parts := strings.Split(k, "/") + if len(parts) == 2 { + protocol = strings.ToLower(parts[1]) + } + containerPort, err := strconv.Atoi(parts[0]) + if err != nil { + return "", err + } + for _, pm := range v { + if containerPort == common.APIServerInternalPort && protocol == "tcp" { + return net.JoinHostPort(pm.HostIP, pm.HostPort), nil + } + } + } + } + var portMappings19 []portMapping19 + if err := json.Unmarshal([]byte(lines[0]), &portMappings19); err != nil { + return "", errors.Errorf("invalid network details: %v", err) + } + for _, pm := range portMappings19 { + if pm.ContainerPort == common.APIServerInternalPort && pm.Protocol == "tcp" { + return net.JoinHostPort(pm.HostIP, strconv.Itoa(int(pm.HostPort))), nil + } + } + } + // TODO: hack until https://github.com/containers/podman/issues/8444 is resolved cmd := exec.Command( "podman", "inspect", "--format", - "{{ json .NetworkSettings.Ports }}", + "{{range .NetworkSettings.Ports }}{{range .}}{{.HostIP}}/{{.HostPort}}{{end}}{{end}}", n.String(), ) + lines, err := exec.OutputLines(cmd) if err != nil { return "", errors.Wrap(err, "failed to get api server port") @@ -199,52 +266,18 @@ func (p *provider) GetAPIServerEndpoint(cluster string) (string, error) { if len(lines) != 1 { return "", errors.Errorf("network details should only be one line, got %d lines", len(lines)) } - - // portMapping19 maps to the standard CNI portmapping capability used in podman 1.9 - // see: https://github.com/containernetworking/cni/blob/spec-v0.4.0/CONVENTIONS.md - type portMapping19 struct { - HostPort int32 `json:"hostPort"` - ContainerPort int32 `json:"containerPort"` - Protocol string `json:"protocol"` - HostIP string `json:"hostIP"` + // output is in the format IP/Port + parts := strings.Split(strings.TrimSpace(lines[0]), "/") + if len(parts) != 2 { + return "", errors.Errorf("network details should be in the format IP/Port, received: %s", parts) } - // portMapping20 maps to the podman 2.0 portmap type - // see: https://github.com/containers/podman/blob/05988fc74fc25f2ad2256d6e011dfb7ad0b9a4eb/libpod/define/container_inspect.go#L134-L143 - type portMapping20 struct { - HostPort string `json:"HostPort"` - HostIP string `json:"HostIp"` - } - - portMappings20 := make(map[string][]portMapping20) - if err := json.Unmarshal([]byte(lines[0]), &portMappings20); err == nil { - for k, v := range portMappings20 { - protocol := "tcp" - parts := strings.Split(k, "/") - if len(parts) == 2 { - protocol = strings.ToLower(parts[1]) - } - containerPort, err := strconv.Atoi(parts[0]) - if err != nil { - return "", err - } - for _, pm := range v { - if containerPort == common.APIServerInternalPort && protocol == "tcp" { - return net.JoinHostPort(pm.HostIP, pm.HostPort), nil - } - } - } - } - var portMappings19 []portMapping19 - if err := json.Unmarshal([]byte(lines[0]), &portMappings19); err != nil { - return "", errors.Errorf("invalid network details: %v", err) - } - for _, pm := range portMappings19 { - if pm.ContainerPort == common.APIServerInternalPort && pm.Protocol == "tcp" { - return net.JoinHostPort(pm.HostIP, strconv.Itoa(int(pm.HostPort))), nil - } + host := parts[0] + port, err := strconv.Atoi(parts[1]) + if err != nil { + return "", errors.Errorf("network port not an integer: %v", err) } - return "", errors.Errorf("unable to find apiserver endpoint information") + return net.JoinHostPort(host, strconv.Itoa(port)), nil } // GetAPIServerInternalEndpoint is part of the providers.Provider interface From 847c825f73941a2b9ec815c03a36c3f892656ee4 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Wed, 2 Dec 2020 22:46:19 +0100 Subject: [PATCH 5/5] podman exclude ipv6 from the CI the problem is that the API is forwarded in localhost, and portmapping doesn' t work from localhost to localhost --- .github/workflows/podman.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/podman.yml b/.github/workflows/podman.yml index 7768e0a941..10a186cf20 100644 --- a/.github/workflows/podman.yml +++ b/.github/workflows/podman.yml @@ -16,6 +16,8 @@ jobs: matrix: ipFamily: [ipv4, ipv6] deployment: [singleNode, multiNode] + exclude: + - ipFamily: ipv6 env: JOB_NAME: "podman-${{ matrix.deployment }}-${{ matrix.ipFamily }}" KIND_EXPERIMENTAL_PROVIDER: "podman"