diff --git a/.github/workflows/podman.yml b/.github/workflows/podman.yml index 67543db062..10a186cf20 100644 --- a/.github/workflows/podman.yml +++ b/.github/workflows/podman.yml @@ -7,7 +7,6 @@ on: - master jobs: - podman: name: Podman runs-on: ubuntu-latest @@ -18,84 +17,98 @@ jobs: ipFamily: [ipv4, ipv6] deployment: [singleNode, multiNode] exclude: - # exclude IPv6 and multinode - ipFamily: ipv6 - - deployment: multiNode env: JOB_NAME: "podman-${{ matrix.deployment }}-${{ matrix.ipFamily }}" KIND_EXPERIMENTAL_PROVIDER: "podman" IP_FAMILY: ${{ matrix.ipFamily }} + PODMAN_VERSION: "stable" steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Verify + run: make verify + + - name: Install kind + run: sudo make install INSTALL_DIR=/usr/local/bin - - name: Check out code into the Go module directory - uses: actions/checkout@v2 - with: - fetch-depth: 0 + - name: Install kubectl + run: | + curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl - - name: Verify - run: make verify + - name: Enable ipv4 and ipv6 forwarding + run: | + sudo sysctl -w net.ipv6.conf.all.forwarding=1 + sudo sysctl -w net.ipv4.ip_forward=1 - - name: Install kind - run: sudo make install INSTALL_DIR=/usr/local/bin + - name: Install podman + run: | + . /etc/os-release + echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/${PODMAN_VERSION}/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list + curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/${PODMAN_VERSION}/xUbuntu_${VERSION_ID}/Release.key | sudo apt-key add - + sudo apt-get update -qq + sudo apt-get -qq -y install podman + # Install network + sudo mkdir -p /etc/cni/net.d + curl -qsSL https://raw.githubusercontent.com/containers/libpod/master/cni/87-podman-bridge.conflist | sudo tee /etc/cni/net.d/87-podman-bridge.conf + curl -qsSL https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz --output /tmp/cni.tgz + sudo mkdir -p /usr/libexec/cni + sudo tar -C /usr/libexec/cni -xvzf /tmp/cni.tgz - - name: Install podman - run: | - . /etc/os-release - echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list - curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key | sudo apt-key add - - sudo apt-get update -qq - sudo apt-get -qq -y install podman - # Install network - sudo mkdir -p /etc/cni/net.d - curl -qsSL https://raw.githubusercontent.com/containers/libpod/master/cni/87-podman-bridge.conflist | sudo tee /etc/cni/net.d/87-podman-bridge.conf - curl -qsSL https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz --output /tmp/cni.tgz - sudo mkdir -p /usr/libexec/cni - sudo tar -C /usr/libexec/cni -xvzf /tmp/cni.tgz + - name: Create single node cluster + if: ${{ matrix.deployment == 'singleNode' }} + run: | + cat <= 2.2.0 +func ensureNetwork(name string) error { + // network already exists + if checkIfNetworkExists(name) { + return nil + } + + // generate unique subnet per network based on the name + // obtained from the ULA fc00::/8 range + // Make N attempts with "probing" in case we happen to collide + subnet := generateULASubnetFromName(name, 0) + err := createNetwork(name, subnet) + if err == nil { + // Success! + return nil + } + + if isUnknownIPv6FlagError(err) { + return createNetwork(name, "") + } + + // Only continue if the error is because of the subnet range + // is already allocated + if !isPoolOverlapError(err) { + return err + } + + // keep trying for ipv6 subnets + const maxAttempts = 5 + for attempt := int32(1); attempt < maxAttempts; attempt++ { + subnet := generateULASubnetFromName(name, attempt) + err = createNetwork(name, subnet) + if err == nil { + // success! + return nil + } else if !isPoolOverlapError(err) { + // unknown error ... + return err + } + } + return errors.New("exhausted attempts trying to find a non-overlapping subnet") + +} + +func createNetwork(name, ipv6Subnet string) error { + if ipv6Subnet == "" { + return exec.Command("podman", "network", "create", "-d=bridge", name).Run() + } + return exec.Command("podman", "network", "create", "-d=bridge", + "--ipv6", "--subnet", ipv6Subnet, name).Run() +} + +func checkIfNetworkExists(name string) bool { + _, err := exec.Output(exec.Command( + "podman", "network", "inspect", + regexp.QuoteMeta(name), + )) + return err == nil +} + +func isUnknownIPv6FlagError(err error) bool { + rerr := exec.RunErrorForError(err) + return rerr != nil && + strings.Contains(string(rerr.Output), "unknown flag: --ipv6") +} + +func isPoolOverlapError(err error) bool { + rerr := exec.RunErrorForError(err) + return rerr != nil && + (strings.Contains(string(rerr.Output), "is being used by a network interface") || + strings.Contains(string(rerr.Output), "is already being used by a cni configuration")) +} + +// generateULASubnetFromName generate an IPv6 subnet based on the +// name and Nth probing attempt +func generateULASubnetFromName(name string, attempt int32) string { + ip := make([]byte, 16) + ip[0] = 0xfc + ip[1] = 0x00 + h := sha1.New() + _, _ = h.Write([]byte(name)) + _ = binary.Write(h, binary.LittleEndian, attempt) + bs := h.Sum(nil) + for i := 2; i < 8; i++ { + ip[i] = bs[i] + } + subnet := &net.IPNet{ + IP: net.IP(ip), + Mask: net.CIDRMask(64, 128), + } + return subnet.String() +} diff --git a/pkg/cluster/internal/providers/podman/node.go b/pkg/cluster/internal/providers/podman/node.go index 4a530f08b2..5285dd2244 100644 --- a/pkg/cluster/internal/providers/podman/node.go +++ b/pkg/cluster/internal/providers/podman/node.go @@ -53,7 +53,7 @@ func (n *node) Role() (string, error) { func (n *node) IP() (ipv4 string, ipv6 string, err error) { // retrieve the IP address of the node using podman inspect cmd := exec.Command("podman", "inspect", - "-f", "{{.NetworkSettings.IPAddress}},{{.NetworkSettings.GlobalIPv6Address}}", + "-f", "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}", n.name, // ... against the "node" container ) lines, err := exec.OutputLines(cmd) diff --git a/pkg/cluster/internal/providers/podman/provider.go b/pkg/cluster/internal/providers/podman/provider.go index 65c35d7047..8cdd1519db 100644 --- a/pkg/cluster/internal/providers/podman/provider.go +++ b/pkg/cluster/internal/providers/podman/provider.go @@ -26,6 +26,7 @@ import ( "strings" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/version" "sigs.k8s.io/kind/pkg/cluster/nodes" "sigs.k8s.io/kind/pkg/cluster/nodeutils" @@ -79,13 +80,24 @@ func (p *provider) Provision(status *cli.Status, cfg *config.Cluster) (err error return err } + // ensure the pre-requesite network exists + networkName := fixedNetworkName + if n := os.Getenv("KIND_EXPERIMENTAL_PODMAN_NETWORK"); n != "" { + p.logger.Warn("WARNING: Overriding podman network due to KIND_EXPERIMENTAL_PODMAN_NETWORK") + p.logger.Warn("WARNING: Here be dragons! This is not supported currently.") + networkName = n + } + if err := ensureNetwork(networkName); err != nil { + return errors.Wrap(err, "failed to ensure podman network") + } + // actually provision the cluster icons := strings.Repeat("📦 ", len(cfg.Nodes)) status.Start(fmt.Sprintf("Preparing nodes %s", icons)) defer func() { status.End(err == nil) }() // plan creating the containers - createContainerFuncs, err := planCreation(cfg) + createContainerFuncs, err := planCreation(cfg, networkName) if err != nil { return err } @@ -174,13 +186,79 @@ func (p *provider) GetAPIServerEndpoint(cluster string) (string, error) { return "", errors.Wrap(err, "failed to get api server endpoint") } - // retrieve the specific port mapping using podman inspect + // TODO: get rid of this once podman settles on how to get the port mapping using podman inspect + // This is only used to get the Kubeconfig server field + v, err := getPodmanVersion() + if err != nil { + return "", errors.Wrap(err, "failed to check podman version") + } + if v.LessThan(version.MustParseSemantic("2.2.0")) { + cmd := exec.Command( + "podman", "inspect", + "--format", + "{{ json .NetworkSettings.Ports }}", + n.String(), + ) + lines, err := exec.OutputLines(cmd) + if err != nil { + return "", errors.Wrap(err, "failed to get api server port") + } + if len(lines) != 1 { + return "", errors.Errorf("network details should only be one line, got %d lines", len(lines)) + } + + // portMapping19 maps to the standard CNI portmapping capability used in podman 1.9 + // see: https://github.com/containernetworking/cni/blob/spec-v0.4.0/CONVENTIONS.md + type portMapping19 struct { + HostPort int32 `json:"hostPort"` + ContainerPort int32 `json:"containerPort"` + Protocol string `json:"protocol"` + HostIP string `json:"hostIP"` + } + // portMapping20 maps to the podman 2.0 portmap type + // see: https://github.com/containers/podman/blob/05988fc74fc25f2ad2256d6e011dfb7ad0b9a4eb/libpod/define/container_inspect.go#L134-L143 + type portMapping20 struct { + HostPort string `json:"HostPort"` + HostIP string `json:"HostIp"` + } + + portMappings20 := make(map[string][]portMapping20) + if err := json.Unmarshal([]byte(lines[0]), &portMappings20); err == nil { + for k, v := range portMappings20 { + protocol := "tcp" + parts := strings.Split(k, "/") + if len(parts) == 2 { + protocol = strings.ToLower(parts[1]) + } + containerPort, err := strconv.Atoi(parts[0]) + if err != nil { + return "", err + } + for _, pm := range v { + if containerPort == common.APIServerInternalPort && protocol == "tcp" { + return net.JoinHostPort(pm.HostIP, pm.HostPort), nil + } + } + } + } + var portMappings19 []portMapping19 + if err := json.Unmarshal([]byte(lines[0]), &portMappings19); err != nil { + return "", errors.Errorf("invalid network details: %v", err) + } + for _, pm := range portMappings19 { + if pm.ContainerPort == common.APIServerInternalPort && pm.Protocol == "tcp" { + return net.JoinHostPort(pm.HostIP, strconv.Itoa(int(pm.HostPort))), nil + } + } + } + // TODO: hack until https://github.com/containers/podman/issues/8444 is resolved cmd := exec.Command( "podman", "inspect", "--format", - "{{ json .NetworkSettings.Ports }}", + "{{range .NetworkSettings.Ports }}{{range .}}{{.HostIP}}/{{.HostPort}}{{end}}{{end}}", n.String(), ) + lines, err := exec.OutputLines(cmd) if err != nil { return "", errors.Wrap(err, "failed to get api server port") @@ -188,52 +266,18 @@ func (p *provider) GetAPIServerEndpoint(cluster string) (string, error) { if len(lines) != 1 { return "", errors.Errorf("network details should only be one line, got %d lines", len(lines)) } - - // portMapping19 maps to the standard CNI portmapping capability used in podman 1.9 - // see: https://github.com/containernetworking/cni/blob/spec-v0.4.0/CONVENTIONS.md - type portMapping19 struct { - HostPort int32 `json:"hostPort"` - ContainerPort int32 `json:"containerPort"` - Protocol string `json:"protocol"` - HostIP string `json:"hostIP"` - } - // portMapping20 maps to the podman 2.0 portmap type - // see: https://github.com/containers/podman/blob/05988fc74fc25f2ad2256d6e011dfb7ad0b9a4eb/libpod/define/container_inspect.go#L134-L143 - type portMapping20 struct { - HostPort string `json:"HostPort"` - HostIP string `json:"HostIp"` - } - - portMappings20 := make(map[string][]portMapping20) - if err := json.Unmarshal([]byte(lines[0]), &portMappings20); err == nil { - for k, v := range portMappings20 { - protocol := "tcp" - parts := strings.Split(k, "/") - if len(parts) == 2 { - protocol = strings.ToLower(parts[1]) - } - containerPort, err := strconv.Atoi(parts[0]) - if err != nil { - return "", err - } - for _, pm := range v { - if containerPort == common.APIServerInternalPort && protocol == "tcp" { - return net.JoinHostPort(pm.HostIP, pm.HostPort), nil - } - } - } - } - var portMappings19 []portMapping19 - if err := json.Unmarshal([]byte(lines[0]), &portMappings19); err != nil { - return "", errors.Errorf("invalid network details: %v", err) + // output is in the format IP/Port + parts := strings.Split(strings.TrimSpace(lines[0]), "/") + if len(parts) != 2 { + return "", errors.Errorf("network details should be in the format IP/Port, received: %s", parts) } - for _, pm := range portMappings19 { - if pm.ContainerPort == common.APIServerInternalPort && pm.Protocol == "tcp" { - return net.JoinHostPort(pm.HostIP, strconv.Itoa(int(pm.HostPort))), nil - } + host := parts[0] + port, err := strconv.Atoi(parts[1]) + if err != nil { + return "", errors.Errorf("network port not an integer: %v", err) } - return "", errors.Errorf("unable to find apiserver endpoint information") + return net.JoinHostPort(host, strconv.Itoa(port)), nil } // GetAPIServerInternalEndpoint is part of the providers.Provider interface @@ -247,14 +291,8 @@ func (p *provider) GetAPIServerInternalEndpoint(cluster string) (string, error) if err != nil { return "", errors.Wrap(err, "failed to get apiserver endpoint") } - // TODO: check cluster IP family and return the correct IP - // This means IPv6 singlestack is broken on podman - ipv4, _, err := n.IP() - if err != nil { - return "", errors.Wrap(err, "failed to get apiserver IP") - } - return net.JoinHostPort(ipv4, fmt.Sprintf("%d", common.APIServerInternalPort)), nil - + // NOTE: we're using the nodes's hostnames which are their names + return net.JoinHostPort(n.String(), fmt.Sprintf("%d", common.APIServerInternalPort)), nil } // node returns a new node handle for this provider diff --git a/pkg/cluster/internal/providers/podman/provision.go b/pkg/cluster/internal/providers/podman/provision.go index 8e4d75ffef..47577d3e56 100644 --- a/pkg/cluster/internal/providers/podman/provision.go +++ b/pkg/cluster/internal/providers/podman/provision.go @@ -32,10 +32,10 @@ import ( ) // planCreation creates a slice of funcs that will create the containers -func planCreation(cfg *config.Cluster) (createContainerFuncs []func() error, err error) { +func planCreation(cfg *config.Cluster, networkName string) (createContainerFuncs []func() error, err error) { // these apply to all container creation nodeNamer := common.MakeNodeNamer(cfg.Name) - genericArgs, err := commonArgs(cfg) + genericArgs, err := commonArgs(cfg, networkName) if err != nil { return nil, err } @@ -135,11 +135,12 @@ func clusterHasImplicitLoadBalancer(cfg *config.Cluster) bool { } // commonArgs computes static arguments that apply to all containers -func commonArgs(cfg *config.Cluster) ([]string, error) { +func commonArgs(cfg *config.Cluster, networkName string) ([]string, error) { // standard arguments all nodes containers need, computed once args := []string{ - "--detach", // run the container detached - "--tty", // allocate a tty for entrypoint logs + "--detach", // run the container detached + "--tty", // allocate a tty for entrypoint logs + "--net", networkName, // attach to its own network // label the node with the cluster ID "--label", fmt.Sprintf("%s=%s", clusterLabelKey, cfg.Name), }