feat: add IPv6-only clusters (#752)

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.8.4
+	go4.org/netipx v0.0.0-20230728184502-ec4c8b891b28
 	golang.org/x/oauth2 v0.12.0
 	golang.org/x/sync v0.3.0
 	google.golang.org/api v0.143.0

go.sum

@@ -446,6 +446,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
 go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
+go4.org/netipx v0.0.0-20230728184502-ec4c8b891b28 h1:zLxFnORHDFTSkJPawMU7LzsuGQJ4MUFS653jJHpORow=
+go4.org/netipx v0.0.0-20230728184502-ec4c8b891b28/go.mod h1:TQvodOM+hJTioNQJilmLXu08JNb8i+ccq418+KWu1/Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

internal/cmd/ktf/environments.go

@@ -49,6 +49,7 @@ func init() { //nolint:gochecknoinits
 	// cluster configurations
 	environmentsCreateCmd.PersistentFlags().String("kubernetes-version", "", "which kubernetes version to use (default: latest for driver)")
 	environmentsCreateCmd.PersistentFlags().Bool("cni-calico", false, "use Calico for cluster CNI instead of the default CNI")
+	environmentsCreateCmd.PersistentFlags().Bool("ipv6-only", false, "only use IPv6")
 
 	// addon configurations
 	environmentsCreateCmd.PersistentFlags().StringArray("addon", nil, "name of an addon to deploy to the testing environment's cluster")
@@ -86,6 +87,10 @@ var environmentsCreateCmd = &cobra.Command{
 		useCalicoCNI, err := cmd.PersistentFlags().GetBool("cni-calico")
 		cobra.CheckErr(err)
 
+		// check if IPv6 was requested
+		useIPv6Only, err := cmd.PersistentFlags().GetBool("ipv6-only")
+		cobra.CheckErr(err)
+
 		// setup the new environment
 		builder := environments.NewBuilder()
 		if !useGeneratedName {
@@ -94,6 +99,9 @@ var environmentsCreateCmd = &cobra.Command{
 		if useCalicoCNI {
 			builder = builder.WithCalicoCNI()
 		}
+		if useIPv6Only {
+			builder = builder.WithIPv6Only()
+		}
 		if kubernetesVersion != "" {
 			version, err := semver.Parse(strings.TrimPrefix(kubernetesVersion, "v"))
 			cobra.CheckErr(err)

pkg/clusters/addons/kong/addon.go

@@ -266,6 +266,16 @@ func (a *Addon) Deploy(ctx context.Context, cluster clusters.Cluster) error {
 		)
 	}
 
+	if cluster.IPFamily() == clusters.IPv6 {
+		a.deployArgs = append(a.deployArgs,
+			"--set", "proxy.address=[::]",
+			"--set", "admin.address=[::1]",
+			"--set", "status.address=[::]",
+			"--set", "cluster.address=[::]",
+			"--set", "ingressController.admissionWebhook.address=[::]",
+		)
+	}
+
 	// if the ingress controller is disabled flag it in the chart and don't install any CRDs
 	if a.ingressControllerDisabled {
 		a.deployArgs = append(a.deployArgs,

pkg/clusters/addons/metallb/metallb.go

@@ -7,9 +7,11 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"net/netip"
 	"os"
 	"time"
 
+	"go4.org/netipx"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -26,7 +28,6 @@ import (
 	"github.com/kong/kubernetes-testing-framework/pkg/clusters/types/kind"
 	"github.com/kong/kubernetes-testing-framework/pkg/utils/docker"
 	"github.com/kong/kubernetes-testing-framework/pkg/utils/kubernetes/kubectl"
-	"github.com/kong/kubernetes-testing-framework/pkg/utils/networking"
 )
 
 // -----------------------------------------------------------------------------
@@ -141,10 +142,8 @@ func (a *addon) DumpDiagnostics(context.Context, clusters.Cluster) (map[string][
 // -----------------------------------------------------------------------------
 
 var (
-	defaultStartIP = net.ParseIP("0.0.0.100")
-	defaultEndIP   = net.ParseIP("0.0.0.250")
-	metalManifest  = "https://github.com/metallb/metallb/config/native?ref=v0.13.11&timeout=2m"
-	secretKeyLen   = 128
+	metalManifest = "https://github.com/metallb/metallb/config/native?ref=v0.13.11&timeout=2m"
+	secretKeyLen  = 128
 )
 
 // -----------------------------------------------------------------------------
@@ -201,11 +200,15 @@ func deployMetallbForKindCluster(ctx context.Context, cluster clusters.Cluster,
 
 func createIPAddressPool(ctx context.Context, cluster clusters.Cluster, dockerNetwork string) error {
 	// get an IP range for the docker container network to use for MetalLB
-	network, err := docker.GetDockerContainerIPNetwork(docker.GetKindContainerID(cluster.Name()), dockerNetwork)
+	// this returns addresses based on the _Docker network_ the cluster runs on, not the cluster itself. this may,
+	// for example, return IPv4 addresses even for an IPv6-only cluster. although unsupported addresses will be listed
+	// in the IPAddressPool, speaker will not actually assign them if they are not compatible with the cluster network.
+	network, network6, err := docker.GetDockerContainerIPNetwork(docker.GetKindContainerID(cluster.Name()), dockerNetwork)
 	if err != nil {
 		return err
 	}
 	ipStart, ipEnd := getIPRangeForMetallb(*network)
+	ip6Start, ip6End := getIPRangeForMetallb(*network6)
 
 	dynamicClient, err := dynamic.NewForConfig(cluster.Config())
 	if err != nil {
@@ -228,7 +231,8 @@ func createIPAddressPool(ctx context.Context, cluster clusters.Cluster, dockerNe
 				},
 				"spec": map[string]interface{}{
 					"addresses": []string{
-						networking.GetIPRangeStr(ipStart, ipEnd),
+						fmt.Sprintf("%s-%s", ipStart, ipEnd),
+						fmt.Sprintf("%s-%s", ip6Start, ip6End),
 					},
 				},
 			},
@@ -297,15 +301,24 @@ func createL2Advertisement(ctx context.Context, cluster clusters.Cluster) error
 	return nil
 }
 
+// TODO use netip throughout. this converts because old public APIs used net/ip instead of net/netip
+
 // getIPRangeForMetallb provides a range of IP addresses to use for MetalLB given an IPv4 Network
 //
-// TODO: Just choosing specific default IPs for now, need to check range validity and dynamically assign IPs.
+// TODO: this just chooses the upper half of the Docker network (minus the network and broadcast addresses for the
+// chosen subnet), although those IPs may be in use. Speaker will happily assign those, but they won't work.
+// In practice this doesn't appear to cause many problems, since the IPs are normally not in use by KIND components
+// (it appears to assign starting from the bottom of the Docker net)
 //
 // See: https://github.com/Kong/kubernetes-testing-framework/issues/24
-func getIPRangeForMetallb(network net.IPNet) (startIP, endIP net.IP) {
-	startIP = networking.ConvertUint32ToIPv4(networking.ConvertIPv4ToUint32(network.IP) | networking.ConvertIPv4ToUint32(defaultStartIP))
-	endIP = networking.ConvertUint32ToIPv4(networking.ConvertIPv4ToUint32(network.IP) | networking.ConvertIPv4ToUint32(defaultEndIP))
-	return
+func getIPRangeForMetallb(network net.IPNet) (startIP, endIP netip.Addr) {
+	// we trust that this is a valid prefix here because we already checked it in docker.GetDockerContainerIPNetwork
+	prefix := netip.MustParsePrefix(network.String())
+	half := prefix.Bits() + 1
+	wholeRange := netipx.RangeOfPrefix(prefix)
+	upperHalfPrefix := netip.PrefixFrom(wholeRange.To(), half).Masked()
+	halfRange := netipx.RangeOfPrefix(upperHalfPrefix)
+	return halfRange.From().Next(), halfRange.To().Prev()
 }
 
 // TODO: needs to be replaced with non-kubectl, just used this originally for speed.

pkg/clusters/addons/metallb/metallb_test.go

@@ -5,17 +5,17 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-
-	"github.com/kong/kubernetes-testing-framework/pkg/utils/networking"
 )
 
 func TestHelperFunctions(t *testing.T) {
 	network := net.IPNet{
 		IP:   net.IPv4(192, 168, 1, 0),
-		Mask: net.IPv4Mask(0, 0, 0, 255),
+		Mask: net.IPv4Mask(255, 255, 255, 0),
 	}
+	// this should choose the upper half of the input network, minus network and broadcast addresses
+	// since we start with 192.168.1.0/24, we should get 192.168.1.128/25. the complete range is
+	// 192.168.1.128-192.168.1.255, and the returned range is thus 192.168.1.129-192.168.1.254.
 	ip1, ip2 := getIPRangeForMetallb(network)
-	assert.Equal(t, ip1.String(), net.IPv4(192, 168, 1, 100).String())
-	assert.Equal(t, ip2.String(), net.IPv4(192, 168, 1, 250).String())
-	assert.Equal(t, networking.GetIPRangeStr(ip1, ip2), "192.168.1.100-192.168.1.250")
+	assert.Equal(t, net.IPv4(192, 168, 1, 129).String(), ip1.String())
+	assert.Equal(t, net.IPv4(192, 168, 1, 254).String(), ip2.String())
 }

pkg/clusters/cluster.go

@@ -15,6 +15,17 @@ import (
 // Type indicates the type of Kubernetes cluster (e.g. Kind, GKE, e.t.c.)
 type Type string
 
+type IPFamily string
+
+const (
+	// IPv4 indicates a Cluster that supports only IPv4 networking.
+	IPv4 IPFamily = "ipv4"
+	// IPv6 indicates a Cluster that supports only IPv6 networking.
+	IPv6 IPFamily = "ipv6"
+	// Dual indicates a Cluster that supports both IPv4 and IPv6 networking.
+	Dual IPFamily = "dual"
+)
+
 // Cluster objects represent a running Kubernetes cluster.
 type Cluster interface {
 	// Name indicates the unique name of the running cluster.
@@ -51,6 +62,9 @@ type Cluster interface {
 	// of said directory and an error.
 	// It uses the provided meta string allow for diagnostics identification.
 	DumpDiagnostics(ctx context.Context, meta string) (string, error)
+
+	// IPFamily returns the cluster's IP networking capabilities.
+	IPFamily() IPFamily
 }
 
 type Builder interface {

pkg/clusters/types/gke/builder.go

@@ -229,6 +229,8 @@ func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) {
 		cfg:             restCFG,
 		addons:          make(clusters.Addons),
 		l:               &sync.RWMutex{},
+		// we simply set this directly for GKE as we lack the ability to create other types of cluster
+		ipFamily: clusters.IPv4,
 	}
 
 	if err := utils.ClusterInitHooks(ctx, cluster); err != nil {

pkg/clusters/types/gke/cluster.go

@@ -36,6 +36,7 @@ type Cluster struct {
 	cfg             *rest.Config
 	addons          clusters.Addons
 	l               *sync.RWMutex
+	ipFamily        clusters.IPFamily
 }
 
 // NewFromExistingWithEnv provides a new clusters.Cluster backed by an existing GKE cluster,
@@ -278,3 +279,7 @@ func (c *Cluster) DumpDiagnostics(ctx context.Context, meta string) (string, err
 
 	return outDir, err
 }
+
+func (c *Cluster) IPFamily() clusters.IPFamily {
+	return c.ipFamily
+}

pkg/clusters/types/kind/builder.go

@@ -25,6 +25,7 @@ type Builder struct {
 	configPath     *string
 	configReader   io.Reader
 	calicoCNI      bool
+	ipv6Only       bool
 }
 
 // NewBuilder provides a new *Builder object.
@@ -74,6 +75,12 @@ func (b *Builder) WithCalicoCNI() *Builder {
 	return b
 }
 
+// WithIPv6Only configures KIND to only use IPv6.
+func (b *Builder) WithIPv6Only() *Builder {
+	b.ipv6Only = true
+	return b
+}
+
 // Build creates and configures clients for a Kind-based Kubernetes clusters.Cluster.
 func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) {
 	deployArgs := make([]string, 0)
@@ -92,6 +99,12 @@ func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) {
 		deployArgs = append(deployArgs, "--wait", "1s")
 	}
 
+	if b.ipv6Only {
+		if err := b.useIPv6Only(); err != nil {
+			return nil, fmt.Errorf("failed configuring IPv6-only networking: %w", err)
+		}
+	}
+
 	var stdin io.Reader
 	if b.configPath != nil {
 		deployArgs = append(deployArgs, "--config", *b.configPath)
@@ -116,13 +129,19 @@ func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) {
 		return nil, err
 	}
 
+	ipFamily := clusters.IPv4
+	if b.ipv6Only {
+		ipFamily = clusters.IPv6
+	}
+
 	cluster := &Cluster{
 		name:       b.Name,
 		client:     kc,
 		cfg:        cfg,
 		addons:     make(clusters.Addons),
 		deployArgs: deployArgs,
 		l:          &sync.RWMutex{},
+		ipFamily:   ipFamily,
 	}
 
 	if b.calicoCNI {

pkg/clusters/types/kind/cluster.go

@@ -38,6 +38,7 @@ type Cluster struct {
 	addons     clusters.Addons
 	deployArgs []string
 	l          *sync.RWMutex
+	ipFamily   clusters.IPFamily
 }
 
 // New provides a new clusters.Cluster backed by a Kind based Kubernetes Cluster.
@@ -158,3 +159,7 @@ func (c *Cluster) DumpDiagnostics(ctx context.Context, meta string) (string, err
 	err = clusters.DumpDiagnostics(ctx, c, meta, outDir)
 	return outDir, err
 }
+
+func (c *Cluster) IPFamily() clusters.IPFamily {
+	return c.ipFamily
+}

pkg/clusters/types/kind/utils.go

@@ -141,6 +141,38 @@ func (b *Builder) disableDefaultCNI() error {
 	return nil
 }
 
+func (b *Builder) useIPv6Only() error {
+	if err := b.ensureConfigFile(); err != nil {
+		return err
+	}
+
+	configYAML, err := os.ReadFile(*b.configPath)
+	if err != nil {
+		return fmt.Errorf("failed reading kind config from %s: %w", *b.configPath, err)
+	}
+
+	kindConfig := v1alpha4.Cluster{}
+	if err := yaml.Unmarshal(configYAML, &kindConfig); err != nil {
+		return fmt.Errorf("failed unmarshalling kind config: %w", err)
+	}
+
+	kindConfig.Networking.IPFamily = v1alpha4.IPv6Family
+	// For Windows/OS X Docker compatibility:
+	// https://kind.sigs.k8s.io/docs/user/configuration/#ip-family
+	kindConfig.Networking.APIServerAddress = "127.0.0.1"
+
+	configYAML, err = yaml.Marshal(kindConfig)
+	if err != nil {
+		return fmt.Errorf("failed marshalling kind config: %w", err)
+	}
+
+	err = os.WriteFile(*b.configPath, configYAML, 0o600) //nolint:gomnd
+	if err != nil {
+		return fmt.Errorf("failed writing kind config %s: %w", *b.configPath, err)
+	}
+	return nil
+}
+
 // exportLogs dumps a kind cluster logs to the specified directory
 func exportLogs(ctx context.Context, name string, outDir string) error {
 	args := []string{"export", "logs", outDir, "--name", name}

pkg/environments/builder.go

@@ -26,6 +26,7 @@ type Builder struct {
 	clusterBuilder    clusters.Builder
 	kubernetesVersion *semver.Version
 	calicoCNI         bool
+	ipv6Only          bool
 }
 
 // NewBuilder generates a new empty Builder for creating Environments.
@@ -78,6 +79,12 @@ func (b *Builder) WithCalicoCNI() *Builder {
 	return b
 }
 
+// WithIPv6Only configures KIND to only use IPv6.
+func (b *Builder) WithIPv6Only() *Builder {
+	b.ipv6Only = true
+	return b
+}
+
 // Build is a blocking call to construct the configured Environment and it's
 // underlying Kubernetes cluster. The amount of time that it blocks depends
 // entirely on the underlying clusters.Cluster implementation that was requested.
@@ -88,6 +95,10 @@ func (b *Builder) Build(ctx context.Context) (env Environment, err error) {
 		return nil, fmt.Errorf("trying to deploy Calico CNI on an existing cluster is not currently supported")
 	}
 
+	if b.ipv6Only && b.existingCluster != nil {
+		return nil, fmt.Errorf("trying to configure IPv6 only on an existing cluster is not currently supported")
+	}
+
 	if b.existingCluster != nil && b.clusterBuilder != nil {
 		return nil, fmt.Errorf("Environment cannot specify both existingCluster and clusterBuilder")
 	}
@@ -114,6 +125,9 @@ func (b *Builder) Build(ctx context.Context) (env Environment, err error) {
 		if b.calicoCNI {
 			builder.WithCalicoCNI()
 		}
+		if b.ipv6Only {
+			builder.WithIPv6Only()
+		}
 		cluster, err = builder.Build(ctx)
 		if err != nil {
 			return nil, err