From 9c91048386a1bd7c179b1c9144622335598636f0 Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Fri, 28 Jul 2023 18:11:51 -0700 Subject: [PATCH] feat: add and test Kong IPv6 support --- pkg/clusters/addons/kong/addon.go | 10 +++++++++ pkg/clusters/addons/metallb/metallb.go | 3 +++ pkg/clusters/cluster.go | 14 ++++++++++++ pkg/clusters/types/gke/builder.go | 2 ++ pkg/clusters/types/gke/cluster.go | 5 +++++ pkg/clusters/types/kind/builder.go | 6 ++++++ pkg/clusters/types/kind/cluster.go | 5 +++++ test/integration/ipv6_test.go | 30 +++++++++++++++++++++++++- 8 files changed, 74 insertions(+), 1 deletion(-) diff --git a/pkg/clusters/addons/kong/addon.go b/pkg/clusters/addons/kong/addon.go index 2d2dc411..53d9e97d 100644 --- a/pkg/clusters/addons/kong/addon.go +++ b/pkg/clusters/addons/kong/addon.go @@ -266,6 +266,16 @@ func (a *Addon) Deploy(ctx context.Context, cluster clusters.Cluster) error { ) } + if cluster.IPFamily() == clusters.IPv6 { + a.deployArgs = append(a.deployArgs, + "--set", "proxy.address=[::]", + "--set", "admin.address=[::1]", + "--set", "status.address=[::]", + "--set", "cluster.address=[::]", + "--set", "ingressController.admissionWebhook.address=[::]", + ) + } + // if the ingress controller is disabled flag it in the chart and don't install any CRDs if a.ingressControllerDisabled { a.deployArgs = append(a.deployArgs, diff --git a/pkg/clusters/addons/metallb/metallb.go b/pkg/clusters/addons/metallb/metallb.go index 443b6b05..000afcb1 100644 --- a/pkg/clusters/addons/metallb/metallb.go +++ b/pkg/clusters/addons/metallb/metallb.go @@ -200,6 +200,9 @@ func deployMetallbForKindCluster(ctx context.Context, cluster clusters.Cluster, func createIPAddressPool(ctx context.Context, cluster clusters.Cluster, dockerNetwork string) error { // get an IP range for the docker container network to use for MetalLB + // this returns addresses based on the _Docker network_ the cluster runs on, not the cluster itself. this may, + // for example, return IPv4 addresses even for an IPv6-only cluster. although unsupported addresses will be listed + // in the IPAddressPool, speaker will not actually assign them if they are not compatible with the cluster network. network, network6, err := docker.GetDockerContainerIPNetwork(docker.GetKindContainerID(cluster.Name()), dockerNetwork) if err != nil { return err diff --git a/pkg/clusters/cluster.go b/pkg/clusters/cluster.go index 3b1d6d10..7bfd1a15 100644 --- a/pkg/clusters/cluster.go +++ b/pkg/clusters/cluster.go @@ -15,6 +15,17 @@ import ( // Type indicates the type of Kubernetes cluster (e.g. Kind, GKE, e.t.c.) type Type string +type IPFamily string + +const ( + // IPv4 indicates a Cluster that supports only IPv4 networking. + IPv4 IPFamily = "ipv4" + // IPv6 indicates a Cluster that supports only IPv6 networking. + IPv6 IPFamily = "ipv6" + // Dual indicates a Cluster that supports both IPv4 and IPv6 networking. + Dual IPFamily = "dual" +) + // Cluster objects represent a running Kubernetes cluster. type Cluster interface { // Name indicates the unique name of the running cluster. @@ -51,6 +62,9 @@ type Cluster interface { // of said directory and an error. // It uses the provided meta string allow for diagnostics identification. DumpDiagnostics(ctx context.Context, meta string) (string, error) + + // IPFamily returns the cluster's IP networking capabilities. + IPFamily() IPFamily } type Builder interface { diff --git a/pkg/clusters/types/gke/builder.go b/pkg/clusters/types/gke/builder.go index 3bb78a3b..8b59f2df 100644 --- a/pkg/clusters/types/gke/builder.go +++ b/pkg/clusters/types/gke/builder.go @@ -229,6 +229,8 @@ func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) { cfg: restCFG, addons: make(clusters.Addons), l: &sync.RWMutex{}, + // we simply set this directly for GKE as we lack the ability to create other types of cluster + ipFamily: clusters.IPv4, } if err := utils.ClusterInitHooks(ctx, cluster); err != nil { diff --git a/pkg/clusters/types/gke/cluster.go b/pkg/clusters/types/gke/cluster.go index cb21f8c4..29148d9b 100644 --- a/pkg/clusters/types/gke/cluster.go +++ b/pkg/clusters/types/gke/cluster.go @@ -36,6 +36,7 @@ type Cluster struct { cfg *rest.Config addons clusters.Addons l *sync.RWMutex + ipFamily clusters.IPFamily } // NewFromExistingWithEnv provides a new clusters.Cluster backed by an existing GKE cluster, @@ -278,3 +279,7 @@ func (c *Cluster) DumpDiagnostics(ctx context.Context, meta string) (string, err return outDir, err } + +func (c *Cluster) IPFamily() clusters.IPFamily { + return c.ipFamily +} diff --git a/pkg/clusters/types/kind/builder.go b/pkg/clusters/types/kind/builder.go index 1da1e9ab..08d65132 100644 --- a/pkg/clusters/types/kind/builder.go +++ b/pkg/clusters/types/kind/builder.go @@ -129,6 +129,11 @@ func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) { return nil, err } + ipFamily := clusters.IPv4 + if b.ipv6Only { + ipFamily = clusters.IPv6 + } + cluster := &Cluster{ name: b.Name, client: kc, @@ -136,6 +141,7 @@ func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) { addons: make(clusters.Addons), deployArgs: deployArgs, l: &sync.RWMutex{}, + ipFamily: ipFamily, } if b.calicoCNI { diff --git a/pkg/clusters/types/kind/cluster.go b/pkg/clusters/types/kind/cluster.go index fed8be1c..45ae3875 100644 --- a/pkg/clusters/types/kind/cluster.go +++ b/pkg/clusters/types/kind/cluster.go @@ -38,6 +38,7 @@ type Cluster struct { addons clusters.Addons deployArgs []string l *sync.RWMutex + ipFamily clusters.IPFamily } // New provides a new clusters.Cluster backed by a Kind based Kubernetes Cluster. @@ -158,3 +159,7 @@ func (c *Cluster) DumpDiagnostics(ctx context.Context, meta string) (string, err err = clusters.DumpDiagnostics(ctx, c, meta, outDir) return outDir, err } + +func (c *Cluster) IPFamily() clusters.IPFamily { + return c.ipFamily +} diff --git a/test/integration/ipv6_test.go b/test/integration/ipv6_test.go index 21d87bf9..929de416 100644 --- a/test/integration/ipv6_test.go +++ b/test/integration/ipv6_test.go @@ -3,14 +3,18 @@ package integration import ( + "fmt" "net" + "net/http" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/kong/kubernetes-testing-framework/pkg/clusters/addons/kong" "github.com/kong/kubernetes-testing-framework/pkg/clusters/addons/metallb" "github.com/kong/kubernetes-testing-framework/pkg/environments" ) @@ -19,7 +23,7 @@ func TestKindClusterWithIPv6(t *testing.T) { t.Parallel() t.Log("configuring the test environment with IPv6 only enabled") - builder := environments.NewBuilder().WithIPv6Only().WithAddons(metallb.New()) + builder := environments.NewBuilder().WithIPv6Only().WithAddons(metallb.New(), kong.New()) t.Log("building the testing environment and Kubernetes cluster") env, err := builder.Build(ctx) @@ -36,4 +40,28 @@ func TestKindClusterWithIPv6(t *testing.T) { require.Nil(t, parsed.To4()) } } + + require.Eventually(t, func() bool { + kongServices := env.Cluster().Client().CoreV1().Services(kong.DefaultNamespace) + service, err := kongServices.Get(ctx, kong.DefaultProxyServiceName, metav1.GetOptions{}) + if err != nil { + return false + } + + if len(service.Status.LoadBalancer.Ingress) == 0 { + return false + } + + if net.ParseIP(service.Status.LoadBalancer.Ingress[0].IP).To4() != nil { + return false + } + kongURL := fmt.Sprintf("http://[%s]", service.Status.LoadBalancer.Ingress[0].IP) + resp, err := http.Get(kongURL) + // we don't care that the proxy has nothing to serve so long as we can talk to it and get a valid HTTP response + if err == nil && resp.StatusCode == http.StatusNotFound { + return true + } + return false + + }, time.Minute*3, time.Second) }