From 0dab64760e88b208e0b7e4dd52a2ada3ca71e230 Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:49:17 -0700 Subject: [PATCH] feat: add IPv6-only clusters (#752) --- go.mod | 1 + go.sum | 2 + internal/cmd/ktf/environments.go | 8 +++ pkg/clusters/addons/kong/addon.go | 10 +++ pkg/clusters/addons/metallb/metallb.go | 37 ++++++++---- pkg/clusters/addons/metallb/metallb_test.go | 12 ++-- pkg/clusters/cluster.go | 14 +++++ pkg/clusters/types/gke/builder.go | 2 + pkg/clusters/types/gke/cluster.go | 5 ++ pkg/clusters/types/kind/builder.go | 19 ++++++ pkg/clusters/types/kind/cluster.go | 5 ++ pkg/clusters/types/kind/utils.go | 32 ++++++++++ pkg/environments/builder.go | 14 +++++ pkg/utils/docker/docker.go | 14 +++-- pkg/utils/networking/networking.go | 4 ++ test/integration/ipv6_test.go | 67 +++++++++++++++++++++ 16 files changed, 223 insertions(+), 23 deletions(-) create mode 100644 test/integration/ipv6_test.go diff --git a/go.mod b/go.mod index 803d8692..93ef7c21 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 + go4.org/netipx v0.0.0-20230728184502-ec4c8b891b28 golang.org/x/oauth2 v0.12.0 golang.org/x/sync v0.3.0 google.golang.org/api v0.143.0 diff --git a/go.sum b/go.sum index a5b70dc3..3cfb4c0a 100644 --- a/go.sum +++ b/go.sum @@ -446,6 +446,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= +go4.org/netipx v0.0.0-20230728184502-ec4c8b891b28 h1:zLxFnORHDFTSkJPawMU7LzsuGQJ4MUFS653jJHpORow= +go4.org/netipx v0.0.0-20230728184502-ec4c8b891b28/go.mod h1:TQvodOM+hJTioNQJilmLXu08JNb8i+ccq418+KWu1/Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/internal/cmd/ktf/environments.go b/internal/cmd/ktf/environments.go index 0c748240..c4ec9bb9 100644 --- a/internal/cmd/ktf/environments.go +++ b/internal/cmd/ktf/environments.go @@ -49,6 +49,7 @@ func init() { //nolint:gochecknoinits // cluster configurations environmentsCreateCmd.PersistentFlags().String("kubernetes-version", "", "which kubernetes version to use (default: latest for driver)") environmentsCreateCmd.PersistentFlags().Bool("cni-calico", false, "use Calico for cluster CNI instead of the default CNI") + environmentsCreateCmd.PersistentFlags().Bool("ipv6-only", false, "only use IPv6") // addon configurations environmentsCreateCmd.PersistentFlags().StringArray("addon", nil, "name of an addon to deploy to the testing environment's cluster") @@ -86,6 +87,10 @@ var environmentsCreateCmd = &cobra.Command{ useCalicoCNI, err := cmd.PersistentFlags().GetBool("cni-calico") cobra.CheckErr(err) + // check if IPv6 was requested + useIPv6Only, err := cmd.PersistentFlags().GetBool("ipv6-only") + cobra.CheckErr(err) + // setup the new environment builder := environments.NewBuilder() if !useGeneratedName { @@ -94,6 +99,9 @@ var environmentsCreateCmd = &cobra.Command{ if useCalicoCNI { builder = builder.WithCalicoCNI() } + if useIPv6Only { + builder = builder.WithIPv6Only() + } if kubernetesVersion != "" { version, err := semver.Parse(strings.TrimPrefix(kubernetesVersion, "v")) cobra.CheckErr(err) diff --git a/pkg/clusters/addons/kong/addon.go b/pkg/clusters/addons/kong/addon.go index 2d2dc411..53d9e97d 100644 --- a/pkg/clusters/addons/kong/addon.go +++ b/pkg/clusters/addons/kong/addon.go @@ -266,6 +266,16 @@ func (a *Addon) Deploy(ctx context.Context, cluster clusters.Cluster) error { ) } + if cluster.IPFamily() == clusters.IPv6 { + a.deployArgs = append(a.deployArgs, + "--set", "proxy.address=[::]", + "--set", "admin.address=[::1]", + "--set", "status.address=[::]", + "--set", "cluster.address=[::]", + "--set", "ingressController.admissionWebhook.address=[::]", + ) + } + // if the ingress controller is disabled flag it in the chart and don't install any CRDs if a.ingressControllerDisabled { a.deployArgs = append(a.deployArgs, diff --git a/pkg/clusters/addons/metallb/metallb.go b/pkg/clusters/addons/metallb/metallb.go index fae38382..000afcb1 100644 --- a/pkg/clusters/addons/metallb/metallb.go +++ b/pkg/clusters/addons/metallb/metallb.go @@ -7,9 +7,11 @@ import ( "fmt" "io" "net" + "net/netip" "os" "time" + "go4.org/netipx" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,7 +28,6 @@ import ( "github.com/kong/kubernetes-testing-framework/pkg/clusters/types/kind" "github.com/kong/kubernetes-testing-framework/pkg/utils/docker" "github.com/kong/kubernetes-testing-framework/pkg/utils/kubernetes/kubectl" - "github.com/kong/kubernetes-testing-framework/pkg/utils/networking" ) // ----------------------------------------------------------------------------- @@ -141,10 +142,8 @@ func (a *addon) DumpDiagnostics(context.Context, clusters.Cluster) (map[string][ // ----------------------------------------------------------------------------- var ( - defaultStartIP = net.ParseIP("0.0.0.100") - defaultEndIP = net.ParseIP("0.0.0.250") - metalManifest = "https://github.com/metallb/metallb/config/native?ref=v0.13.11&timeout=2m" - secretKeyLen = 128 + metalManifest = "https://github.com/metallb/metallb/config/native?ref=v0.13.11&timeout=2m" + secretKeyLen = 128 ) // ----------------------------------------------------------------------------- @@ -201,11 +200,15 @@ func deployMetallbForKindCluster(ctx context.Context, cluster clusters.Cluster, func createIPAddressPool(ctx context.Context, cluster clusters.Cluster, dockerNetwork string) error { // get an IP range for the docker container network to use for MetalLB - network, err := docker.GetDockerContainerIPNetwork(docker.GetKindContainerID(cluster.Name()), dockerNetwork) + // this returns addresses based on the _Docker network_ the cluster runs on, not the cluster itself. this may, + // for example, return IPv4 addresses even for an IPv6-only cluster. although unsupported addresses will be listed + // in the IPAddressPool, speaker will not actually assign them if they are not compatible with the cluster network. + network, network6, err := docker.GetDockerContainerIPNetwork(docker.GetKindContainerID(cluster.Name()), dockerNetwork) if err != nil { return err } ipStart, ipEnd := getIPRangeForMetallb(*network) + ip6Start, ip6End := getIPRangeForMetallb(*network6) dynamicClient, err := dynamic.NewForConfig(cluster.Config()) if err != nil { @@ -228,7 +231,8 @@ func createIPAddressPool(ctx context.Context, cluster clusters.Cluster, dockerNe }, "spec": map[string]interface{}{ "addresses": []string{ - networking.GetIPRangeStr(ipStart, ipEnd), + fmt.Sprintf("%s-%s", ipStart, ipEnd), + fmt.Sprintf("%s-%s", ip6Start, ip6End), }, }, }, @@ -297,15 +301,24 @@ func createL2Advertisement(ctx context.Context, cluster clusters.Cluster) error return nil } +// TODO use netip throughout. this converts because old public APIs used net/ip instead of net/netip + // getIPRangeForMetallb provides a range of IP addresses to use for MetalLB given an IPv4 Network // -// TODO: Just choosing specific default IPs for now, need to check range validity and dynamically assign IPs. +// TODO: this just chooses the upper half of the Docker network (minus the network and broadcast addresses for the +// chosen subnet), although those IPs may be in use. Speaker will happily assign those, but they won't work. +// In practice this doesn't appear to cause many problems, since the IPs are normally not in use by KIND components +// (it appears to assign starting from the bottom of the Docker net) // // See: https://github.com/Kong/kubernetes-testing-framework/issues/24 -func getIPRangeForMetallb(network net.IPNet) (startIP, endIP net.IP) { - startIP = networking.ConvertUint32ToIPv4(networking.ConvertIPv4ToUint32(network.IP) | networking.ConvertIPv4ToUint32(defaultStartIP)) - endIP = networking.ConvertUint32ToIPv4(networking.ConvertIPv4ToUint32(network.IP) | networking.ConvertIPv4ToUint32(defaultEndIP)) - return +func getIPRangeForMetallb(network net.IPNet) (startIP, endIP netip.Addr) { + // we trust that this is a valid prefix here because we already checked it in docker.GetDockerContainerIPNetwork + prefix := netip.MustParsePrefix(network.String()) + half := prefix.Bits() + 1 + wholeRange := netipx.RangeOfPrefix(prefix) + upperHalfPrefix := netip.PrefixFrom(wholeRange.To(), half).Masked() + halfRange := netipx.RangeOfPrefix(upperHalfPrefix) + return halfRange.From().Next(), halfRange.To().Prev() } // TODO: needs to be replaced with non-kubectl, just used this originally for speed. diff --git a/pkg/clusters/addons/metallb/metallb_test.go b/pkg/clusters/addons/metallb/metallb_test.go index 0319ee3a..cb333332 100644 --- a/pkg/clusters/addons/metallb/metallb_test.go +++ b/pkg/clusters/addons/metallb/metallb_test.go @@ -5,17 +5,17 @@ import ( "testing" "github.com/stretchr/testify/assert" - - "github.com/kong/kubernetes-testing-framework/pkg/utils/networking" ) func TestHelperFunctions(t *testing.T) { network := net.IPNet{ IP: net.IPv4(192, 168, 1, 0), - Mask: net.IPv4Mask(0, 0, 0, 255), + Mask: net.IPv4Mask(255, 255, 255, 0), } + // this should choose the upper half of the input network, minus network and broadcast addresses + // since we start with 192.168.1.0/24, we should get 192.168.1.128/25. the complete range is + // 192.168.1.128-192.168.1.255, and the returned range is thus 192.168.1.129-192.168.1.254. ip1, ip2 := getIPRangeForMetallb(network) - assert.Equal(t, ip1.String(), net.IPv4(192, 168, 1, 100).String()) - assert.Equal(t, ip2.String(), net.IPv4(192, 168, 1, 250).String()) - assert.Equal(t, networking.GetIPRangeStr(ip1, ip2), "192.168.1.100-192.168.1.250") + assert.Equal(t, net.IPv4(192, 168, 1, 129).String(), ip1.String()) + assert.Equal(t, net.IPv4(192, 168, 1, 254).String(), ip2.String()) } diff --git a/pkg/clusters/cluster.go b/pkg/clusters/cluster.go index 3b1d6d10..7bfd1a15 100644 --- a/pkg/clusters/cluster.go +++ b/pkg/clusters/cluster.go @@ -15,6 +15,17 @@ import ( // Type indicates the type of Kubernetes cluster (e.g. Kind, GKE, e.t.c.) type Type string +type IPFamily string + +const ( + // IPv4 indicates a Cluster that supports only IPv4 networking. + IPv4 IPFamily = "ipv4" + // IPv6 indicates a Cluster that supports only IPv6 networking. + IPv6 IPFamily = "ipv6" + // Dual indicates a Cluster that supports both IPv4 and IPv6 networking. + Dual IPFamily = "dual" +) + // Cluster objects represent a running Kubernetes cluster. type Cluster interface { // Name indicates the unique name of the running cluster. @@ -51,6 +62,9 @@ type Cluster interface { // of said directory and an error. // It uses the provided meta string allow for diagnostics identification. DumpDiagnostics(ctx context.Context, meta string) (string, error) + + // IPFamily returns the cluster's IP networking capabilities. + IPFamily() IPFamily } type Builder interface { diff --git a/pkg/clusters/types/gke/builder.go b/pkg/clusters/types/gke/builder.go index 3bb78a3b..8b59f2df 100644 --- a/pkg/clusters/types/gke/builder.go +++ b/pkg/clusters/types/gke/builder.go @@ -229,6 +229,8 @@ func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) { cfg: restCFG, addons: make(clusters.Addons), l: &sync.RWMutex{}, + // we simply set this directly for GKE as we lack the ability to create other types of cluster + ipFamily: clusters.IPv4, } if err := utils.ClusterInitHooks(ctx, cluster); err != nil { diff --git a/pkg/clusters/types/gke/cluster.go b/pkg/clusters/types/gke/cluster.go index cb21f8c4..29148d9b 100644 --- a/pkg/clusters/types/gke/cluster.go +++ b/pkg/clusters/types/gke/cluster.go @@ -36,6 +36,7 @@ type Cluster struct { cfg *rest.Config addons clusters.Addons l *sync.RWMutex + ipFamily clusters.IPFamily } // NewFromExistingWithEnv provides a new clusters.Cluster backed by an existing GKE cluster, @@ -278,3 +279,7 @@ func (c *Cluster) DumpDiagnostics(ctx context.Context, meta string) (string, err return outDir, err } + +func (c *Cluster) IPFamily() clusters.IPFamily { + return c.ipFamily +} diff --git a/pkg/clusters/types/kind/builder.go b/pkg/clusters/types/kind/builder.go index b11e552a..08d65132 100644 --- a/pkg/clusters/types/kind/builder.go +++ b/pkg/clusters/types/kind/builder.go @@ -25,6 +25,7 @@ type Builder struct { configPath *string configReader io.Reader calicoCNI bool + ipv6Only bool } // NewBuilder provides a new *Builder object. @@ -74,6 +75,12 @@ func (b *Builder) WithCalicoCNI() *Builder { return b } +// WithIPv6Only configures KIND to only use IPv6. +func (b *Builder) WithIPv6Only() *Builder { + b.ipv6Only = true + return b +} + // Build creates and configures clients for a Kind-based Kubernetes clusters.Cluster. func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) { deployArgs := make([]string, 0) @@ -92,6 +99,12 @@ func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) { deployArgs = append(deployArgs, "--wait", "1s") } + if b.ipv6Only { + if err := b.useIPv6Only(); err != nil { + return nil, fmt.Errorf("failed configuring IPv6-only networking: %w", err) + } + } + var stdin io.Reader if b.configPath != nil { deployArgs = append(deployArgs, "--config", *b.configPath) @@ -116,6 +129,11 @@ func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) { return nil, err } + ipFamily := clusters.IPv4 + if b.ipv6Only { + ipFamily = clusters.IPv6 + } + cluster := &Cluster{ name: b.Name, client: kc, @@ -123,6 +141,7 @@ func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) { addons: make(clusters.Addons), deployArgs: deployArgs, l: &sync.RWMutex{}, + ipFamily: ipFamily, } if b.calicoCNI { diff --git a/pkg/clusters/types/kind/cluster.go b/pkg/clusters/types/kind/cluster.go index fed8be1c..45ae3875 100644 --- a/pkg/clusters/types/kind/cluster.go +++ b/pkg/clusters/types/kind/cluster.go @@ -38,6 +38,7 @@ type Cluster struct { addons clusters.Addons deployArgs []string l *sync.RWMutex + ipFamily clusters.IPFamily } // New provides a new clusters.Cluster backed by a Kind based Kubernetes Cluster. @@ -158,3 +159,7 @@ func (c *Cluster) DumpDiagnostics(ctx context.Context, meta string) (string, err err = clusters.DumpDiagnostics(ctx, c, meta, outDir) return outDir, err } + +func (c *Cluster) IPFamily() clusters.IPFamily { + return c.ipFamily +} diff --git a/pkg/clusters/types/kind/utils.go b/pkg/clusters/types/kind/utils.go index 47abe82b..d2a201a9 100644 --- a/pkg/clusters/types/kind/utils.go +++ b/pkg/clusters/types/kind/utils.go @@ -141,6 +141,38 @@ func (b *Builder) disableDefaultCNI() error { return nil } +func (b *Builder) useIPv6Only() error { + if err := b.ensureConfigFile(); err != nil { + return err + } + + configYAML, err := os.ReadFile(*b.configPath) + if err != nil { + return fmt.Errorf("failed reading kind config from %s: %w", *b.configPath, err) + } + + kindConfig := v1alpha4.Cluster{} + if err := yaml.Unmarshal(configYAML, &kindConfig); err != nil { + return fmt.Errorf("failed unmarshalling kind config: %w", err) + } + + kindConfig.Networking.IPFamily = v1alpha4.IPv6Family + // For Windows/OS X Docker compatibility: + // https://kind.sigs.k8s.io/docs/user/configuration/#ip-family + kindConfig.Networking.APIServerAddress = "127.0.0.1" + + configYAML, err = yaml.Marshal(kindConfig) + if err != nil { + return fmt.Errorf("failed marshalling kind config: %w", err) + } + + err = os.WriteFile(*b.configPath, configYAML, 0o600) //nolint:gomnd + if err != nil { + return fmt.Errorf("failed writing kind config %s: %w", *b.configPath, err) + } + return nil +} + // exportLogs dumps a kind cluster logs to the specified directory func exportLogs(ctx context.Context, name string, outDir string) error { args := []string{"export", "logs", outDir, "--name", name} diff --git a/pkg/environments/builder.go b/pkg/environments/builder.go index 17d59875..62ea6022 100644 --- a/pkg/environments/builder.go +++ b/pkg/environments/builder.go @@ -26,6 +26,7 @@ type Builder struct { clusterBuilder clusters.Builder kubernetesVersion *semver.Version calicoCNI bool + ipv6Only bool } // NewBuilder generates a new empty Builder for creating Environments. @@ -78,6 +79,12 @@ func (b *Builder) WithCalicoCNI() *Builder { return b } +// WithIPv6Only configures KIND to only use IPv6. +func (b *Builder) WithIPv6Only() *Builder { + b.ipv6Only = true + return b +} + // Build is a blocking call to construct the configured Environment and it's // underlying Kubernetes cluster. The amount of time that it blocks depends // entirely on the underlying clusters.Cluster implementation that was requested. @@ -88,6 +95,10 @@ func (b *Builder) Build(ctx context.Context) (env Environment, err error) { return nil, fmt.Errorf("trying to deploy Calico CNI on an existing cluster is not currently supported") } + if b.ipv6Only && b.existingCluster != nil { + return nil, fmt.Errorf("trying to configure IPv6 only on an existing cluster is not currently supported") + } + if b.existingCluster != nil && b.clusterBuilder != nil { return nil, fmt.Errorf("Environment cannot specify both existingCluster and clusterBuilder") } @@ -114,6 +125,9 @@ func (b *Builder) Build(ctx context.Context) (env Environment, err error) { if b.calicoCNI { builder.WithCalicoCNI() } + if b.ipv6Only { + builder.WithIPv6Only() + } cluster, err = builder.Build(ctx) if err != nil { return nil, err diff --git a/pkg/utils/docker/docker.go b/pkg/utils/docker/docker.go index bdda1ff6..338d08c7 100644 --- a/pkg/utils/docker/docker.go +++ b/pkg/utils/docker/docker.go @@ -24,19 +24,23 @@ func InspectDockerContainer(containerID string) (*types.ContainerJSON, error) { return &containerJSON, err } +// TODO should be converted to net/ip to net/netip, but this requires a breaking change to a public function + // GetDockerContainerIPNetwork supports retreiving the *net.IP4Net of a container specified // by name (and a specified network name for the case of multiple networks). -func GetDockerContainerIPNetwork(containerID, networkName string) (*net.IPNet, error) { +func GetDockerContainerIPNetwork(containerID, networkName string) (*net.IPNet, *net.IPNet, error) { container, err := InspectDockerContainer(containerID) if err != nil { - return nil, err + return nil, nil, err } dockerNetwork := container.NetworkSettings.Networks[networkName] _, network, err := net.ParseCIDR(fmt.Sprintf("%s/%d", dockerNetwork.Gateway, dockerNetwork.IPPrefixLen)) - if err != nil { - return nil, err + _, network6, err6 := net.ParseCIDR(fmt.Sprintf("%s/%d", dockerNetwork.IPv6Gateway, dockerNetwork.GlobalIPv6PrefixLen)) + + if network == nil && network6 == nil { + return nil, nil, fmt.Errorf("no addresses found, IPv4Error(\"%s\"), IPv6Error(\"%s\")", err, err6) } - return network, nil + return network, network6, nil } diff --git a/pkg/utils/networking/networking.go b/pkg/utils/networking/networking.go index 892c6e0c..92e0cf61 100644 --- a/pkg/utils/networking/networking.go +++ b/pkg/utils/networking/networking.go @@ -10,6 +10,10 @@ import ( // Public Functions - Helper // ----------------------------------------------------------------------------- +// TODO the tools in this file are no longer used internally. they have been replaced with stdlib+third party packages. +// they remain here because they were public and removing them counts as a breaking change, but we probably should +// remove them. + const ( ipv4len = 16 ipv4bytes = 4 diff --git a/test/integration/ipv6_test.go b/test/integration/ipv6_test.go new file mode 100644 index 00000000..929de416 --- /dev/null +++ b/test/integration/ipv6_test.go @@ -0,0 +1,67 @@ +//go:build integration_tests + +package integration + +import ( + "fmt" + "net" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kong/kubernetes-testing-framework/pkg/clusters/addons/kong" + "github.com/kong/kubernetes-testing-framework/pkg/clusters/addons/metallb" + "github.com/kong/kubernetes-testing-framework/pkg/environments" +) + +func TestKindClusterWithIPv6(t *testing.T) { + t.Parallel() + + t.Log("configuring the test environment with IPv6 only enabled") + builder := environments.NewBuilder().WithIPv6Only().WithAddons(metallb.New(), kong.New()) + + t.Log("building the testing environment and Kubernetes cluster") + env, err := builder.Build(ctx) + require.NoError(t, err) + + t.Log("waiting for the testing environment to be ready") + require.NoError(t, <-env.WaitForReady(ctx)) + defer func() { assert.NoError(t, env.Cleanup(ctx)) }() + + endpoints, err := env.Cluster().Client().CoreV1().Endpoints(corev1.NamespaceDefault).Get(ctx, "kubernetes", metav1.GetOptions{}) + for _, subset := range endpoints.Subsets { + for _, addr := range subset.Addresses { + parsed := net.ParseIP(addr.IP) + require.Nil(t, parsed.To4()) + } + } + + require.Eventually(t, func() bool { + kongServices := env.Cluster().Client().CoreV1().Services(kong.DefaultNamespace) + service, err := kongServices.Get(ctx, kong.DefaultProxyServiceName, metav1.GetOptions{}) + if err != nil { + return false + } + + if len(service.Status.LoadBalancer.Ingress) == 0 { + return false + } + + if net.ParseIP(service.Status.LoadBalancer.Ingress[0].IP).To4() != nil { + return false + } + kongURL := fmt.Sprintf("http://[%s]", service.Status.LoadBalancer.Ingress[0].IP) + resp, err := http.Get(kongURL) + // we don't care that the proxy has nothing to serve so long as we can talk to it and get a valid HTTP response + if err == nil && resp.StatusCode == http.StatusNotFound { + return true + } + return false + + }, time.Minute*3, time.Second) +}