From caf882e90836f16819f4e86743fbd8c83ebb5bd5 Mon Sep 17 00:00:00 2001 From: Manabu McCloskey Date: Thu, 29 Aug 2024 15:15:45 -0700 Subject: [PATCH] ensure ingress-nginx node label is applied (#377) Signed-off-by: Manabu McCloskey --- pkg/kind/cluster.go | 44 +++++++++++++++++-- pkg/kind/cluster_test.go | 38 ++++++++++++---- pkg/kind/testdata/expected/label-only.yaml | 21 +++++++++ pkg/kind/testdata/expected/no-port-multi.yaml | 19 ++++++++ .../{no-necessary-port.yaml => no-port.yaml} | 5 ++- ...ssary-port-present.yaml => port-only.yaml} | 8 ++-- pkg/kind/testdata/label-only.yaml | 15 +++++++ pkg/kind/testdata/no-node.yaml | 2 + pkg/kind/testdata/no-port-multi.yaml | 13 ++++++ .../{no-necessary-port.yaml => no-port.yaml} | 0 ...ssary-port-present.yaml => port-only.yaml} | 6 +-- 11 files changed, 152 insertions(+), 19 deletions(-) create mode 100644 pkg/kind/testdata/expected/label-only.yaml create mode 100644 pkg/kind/testdata/expected/no-port-multi.yaml rename pkg/kind/testdata/expected/{no-necessary-port.yaml => no-port.yaml} (82%) rename pkg/kind/testdata/expected/{necessary-port-present.yaml => port-only.yaml} (74%) create mode 100644 pkg/kind/testdata/label-only.yaml create mode 100644 pkg/kind/testdata/no-node.yaml create mode 100644 pkg/kind/testdata/no-port-multi.yaml rename pkg/kind/testdata/{no-necessary-port.yaml => no-port.yaml} (100%) rename pkg/kind/testdata/{necessary-port-present.yaml => port-only.yaml} (73%) diff --git a/pkg/kind/cluster.go b/pkg/kind/cluster.go index 38984ced..08e9988b 100644 --- a/pkg/kind/cluster.go +++ b/pkg/kind/cluster.go @@ -19,6 +19,11 @@ import ( "sigs.k8s.io/yaml" ) +const ( + ingressNginxNodeLabelKey = "ingress-ready" + ingressNginxNodeLabelValue = "true" +) + var ( setupLog = log.Log.WithName("setup") ) @@ -241,20 +246,43 @@ func (c *Cluster) ensureCorrectConfig(in []byte) (kindv1alpha4.Cluster, error) { if err != nil { return kindv1alpha4.Cluster{}, fmt.Errorf("parsing kind config: %w", err) } - + // the port and ingress-nginx label must be on the same node to ensure nginx runs on the node with the right port. appendNecessaryPort := true + appendIngressNodeLabel := true + // pick the first node for the ingress-nginx if we need to configure node port. + nodePosition := 0 + + if parsedCluster.Nodes == nil || len(parsedCluster.Nodes) == 0 { + return kindv1alpha4.Cluster{}, fmt.Errorf("provided kind config does not have the node field defined") + } + nodes: for i := range parsedCluster.Nodes { node := parsedCluster.Nodes[i] for _, pm := range node.ExtraPortMappings { if strconv.Itoa(int(pm.HostPort)) == c.cfg.Port { appendNecessaryPort = false + nodePosition = i + if node.Labels != nil { + v, ok := node.Labels[ingressNginxNodeLabelKey] + if ok && v == ingressNginxNodeLabelValue { + appendIngressNodeLabel = false + } + } + break nodes + } + } + if node.Labels != nil { + v, ok := node.Labels[ingressNginxNodeLabelKey] + if ok && v == ingressNginxNodeLabelValue { + appendIngressNodeLabel = false + nodePosition = i break nodes } } } - if appendNecessaryPort && len(parsedCluster.Nodes) != 0 { + if appendNecessaryPort { hp, err := strconv.Atoi(c.cfg.Port) if err != nil { return kindv1alpha4.Cluster{}, fmt.Errorf("converting port, %s, to int: %w", c.cfg.Port, err) @@ -262,7 +290,17 @@ nodes: // either "80" or "443". No need to check for err cp, _ := strconv.Atoi(containerPort) - parsedCluster.Nodes[0].ExtraPortMappings = append(parsedCluster.Nodes[0].ExtraPortMappings, kindv1alpha4.PortMapping{ContainerPort: int32(cp), HostPort: int32(hp)}) + if parsedCluster.Nodes[nodePosition].ExtraPortMappings == nil { + parsedCluster.Nodes[nodePosition].ExtraPortMappings = make([]kindv1alpha4.PortMapping, 0, 1) + } + parsedCluster.Nodes[nodePosition].ExtraPortMappings = + append(parsedCluster.Nodes[nodePosition].ExtraPortMappings, kindv1alpha4.PortMapping{ContainerPort: int32(cp), HostPort: int32(hp), Protocol: "TCP"}) + } + if appendIngressNodeLabel { + if parsedCluster.Nodes[nodePosition].Labels == nil { + parsedCluster.Nodes[nodePosition].Labels = make(map[string]string) + } + parsedCluster.Nodes[nodePosition].Labels[ingressNginxNodeLabelKey] = ingressNginxNodeLabelValue } return parsedCluster, nil diff --git a/pkg/kind/cluster_test.go b/pkg/kind/cluster_test.go index 7f7bcc1b..d4bfeed9 100644 --- a/pkg/kind/cluster_test.go +++ b/pkg/kind/cluster_test.go @@ -98,21 +98,38 @@ func TestGetConfigCustom(t *testing.T) { inputPath string outputPath string hostPort string - Protocol string + protocol string + error bool } cases := []testCase{ { - inputPath: "testdata/no-necessary-port.yaml", - outputPath: "testdata/expected/no-necessary-port.yaml", + inputPath: "testdata/no-port.yaml", + outputPath: "testdata/expected/no-port.yaml", hostPort: "8443", - Protocol: "https", + protocol: "https", }, { - inputPath: "testdata/necessary-port-present.yaml", - outputPath: "testdata/expected/necessary-port-present.yaml", + inputPath: "testdata/port-only.yaml", + outputPath: "testdata/expected/port-only.yaml", hostPort: "80", - Protocol: "http", + protocol: "http", + }, + { + inputPath: "testdata/no-port-multi.yaml", + outputPath: "testdata/expected/no-port-multi.yaml", + hostPort: "8443", + protocol: "https", + }, + { + inputPath: "testdata/label-only.yaml", + outputPath: "testdata/expected/label-only.yaml", + hostPort: "8443", + protocol: "https", + }, + { + inputPath: "testdata/no-node", + error: true, }, } @@ -120,15 +137,18 @@ func TestGetConfigCustom(t *testing.T) { c, _ := NewCluster("testcase", "v1.26.3", "", v.inputPath, "", util.CorePackageTemplateConfig{ Host: "cnoe.localtest.me", Port: v.hostPort, - Protocol: v.Protocol, + Protocol: v.protocol, }) b, err := c.getConfig() + if v.error { + assert.Error(t, err) + continue + } assert.NoError(t, err) expected, _ := os.ReadFile(v.outputPath) assert.YAMLEq(t, string(expected), string(b)) } - } // Mock provider for testing diff --git a/pkg/kind/testdata/expected/label-only.yaml b/pkg/kind/testdata/expected/label-only.yaml new file mode 100644 index 00000000..762fe21c --- /dev/null +++ b/pkg/kind/testdata/expected/label-only.yaml @@ -0,0 +1,21 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: {} +nodes: + - role: control-plane + extraPortMappings: + - containerPort: 31337 + hostPort: 31337 + - containerPort: 31340 + hostPort: 31340 + - containerPort: 31333 + hostPort: 31333 + - role: worker + image: "abc" + labels: + ingress-ready: "true" + extraPortMappings: + - containerPort: 443 + hostPort: 8443 + protocol: TCP + diff --git a/pkg/kind/testdata/expected/no-port-multi.yaml b/pkg/kind/testdata/expected/no-port-multi.yaml new file mode 100644 index 00000000..d20814c3 --- /dev/null +++ b/pkg/kind/testdata/expected/no-port-multi.yaml @@ -0,0 +1,19 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: {} +nodes: + - role: control-plane + labels: + ingress-ready: "true" + extraPortMappings: + - containerPort: 31337 + hostPort: 31337 + - containerPort: 31340 + hostPort: 31340 + - containerPort: 31333 + hostPort: 31333 + - containerPort: 443 + hostPort: 8443 + protocol: TCP + - role: worker + image: "abc" diff --git a/pkg/kind/testdata/expected/no-necessary-port.yaml b/pkg/kind/testdata/expected/no-port.yaml similarity index 82% rename from pkg/kind/testdata/expected/no-necessary-port.yaml rename to pkg/kind/testdata/expected/no-port.yaml index bff4ddb2..60610f67 100644 --- a/pkg/kind/testdata/expected/no-necessary-port.yaml +++ b/pkg/kind/testdata/expected/no-port.yaml @@ -3,6 +3,8 @@ apiVersion: kind.x-k8s.io/v1alpha4 networking: {} nodes: - role: control-plane + labels: + ingress-ready: "true" extraMounts: - containerPath: /var/lib/kubelet/config.json hostPath: ~/.docker/config.json @@ -14,4 +16,5 @@ nodes: - containerPort: 31333 hostPort: 31333 - containerPort: 443 - hostPort: 8443 \ No newline at end of file + hostPort: 8443 + protocol: TCP diff --git a/pkg/kind/testdata/expected/necessary-port-present.yaml b/pkg/kind/testdata/expected/port-only.yaml similarity index 74% rename from pkg/kind/testdata/expected/necessary-port-present.yaml rename to pkg/kind/testdata/expected/port-only.yaml index 0ba23326..4df18dc4 100644 --- a/pkg/kind/testdata/expected/necessary-port-present.yaml +++ b/pkg/kind/testdata/expected/port-only.yaml @@ -3,9 +3,6 @@ apiVersion: kind.x-k8s.io/v1alpha4 networking: {} nodes: - role: control-plane - extraMounts: - - containerPath: /var/lib/kubelet/config.json - hostPath: ~/.docker/config.json extraPortMappings: - containerPort: 31337 hostPort: 31337 @@ -13,5 +10,10 @@ nodes: hostPort: 31340 - containerPort: 31333 hostPort: 31333 + - role: worker + labels: + ingress-ready: "true" + extraPortMappings: - containerPort: 80 hostPort: 80 + protocol: TCP diff --git a/pkg/kind/testdata/label-only.yaml b/pkg/kind/testdata/label-only.yaml new file mode 100644 index 00000000..4beba13c --- /dev/null +++ b/pkg/kind/testdata/label-only.yaml @@ -0,0 +1,15 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + extraPortMappings: + - containerPort: 31337 + hostPort: 31337 + - containerPort: 31340 + hostPort: 31340 + - containerPort: 31333 + hostPort: 31333 + - role: worker + image: "abc" + labels: + ingress-ready: "true" diff --git a/pkg/kind/testdata/no-node.yaml b/pkg/kind/testdata/no-node.yaml new file mode 100644 index 00000000..18eb9ae2 --- /dev/null +++ b/pkg/kind/testdata/no-node.yaml @@ -0,0 +1,2 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 diff --git a/pkg/kind/testdata/no-port-multi.yaml b/pkg/kind/testdata/no-port-multi.yaml new file mode 100644 index 00000000..e43bd7f8 --- /dev/null +++ b/pkg/kind/testdata/no-port-multi.yaml @@ -0,0 +1,13 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + extraPortMappings: + - containerPort: 31337 + hostPort: 31337 + - containerPort: 31340 + hostPort: 31340 + - containerPort: 31333 + hostPort: 31333 + - role: worker + image: "abc" diff --git a/pkg/kind/testdata/no-necessary-port.yaml b/pkg/kind/testdata/no-port.yaml similarity index 100% rename from pkg/kind/testdata/no-necessary-port.yaml rename to pkg/kind/testdata/no-port.yaml diff --git a/pkg/kind/testdata/necessary-port-present.yaml b/pkg/kind/testdata/port-only.yaml similarity index 73% rename from pkg/kind/testdata/necessary-port-present.yaml rename to pkg/kind/testdata/port-only.yaml index aceee5c6..50f1652a 100644 --- a/pkg/kind/testdata/necessary-port-present.yaml +++ b/pkg/kind/testdata/port-only.yaml @@ -2,9 +2,6 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - extraMounts: - - containerPath: /var/lib/kubelet/config.json - hostPath: ~/.docker/config.json extraPortMappings: - containerPort: 31337 hostPort: 31337 @@ -12,5 +9,8 @@ nodes: hostPort: 31340 - containerPort: 31333 hostPort: 31333 + - role: worker + extraPortMappings: - containerPort: 80 hostPort: 80 + protocol: TCP