Fix udp service access through k3d-proxy

This commit separates PORTS and UDP_PORTS and configures nginx appropriately. Signed-off-by: Dinar Valeev <[email protected]>
k3d-io · Feb 8, 2021 · b454a5f · b454a5f
1 parent 58d37be
commit b454a5f
Show file tree

Hide file tree

Showing 6 changed files with 87 additions and 6 deletions.
diff --git a/go.mod b/go.mod
@@ -21,6 +21,7 @@ require (
 	github.com/gogo/googleapis v1.4.0 // indirect
 	github.com/heroku/docker-registry-client v0.0.0-20190909225348-afc9e1acc3d5
 	github.com/imdario/mergo v0.3.9
+	github.com/iwilltry42/confd v0.16.1 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.3.3 // indirect

diff --git a/go.sum b/go.sum
diff --git a/pkg/client/cluster.go b/pkg/client/cluster.go
@@ -506,9 +506,15 @@ ClusterCreatOpts:
 			}
 
 			// generate comma-separated list of extra ports to forward
-			ports := k3d.DefaultAPIPort
+			ports := []string{k3d.DefaultAPIPort}
+			var udp_ports []string
 			for exposedPort := range cluster.ServerLoadBalancer.Ports {
-				ports += "," + exposedPort.Port()
+				if exposedPort.Proto() == "udp" {
+					udp_ports = append(udp_ports, exposedPort.Port())
+					continue
+				}
+				ports = append(ports, exposedPort.Port())
+
 			}
 
 			if cluster.ServerLoadBalancer.Ports == nil {
@@ -523,14 +529,17 @@ ClusterCreatOpts:
 				Ports: cluster.ServerLoadBalancer.Ports,
 				Env: []string{
 					fmt.Sprintf("SERVERS=%s", servers),
-					fmt.Sprintf("PORTS=%s", ports),
-					fmt.Sprintf("WORKER_PROCESSES=%d", len(strings.Split(ports, ","))),
+					fmt.Sprintf("PORTS=%s", strings.Join(ports, ",")),
+					fmt.Sprintf("WORKER_PROCESSES=%d", len(ports)),
 				},
 				Role:     k3d.LoadBalancerRole,
 				Labels:   clusterCreateOpts.GlobalLabels, // TODO: createLoadBalancer: add more expressive labels
 				Networks: []string{cluster.Network.Name},
 				Restart:  true,
 			}
+			if len(udp_ports) > 0 {
+				lbNode.Env = append(lbNode.Env, fmt.Sprintf("UDP_PORTS=%s", strings.Join(udp_ports, ",")))
+			}
 			cluster.Nodes = append(cluster.Nodes, lbNode) // append lbNode to list of cluster nodes, so it will be considered during rollback
 			log.Infof("Creating LoadBalancer '%s'", lbNode.Name)
 			if err := NodeCreate(clusterCreateCtx, runtime, lbNode, k3d.NodeCreateOpts{}); err != nil {

diff --git a/proxy/conf.d/nginx.toml b/proxy/conf.d/nginx.toml
@@ -4,4 +4,4 @@ dest = "/etc/nginx/nginx.conf"
 keys = [
     "SERVERS",
     "PORTS",
-]
+]
diff --git a/proxy/templates/nginx.tmpl b/proxy/templates/nginx.tmpl
@@ -1,5 +1,6 @@
 {{- $servers := split (getenv "SERVERS") "," -}}
 {{- $ports := split (getenv "PORTS") "," -}}
+{{- $udp_ports := split (getenv "UDP_PORTS") "," -}}
 error_log stderr notice;
 
 worker_processes auto;
@@ -24,4 +25,20 @@ stream {
     proxy_connect_timeout 2s;
   }
   {{- end }}
-}
+  {{- range $port := $udp_ports }}
+  {{- if $port }}
+  upstream server_nodes_{{ $port }} {
+    {{- range $server := $servers }}
+    server {{ $server }}:{{ $port }} max_fails=1 fail_timeout=10s;
+    {{- end }}
+  }
+
+  server {
+    listen        {{ $port }} udp;
+    proxy_pass    server_nodes_{{ $port }};
+    proxy_timeout 600;
+    proxy_connect_timeout 2s;
+  }
+  {{- end }}
+  {{- end }}
+}
diff --git a/vendor/modules.txt b/vendor/modules.txt
@@ -205,6 +205,8 @@ github.com/heroku/docker-registry-client/registry
 github.com/imdario/mergo
 # github.com/inconshreveable/mousetrap v1.0.0
 github.com/inconshreveable/mousetrap
+# github.com/iwilltry42/confd v0.16.1
+## explicit
 # github.com/json-iterator/go v1.1.8
 github.com/json-iterator/go
 # github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de