vm based providers: Expose a UDP port for DNS (#867)

* base image: Add debug packages Those packages add helpfull commands such as ping, nslookup, tcpdump, that are needed when developing network features. Signed-off-by: Or Shoval <[email protected]> * vm based providers: Expose a UDP port for DNS Expose a custom UDP port (selected 31111) by the vm based providers. This allows to expose a nodePort of a service, set it's node port to 31111, and then it will be reachable from the host itself. Signed-off-by: Or Shoval <[email protected]> Signed-off-by: Or Shoval <[email protected]>
kubevirt · Nov 2, 2022 · 8cca8c0 · 8cca8c0
1 parent 2fea446
commit 8cca8c0
Show file tree

Hide file tree

Showing 9 changed files with 71 additions and 28 deletions.
diff --git a/cluster-provision/centos8/Dockerfile b/cluster-provision/centos8/Dockerfile
@@ -3,7 +3,7 @@ FROM quay.io/kubevirtci/fedora@sha256:486fd5578f93fbc57a519e34ad4b7cac927c3f8a95
 
 ARG centos_version
 
-RUN dnf -y install jq iptables iproute dnsmasq qemu openssh-clients screen && dnf clean all
+RUN dnf -y install jq iptables iproute dnsmasq qemu openssh-clients screen bind-utils tcpdump iputils && dnf clean all
 
 WORKDIR /
 

diff --git a/cluster-provision/centos8/scripts/vm.sh b/cluster-provision/centos8/scripts/vm.sh
@@ -73,16 +73,28 @@ else
   iptables -t nat -A OUTPUT -p tcp --dport 22${n} -j DNAT --to-destination 192.168.66.1${n}:22
 fi
 
+function create_ip_rules {
+  protocol=$1
+  shift
+  if [ "$ROOTLESS" -ne 1 ]; then
+    for port in "$@"; do
+      iptables -t nat -A PREROUTING -p ${protocol} -i eth0 -m ${protocol} --dport ${port} -j DNAT --to-destination 192.168.66.101:${port}
+    done
+  else
+    for port in "$@"; do
+      # Add DNAT rule for rootless podman (traffic originating from loopback adapter)
+      iptables -t nat -A OUTPUT -p ${protocol} --dport ${port} -j DNAT --to-destination 192.168.66.101:${port}
+    done
+  fi
+}
+
 # Route ports from container to VM for first node
 if [ "$n" = "01" ] ; then
-  for port in 6443 8443 80 443 30007 30008 31001; do
-    if [ "$ROOTLESS" -ne 1 ]; then
-      iptables -t nat -A PREROUTING -p tcp -i eth0 -m tcp --dport ${port} -j DNAT --to-destination 192.168.66.1${n}:${port}
-    else
-      # Add DNAT rule for rootless podman (traffic originating from loopback adapter)
-      iptables -t nat -A OUTPUT -p tcp --dport ${port} -j DNAT --to-destination 192.168.66.1${n}:${port}
-    fi
-  done
+  tcp_ports=( 6443 8443 80 443 30007 30008 31001 )
+  create_ip_rules "tcp" "${tcp_ports[@]}"
+
+  udp_ports=( 31111 )
+  create_ip_rules "udp" "${udp_ports[@]}"
 fi
 
 # For backward compatibility, so that we can just copy over the newer files

diff --git a/cluster-provision/gocli/cmd/ports.go b/cluster-provision/gocli/cmd/ports.go
@@ -32,7 +32,7 @@ Known port names are 'ssh', 'registry', 'ocp', 'k8s', 'prometheus' and 'grafana'
 
 			if len(args) == 1 {
 				switch args[0] {
-				case utils.PortNameSSH, utils.PortNameSSHWorker, utils.PortNameAPI, utils.PortNameOCP, utils.PortNameOCPConsole, utils.PortNameRegistry, utils.PortNameVNC, utils.PortNameHTTP, utils.PortNameHTTPS, utils.PortNamePrometheus, utils.PortNameGrafana, utils.PortNameUploadProxy:
+				case utils.PortNameSSH, utils.PortNameSSHWorker, utils.PortNameAPI, utils.PortNameOCP, utils.PortNameOCPConsole, utils.PortNameRegistry, utils.PortNameVNC, utils.PortNameHTTP, utils.PortNameHTTPS, utils.PortNamePrometheus, utils.PortNameGrafana, utils.PortNameUploadProxy, utils.PortNameDNS:
 					return nil
 				default:
 					return fmt.Errorf("unknown port name %s", args[0])
@@ -109,6 +109,8 @@ func ports(cmd *cobra.Command, args []string) error {
 			err = utils.PrintPublicPort(utils.PortGrafana, container.NetworkSettings.Ports)
 		case utils.PortNameUploadProxy:
 			err = utils.PrintPublicPort(utils.PortUploadProxy, container.NetworkSettings.Ports)
+		case utils.PortNameDNS:
+			err = utils.PrintPublicPort(utils.PortDNS, container.NetworkSettings.Ports)
 		}
 
 		if err != nil {

diff --git a/cluster-provision/gocli/cmd/provision.go b/cluster-provision/gocli/cmd/provision.go
@@ -108,8 +108,8 @@ func provisionCluster(cmd *cobra.Command, args []string) (retErr error) {
 
 	portMap := nat.PortMap{}
 
-	utils.AppendIfExplicit(portMap, utils.PortSSH, cmd.Flags(), "ssh-port")
-	utils.AppendIfExplicit(portMap, utils.PortVNC, cmd.Flags(), "vnc-port")
+	utils.AppendTCPIfExplicit(portMap, utils.PortSSH, cmd.Flags(), "ssh-port")
+	utils.AppendTCPIfExplicit(portMap, utils.PortVNC, cmd.Flags(), "vnc-port")
 
 	qemuArgs, err := cmd.Flags().GetString("qemu-args")
 	if err != nil {

diff --git a/cluster-provision/gocli/cmd/run.go b/cluster-provision/gocli/cmd/run.go
@@ -91,6 +91,7 @@ func NewRunCommand() *cobra.Command {
 	run.Flags().Uint("ssh-port", 0, "port on localhost for ssh server")
 	run.Flags().Uint("prometheus-port", 0, "port on localhost for prometheus server")
 	run.Flags().Uint("grafana-port", 0, "port on localhost for grafana server")
+	run.Flags().Uint("dns-port", 0, "port on localhost for dns server")
 	run.Flags().String("nfs-data", "", "path to data which should be exposed via nfs to the nodes")
 	run.Flags().Bool("enable-ceph", false, "enables dynamic storage provisioning using Ceph")
 	run.Flags().Bool("enable-istio", false, "deploys Istio service mesh")
@@ -147,15 +148,16 @@ func run(cmd *cobra.Command, args []string) (retErr error) {
 
 	portMap := nat.PortMap{}
 
-	utils.AppendIfExplicit(portMap, utils.PortSSH, cmd.Flags(), "ssh-port")
-	utils.AppendIfExplicit(portMap, utils.PortVNC, cmd.Flags(), "vnc-port")
-	utils.AppendIfExplicit(portMap, utils.PortHTTP, cmd.Flags(), "http-port")
-	utils.AppendIfExplicit(portMap, utils.PortHTTPS, cmd.Flags(), "https-port")
-	utils.AppendIfExplicit(portMap, utils.PortAPI, cmd.Flags(), "k8s-port")
-	utils.AppendIfExplicit(portMap, utils.PortOCP, cmd.Flags(), "ocp-port")
-	utils.AppendIfExplicit(portMap, utils.PortRegistry, cmd.Flags(), "registry-port")
-	utils.AppendIfExplicit(portMap, utils.PortPrometheus, cmd.Flags(), "prometheus-port")
-	utils.AppendIfExplicit(portMap, utils.PortGrafana, cmd.Flags(), "grafana-port")
+	utils.AppendTCPIfExplicit(portMap, utils.PortSSH, cmd.Flags(), "ssh-port")
+	utils.AppendTCPIfExplicit(portMap, utils.PortVNC, cmd.Flags(), "vnc-port")
+	utils.AppendTCPIfExplicit(portMap, utils.PortHTTP, cmd.Flags(), "http-port")
+	utils.AppendTCPIfExplicit(portMap, utils.PortHTTPS, cmd.Flags(), "https-port")
+	utils.AppendTCPIfExplicit(portMap, utils.PortAPI, cmd.Flags(), "k8s-port")
+	utils.AppendTCPIfExplicit(portMap, utils.PortOCP, cmd.Flags(), "ocp-port")
+	utils.AppendTCPIfExplicit(portMap, utils.PortRegistry, cmd.Flags(), "registry-port")
+	utils.AppendTCPIfExplicit(portMap, utils.PortPrometheus, cmd.Flags(), "prometheus-port")
+	utils.AppendTCPIfExplicit(portMap, utils.PortGrafana, cmd.Flags(), "grafana-port")
+	utils.AppendUDPIfExplicit(portMap, utils.PortDNS, cmd.Flags(), "dns-port")
 
 	qemuArgs, err := cmd.Flags().GetString("qemu-args")
 	if err != nil {

diff --git a/cluster-provision/gocli/cmd/utils/ports.go b/cluster-provision/gocli/cmd/utils/ports.go
@@ -32,6 +32,8 @@ const (
 	PortGrafana = 30008
 	//PortUploadProxy contains CDI UploadProxy port
 	PortUploadProxy = 31001
+	//PortDNS contains DNS port
+	PortDNS = 31111
 
 	// PortNameSSH contains control-plane node SSH port name
 	PortNameSSH = "ssh"
@@ -58,13 +60,15 @@ const (
 	PortNameGrafana = "grafana"
 	// PortNameUploadProxy contains CDI UploadProxy port
 	PortNameUploadProxy = "uploadproxy"
+	// PortNameDNS contains UDP port
+	PortNameDNS = "dns"
 )
 
 // GetPublicPort returns public port by private port
 func GetPublicPort(port uint16, ports nat.PortMap) (uint16, error) {
-	portStr := strconv.Itoa(int(port)) + "/tcp"
+	portStr := strconv.Itoa(int(port))
 	for k, p := range ports {
-		if k == nat.Port(portStr) {
+		if k == nat.Port(portStr+"/tcp") || k == nat.Port(portStr+"/udp") {
 			if len(p) > 0 {
 				publicPort, err := strconv.Atoi(p[0].HostPort)
 				if err != nil {
@@ -89,9 +93,18 @@ func PrintPublicPort(port uint16, ports nat.PortMap) error {
 	return nil
 }
 
-// TCPPortOrDie returns net.Port object or panic if cast failed
+// TCPPortOrDie returns net.Port TCP object or panic if cast failed
 func TCPPortOrDie(port int) nat.Port {
-	p, err := nat.NewPort("tcp", strconv.Itoa(port))
+	return portOrDie(port, "tcp")
+}
+
+// UDPPortOrDie returns net.Port UDP object or panic if cast failed
+func UDPPortOrDie(port int) nat.Port {
+	return portOrDie(port, "udp")
+}
+
+func portOrDie(port int, protocol string) nat.Port {
+	p, err := nat.NewPort(protocol, strconv.Itoa(port))
 	if err != nil {
 		panic(err)
 	}

diff --git a/cluster-provision/gocli/cmd/utils/utils.go b/cluster-provision/gocli/cmd/utils/utils.go
@@ -7,15 +7,24 @@ import (
 	"github.com/spf13/pflag"
 )
 
-// AppendIfExplicit append port to the portMap if the port flag exists
-func AppendIfExplicit(ports nat.PortMap, exposedPort int, flagSet *pflag.FlagSet, flagName string) error {
+// AppendTCPIfExplicit append TCP port to the portMap if the port flag exists
+func AppendTCPIfExplicit(ports nat.PortMap, exposedPort int, flagSet *pflag.FlagSet, flagName string) error {
+	return appendIfExplicit(ports, exposedPort, flagSet, flagName, TCPPortOrDie)
+}
+
+// AppendUDPIfExplicit append UDP port to the portMap if the port flag exists
+func AppendUDPIfExplicit(ports nat.PortMap, exposedPort int, flagSet *pflag.FlagSet, flagName string) error {
+	return appendIfExplicit(ports, exposedPort, flagSet, flagName, UDPPortOrDie)
+}
+
+func appendIfExplicit(ports nat.PortMap, exposedPort int, flagSet *pflag.FlagSet, flagName string, portFn func(port int) nat.Port) error {
 	flag := flagSet.Lookup(flagName)
 	if flag != nil && flag.Changed {
 		publicPort, err := flagSet.GetUint(flagName)
 		if err != nil {
 			return err
 		}
-		port := TCPPortOrDie(exposedPort)
+		port := portFn(exposedPort)
 		ports[port] = []nat.PortBinding{
 			{
 				HostIP:   "127.0.0.1",

diff --git a/cluster-provision/gocli/containers/dnsmasq.go b/cluster-provision/gocli/containers/dnsmasq.go
@@ -56,6 +56,7 @@ func DNSMasq(cli *client.Client, ctx context.Context, options *DNSMasqOptions) (
 			utils.TCPPortOrDie(utils.PortPrometheus):  {},
 			utils.TCPPortOrDie(utils.PortGrafana):     {},
 			utils.TCPPortOrDie(utils.PortUploadProxy): {},
+			utils.UDPPortOrDie(utils.PortDNS):         {},
 		},
 	}, &container.HostConfig{
 		Privileged:      true,

diff --git a/cluster-up/cluster/ephemeral-provider-common.sh b/cluster-up/cluster/ephemeral-provider-common.sh
@@ -87,6 +87,10 @@ function _registry_volume() {
 function _add_common_params() {
     # shellcheck disable=SC2155
     local params="--nodes ${KUBEVIRT_NUM_NODES} --memory ${KUBEVIRT_MEMORY_SIZE} --cpu 6 --secondary-nics ${KUBEVIRT_NUM_SECONDARY_NICS} --random-ports --background --prefix $provider_prefix ${KUBEVIRT_PROVIDER} ${KUBEVIRT_PROVIDER_EXTRA_ARGS}"
+
+    dns_host_port=53
+    params=" --dns-port $dns_host_port $params"
+
     if [[ $TARGET =~ windows_sysprep.* ]] && [ -n "$WINDOWS_SYSPREP_NFS_DIR" ]; then
         params=" --nfs-data $WINDOWS_SYSPREP_NFS_DIR $params"
     elif [[ $TARGET =~ windows.* ]] && [ -n "$WINDOWS_NFS_DIR" ]; then