From 26cd3a6a197194f230f418d4c12f4e234160bb0d Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Sat, 13 Aug 2022 13:31:41 +0800 Subject: [PATCH 1/2] kubectl-ko: fix destination mac --- dist/images/kubectl-ko | 249 ++++++++++++++++++++++++++++++----------- 1 file changed, 181 insertions(+), 68 deletions(-) diff --git a/dist/images/kubectl-ko b/dist/images/kubectl-ko index 4f8a2f6813c..9dc0f0392b8 100755 --- a/dist/images/kubectl-ko +++ b/dist/images/kubectl-ko @@ -25,6 +25,105 @@ showHelp(){ echo " reload restart all kube-ovn components" } +# usage: ipv4_to_hex 192.168.0.1 +ipv4_to_hex(){ + printf "%02x" ${1//./ } +} + +# convert hex to dec (portable version) +hex2dec(){ + for i in $(echo "$@"); do + printf "%d\n" "$(( 0x$i ))" + done +} + +# https://github.com/chmduquesne/wg-ip +# usage: expand_ipv6 2001::1 +expand_ipv6(){ + local ip=$1 + + # prepend 0 if we start with : + echo $ip | grep -qs "^:" && ip="0${ip}" + + # expand :: + if echo $ip | grep -qs "::"; then + local colons=$(echo $ip | sed 's/[^:]//g') + local missing=$(echo ":::::::::" | sed "s/$colons//") + local expanded=$(echo $missing | sed 's/:/:0/g') + ip=$(echo $ip | sed "s/::/$expanded/") + fi + + local blocks=$(echo $ip | grep -o "[0-9a-f]\+") + set $blocks + + printf "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n" \ + $(hex2dec $@) +} + +# convert an IPv6 address to bytes +ipv6_bytes(){ + for x in $(expand_ipv6 $1 | tr ':' ' '); do + printf "%d %d " $((0x$x >> 8 & 0xff)) $((0x$x & 0xff)) + done + echo +} + +# usage: ipIsInCidr 192.168.0.1 192.168.0.0/24 +# return: 0 for true, 1 for false +ipIsInCidr(){ + local ip=$1 + local cidr=$2 + + if [[ $ip =~ .*:.* ]]; then + # IPv6 + cidr=${cidr#*,} + local network=${cidr%/*} + local prefix=${cidr#*/} + local ip_bytes=($(ipv6_bytes $ip)) + local network_bytes=($(ipv6_bytes $network)) + for ((i=0; i<${#ip_bytes[*]}; i++)); do + if [ ${ip_bytes[$i]} -eq ${network_bytes[$i]} ]; then + continue + fi + + if [ $((($i+1)*8)) -le $prefix ]; then + return 1 + fi + if [ $(($i*8)) -ge $prefix ]; then + return 0 + fi + if [ $((($i+1)*8)) -le $prefix ]; then + return 1 + fi + + local bits=$(($prefix-$i*8)) + local mask=$((0xff<<$bits & 0xff)) + # TODO: check whether the IP is network/broadcast address + if [ $((${ip_bytes[$i]} & $mask)) -ne ${network_bytes[$i]} ]; then + return 1 + fi + done + + return 0 + fi + + # IPv4 + cidr=${cidr%,*} + local network=${cidr%/*} + local prefix=${cidr#*/} + local ip_hex=$(ipv4_to_hex $ip) + local ip_dec=$((0x$ip_hex)) + local network_hex=$(ipv4_to_hex $network) + local network_dec=$((0x$network_hex)) + local broadcast_dec=$(($network_dec + 2**$prefix - 1)) + # TODO: check whether the IP is network/broadcast address + if [ $ip_dec -gt $network_dec -a $ip_dec -lt $broadcast_dec ]; then + return 0 + fi + + return 1 +} + tcpdump(){ namespacedPod="$1"; shift namespace=$(echo "$namespacedPod" | cut -d "/" -f1) @@ -69,12 +168,18 @@ tcpdump(){ trace(){ namespacedPod="$1" - namespace=$(echo "$1" | cut -d "/" -f1) - podName=$(echo "$1" | cut -d "/" -f2) - if [ "$podName" = "$1" ]; then + namespace=$(echo "$namespacedPod" | cut -d "/" -f1) + podName=$(echo "$namespacedPod" | cut -d "/" -f2) + if [ "$podName" = "$namespacedPod" ]; then namespace="default" fi + hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork}) + if [ "$hostNetwork" = "true" ]; then + echo "Can not trace host network pod" + exit 1 + fi + dst="$2" if [ -z "$dst" ]; then echo "need a target ip address" @@ -90,18 +195,6 @@ trace(){ proto="6" fi - hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork}) - if [ "$hostNetwork" = "true" ]; then - echo "Can not trace host network pod" - exit 1 - fi - - ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch}) - if [ -z "$ls" ]; then - echo "pod address not ready" - exit 1 - fi - podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath="{.status.podIPs[*].ip}")) if [ ${#podIPs[@]} -eq 0 ]; then podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/ip_address} | sed 's/,/ /g')) @@ -111,9 +204,6 @@ trace(){ fi fi - mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address}) - nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName}) - podIP="" for ip in ${podIPs[@]}; do if [ "$af" = "4" ]; then @@ -128,64 +218,87 @@ trace(){ done if [ -z "$podIP" ]; then - echo "Pod has no IPv$af address" + echo "Pod $namespacedPod has no IPv$af address" exit 1 fi - gwMac="" - vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan}) - logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway}) - if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then - gateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.gateway}) - if [[ "$gateway" =~ .*,.* ]]; then - if [ "$af" = "4" ]; then - gateway=${gateway%%,*} - else - gateway=${gateway##*,} - fi - fi + ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch}) + if [ -z "$ls" ]; then + echo "pod address not ready" + exit 1 + fi - ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep -w kube-ovn-cni | grep " $nodeName " | awk '{print $1}') - if [ -z "$ovnCni" ]; then - echo "No kube-ovn-cni Pod running on node $nodeName" - exit 1 - fi + local cidr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/cidr}) + mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address}) + nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName}) - nicName=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r') - if [ -z "$nicName" ]; then - echo "nic doesn't exist on node $nodeName" - exit 1 + dstMac="" + if ipIsInCidr $dst $cidr; then + if [ $af -eq 4 ]; then + set +o pipefail + dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -w "$(echo $dst | tr . '\.')" | awk '{print $1}') + set -o pipefail + else + set +o pipefail + dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -i " $dst\$" | awk '{print $1}') + set -o pipefail fi + fi + if [ -z "$dstMac" ]; then + vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan}) + logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway}) + if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then + gateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.gateway}) + if [[ "$gateway" =~ .*,.* ]]; then + if [ "$af" = "4" ]; then + gateway=${gateway%%,*} + else + gateway=${gateway##*,} + fi + fi - podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type}) - podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//') - if [ "$podNicType" != "internal-port" ]; then - nicName="eth0" - fi + ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep -w kube-ovn-cni | grep " $nodeName " | awk '{print $1}') + if [ -z "$ovnCni" ]; then + echo "No kube-ovn-cni Pod running on node $nodeName" + exit 1 + fi - if [[ "$gateway" =~ .*:.* ]]; then - cmd="ndisc6 -q $gateway $nicName" - output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ndisc6 -q "$gateway" "$nicName") - else - cmd="arping -c3 -C1 -i1 -I $nicName $gateway" - output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c3 -C1 -i1 -I "$nicName" "$gateway") - fi + nicName=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r') + if [ -z "$nicName" ]; then + echo "nic doesn't exist on node $nodeName" + exit 1 + fi - if [ $? -ne 0 ]; then - echo "failed to run '$cmd' in Pod's netns" - exit 1 - fi - gwMac=$(echo "$output" | grep -o -E '([[:xdigit:]]{1,2}:){5}[[:xdigit:]]{1,2}') - else - lr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_router}) - if [ -z "$lr" ]; then - lr=$(kubectl get subnet "$ls" -o jsonpath={.spec.vpc}) + podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type}) + podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//') + if [ "$podNicType" != "internal-port" ]; then + nicName="eth0" + fi + + if [[ "$gateway" =~ .*:.* ]]; then + cmd="ndisc6 -q $gateway $nicName" + output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ndisc6 -q "$gateway" "$nicName") + else + cmd="arping -c3 -C1 -i1 -I $nicName $gateway" + output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c3 -C1 -i1 -I "$nicName" "$gateway") + fi + + if [ $? -ne 0 ]; then + echo "failed to run '$cmd' in Pod's netns" + exit 1 + fi + dstMac=$(echo "$output" | grep -o -E '([[:xdigit:]]{1,2}:){5}[[:xdigit:]]{1,2}') + else + lr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_router}) + if [ -z "$lr" ]; then + lr=$(kubectl get subnet "$ls" -o jsonpath={.spec.vpc}) + fi + dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name="$lr"-"$ls" | tr -d '\r') fi - gwMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name="$lr"-"$ls" | tr -d '\r') fi - if [ -z "$gwMac" ]; then - echo "get gw mac failed" + if [ -z "$dstMac" ]; then + echo "failed to get destination mac" exit 1 fi @@ -193,11 +306,11 @@ trace(){ case $type in icmp) set -x - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $podIP && eth.dst == $gwMac && ip$af.dst == $dst" + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst" ;; tcp|udp) set -x - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $podIP && eth.dst == $gwMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4" + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4" ;; *) echo "type $type not supported" @@ -222,11 +335,11 @@ trace(){ case $type in icmp) set -x - kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$gwMac" + kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac" ;; tcp|udp) set -x - kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$gwMac,${type}_src=1000,${type}_dst=$4" + kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac,${type}_src=1000,${type}_dst=$4" ;; *) echo "type $type not supported" From a46a18a32d3a80eccebef89d0bb6975e66ee4c2d Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Mon, 15 Aug 2022 11:34:38 +0800 Subject: [PATCH 2/2] some optimization --- dist/images/kubectl-ko | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/dist/images/kubectl-ko b/dist/images/kubectl-ko index 9dc0f0392b8..65b2def8147 100755 --- a/dist/images/kubectl-ko +++ b/dist/images/kubectl-ko @@ -174,18 +174,18 @@ trace(){ namespace="default" fi - hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork}) - if [ "$hostNetwork" = "true" ]; then - echo "Can not trace host network pod" - exit 1 - fi - dst="$2" if [ -z "$dst" ]; then echo "need a target ip address" exit 1 fi + hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork}) + if [ "$hostNetwork" = "true" ]; then + echo "Can not trace host network pod" + exit 1 + fi + af="4" nw="nw" proto="" @@ -234,15 +234,13 @@ trace(){ dstMac="" if ipIsInCidr $dst $cidr; then + set +o pipefail if [ $af -eq 4 ]; then - set +o pipefail dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -w "$(echo $dst | tr . '\.')" | awk '{print $1}') - set -o pipefail else - set +o pipefail dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -i " $dst\$" | awk '{print $1}') - set -o pipefail fi + set -o pipefail fi if [ -z "$dstMac" ]; then vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan})