uperf-server-start

#!/bin/bash
exec >uperf-server-stderrout.txt 
exec 2>&1

. /usr/bin/uperf-base || (echo "/usr/bin/uperf-base not found"; exit 1)

dump_runtime
validate_label
validate_sw_prereqs jq ss uperf getopt ip

# defaults
control_port=""
data_port=""
ifname=""
cpu_pin=""

longopts="ifname:,cpu-pin:,passthru:"
opts=$(getopt -q -o "" --longoptions "$longopts" -n "getopt.sh" -- "$@");
if [ $? -ne 0 ]; then
    exit_error  "Unrecognized option specified"
fi
eval set -- "$opts";
while true; do
    case "$1" in
	--cpu-pin)
	    shift;
	    cpu_pin=$1
	    echo "cpu_pin=${cpu_pin}"
	    shift;
	    ;;
        --ifname)
            shift;
            ifname="$1"
            echo "ifname=$ifname"
            shift
            ;;
        --test-type|--nthreads|--protocol|--rsize|--wsize|--remotehost|--duration|--think)
            shift;
            shift;
            ;;
        --passthru)     # uperf options that user wants passthru
            shift;
            pt_opts=$1
            echo "pt_opts=$pt_opts"
            shift
            ;;
        --)
            shift;
            break
            ;;
        *)
            exit_error "Invalid option: $1"
    esac
done

case "${cpu_pin}" in
    ""|"numa")
	;;
    *)
	exit_error "unsupported cpu-pin value '${cpu_pin}'"
	;;
esac

# A benchmark server can provide both an IP and ports to the client.
# The server must package these in a JSON, which will be sent to
# the endpoint.  The endpoint may make adjustments to the IP and
# ports depending on the network and security model.

# There are always 2 ports for these uperf tests, and ports should always
# start at 30000 to avoid port number reassignment
let "control_port = 2 * $id"
let "control_port = $control_port + 30000"
let "data_port = $control_port + 1"

# If --remotehost is not used by the user, they can pass --ifname, a netdev name
# like "eth0", and the server (below) will try to resolve this to the actual IP address.
# This allows the user to not know the IP address in advance.
ip=""
ip -j a >ip.json
if [ ! -z "$ifname" ]; then
    if echo $ifname | grep -q '%id:'; then
        # We have the role ID variable embedded, so we need to extract the value
        before_var=`echo $ifname | sed -e 's/\(.*\)%id:.*%/\1/'`
        after_var=`echo $ifname | sed -e 's/.*%id:.*%\(.*\)/\1/'`
        var=`echo $ifname | sed -e 's/%id:\(.*\)%/\1/' | jq -r '."'$id'"'`
        ifname="${before_var}${var}${after_var}"
    fi
    if [ "${ifname}" == "default-route" ]; then
	echo "Looking for default route interface..."
	ip -j route > route.json
	ifname=$(jq --raw-output 'to_entries | .[] | if (.value.dst == "default") then .value.dev else null end' route.json | grep -v null)
	if [ -z "${ifname}" ]; then
	    exit_error "could not find default route interface"
	else
	    echo "Found default route on interface ${ifname}"
	fi
    else
	echo "--ifname=$ifname was found"
    fi
    echo "Searching for IP for interface ${ifname}..."
    # Find the element in the 'ip a' array that matches the ifname we are looking for
    ifname_idx=`jq 'to_entries | .[] | if (.value.ifname == "'$ifname'") then .key else null end' ip.json | grep -v null`
    if [ ! -z "$ifname_idx" ]; then
        echo "$ifname matches index $ifname_idx, looking for ipv4 (inet) in addr_info:"
        jq '.['$ifname_idx'].addr_info' ip.json
        # Find the first ipv4 address
        addr_idx=`jq '.['$ifname_idx'].addr_info | to_entries | .[] | if (.value.family == "inet") then .key else null end' ip.json | grep -v null | head -1`
        if [ ! -z $addr_idx ]; then
            echo "Found inet in index $addr_idx:"
            jq -r '.['$ifname_idx'].addr_info['$addr_idx']' ip.json
            ip=`jq -r '.['$ifname_idx'].addr_info['$addr_idx'].local' ip.json`
        else
            addr_info=`jq '.['$ifname_idx'].addr_info' ip.json`
            exit_error "An IPV4 address could not be found for $ifname in addr_info: $addr_info"
        fi
    else
        interfaces=`jq -r '. | to_entries | .[] | .value.ifname' ip.json`
        exit_error "could not find netdev $ifname, interfaces are: $interfaces"
    fi
    if [ -z "$ip" ]; then
        exit_error "IP was not found from $ifname"
    else
        echo "IP was found: $ip"
    fi
else
    echo "Not going to look for an IP since --ifname was not used."
fi

# Queue ip/port info, to be sent to endpoint
echo '{"recipient":{"type":"all","id":"all"},"user-object":{"svc":{"ip":"'$ip'","ports":['$control_port,$data_port']}}}' >msgs/tx/svc

echo "Existing listening ports:"
ss -tlnp
echo

# Kill any existing process using this port
echo "Attempt to kill port $control_port via 'ss' if it exists:"
ss -tlnpK '( sport = '$control_port' )'
echo

pid=`ss -Otlnp '( sport = '$control_port' )' | awk '{print $6}' | grep -v Peer | awk -Fpid= '{print $2}' | awk -F, '{print $1}' | sort | uniq`
if [ ! -z "$pid" ]; then
    echo "Trying to force kill PID $pid with 'kill -9'"
    kill -9 $pid
fi

echo "Existing listening ports (after kill):"
ss -tlnp
echo
#TODO: exit with error if ports still used

ifname_numa_node=-1
ifname_numa_node_cpus=-1
if [ -e /sys/class/net/${ifname}/device/numa_node ]; then
    ifname_numa_node=$(cat /sys/class/net/${ifname}/device/numa_node)
    if [ $ifname_numa_node == "-1" ] && [ "${cpu_pin}" == "numa" ]; then
        exit_error "PCIe has no NUMA locality, but cpu-pin is numa"
    fi
    ifname_numa_node_cpus=$(cat /sys/devices/system/node/node${ifname_numa_node}/cpulist)
elif [ "${cpu_pin}" == "numa" ]; then
    exit_error "cannot determine numa node for interface '${ifname}' and cpu-pin is numa"
fi

echo "ifname=${ifname}"
echo "ifname_numa_node=${ifname_numa_node}"

cpu_pin_cmd=""
case "${cpu_pin}" in
    "numa")
	cpu_pin_cmd="taskset --cpu-list ${ifname_numa_node_cpus}"
	;;
esac

cmd="${cpu_pin_cmd} uperf -s -P $control_port"

# strip ',' from the passthru options 
passthru_opts=$(echo "$pt_opts" | sed 's/,/ /g')
options+=" $passthru_opts"
echo "passthu optins run: $options"

cmd+="$options"

echo "going to run: $cmd"
export MALLOC_CHECK_=2
$cmd 2>&1 >uperf-server-start.txt &
pid=$!
echo $pid >uperf-server.pid
# Wait a little bit to see if the server errored out
sleep 10
if grep -q -i error uperf-server-start.txt; then
    exit_error "Server failed to start: `grep -i error uperf-server-start.txt`"
fi
echo ps:
ps aux | grep uperf
echo
echo ss -tlnp:
ss -tlnp
exit 0