[ FAB-5251 ] Changes to optimize fvt tests

Initial benchmark testing turned up a potential
improvement in haproxy. From the haproxy doc:

   "Setting "option http-server-close" enables HTTP
   connection-close mode on the server side while
   keeping the ability to support HTTP keep-alive
   and pipelining on the client side.  This provides
   the lowest latency on the client side (slow network)
   and the fastest session reuse on the server side to
   save server resources..."

In addition, minor updates to polling timeouts using
sub-second sleeps (where needed) improved fvt runtime
by 30-40%.

On x, 28% improvement:
5 minutes, 32 seconds runtime
3 minutes, 58 seconds runtime

On z, 43% improvement:
4 minutes, 4 seconds runtime
2 minutes, 19 seconds runtime

Lastly, the verifyServerTraffic procedure in fabric-ca_utils,
which verifies the correct roundrobin distribution of client
requests, needed to be updated to account for >10 servers.
The search for 'server[[:digit:]]' needs to find the word boundary,
denoted in the haproxy stats table by a doulble quote, hence:
  server[[:digit:]]+\"

Change-Id: I13d86808efb06129448efac6e7c5aebdcf4e5177
Signed-off-by: rennman <[email protected]>

scripts/fvt/enrollments_test.sh

@@ -238,7 +238,7 @@ trap "CleanUp 1; exit 1" INT
    $SCRIPTDIR/fabric-ca_setup.sh -R -x $CA_CFG_PATH
    test -d $CA_CFG_PATH || mkdir $CA_CFG_PATH
    genServerConfig invalid
-   $SCRIPTDIR/fabric-ca_setup.sh -S -X -g $SERVERCONFIG
+   $SCRIPTDIR/fabric-ca_setup.sh -o 1 -S -X -g $SERVERCONFIG
    test $? -eq 0 && ErrorMsg "user enrollment > global setting"
 
 $SCRIPTDIR/fabric-ca_setup.sh -L

scripts/fvt/fabric-ca_setup.sh

@@ -209,7 +209,7 @@ function startFabricCa() {
    until test "$started" = "$server_addr:${USER_CA_PORT-$CA_DEFAULT_PORT}" -o "$now" -gt "$timeout"; do
       started=$(ss -ltnp src $server_addr:${USER_CA_PORT-$CA_DEFAULT_PORT} | awk 'NR!=1 {print $4}')
       test "$started" = "$server_addr:${USER_CA_PORT-$CA_DEFAULT_PORT}" && break
-      sleep .5
+      sleep .1
       let now+=1
    done
    printf "FABRIC_CA server on $server_addr:${USER_CA_PORT-$CA_DEFAULT_PORT} "

scripts/fvt/fabric-ca_utils

@@ -162,7 +162,7 @@ verifyServerTraffic() {
     : ${op:="eq"}
 
    result=$(curl -s http://${haproxy_addr}/ |
-     awk -v s="$server_name" '$0~s'|html2text|
+     awk -v s="$server_name\"" '$0~s'|html2text|
         awk -v c="$code" '
            /Cum. sessions:/ {sessions=$NF}
            $0~c {gsub(/[(%)]/,"",$NF);status=$NF}

scripts/fvt/reenroll_test.sh

@@ -22,6 +22,14 @@ past=$(date +"$prev_year%m%d%H%M%SZ")
 now=$(date +"%g%m%d%H%M%SZ")
 future=$(date +"$next_year%m%d%H%M%SZ")
 
+NUM_SERVERS=4
+USER_SERVER_RATIO=8 
+for u in $(eval echo {1..$((NUM_SERVERS*USER_SERVER_RATIO-1))}); do
+   USERS[u]="user$u"
+done
+NUM_USERS=${#USERS[*]}
+EXPECTED_DISTRIBUTION=$(((NUM_USERS+1)*2/$NUM_SERVERS))
+
 . $SCRIPTDIR/fabric-ca_utils
 
 while getopts "dx:" option; do
@@ -72,41 +80,35 @@ for driver in sqlite3 postgres mysql; do
    echo "------> BEGIN TESTING $driver <----------"
    # note MAX_ENROLLMENTS defaults to '1'
    $SCRIPTDIR/fabric-ca_setup.sh -R -d $driver -x $CA_CFG_PATH
-   $SCRIPTDIR/fabric-ca_setup.sh -I -S -X -n4 -d $driver -x $CA_CFG_PATH
+   $SCRIPTDIR/fabric-ca_setup.sh -I -S -X -n $NUM_SERVERS -d $driver -x $CA_CFG_PATH
    if test $? -ne 0; then
       ErrorMsg "Failed to setup server"
       continue
    fi
    enroll $REGISTRAR
+   enroll $REGISTRAR
    if test $? -ne 0; then
       ErrorMsg "Failed to enroll $REGISTRAR"
       continue
    fi
 
-   for i in {1..4}; do
+   for i in $(eval echo {1..$NUM_USERS}); do
       OUT=$(register $REGISTRAR user${i})
       pswd[$i]=$(echo $OUT | tail -n1 | awk '{print $NF}')
       echo $pswd
    done
 
-   for i in {1..4}; do
+   for i in $(eval echo {1..$NUM_USERS}); do
       enroll user${i} ${pswd[i]}
       test $? -ne 0 && ErrorMsg "Failed to reenroll user${i}"
    done
 
-   for i in 1 2 3 4 1 2 3; do
-      # sqaure up the number of requests to each of 4 servers by additional requests
-       reenroll user${i} "$CA_CFG_PATH/user${i}/$MSP_CERT_DIR/cert.pem" "$CA_CFG_PATH/user${i}/$MSP_KEY_DIR/key.pem"
-       test $? -ne 0 && ErrorMsg "Failed to reenroll user${i}"
-   done
-   # all servers should register 4 successful requests
-   # but...it's only available when tls is disabled
-   if test "$FABRIC_TLS" = 'false'; then
-      for s in {1..4}; do
+   if ! $(${FABRIC_TLS:-false}); then
+      for s in $(eval echo {1..$NUM_SERVERS}); do
          curl -s http://${HOST}/ | awk -v s="server${s}" '$0~s'|html2text|grep HTTP
-         verifyServerTraffic $HOST server${s} 4
+         verifyServerTraffic $HOST server${s} $EXPECTED_DISTRIBUTION
          test $? -ne 0 && ErrorMsg "Distributed traffic to server FAILED"
-         sleep 1
+         sleep .1
       done
    fi
 

scripts/fvt/registerAndEnroll.sh

@@ -38,7 +38,6 @@ for i in $USERNAME; do
                                    tail -n1 | awk '{print $NF}')
    enroll $i $pswd
    RC=$((RC+$?))
-   sleep 1
 done
 
 exit $RC

scripts/fvt/reregister_test.sh

@@ -15,6 +15,9 @@ USERNAME="testuser99"
 HTTP_PORT="3755"
 RC=0
 
+NUM_SERVERS=4
+NUM_BAD_REQ=16
+
 while getopts "dx:" option; do
   case "$option" in
      d)   FABRIC_CA_DEBUG="true" ;;
@@ -38,7 +41,7 @@ trap "kill $HTTP_PID; CleanUp 1; exit 1" INT
 export FABRIC_CA_DEBUG
 for driver in sqlite3 postgres mysql; do
    $SCRIPTDIR/fabric-ca_setup.sh -R -x $CA_CFG_PATH -d $driver
-   $SCRIPTDIR/fabric-ca_setup.sh -I -S -X -n4 -t rsa -l 2048 -d $driver -x $CA_CFG_PATH
+   $SCRIPTDIR/fabric-ca_setup.sh -I -S -X -n $NUM_SERVERS -t rsa -l 2048 -d $driver -x $CA_CFG_PATH
    if test $? -ne 0; then
       ErrorMsg "Failed to setup fabric-ca server"
       continue
@@ -56,18 +59,19 @@ for driver in sqlite3 postgres mysql; do
       continue
    fi
 
-   for i in {2..8}; do
+   for u in $(eval echo {1..$NUM_BAD_REQ}); do
       register $REGISTRAR $USERNAME
       test $? -eq 0 && ErrorMsg "Duplicate registration of $USERNAME"
    done
 
    # all servers should register = number of successful requests
    # but...it's only available when tls is disabled
-   if test "$FABRIC_TLS" = 'false'; then
-      for s in {1..4}; do
-         verifyServerTraffic $HOST server${s} 10 "" "" lt
+   if ! $(${FABRIC_TLS:-false}); then
+      for s in $(eval echo {1..$NUM_SERVERS}); do
+         curl -s http://${HOST}/ | awk -v s="server${s}" '$0~s'|html2text|grep HTTP
+         verifyServerTraffic $HOST server${s} 0 0 "HTTP 5xx" gt
          test $? -eq 0 || ErrorMsg "verifyServerTraffic failed"
-         sleep 1
+         sleep .1
       done
    fi
 

scripts/fvt/roundrobin_test.sh

@@ -12,6 +12,13 @@ HOST="127.0.0.1:10888"
 RC=0
 HTTP_PORT="3755"
 CA_CFG_PATH="/tmp/fabric-ca/roundrobin"
+NUM_SERVERS=4
+USER_SERVER_RATIO=8
+for u in $(eval echo {1..$((NUM_SERVERS*USER_SERVER_RATIO-1))}); do
+   USERS[u]="user$u"
+done
+NUM_USERS=${#USERS[*]}
+EXPECTED_DISTRIBUTION=$(((NUM_USERS+1)*2/$NUM_SERVERS))
 export CA_CFG_PATH
 
 cd $TESTDATA
@@ -23,15 +30,18 @@ trap "kill $HTTP_PID; CleanUp 1; exit 1" INT
 
 for driver in sqlite3 mysql postgres ; do
    $SCRIPTDIR/fabric-ca_setup.sh -R -x $CA_CFG_PATH
-   $SCRIPTDIR/fabric-ca_setup.sh -I -S -X -n4 -t rsa -l 2048 -d $driver
+   $SCRIPTDIR/fabric-ca_setup.sh -I -S -X -n $NUM_SERVERS -t rsa -l 2048 -d $driver
    test $? -ne 0 && ErrorExit "Failed to setup server"
-   $SCRIPTDIR/registerAndEnroll.sh -u 'user1 user2 user3 user4 user5 user6 user7 user8 user9'
+   $SCRIPTDIR/registerAndEnroll.sh -u "${USERS[*]}"
    test $? -ne 0 && ErrorMsg "registerAndEnroll failed"
    reenroll admin
-   if test "$FABRIC_TLS" = 'false'; then
-      for s in 1 2 3 4; do
-         curl -s http://$HOST/ | awk -v s="server${s}" '$0~s'|html2text | egrep "HTTP|server${s}"
-         verifyServerTraffic $HOST server${s} 5
+   if ! $(${FABRIC_TLS:-false}); then 
+      for s in $(eval echo {1..$NUM_SERVERS}); do
+         curl -s http://$HOST/ | 
+            awk -v s="server${s}\"" '$0~s'|
+               html2text |
+                  egrep "HTTP|server${s}"
+         verifyServerTraffic $HOST server${s} $EXPECTED_DISTRIBUTION
          test $? -ne 0 && ErrorMsg "verifyServerTraffic failed"
       done
    fi