[ FAB-6320 ] Trim logs for CI builds

Change the run_fvt_tests script to only print full log for failures. Change-Id: Ief018fe7a828c2bc07f225c0e5ab0e25656c2ad5 Signed-off-by: Allen Bailey <[email protected]>
hyperledger · Sep 28, 2017 · 21ee6a1 · 21ee6a1
1 parent 1f130bc
commit 21ee6a1
Show file tree

Hide file tree

Showing 12 changed files with 38 additions and 62 deletions.
diff --git a/images/fabric-ca-fvt/payload/postgres_setup.sh b/images/fabric-ca-fvt/payload/postgres_setup.sh
@@ -3,12 +3,12 @@ RC=0
 
 # Configure and start postgres
 echo $PGUSER:$PGUSER | chpasswd
-mkdir -p $PGDATA && chown postgres:postgres $PGDATA
+mkdir -p $PGDATA && chown $PGUSER:$PGUSER $PGDATA
 su $PGUSER -c "/usr/lib/postgresql/$PGVER/bin/initdb -D $PGDATA"
 su $PGUSER -c "/usr/lib/postgresql/$PGVER/bin/pg_ctl start -D $PGDATA" &&\
                    sleep 10 &&\
-                   psql -U postgres -h localhost -c "ALTER USER $PGUSER WITH PASSWORD '$PGPASSWORD';" &&\
-                   su postgres -c "/usr/lib/postgresql/$PGVER/bin/pg_ctl stop"
+                   psql -U $PGUSER -h localhost -c "ALTER USER $PGUSER WITH PASSWORD '$PGPASSWORD';" &&\
+                   su $PGUSER -c "/usr/lib/postgresql/$PGVER/bin/pg_ctl stop"
 let RC+=$?
 echo "host all  all    0.0.0.0/0  trust" >> ${PGDATA}/pg_hba.conf
 echo "listen_addresses='*'" >> ${PGDATA}/postgresql.conf
@@ -22,7 +22,16 @@ chmod 600 $PGDATA/FabricTlsServer*.pem || let RC+=1
 sed -i "s/\(^[[:blank:]]*\)#*\([[:blank:]]*ssl[[:blank:]]*=[[:blank:]]*\).*/\1\2on/;\
 s/\(^[[:blank:]]*\)#*\([[:blank:]]*ssl_cert_file[[:blank:]]*=[[:blank:]]*\).*/\1\2'$TLS_SERVER_CERT'/;\
 s/\(^[[:blank:]]*\)#*\([[:blank:]]*ssl_key_file[[:blank:]]*=[[:blank:]]*\).*/\1\2'$TLS_SERVER_KEY'/;\
-s/\(^[[:blank:]]*\)#*\([[:blank:]]*ssl_ca_file[[:blank:]]*=[[:blank:]]*\).*/\1\2'$TLS_BUNDLE'/" $PGDATA/postgresql.conf || let RC+=1
+s/\(^[[:blank:]]*\)#*\([[:blank:]]*ssl_ca_file[[:blank:]]*=[[:blank:]]*\).*/\1\2'$TLS_BUNDLE'/;\
+s/\(^[[:blank:]]*\)#*\([[:blank:]]*log_destination[[:blank:]]*=[[:blank:]]*\).*/\1\2'syslog'/;\
+s/\(^[[:blank:]]*\)#*\([[:blank:]]*logging_collector[[:blank:]]*=[[:blank:]]*\).*/\1\2on/;\
+s/\(^[[:blank:]]*\)#*\([[:blank:]]*log_directory[[:blank:]]*=[[:blank:]]*\).*/\1\2'pg_log'/;\
+s/\(^[[:blank:]]*\)#*\([[:blank:]]*log_filename[[:blank:]]*=[[:blank:]]*\).*/\1\2'postgresql-%Y-%m-%d_%H%M%S.log'/;\
+s/\(^[[:blank:]]*\)#*\([[:blank:]]*log_file_mode[[:blank:]]*=[[:blank:]]*\).*/\1\2'0644'/;\
+s/\(^[[:blank:]]*\)#*\([[:blank:]]*log_truncate_on_rotation[[:blank:]]*=[[:blank:]]*\).*/\1\2on/;\
+s/\(^[[:blank:]]*\)#*\([[:blank:]]*log_rotation_size[[:blank:]]*=[[:blank:]]*\).*/\1\210MB/;\
+s/\(^[[:blank:]]*\)#*\([[:blank:]]*syslog_facility[[:blank:]]*=[[:blank:]]*\).*/\1\2'LOCAL0'/;\
+s/\(^[[:blank:]]*\)#*\([[:blank:]]*syslog_ident[[:blank:]]*=[[:blank:]]*\).*/\1\2'$PGUSER'/" $PGDATA/postgresql.conf || let RC+=1
 
 # Generate version-agnostic postgres command
 ln -s /usr/lib/postgresql/$PGVER/bin/postgres /usr/local/bin/postgres && chmod 777 /usr/local/bin/postgres || let RC+=1

diff --git a/images/fabric-ca-fvt/payload/system_update.sh b/images/fabric-ca-fvt/payload/system_update.sh
@@ -30,6 +30,7 @@ apt-get -y autoremove
 sed -i 's/^[[:blank:]]*#\([[:blank:]]*.*imudp.*\)/\1/' /etc/rsyslog.conf
 rm /etc/rsyslog.d/*haproxy*conf
 printf "local2.*    /var/log/haproxy.log\n& ~\n" > /etc/rsyslog.d/haproxy.conf
+printf "local0.*    /var/log/postgres.log\n& ~\n" > /etc/rsyslog.d/postgres.conf
 
 # Use python2, not 3
 ln -s /usr/bin/python2.7 /usr/local/bin/python && chmod 777 /usr/local/bin/python || let RC+=1

diff --git a/scripts/fvt/cdp_exploit_test.sh b/scripts/fvt/cdp_exploit_test.sh
@@ -6,7 +6,7 @@
 #
 RC=0
 CADOMAIN="FVT"
-TESTCASE="crl_limit"
+: ${TESTCASE:="crl_limit"}
 FABRIC_CA="$GOPATH/src/github.com/hyperledger/fabric-ca"
 SCRIPTDIR="$FABRIC_CA/scripts/fvt"
 . $SCRIPTDIR/fabric-ca_utils

diff --git a/scripts/fvt/db_test.sh b/scripts/fvt/db_test.sh
@@ -5,7 +5,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-TESTCASE="db_resiliency"
+: ${TESTCASE:="db_resiliency"}
 FABRIC_CA="$GOPATH/src/github.com/hyperledger/fabric-ca"
 echo $FABRIC_CA
 FABRIC_CAEXEC="$FABRIC_CA/bin/fabric-ca"

diff --git a/scripts/fvt/intermediateca_test.sh b/scripts/fvt/intermediateca_test.sh
@@ -5,7 +5,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-TESTCASE="intermediateca-test"
+: ${TESTCASE:="intermediateca-test"}
 TDIR=/tmp/$TESTCASE
 FABRIC_CA="$GOPATH/src/github.com/hyperledger/fabric-ca"
 SCRIPTDIR="$FABRIC_CA/scripts/fvt"

diff --git a/scripts/fvt/multica_test.sh b/scripts/fvt/multica_test.sh
@@ -5,7 +5,8 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-TESTCASE="multica-test"
+
+: ${TESTCASE:="multica-test"}
 TDIR=/tmp/$TESTCASE
 FABRIC_CA="$GOPATH/src/github.com/hyperledger/fabric-ca"
 SCRIPTDIR="$FABRIC_CA/scripts/fvt"
@@ -26,7 +27,7 @@ function createRootCA() {
    # Start RootCA
    mkdir -p "$TDIR/ca0"
    $SCRIPTDIR/fabric-ca_setup.sh -I -x "$TDIR/ca0" -d $driver -m $MAXENROLL
-   sed -i "/^ca:/,/^[^\t ]/ s@\(\(cert\|key\)file:\).*@\1@" /tmp/multica-test/ca0/runFabricCaFvt.yaml
+   sed -i "/^ca:/,/^[^\t ]/ s@\(\(cert\|key\)file:\).*@\1@" $TDIR/ca0/runFabricCaFvt.yaml
    FABRIC_CA_SERVER_HOME="$TDIR/ca0" fabric-ca-server start -d --cacount $NUMINTCAS \
                                       --csr.hosts $ROOT_CA_ADDR --address $ROOT_CA_ADDR \
                                       -c $TDIR/ca0/runFabricCaFvt.yaml 2>&1 |

diff --git a/scripts/fvt/passwordsInLog_test.sh b/scripts/fvt/passwordsInLog_test.sh
@@ -45,7 +45,7 @@ function passWordSub() {
 }
 
 RC=0
-TESTCASE="passwordsInLog"
+: ${TESTCASE:="passwordsInLog"}
 TESTDIR="/tmp/$TESTCASE"
 mkdir -p $TESTDIR
 

diff --git a/scripts/fvt/reenroll_test.sh b/scripts/fvt/reenroll_test.sh
@@ -23,7 +23,7 @@ now=$(date +"%g%m%d%H%M%SZ")
 future=$(date +"$next_year%m%d%H%M%SZ")
 
 NUM_SERVERS=4
-USER_SERVER_RATIO=8 
+USER_SERVER_RATIO=8
 for u in $(eval echo {1..$((NUM_SERVERS*USER_SERVER_RATIO-1))}); do
    USERS[u]="user$u"
 done

diff --git a/scripts/fvt/roundrobin_test.sh b/scripts/fvt/roundrobin_test.sh
@@ -35,10 +35,10 @@ for driver in sqlite3 mysql postgres ; do
    $SCRIPTDIR/registerAndEnroll.sh -u "${USERS[*]}"
    test $? -ne 0 && ErrorMsg "registerAndEnroll failed"
    reenroll admin
-   if ! $(${FABRIC_TLS:-false}); then 
+   if ! $(${FABRIC_TLS:-false}); then
       nums=$((NUM_SERVERS-1))
       for s in $(eval echo {0..$nums}); do
-         curl -s http://$HOST/ | 
+         curl -s http://$HOST/ |
             awk -v s="server${s}\"" '$0~s'|
                html2text |
                   egrep "HTTP|server${s}"

diff --git a/scripts/fvt/staging/intermediateca_test.sh b/scripts/fvt/staging/intermediateca_test.sh
diff --git a/scripts/run_benchmarks b/scripts/run_benchmarks
@@ -102,7 +102,7 @@ fi
 echo "Running all benchmarks ..."
 if test -n "$CREATE_PROFILE"; then
   # go test does not allow to specify multiple packages when benchmarks are profiled
-  # so, run go test in each package with profiling options. This will create benchmark 
+  # so, run go test in each package with profiling options. This will create benchmark
   # and profile files in each package
   for pkg in $PKGS
   do
@@ -114,4 +114,4 @@ if test -n "$CREATE_PROFILE"; then
 else
   go test -run=^$ -bench=. -benchmem -timeout=20m $PKGS | tee $BENCHMARK_FILE_NAME
   exitrc ${PIPESTATUS[0]}
-fi
+fi
diff --git a/scripts/run_fvt_tests b/scripts/run_fvt_tests
@@ -16,28 +16,27 @@ export FABRIC_TLS
 
 function runTests() {
    echo "Running fvt tests ..."
+   echo ""
    tests="$(find $SCRIPTDIR -maxdepth 1 -name "*test.*sh")"
 
    for FABRIC_TLS in "true" "false"; do
       for cmd in $tests; do
-        echo ""
-        echo "*******************"
-        export TESTCASE="${cmd##*/}"
-        echo "${cmd}"
-        echo "*******************"
-        ${cmd}
-        RC=$((RC+$?))
+        export TESTCASE="${cmd##*/}-TLS-${FABRIC_TLS}"
+        echo "*******************"  | tee -a $RESULTLOG 2>&1
+        printf " Running $TESTCASE " |tee -a $RESULTLOG 2>&1
+        ${cmd} >> $RESULTLOG 2>&1
+        rc=$?
+        test $rc -eq 0 && echo PASSED || echo FAILED
+        test $rc -ne 0 && awk -v b=$TESTCASE -v e="test ended." '$0~b,$0~e' $RESULTLOG
+        RC=$((RC+$rc))
         $SCRIPTDIR/fabric-ca_setup.sh -R
-        echo ""
-        echo ""
-        echo ""
       done
    done
 }
 
-TimeStamp
-runTests > >( tee $RESULTLOG )
-grep RC: $RESULTLOG
+TimeStamp | tee $RESULTLOG
+runTests
+grep -a RC: $RESULTLOG
 echo "Finished running fvt tests"
 TESTCASE="fabric-ca-fvt"
 CleanUp "$RC"