Phylsbranch

DaosTestMulti.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+set -ex
+
+if [ "$JENKINS_URL" = "http://localhost:8080/" ]; then
+    NFS_SERVER="192.168.121.1"
+else
+    HOSTPREFIX="wolf-53"
+fi
+NFS_SERVER=${NFS_SERVER:-$HOSTPREFIX}
+
+# leave this empty to run on the centos7 builder
+CLIENT_VM=${HOSTPREFIX}vm1
+
+# shellcheck disable=SC1091
+. .build_vars.sh
+
+daospath=${SL_OMPI_PREFIX%/install}
+CPATH="${daospath}"/install/include/:"$CPATH"
+PATH="${daospath}"/install/bin/:"${daospath}"/install/sbin:"$PATH"
+LD_LIBRARY_PATH="${daospath}"/install/lib:"${daospath}"/install/lib/daos_srv:$LD_LIBRARY_PATH
+export CPATH PATH LD_LIBRARY_PATH
+export CRT_PHY_ADDR_STR="ofi+sockets"
+
+cat <<EOF > hostfile
+${HOSTPREFIX}vm1 slots=1
+${HOSTPREFIX}vm2 slots=1
+${HOSTPREFIX}vm3 slots=1
+${HOSTPREFIX}vm4 slots=1
+${HOSTPREFIX}vm5 slots=1
+${HOSTPREFIX}vm6 slots=1
+${HOSTPREFIX}vm7 slots=1
+${HOSTPREFIX}vm8 slots=1
+EOF
+
+# shellcheck disable=SC1091
+#. scons_local/utils/setup_local.sh
+
+export SERVER_OFI_INTERFACE=eth0
+if [ -n "$CLIENT_VM" ]; then
+    export CLIENT_OFI_INTERFACE=eth0
+else
+    export CLIENT_OFI_INTERFACE=virbr1
+fi
+
+pdsh -R ssh -S -w ${HOSTPREFIX}vm[1-8] "sudo bash -c 'echo \"1\" > /proc/sys/kernel/sysrq'
+if grep /mnt/daos\\  /proc/mounts; then
+    sudo umount /mnt/daos
+else
+    if [ ! -d /mnt/daos ]; then
+        sudo mkdir -p /mnt/daos
+    fi
+fi
+sudo mkdir -p $daospath
+sudo mount -t nfs $NFS_SERVER:$PWD $daospath
+sudo mount -t tmpfs -o size=16G tmpfs /mnt/daos
+df -h /mnt/daos" 2>&1 | dshbak -c
+
+TESTS=${1:-"-mpceiACoRO"}
+
+rm -f "${daospath}"/daos.log
+
+# shellcheck disable=SC2029
+ssh "$CLIENT_VM" "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH
+\"${daospath}\"/install/bin/orterun --np 8 --hostfile \"$daospath\"/hostfile --enable-recovery --report-uri \"$daospath\"/urifile  -x LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH\" -x CRT_PHY_ADDR_STR=\"$CRT_PHY_ADDR_STR\" -x D_LOG_FILE=\"$daospath\"/daos.log -x ABT_ENV_MAX_NUM_XSTREAMS=100 -x ABT_MAX_NUM_XSTREAMS=100 -x PATH=\"$PATH\" -x OFI_PORT=23350 -x OFI_INTERFACE=\"$SERVER_OFI_INTERFACE\" \"${daospath}\"/install/bin/daos_server -g daos_server -c 1 >/tmp/daos_server.out 2>&1 &
+daos_server_pid=\$!
+if ! kill -0 \$daos_server_pid 2>/dev/null; then
+    wait \$daos_server_pid
+    exit \${PIPESTATUS[0]}
+else
+    echo \"\$daos_server_pid\" > /tmp/daos_server_pid
+fi"
+# shellcheck disable=SC2154
+trap 'ssh "$CLIENT_VM" "daos_server_pid=\$(cat /tmp/daos_server_pid)
+if kill -0 \$daos_server_pid 2>/dev/null; then
+    kill \$daos_server_pid
+fi"
+pdsh -R ssh -S -w ${HOSTPREFIX}vm[1-8] "x=0
+while [ \$x -lt 30 ] && grep $daospath /proc/mounts && ! sudo umount $daospath; do
+    sleep 1
+    let x=\$x+1
+done
+sudo rmdir $daospath" 2>&1 | dshbak -c
+ls -l "$daospath"/daos.log' EXIT
+
+sleep 5
+
+# shellcheck disable=SC2029
+if ! ssh "$CLIENT_VM" "daos_server_pid=\$(cat /tmp/daos_server_pid)
+    if ! kill -0 \$daos_server_pid 2>/dev/null; then
+        exit 199
+    fi
+    # cmocka's XML results are not JUnit compliant as it creates multiple
+    # <testsuites> blocks and only one is allowed
+    trap 'set -x; cat \"${daospath}\"/results.xml; sed -i -e '\"'\"'2!{/<testsuites>/d;}'\"'\"' -e '\"'\"'\$!{/<\\/testsuites>/d;}'\"'\"' \"${daospath}\"/results.xml; cat \"${daospath}\"/results.xml;' EXIT
+    rm -f \"${daospath}\"/results.xml
+    CMOCKA_XML_FILE=\"${daospath}\"/results.xml CMOCKA_MESSAGE_OUTPUT=xml \"${daospath}\"/install/bin/orterun --output-filename \"$daospath\"/daos_test.out --np 1 --ompi-server file:\"$daospath\"/urifile -x ABT_ENV_MAX_NUM_XSTREAMS=100 -x PATH=\"$PATH\" -x CRT_PHY_ADDR_STR=\"$CRT_PHY_ADDR_STR\" -x ABT_MAX_NUM_XSTREAMS=100 -x LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH\" -x D_LOG_FILE=\"$daospath\"/daos.log -x OFI_INTERFACE=\"$CLIENT_OFI_INTERFACE\" daos_test \"$TESTS\""; then
+    if [ "${PIPESTATUS[0]}" = 199 ]; then
+        echo "daos_server not running"
+        pdsh -R ssh -S -w ${HOSTPREFIX}vm[1-8] "if [ -f /tmp/daos_server.out ]; then cat /tmp/daos_server.out; fi" | dshbak -c
+        trap 'pdsh -R ssh -S -w ${HOSTPREFIX}vm[1-8] "x=0
+while [ \$x -lt 30 ] && grep $daospath /proc/mounts && ! sudo umount $daospath; do
+    sleep 1
+    let x=\$x+1
+done
+sudo rmdir $daospath" 2>&1 | dshbak -c' EXIT
+        exit 199
+    else
+        echo "Running daos_test failed"
+        exit 1
+    fi
+fi

Jenkinsfile

@@ -0,0 +1,232 @@
+// My copy
+// Trying to retrigger
+pipeline {
+    agent none
+
+    environment {
+        SHELL = '/bin/bash'
+    }
+
+    // triggers {
+        // Jenkins instances behind firewalls can't get webhooks
+        // sadly, this doesn't seem to work
+        // pollSCM('* * * * *')
+        // See if cron works since pollSCM above isn't
+        // cron('H 22 * * *')
+    // }
+
+    options {
+        // preserve stashes so that jobs can be started at the test stage
+        preserveStashes(buildCount: 5)
+    }
+
+    stages {
+        stage('Pre-build') {
+            parallel {
+                stage('check_modules.sh') {
+                    agent {
+                        dockerfile {
+                            filename 'Dockerfile.centos:7'
+                            dir 'utils/docker'
+                            label 'docker_runner'
+                            additionalBuildArgs  '--build-arg NOBUILD=1 --build-arg UID=$(id -u)'
+                        }
+                    }
+                    steps {
+                        githubNotify description: 'checkmodules.sh',  context: 'checkmodules.sh', status: 'PENDING'
+                        sh '''pushd scons_local
+                              git fetch https://review.hpdd.intel.com/coral/scons_local refs/changes/13/33013/9
+                              popd
+                              git submodule update --init --recursive
+                              utils/check_modules.sh'''
+                    }
+                    post {
+                        always {
+                            archiveArtifacts artifacts: 'pylint.log', allowEmptyArchive: true
+                        }
+                        success {
+                            githubNotify description: 'checkmodules.sh',  context: 'checkmodules.sh', status: 'SUCCESS'
+                        }
+                        unstable {
+                            githubNotify description: 'checkmodules.sh',  context: 'checkmodules.sh', status: 'FAILURE'
+                        }
+                    }
+                }
+            }
+        }
+        stage('Build') {
+            parallel {
+                stage('Build on CentOS 7') {
+                    agent {
+                        dockerfile {
+                            filename 'Dockerfile.centos:7'
+                            dir 'utils/docker'
+                            label 'docker_runner'
+                            additionalBuildArgs  '--build-arg NOBUILD=1 --build-arg UID=$(id -u)'
+                        }
+                    }
+                    steps {
+                        githubNotify description: 'CentOS 7 Build',  context: 'build/centos7', status: 'PENDING'
+                        checkout scm
+                        sh '''git submodule update --init --recursive
+                              scons -c
+                              # scons -c is not perfect so get out the big hammer
+                              rm -rf _build.external install build
+                              pushd scons_local
+                              git fetch https://review.hpdd.intel.com/coral/scons_local refs/changes/13/33013/5
+                              popd
+                              utils/fetch_go_packages.sh -i .
+                              SCONS_ARGS="--update-prereq=all --build-deps=yes USE_INSTALLED=all install"
+                              if ! scons $SCONS_ARGS; then
+                                  if ! scons --config=force $SCONS_ARGS; then
+                                      rc=\${PIPESTATUS[0]}
+                                      cat config.log || true
+                                      exit \$rc
+                                  fi
+                              fi'''
+                        stash name: 'CentOS-install', includes: 'install/**'
+                        stash name: 'CentOS-build-vars', includes: '.build_vars.*'
+                        stash name: 'CentOS-tests', includes: 'build/src/rdb/raft/src/tests_main, build/src/common/tests/btree_direct, build/src/common/tests/btree, src/common/tests/btree.sh, build/src/common/tests/sched, build/src/client/api/tests/eq_tests, src/vos/tests/evt_ctl.sh, build/src/vos/vea/tests/vea_ut, src/rdb/raft_tests/raft_tests.py'
+                    }
+                    post {
+                        success {
+                            githubNotify description: 'CentOS 7 Build',  context: 'build/centos7', status: 'SUCCESS'
+                        }
+                        unstable {
+                            githubNotify description: 'CentOS 7 Build',  context: 'build/centos7', status: 'FAILURE'
+                        }
+                    }
+                }
+                stage('Build on Ubuntu 16.04') {
+                    agent {
+                        label 'docker_runner'
+                    }
+                    steps {
+                        githubNotify description: 'Ubuntu 18 Build',  context: 'build/ubuntu18', status: 'PENDING'
+                        echo "Building on Ubuntu is broken for the moment"
+                    }
+                    post {
+                        success {
+                            githubNotify description: 'Ubuntu 18 Build',  context: 'build/ubuntu18', status: 'SUCCESS'
+                        }
+                        unstable {
+                            githubNotify description: 'Ubuntu 18 Build',  context: 'build/ubuntu18', status: 'FAILURE'
+                        }
+                    }
+                }
+            }
+        }
+        stage('Test') {
+            parallel {
+                stage('run_test.sh') {
+                    agent {
+                        label 'single'
+                    }
+                    steps {
+                        githubNotify description: 'run_test.sh',  context: 'test/run_test.sh', status: 'PENDING'
+                        dir('install') {
+                            deleteDir()
+                        }
+                        unstash 'CentOS-tests'
+                        unstash 'CentOS-install'
+                        unstash 'CentOS-build-vars'
+                        sh '''HOSTPREFIX=wolf-53 bash -x utils/run_test.sh --init
+                              mv /tmp/daos.log daos-run_test.sh.log'''
+                    }
+                    post {
+                        always {
+                            archiveArtifacts artifacts: 'daos-run_test.sh.log'
+                        }
+                        success {
+                            githubNotify description: 'run_test.sh',  context: 'test/run_test.sh', status: 'SUCCESS'
+                        }
+                        unstable {
+                            githubNotify description: 'run_test.sh',  context: 'test/run_test.sh', status: 'FAILURE'
+                        }
+                    }
+                }
+                stage('DaosTestMulti All') {
+                    agent {
+                        label 'cluster_provisioner'
+                    }
+                    steps {
+                        githubNotify description: 'DaosTestMulti All',  context: 'test/daostestmulti_all', status: 'PENDING'
+                        dir('install') {
+                            deleteDir()
+                        }
+                        unstash 'CentOS-install'
+                        sh '''trap 'mv daos{,-DaosTestMulti-All}.log
+                                    [ -f results.xml ] && mv -f results{,-DaosTestMulti-All}.xml' EXIT
+                              bash DaosTestMulti.sh || true'''
+                    }
+                    post {
+                        always {
+                            archiveArtifacts artifacts: 'daos-DaosTestMulti-All.log, results-DaosTestMulti-All.xml'
+                            junit allowEmptyResults: true, testResults: 'results-DaosTestMulti-All.xml'
+                        }
+                        success {
+                            githubNotify description: 'DaosTestMulti All',  context: 'test/daostestmulti_all', status: 'SUCCESS'
+                        }
+                        unstable {
+                            githubNotify description: 'DaosTestMulti All',  context: 'test/daostestmulti_all', status: 'FAILURE'
+                        }
+                    }
+                }
+                stage('DaosTestMulti Degraded') {
+                    agent {
+                        label 'cluster_provisioner'
+                    }
+                    steps {
+                        githubNotify description: 'DaosTestMulti Degraded',  context: 'test/daostestmulti_degraded', status: 'PENDING'
+                        dir('install') {
+                            deleteDir()
+                        }
+                        unstash 'CentOS-install'
+                        sh '''trap 'mv daos{,-DaosTestMulti-Degraded}.log
+                                    [ -f results.xml ] && mv -f results{,-DaosTestMulti-Degraded}.xml' EXIT
+                              bash DaosTestMulti.sh -d || true'''
+                    }
+                    post {
+                        always {
+                            archiveArtifacts artifacts: 'daos-DaosTestMulti-Degraded.log, results-DaosTestMulti-Degraded.xml'
+                            junit allowEmptyResults: true, testResults: 'results-DaosTestMulti-Degraded.xml'
+                        }
+                        success {
+                            githubNotify description: 'DaosTestMulti Degraded',  context: 'test/daostestmulti_degraded', status: 'SUCCESS'
+                        }
+                        unstable {
+                            githubNotify description: 'DaosTestMulti Degraded',  context: 'test/daostestmulti_degraded', status: 'FAILURE'
+                        }
+                    }
+                }
+                stage('DaosTestMulti Rebuild') {
+                    agent {
+                        label 'cluster_provisioner'
+                    }
+                    steps {
+                        githubNotify description: 'DaosTestMulti Rebuild',  context: 'test/daostestmulti_rebuild', status: 'PENDING'
+                        dir('install') {
+                            deleteDir()
+                        }
+                        unstash 'CentOS-install'
+                        sh '''trap 'mv daos{,-DaosTestMulti-Rebuild}.log
+                                    [ -f results.xml ] && mv -f results{,-DaosTestMulti-Rebuild}.xml' EXIT
+                              bash DaosTestMulti.sh -r || true'''
+                    }
+                    post {
+                        always {
+                            archiveArtifacts artifacts: 'daos-DaosTestMulti-Rebuild.log, results-DaosTestMulti-Rebuild.xml'
+                            junit allowEmptyResults: true, testResults: 'results-DaosTestMulti-Rebuild.xml'
+                        }
+                        success {
+                            githubNotify description: 'DaosTestMulti Rebuild',  context: 'test/daostestmulti_rebuild', status: 'SUCCESS'
+                        }
+                        unstable {
+                            githubNotify description: 'DaosTestMulti Rebuild',  context: 'test/daostestmulti_rebuild', status: 'FAILURE'
+                        }
+                    }
+                }
+            }
+        }
+    }
+}