save resulting notebooks to better help troubleshoot errors

.gitignore

@@ -11,3 +11,4 @@ esgf-compute-api-*/
 # files produced by build
 conda-list-explicit-birdy.txt
 environment-export-birdy.yml
+buildout/

Jenkinsfile

@@ -5,7 +5,7 @@ pipeline {
     // https://jenkins.io/doc/book/pipeline/syntax/
     agent {
         docker {
-            image "pavics/workflow-tests:190312.1"
+            image "pavics/workflow-tests:190321"
             label 'linux && docker'
         }
     }
@@ -17,6 +17,9 @@ pipeline {
                description: 'https://github.com/Ouranosinc/pavics-sdi branch to test against.', trim: true)
         booleanParam(name: 'VERIFY_SSL', defaultValue: true,
                      description: 'Check the box to verify SSL certificate for https connections to PAVICS host.')
+        booleanParam(name: 'SAVE_RESULTING_NOTEBOOK', defaultValue: true,
+                     description: '''Check the box to save the resulting notebooks of the run.
+Note this is another run, will double the time and no guaranty to have same error as the run from py.test.''')
     }
 
     triggers {
@@ -34,7 +37,9 @@ pipeline {
                          string(credentialsId: 'esgf_auth_token',
                                 variable: 'ESGF_AUTH_TOKEN')
                          ]) {
-                        sh("VERIFY_SSL=${params.VERIFY_SSL} ./testall")
+                        sh("VERIFY_SSL=${params.VERIFY_SSL} \
+                            SAVE_RESULTING_NOTEBOOK=${params.SAVE_RESULTING_NOTEBOOK} \
+                            ./testall")
                     }
                 }
             }
@@ -43,7 +48,7 @@ pipeline {
 
     post {
         always {
-            archiveArtifacts(artifacts: 'environment-export-birdy.yml, conda-list-explicit-birdy.txt, notebooks/*.ipynb, pavics-sdi-*/docs/source/notebooks/*.ipynb',
+            archiveArtifacts(artifacts: 'environment-export-birdy.yml, conda-list-explicit-birdy.txt, notebooks/*.ipynb, pavics-sdi-*/docs/source/notebooks/*.ipynb, buildout/*.output.ipynb',
                              fingerprint: true)
         }
 	unsuccessful {  // Run if the current builds status is "Aborted", "Failure" or "Unstable"
@@ -59,7 +64,6 @@ pipeline {
         ansiColor('xterm')
         timestamps()
         timeout(time: 1, unit: 'HOURS')
-        disableConcurrentBuilds()
         // trying to keep 2 months worth of history with buffer for manual
         // build trigger
         buildDiscarder(logRotator(numToKeepStr: '100'))

README.md

@@ -19,6 +19,19 @@ Test user-level workflow.
 # run against another PAVICS host than pavics.ouranos.ca
 # this assume the PAVICS host hardcoded inside the notebooks is pavics.ouranos.ca
 PAVICS_HOST=host.example.com ./runtest
+
+# disable SSL cert verification for notebooks that support this flag
+# useful together with PAVICS_HOST to hit hosts using self-signed SSL cert
+DISABLE_VERIFY_SSL=1 ./runtest
+
+# save output of test run as a notebook, ending with .outout.ipynb
+# each input notebook will have corresponding .output.ipynb file under
+#   buildout/ dir
+# CAVEAT:
+#   * run time is double as a different run is needed
+#   * might not contain the exact same error as the original run since it's a
+#     different run
+SAVE_RESULTING_NOTEBOOK=true ./runtest
 ```
 
 ## Design considerations

docker/Dockerfile

@@ -27,4 +27,4 @@ RUN python -m ipykernel install --name birdy
 # anything accidentally
 # this is for debug only, all dependencies should be specified in
 # environment.yml above
-# RUN conda install -n birdy -c birdhouse -c conda-forge -c default nbdime
+# RUN conda install -n birdy -c birdhouse -c conda-forge -c defaults nbdime

docker/environment.yml

@@ -17,3 +17,5 @@ dependencies:
   - nbval
   # to edit .ipynb
   - jupyter
+  # to diff .ipynb files
+  - nbdime

launchcontainer

@@ -1,7 +1,7 @@
 #!/bin/sh -x
 
 if [ -z "$DOCKER_IMAGE" ]; then
-    DOCKER_IMAGE="pavics/workflow-tests:190312.1"
+    DOCKER_IMAGE="pavics/workflow-tests:190321"
 fi
 
 UID="`id -u`"

launchnotebook

@@ -7,7 +7,7 @@ if [ -z "$PORT" ]; then
 fi
 
 if [ -z "$DOCKER_IMAGE" ]; then
-    DOCKER_IMAGE="pavics/workflow-tests:190312.1"
+    DOCKER_IMAGE="pavics/workflow-tests:190321"
 fi
 
 UID="`id -u`"

notebooktoken

@@ -0,0 +1,3 @@
+#!/bin/sh -x
+
+docker exec -it birdy-notebook su jenkins -s /bin/bash -c 'jupyter notebook list'

runtest

@@ -18,3 +18,31 @@ fi
 export PYTHONWARNINGS="ignore:Unverified HTTPS request"
 
 py.test --nbval $NOTEBOOKS --sanitize-with notebooks/output-sanitize.cfg
+EXIT_CODE="$?"
+
+# lowercase SAVE_RESULTING_NOTEBOOK string
+SAVE_RESULTING_NOTEBOOK="`echo "$SAVE_RESULTING_NOTEBOOK" | tr '[:upper:]' '[:lower:]'`"
+
+
+# save notebooks resulting from the run
+# this might not be the same as what py.test have seen since it's another run
+
+# user can manually diff the original with the resulting notebooks this way:
+# nbdiff original.ipynb resulting.ipynb.output.ipynb (conda install nbdime)
+
+# work-around as nbval can not save the result of the run
+# see https://github.com/computationalmodelling/nbval/issues/112
+
+if [ x"$SAVE_RESULTING_NOTEBOOK" = xtrue ]; then
+    mkdir -p buildout
+    for nb in $NOTEBOOKS; do
+        filename="`basename "$nb"`"
+        filename="`echo "$filename" | sed "s/.ipynb$//"`"  # remove .ipynb ext
+        jupyter nbconvert --to notebook --execute \
+            --ExecutePreprocessor.timeout=60 --allow-errors \
+            --output-dir buildout --output ${filename}.output.ipynb $nb
+    done
+fi
+
+# exit with return code from py.test
+exit $EXIT_CODE