From 439e7d54062b93397bc0d06e516e52f9d2f1f9b1 Mon Sep 17 00:00:00 2001
From: bud42 <bdboyd42@gmail.com>
Date: Fri, 26 Oct 2018 14:36:37 -0400
Subject: [PATCH 1/7] Add support for setting dax_version_hash

---
 .DS_Store              | Bin 14340 -> 14340 bytes
 dax/dax_tools_utils.py |  25 ++++++++++++++++++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/.DS_Store b/.DS_Store
index ffbb913d599d8bdfe6e8a5e1868b9e33bf5761d0..4e084a62a646978ed272cdca1d17e52db2479826 100644
GIT binary patch
delta 1030
zcma)*OH30{6o$`FE8dASItZ0gD6dM3sDKI)(Ex>NRYZ-VVthdBj9P@YwpBqQjRuVo
zQR5`OjWHTyd`05}<0E0=!so&S7skYm8aFI7fsHZ7+YUiPTsVu#ob#P~?w#+<bOpPD
zxASClYj$}?LanG>Og0@i7i!1kGOgO2lQO|<&9zT0E-kNg^~*^UlSPx5s$H03){dH`
zsG(A*_j_8^IFT$;29s^9VTaeZeTOg7uGeQtIf8tjtyeq|wJxCeHv3dXug{Y57+V+1
z*MwAl*VAIYR3OM6Rv-55)*Fk{#Ueot3-#N4{^6xF#S$hfQDIe!$E&vJ?PiN}m|QEY
z*{O!YzJOm(&J!zG5~c|an*yz^sy`Ce(@v>|$<Nuw*yc+-;mER}>i=^rH)CsK#HlE%
zVx!^9R+X<m#b4Q0RZP)%ED$3r%_J8!P!olygL>!`U8H;TfO_d6J)<}Djy}*A`bOVr
zkcQ|73@{=QV<96A7G%JR0?a}QO5wy})S?bcu@dXC0gY&a3O@qaju6_g7yHnK{pi67
z<ebDAoW*6_#4X&$9rWP^Ug8zr;Um6c07KgAiFJwYh3-X*M-4W6{QC*9ai*9irJ8f>
z1zJ|RRU1g2YqVyHlGc%KkEU7DGjeT`B*#Rt{#ZE9A&GpHLOhu2;4ER76cuw~;8W9C
zNvSBwV`N3p*Uc#xMcK$FsMCr{P80d4bXw!$l#=+jB>d$%I$HLBxqe0c^oe`^32?h2
zQn=q-<f9N}s6sX7V*%V)juoiKDy-&qJ@8@^TH=O-Xh#S3pc7Fz4&o3FbInI^5~sQ4
zb2yJHxQc7IjvKg-UOd8MJkgw%RPBJtF?sOD)Bf&<eNKK(11aR8w$^~+^0oxLJi~N)
z_pPar*~5J`_dfT=D~%K(mBJMHyH@9y@-h<|o9CAC)kZ3$E#!+;xkfs6*6m`Br^8)F
f_@5pB#e-XCtmHpJFz!;zChb@Dq`PObn^S)Sp_uoB

delta 1010
zcmaiyX>3eU6vxkBhrGwVdfVx=RXT>z(M{X2j@nu+sx2ZdYKu(!raY#dGE<Z=u|`5-
zd2J*@5F$Z>XoWA95KCeS(nKr?lGX<jB9-_exHE5p55)a&lY9Q>-gEBnf3`PmZ@Q2n
zkZQPuyC18p(dz<@Rr&^>@i{ZkI4nHI-W;1T#A<aoiwY>ZROM`ZYWh%DPQjSMDW#nv
zA<>#-W6aRAvchpwR5cnIkj8mUbFIF*Sr4@cr=moo_Q9Mj(YztwtbnH1>po3fQp|kk
zV9pmul#(?*xofct!!tRn4$5|fH<p+?g0bpVdZQ+Su~P1ljL-9XYkhv<(Nwe0*l~<6
zSnFF8)B}y8A{Lx5k@1?kKtqGCF%(QdZY*3>%(>ysPD+aWWSZ<_VWf{@O^bA4BqXMU
zRRK~com}LhN~)%M3eXnXLr1BDuF^HSPWP#kUQ;)HqObIget-b<g%vg=BLxnOKt2lK
zMj@tS2FfrO^HGC^ScE#Pz)GxwAI;c^P1uYuc3~e5z(6aG;1o{VaRz76fm^tZJGhI-
zc!8IAg>HOA555==5@*Gi%`7Wt+5dFU{)c<d<4wzwQ&NYxa`H!ypIl-*v{qTTojHs^
zt}8qsZQvm8<Qa;%rwFApa%7o2Ta{)BM2|d^=P8QlWom9dN|pYII8z(TJc`g%(c4rf
zut}=)N?1&_h)q?bBchL~PGd9F1UWT=oaS^98EJ|A!|H#4Kc$!Sp1vXu3i?aX1101v
z33wceqkt<=iP=~nAuqyWEJZz*Lzkc<nAc(*TCf${umd|K?EMmUE81`n$8a1ca1y6d
zaUK_N5$(8yYq*Y^QTUJW%xJcGhV4E$r)SvZ8^yBYIMT>V>l*@EX{|p{D<ew##FCD#
z%k58Wd5Y^p+tRM{=c1Vw3XzY36#6sQ)K6sCdo9!Zmdo1~%A@6^N3u$Pz4*59oyC78
h_R25)cg4TyO)K10a@L!oRQJ9i<GUld{g7j2%1^2O_Obu~

diff --git a/dax/dax_tools_utils.py b/dax/dax_tools_utils.py
index 259379f7..3309e89b 100644
--- a/dax/dax_tools_utils.py
+++ b/dax/dax_tools_utils.py
@@ -816,6 +816,27 @@ def get_version_assessor(assessor_path):
     return version
 
 
+def get_dax_version_hash_assessor(assessor_path):
+    """
+    Get the dax_version_hash of assessor we are uploading from text file
+
+    :param assessor_path: path for the assessor
+    :return: version of the assessor from the version.txt file
+    """
+    dax_version_hash = ''
+    fpath = os.path.join(assessor_path, 'dax_version_hash.txt')
+
+    try:
+        with open(fpath, 'r') as f_obj:
+            dax_version_hash = f_obj.read().strip()
+            f_obj.close()
+
+    except IOError as e:
+        LOGGER.warn('failed to read dax_version_hash:' + fpath + ':' + e)
+
+    return dax_version_hash
+
+
 def generate_snapshots(assessor_path):
     """
     Generate Snapshots from the PDF if it exists.
@@ -967,6 +988,7 @@ def upload_assessor(xnat, assessor_dict, assessor_path):
     """
     # get spiderpath from version.txt file:
     version = get_version_assessor(assessor_path)
+    dax_version_hash = get_dax_version_hash_assessor(assessor_path)
     session_obj = XnatUtils.select_obj(xnat,
                                        assessor_dict['project_id'],
                                        assessor_dict['subject_label'],
@@ -1028,7 +1050,8 @@ def upload_assessor(xnat, assessor_dict, assessor_path):
                 xsitype + '/jobnode': ctask.get_jobnode(),
                 xsitype + '/memused': ctask.get_memused(),
                 xsitype + '/walltimeused': ctask.get_walltime(),
-                xsitype + '/jobstartdate': ctask.get_jobstartdate()
+                xsitype + '/jobstartdate': ctask.get_jobstartdate(),
+                xsitype + '/dax_version_hash': dax_version_hash
             })
 
             # Delete the task from diskq

From 7bcab9ee18c39ac412977be5f6c8844deae97c83 Mon Sep 17 00:00:00 2001
From: bud42 <bdboyd42@gmail.com>
Date: Sun, 28 Oct 2018 12:33:18 -0400
Subject: [PATCH 2/7] Update singularity job template

---
 dax/processors.py                                |  2 +-
 dax/templates/SLURM/singularity_job_template.txt | 11 ++++++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/dax/processors.py b/dax/processors.py
index 27b89abc..30df888e 100644
--- a/dax/processors.py
+++ b/dax/processors.py
@@ -1007,7 +1007,7 @@ def build_text(self, var2val, input_list, jobdir, dstdir):
         cmd += 'INDIR=$JOBDIR/INPUTS\n'
         cmd += 'OUTDIR=$JOBDIR/OUTPUTS\n'
         cmd += 'DSTDIR={}\n\n'.format(dstdir)
-
+        cmd += 'CONTAINERPATH={}\n\n'.format(self.container_path)
         # Append the main command
         cmd += 'MAINCMD=\"'
         cmd += self.command.format(**var2val)
diff --git a/dax/templates/SLURM/singularity_job_template.txt b/dax/templates/SLURM/singularity_job_template.txt
index 6ec58b29..95018e3a 100644
--- a/dax/templates/SLURM/singularity_job_template.txt
+++ b/dax/templates/SLURM/singularity_job_template.txt
@@ -22,12 +22,14 @@ date
 #DSTDIR=
 #INLIST=
 #OUTLIST=
+#CONTAINERPATH=
 #MAINCMD=
 ${job_cmds}
 #=============================================================================
 echo $DSTDIR
 echo $INDIR
 echo $OUTDIR
+echo $CONTAINERPATH
 
 mkdir -p $INDIR
 mkdir -p $OUTDIR
@@ -40,21 +42,21 @@ for IN in "${INLIST[@]}"; do
     elif [ $col2 == "DIRJ" ]; then
         CMD="curl -s -n $col3?format=zip -o $INDIR/${col1}.zip && unzip -j $INDIR/${col1}.zip -d $INDIR/$col1"
     else
-        CMD="curl -s -n $col3?format=zip -o $INDIR/${col1}.zip && unzip $INDIR/${col1}.zip -d $INDIR/$col1"
+        CMD="curl -s -n '$col3?format=zip&structure=simplified' -o $INDIR/${col1}.zip && unzip $INDIR/${col1}.zip -d $INDIR/$col1 && mv $INDIR/$col1/*/out/* $INDIR/$col1"
     fi
     echo $CMD
     eval $CMD
 done
 
 # Run main command
-module load GCC Singularity
 echo $_JAVA_OPTIONS
 echo $MAINCMD
 eval $MAINCMD
 
-# Write version file
+# Write version files
 mkdir -p $DSTDIR
 echo $VERSION > $DSTDIR/version.txt
+sha256sum $CONTAINERPATH | awk '{print $1}' > $DSTDIR/dax_version_hash.txt
 
 # Handle outputs
 errors=0
@@ -84,4 +86,7 @@ done
 if [ $errors -gt 0 ] || [ $haspdf != 1 ]; then echo "JOB_FAILED" && touch $DSTDIR/JOB_FAILED.txt;
 else echo "COMPLETE" && touch $DSTDIR/READY_TO_UPLOAD.txt; fi
 
+rm -rf $INDIR $OUTDIR
+
 echo "DONE"
+

From f151201db6ad644f1bfdbacebfc70fa3828f8153 Mon Sep 17 00:00:00 2001
From: bud42 <bdboyd42@gmail.com>
Date: Mon, 29 Oct 2018 12:04:50 -0400
Subject: [PATCH 3/7] Handle input assessor attributes

---
 dax/processors.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/dax/processors.py b/dax/processors.py
index 30df888e..77103704 100644
--- a/dax/processors.py
+++ b/dax/processors.py
@@ -937,7 +937,7 @@ def build_cmds(self, assr, jobdir):
         """
         assr_label = assr.label()
         dstdir = os.path.join(DAX_Settings().get_results_dir(), assr_label)
-        
+
         # Find values for the xnat inputs
         var2val, input_list = self.parser.find_inputs(assr)
 
@@ -968,6 +968,10 @@ def build_cmds(self, assr, jobdir):
                 _ref = attr_in['ref']
                 _refval = assr_inputs[_ref].rsplit('/', 1)[1]
                 _val = assr.parent().scan(_refval).attrs.get(_attr)
+            elif _obj == 'assessor':
+                _ref = attr_in['ref']
+                _refval = assr_inputs[_ref].rsplit('/', 1)[1]
+                _val = assr.parent().assessor(_refval).attrs.get(_attr)
             else:
                 LOGGER.error('invalid YAML')
                 err = 'YAML File:contains invalid attribute:{}'

From 78653ac79d8abbae6d32ad8adc791a0fe9baedc8 Mon Sep 17 00:00:00 2001
From: bud42 <bdboyd42@gmail.com>
Date: Mon, 29 Oct 2018 12:31:38 -0400
Subject: [PATCH 4/7] Fix bug with XnatSwitchProcessStatus

---
 bin/Xnat_tools/XnatSwitchProcessStatus | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/bin/Xnat_tools/XnatSwitchProcessStatus b/bin/Xnat_tools/XnatSwitchProcessStatus
index 1adc1933..14184fb4 100755
--- a/bin/Xnat_tools/XnatSwitchProcessStatus
+++ b/bin/Xnat_tools/XnatSwitchProcessStatus
@@ -450,11 +450,13 @@ def set_need_inputs_proctype(xnat, assessor, need_inputs, full_regex=False):
             else:
                 set_proc_status(assessor_obj, task.NEED_INPUTS)
 
-            li_assessors = xnat.get_assessor_out_resources(
+            li_assessors = XnatUtils.list_assessor_out_resources(
+                xnat,
                 a_linked_dict['project_id'],
                 a_linked_dict['subject_id'],
                 a_linked_dict['session_id'],
                 a_linked_dict['label'])
+
             for resource in li_assessors:
                 delete_assr_resource(assessor_obj, resource['label'])
 

From 7ca1bee3ef76c3c4869735865bf6b323180957f7 Mon Sep 17 00:00:00 2001
From: bud42 <bdboyd42@gmail.com>
Date: Mon, 29 Oct 2018 12:36:02 -0400
Subject: [PATCH 5/7] Fix another bug in XnatSwitchProcessStatus

---
 bin/Xnat_tools/XnatSwitchProcessStatus | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/Xnat_tools/XnatSwitchProcessStatus b/bin/Xnat_tools/XnatSwitchProcessStatus
index 14184fb4..b1fbeb65 100755
--- a/bin/Xnat_tools/XnatSwitchProcessStatus
+++ b/bin/Xnat_tools/XnatSwitchProcessStatus
@@ -569,7 +569,8 @@ def set_status_assessor(xnat, assessor, status,
                 for resource in resources:
                     delete_assr_resource(assessor_obj, resource)
             else:
-                resources = xnat.get_assessor_out_resources(
+                resources = XnatUtils.list_assessor_out_resources(
+                    xnat,
                     assessor['project_id'], assessor['subject_label'],
                     assessor['session_label'], assessor['label'])
                 for resource in resources:

From 4c46697857d06e492ddd32459019d60f72acba81 Mon Sep 17 00:00:00 2001
From: bud42 <bdboyd42@gmail.com>
Date: Mon, 29 Oct 2018 12:54:15 -0400
Subject: [PATCH 6/7] Set a default for fmatch when type is FILE

---
 dax/processor_parser.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/dax/processor_parser.py b/dax/processor_parser.py
index 1fff958c..aeaba004 100644
--- a/dax/processor_parser.py
+++ b/dax/processor_parser.py
@@ -275,7 +275,13 @@ def find_inputs(self, assr):
 
             if 'fmatch' in cur_res:
                 fmatch = cur_res['fmatch']
+            elif cur_res['ftype'] == 'FILE':
+                # Default to all
+                fmatch = '*'
+            else:
+                fmatch = None
 
+            if fmatch:
                 # Get list of all files in the resource
                 file_list = robj.files().get()
 

From 2b8d3c35a003a8729b1c2640d6b0a21470d4b312 Mon Sep 17 00:00:00 2001
From: bud42 <bdboyd42@gmail.com>
Date: Mon, 29 Oct 2018 14:49:06 -0400
Subject: [PATCH 7/7] Fix handling of missing dax_version_hash

---
 dax/dax_tools_utils.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/dax/dax_tools_utils.py b/dax/dax_tools_utils.py
index 3309e89b..d1261ef9 100644
--- a/dax/dax_tools_utils.py
+++ b/dax/dax_tools_utils.py
@@ -829,10 +829,9 @@ def get_dax_version_hash_assessor(assessor_path):
     try:
         with open(fpath, 'r') as f_obj:
             dax_version_hash = f_obj.read().strip()
-            f_obj.close()
 
     except IOError as e:
-        LOGGER.warn('failed to read dax_version_hash:' + fpath + ':' + e)
+        LOGGER.warn('failed to read dax_version_hash:' + str(e))
 
     return dax_version_hash