try to fix linting

.github/workflows/check_tpv_config.yml

@@ -31,7 +31,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install total-perspective-vortex==$TPV_VERSION pyyaml galaxy-data
+        pip install total-perspective-vortex==$TPV_VERSION pyyaml galaxy-app
       env:
         TPV_VERSION: '2.1.0'
     - name: Check TPV files and job conf
@@ -42,42 +42,57 @@ jobs:
         import re
         import subprocess
         import os
+        import shutil
 
-        with open('.vault_pass.txt') as password_file:
+        working_dir = '.github/workflows/check_tpv_config'
+        hosts_file = os.path.join(working_dir, 'hosts')
+        playbook_file = os.path.join(working_dir, 'template_playbook.yml')
+        files_dir = os.path.join(working_dir, 'files')
+        templates_dir = os.path.join(working_dir, 'templates')
+        os.mkdir(files_dir)
+        os.mkdir(templates_dir)
+
+        tpv_dir = 'files/galaxy/dynamic_job_rules/production/total_perspective_vortex'
+        tpv_config_files = [os.path.join(tpv_dir, f) for f in os.listdir(tpv_dir)]
+        job_conf_template = 'templates/galaxy/config/aarnet_job_conf.yml.j2'
+        tpv_files = []
+
+        replace_patterns = ['\{\{\s?vault_\w+\s?\}\}', '\{\{\s?hostname\s?\}\}']
+
+        with open('.vault_pass.txt', 'w') as password_file:
           password_file.write('FalsePassword')
 
         def template_file(fname):
-          output_fname = fname.replace('.j2', '')
-          playbook = '.github/workflows/template_playbook.yml'
-          vars_files = [
-            'host_vars/aarnet.yml',
-            'host_vars/aarnet.usegalaxy.org.au',
-          ]
-          extra_vars = f'src={fname} dest={output_fname} {" ".join(["@"+vf for vf in vars_files])}'
-          ansible_command = f'ansible-playbook {playbook} --extra-vars \'{extra_vars}\''
-          subprocess.check_output(ansible_command, shell=True)
+          output_fname = fname.replace(templates_dir, files_dir).replace('.j2', '')
+          extra_vars = f'src={fname.replace(templates_dir+"/", "")} dest={os.path.realpath(output_fname)}'
+          ansible_command = f'ansible-playbook -i {hosts_file} {playbook_file} --extra-vars \'{extra_vars}\''
+          try:
+            subprocess.check_output(ansible_command, shell=True)
+          except subprocess.CalledProcessError as e:
+            print(e.output)
+            raise Exception('Failed to template {os.path.basename(fname)}')
           return output_fname
 
         def sanitize_text(fname):
-          output_fname = fname.replace('.yml.j2', 'sanitized.yml.j2')
           with open(fname) as handle:
             text = handle.read()
-            sanitized_text = re.sub('\{\{\s?vault_\w+\s?\}\}', 'text_replaced', text)
-          with open(output_fname, 'w') as handle:
-            handle.write(sanitized_text)
-          return output_fname
-
-        tpv_dir = 'files/galaxy/dynamic_job_rules/production/total_perspective_vortex'
-        tpv_templates = [os.path.join(tpv_dir, f) for f in os.listdir(tpv_dir) if f.endswith('yml.j2')]
-        tpv_files = [os.path.join(tpv_dir, f) for f in os.listdir(tpv_dir) if f.endswith('yml.j2')]
-        job_conf_template = 'templates/galaxy/config/aarnet_job_conf.yml.j2'
+            for rp in replace_patterns:
+              text = re.sub(rp, 'text_replaced', text)
+          with open(fname, 'w') as handle:
+            handle.write(text)
 
-        for t in tpv_templates + [job_conf_template]:
-          # substitute all vault passwords
-          sanitized_template = sanitize_text(t)
-          templated_file = template_file(sanitized_template)
-          if 'tpv_dir' in t:
-            tpv_files.append(templated_file)
+        for t in tpv_config_files + [job_conf_template]:
+          if t.endswith('.yml'):
+            test_fname = os.path.join(files_dir, os.path.basename(t))
+            shutil.copy(t, test_fname)
+            tpv_files.append(test_fname)
+          elif t.endswith('.yml.j2'):
+            test_fname = os.path.join(templates_dir, os.path.basename(t))
+            shutil.copy(t, test_fname)
+            sanitize_text(test_fname)
+            templated_file = template_file(test_fname)
+            if 'tpv_dir' in t:
+              tpv_files.append(templated_file)
 
         for t in tpv_files:
           subprocess.check_output(f'tpv lint {t}', shell=True)

.github/workflows/check_tpv_config/hosts

@@ -0,0 +1,2 @@
+[defaults]
+localhost ansible_connection=local

.github/workflows/check_tpv_config/template_playbook.yml

@@ -0,0 +1,11 @@
+---
+- hosts: localhost
+  vars_files:
+  - ../../../group_vars/aarnet.yml
+  - ../../../group_vars/galaxyservers.yml
+  - ../../../host_vars/aarnet.usegalaxy.org.au.yml
+  tasks:
+  - name: Template file
+    template:
+      src: "{{ src }}"
+      dest: "{{ dest }}"