From 1db47bcd654e0d11394a76b8190987085dd9d274 Mon Sep 17 00:00:00 2001 From: cat-bro Date: Tue, 21 Mar 2023 13:39:03 +1100 Subject: [PATCH] try to fix linting --- .github/workflows/check_tpv_config.yml | 69 +++++++++++-------- .github/workflows/check_tpv_config/hosts | 2 + .../check_tpv_config/template_playbook.yml | 11 +++ .github/workflows/template_playbook.yml | 7 -- 4 files changed, 55 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/check_tpv_config/hosts create mode 100644 .github/workflows/check_tpv_config/template_playbook.yml delete mode 100644 .github/workflows/template_playbook.yml diff --git a/.github/workflows/check_tpv_config.yml b/.github/workflows/check_tpv_config.yml index 6f3fe1538..0d7888eed 100644 --- a/.github/workflows/check_tpv_config.yml +++ b/.github/workflows/check_tpv_config.yml @@ -31,7 +31,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install total-perspective-vortex==$TPV_VERSION pyyaml galaxy-data + pip install total-perspective-vortex==$TPV_VERSION pyyaml galaxy-app env: TPV_VERSION: '2.1.0' - name: Check TPV files and job conf @@ -42,42 +42,57 @@ jobs: import re import subprocess import os + import shutil - with open('.vault_pass.txt') as password_file: + working_dir = '.github/workflows/check_tpv_config' + hosts_file = os.path.join(working_dir, 'hosts') + playbook_file = os.path.join(working_dir, 'template_playbook.yml') + files_dir = os.path.join(working_dir, 'files') + templates_dir = os.path.join(working_dir, 'templates') + os.mkdir(files_dir) + os.mkdir(templates_dir) + + tpv_dir = 'files/galaxy/dynamic_job_rules/production/total_perspective_vortex' + tpv_config_files = [os.path.join(tpv_dir, f) for f in os.listdir(tpv_dir)] + job_conf_template = 'templates/galaxy/config/aarnet_job_conf.yml.j2' + tpv_files = [] + + replace_patterns = ['\{\{\s?vault_\w+\s?\}\}', '\{\{\s?hostname\s?\}\}'] + + with open('.vault_pass.txt', 'w') as password_file: password_file.write('FalsePassword') def template_file(fname): - output_fname = fname.replace('.j2', '') - playbook = '.github/workflows/template_playbook.yml' - vars_files = [ - 'host_vars/aarnet.yml', - 'host_vars/aarnet.usegalaxy.org.au', - ] - extra_vars = f'src={fname} dest={output_fname} {" ".join(["@"+vf for vf in vars_files])}' - ansible_command = f'ansible-playbook {playbook} --extra-vars \'{extra_vars}\'' - subprocess.check_output(ansible_command, shell=True) + output_fname = fname.replace(templates_dir, files_dir).replace('.j2', '') + extra_vars = f'src={fname.replace(templates_dir+"/", "")} dest={os.path.realpath(output_fname)}' + ansible_command = f'ansible-playbook -i {hosts_file} {playbook_file} --extra-vars \'{extra_vars}\'' + try: + subprocess.check_output(ansible_command, shell=True) + except subprocess.CalledProcessError as e: + print(e.output) + raise Exception('Failed to template {os.path.basename(fname)}') return output_fname def sanitize_text(fname): - output_fname = fname.replace('.yml.j2', 'sanitized.yml.j2') with open(fname) as handle: text = handle.read() - sanitized_text = re.sub('\{\{\s?vault_\w+\s?\}\}', 'text_replaced', text) - with open(output_fname, 'w') as handle: - handle.write(sanitized_text) - return output_fname - - tpv_dir = 'files/galaxy/dynamic_job_rules/production/total_perspective_vortex' - tpv_templates = [os.path.join(tpv_dir, f) for f in os.listdir(tpv_dir) if f.endswith('yml.j2')] - tpv_files = [os.path.join(tpv_dir, f) for f in os.listdir(tpv_dir) if f.endswith('yml.j2')] - job_conf_template = 'templates/galaxy/config/aarnet_job_conf.yml.j2' + for rp in replace_patterns: + text = re.sub(rp, 'text_replaced', text) + with open(fname, 'w') as handle: + handle.write(text) - for t in tpv_templates + [job_conf_template]: - # substitute all vault passwords - sanitized_template = sanitize_text(t) - templated_file = template_file(sanitized_template) - if 'tpv_dir' in t: - tpv_files.append(templated_file) + for t in tpv_config_files + [job_conf_template]: + if t.endswith('.yml'): + test_fname = os.path.join(files_dir, os.path.basename(t)) + shutil.copy(t, test_fname) + tpv_files.append(test_fname) + elif t.endswith('.yml.j2'): + test_fname = os.path.join(templates_dir, os.path.basename(t)) + shutil.copy(t, test_fname) + sanitize_text(test_fname) + templated_file = template_file(test_fname) + if 'tpv_dir' in t: + tpv_files.append(templated_file) for t in tpv_files: subprocess.check_output(f'tpv lint {t}', shell=True) \ No newline at end of file diff --git a/.github/workflows/check_tpv_config/hosts b/.github/workflows/check_tpv_config/hosts new file mode 100644 index 000000000..e46aecbf9 --- /dev/null +++ b/.github/workflows/check_tpv_config/hosts @@ -0,0 +1,2 @@ +[defaults] +localhost ansible_connection=local \ No newline at end of file diff --git a/.github/workflows/check_tpv_config/template_playbook.yml b/.github/workflows/check_tpv_config/template_playbook.yml new file mode 100644 index 000000000..affa3ad1d --- /dev/null +++ b/.github/workflows/check_tpv_config/template_playbook.yml @@ -0,0 +1,11 @@ +--- +- hosts: localhost + vars_files: + - ../../../group_vars/aarnet.yml + - ../../../group_vars/galaxyservers.yml + - ../../../host_vars/aarnet.usegalaxy.org.au.yml + tasks: + - name: Template file + template: + src: "{{ src }}" + dest: "{{ dest }}" \ No newline at end of file diff --git a/.github/workflows/template_playbook.yml b/.github/workflows/template_playbook.yml deleted file mode 100644 index 5e16fac06..000000000 --- a/.github/workflows/template_playbook.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- hosts: localhost - tasks: - - name: Template file - template: - src: "{{ src }}" - dest: "{{ dest }}" \ No newline at end of file