From 9f1e1069c0b7e792155ca1c6b69ffd359a9d27a1 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Tue, 14 Dec 2021 10:04:48 -0500 Subject: [PATCH 1/7] Wrap wait_for_job in try/except block --- lib/workflow.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/lib/workflow.py b/lib/workflow.py index e9503bf..1d70738 100644 --- a/lib/workflow.py +++ b/lib/workflow.py @@ -391,16 +391,20 @@ def wait_for_jobs(gi: GalaxyInstance, invocations: dict): job_id = step['job_id'] if job_id is not None: print(f"Waiting for job {job_id} on {lib.GALAXY_SERVER}") - status = gi.jobs.wait_for_job(job_id, 86400, 10, False) - data = gi.jobs.show_job(job_id, full_details=True) - metrics = { - 'workflow_id': wfid, - 'history_id': hid, - 'metrics': data, - 'status': status, - 'server': lib.GALAXY_SERVER - } - output_path = os.path.join(METRICS_DIR, f"{job_id}.json") - with open(output_path, "w") as f: - json.dump(metrics, f, indent=4) - print(f"Wrote metrics to {output_path}") + try: + # TDOD Should retry if anything throws an exception. + status = gi.jobs.wait_for_job(job_id, 86400, 10, False) + data = gi.jobs.show_job(job_id, full_details=True) + metrics = { + 'workflow_id': wfid, + 'history_id': hid, + 'metrics': data, + 'status': status, + 'server': lib.GALAXY_SERVER + } + output_path = os.path.join(METRICS_DIR, f"{job_id}.json") + with open(output_path, "w") as f: + json.dump(metrics, f, indent=4) + print(f"Wrote metrics to {output_path}") + except Exception as e: + print(f"ERROR: {e}") From ee31c30cd5a8310d74b38aa75c3244703255c5e4 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Thu, 16 Dec 2021 11:50:00 -0500 Subject: [PATCH 2/7] Bump version to 1.4.1-dev --- abm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/abm.py b/abm.py index c6aa12c..c713897 100644 --- a/abm.py +++ b/abm.py @@ -20,7 +20,7 @@ log = logging.getLogger('abm') log.setLevel(logging.ERROR) -VERSION = '1.4.0' +VERSION = '1.4.1-dev' BOLD = '\033[1m' CLEAR = '\033[0m' From 737343495a0498573fcfa41dba6e69e14ecdd55b Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Thu, 16 Dec 2021 11:50:24 -0500 Subject: [PATCH 3/7] New sample run config --- config/dna2.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 config/dna2.yml diff --git a/config/dna2.yml b/config/dna2.yml new file mode 100644 index 0000000..fbaa32e --- /dev/null +++ b/config/dna2.yml @@ -0,0 +1,30 @@ +- workflow_id: d8beac5a0d816eee + output_history_base_name: DNA 29C 116G-MEM + runs: + - history_name: HG00171 + inputs: + - name: FASTQ Dataset + dataset_id: 3ef6e7918b01e665 + - history_name: HG00152 + inputs: + - name: FASTQ Dataset + dataset_id: daf58ed87714bfc3 + - history_name: HG00599 + inputs: + - name: FASTQ Dataset + dataset_id: 1c8ebd93f5710bc6 + - history_name: HG01167 + inputs: + - name: FASTQ Dataset + dataset_id: c49cc12509f57ec1 + - history_name: HG003 + inputs: + - name: FASTQ Dataset + dataset_id: 845199e085d0abbe +- workflow_id: b94314cb9cb46380 + output_history_base_name: PairedDNA 29C 116G-MEM + runs: + - history_name: SRS9540951 + inputs: + - name: FASTQ Dataset + dataset_id: 4b0ef3aaf341f57b From 4fd2c39ead2bdb9f843c555c33caa03a7812471d Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Thu, 16 Dec 2021 11:51:20 -0500 Subject: [PATCH 4/7] Reorder nested loops while running experiment --- lib/benchmark.py | 71 +++++++++++++++++++++++++++++++++--------------- lib/common.py | 3 +- lib/workflow.py | 11 ++++++++ 3 files changed, 61 insertions(+), 24 deletions(-) diff --git a/lib/benchmark.py b/lib/benchmark.py index 2c0906c..b39880d 100644 --- a/lib/benchmark.py +++ b/lib/benchmark.py @@ -38,27 +38,49 @@ def run(args: list): profiles = load_profiles() num_runs = config['runs'] - for n in range(num_runs): + for cloud in config['cloud']: + if cloud not in profiles: + print(f"WARNING: No profile found for {cloud}") + continue + if not set_active_profile(cloud): + print(f"ERROR: Unable to set the profile for {cloud}") + continue + if lib.KUBECONFIG is None: + print(f"ERROR: No kubeconfig set for {cloud}") + continue print("------------------------") - print(f"Benchmarking run #{n+1}") - for cloud in config['cloud']: - if cloud not in profiles: - print(f"WARNING: no profile for instance {cloud}") + print(f"Benchmarking: {cloud}") + for conf in config['job_configs']: + job_conf_path = f"rules/{conf}.yml" + if not helm.update([job_conf_path]): + print(f"WARNING: job conf not found {conf}") continue - if not set_active_profile(cloud): - print(f"WARNING: unable to set {cloud} as the active profile") - if lib.KUBECONFIG is None: - print(f"WARNGING: no kubeconfig for instance {cloud}") - continue - for job_conf in config['job_configs']: - job_conf_path = f"rules/{job_conf}.yml" - if not helm.update([job_conf_path]): - print(f"WARNING: job conf not found {job_conf}") - continue - history_name_prefix = f"Run {n} {job_conf}" + for n in range(num_runs): + history_name_prefix = f"{n} {cloud} {conf}" for workflow_conf in config['workflow_conf']: workflow.run([workflow_conf, history_name_prefix]) + # for n in range(num_runs): + # print("------------------------") + # print(f"Benchmarking run #{n+1}") + # for cloud in config['cloud']: + # if cloud not in profiles: + # print(f"WARNING: no profile for instance {cloud}") + # continue + # if not set_active_profile(cloud): + # print(f"WARNING: unable to set {cloud} as the active profile") + # if lib.KUBECONFIG is None: + # print(f"WARNGING: no kubeconfig for instance {cloud}") + # continue + # for job_conf in config['job_configs']: + # job_conf_path = f"rules/{job_conf}.yml" + # if not helm.update([job_conf_path]): + # print(f"WARNING: job conf not found {job_conf}") + # continue + # history_name_prefix = f"Run {n} {job_conf}" + # for workflow_conf in config['workflow_conf']: + # workflow.run([workflow_conf, history_name_prefix]) + def test(args: list): print(common.GALAXY_SERVER) @@ -73,15 +95,19 @@ def summarize(args: list): :return: None """ row = [''] * 12 + print("Workflow,History,Server,Tool ID,State,Slots,Memory,Runtime,CPU,Memory Limit,Memory Max usage,Memory Soft Limit") for file in os.listdir(METRICS_DIR): input_path = os.path.join(METRICS_DIR, file) with open(input_path, 'r') as f: data = json.load(f) - row[0] = data['workflow_id'] - row[1] = data['history_id'] - row[2] = data['server'] if data['server'] is not None else 'https://iu1.usegvl.org/galaxy' - row[3] = data['metrics']['tool_id'] - row[4] = data['metrics']['state'] + row[0] = data['run'] + row[1] = data['cloud'] + row[2] = data['conf'] + row[3] = data['workflow_id'] + row[4] = data['history_id'] + row[5] = data['server'] if data['server'] is not None else 'https://iu1.usegvl.org/galaxy' + row[6] = data['metrics']['tool_id'] + row[7] = data['metrics']['state'] add_metrics_to_row(data['metrics']['job_metrics'], row) print(','.join(row)) @@ -91,7 +117,8 @@ def add_metrics_to_row(metrics_list: list, row: list): for job_metrics in metrics_list: if job_metrics['name'] in accept_metrics: index = accept_metrics.index(job_metrics['name']) - row[index + 5] = job_metrics['raw_value'] + row[index + 8] = job_metrics['raw_value'] + # row.append(job_metrics['raw_value']) diff --git a/lib/common.py b/lib/common.py index c51e18b..df3c830 100644 --- a/lib/common.py +++ b/lib/common.py @@ -41,9 +41,8 @@ def connect(): def set_active_profile(profile_name: str): - print(f"Parsing profile for {profile_name}") + # print(f"Parsing profile for {profile_name}") lib.GALAXY_SERVER, lib.API_KEY, lib.KUBECONFIG = parse_profile(profile_name) - print(lib.KUBECONFIG) return lib.GALAXY_SERVER != None diff --git a/lib/workflow.py b/lib/workflow.py index 1d70738..44b23e7 100644 --- a/lib/workflow.py +++ b/lib/workflow.py @@ -230,6 +230,11 @@ def run(args: list): print(f"Wrote invocation data to {output_path}") invocations = gi.invocations.wait_for_invocation(id, 86400, 10, False) print("Waiting for jobs") + if len(args) > 1: + for parts in args[1].split(): + invocations['run'] = parts[0] + invocations['cloud'] = parts[1] + invocations['job_conf'] = parts[2] wait_for_jobs(gi, invocations) print("Benchmarking run complete") return True @@ -387,6 +392,9 @@ def wait_for_jobs(gi: GalaxyInstance, invocations: dict): """ wfid = invocations['workflow_id'] hid = invocations['history_id'] + run = invocations['run'] + cloud = invocations['cloud'] + conf = invocations['job_conf'] for step in invocations['steps']: job_id = step['job_id'] if job_id is not None: @@ -396,6 +404,9 @@ def wait_for_jobs(gi: GalaxyInstance, invocations: dict): status = gi.jobs.wait_for_job(job_id, 86400, 10, False) data = gi.jobs.show_job(job_id, full_details=True) metrics = { + 'run': run, + 'cloud': cloud, + 'job_conf': conf, 'workflow_id': wfid, 'history_id': hid, 'metrics': data, From ef112bb2b8563654e91222dfff69c48cd954ed38 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Thu, 16 Dec 2021 12:49:37 -0500 Subject: [PATCH 5/7] Fix path for loading menu.yml --- abm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/abm.py b/abm.py index c713897..b754309 100644 --- a/abm.py +++ b/abm.py @@ -118,7 +118,7 @@ def alias(shortcut, fullname): def parse_menu(): log.debug('parse_menu') - menu_config = f'{os.path.dirname(__file__)}/lib/menu.yml' + menu_config = f'{os.path.dirname(os.path.abspath(__file__))}/lib/menu.yml' if not os.path.exists(menu_config): print(f"ERROR: Unable to load the menu configuration from {menu_config}") sys.exit(1) From ded8aac3ee1fdfcabe026d39690b38ed9a03d17d Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Fri, 17 Dec 2021 17:13:12 -0500 Subject: [PATCH 6/7] Update README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 489daef..36f6eb6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Benchmarking Scripts +# Automated Benchmarking An opinionated Python Bioblend script for automating benchmarking tasks in Galaxy. @@ -7,8 +7,8 @@ An opinionated Python Bioblend script for automating benchmarking tasks in Galax 1. Clone this repository. ```bash - git clone https://github.com/ksuderman/bioblend-scripts.git - cd bioblend-scripts + git clone https://github.com/galaxyproject/gxabm.git + cd gxabm ``` 1. Create a virtual env and install the required libraries ```bash From 2fe024dd2f3f986be5409322d8234088abd7a25a Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Sun, 2 Jan 2022 14:04:15 -0500 Subject: [PATCH 7/7] Reorder benchmark execution to minimize helm updates --- abm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/abm.py b/abm.py index b754309..8d87e01 100644 --- a/abm.py +++ b/abm.py @@ -18,9 +18,9 @@ from lib import job, dataset, workflow, history, library, folder, benchmark, helm, kubectl, config log = logging.getLogger('abm') -log.setLevel(logging.ERROR) +log.setLevel(logging.INFO) -VERSION = '1.4.1-dev' +VERSION = '1.4.1' BOLD = '\033[1m' CLEAR = '\033[0m'