From 3512a556ee5ab2d37631ec40a20fe8cae7404bfe Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Thu, 1 Jun 2023 11:27:28 -0400 Subject: [PATCH 1/7] Change wait_for_invocation terminal states --- abm/lib/benchmark.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/abm/lib/benchmark.py b/abm/lib/benchmark.py index 7126101..b125c4e 100644 --- a/abm/lib/benchmark.py +++ b/abm/lib/benchmark.py @@ -161,7 +161,8 @@ def run(context: Context, workflow_path, history_prefix: str, experiment: str): new_history_name = f"{experiment} {new_history_name}" invocation = gi.workflows.invoke_workflow(wfid, inputs=inputs, history_name=new_history_name) id = invocation['id'] - invocations = gi.invocations.wait_for_invocation(id, 86400, 10, False) + #invocations = gi.invocations.wait_for_invocation(id, 86400, 10, False) + invocations = gi.invocations.wait_for_invocation(id, 86400, 10, True) print("Waiting for jobs") if history_prefix is not None: parts = history_prefix.split() From 48de9570ae381d116a0cccd37465c72f976b8e4b Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Fri, 23 Jun 2023 11:59:35 -0400 Subject: [PATCH 2/7] Fix error message for config url --- abm/lib/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/abm/lib/config.py b/abm/lib/config.py index 6942e3a..34da927 100644 --- a/abm/lib/config.py +++ b/abm/lib/config.py @@ -59,7 +59,7 @@ def key(context: Context, args: list): def url(context: Context, args: list): if len(args) != 2: - print(f"USAGE: abm config key ") + print(f"USAGE: abm config url ") return profile_name = args[0] url = args[1] From a6c5326bee400fc5069ec38f1e0a375547d0b9fc Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Mon, 26 Jun 2023 16:29:30 -0400 Subject: [PATCH 3/7] Add dataset to history by name --- abm/lib/dataset.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/abm/lib/dataset.py b/abm/lib/dataset.py index ba9a987..103d78f 100644 --- a/abm/lib/dataset.py +++ b/abm/lib/dataset.py @@ -1,7 +1,7 @@ import json from bioblend.galaxy import dataset_collections -from common import connect, Context, print_json, _get_dataset_data, _make_dataset_element +from common import connect, Context, print_json, _get_dataset_data, _make_dataset_element, find_history from pprint import pprint from pathlib import Path @@ -147,6 +147,7 @@ def collection(context: Context, args: list): def import_from_config(context: Context, args: list): gi = None key = None + history = None kwargs = {} while len(args) > 0: arg = args.pop(0) @@ -178,6 +179,9 @@ def import_from_config(context: Context, args: list): if gi is None: gi = connect(context) + if history is not None: + history = find_history(gi, history) + response = gi.tools.put_url(url, history, **kwargs) print(json.dumps(response, indent=4)) From 5a6721ec1b7227ca4da757753a86db6382aed088 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Sun, 9 Jul 2023 18:13:02 -0400 Subject: [PATCH 4/7] Restore the version number --- abm/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/abm/VERSION b/abm/VERSION index 2c9b4ef..3a591c1 100644 --- a/abm/VERSION +++ b/abm/VERSION @@ -1 +1 @@ -2.7.3 +2.8.0-dev.1 From 641b335a06a02c1e3fe128b82c24cae33c4a6ac8 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Mon, 10 Jul 2023 12:57:06 -0400 Subject: [PATCH 5/7] Add a slight delay before we start waiting on pods --- abm/lib/helm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/abm/lib/helm.py b/abm/lib/helm.py index 6161323..ca0a6b4 100644 --- a/abm/lib/helm.py +++ b/abm/lib/helm.py @@ -37,6 +37,8 @@ def update(context: Context, args: list): return False print(f"Applying rules {values} to {context.GALAXY_SERVER}") + print(f"Helm update namespace: {namespace}") + print(f"Helm update chart: {chart}") #command = f'{helm} upgrade galaxy {chart} -n {namespace} --reuse-values --set-file jobs.rules."container_mapper_rules\.yml".content={rules}' command = f'{helm} upgrade galaxy {chart} -n {namespace} --reuse-values -f {values}' env = get_env(context) @@ -51,6 +53,8 @@ def update(context: Context, args: list): return False print('Waiting for the new deployments to come online') + # Give kubernetes a moment to start processing the update. + time.sleep(30) wait_until_ready(namespace, env) return True From a599029b4f0ded2210ed4082a894d61748c117ec Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Mon, 10 Jul 2023 12:57:34 -0400 Subject: [PATCH 6/7] Add some debugging output --- abm/lib/experiment.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/abm/lib/experiment.py b/abm/lib/experiment.py index 5e02127..ce7c115 100644 --- a/abm/lib/experiment.py +++ b/abm/lib/experiment.py @@ -70,6 +70,8 @@ def run_on_cloud(cloud: str, config: dict): chart = config['galaxy']['chart'] if 'job_configs' in config and len(config['job_configs']) > 0: for conf in config['job_configs']: + rules_file = f"rules/{conf}.yml" + print(f"Applying {rules_file} namespace:{namespace} chart:{chart}") if not helm.update(context, [f"rules/{conf}.yml", namespace, chart]): log.warning(f"job configuration not found: rules/{conf}.yml") continue From c9b05785102372cad7ebee65516a7410b4048786 Mon Sep 17 00:00:00 2001 From: Keith Suderman Date: Mon, 10 Jul 2023 12:58:26 -0400 Subject: [PATCH 7/7] Output history name used for benchmark --- abm/lib/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/abm/lib/benchmark.py b/abm/lib/benchmark.py index 61ccfea..deb6f03 100644 --- a/abm/lib/benchmark.py +++ b/abm/lib/benchmark.py @@ -203,7 +203,7 @@ def run(context: Context, workflow_path, history_prefix: str, experiment: str): inputs[input[0]] = {'id': dsid, 'src': 'hda', 'size': dssize} else: raise Exception(f'Invalid input value') - print(f"Running workflow {wfid}") + print(f"Running workflow {wfid} in history {new_history_name}") invocation = gi.workflows.invoke_workflow(wfid, inputs=inputs, history_name=new_history_name) id = invocation['id'] #invocations = gi.invocations.wait_for_invocation(id, 86400, 10, False)