From 8b0c7dfffd96ca73ff9341c15ee0a6c41303e051 Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Thu, 26 Aug 2021 12:20:05 -0700 Subject: [PATCH 001/126] Re-added support for all brokers with task priority (#324) * Re-added support for all brokers with task priority --- CHANGELOG.md | 4 ++++ merlin/config/utils.py | 11 ++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d56aef56e..2f252fec6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed +- task priority support for amqp, amqps, rediss, redis+socket brokers ## [1.8.0] diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 9c4c95027..872f2472b 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -8,6 +8,11 @@ class Priority(enum.Enum): mid = 2 low = 3 +def is_rabbit_broker(broker): + return broker in ["rabbitmq", "amqps", "amqp"] + +def is_redis_broker(broker): + return broker in ["redis", "rediss", "redis+socket"] def get_priority(priority): broker = CONFIG.broker.name.lower() @@ -18,16 +23,16 @@ def get_priority(priority): ) if priority == Priority.mid: return 5 - if broker == "rabbitmq": + if is_rabbit_broker(broker): if priority == Priority.low: return 1 if priority == Priority.high: return 10 - if broker == "redis": + if is_redis_broker(broker): if priority == Priority.low: return 10 if priority == Priority.high: return 1 raise ValueError( - "Function get_priority has reached unknown state! Check input parameter and broker." + f"Function get_priority has reached unknown state! Maybe unsupported broker {broker}?" ) From 6d06997855e39c79a4de4188e0a30281d5c17cd2 Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Thu, 26 Aug 2021 13:12:56 -0700 Subject: [PATCH 002/126] Bugfix for merlin purge (#327) * Bugfix for merlin purge * Address reviewer comments --- CHANGELOG.md | 1 + merlin/display.py | 2 +- merlin/spec/specification.py | 3 ++- merlin/study/celeryadapter.py | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f252fec6..26d194cb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Fixed +- merlin purge queue name conflict & shell quote escape - task priority support for amqp, amqps, rediss, redis+socket brokers ## [1.8.0] diff --git a/merlin/display.py b/merlin/display.py index 67b0534b5..d3fe4a3a7 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -190,5 +190,5 @@ def print_info(args): info_str += 'echo " $ ' + x + '" && ' + x + "\n" info_str += "echo \n" info_str += r"echo \"echo \$PYTHONPATH\" && echo $PYTHONPATH" - subprocess.call(info_str, shell=True) + _ = subprocess.run(info_str, shell=True) print("") diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index e9e324377..35be16c79 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -35,6 +35,7 @@ """ import logging import os +import shlex from io import StringIO import yaml @@ -368,7 +369,7 @@ def make_queue_string(self, steps): param steps: a list of step names """ queues = ",".join(set(self.get_queue_list(steps))) - return f'"{queues}"' + return shlex.quote(queues) def get_worker_names(self): result = [] diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 86a19a875..cd15b4128 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -388,7 +388,7 @@ def purge_celery_tasks(queues, force): force_com = " -f " purge_command = " ".join(["celery -A merlin purge", force_com, "-Q", queues]) LOG.debug(purge_command) - return subprocess.call(purge_command.split()) + return subprocess.run(purge_command, shell=True).returncode def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): From 9037763ec0c60357f27d7d62b4c351a9233dccdf Mon Sep 17 00:00:00 2001 From: Alexander Winter <80929291+AlexanderWinterLLNL@users.noreply.github.com> Date: Fri, 3 Sep 2021 10:48:57 -0700 Subject: [PATCH 003/126] Refactor of Merlin internals and CI to be flake8 compliant (#326) * Refactored source to be flake8 compliant. Many changes are cosmetic/style focused - to spacing, removing unused vars, using r strings, etc. Notable changes which *could* have changed behavior included pulling out a considerable chunk of batch_worker_launch into a subfunction, construct_worker_launch_command. Added typing to most of the files touched by the flake8 refactor (any that were missed will be caught in the PyLint refactor). * The Makefile was updated in a few ways - generally cleaned up to be a bit neater (so targets are defined approx. as they're encountered/needed in a workflow), a few targets were added (one to check the whole CI pipeline before pushing), the testing targets were updated to activate the venv before running the test suites. Co-authored-by: Alexander Cameron Winter --- .github/workflows/push-pr_workflow.yml | 2 +- CHANGELOG.md | 2 +- Makefile | 166 +++++++++--------- config.mk | 2 +- docs/source/conf.py | 9 +- merlin/ascii_art.py | 2 + merlin/common/opennpylib.py | 18 +- merlin/common/tasks.py | 41 +++-- merlin/config/configfile.py | 61 +++++-- merlin/config/utils.py | 18 +- .../workflows/flux/scripts/flux_info.py | 25 +-- .../null_spec/scripts/launch_jobs.py | 47 ++--- merlin/spec/specification.py | 9 +- merlin/study/batch.py | 102 ++++++----- merlin/study/celeryadapter.py | 49 ++++-- merlin/utils.py | 6 +- setup.cfg | 2 +- setup.py | 9 +- tests/integration/run_tests.py | 11 +- tests/integration/test_definitions.py | 20 +-- tests/unit/common/test_sample_index.py | 1 - tests/unit/study/test_study.py | 21 +-- 22 files changed, 351 insertions(+), 272 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 739d9eb4a..b1a62e791 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -38,7 +38,7 @@ jobs: # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=15 --statistics --max-line-length=88 + flake8 . --count --max-complexity=15 --statistics --max-line-length=127 - name: Run pytest over unit test suite run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 26d194cb7..59370e7b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,12 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - ## [Unreleased] ### Fixed - merlin purge queue name conflict & shell quote escape - task priority support for amqp, amqps, rediss, redis+socket brokers +- Flake8 compliance ## [1.8.0] diff --git a/Makefile b/Makefile index ef6d60c6b..98122ce23 100644 --- a/Makefile +++ b/Makefile @@ -29,96 +29,88 @@ ############################################################################### include config.mk -.PHONY : all -.PHONY : install-dev .PHONY : virtualenv -.PHONY : install-workflow-deps .PHONY : install-merlin -.PHONY : clean-output -.PHONY : clean-docs -.PHONY : clean-release -.PHONY : clean-py -.PHONY : clean -.PHONY : release +.PHONY : install-workflow-deps +.PHONY : install-merlin-dev .PHONY : unit-tests -.PHONY : cli-tests +.PHONY : e2e-tests .PHONY : tests .PHONY : fix-style .PHONY : check-style +.PHONY : check-push .PHONY : check-camel-case .PHONY : checks .PHONY : reqlist -.PHONY : check-variables - - -all: install-dev install-merlin install-workflow-deps - - -# install requirements -install-dev: virtualenv - $(PIP) install -r requirements/dev.txt - - -check-variables: - - echo MAX_LINE_LENGTH $(MAX_LINE_LENGTH) - +.PHONY : release +.PHONY : clean-release +.PHONY : clean-output +.PHONY : clean-docs +.PHONY : clean-py +.PHONY : clean -# this only works outside the venv +# this only works outside the venv - if run from inside a custom venv, or any target that depends on this, +# you will break your venv. virtualenv: - $(PYTHON) -m venv $(VENV) --prompt $(PENV) --system-site-packages - $(PIP) install --upgrade pip - + $(PYTHON) -m venv $(VENV) --prompt $(PENV) --system-site-packages; \ + $(PIP) install --upgrade pip; \ + $(PIP) install -r requirements/release.txt; \ -install-workflow-deps: - $(PIP) install -r $(WKFW)feature_demo/requirements.txt +# install merlin into the virtual environment +install-merlin: virtualenv + $(PIP) install -e .; \ + merlin config; \ -install-merlin: - $(PIP) install -e . +# install the example workflow to enable integrated testing +install-workflow-deps: virtualenv install-merlin + $(PIP) install -r $(WKFW)feature_demo/requirements.txt; \ -# remove python bytecode files -clean-py: - -find $(MRLN) -name "*.py[cod]" -exec rm -f {} \; - -find $(MRLN) -name "__pycache__" -type d -exec rm -rf {} \; - +# install requirements +install-merlin-dev: virtualenv install-workflow-deps + $(PIP) install -r requirements/dev.txt; \ -# remove all studies/ directories -clean-output: - -find $(MRLN) -name "studies*" -type d -exec rm -rf {} \; - -find . -maxdepth 1 -name "studies*" -type d -exec rm -rf {} \; - -find . -maxdepth 1 -name "merlin.log" -type f -exec rm -rf {} \; +# tests require a valid dev install of merlin +unit-tests: + . $(VENV)/bin/activate; \ + $(PYTHON) -m pytest $(UNIT); \ -# remove doc build files -clean-docs: - rm -rf $(DOCS)/build +# run CLI tests - these require an active install of merlin in a venv +e2e-tests: + . $(VENV)/bin/activate; \ + $(PYTHON) $(TEST)/integration/run_tests.py --local; \ -clean-release: - rm -rf dist - rm -rf build +e2e-tests-diagnostic: + $(PYTHON) $(TEST)/integration/run_tests.py --local --verbose -# remove unwanted files -clean: clean-py clean-docs clean-release +# run unit and CLI tests +tests: unit-tests e2e-tests -release: - $(PYTHON) setup.py sdist bdist_wheel +# run code style checks +check-style: + . $(VENV)/bin/activate + -$(PYTHON) -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics + -$(PYTHON) -m flake8 . --count --max-complexity=15 --statistics --max-line-length=127 + -black --check --target-version py36 $(MRLN) -unit-tests: - -$(PYTHON) -m pytest $(UNIT) +check-push: tests check-style -# run CLI tests -cli-tests: - -$(PYTHON) $(TEST)/integration/run_tests.py --local +# finds all strings in project that begin with a lowercase letter, +# contain only letters and numbers, and contain at least one lowercase +# letter and at least one uppercase letter. +check-camel-case: clean-py + grep -rnw --exclude=lbann_pb2.py $(MRLN) -e "[a-z]\([A-Z0-9]*[a-z][a-z0-9]*[A-Z]\|[a-z0-9]*[A-Z][A-Z0-9]*[a-z]\)[A-Za-z0-9]*" -# run unit and CLI tests -tests: unit-tests cli-tests +# run all checks +checks: check-style check-camel-case # automatically make python files pep 8-compliant @@ -132,31 +124,14 @@ fix-style: black --target-version py36 *.py -# run code style checks -check-style: - -$(PYTHON) -m flake8 --max-complexity $(MAX_COMPLEXITY) --max-line-length $(MAX_LINE_LENGTH) --exclude ascii_art.py $(MRLN) - -black --check --target-version py36 $(MRLN) - - -# finds all strings in project that begin with a lowercase letter, -# contain only letters and numbers, and contain at least one lowercase -# letter and at least one uppercase letter. -check-camel-case: clean-py - grep -rnw --exclude=lbann_pb2.py $(MRLN) -e "[a-z]\([A-Z0-9]*[a-z][a-z0-9]*[A-Z]\|[a-z0-9]*[A-Z][A-Z0-9]*[a-z]\)[A-Za-z0-9]*" - - -# run all checks -checks: check-style check-camel-case - - # Increment the Merlin version. USE ONLY ON DEVELOP BEFORE MERGING TO MASTER. -# Use like this: make VER=?.?.? verison +# Use like this: make VER=?.?.? version version: - # do merlin/__init__.py +# do merlin/__init__.py sed -i 's/__version__ = "$(VSTRING)"/__version__ = "$(VER)"/g' merlin/__init__.py - # do CHANGELOG.md +# do CHANGELOG.md sed -i 's/## \[Unreleased\]/## [$(VER)]/g' CHANGELOG.md - # do all file headers (works on linux) +# do all file headers (works on linux) find merlin/ -type f -print0 | xargs -0 sed -i 's/Version: $(VSTRING)/Version: $(VER)/g' find *.py -type f -print0 | xargs -0 sed -i 's/Version: $(VSTRING)/Version: $(VER)/g' find tests/ -type f -print0 | xargs -0 sed -i 's/Version: $(VSTRING)/Version: $(VER)/g' @@ -165,3 +140,34 @@ version: # Make a list of all dependencies/requirements reqlist: johnnydep merlin --output-format pinned + + +release: + $(PYTHON) setup.py sdist bdist_wheel + + +clean-release: + rm -rf dist + rm -rf build + + +# remove python bytecode files +clean-py: + -find $(MRLN) -name "*.py[cod]" -exec rm -f {} \; + -find $(MRLN) -name "__pycache__" -type d -exec rm -rf {} \; + + +# remove all studies/ directories +clean-output: + -find $(MRLN) -name "studies*" -type d -exec rm -rf {} \; + -find . -maxdepth 1 -name "studies*" -type d -exec rm -rf {} \; + -find . -maxdepth 1 -name "merlin.log" -type f -exec rm -rf {} \; + + +# remove doc build files +clean-docs: + rm -rf $(DOCS)/build + + +# remove unwanted files +clean: clean-py clean-docs clean-release diff --git a/config.mk b/config.mk index 4aa07ef38..38a87a9bf 100644 --- a/config.mk +++ b/config.mk @@ -1,7 +1,7 @@ PYTHON?=python3 PYV=$(shell $(PYTHON) -c "import sys;t='{v[0]}_{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)") PYVD=$(shell $(PYTHON) -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)") -VENV?=venv_merlin_py$(PYV) +VENV?=venv_merlin_py_$(PYV) PIP?=$(VENV)/bin/pip MRLN=merlin TEST=tests diff --git a/docs/source/conf.py b/docs/source/conf.py index 385701da2..5b8c881d0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -43,10 +43,10 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -#extensions = [ +# extensions = [ # 'sphinx.ext.autodoc', # 'sphinx.ext.intersphinx', -#] +# ] extensions = [] # Add any paths that contain templates here, relative to this directory. @@ -104,8 +104,8 @@ html_context = { 'css_files': [ '_static/theme_overrides.css', # override wide tables in RTD theme - ], - } + ], +} # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -183,6 +183,7 @@ intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} + def setup(app): app.add_stylesheet('custom.css') app.add_javascript("custom.js") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 78c8b5406..1be8f9374 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -28,6 +28,8 @@ # SOFTWARE. ############################################################################### +# pylint: skip-file + """ Holds ascii art strings. """ diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index d97042576..a8551da74 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -81,6 +81,7 @@ print a.dtype # dtype of array """ +from typing import List, Tuple import numpy as np @@ -280,16 +281,19 @@ def to_array(self): class OpenNPYList: - def __init__(self, l): - self.filenames = l - self.files = [OpenNPY(_) for _ in self.filenames] + def __init__(self, filename_strs: List[str]): + self.filenames: List[str] = filename_strs + self.files: List[OpenNPY] = [OpenNPY(file_str) for file_str in self.filenames] + i: OpenNPY for i in self.files: i.load_header() - self.shapes = [_.hdr["shape"] for _ in self.files] - for i in self.shapes[1:]: + self.shapes: List[Tuple[int]] = [openNPY_obj.hdr["shape"] for openNPY_obj in self.files] + k: Tuple[int] + for k in self.shapes[1:]: # Match subsequent axes shapes. - assert i[1:] == self.shapes[0][1:] - self.tells = np.cumsum([_[0] for _ in self.shapes]) # Tell locations. + if k[1:] != self.shapes[0][1:]: + raise AttributeError(f"Mismatch in subsequent axes shapes: {k[1:]} != {self.shapes[0][1:]}") + self.tells: np.ndarray = np.cumsum([arr_shape[0] for arr_shape in self.shapes]) # Tell locations. self.tells = np.hstack(([0], self.tells)) def close(self): diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index f55395fcd..b2b196129 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -33,6 +33,7 @@ import logging import os +from typing import Any, Dict, Optional from celery import chain, chord, group, shared_task, signature from celery.exceptions import MaxRetriesExceededError, OperationalError, TimeoutError @@ -70,13 +71,13 @@ STOP_COUNTDOWN = 60 -@shared_task( +@shared_task( # noqa: C901 bind=True, autoretry_for=retry_exceptions, retry_backoff=True, priority=get_priority(Priority.high), ) -def merlin_step(self, *args, **kwargs): +def merlin_step(self, *args: Any, **kwargs: Any) -> Optional[ReturnCode]: # noqa: C901 """ Executes a Merlin Step :param args: The arguments, one of which should be an instance of Step @@ -88,25 +89,27 @@ def merlin_step(self, *args, **kwargs): "next_in_chain": } # merlin_step will be added to the current chord # with next_in_chain as an argument """ - step = None + step: Optional[Step] = None LOG.debug(f"args is {len(args)} long") - for a in args: - if isinstance(a, Step): - step = a + arg: Any + for arg in args: + if isinstance(arg, Step): + step = arg else: - LOG.debug(f"discard argument {a}") + LOG.debug(f"discard argument {arg}, not of type Step.") - config = kwargs.pop("adapter_config", {"type": "local"}) - next_in_chain = kwargs.pop("next_in_chain", None) + config: Dict[str, str] = kwargs.pop("adapter_config", {"type": "local"}) + next_in_chain: Optional[Step] = kwargs.pop("next_in_chain", None) if step: self.max_retries = step.max_retries - step_name = step.name() - step_dir = step.get_workspace() + step_name: str = step.name() + step_dir: str = step.get_workspace() LOG.debug(f"merlin_step: step_name '{step_name}' step_dir '{step_dir}'") - finished_filename = os.path.join(step_dir, "MERLIN_FINISHED") + finished_filename: str = os.path.join(step_dir, "MERLIN_FINISHED") # if we've already finished this task, skip it + result: ReturnCode if os.path.exists(finished_filename): LOG.info(f"Skipping step '{step_name}' in '{step_dir}'.") result = ReturnCode.OK @@ -299,12 +302,12 @@ def add_merlin_expanded_chain_to_chord( new_chain.append(new_step) all_chains.append(new_chain) - LOG.debug(f"adding chain to chord") + LOG.debug("adding chain to chord") add_chains_to_chord(self, all_chains) - LOG.debug(f"chain added to chord") + LOG.debug("chain added to chord") else: # recurse down the sample_index hierarchy - LOG.debug(f"recursing down sample_index hierarchy") + LOG.debug("recursing down sample_index hierarchy") for next_index in sample_index.children.values(): next_index.name = os.path.join(sample_index.name, next_index.name) LOG.debug("generating next step") @@ -486,7 +489,7 @@ def expand_tasks_with_samples( if needs_expansion: # prepare_chain_workspace(sample_index, steps) sample_index.name = "" - LOG.debug(f"queuing merlin expansion tasks") + LOG.debug("queuing merlin expansion tasks") found_tasks = False conditions = [ lambda c: c.is_great_grandparent_of_leaf, @@ -527,9 +530,9 @@ def expand_tasks_with_samples( ) found_tasks = True else: - LOG.debug(f"queuing simple chain task") + LOG.debug("queuing simple chain task") add_simple_chain_to_chord(self, task_type, steps, adapter_config) - LOG.debug(f"simple chain task queued") + LOG.debug("simple chain task queued") @shared_task( @@ -554,7 +557,7 @@ def shutdown_workers(self, shutdown_queues): if shutdown_queues is not None: LOG.warning(f"Shutting down workers in queues {shutdown_queues}!") else: - LOG.warning(f"Shutting down workers in all queues!") + LOG.warning("Shutting down workers in all queues!") return stop_workers("celery", None, shutdown_queues, None) diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 74c804e58..2ec10ee3b 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -36,6 +36,7 @@ import logging import os import ssl +from typing import Dict, List, Optional, Union from merlin.config import Config from merlin.utils import load_yaml @@ -202,7 +203,10 @@ def get_cert_file(server_type, config, cert_name, cert_path): return cert_file -def get_ssl_entries(server_type, server_name, server_config, cert_path): +def get_ssl_entries(server_type: str, + server_name: str, + server_config: Config, + cert_path: str) -> str: """ Check if a ssl certificate file is present in the config @@ -211,17 +215,17 @@ def get_ssl_entries(server_type, server_name, server_config, cert_path): :param server_config : The server config :param cert_path : The optional cert path """ - server_ssl = {} + server_ssl: Dict[str, Union[str, ssl.VerifyMode]] = {} - keyfile = get_cert_file(server_type, server_config, "keyfile", cert_path) + keyfile: Optional[str] = get_cert_file(server_type, server_config, "keyfile", cert_path) if keyfile: server_ssl["keyfile"] = keyfile - certfile = get_cert_file(server_type, server_config, "certfile", cert_path) + certfile: Optional[str] = get_cert_file(server_type, server_config, "certfile", cert_path) if certfile: server_ssl["certfile"] = certfile - ca_certsfile = get_cert_file(server_type, server_config, "ca_certs", cert_path) + ca_certsfile: Optional[str] = get_cert_file(server_type, server_config, "ca_certs", cert_path) if ca_certsfile: server_ssl["ca_certs"] = ca_certsfile @@ -245,8 +249,21 @@ def get_ssl_entries(server_type, server_name, server_config, cert_path): if server_ssl and "cert_reqs" not in server_ssl.keys(): server_ssl["cert_reqs"] = ssl.CERT_REQUIRED - ssl_map = {} + ssl_map: Dict[str, str] = process_ssl_map(server_name) + if server_ssl and ssl_map: + server_ssl = merge_sslmap(server_ssl, ssl_map) + + return server_ssl + + +def process_ssl_map(server_name: str) -> Optional[Dict[str, str]]: + """ + Process a special map for rediss and mysql. + + :param server_name : The server name for output + """ + ssl_map: Dict[str, str] = {} # The redis server requires key names with ssl_ if server_name == "rediss": ssl_map = { @@ -260,18 +277,28 @@ def get_ssl_entries(server_type, server_name, server_config, cert_path): if "mysql" in server_name: ssl_map = {"keyfile": "ssl_key", "certfile": "ssl_cert", "ca_certs": "ssl_ca"} - if server_ssl and ssl_map: - new_server_ssl = {} - sk = server_ssl.keys() - smk = ssl_map.keys() - for k in sk: - if k in smk: - new_server_ssl[ssl_map[k]] = server_ssl[k] - else: - new_server_ssl[k] = server_ssl[k] - server_ssl = new_server_ssl + return ssl_map - return server_ssl + +def merge_sslmap(server_ssl: Dict[str, Union[str, ssl.VerifyMode]], + ssl_map: Dict[str, str]) -> Dict: + """ + The different servers have different key var expectations, this updates the keys of the ssl_server dict with keys from + the ssl_map if using rediss or mysql. + + : param server_ssl : the dict constructed in get_ssl_entries, here updated with keys from ssl_map + : param ssl_map : the dict holding special key:value pairs for rediss and mysql + """ + new_server_ssl: Dict[str, Union[str, ssl.VerifyMode]] = {} + sk: List[str] = server_ssl.keys() + smk: List[str] = ssl_map.keys() + k: str + for k in sk: + if k in smk: + new_server_ssl[ssl_map[k]] = server_ssl[k] + else: + new_server_ssl[k] = server_ssl[k] + server_ssl = new_server_ssl app_config = get_config(None) diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 872f2472b..d26197025 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -1,4 +1,5 @@ import enum +from typing import List from merlin.config.configfile import CONFIG @@ -8,17 +9,20 @@ class Priority(enum.Enum): mid = 2 low = 3 -def is_rabbit_broker(broker): + +def is_rabbit_broker(broker: str) -> bool: return broker in ["rabbitmq", "amqps", "amqp"] -def is_redis_broker(broker): + +def is_redis_broker(broker: str) -> bool: return broker in ["redis", "rediss", "redis+socket"] -def get_priority(priority): - broker = CONFIG.broker.name.lower() - priorities = [Priority.high, Priority.mid, Priority.low] - if priority not in priorities: - raise ValueError( + +def get_priority(priority: Priority) -> int: + broker: str = CONFIG.broker.name.lower() + priorities: List[Priority] = [Priority.high, Priority.mid, Priority.low] + if not isinstance(priority, Priority): + raise TypeError( f"Unrecognized priority '{priority}'! Priority enum options: {[x.name for x in priorities]}" ) if priority == Priority.mid: diff --git a/merlin/examples/workflows/flux/scripts/flux_info.py b/merlin/examples/workflows/flux/scripts/flux_info.py index 04991164d..0383b9970 100755 --- a/merlin/examples/workflows/flux/scripts/flux_info.py +++ b/merlin/examples/workflows/flux/scripts/flux_info.py @@ -23,6 +23,7 @@ import json import os import subprocess +from typing import Dict, Union, IO import flux from flux import kvs @@ -68,24 +69,28 @@ except BaseException: top_dir = "job" - def get_data_dict(key): - kwargs = { + def get_data_dict(key: str) -> Dict: + kwargs: Dict[str, Union[str, bool, os.Environ]] = { "env": os.environ, "shell": True, "universal_newlines": True, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE, } - flux_com = f"flux kvs get {key}" - p = subprocess.Popen(flux_com, **kwargs) + flux_com: str = f"flux kvs get {key}" + p: subprocess.Popen = subprocess.Popen(flux_com, **kwargs) + stdout: IO[str] + stderr: IO[str] stdout, stderr = p.communicate() - data = {} - for l in stdout.split("/n"): - for s in l.strip().split(): - if "timestamp" in s: - jstring = s.replace("'", '"') - d = json.loads(jstring) + data: Dict = {} + line: str + for line in stdout.split("/n"): + token: str + for token in line.strip().split(): + if "timestamp" in token: + jstring: str = token.replace("'", '"') + d: Dict = json.loads(jstring) data[d["name"]] = d["timestamp"] return data diff --git a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py index b801f5764..bb5ba8a4d 100644 --- a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py +++ b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py @@ -3,26 +3,28 @@ import shutil import socket import subprocess +from typing import List -parser = argparse.ArgumentParser(description="Launch 35 merlin workflow jobs") +parser: argparse.ArgumentParser = argparse.ArgumentParser(description="Launch 35 merlin workflow jobs") parser.add_argument("run_id", type=int, help="The ID of this run") parser.add_argument("output_path", type=str, help="the output path") parser.add_argument("spec_path", type=str, help="path to the spec to run") parser.add_argument("script_path", type=str, help="path to the make samples script") -args = parser.parse_args() +args: argparse.Namespace = parser.parse_args() -machine = socket.gethostbyaddr(socket.gethostname())[0] +machine: str = socket.gethostbyaddr(socket.gethostname())[0] if "quartz" in machine: machine = "quartz" elif "pascal" in machine: machine = "pascal" # launch n_samples * n_conc merlin workflow jobs -submit_path = os.path.abspath(os.path.dirname(os.path.abspath(__file__))) -concurrencies = [2 ** 0, 2 ** 1, 2 ** 2, 2 ** 3, 2 ** 4, 2 ** 5, 2 ** 6] -samples = [10 ** 1, 10 ** 2, 10 ** 3, 10 ** 4, 10 ** 5] -nodes = [] +submit_path: str = os.path.abspath(os.path.dirname(os.path.abspath(__file__))) +concurrencies: List[int] = [2 ** 0, 2 ** 1, 2 ** 2, 2 ** 3, 2 ** 4, 2 ** 5, 2 ** 6] +samples: List[int] = [10 ** 1, 10 ** 2, 10 ** 3, 10 ** 4, 10 ** 5] +nodes: List = [] +c: int for c in concurrencies: if c > 32: nodes.append(int(c / 32)) @@ -35,22 +37,27 @@ # concurrencies = [2 ** 3] # samples = [10 ** 5] -output_path = os.path.join(args.output_path, f"run_{args.run_id}") +output_path: str = os.path.join(args.output_path, f"run_{args.run_id}") os.makedirs(output_path, exist_ok=True) -for i, concurrency in enumerate(concurrencies): - c_name = os.path.join(output_path, f"c_{concurrency}") +ii: int +concurrency: int +for ii, concurrency in enumerate(concurrencies): + c_name: str = os.path.join(output_path, f"c_{concurrency}") if not os.path.isdir(c_name): os.mkdir(c_name) os.chdir(c_name) - for j, sample in enumerate(samples): - s_name = os.path.join(c_name, f"s_{sample}") + jj: int + sample: int + for jj, sample in enumerate(samples): + s_name: str = os.path.join(c_name, f"s_{sample}") if not os.path.isdir(s_name): os.mkdir(s_name) os.chdir(s_name) os.mkdir("scripts") - samp_per_worker = float(sample) / float(concurrency) - # if (samp_per_worker / 60) > times[j]: - # print(f"c{concurrency}_s{sample} : {round(samp_per_worker / 60, 0)}m.\ttime: {times[j]}m.\tdiff: {round((samp_per_worker / 60) - times[j], 0)}m") + samp_per_worker: float = float(sample) / float(concurrency) + # if (samp_per_worker / 60) > times[jj]: + # print(f"c{concurrency}_s{sample} : {round(samp_per_worker / 60, 0)}m.\ttime: {times[jj]}m.\tdiff: {round((samp_per_worker / 60) - times[jj], 0)}m") + real_time: int if (samp_per_worker / 60) < 1.0: real_time = 4 elif (samp_per_worker / 60) < 3.0: @@ -70,11 +77,11 @@ partition = "pbatch" if real_time > 1440: real_time = 1440 - submit = "submit.sbatch" - command = f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[i]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[i])} {args.run_id} {concurrency}" + submit: str = "submit.sbatch" + command: str = f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[ii]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[ii])} {args.run_id} {concurrency}" shutil.copyfile(os.path.join(submit_path, submit), submit) shutil.copyfile(args.spec_path, "spec.yaml") shutil.copyfile(args.script_path, os.path.join("scripts", "make_samples.py")) - lines = subprocess.check_output(command, shell=True).decode("ascii") - os.chdir(f"..") - os.chdir(f"..") + lines: str = subprocess.check_output(command, shell=True).decode("ascii") + os.chdir("..") + os.chdir("..") diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 35be16c79..d01617a11 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -124,11 +124,10 @@ def load_merlin_block(stream): merlin_block = yaml.safe_load(stream)["merlin"] except KeyError: merlin_block = {} - LOG.warning( - f"Workflow specification missing \n " - f"encouraged 'merlin' section! Run 'merlin example' for examples.\n" - f"Using default configuration with no sampling." - ) + warning_msg: str = ("Workflow specification missing \n " + "encouraged 'merlin' section! Run 'merlin example' for examples.\n" + "Using default configuration with no sampling.") + LOG.warning(warning_msg) return merlin_block def process_spec_defaults(self): diff --git a/merlin/study/batch.py b/merlin/study/batch.py index fd2e3296c..d4eea47e9 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -37,6 +37,7 @@ """ import logging import os +from typing import Dict, Union, Optional from merlin.utils import get_yaml_var @@ -110,16 +111,19 @@ def get_node_count(default=1): return default -def batch_worker_launch(spec, com, nodes=None, batch=None): +def batch_worker_launch(spec: Dict, + com: str, + nodes: Optional[Union[str, int]] = None, + batch: Optional[Dict] = None) -> str: """ The configuration in the batch section of the merlin spec is used to create the worker launch line, which may be different from a simulation launch. - com (str): The command to launch with batch configuration - nodes (int): The number of nodes to use in the batch launch - batch (dict): An optional batch override from the worker config - + : param spec : (Dict) workflow specification + : param com : (str): The command to launch with batch configuration + : param nodes : (Optional[Union[str, int]]): The number of nodes to use in the batch launch + : param batch : (Optional[Dict]): An optional batch override from the worker config """ if batch is None: try: @@ -128,7 +132,7 @@ def batch_worker_launch(spec, com, nodes=None, batch=None): LOG.error("The batch section is required in the specification file.") raise - btype = get_yaml_var(batch, "type", "local") + btype: str = get_yaml_var(batch, "type", "local") # A jsrun submission cannot be run under a parent jsrun so # all non flux lsf submissions need to be local. @@ -142,61 +146,73 @@ def batch_worker_launch(spec, com, nodes=None, batch=None): # Get the number of nodes from the environment if unset if nodes is None or nodes == "all": nodes = get_node_count(default=1) + elif not isinstance(nodes, int): + raise TypeError("Nodes was passed into batch_worker_launch with an invalid type (likely a string other than 'all').") - bank = get_yaml_var(batch, "bank", "") - queue = get_yaml_var(batch, "queue", "") - shell = get_yaml_var(batch, "shell", "bash") - walltime = get_yaml_var(batch, "walltime", "") + shell: str = get_yaml_var(batch, "shell", "bash") - launch_pre = get_yaml_var(batch, "launch_pre", "") - launch_args = get_yaml_var(batch, "launch_args", "") - worker_launch = get_yaml_var(batch, "worker_launch", "") + launch_pre: str = get_yaml_var(batch, "launch_pre", "") + launch_args: str = get_yaml_var(batch, "launch_args", "") + launch_command: str = get_yaml_var(batch, "worker_launch", "") - if btype == "flux": - launcher = get_batch_type() - else: - launcher = get_batch_type() - - launchs = worker_launch - if not launchs: - if btype == "slurm" or launcher == "slurm": - launchs = f"srun -N {nodes} -n {nodes}" - if bank: - launchs += f" -A {bank}" - if queue: - launchs += f" -p {queue}" - if walltime: - launchs += f" -t {walltime}" - if launcher == "lsf": - # The jsrun utility does not have a time argument - launchs = f"jsrun -a 1 -c ALL_CPUS -g ALL_GPUS --bind=none -n {nodes}" - - launchs += f" {launch_args}" + if not launch_command: + launch_command = construct_worker_launch_command(batch, btype, nodes) + + launch_command += f" {launch_args}" # Allow for any pre launch manipulation, e.g. module load # hwloc/1.11.10-cuda if launch_pre: - launchs = f"{launch_pre} {launchs}" - - worker_cmd = f"{launchs} {com}" + launch_command = f"{launch_pre} {launch_command}" + worker_cmd: str = "" if btype == "flux": - flux_path = get_yaml_var(batch, "flux_path", "") - flux_opts = get_yaml_var(batch, "flux_start_opts", "") - flux_exec_workers = get_yaml_var(batch, "flux_exec_workers", True) + flux_path: str = get_yaml_var(batch, "flux_path", "") + flux_opts: Union[str, Dict] = get_yaml_var(batch, "flux_start_opts", "") + flux_exec_workers: Union[str, Dict, bool] = get_yaml_var(batch, "flux_exec_workers", True) - flux_exec = "" + flux_exec: str = "" if flux_exec_workers: flux_exec = "flux exec" if "/" in flux_path: flux_path += "/" - flux_exe = os.path.join(flux_path, "flux") + flux_exe: str = os.path.join(flux_path, "flux") - launch = ( - f"{launchs} {flux_exe} start {flux_opts} {flux_exec} `which {shell}` -c" + launch: str = ( + f"{launch_command} {flux_exe} start {flux_opts} {flux_exec} `which {shell}` -c" ) worker_cmd = f'{launch} "{com}"' + else: + worker_cmd = f"{launch_command} {com}" return worker_cmd + + +def construct_worker_launch_command(batch: Optional[Dict], btype: str, nodes: int) -> str: + """ + If no 'worker_launch' is found in the batch yaml, this method constructs the needed launch command. + + : param batch : (Optional[Dict]): An optional batch override from the worker config + : param btype : (str): The type of batch (flux, local, lsf) + : param nodes : (int): The number of nodes to use in the batch launch + """ + launch_command: str = "" + workload_manager: str = get_batch_type() + bank: str = get_yaml_var(batch, "bank", "") + queue: str = get_yaml_var(batch, "queue", "") + walltime: str = get_yaml_var(batch, "walltime", "") + if btype == "slurm" or workload_manager == "slurm": + launch_command = f"srun -N {nodes} -n {nodes}" + if bank: + launch_command += f" -A {bank}" + if queue: + launch_command += f" -p {queue}" + if walltime: + launch_command += f" -t {walltime}" + if workload_manager == "lsf": + # The jsrun utility does not have a time argument + launch_command = f"jsrun -a 1 -c ALL_CPUS -g ALL_GPUS --bind=none -n {nodes}" + + return launch_command diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index cd15b4128..7eebd43fb 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -223,24 +223,9 @@ def start_celery_workers(spec, steps, celery_args, just_return_command): local_queues = [] for worker_name, worker_val in workers.items(): - worker_machines = get_yaml_var(worker_val, "machines", None) - if worker_machines: - LOG.debug("check machines = ", check_machines(worker_machines)) - if not check_machines(worker_machines): - continue - - if yenv: - output_path = get_yaml_var(yenv, "OUTPUT_PATH", None) - if output_path and not os.path.exists(output_path): - hostname = socket.gethostname() - LOG.error( - f"The output path, {output_path}, is not accessible on this host, {hostname}" - ) - else: - LOG.warning( - "The env:variables section does not have an OUTPUT_PATH" - "specified, multi-machine checks cannot be performed." - ) + skip_loop_step: bool = examine_and_log_machines(worker_val, yenv) + if skip_loop_step: + continue worker_args = get_yaml_var(worker_val, "args", celery_args) with suppress(KeyError): @@ -260,7 +245,7 @@ def start_celery_workers(spec, steps, celery_args, just_return_command): # Add a per worker log file (debug) if LOG.isEnabledFor(logging.DEBUG): LOG.debug("Redirecting worker output to individual log files") - worker_args += f" --logfile %p.%i" + worker_args += " --logfile %p.%i" # Get the celery command celery_com = launch_celery_workers( @@ -324,6 +309,32 @@ def start_celery_workers(spec, steps, celery_args, just_return_command): return str(worker_list) +def examine_and_log_machines(worker_val, yenv) -> bool: + """ + Examines whether a worker should be skipped in a step of start_celery_workers(), logs errors in output path for a celery + worker. + """ + worker_machines = get_yaml_var(worker_val, "machines", None) + if worker_machines: + LOG.debug("check machines = ", check_machines(worker_machines)) + if not check_machines(worker_machines): + return True + + if yenv: + output_path = get_yaml_var(yenv, "OUTPUT_PATH", None) + if output_path and not os.path.exists(output_path): + hostname = socket.gethostname() + LOG.error( + f"The output path, {output_path}, is not accessible on this host, {hostname}" + ) + else: + LOG.warning( + "The env:variables section does not have an OUTPUT_PATH" + "specified, multi-machine checks cannot be performed." + ) + return False + + def verify_args(spec, worker_args, worker_name, overlap): """Examines the args passed to a worker for completeness.""" parallel = batch_check_parallel(spec) diff --git a/merlin/utils.py b/merlin/utils.py index b06495b21..bb8df74c2 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -386,15 +386,15 @@ def get_flux_version(flux_path, no_errors=False): except FileNotFoundError as e: if not no_errors: LOG.error(f"The flux path {flux_path} canot be found") - LOG.error(f"Suppress this error with no_errors=True") + LOG.error("Suppress this error with no_errors=True") raise e try: flux_ver = re.search(r"\s*([\d.]+)", ps[0]).group(1) except (ValueError, TypeError) as e: if not no_errors: - LOG.error(f"The flux version canot be determined") - LOG.error(f"Suppress this error with no_errors=True") + LOG.error("The flux version cannot be determined") + LOG.error("Suppress this error with no_errors=True") raise e else: flux_ver = DEFAULT_FLUX_VERSION diff --git a/setup.cfg b/setup.cfg index 8161403db..eccb60e46 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,7 @@ ignore = E203, E266, E501, W503 max-line-length = 127 max-complexity = 15 select = B,C,E,F,W,T4 -exclude = .git,__pycache__,ascii_art.py,merlin/examples/* +exclude = .git,__pycache__,ascii_art.py,merlin/examples/*,*venv* [mypy] files=best_practices,test diff --git a/setup.py b/setup.py index c62d698f3..0a338e961 100644 --- a/setup.py +++ b/setup.py @@ -44,8 +44,9 @@ def readme(): # The reqs code from celery setup.py -def _strip_comments(l): - return l.split("#", 1)[0].strip() +def _strip_comments(line: str): + """Removes comments from a line passed in from _reqs().""" + return line.split("#", 1)[0].strip() def _pip_requirement(req): @@ -59,8 +60,8 @@ def _reqs(*f): return [ _pip_requirement(r) for r in ( - _strip_comments(l) - for l in open(os.path.join(os.getcwd(), "requirements", *f)).readlines() + _strip_comments(line) + for line in open(os.path.join(os.getcwd(), "requirements", *f)).readlines() ) if r ] diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 7fd52c7c0..9fa84d7dd 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -106,13 +106,10 @@ def process_test_result(passed, info, is_verbose, exit): print("pass") if info["violated_condition"] is not None: - message = info["violated_condition"][0] - condition_id = info["violated_condition"][1] + 1 - n_conditions = info["violated_condition"][2] - print( - f"\tCondition {condition_id} of {n_conditions}: " - + str(info["violated_condition"][0]) - ) + msg: str = str(info["violated_condition"][0]) + condition_id: str = info["violated_condition"][1] + 1 + n_conditions: str = info["violated_condition"][2] + print(f"\tCondition {condition_id} of {n_conditions}: {msg}") if is_verbose is True: print(f"\tcommand: {info['command']}") print(f"\telapsed time: {round(info['total_time'], 2)} s") diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 996d90273..edddeed59 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -105,7 +105,7 @@ def define_tests(): ), } example_tests = { - "example failure": (f"merlin example failure", HasRegex("not found"), "local"), + "example failure": ("merlin example failure", HasRegex("not found"), "local"), "example simple_chain": ( f"merlin example simple_chain ; {run} simple_chain.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR} ; rm simple_chain.yaml", HasReturnCode(), @@ -217,13 +217,13 @@ def define_tests(): [ HasReturnCode(), ProvenanceYAMLFileHasRegex( - regex="HELLO: \$\(SCRIPTS\)/hello_world.py", + regex=r"HELLO: \$\(SCRIPTS\)/hello_world.py", name="feature_demo", output_path=OUTPUT_DIR, provenance_type="orig", ), ProvenanceYAMLFileHasRegex( - regex="name: \$\(NAME\)", + regex=r"name: \$\(NAME\)", name="feature_demo", output_path=OUTPUT_DIR, provenance_type="partial", @@ -241,7 +241,7 @@ def define_tests(): provenance_type="expanded", ), ProvenanceYAMLFileHasRegex( - regex="\$\(NAME\)", + regex=r"\$\(NAME\)", name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", @@ -287,13 +287,13 @@ def define_tests(): f"{run} {demo} --pgen {demo_pgen} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local", [ ProvenanceYAMLFileHasRegex( - regex="\[0.3333333", + regex=r"\[0.3333333", name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", ), ProvenanceYAMLFileHasRegex( - regex="\[0.5", + regex=r"\[0.5", name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", @@ -304,25 +304,25 @@ def define_tests(): "local", ), } - provenence_equality_checks = { + provenence_equality_checks = { # noqa: F841 "local provenance spec equality": ( f"{run} {simple} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local ; cp $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml') ./{OUTPUT_DIR}/FILE1 ; rm -rf ./{OUTPUT_DIR}/simple_chain_* ; {run} ./{OUTPUT_DIR}/FILE1 --vars OUTPUT_PATH=./{OUTPUT_DIR} --local ; cmp ./{OUTPUT_DIR}/FILE1 $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml')", HasReturnCode(), "local", ), } - style_checks = { + style_checks = { # noqa: F841 "black check merlin": (f"{black} merlin/", HasReturnCode(), "local"), "black check tests": (f"{black} tests/", HasReturnCode(), "local"), } dependency_checks = { "deplic no GNU": ( - f"deplic ./", + "deplic ./", [HasRegex("GNU", negate=True), HasRegex("GPL", negate=True)], "local", ), } - distributed_tests = { + distributed_tests = { # noqa: F841 "run and purge feature_demo": ( f"{run} {demo} ; {purge} {demo} -f", HasReturnCode(), diff --git a/tests/unit/common/test_sample_index.py b/tests/unit/common/test_sample_index.py index ba18db1e8..eddeb1524 100644 --- a/tests/unit/common/test_sample_index.py +++ b/tests/unit/common/test_sample_index.py @@ -1,6 +1,5 @@ import os import shutil -import unittest from contextlib import suppress from merlin.common.sample_index_factory import create_hierarchy, read_hierarchy diff --git a/tests/unit/study/test_study.py b/tests/unit/study/test_study.py index 2b04e1df9..54d531458 100644 --- a/tests/unit/study/test_study.py +++ b/tests/unit/study/test_study.py @@ -6,6 +6,8 @@ import tempfile import unittest +import pytest + from merlin.study.step import Step from merlin.study.study import MerlinStudy @@ -277,27 +279,22 @@ def test_column_label_conflict(self): If there is a common key between Maestro's global.parameters and Merlin's sample/column_labels, an error should be raised. """ - merlin_spec_conflict = os.path.join(self.tmpdir, "basic_ensemble_conflict.yaml") + merlin_spec_conflict: str = os.path.join(self.tmpdir, "basic_ensemble_conflict.yaml") with open(merlin_spec_conflict, "w+") as _file: _file.write(MERLIN_SPEC_CONFLICT) - try: - study_conflict = MerlinStudy(merlin_spec_conflict) - except ValueError: - pass - else: - assert False + # for some reason flake8 doesn't believe variables instantiated inside the try/with context are assigned + with pytest.raises(ValueError): + study_conflict: MerlinStudy = MerlinStudy(merlin_spec_conflict) # noqa: F841 def test_no_env(self): """ A MerlinStudy should be able to support a MerlinSpec that does not contain the optional `env` section. """ - merlin_spec_no_env_filepath = os.path.join( - self.tmpdir, "basic_ensemble_no_env.yaml" - ) + merlin_spec_no_env_filepath: str = os.path.join(self.tmpdir, "basic_ensemble_no_env.yaml") with open(merlin_spec_no_env_filepath, "w+") as _file: _file.write(MERLIN_SPEC_NO_ENV) try: - study_no_env = MerlinStudy(merlin_spec_no_env_filepath) + study_no_env: MerlinStudy = MerlinStudy(merlin_spec_no_env_filepath) # noqa: F841 except Exception as e: - assert False + assert False, f"Encountered unexpected exception, {e}, for viable MerlinSpec without optional 'env' section." From 7cf71f0d1887dde4e319853442b1565b651c4dff Mon Sep 17 00:00:00 2001 From: Alexander Winter <80929291+AlexanderWinterLLNL@users.noreply.github.com> Date: Fri, 3 Sep 2021 15:45:16 -0700 Subject: [PATCH 004/126] Release work: reformatting via Black, version update, minor test change. (#329) Reformatted source using Black to be PEP compliant. Updated version number to 1.8.1. Minor modification to unit testing for test_no_env and test_column_conflict - this removed an unnecessary Flake8 stepover by using PyTest and examining test objects more thoroughly. Co-authored-by: Alexander Cameron Winter --- CHANGELOG.md | 2 +- Makefile | 2 +- merlin/__init__.py | 4 +-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 14 ++++++--- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- .../security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/configfile.py | 26 +++++++++------- merlin/config/results_backend.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- .../workflows/flux/scripts/flux_info.py | 2 +- .../null_spec/scripts/launch_jobs.py | 4 ++- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/specification.py | 10 ++++--- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 30 +++++++++++-------- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/run_tests.py | 2 +- tests/unit/study/test_study.py | 25 ++++++++++++---- 49 files changed, 117 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59370e7b7..d6cef3eed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.8.1] ### Fixed - merlin purge queue name conflict & shell quote escape diff --git a/Makefile b/Makefile index 98122ce23..733cf9c8c 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index 84979cef9..b94a89202 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.8.0" +__version__ = "1.8.1" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 1be8f9374..5b73d4a38 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index 316a32fcd..91ff70f9d 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 783d663bc..b0ab8ba53 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 783d663bc..b0ab8ba53 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index acc58a720..a3becf73a 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 84e9a709a..b1d66645e 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index a8551da74..b44bfead5 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -287,13 +287,19 @@ def __init__(self, filename_strs: List[str]): i: OpenNPY for i in self.files: i.load_header() - self.shapes: List[Tuple[int]] = [openNPY_obj.hdr["shape"] for openNPY_obj in self.files] + self.shapes: List[Tuple[int]] = [ + openNPY_obj.hdr["shape"] for openNPY_obj in self.files + ] k: Tuple[int] for k in self.shapes[1:]: # Match subsequent axes shapes. if k[1:] != self.shapes[0][1:]: - raise AttributeError(f"Mismatch in subsequent axes shapes: {k[1:]} != {self.shapes[0][1:]}") - self.tells: np.ndarray = np.cumsum([arr_shape[0] for arr_shape in self.shapes]) # Tell locations. + raise AttributeError( + f"Mismatch in subsequent axes shapes: {k[1:]} != {self.shapes[0][1:]}" + ) + self.tells: np.ndarray = np.cumsum( + [arr_shape[0] for arr_shape in self.shapes] + ) # Tell locations. self.tells = np.hstack(([0], self.tells)) def close(self): diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index c8b954aa7..b932e4b24 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index e329d4ef6..44a3c6b8c 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 783d663bc..b0ab8ba53 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index abd17f20f..554fd87e4 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index dca8c5562..ec540bf23 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index b2b196129..714818736 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 964ab0b6d..acf06b74b 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index d3f11b336..d3d9fc75f 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index c80a10676..cfd0b366a 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 2ec10ee3b..a08c53553 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -203,10 +203,9 @@ def get_cert_file(server_type, config, cert_name, cert_path): return cert_file -def get_ssl_entries(server_type: str, - server_name: str, - server_config: Config, - cert_path: str) -> str: +def get_ssl_entries( + server_type: str, server_name: str, server_config: Config, cert_path: str +) -> str: """ Check if a ssl certificate file is present in the config @@ -217,15 +216,21 @@ def get_ssl_entries(server_type: str, """ server_ssl: Dict[str, Union[str, ssl.VerifyMode]] = {} - keyfile: Optional[str] = get_cert_file(server_type, server_config, "keyfile", cert_path) + keyfile: Optional[str] = get_cert_file( + server_type, server_config, "keyfile", cert_path + ) if keyfile: server_ssl["keyfile"] = keyfile - certfile: Optional[str] = get_cert_file(server_type, server_config, "certfile", cert_path) + certfile: Optional[str] = get_cert_file( + server_type, server_config, "certfile", cert_path + ) if certfile: server_ssl["certfile"] = certfile - ca_certsfile: Optional[str] = get_cert_file(server_type, server_config, "ca_certs", cert_path) + ca_certsfile: Optional[str] = get_cert_file( + server_type, server_config, "ca_certs", cert_path + ) if ca_certsfile: server_ssl["ca_certs"] = ca_certsfile @@ -280,8 +285,9 @@ def process_ssl_map(server_name: str) -> Optional[Dict[str, str]]: return ssl_map -def merge_sslmap(server_ssl: Dict[str, Union[str, ssl.VerifyMode]], - ssl_map: Dict[str, str]) -> Dict: +def merge_sslmap( + server_ssl: Dict[str, Union[str, ssl.VerifyMode]], ssl_map: Dict[str, str] +) -> Dict: """ The different servers have different key var expectations, this updates the keys of the ssl_server dict with keys from the ssl_map if using rediss or mysql. diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 23c4c39ce..6c3627e30 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 783d663bc..b0ab8ba53 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index d3fe4a3a7..b8f080ab3 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 783d663bc..b0ab8ba53 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index baff4c089..152fe8763 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 71a198002..460f31e55 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/workflows/flux/scripts/flux_info.py b/merlin/examples/workflows/flux/scripts/flux_info.py index 0383b9970..cc3114f04 100755 --- a/merlin/examples/workflows/flux/scripts/flux_info.py +++ b/merlin/examples/workflows/flux/scripts/flux_info.py @@ -23,7 +23,7 @@ import json import os import subprocess -from typing import Dict, Union, IO +from typing import IO, Dict, Union import flux from flux import kvs diff --git a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py index bb5ba8a4d..4b8671885 100644 --- a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py +++ b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py @@ -6,7 +6,9 @@ from typing import List -parser: argparse.ArgumentParser = argparse.ArgumentParser(description="Launch 35 merlin workflow jobs") +parser: argparse.ArgumentParser = argparse.ArgumentParser( + description="Launch 35 merlin workflow jobs" +) parser.add_argument("run_id", type=int, help="The ID of this run") parser.add_argument("output_path", type=str, help="the output path") parser.add_argument("spec_path", type=str, help="path to the spec to run") diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index eb15b1d2e..2481247df 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index ed544e595..7e692765a 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 3712cf6e2..4a159a717 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 4cc1a872e..d906796cd 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index e70284b72..0e852846a 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 783d663bc..b0ab8ba53 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 82260ef55..ea8483682 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 81a9dff7e..50506dfec 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index ca2705fc7..128eb679b 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index d01617a11..59446b2ec 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -124,9 +124,11 @@ def load_merlin_block(stream): merlin_block = yaml.safe_load(stream)["merlin"] except KeyError: merlin_block = {} - warning_msg: str = ("Workflow specification missing \n " - "encouraged 'merlin' section! Run 'merlin example' for examples.\n" - "Using default configuration with no sampling.") + warning_msg: str = ( + "Workflow specification missing \n " + "encouraged 'merlin' section! Run 'merlin example' for examples.\n" + "Using default configuration with no sampling." + ) LOG.warning(warning_msg) return merlin_block diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 783d663bc..b0ab8ba53 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index d4eea47e9..00736c696 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -37,7 +37,7 @@ """ import logging import os -from typing import Dict, Union, Optional +from typing import Dict, Optional, Union from merlin.utils import get_yaml_var @@ -111,10 +111,12 @@ def get_node_count(default=1): return default -def batch_worker_launch(spec: Dict, - com: str, - nodes: Optional[Union[str, int]] = None, - batch: Optional[Dict] = None) -> str: +def batch_worker_launch( + spec: Dict, + com: str, + nodes: Optional[Union[str, int]] = None, + batch: Optional[Dict] = None, +) -> str: """ The configuration in the batch section of the merlin spec is used to create the worker launch line, which may be @@ -147,7 +149,9 @@ def batch_worker_launch(spec: Dict, if nodes is None or nodes == "all": nodes = get_node_count(default=1) elif not isinstance(nodes, int): - raise TypeError("Nodes was passed into batch_worker_launch with an invalid type (likely a string other than 'all').") + raise TypeError( + "Nodes was passed into batch_worker_launch with an invalid type (likely a string other than 'all')." + ) shell: str = get_yaml_var(batch, "shell", "bash") @@ -169,7 +173,9 @@ def batch_worker_launch(spec: Dict, if btype == "flux": flux_path: str = get_yaml_var(batch, "flux_path", "") flux_opts: Union[str, Dict] = get_yaml_var(batch, "flux_start_opts", "") - flux_exec_workers: Union[str, Dict, bool] = get_yaml_var(batch, "flux_exec_workers", True) + flux_exec_workers: Union[str, Dict, bool] = get_yaml_var( + batch, "flux_exec_workers", True + ) flux_exec: str = "" if flux_exec_workers: @@ -180,9 +186,7 @@ def batch_worker_launch(spec: Dict, flux_exe: str = os.path.join(flux_path, "flux") - launch: str = ( - f"{launch_command} {flux_exe} start {flux_opts} {flux_exec} `which {shell}` -c" - ) + launch: str = f"{launch_command} {flux_exe} start {flux_opts} {flux_exec} `which {shell}` -c" worker_cmd = f'{launch} "{com}"' else: worker_cmd = f"{launch_command} {com}" @@ -190,7 +194,9 @@ def batch_worker_launch(spec: Dict, return worker_cmd -def construct_worker_launch_command(batch: Optional[Dict], btype: str, nodes: int) -> str: +def construct_worker_launch_command( + batch: Optional[Dict], btype: str, nodes: int +) -> str: """ If no 'worker_launch' is found in the batch yaml, this method constructs the needed launch command. diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 7eebd43fb..12ce3911f 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 4a124cfb6..22f3d7629 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 4aa7df048..0a8d367ea 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index e6a46c183..bdde0a9b4 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 8b0b2e1b4..ad590d79e 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index bb8df74c2..c1914735c 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index 0a338e961..db5ed9a74 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 9fa84d7dd..a43a3f51e 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/unit/study/test_study.py b/tests/unit/study/test_study.py index 54d531458..126a8f802 100644 --- a/tests/unit/study/test_study.py +++ b/tests/unit/study/test_study.py @@ -150,6 +150,7 @@ """ +# TODO many of these more resemble integration tests than unit tests, may want to review unit tests to make it more granular. def test_get_task_queue_default(): """ Given a steps dictionary that sets the task queue to `test_queue` return @@ -279,22 +280,36 @@ def test_column_label_conflict(self): If there is a common key between Maestro's global.parameters and Merlin's sample/column_labels, an error should be raised. """ - merlin_spec_conflict: str = os.path.join(self.tmpdir, "basic_ensemble_conflict.yaml") + merlin_spec_conflict: str = os.path.join( + self.tmpdir, "basic_ensemble_conflict.yaml" + ) with open(merlin_spec_conflict, "w+") as _file: _file.write(MERLIN_SPEC_CONFLICT) # for some reason flake8 doesn't believe variables instantiated inside the try/with context are assigned with pytest.raises(ValueError): - study_conflict: MerlinStudy = MerlinStudy(merlin_spec_conflict) # noqa: F841 + study_conflict: MerlinStudy = MerlinStudy(merlin_spec_conflict) + assert ( + not study_conflict + ), "study_conflict completed construction without raising a ValueError." + # TODO the pertinent attribute for study_no_env should be examined and asserted to be empty def test_no_env(self): """ A MerlinStudy should be able to support a MerlinSpec that does not contain the optional `env` section. """ - merlin_spec_no_env_filepath: str = os.path.join(self.tmpdir, "basic_ensemble_no_env.yaml") + merlin_spec_no_env_filepath: str = os.path.join( + self.tmpdir, "basic_ensemble_no_env.yaml" + ) with open(merlin_spec_no_env_filepath, "w+") as _file: _file.write(MERLIN_SPEC_NO_ENV) try: - study_no_env: MerlinStudy = MerlinStudy(merlin_spec_no_env_filepath) # noqa: F841 + study_no_env: MerlinStudy = MerlinStudy(merlin_spec_no_env_filepath) + bad_type_err: str = ( + f"study_no_env failed construction, is type {type(study_no_env)}." + ) + assert isinstance(study_no_env, MerlinStudy), bad_type_err except Exception as e: - assert False, f"Encountered unexpected exception, {e}, for viable MerlinSpec without optional 'env' section." + assert ( + False + ), f"Encountered unexpected exception, {e}, for viable MerlinSpec without optional 'env' section." From 22e7f6085503bad62ec7560b4fa9f1bc17e67780 Mon Sep 17 00:00:00 2001 From: Alexander Winter <80929291+AlexanderWinterLLNL@users.noreply.github.com> Date: Fri, 3 Sep 2021 16:55:24 -0700 Subject: [PATCH 005/126] Release 1.8.1 (#330) * Re-added support for all brokers with task priority (#324) * Re-added support for all brokers with task priority * Bugfix for merlin purge (#327) * Bugfix for merlin purge * Address reviewer comments * Refactor of Merlin internals and CI to be flake8 compliant (#326) * Refactored source to be flake8 compliant. Many changes are cosmetic/style focused - to spacing, removing unused vars, using r strings, etc. Notable changes which *could* have changed behavior included pulling out a considerable chunk of batch_worker_launch into a subfunction, construct_worker_launch_command. Added typing to most of the files touched by the flake8 refactor (any that were missed will be caught in the PyLint refactor). * The Makefile was updated in a few ways - generally cleaned up to be a bit neater (so targets are defined approx. as they're encountered/needed in a workflow), a few targets were added (one to check the whole CI pipeline before pushing), the testing targets were updated to activate the venv before running the test suites. Co-authored-by: Alexander Cameron Winter * Release work: reformatting via Black, version update, minor test change. (#329) Reformatted source using Black to be PEP compliant. Updated version number to 1.8.1. Minor modification to unit testing for test_no_env and test_column_conflict - this removed an unnecessary Flake8 stepover by using PyTest and examining test objects more thoroughly. Co-authored-by: Alexander Cameron Winter Co-authored-by: Luc Peterson Co-authored-by: Alexander Cameron Winter --- .github/workflows/push-pr_workflow.yml | 2 +- CHANGELOG.md | 5 + Makefile | 168 +++++++++--------- config.mk | 2 +- docs/source/conf.py | 9 +- merlin/__init__.py | 4 +- merlin/ascii_art.py | 4 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 26 ++- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- .../security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 43 ++--- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/configfile.py | 69 +++++-- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 25 ++- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 4 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- .../workflows/flux/scripts/flux_info.py | 25 +-- .../null_spec/scripts/launch_jobs.py | 49 ++--- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/specification.py | 14 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 112 +++++++----- merlin/study/celeryadapter.py | 53 +++--- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 8 +- setup.cfg | 2 +- setup.py | 11 +- tests/integration/run_tests.py | 13 +- tests/integration/test_definitions.py | 20 +-- tests/unit/common/test_sample_index.py | 1 - tests/unit/study/test_study.py | 34 ++-- 56 files changed, 447 insertions(+), 320 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 739d9eb4a..b1a62e791 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -38,7 +38,7 @@ jobs: # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=15 --statistics --max-line-length=88 + flake8 . --count --max-complexity=15 --statistics --max-line-length=127 - name: Run pytest over unit test suite run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index d56aef56e..d6cef3eed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,12 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.8.1] +### Fixed +- merlin purge queue name conflict & shell quote escape +- task priority support for amqp, amqps, rediss, redis+socket brokers +- Flake8 compliance ## [1.8.0] diff --git a/Makefile b/Makefile index ef6d60c6b..733cf9c8c 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -29,96 +29,88 @@ ############################################################################### include config.mk -.PHONY : all -.PHONY : install-dev .PHONY : virtualenv -.PHONY : install-workflow-deps .PHONY : install-merlin -.PHONY : clean-output -.PHONY : clean-docs -.PHONY : clean-release -.PHONY : clean-py -.PHONY : clean -.PHONY : release +.PHONY : install-workflow-deps +.PHONY : install-merlin-dev .PHONY : unit-tests -.PHONY : cli-tests +.PHONY : e2e-tests .PHONY : tests .PHONY : fix-style .PHONY : check-style +.PHONY : check-push .PHONY : check-camel-case .PHONY : checks .PHONY : reqlist -.PHONY : check-variables - - -all: install-dev install-merlin install-workflow-deps - - -# install requirements -install-dev: virtualenv - $(PIP) install -r requirements/dev.txt - - -check-variables: - - echo MAX_LINE_LENGTH $(MAX_LINE_LENGTH) - +.PHONY : release +.PHONY : clean-release +.PHONY : clean-output +.PHONY : clean-docs +.PHONY : clean-py +.PHONY : clean -# this only works outside the venv +# this only works outside the venv - if run from inside a custom venv, or any target that depends on this, +# you will break your venv. virtualenv: - $(PYTHON) -m venv $(VENV) --prompt $(PENV) --system-site-packages - $(PIP) install --upgrade pip - + $(PYTHON) -m venv $(VENV) --prompt $(PENV) --system-site-packages; \ + $(PIP) install --upgrade pip; \ + $(PIP) install -r requirements/release.txt; \ -install-workflow-deps: - $(PIP) install -r $(WKFW)feature_demo/requirements.txt +# install merlin into the virtual environment +install-merlin: virtualenv + $(PIP) install -e .; \ + merlin config; \ -install-merlin: - $(PIP) install -e . +# install the example workflow to enable integrated testing +install-workflow-deps: virtualenv install-merlin + $(PIP) install -r $(WKFW)feature_demo/requirements.txt; \ -# remove python bytecode files -clean-py: - -find $(MRLN) -name "*.py[cod]" -exec rm -f {} \; - -find $(MRLN) -name "__pycache__" -type d -exec rm -rf {} \; - +# install requirements +install-merlin-dev: virtualenv install-workflow-deps + $(PIP) install -r requirements/dev.txt; \ -# remove all studies/ directories -clean-output: - -find $(MRLN) -name "studies*" -type d -exec rm -rf {} \; - -find . -maxdepth 1 -name "studies*" -type d -exec rm -rf {} \; - -find . -maxdepth 1 -name "merlin.log" -type f -exec rm -rf {} \; +# tests require a valid dev install of merlin +unit-tests: + . $(VENV)/bin/activate; \ + $(PYTHON) -m pytest $(UNIT); \ -# remove doc build files -clean-docs: - rm -rf $(DOCS)/build +# run CLI tests - these require an active install of merlin in a venv +e2e-tests: + . $(VENV)/bin/activate; \ + $(PYTHON) $(TEST)/integration/run_tests.py --local; \ -clean-release: - rm -rf dist - rm -rf build +e2e-tests-diagnostic: + $(PYTHON) $(TEST)/integration/run_tests.py --local --verbose -# remove unwanted files -clean: clean-py clean-docs clean-release +# run unit and CLI tests +tests: unit-tests e2e-tests -release: - $(PYTHON) setup.py sdist bdist_wheel +# run code style checks +check-style: + . $(VENV)/bin/activate + -$(PYTHON) -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics + -$(PYTHON) -m flake8 . --count --max-complexity=15 --statistics --max-line-length=127 + -black --check --target-version py36 $(MRLN) -unit-tests: - -$(PYTHON) -m pytest $(UNIT) +check-push: tests check-style -# run CLI tests -cli-tests: - -$(PYTHON) $(TEST)/integration/run_tests.py --local +# finds all strings in project that begin with a lowercase letter, +# contain only letters and numbers, and contain at least one lowercase +# letter and at least one uppercase letter. +check-camel-case: clean-py + grep -rnw --exclude=lbann_pb2.py $(MRLN) -e "[a-z]\([A-Z0-9]*[a-z][a-z0-9]*[A-Z]\|[a-z0-9]*[A-Z][A-Z0-9]*[a-z]\)[A-Za-z0-9]*" -# run unit and CLI tests -tests: unit-tests cli-tests +# run all checks +checks: check-style check-camel-case # automatically make python files pep 8-compliant @@ -132,31 +124,14 @@ fix-style: black --target-version py36 *.py -# run code style checks -check-style: - -$(PYTHON) -m flake8 --max-complexity $(MAX_COMPLEXITY) --max-line-length $(MAX_LINE_LENGTH) --exclude ascii_art.py $(MRLN) - -black --check --target-version py36 $(MRLN) - - -# finds all strings in project that begin with a lowercase letter, -# contain only letters and numbers, and contain at least one lowercase -# letter and at least one uppercase letter. -check-camel-case: clean-py - grep -rnw --exclude=lbann_pb2.py $(MRLN) -e "[a-z]\([A-Z0-9]*[a-z][a-z0-9]*[A-Z]\|[a-z0-9]*[A-Z][A-Z0-9]*[a-z]\)[A-Za-z0-9]*" - - -# run all checks -checks: check-style check-camel-case - - # Increment the Merlin version. USE ONLY ON DEVELOP BEFORE MERGING TO MASTER. -# Use like this: make VER=?.?.? verison +# Use like this: make VER=?.?.? version version: - # do merlin/__init__.py +# do merlin/__init__.py sed -i 's/__version__ = "$(VSTRING)"/__version__ = "$(VER)"/g' merlin/__init__.py - # do CHANGELOG.md +# do CHANGELOG.md sed -i 's/## \[Unreleased\]/## [$(VER)]/g' CHANGELOG.md - # do all file headers (works on linux) +# do all file headers (works on linux) find merlin/ -type f -print0 | xargs -0 sed -i 's/Version: $(VSTRING)/Version: $(VER)/g' find *.py -type f -print0 | xargs -0 sed -i 's/Version: $(VSTRING)/Version: $(VER)/g' find tests/ -type f -print0 | xargs -0 sed -i 's/Version: $(VSTRING)/Version: $(VER)/g' @@ -165,3 +140,34 @@ version: # Make a list of all dependencies/requirements reqlist: johnnydep merlin --output-format pinned + + +release: + $(PYTHON) setup.py sdist bdist_wheel + + +clean-release: + rm -rf dist + rm -rf build + + +# remove python bytecode files +clean-py: + -find $(MRLN) -name "*.py[cod]" -exec rm -f {} \; + -find $(MRLN) -name "__pycache__" -type d -exec rm -rf {} \; + + +# remove all studies/ directories +clean-output: + -find $(MRLN) -name "studies*" -type d -exec rm -rf {} \; + -find . -maxdepth 1 -name "studies*" -type d -exec rm -rf {} \; + -find . -maxdepth 1 -name "merlin.log" -type f -exec rm -rf {} \; + + +# remove doc build files +clean-docs: + rm -rf $(DOCS)/build + + +# remove unwanted files +clean: clean-py clean-docs clean-release diff --git a/config.mk b/config.mk index 4aa07ef38..38a87a9bf 100644 --- a/config.mk +++ b/config.mk @@ -1,7 +1,7 @@ PYTHON?=python3 PYV=$(shell $(PYTHON) -c "import sys;t='{v[0]}_{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)") PYVD=$(shell $(PYTHON) -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)") -VENV?=venv_merlin_py$(PYV) +VENV?=venv_merlin_py_$(PYV) PIP?=$(VENV)/bin/pip MRLN=merlin TEST=tests diff --git a/docs/source/conf.py b/docs/source/conf.py index 385701da2..5b8c881d0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -43,10 +43,10 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -#extensions = [ +# extensions = [ # 'sphinx.ext.autodoc', # 'sphinx.ext.intersphinx', -#] +# ] extensions = [] # Add any paths that contain templates here, relative to this directory. @@ -104,8 +104,8 @@ html_context = { 'css_files': [ '_static/theme_overrides.css', # override wide tables in RTD theme - ], - } + ], +} # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -183,6 +183,7 @@ intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} + def setup(app): app.add_stylesheet('custom.css') app.add_javascript("custom.js") diff --git a/merlin/__init__.py b/merlin/__init__.py index 84979cef9..b94a89202 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.8.0" +__version__ = "1.8.1" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 78c8b5406..5b73d4a38 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -28,6 +28,8 @@ # SOFTWARE. ############################################################################### +# pylint: skip-file + """ Holds ascii art strings. """ diff --git a/merlin/celery.py b/merlin/celery.py index 316a32fcd..91ff70f9d 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 783d663bc..b0ab8ba53 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 783d663bc..b0ab8ba53 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index acc58a720..a3becf73a 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 84e9a709a..b1d66645e 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index d97042576..b44bfead5 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -81,6 +81,7 @@ print a.dtype # dtype of array """ +from typing import List, Tuple import numpy as np @@ -280,16 +281,25 @@ def to_array(self): class OpenNPYList: - def __init__(self, l): - self.filenames = l - self.files = [OpenNPY(_) for _ in self.filenames] + def __init__(self, filename_strs: List[str]): + self.filenames: List[str] = filename_strs + self.files: List[OpenNPY] = [OpenNPY(file_str) for file_str in self.filenames] + i: OpenNPY for i in self.files: i.load_header() - self.shapes = [_.hdr["shape"] for _ in self.files] - for i in self.shapes[1:]: + self.shapes: List[Tuple[int]] = [ + openNPY_obj.hdr["shape"] for openNPY_obj in self.files + ] + k: Tuple[int] + for k in self.shapes[1:]: # Match subsequent axes shapes. - assert i[1:] == self.shapes[0][1:] - self.tells = np.cumsum([_[0] for _ in self.shapes]) # Tell locations. + if k[1:] != self.shapes[0][1:]: + raise AttributeError( + f"Mismatch in subsequent axes shapes: {k[1:]} != {self.shapes[0][1:]}" + ) + self.tells: np.ndarray = np.cumsum( + [arr_shape[0] for arr_shape in self.shapes] + ) # Tell locations. self.tells = np.hstack(([0], self.tells)) def close(self): diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index c8b954aa7..b932e4b24 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index e329d4ef6..44a3c6b8c 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 783d663bc..b0ab8ba53 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index abd17f20f..554fd87e4 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index dca8c5562..ec540bf23 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index f55395fcd..714818736 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -33,6 +33,7 @@ import logging import os +from typing import Any, Dict, Optional from celery import chain, chord, group, shared_task, signature from celery.exceptions import MaxRetriesExceededError, OperationalError, TimeoutError @@ -70,13 +71,13 @@ STOP_COUNTDOWN = 60 -@shared_task( +@shared_task( # noqa: C901 bind=True, autoretry_for=retry_exceptions, retry_backoff=True, priority=get_priority(Priority.high), ) -def merlin_step(self, *args, **kwargs): +def merlin_step(self, *args: Any, **kwargs: Any) -> Optional[ReturnCode]: # noqa: C901 """ Executes a Merlin Step :param args: The arguments, one of which should be an instance of Step @@ -88,25 +89,27 @@ def merlin_step(self, *args, **kwargs): "next_in_chain": } # merlin_step will be added to the current chord # with next_in_chain as an argument """ - step = None + step: Optional[Step] = None LOG.debug(f"args is {len(args)} long") - for a in args: - if isinstance(a, Step): - step = a + arg: Any + for arg in args: + if isinstance(arg, Step): + step = arg else: - LOG.debug(f"discard argument {a}") + LOG.debug(f"discard argument {arg}, not of type Step.") - config = kwargs.pop("adapter_config", {"type": "local"}) - next_in_chain = kwargs.pop("next_in_chain", None) + config: Dict[str, str] = kwargs.pop("adapter_config", {"type": "local"}) + next_in_chain: Optional[Step] = kwargs.pop("next_in_chain", None) if step: self.max_retries = step.max_retries - step_name = step.name() - step_dir = step.get_workspace() + step_name: str = step.name() + step_dir: str = step.get_workspace() LOG.debug(f"merlin_step: step_name '{step_name}' step_dir '{step_dir}'") - finished_filename = os.path.join(step_dir, "MERLIN_FINISHED") + finished_filename: str = os.path.join(step_dir, "MERLIN_FINISHED") # if we've already finished this task, skip it + result: ReturnCode if os.path.exists(finished_filename): LOG.info(f"Skipping step '{step_name}' in '{step_dir}'.") result = ReturnCode.OK @@ -299,12 +302,12 @@ def add_merlin_expanded_chain_to_chord( new_chain.append(new_step) all_chains.append(new_chain) - LOG.debug(f"adding chain to chord") + LOG.debug("adding chain to chord") add_chains_to_chord(self, all_chains) - LOG.debug(f"chain added to chord") + LOG.debug("chain added to chord") else: # recurse down the sample_index hierarchy - LOG.debug(f"recursing down sample_index hierarchy") + LOG.debug("recursing down sample_index hierarchy") for next_index in sample_index.children.values(): next_index.name = os.path.join(sample_index.name, next_index.name) LOG.debug("generating next step") @@ -486,7 +489,7 @@ def expand_tasks_with_samples( if needs_expansion: # prepare_chain_workspace(sample_index, steps) sample_index.name = "" - LOG.debug(f"queuing merlin expansion tasks") + LOG.debug("queuing merlin expansion tasks") found_tasks = False conditions = [ lambda c: c.is_great_grandparent_of_leaf, @@ -527,9 +530,9 @@ def expand_tasks_with_samples( ) found_tasks = True else: - LOG.debug(f"queuing simple chain task") + LOG.debug("queuing simple chain task") add_simple_chain_to_chord(self, task_type, steps, adapter_config) - LOG.debug(f"simple chain task queued") + LOG.debug("simple chain task queued") @shared_task( @@ -554,7 +557,7 @@ def shutdown_workers(self, shutdown_queues): if shutdown_queues is not None: LOG.warning(f"Shutting down workers in queues {shutdown_queues}!") else: - LOG.warning(f"Shutting down workers in all queues!") + LOG.warning("Shutting down workers in all queues!") return stop_workers("celery", None, shutdown_queues, None) diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 964ab0b6d..acf06b74b 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index d3f11b336..d3d9fc75f 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index c80a10676..cfd0b366a 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 74c804e58..a08c53553 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -36,6 +36,7 @@ import logging import os import ssl +from typing import Dict, List, Optional, Union from merlin.config import Config from merlin.utils import load_yaml @@ -202,7 +203,9 @@ def get_cert_file(server_type, config, cert_name, cert_path): return cert_file -def get_ssl_entries(server_type, server_name, server_config, cert_path): +def get_ssl_entries( + server_type: str, server_name: str, server_config: Config, cert_path: str +) -> str: """ Check if a ssl certificate file is present in the config @@ -211,17 +214,23 @@ def get_ssl_entries(server_type, server_name, server_config, cert_path): :param server_config : The server config :param cert_path : The optional cert path """ - server_ssl = {} + server_ssl: Dict[str, Union[str, ssl.VerifyMode]] = {} - keyfile = get_cert_file(server_type, server_config, "keyfile", cert_path) + keyfile: Optional[str] = get_cert_file( + server_type, server_config, "keyfile", cert_path + ) if keyfile: server_ssl["keyfile"] = keyfile - certfile = get_cert_file(server_type, server_config, "certfile", cert_path) + certfile: Optional[str] = get_cert_file( + server_type, server_config, "certfile", cert_path + ) if certfile: server_ssl["certfile"] = certfile - ca_certsfile = get_cert_file(server_type, server_config, "ca_certs", cert_path) + ca_certsfile: Optional[str] = get_cert_file( + server_type, server_config, "ca_certs", cert_path + ) if ca_certsfile: server_ssl["ca_certs"] = ca_certsfile @@ -245,8 +254,21 @@ def get_ssl_entries(server_type, server_name, server_config, cert_path): if server_ssl and "cert_reqs" not in server_ssl.keys(): server_ssl["cert_reqs"] = ssl.CERT_REQUIRED - ssl_map = {} + ssl_map: Dict[str, str] = process_ssl_map(server_name) + if server_ssl and ssl_map: + server_ssl = merge_sslmap(server_ssl, ssl_map) + + return server_ssl + + +def process_ssl_map(server_name: str) -> Optional[Dict[str, str]]: + """ + Process a special map for rediss and mysql. + + :param server_name : The server name for output + """ + ssl_map: Dict[str, str] = {} # The redis server requires key names with ssl_ if server_name == "rediss": ssl_map = { @@ -260,18 +282,29 @@ def get_ssl_entries(server_type, server_name, server_config, cert_path): if "mysql" in server_name: ssl_map = {"keyfile": "ssl_key", "certfile": "ssl_cert", "ca_certs": "ssl_ca"} - if server_ssl and ssl_map: - new_server_ssl = {} - sk = server_ssl.keys() - smk = ssl_map.keys() - for k in sk: - if k in smk: - new_server_ssl[ssl_map[k]] = server_ssl[k] - else: - new_server_ssl[k] = server_ssl[k] - server_ssl = new_server_ssl + return ssl_map - return server_ssl + +def merge_sslmap( + server_ssl: Dict[str, Union[str, ssl.VerifyMode]], ssl_map: Dict[str, str] +) -> Dict: + """ + The different servers have different key var expectations, this updates the keys of the ssl_server dict with keys from + the ssl_map if using rediss or mysql. + + : param server_ssl : the dict constructed in get_ssl_entries, here updated with keys from ssl_map + : param ssl_map : the dict holding special key:value pairs for rediss and mysql + """ + new_server_ssl: Dict[str, Union[str, ssl.VerifyMode]] = {} + sk: List[str] = server_ssl.keys() + smk: List[str] = ssl_map.keys() + k: str + for k in sk: + if k in smk: + new_server_ssl[ssl_map[k]] = server_ssl[k] + else: + new_server_ssl[k] = server_ssl[k] + server_ssl = new_server_ssl app_config = get_config(None) diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 23c4c39ce..6c3627e30 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 9c4c95027..d26197025 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -1,4 +1,5 @@ import enum +from typing import List from merlin.config.configfile import CONFIG @@ -9,25 +10,33 @@ class Priority(enum.Enum): low = 3 -def get_priority(priority): - broker = CONFIG.broker.name.lower() - priorities = [Priority.high, Priority.mid, Priority.low] - if priority not in priorities: - raise ValueError( +def is_rabbit_broker(broker: str) -> bool: + return broker in ["rabbitmq", "amqps", "amqp"] + + +def is_redis_broker(broker: str) -> bool: + return broker in ["redis", "rediss", "redis+socket"] + + +def get_priority(priority: Priority) -> int: + broker: str = CONFIG.broker.name.lower() + priorities: List[Priority] = [Priority.high, Priority.mid, Priority.low] + if not isinstance(priority, Priority): + raise TypeError( f"Unrecognized priority '{priority}'! Priority enum options: {[x.name for x in priorities]}" ) if priority == Priority.mid: return 5 - if broker == "rabbitmq": + if is_rabbit_broker(broker): if priority == Priority.low: return 1 if priority == Priority.high: return 10 - if broker == "redis": + if is_redis_broker(broker): if priority == Priority.low: return 10 if priority == Priority.high: return 1 raise ValueError( - "Function get_priority has reached unknown state! Check input parameter and broker." + f"Function get_priority has reached unknown state! Maybe unsupported broker {broker}?" ) diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 783d663bc..b0ab8ba53 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index 67b0534b5..b8f080ab3 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -190,5 +190,5 @@ def print_info(args): info_str += 'echo " $ ' + x + '" && ' + x + "\n" info_str += "echo \n" info_str += r"echo \"echo \$PYTHONPATH\" && echo $PYTHONPATH" - subprocess.call(info_str, shell=True) + _ = subprocess.run(info_str, shell=True) print("") diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 783d663bc..b0ab8ba53 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index baff4c089..152fe8763 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 71a198002..460f31e55 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/workflows/flux/scripts/flux_info.py b/merlin/examples/workflows/flux/scripts/flux_info.py index 04991164d..cc3114f04 100755 --- a/merlin/examples/workflows/flux/scripts/flux_info.py +++ b/merlin/examples/workflows/flux/scripts/flux_info.py @@ -23,6 +23,7 @@ import json import os import subprocess +from typing import IO, Dict, Union import flux from flux import kvs @@ -68,24 +69,28 @@ except BaseException: top_dir = "job" - def get_data_dict(key): - kwargs = { + def get_data_dict(key: str) -> Dict: + kwargs: Dict[str, Union[str, bool, os.Environ]] = { "env": os.environ, "shell": True, "universal_newlines": True, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE, } - flux_com = f"flux kvs get {key}" - p = subprocess.Popen(flux_com, **kwargs) + flux_com: str = f"flux kvs get {key}" + p: subprocess.Popen = subprocess.Popen(flux_com, **kwargs) + stdout: IO[str] + stderr: IO[str] stdout, stderr = p.communicate() - data = {} - for l in stdout.split("/n"): - for s in l.strip().split(): - if "timestamp" in s: - jstring = s.replace("'", '"') - d = json.loads(jstring) + data: Dict = {} + line: str + for line in stdout.split("/n"): + token: str + for token in line.strip().split(): + if "timestamp" in token: + jstring: str = token.replace("'", '"') + d: Dict = json.loads(jstring) data[d["name"]] = d["timestamp"] return data diff --git a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py index b801f5764..4b8671885 100644 --- a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py +++ b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py @@ -3,26 +3,30 @@ import shutil import socket import subprocess +from typing import List -parser = argparse.ArgumentParser(description="Launch 35 merlin workflow jobs") +parser: argparse.ArgumentParser = argparse.ArgumentParser( + description="Launch 35 merlin workflow jobs" +) parser.add_argument("run_id", type=int, help="The ID of this run") parser.add_argument("output_path", type=str, help="the output path") parser.add_argument("spec_path", type=str, help="path to the spec to run") parser.add_argument("script_path", type=str, help="path to the make samples script") -args = parser.parse_args() +args: argparse.Namespace = parser.parse_args() -machine = socket.gethostbyaddr(socket.gethostname())[0] +machine: str = socket.gethostbyaddr(socket.gethostname())[0] if "quartz" in machine: machine = "quartz" elif "pascal" in machine: machine = "pascal" # launch n_samples * n_conc merlin workflow jobs -submit_path = os.path.abspath(os.path.dirname(os.path.abspath(__file__))) -concurrencies = [2 ** 0, 2 ** 1, 2 ** 2, 2 ** 3, 2 ** 4, 2 ** 5, 2 ** 6] -samples = [10 ** 1, 10 ** 2, 10 ** 3, 10 ** 4, 10 ** 5] -nodes = [] +submit_path: str = os.path.abspath(os.path.dirname(os.path.abspath(__file__))) +concurrencies: List[int] = [2 ** 0, 2 ** 1, 2 ** 2, 2 ** 3, 2 ** 4, 2 ** 5, 2 ** 6] +samples: List[int] = [10 ** 1, 10 ** 2, 10 ** 3, 10 ** 4, 10 ** 5] +nodes: List = [] +c: int for c in concurrencies: if c > 32: nodes.append(int(c / 32)) @@ -35,22 +39,27 @@ # concurrencies = [2 ** 3] # samples = [10 ** 5] -output_path = os.path.join(args.output_path, f"run_{args.run_id}") +output_path: str = os.path.join(args.output_path, f"run_{args.run_id}") os.makedirs(output_path, exist_ok=True) -for i, concurrency in enumerate(concurrencies): - c_name = os.path.join(output_path, f"c_{concurrency}") +ii: int +concurrency: int +for ii, concurrency in enumerate(concurrencies): + c_name: str = os.path.join(output_path, f"c_{concurrency}") if not os.path.isdir(c_name): os.mkdir(c_name) os.chdir(c_name) - for j, sample in enumerate(samples): - s_name = os.path.join(c_name, f"s_{sample}") + jj: int + sample: int + for jj, sample in enumerate(samples): + s_name: str = os.path.join(c_name, f"s_{sample}") if not os.path.isdir(s_name): os.mkdir(s_name) os.chdir(s_name) os.mkdir("scripts") - samp_per_worker = float(sample) / float(concurrency) - # if (samp_per_worker / 60) > times[j]: - # print(f"c{concurrency}_s{sample} : {round(samp_per_worker / 60, 0)}m.\ttime: {times[j]}m.\tdiff: {round((samp_per_worker / 60) - times[j], 0)}m") + samp_per_worker: float = float(sample) / float(concurrency) + # if (samp_per_worker / 60) > times[jj]: + # print(f"c{concurrency}_s{sample} : {round(samp_per_worker / 60, 0)}m.\ttime: {times[jj]}m.\tdiff: {round((samp_per_worker / 60) - times[jj], 0)}m") + real_time: int if (samp_per_worker / 60) < 1.0: real_time = 4 elif (samp_per_worker / 60) < 3.0: @@ -70,11 +79,11 @@ partition = "pbatch" if real_time > 1440: real_time = 1440 - submit = "submit.sbatch" - command = f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[i]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[i])} {args.run_id} {concurrency}" + submit: str = "submit.sbatch" + command: str = f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[ii]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[ii])} {args.run_id} {concurrency}" shutil.copyfile(os.path.join(submit_path, submit), submit) shutil.copyfile(args.spec_path, "spec.yaml") shutil.copyfile(args.script_path, os.path.join("scripts", "make_samples.py")) - lines = subprocess.check_output(command, shell=True).decode("ascii") - os.chdir(f"..") - os.chdir(f"..") + lines: str = subprocess.check_output(command, shell=True).decode("ascii") + os.chdir("..") + os.chdir("..") diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index eb15b1d2e..2481247df 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index ed544e595..7e692765a 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 3712cf6e2..4a159a717 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 4cc1a872e..d906796cd 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index e70284b72..0e852846a 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 783d663bc..b0ab8ba53 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 82260ef55..ea8483682 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 81a9dff7e..50506dfec 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index ca2705fc7..128eb679b 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index e9e324377..59446b2ec 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -35,6 +35,7 @@ """ import logging import os +import shlex from io import StringIO import yaml @@ -123,11 +124,12 @@ def load_merlin_block(stream): merlin_block = yaml.safe_load(stream)["merlin"] except KeyError: merlin_block = {} - LOG.warning( - f"Workflow specification missing \n " - f"encouraged 'merlin' section! Run 'merlin example' for examples.\n" - f"Using default configuration with no sampling." + warning_msg: str = ( + "Workflow specification missing \n " + "encouraged 'merlin' section! Run 'merlin example' for examples.\n" + "Using default configuration with no sampling." ) + LOG.warning(warning_msg) return merlin_block def process_spec_defaults(self): @@ -368,7 +370,7 @@ def make_queue_string(self, steps): param steps: a list of step names """ queues = ",".join(set(self.get_queue_list(steps))) - return f'"{queues}"' + return shlex.quote(queues) def get_worker_names(self): result = [] diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 783d663bc..b0ab8ba53 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index fd2e3296c..00736c696 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -37,6 +37,7 @@ """ import logging import os +from typing import Dict, Optional, Union from merlin.utils import get_yaml_var @@ -110,16 +111,21 @@ def get_node_count(default=1): return default -def batch_worker_launch(spec, com, nodes=None, batch=None): +def batch_worker_launch( + spec: Dict, + com: str, + nodes: Optional[Union[str, int]] = None, + batch: Optional[Dict] = None, +) -> str: """ The configuration in the batch section of the merlin spec is used to create the worker launch line, which may be different from a simulation launch. - com (str): The command to launch with batch configuration - nodes (int): The number of nodes to use in the batch launch - batch (dict): An optional batch override from the worker config - + : param spec : (Dict) workflow specification + : param com : (str): The command to launch with batch configuration + : param nodes : (Optional[Union[str, int]]): The number of nodes to use in the batch launch + : param batch : (Optional[Dict]): An optional batch override from the worker config """ if batch is None: try: @@ -128,7 +134,7 @@ def batch_worker_launch(spec, com, nodes=None, batch=None): LOG.error("The batch section is required in the specification file.") raise - btype = get_yaml_var(batch, "type", "local") + btype: str = get_yaml_var(batch, "type", "local") # A jsrun submission cannot be run under a parent jsrun so # all non flux lsf submissions need to be local. @@ -142,61 +148,77 @@ def batch_worker_launch(spec, com, nodes=None, batch=None): # Get the number of nodes from the environment if unset if nodes is None or nodes == "all": nodes = get_node_count(default=1) + elif not isinstance(nodes, int): + raise TypeError( + "Nodes was passed into batch_worker_launch with an invalid type (likely a string other than 'all')." + ) - bank = get_yaml_var(batch, "bank", "") - queue = get_yaml_var(batch, "queue", "") - shell = get_yaml_var(batch, "shell", "bash") - walltime = get_yaml_var(batch, "walltime", "") + shell: str = get_yaml_var(batch, "shell", "bash") - launch_pre = get_yaml_var(batch, "launch_pre", "") - launch_args = get_yaml_var(batch, "launch_args", "") - worker_launch = get_yaml_var(batch, "worker_launch", "") + launch_pre: str = get_yaml_var(batch, "launch_pre", "") + launch_args: str = get_yaml_var(batch, "launch_args", "") + launch_command: str = get_yaml_var(batch, "worker_launch", "") - if btype == "flux": - launcher = get_batch_type() - else: - launcher = get_batch_type() - - launchs = worker_launch - if not launchs: - if btype == "slurm" or launcher == "slurm": - launchs = f"srun -N {nodes} -n {nodes}" - if bank: - launchs += f" -A {bank}" - if queue: - launchs += f" -p {queue}" - if walltime: - launchs += f" -t {walltime}" - if launcher == "lsf": - # The jsrun utility does not have a time argument - launchs = f"jsrun -a 1 -c ALL_CPUS -g ALL_GPUS --bind=none -n {nodes}" - - launchs += f" {launch_args}" + if not launch_command: + launch_command = construct_worker_launch_command(batch, btype, nodes) + + launch_command += f" {launch_args}" # Allow for any pre launch manipulation, e.g. module load # hwloc/1.11.10-cuda if launch_pre: - launchs = f"{launch_pre} {launchs}" - - worker_cmd = f"{launchs} {com}" + launch_command = f"{launch_pre} {launch_command}" + worker_cmd: str = "" if btype == "flux": - flux_path = get_yaml_var(batch, "flux_path", "") - flux_opts = get_yaml_var(batch, "flux_start_opts", "") - flux_exec_workers = get_yaml_var(batch, "flux_exec_workers", True) + flux_path: str = get_yaml_var(batch, "flux_path", "") + flux_opts: Union[str, Dict] = get_yaml_var(batch, "flux_start_opts", "") + flux_exec_workers: Union[str, Dict, bool] = get_yaml_var( + batch, "flux_exec_workers", True + ) - flux_exec = "" + flux_exec: str = "" if flux_exec_workers: flux_exec = "flux exec" if "/" in flux_path: flux_path += "/" - flux_exe = os.path.join(flux_path, "flux") + flux_exe: str = os.path.join(flux_path, "flux") - launch = ( - f"{launchs} {flux_exe} start {flux_opts} {flux_exec} `which {shell}` -c" - ) + launch: str = f"{launch_command} {flux_exe} start {flux_opts} {flux_exec} `which {shell}` -c" worker_cmd = f'{launch} "{com}"' + else: + worker_cmd = f"{launch_command} {com}" return worker_cmd + + +def construct_worker_launch_command( + batch: Optional[Dict], btype: str, nodes: int +) -> str: + """ + If no 'worker_launch' is found in the batch yaml, this method constructs the needed launch command. + + : param batch : (Optional[Dict]): An optional batch override from the worker config + : param btype : (str): The type of batch (flux, local, lsf) + : param nodes : (int): The number of nodes to use in the batch launch + """ + launch_command: str = "" + workload_manager: str = get_batch_type() + bank: str = get_yaml_var(batch, "bank", "") + queue: str = get_yaml_var(batch, "queue", "") + walltime: str = get_yaml_var(batch, "walltime", "") + if btype == "slurm" or workload_manager == "slurm": + launch_command = f"srun -N {nodes} -n {nodes}" + if bank: + launch_command += f" -A {bank}" + if queue: + launch_command += f" -p {queue}" + if walltime: + launch_command += f" -t {walltime}" + if workload_manager == "lsf": + # The jsrun utility does not have a time argument + launch_command = f"jsrun -a 1 -c ALL_CPUS -g ALL_GPUS --bind=none -n {nodes}" + + return launch_command diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 86a19a875..12ce3911f 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -223,24 +223,9 @@ def start_celery_workers(spec, steps, celery_args, just_return_command): local_queues = [] for worker_name, worker_val in workers.items(): - worker_machines = get_yaml_var(worker_val, "machines", None) - if worker_machines: - LOG.debug("check machines = ", check_machines(worker_machines)) - if not check_machines(worker_machines): - continue - - if yenv: - output_path = get_yaml_var(yenv, "OUTPUT_PATH", None) - if output_path and not os.path.exists(output_path): - hostname = socket.gethostname() - LOG.error( - f"The output path, {output_path}, is not accessible on this host, {hostname}" - ) - else: - LOG.warning( - "The env:variables section does not have an OUTPUT_PATH" - "specified, multi-machine checks cannot be performed." - ) + skip_loop_step: bool = examine_and_log_machines(worker_val, yenv) + if skip_loop_step: + continue worker_args = get_yaml_var(worker_val, "args", celery_args) with suppress(KeyError): @@ -260,7 +245,7 @@ def start_celery_workers(spec, steps, celery_args, just_return_command): # Add a per worker log file (debug) if LOG.isEnabledFor(logging.DEBUG): LOG.debug("Redirecting worker output to individual log files") - worker_args += f" --logfile %p.%i" + worker_args += " --logfile %p.%i" # Get the celery command celery_com = launch_celery_workers( @@ -324,6 +309,32 @@ def start_celery_workers(spec, steps, celery_args, just_return_command): return str(worker_list) +def examine_and_log_machines(worker_val, yenv) -> bool: + """ + Examines whether a worker should be skipped in a step of start_celery_workers(), logs errors in output path for a celery + worker. + """ + worker_machines = get_yaml_var(worker_val, "machines", None) + if worker_machines: + LOG.debug("check machines = ", check_machines(worker_machines)) + if not check_machines(worker_machines): + return True + + if yenv: + output_path = get_yaml_var(yenv, "OUTPUT_PATH", None) + if output_path and not os.path.exists(output_path): + hostname = socket.gethostname() + LOG.error( + f"The output path, {output_path}, is not accessible on this host, {hostname}" + ) + else: + LOG.warning( + "The env:variables section does not have an OUTPUT_PATH" + "specified, multi-machine checks cannot be performed." + ) + return False + + def verify_args(spec, worker_args, worker_name, overlap): """Examines the args passed to a worker for completeness.""" parallel = batch_check_parallel(spec) @@ -388,7 +399,7 @@ def purge_celery_tasks(queues, force): force_com = " -f " purge_command = " ".join(["celery -A merlin purge", force_com, "-Q", queues]) LOG.debug(purge_command) - return subprocess.call(purge_command.split()) + return subprocess.run(purge_command, shell=True).returncode def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 4a124cfb6..22f3d7629 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 4aa7df048..0a8d367ea 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index e6a46c183..bdde0a9b4 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 8b0b2e1b4..ad590d79e 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index b06495b21..c1914735c 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -386,15 +386,15 @@ def get_flux_version(flux_path, no_errors=False): except FileNotFoundError as e: if not no_errors: LOG.error(f"The flux path {flux_path} canot be found") - LOG.error(f"Suppress this error with no_errors=True") + LOG.error("Suppress this error with no_errors=True") raise e try: flux_ver = re.search(r"\s*([\d.]+)", ps[0]).group(1) except (ValueError, TypeError) as e: if not no_errors: - LOG.error(f"The flux version canot be determined") - LOG.error(f"Suppress this error with no_errors=True") + LOG.error("The flux version cannot be determined") + LOG.error("Suppress this error with no_errors=True") raise e else: flux_ver = DEFAULT_FLUX_VERSION diff --git a/setup.cfg b/setup.cfg index 8161403db..eccb60e46 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,7 @@ ignore = E203, E266, E501, W503 max-line-length = 127 max-complexity = 15 select = B,C,E,F,W,T4 -exclude = .git,__pycache__,ascii_art.py,merlin/examples/* +exclude = .git,__pycache__,ascii_art.py,merlin/examples/*,*venv* [mypy] files=best_practices,test diff --git a/setup.py b/setup.py index c62d698f3..db5ed9a74 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -44,8 +44,9 @@ def readme(): # The reqs code from celery setup.py -def _strip_comments(l): - return l.split("#", 1)[0].strip() +def _strip_comments(line: str): + """Removes comments from a line passed in from _reqs().""" + return line.split("#", 1)[0].strip() def _pip_requirement(req): @@ -59,8 +60,8 @@ def _reqs(*f): return [ _pip_requirement(r) for r in ( - _strip_comments(l) - for l in open(os.path.join(os.getcwd(), "requirements", *f)).readlines() + _strip_comments(line) + for line in open(os.path.join(os.getcwd(), "requirements", *f)).readlines() ) if r ] diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 7fd52c7c0..a43a3f51e 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.0. +# This file is part of Merlin, Version: 1.8.1. # # For details, see https://github.com/LLNL/merlin. # @@ -106,13 +106,10 @@ def process_test_result(passed, info, is_verbose, exit): print("pass") if info["violated_condition"] is not None: - message = info["violated_condition"][0] - condition_id = info["violated_condition"][1] + 1 - n_conditions = info["violated_condition"][2] - print( - f"\tCondition {condition_id} of {n_conditions}: " - + str(info["violated_condition"][0]) - ) + msg: str = str(info["violated_condition"][0]) + condition_id: str = info["violated_condition"][1] + 1 + n_conditions: str = info["violated_condition"][2] + print(f"\tCondition {condition_id} of {n_conditions}: {msg}") if is_verbose is True: print(f"\tcommand: {info['command']}") print(f"\telapsed time: {round(info['total_time'], 2)} s") diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 996d90273..edddeed59 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -105,7 +105,7 @@ def define_tests(): ), } example_tests = { - "example failure": (f"merlin example failure", HasRegex("not found"), "local"), + "example failure": ("merlin example failure", HasRegex("not found"), "local"), "example simple_chain": ( f"merlin example simple_chain ; {run} simple_chain.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR} ; rm simple_chain.yaml", HasReturnCode(), @@ -217,13 +217,13 @@ def define_tests(): [ HasReturnCode(), ProvenanceYAMLFileHasRegex( - regex="HELLO: \$\(SCRIPTS\)/hello_world.py", + regex=r"HELLO: \$\(SCRIPTS\)/hello_world.py", name="feature_demo", output_path=OUTPUT_DIR, provenance_type="orig", ), ProvenanceYAMLFileHasRegex( - regex="name: \$\(NAME\)", + regex=r"name: \$\(NAME\)", name="feature_demo", output_path=OUTPUT_DIR, provenance_type="partial", @@ -241,7 +241,7 @@ def define_tests(): provenance_type="expanded", ), ProvenanceYAMLFileHasRegex( - regex="\$\(NAME\)", + regex=r"\$\(NAME\)", name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", @@ -287,13 +287,13 @@ def define_tests(): f"{run} {demo} --pgen {demo_pgen} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local", [ ProvenanceYAMLFileHasRegex( - regex="\[0.3333333", + regex=r"\[0.3333333", name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", ), ProvenanceYAMLFileHasRegex( - regex="\[0.5", + regex=r"\[0.5", name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", @@ -304,25 +304,25 @@ def define_tests(): "local", ), } - provenence_equality_checks = { + provenence_equality_checks = { # noqa: F841 "local provenance spec equality": ( f"{run} {simple} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local ; cp $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml') ./{OUTPUT_DIR}/FILE1 ; rm -rf ./{OUTPUT_DIR}/simple_chain_* ; {run} ./{OUTPUT_DIR}/FILE1 --vars OUTPUT_PATH=./{OUTPUT_DIR} --local ; cmp ./{OUTPUT_DIR}/FILE1 $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml')", HasReturnCode(), "local", ), } - style_checks = { + style_checks = { # noqa: F841 "black check merlin": (f"{black} merlin/", HasReturnCode(), "local"), "black check tests": (f"{black} tests/", HasReturnCode(), "local"), } dependency_checks = { "deplic no GNU": ( - f"deplic ./", + "deplic ./", [HasRegex("GNU", negate=True), HasRegex("GPL", negate=True)], "local", ), } - distributed_tests = { + distributed_tests = { # noqa: F841 "run and purge feature_demo": ( f"{run} {demo} ; {purge} {demo} -f", HasReturnCode(), diff --git a/tests/unit/common/test_sample_index.py b/tests/unit/common/test_sample_index.py index ba18db1e8..eddeb1524 100644 --- a/tests/unit/common/test_sample_index.py +++ b/tests/unit/common/test_sample_index.py @@ -1,6 +1,5 @@ import os import shutil -import unittest from contextlib import suppress from merlin.common.sample_index_factory import create_hierarchy, read_hierarchy diff --git a/tests/unit/study/test_study.py b/tests/unit/study/test_study.py index 2b04e1df9..126a8f802 100644 --- a/tests/unit/study/test_study.py +++ b/tests/unit/study/test_study.py @@ -6,6 +6,8 @@ import tempfile import unittest +import pytest + from merlin.study.step import Step from merlin.study.study import MerlinStudy @@ -148,6 +150,7 @@ """ +# TODO many of these more resemble integration tests than unit tests, may want to review unit tests to make it more granular. def test_get_task_queue_default(): """ Given a steps dictionary that sets the task queue to `test_queue` return @@ -277,27 +280,36 @@ def test_column_label_conflict(self): If there is a common key between Maestro's global.parameters and Merlin's sample/column_labels, an error should be raised. """ - merlin_spec_conflict = os.path.join(self.tmpdir, "basic_ensemble_conflict.yaml") + merlin_spec_conflict: str = os.path.join( + self.tmpdir, "basic_ensemble_conflict.yaml" + ) with open(merlin_spec_conflict, "w+") as _file: _file.write(MERLIN_SPEC_CONFLICT) - try: - study_conflict = MerlinStudy(merlin_spec_conflict) - except ValueError: - pass - else: - assert False - + # for some reason flake8 doesn't believe variables instantiated inside the try/with context are assigned + with pytest.raises(ValueError): + study_conflict: MerlinStudy = MerlinStudy(merlin_spec_conflict) + assert ( + not study_conflict + ), "study_conflict completed construction without raising a ValueError." + + # TODO the pertinent attribute for study_no_env should be examined and asserted to be empty def test_no_env(self): """ A MerlinStudy should be able to support a MerlinSpec that does not contain the optional `env` section. """ - merlin_spec_no_env_filepath = os.path.join( + merlin_spec_no_env_filepath: str = os.path.join( self.tmpdir, "basic_ensemble_no_env.yaml" ) with open(merlin_spec_no_env_filepath, "w+") as _file: _file.write(MERLIN_SPEC_NO_ENV) try: - study_no_env = MerlinStudy(merlin_spec_no_env_filepath) + study_no_env: MerlinStudy = MerlinStudy(merlin_spec_no_env_filepath) + bad_type_err: str = ( + f"study_no_env failed construction, is type {type(study_no_env)}." + ) + assert isinstance(study_no_env, MerlinStudy), bad_type_err except Exception as e: - assert False + assert ( + False + ), f"Encountered unexpected exception, {e}, for viable MerlinSpec without optional 'env' section." From 96967d251ed375c618a934b5ab25c582a9dfa91f Mon Sep 17 00:00:00 2001 From: Alexander Winter <80929291+AlexanderWinterLLNL@users.noreply.github.com> Date: Thu, 23 Sep 2021 15:54:02 -0700 Subject: [PATCH 006/126] CI changes: separate linting and testing, add caching, add check for CHANGELOG update (#334) * CI changes: separated linting and testing, added caching, added a test to check for a CHANGELOG update when pulling. Co-authored-by: Alexander Cameron Winter --- .github/workflows/push-pr_workflow.yml | 57 ++++++++++++++++++++++---- CHANGELOG.md | 10 +++++ Makefile | 1 + 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index b1a62e791..b50c8dbb0 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -3,8 +3,50 @@ name: Python CI on: [push, pull_request] jobs: - build: + changelog: + name: CHANGELOG.md updated + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + + steps: + - uses: actions/checkout@v1 + + - name: Check that CHANGELOG has been updated + run: | + # If this step fails, this means you haven't updated the CHANGELOG.md + # file with notes on your contribution. + git diff --name-only $(git merge-base origin/main HEAD) | grep '^CHANGELOG.md$' && echo "Thanks for helping keep our CHANGELOG up-to-date!" + + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Check cache + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install --upgrade -r requirements.txt; fi + pip3 install --upgrade -r requirements/dev.txt + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --max-complexity=15 --statistics --max-line-length=127 + + build: runs-on: ubuntu-latest strategy: matrix: @@ -17,6 +59,12 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Check cache + uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + - name: Install dependencies run: | python3 -m pip install --upgrade pip @@ -33,13 +81,6 @@ jobs: merlin example feature_demo pip3 install -r feature_demo/requirements.txt - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --max-complexity=15 --statistics --max-line-length=127 - - name: Run pytest over unit test suite run: | python3 -m pytest tests/unit/ diff --git a/CHANGELOG.md b/CHANGELOG.md index d6cef3eed..5a97f4b64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed +- CI now splits linting and testing into different tasks for better utilization of + parallel runners, significant and scalable speed gain over previous setup +- CI now uses caching to restore environment of dependencies, reducing CI runtime + significantly again beyond the previous improvement. Examines for potential updates to + dependencies so the environment doesn't become stale. +- CI now examines that the CHANGELOG is updated on PRs. + ## [1.8.1] ### Fixed diff --git a/Makefile b/Makefile index 733cf9c8c..0f5d850e7 100644 --- a/Makefile +++ b/Makefile @@ -102,6 +102,7 @@ check-style: check-push: tests check-style + # finds all strings in project that begin with a lowercase letter, # contain only letters and numbers, and contain at least one lowercase # letter and at least one uppercase letter. From 60598154fe9255ce5bcbd5c0fe28a56be9672309 Mon Sep 17 00:00:00 2001 From: Alexander Winter <80929291+AlexanderWinterLLNL@users.noreply.github.com> Date: Fri, 24 Sep 2021 12:29:09 -0700 Subject: [PATCH 007/126] Pylint ci refactor (#331) First pass at incorporating PyLint. * Implemented no-fail Pylint examination in CI pipeline (no-fail as the refactor is incomplete), added as a dev dependency. PyLint settings configured in setup.cfg. * Style changes/refactor to numerous files for Pylint compatibility: celery.py, opennpylib, config/__init__.py, broker.py, configfile.py, log_formatter.py, main.py, router.py. * Type hinting to functions/classes touched by PyLint alterations * Corrected over-examination of dependency test in integrated testing suite to enable PyLint as a dev dependency - the test suite now only examines for GPL license dependency by examining requirements/release.txt * Minor Makefile changes to accommodate PyLint Co-authored-by: Alexander Cameron Winter --- .github/workflows/push-pr_workflow.yml | 11 +- CHANGELOG.md | 4 + Makefile | 41 ++- merlin/celery.py | 65 ++-- merlin/common/opennpylib.py | 1 + merlin/config/__init__.py | 23 +- merlin/config/broker.py | 22 +- merlin/config/configfile.py | 34 +- merlin/log_formatter.py | 2 + merlin/main.py | 412 ++++++++++++++----------- merlin/router.py | 7 +- requirements/dev.txt | 1 + setup.cfg | 5 + tests/integration/test_definitions.py | 3 +- 14 files changed, 373 insertions(+), 258 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index b50c8dbb0..617481448 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -3,7 +3,7 @@ name: Python CI on: [push, pull_request] jobs: - changelog: + Changelog: name: CHANGELOG.md updated runs-on: ubuntu-latest if: github.event_name == 'pull_request' @@ -17,7 +17,7 @@ jobs: # file with notes on your contribution. git diff --name-only $(git merge-base origin/main HEAD) | grep '^CHANGELOG.md$' && echo "Thanks for helping keep our CHANGELOG up-to-date!" - lint: + Lint: runs-on: ubuntu-latest steps: @@ -46,7 +46,12 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --max-complexity=15 --statistics --max-line-length=127 - build: + - name: Lint with PyLint + run: | + python3 -m pylint merlin --rcfile=setup.cfg --exit-zero + python3 -m pylint tests --rcfile=setup.cfg --exit-zero + + Test-suite: runs-on: ubuntu-latest strategy: matrix: diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a97f4b64..bf52a3bd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 significantly again beyond the previous improvement. Examines for potential updates to dependencies so the environment doesn't become stale. - CI now examines that the CHANGELOG is updated on PRs. +- Added PyLint pipeline to Github Actions CI (currently no-fail-exit). +- Corrected integration test for dependency to only examine release dependencies. +- PyLint adherence for: celery.py, opennplib.py, config/__init__.py, broker.py, + configfile.py, formatter.py, main.py, router.py ## [1.8.1] diff --git a/Makefile b/Makefile index 0f5d850e7..a0b96f757 100644 --- a/Makefile +++ b/Makefile @@ -32,12 +32,15 @@ include config.mk .PHONY : virtualenv .PHONY : install-merlin .PHONY : install-workflow-deps -.PHONY : install-merlin-dev +.PHONY : install-dev .PHONY : unit-tests .PHONY : e2e-tests .PHONY : tests -.PHONY : fix-style +.PHONY : check-flake8 +.PHONY : check-black +.PHONY : check-pylint .PHONY : check-style +.PHONY : fix-style .PHONY : check-push .PHONY : check-camel-case .PHONY : checks @@ -49,6 +52,7 @@ include config.mk .PHONY : clean-py .PHONY : clean + # this only works outside the venv - if run from inside a custom venv, or any target that depends on this, # you will break your venv. virtualenv: @@ -56,6 +60,7 @@ virtualenv: $(PIP) install --upgrade pip; \ $(PIP) install -r requirements/release.txt; \ + # install merlin into the virtual environment install-merlin: virtualenv $(PIP) install -e .; \ @@ -68,7 +73,7 @@ install-workflow-deps: virtualenv install-merlin # install requirements -install-merlin-dev: virtualenv install-workflow-deps +install-dev: virtualenv install-merlin install-workflow-deps $(PIP) install -r requirements/dev.txt; \ @@ -85,19 +90,37 @@ e2e-tests: e2e-tests-diagnostic: - $(PYTHON) $(TEST)/integration/run_tests.py --local --verbose + . $(VENV)/bin/activate; \ + $(PYTHON) $(TEST)/integration/run_tests.py --local --verbose; \ # run unit and CLI tests tests: unit-tests e2e-tests +check-flake8: + . $(VENV)/bin/activate; \ + echo "Flake8 linting for invalid source (bad syntax, undefined variables)..."; \ + $(PYTHON) -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics; \ + echo "Flake8 linting failure for CI..."; \ + $(PYTHON) -m flake8 . --count --max-complexity=15 --statistics --max-line-length=127; \ + + +check-black: + . $(VENV)/bin/activate; \ + $(PYTHON) -m black --check --target-version py36 $(MRLN); \ + + +check-pylint: + . $(VENV)/bin/activate; \ + echo "PyLinting merlin source..."; \ + $(PYTHON) -m pylint merlin --rcfile=setup.cfg --ignore-patterns="$(VENV)/" --disable=logging-fstring-interpolation; \ + echo "PyLinting merlin tests..."; \ + $(PYTHON) -m pylint tests --rcfile=setup.cfg; \ + + # run code style checks -check-style: - . $(VENV)/bin/activate - -$(PYTHON) -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics - -$(PYTHON) -m flake8 . --count --max-complexity=15 --statistics --max-line-length=127 - -black --check --target-version py36 $(MRLN) +check-style: check-flake8 check-black check-pylint check-push: tests check-style diff --git a/merlin/celery.py b/merlin/celery.py index 91ff70f9d..105e241c3 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -33,6 +33,7 @@ import logging import os +from typing import Dict, Optional, Union import billiard import psutil @@ -47,41 +48,43 @@ from merlin.utils import nested_namespace_to_dicts -LOG = logging.getLogger(__name__) +LOG: logging.Logger = logging.getLogger(__name__) merlin.common.security.encrypt_backend_traffic.set_backend_funcs() -broker_ssl = True -results_ssl = False +BROKER_SSL: bool = True +RESULTS_SSL: bool = False +BROKER_URI: Optional[str] = "" +RESULTS_BACKEND_URI: Optional[str] = "" try: BROKER_URI = broker.get_connection_string() - LOG.debug(f"broker: {broker.get_connection_string(include_password=False)}") - broker_ssl = broker.get_ssl_config() - LOG.debug(f"broker_ssl = {broker_ssl}") + LOG.debug("broker: %s", broker.get_connection_string(include_password=False)) + BROKER_SSL = broker.get_ssl_config() + LOG.debug("broker_ssl = %s", BROKER_SSL) RESULTS_BACKEND_URI = results_backend.get_connection_string() - results_ssl = results_backend.get_ssl_config(celery_check=True) + RESULTS_SSL = results_backend.get_ssl_config(celery_check=True) LOG.debug( - f"results: {results_backend.get_connection_string(include_password=False)}" + "results: %s", results_backend.get_connection_string(include_password=False) ) - LOG.debug(f"results: redis_backed_use_ssl = {results_ssl}") + LOG.debug("results: redis_backed_use_ssl = %s", RESULTS_SSL) except ValueError: # These variables won't be set if running with '--local'. BROKER_URI = None RESULTS_BACKEND_URI = None # initialize app with essential properties -app = Celery( +app: Celery = Celery( "merlin", broker=BROKER_URI, backend=RESULTS_BACKEND_URI, - broker_use_ssl=broker_ssl, - redis_backend_use_ssl=results_ssl, + broker_use_ssl=BROKER_SSL, + redis_backend_use_ssl=RESULTS_SSL, task_routes=(route_for_task,), ) # set task priority defaults to prioritize workflow tasks over task-expansion tasks -task_priority_defaults = { +task_priority_defaults: Dict[str, Union[int, Priority]] = { "task_queue_max_priority": 10, "task_default_priority": get_priority(Priority.mid), } @@ -97,15 +100,15 @@ # load config overrides from app.yaml if ( - (not hasattr(CONFIG.celery, "override")) + not hasattr(CONFIG.celery, "override") or (CONFIG.celery.override is None) - or (len(nested_namespace_to_dicts(CONFIG.celery.override)) == 0) + or (not nested_namespace_to_dicts(CONFIG.celery.override)) # only true if len == 0 ): LOG.debug("Skipping celery config override; 'celery.override' field is empty.") else: - override_dict = nested_namespace_to_dicts(CONFIG.celery.override) - override_str = "" - i = 0 + override_dict: Dict = nested_namespace_to_dicts(CONFIG.celery.override) + override_str: str = "" + i: int = 0 for k, v in override_dict.items(): if k not in str(app.conf.__dict__): raise ValueError(f"'{k}' is not a celery configuration.") @@ -114,7 +117,8 @@ override_str += "\n" i += 1 LOG.info( - f"Overriding default celery config with 'celery.override' in 'app.yaml':\n{override_str}" + "Overriding default celery config with 'celery.override' in 'app.yaml':\n%s", + override_str, ) app.conf.update(**override_dict) @@ -122,8 +126,9 @@ app.autodiscover_tasks(["merlin.common"]) +# Pylint believes the args are unused, I believe they're used after decoration @worker_process_init.connect() -def setup(**kwargs): +def setup(**kwargs): # pylint: disable=W0613 """ Set affinity for the worker on startup (works on toss3 nodes) @@ -131,10 +136,16 @@ def setup(**kwargs): """ if "CELERY_AFFINITY" in os.environ and int(os.environ["CELERY_AFFINITY"]) > 1: # Number of cpus between workers. - cpu_skip = int(os.environ["CELERY_AFFINITY"]) - npu = psutil.cpu_count() - p = psutil.Process() - current = billiard.current_process() - prefork_id = current._identity[0] - 1 # range 0:nworkers-1 - cpu_slot = (prefork_id * cpu_skip) % npu - p.cpu_affinity(list(range(cpu_slot, cpu_slot + cpu_skip))) + cpu_skip: int = int(os.environ["CELERY_AFFINITY"]) + npu: int = psutil.cpu_count() + process: psutil.Process = psutil.Process() + # pylint is upset that typing accesses a protected class, ignoring W0212 + # pylint is upset that billiard doesn't have a current_process() method - it does + current: billiard.process._MainProcess = ( + billiard.current_process() # pylint: disable=W0212, E1101 + ) + prefork_id: int = ( + current._identity[0] - 1 # pylint: disable=W0212 + ) # range 0:nworkers-1 + cpu_slot: int = (prefork_id * cpu_skip) % npu + process.cpu_affinity(list(range(cpu_slot, cpu_slot + cpu_skip))) diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index b44bfead5..1aa35c4a0 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -81,6 +81,7 @@ print a.dtype # dtype of array """ + from typing import List, Tuple import numpy as np diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index d3d9fc75f..7ea6493a9 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -32,6 +32,9 @@ Used to store the application configuration. """ +from types import SimpleNamespace +from typing import Dict, List, Optional + from merlin.utils import nested_dict_to_namespaces @@ -43,20 +46,20 @@ class Config: """ def __init__(self, app_dict): - self.load_app(app_dict) + # I think this ends up a SimpleNamespace from load_app_into_namespaces, but it seems like it should be typed as + # the app var in celery.py, as celery.app.base.Celery + self.celery: Optional[SimpleNamespace] + self.broker: Optional[SimpleNamespace] + self.results_backend: Optional[SimpleNamespace] + self.load_app_into_namespaces(app_dict) - def load_namespaces(self, dic, fields): + def load_app_into_namespaces(self, app_dict: Dict) -> None: """ - TODO + Makes the application dictionary into a namespace, sets the attributes of the Config from the namespace values. """ + fields: List[str] = ["celery", "broker", "results_backend"] for field in fields: try: - setattr(self, field, nested_dict_to_namespaces(dic[field])) + setattr(self, field, nested_dict_to_namespaces(app_dict[field])) except KeyError: pass - - def load_app(self, dic): - """ - Makes the application dictionary into a namespace. - """ - self.load_namespaces(dic, ["celery", "broker", "results_backend"]) diff --git a/merlin/config/broker.py b/merlin/config/broker.py index cfd0b366a..f841b0e1b 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -34,7 +34,9 @@ import getpass import logging import os +import ssl from os.path import expanduser +from typing import Dict, List, Optional, Union from merlin.config.configfile import CONFIG, get_ssl_entries @@ -45,12 +47,12 @@ from urllib.parse import quote -LOG = logging.getLogger(__name__) +LOG: logging.Logger = logging.getLogger(__name__) -BROKERS = ["rabbitmq", "redis", "rediss", "redis+socket", "amqps", "amqp"] +BROKERS: List[str] = ["rabbitmq", "redis", "rediss", "redis+socket", "amqps", "amqp"] -RABBITMQ_CONNECTION = "{conn}://{username}:{password}@{server}:{port}/{vhost}" -REDISSOCK_CONNECTION = "redis+socket://{path}?virtual_host={db_num}" +RABBITMQ_CONNECTION: str = "{conn}://{username}:{password}@{server}:{port}/{vhost}" +REDISSOCK_CONNECTION: str = "redis+socket://{path}?virtual_host={db_num}" USER = getpass.getuser() @@ -239,12 +241,15 @@ def _sort_valid_broker(broker, config_path, include_password): return get_redis_connection(config_path, include_password, ssl=True) -def get_ssl_config(): +def get_ssl_config() -> Union[bool, Dict[str, Union[str, ssl.VerifyMode]]]: """ Return the ssl config based on the configuration specified in the `app.yaml` config file. + + :return: Returns either False if no ssl + :rtype: Union[bool, Dict[str, Union[str, ssl.VerifyMode]]] """ - broker = "" + broker: Union[bool, str] = "" try: broker = CONFIG.broker.url.split(":")[0] except AttributeError: @@ -258,12 +263,15 @@ def get_ssl_config(): if broker not in BROKERS: return False + certs_path: Optional[str] try: certs_path = CONFIG.celery.certs except AttributeError: certs_path = None - broker_ssl = get_ssl_entries("Broker", broker, CONFIG.broker, certs_path) + broker_ssl: Union[bool, Dict[str, Union[str, ssl.VerifyMode]]] = get_ssl_entries( + "Broker", broker, CONFIG.broker, certs_path + ) if not broker_ssl: broker_ssl = True diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index a08c53553..9ebad3007 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -42,13 +42,13 @@ from merlin.utils import load_yaml -LOG = logging.getLogger(__name__) +LOG: logging.Logger = logging.getLogger(__name__) -APP_FILENAME = "app.yaml" -CONFIG = None +APP_FILENAME: str = "app.yaml" +CONFIG: Optional[Config] = None -USER_HOME = os.path.expanduser("~") -MERLIN_HOME = os.path.join(USER_HOME, ".merlin") +USER_HOME: str = os.path.expanduser("~") +MERLIN_HOME: str = os.path.join(USER_HOME, ".merlin") def load_config(filepath): @@ -111,21 +111,23 @@ def load_default_user_names(config): config["broker"]["vhost"] = vhost -def get_config(path): +def get_config(path: Optional[str]) -> Dict: """ Load a merlin configuration file and return a dictionary of the configurations. - :param path : The path to search for the config file. + :param [Optional[str]] path : The path to search for the config file. + :return: the config file to coordinate brokers/results backend/task manager." + :rtype: A Dict with all the config data. """ - filepath = find_config_file(path) + filepath: Optional[str] = find_config_file(path) if filepath is None: raise ValueError( f"Cannot find a merlin config file! Run 'merlin config' and edit the file '{MERLIN_HOME}/{APP_FILENAME}'" ) - config = load_config(filepath) + config: Dict = load_config(filepath) load_defaults(config) return config @@ -205,14 +207,16 @@ def get_cert_file(server_type, config, cert_name, cert_path): def get_ssl_entries( server_type: str, server_name: str, server_config: Config, cert_path: str -) -> str: +) -> Dict[str, Union[str, ssl.VerifyMode]]: """ Check if a ssl certificate file is present in the config - :param server_type : The server type - :param server_name : The server name for output - :param server_config : The server config - :param cert_path : The optional cert path + :param [str] server_type : The server type + :param [str] server_name : The server name for output + :param [Config] server_config : The server config + :param [str] cert_path : The optional cert path + :return : The data needed to manage an ssl certification. + :rtype : A Dict. """ server_ssl: Dict[str, Union[str, ssl.VerifyMode]] = {} @@ -307,5 +311,5 @@ def merge_sslmap( server_ssl = new_server_ssl -app_config = get_config(None) +app_config: Dict = get_config(None) CONFIG = Config(app_config) diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 7e692765a..b336e7715 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -1,3 +1,5 @@ +"""This module handles setting up the extensive logging system in Merlin.""" + ############################################################################### # Copyright (c) 2019, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory diff --git a/merlin/main.py b/merlin/main.py index 4a159a717..fc6fa9804 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -1,3 +1,5 @@ +"""The top level main function for invoking Merlin.""" + ############################################################################### # Copyright (c) 2019, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory @@ -39,10 +41,12 @@ from argparse import ( ArgumentDefaultsHelpFormatter, ArgumentParser, + Namespace, RawDescriptionHelpFormatter, RawTextHelpFormatter, ) from contextlib import suppress +from typing import Dict, List, Optional, Union from merlin import VERSION, router from merlin.ascii_art import banner_small @@ -68,12 +72,15 @@ def error(self, message): sys.exit(2) -def verify_filepath(filepath): +def verify_filepath(filepath: str) -> str: """ Verify that the filepath argument is a valid file. - :param `filepath`: the path of a file + :param [str] `filepath`: the path of a file + + :return: the verified absolute filepath with expanded environment variables. + :rtype: str """ filepath = os.path.abspath(os.path.expandvars(os.path.expanduser(filepath))) if not os.path.isfile(filepath): @@ -81,42 +88,51 @@ def verify_filepath(filepath): return filepath -def verify_dirpath(dirpath): +def verify_dirpath(dirpath: str) -> str: """ Verify that the dirpath argument is a valid directory. - :param `dirpath`: the path of a directory + :param [str] `dirpath`: the path of a directory + + :return: returns the absolute path with expanded environment vars for a given dirpath. + :rtype: str """ - dirpath = os.path.abspath(os.path.expandvars(os.path.expanduser(dirpath))) + dirpath: str = os.path.abspath(os.path.expandvars(os.path.expanduser(dirpath))) if not os.path.isdir(dirpath): raise ValueError(f"'{dirpath}' is not a valid directory path") return dirpath -def parse_override_vars(variables_list): +def parse_override_vars( + variables_list: Optional[List[str]], +) -> Optional[Dict[str, Union[str, int]]]: """ Parse a list of variables from command line syntax into a valid dictionary of variable keys and values. - :param `variables_list`: a list of strings, e.g. ["KEY=val",...] + :param [List[str]] `variables_list`: an optional list of strings, e.g. ["KEY=val",...] + + :return: returns either None or a Dict keyed with strs, linked to strs and ints. + :rtype: Dict """ if variables_list is None: return None LOG.debug(f"Command line override variables = {variables_list}") - result = {} + result: Dict[str, Union[str, int]] = {} + arg: str for arg in variables_list: try: if "=" not in arg: raise ValueError( "--vars requires '=' operator. See 'merlin run --help' for an example." ) - entry = arg.split("=") + entry: str = arg.split("=") if len(entry) != 2: raise ValueError( "--vars requires ONE '=' operator (without spaces) per variable assignment." ) - key = entry[0] + key: str = entry[0] if key is None or key == "" or "$" in key: raise ValueError( "--vars requires valid variable names comprised of alphanumeric characters and underscores." @@ -126,16 +142,16 @@ def parse_override_vars(variables_list): f"Cannot override reserved word '{key}'! Reserved words are: {RESERVED}." ) - val = entry[1] + val: Union[str, int] = entry[1] with suppress(ValueError): int(val) val = int(val) result[key] = val - except BaseException as e: + except BaseException as excpt: raise ValueError( - f"{e} Bad '--vars' formatting on command line. See 'merlin run --help' for an example." - ) + f"{excpt} Bad '--vars' formatting on command line. See 'merlin run --help' for an example." + ) from excpt return result @@ -151,16 +167,16 @@ def get_merlin_spec_with_override(args): return spec, filepath -def process_run(args): +def process_run(args: Namespace) -> None: """ CLI command for running a study. - :param `args`: parsed CLI arguments + :param [Namespace] `args`: parsed CLI arguments """ print(banner_small) - filepath = verify_filepath(args.specification) - variables_dict = parse_override_vars(args.variables) - samples_file = None + filepath: str = verify_filepath(args.specification) + variables_dict: str = parse_override_vars(args.variables) + samples_file: Optional[str] = None if args.samples_file: samples_file = verify_filepath(args.samples_file) @@ -172,7 +188,7 @@ def process_run(args): if args.pgen_file: verify_filepath(args.pgen_file) - study = MerlinStudy( + study: MerlinStudy = MerlinStudy( filepath, override_vars=variables_dict, samples_file=samples_file, @@ -184,27 +200,27 @@ def process_run(args): router.run_task_server(study, args.run_mode) -def process_restart(args): +def process_restart(args: Namespace) -> None: """ CLI command for restarting a study. - :param `args`: parsed CLI arguments + :param [Namespace] `args`: parsed CLI arguments """ print(banner_small) - restart_dir = verify_dirpath(args.restart_dir) - filepath = os.path.join(args.restart_dir, "merlin_info", "*.expanded.yaml") - possible_specs = glob.glob(filepath) - if len(possible_specs) == 0: + restart_dir: str = verify_dirpath(args.restart_dir) + filepath: str = os.path.join(args.restart_dir, "merlin_info", "*.expanded.yaml") + possible_specs: Optional[List[str]] = glob.glob(filepath) + if not possible_specs: # len == 0 raise ValueError( f"'{filepath}' does not match any provenance spec file to restart from." ) - elif len(possible_specs) > 1: + if len(possible_specs) > 1: raise ValueError( f"'{filepath}' matches more than one provenance spec file to restart from." ) - filepath = verify_filepath(possible_specs[0]) + filepath: str = verify_filepath(possible_specs[0]) LOG.info(f"Restarting workflow at '{restart_dir}'") - study = MerlinStudy(filepath, restart_dir=restart_dir) + study: MerlinStudy = MerlinStudy(filepath, restart_dir=restart_dir) router.run_task_server(study, args.run_mode) @@ -297,25 +313,30 @@ def print_info(args): :param `args`: parsed CLI arguments """ - from merlin import display + # if this is moved to the toplevel per standard style, merlin is unable to generate the (needed) default config file + from merlin import display # pylint: disable=import-outside-toplevel display.print_info(args) -def config_merlin(args): +def config_merlin(args: Namespace) -> None: """ CLI command to setup default merlin config. - :param `args`: parsed CLI arguments + :param [Namespace] `args`: parsed CLI arguments """ - output_dir = args.output_dir + output_dir: Optional[str] = args.output_dir if output_dir is None: - USER_HOME = os.path.expanduser("~") - output_dir = os.path.join(USER_HOME, ".merlin") - _ = router.create_config(args.task_server, output_dir, args.broker) + user_home: str = os.path.expanduser("~") + output_dir: str = os.path.join(user_home, ".merlin") + router.create_config(args.task_server, output_dir, args.broker) -def process_example(args): +def process_example(args: Namespace) -> None: + """Either lists all example workflows, or sets up an example as a workflow to be run at root dir. + + :param [Namespace] `args`: parsed CLI arguments + """ if args.workflow == "list": print(list_examples()) else: @@ -338,18 +359,18 @@ def process_monitor(args): LOG.info("Monitor: ... stop condition met") -def setup_argparse(): +def setup_argparse() -> None: """ Setup argparse and any CLI options we want available via the package. """ - parser = HelpParser( + parser: HelpParser = HelpParser( prog="merlin", description=banner_small, formatter_class=RawDescriptionHelpFormatter, epilog="See merlin --help for more info", ) parser.add_argument("-v", "--version", action="version", version=VERSION) - subparsers = parser.add_subparsers(dest="subparsers") + subparsers: ArgumentParser = parser.add_subparsers(dest="subparsers") subparsers.required = True # merlin --level @@ -365,7 +386,7 @@ def setup_argparse(): ) # merlin run - run = subparsers.add_parser( + run: ArgumentParser = subparsers.add_parser( "run", help="Run a workflow using a Merlin or Maestro YAML study " "specification.", formatter_class=ArgumentDefaultsHelpFormatter, @@ -392,7 +413,7 @@ def setup_argparse(): help="Specify desired Merlin variable values to override those found in the specification. Space-delimited. " "Example: '--vars LEARN=path/to/new_learn.py EPOCHS=3'", ) - # TODO add all supported formats to doc string + # TODO add all supported formats to doc string # pylint: disable=fixme run.add_argument( "--samplesfile", action="store", @@ -434,7 +455,7 @@ def setup_argparse(): ) # merlin restart - restart = subparsers.add_parser( + restart: ArgumentParser = subparsers.add_parser( "restart", help="Restart a workflow using an existing Merlin workspace.", formatter_class=ArgumentDefaultsHelpFormatter, @@ -452,53 +473,8 @@ def setup_argparse(): help="Run locally instead of distributed", ) - # merlin run-workers - run_workers = subparsers.add_parser( - "run-workers", - help="Run the workers associated with the Merlin YAML study " - "specification. Does -not- queue tasks, just workers tied " - "to the correct queues.", - formatter_class=ArgumentDefaultsHelpFormatter, - ) - run_workers.set_defaults(func=launch_workers) - run_workers.add_argument( - "specification", type=str, help="Path to a Merlin YAML spec file" - ) - run_workers.add_argument( - "--worker-args", - type=str, - dest="worker_args", - default="", - help="celery worker arguments in quotes.", - ) - run_workers.add_argument( - "--steps", - nargs="+", - type=str, - dest="worker_steps", - default=["all"], - help="The specific steps in the YAML file you want workers for", - ) - run_workers.add_argument( - "--echo", - action="store_true", - default=False, - dest="worker_echo_only", - help="Just echo the command; do not actually run it", - ) - run_workers.add_argument( - "--vars", - action="store", - dest="variables", - type=str, - nargs="+", - default=None, - help="Specify desired Merlin variable values to override those found in the specification. Space-delimited. " - "Example: '--vars LEARN=path/to/new_learn.py EPOCHS=3'", - ) - # merlin purge - purge = subparsers.add_parser( + purge: ArgumentParser = subparsers.add_parser( "purge", help="Remove all tasks from all merlin queues (default). " "If a user would like to purge only selected queues use: " @@ -523,7 +499,8 @@ def setup_argparse(): type=str, dest="purge_steps", default=["all"], - help="The specific steps in the YAML file from which you want to purge the queues. The input is a space separated list.", + help="The specific steps in the YAML file from which you want to purge the queues. \ + The input is a space separated list.", ) purge.add_argument( "--vars", @@ -536,94 +513,7 @@ def setup_argparse(): "Example: '--vars MY_QUEUE=hello'", ) - # merlin stop-workers - stop = subparsers.add_parser( - "stop-workers", help="Attempt to stop all task server workers." - ) - stop.set_defaults(func=stop_workers) - stop.add_argument( - "--spec", - type=str, - default=None, - help="Path to a Merlin YAML spec file from which to read worker names to stop.", - ) - stop.add_argument( - "--task_server", - type=str, - default="celery", - help="Task server type from which to stop workers.\ - Default: %(default)s", - ) - stop.add_argument( - "--queues", type=str, default=None, nargs="+", help="specific queues to stop" - ) - stop.add_argument( - "--workers", - type=str, - default=None, - help="regex match for specific workers to stop", - ) - - # merlin query-workers - query = subparsers.add_parser( - "query-workers", help="List connected task server workers." - ) - query.set_defaults(func=query_workers) - query.add_argument( - "--task_server", - type=str, - default="celery", - help="Task server type from which to query workers.\ - Default: %(default)s", - ) - - # merlin status - status = subparsers.add_parser( - "status", - help="List server stats (name, number of tasks to do, \ - number of connected workers) for a workflow spec.", - ) - status.set_defaults(func=query_status) - status.add_argument( - "specification", type=str, help="Path to a Merlin YAML spec file" - ) - status.add_argument( - "--steps", - nargs="+", - type=str, - dest="steps", - default=["all"], - help="The specific steps in the YAML file you want to query", - ) - status.add_argument( - "--task_server", - type=str, - default="celery", - help="Task server type.\ - Default: %(default)s", - ) - status.add_argument( - "--vars", - action="store", - dest="variables", - type=str, - nargs="+", - default=None, - help="Specify desired Merlin variable values to override those found in the specification. Space-delimited. " - "Example: '--vars LEARN=path/to/new_learn.py EPOCHS=3'", - ) - status.add_argument( - "--csv", type=str, help="csv file to dump status report to", default=None - ) - - # merlin info - info = subparsers.add_parser( - "info", - help="display info about the merlin configuration and the python configuration. Useful for debugging.", - ) - info.set_defaults(func=print_info) - - mconfig = subparsers.add_parser( + mconfig: ArgumentParser = subparsers.add_parser( "config", help="Create a default merlin server config file in ~/.merlin", formatter_class=ArgumentDefaultsHelpFormatter, @@ -653,7 +543,7 @@ def setup_argparse(): ) # merlin example - example = subparsers.add_parser( + example: ArgumentParser = subparsers.add_parser( "example", help="Generate an example merlin workflow.", formatter_class=RawTextHelpFormatter, @@ -675,8 +565,107 @@ def setup_argparse(): ) example.set_defaults(func=process_example) + generate_worker_touching_parsers(subparsers) + + generate_diagnostic_parsers(subparsers) + + return parser + + +def generate_worker_touching_parsers(subparsers: ArgumentParser) -> None: + """All CLI arg parsers directly controlling or invoking workers are generated here. + + :param [ArgumentParser] `subparsers`: the subparsers needed for every CLI command that directly controls or invokes + workers. + """ + # merlin run-workers + run_workers: ArgumentParser = subparsers.add_parser( + "run-workers", + help="Run the workers associated with the Merlin YAML study " + "specification. Does -not- queue tasks, just workers tied " + "to the correct queues.", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + run_workers.set_defaults(func=launch_workers) + run_workers.add_argument( + "specification", type=str, help="Path to a Merlin YAML spec file" + ) + run_workers.add_argument( + "--worker-args", + type=str, + dest="worker_args", + default="", + help="celery worker arguments in quotes.", + ) + run_workers.add_argument( + "--steps", + nargs="+", + type=str, + dest="worker_steps", + default=["all"], + help="The specific steps in the YAML file you want workers for", + ) + run_workers.add_argument( + "--echo", + action="store_true", + default=False, + dest="worker_echo_only", + help="Just echo the command; do not actually run it", + ) + run_workers.add_argument( + "--vars", + action="store", + dest="variables", + type=str, + nargs="+", + default=None, + help="Specify desired Merlin variable values to override those found in the specification. Space-delimited. " + "Example: '--vars LEARN=path/to/new_learn.py EPOCHS=3'", + ) + + # merlin query-workers + query: ArgumentParser = subparsers.add_parser( + "query-workers", help="List connected task server workers." + ) + query.set_defaults(func=query_workers) + query.add_argument( + "--task_server", + type=str, + default="celery", + help="Task server type from which to query workers.\ + Default: %(default)s", + ) + + # merlin stop-workers + stop: ArgumentParser = subparsers.add_parser( + "stop-workers", help="Attempt to stop all task server workers." + ) + stop.set_defaults(func=stop_workers) + stop.add_argument( + "--spec", + type=str, + default=None, + help="Path to a Merlin YAML spec file from which to read worker names to stop.", + ) + stop.add_argument( + "--task_server", + type=str, + default="celery", + help="Task server type from which to stop workers.\ + Default: %(default)s", + ) + stop.add_argument( + "--queues", type=str, default=None, nargs="+", help="specific queues to stop" + ) + stop.add_argument( + "--workers", + type=str, + default=None, + help="regex match for specific workers to stop", + ) + # merlin monitor - monitor = subparsers.add_parser( + monitor: ArgumentParser = subparsers.add_parser( "monitor", help="Check for active workers on an allocation.", formatter_class=RawTextHelpFormatter, @@ -718,7 +707,58 @@ def setup_argparse(): ) monitor.set_defaults(func=process_monitor) - return parser + +def generate_diagnostic_parsers(subparsers: ArgumentParser) -> None: + """All CLI arg parsers generally used diagnostically are generated here. + + :param [ArgumentParser] `subparsers`: the subparsers needed for every CLI command that handles diagnostics for a + Merlin job. + """ + # merlin status + status: ArgumentParser = subparsers.add_parser( + "status", + help="List server stats (name, number of tasks to do, \ + number of connected workers) for a workflow spec.", + ) + status.set_defaults(func=query_status) + status.add_argument( + "specification", type=str, help="Path to a Merlin YAML spec file" + ) + status.add_argument( + "--steps", + nargs="+", + type=str, + dest="steps", + default=["all"], + help="The specific steps in the YAML file you want to query", + ) + status.add_argument( + "--task_server", + type=str, + default="celery", + help="Task server type.\ + Default: %(default)s", + ) + status.add_argument( + "--vars", + action="store", + dest="variables", + type=str, + nargs="+", + default=None, + help="Specify desired Merlin variable values to override those found in the specification. Space-delimited. " + "Example: '--vars LEARN=path/to/new_learn.py EPOCHS=3'", + ) + status.add_argument( + "--csv", type=str, help="csv file to dump status report to", default=None + ) + + # merlin info + info: ArgumentParser = subparsers.add_parser( + "info", + help="display info about the merlin configuration and the python configuration. Useful for debugging.", + ) + info.set_defaults(func=print_info) def main(): @@ -735,10 +775,16 @@ def main(): try: args.func(args) - except Exception as e: + # pylint complains that this exception is too broad - being at the literal top of the program stack, + # it's ok. + except Exception as excpt: # pylint: disable=broad-except LOG.debug(traceback.format_exc()) - LOG.error(str(e)) + LOG.error(str(excpt)) return 1 + # All paths in a function ought to return an exit code, or none of them should. Given the + # distributed nature of Merlin, maybe it doesn't make sense for it to exit 0 until the work is completed, but + # if the work is dispatched with no errors, that is a 'successful' Merlin run - any other failures are runtime. + return 0 if __name__ == "__main__": diff --git a/merlin/router.py b/merlin/router.py index 0e852846a..ea2e9ee2c 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -207,12 +207,13 @@ def route_for_task(name, args, kwargs, options, task=None, **kw): return {"queue": queue} -def create_config(task_server, config_dir, broker): +def create_config(task_server: str, config_dir: str, broker: str) -> None: """ Create a config for the given task server. - :param `task_server`: The task server from which to stop workers. - :param `config_dir`: Optional directory to install the config. + :param [str] `task_server`: The task server from which to stop workers. + :param [str] `config_dir`: Optional directory to install the config. + :param [str] `broker`: string indicated the broker, used to check for redis. """ LOG.info("Creating config ...") diff --git a/requirements/dev.txt b/requirements/dev.txt index b9a546367..c66854a61 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,6 +4,7 @@ dep-license flake8 isort pytest +pylint twine sphinx>=2.0.0 alabaster diff --git a/setup.cfg b/setup.cfg index eccb60e46..b81cb1afa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,6 +16,11 @@ max-complexity = 15 select = B,C,E,F,W,T4 exclude = .git,__pycache__,ascii_art.py,merlin/examples/*,*venv* + +[pylint.FORMAT] +ignore=*venv* + + [mypy] files=best_practices,test ignore_missing_imports=true diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index edddeed59..f9c4e9620 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -38,6 +38,7 @@ def define_tests(): lsf = f"{examples}/lsf/lsf_par.yaml" black = "black --check --target-version py36" config_dir = "./CLI_TEST_MERLIN_CONFIG" + release_dependencies = "./requirements/release.txt" basic_checks = { "merlin": ("merlin", HasReturnCode(1), "local"), @@ -317,7 +318,7 @@ def define_tests(): } dependency_checks = { "deplic no GNU": ( - "deplic ./", + f"deplic {release_dependencies}", [HasRegex("GNU", negate=True), HasRegex("GPL", negate=True)], "local", ), From 42fe0e1055397ef2555e18d8808c9795828aa1db Mon Sep 17 00:00:00 2001 From: Alexander Winter <80929291+AlexanderWinterLLNL@users.noreply.github.com> Date: Mon, 11 Oct 2021 10:11:50 -0700 Subject: [PATCH 008/126] Enforce black and isort in the CI (#336) * Integrated Black and isort to the CI This incorporates Black and iSort into the CI as it is applied in the Makefile and documented standards. This improves style enforcement so that it is at the point of each PR rather than each release. Applies Black and iSort to the codebase. Co-authored-by: Alexander Cameron Winter Co-authored-by: KaseyNagleLLNL <88013612+KaseyNagleLLNL@users.noreply.github.com> --- .github/workflows/push-pr_workflow.yml | 20 ++++- CHANGELOG.md | 3 +- Makefile | 28 ++++-- merlin/celery.py | 12 +-- merlin/common/opennpylib.py | 38 +++------ merlin/common/sample_index.py | 40 +++------ merlin/common/sample_index_factory.py | 12 +-- merlin/common/tasks.py | 85 +++++-------------- merlin/config/broker.py | 4 +- merlin/config/configfile.py | 20 ++--- merlin/config/results_backend.py | 10 +-- merlin/config/utils.py | 8 +- merlin/display.py | 8 +- .../feature_demo/scripts/hello_world.py | 16 +--- .../examples/workflows/hello/make_samples.py | 4 +- .../hpc_demo/cumulative_sample_processor.py | 21 ++--- .../workflows/hpc_demo/faker_sample.py | 4 +- .../workflows/hpc_demo/sample_collector.py | 18 +--- .../workflows/hpc_demo/sample_processor.py | 12 +-- .../cumulative_sample_processor.py | 21 ++--- .../workflows/iterative_demo/faker_sample.py | 4 +- .../iterative_demo/sample_collector.py | 18 +--- .../iterative_demo/sample_processor.py | 12 +-- .../null_spec/scripts/launch_jobs.py | 4 +- .../null_spec/scripts/make_samples.py | 8 +- .../null_spec/scripts/read_output.py | 24 ++---- .../null_spec/scripts/read_output_chain.py | 24 ++---- .../openfoam_wf/scripts/combine_outputs.py | 8 +- .../openfoam_wf/scripts/make_samples.py | 4 +- .../scripts/combine_outputs.py | 8 +- .../scripts/make_samples.py | 4 +- .../optimization/scripts/optimizer.py | 24 ++---- .../optimization/scripts/visualizer.py | 28 ++---- merlin/main.py | 82 +++++------------- merlin/merlin_templates.py | 4 +- merlin/router.py | 8 +- merlin/spec/expansion.py | 14 ++- merlin/spec/override.py | 4 +- merlin/spec/specification.py | 48 +++-------- merlin/study/batch.py | 12 +-- merlin/study/celeryadapter.py | 53 +++--------- merlin/study/dag.py | 13 +-- merlin/study/script_adapter.py | 16 +--- merlin/study/step.py | 15 +--- merlin/study/study.py | 56 +++--------- merlin/utils.py | 22 ++--- setup.py | 5 +- tests/integration/conditions.py | 5 +- tests/integration/run_tests.py | 15 +--- tests/integration/test_definitions.py | 16 +--- tests/unit/common/test_sample_index.py | 12 +-- tests/unit/spec/test_specification.py | 12 +-- tests/unit/study/test_study.py | 44 +++------- tests/unit/utils/test_time_formats.py | 20 ++--- 54 files changed, 284 insertions(+), 746 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 617481448..8a9057616 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -13,12 +13,14 @@ jobs: - name: Check that CHANGELOG has been updated run: | - # If this step fails, this means you haven't updated the CHANGELOG.md - # file with notes on your contribution. + # If this step fails, this means you haven't updated the CHANGELOG.md file with notes on your contribution. git diff --name-only $(git merge-base origin/main HEAD) | grep '^CHANGELOG.md$' && echo "Thanks for helping keep our CHANGELOG up-to-date!" Lint: runs-on: ubuntu-latest + env: + MAX_LINE_LENGTH: 127 + MAX_COMPLEXITY: 15 steps: - uses: actions/checkout@v2 @@ -44,7 +46,19 @@ jobs: # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --max-complexity=15 --statistics --max-line-length=127 + flake8 . --count --max-complexity=$MAX_COMPLEXITY --statistics --max-line-length=$MAX_LINE_LENGTH + + - name: Lint with isort + run: | + python3 -m isort --check --line-length $MAX_LINE_LENGTH merlin + python3 -m isort --check --line-length $MAX_LINE_LENGTH tests + python3 -m isort --check --line-length $MAX_LINE_LENGTH *.py + + - name: Lint with Black + run: | + python3 -m black --check --line-length $MAX_LINE_LENGTH --target-version py36 merlin + python3 -m black --check --line-length $MAX_LINE_LENGTH --target-version py36 tests + python3 -m black --check --line-length $MAX_LINE_LENGTH --target-version py36 *.py - name: Lint with PyLint run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index bf52a3bd0..ffde16acb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added PyLint pipeline to Github Actions CI (currently no-fail-exit). - Corrected integration test for dependency to only examine release dependencies. - PyLint adherence for: celery.py, opennplib.py, config/__init__.py, broker.py, - configfile.py, formatter.py, main.py, router.py +configfile.py, formatter.py, main.py, router.py +- Integrated Black and isort into CI ## [1.8.1] diff --git a/Makefile b/Makefile index a0b96f757..d9aa80e18 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,7 @@ include config.mk .PHONY : tests .PHONY : check-flake8 .PHONY : check-black +.PHONY : check-isort .PHONY : check-pylint .PHONY : check-style .PHONY : fix-style @@ -108,7 +109,16 @@ check-flake8: check-black: . $(VENV)/bin/activate; \ - $(PYTHON) -m black --check --target-version py36 $(MRLN); \ + $(PYTHON) -m black --check --line-length $(MAX_LINE_LENGTH) --target-version py36 $(MRLN); \ + $(PYTHON) -m black --check --line-length $(MAX_LINE_LENGTH) --target-version py36 $(TEST); \ + $(PYTHON) -m black --check --line-length $(MAX_LINE_LENGTH) --target-version py36 *.py; \ + + +check-isort: + . $(VENV)/bin/activate; \ + $(PYTHON) -m isort --check --line-length $(MAX_LINE_LENGTH) merlin; \ + $(PYTHON) -m isort --check --line-length $(MAX_LINE_LENGTH) tests; \ + $(PYTHON) -m isort --check --line-length $(MAX_LINE_LENGTH) *.py; \ check-pylint: @@ -120,7 +130,7 @@ check-pylint: # run code style checks -check-style: check-flake8 check-black check-pylint +check-style: check-flake8 check-black check-isort check-pylint check-push: tests check-style @@ -139,13 +149,13 @@ checks: check-style check-camel-case # automatically make python files pep 8-compliant fix-style: - pip3 install -r requirements/dev.txt -U - isort -rc $(MRLN) - isort -rc $(TEST) - isort *.py - black --target-version py36 $(MRLN) - black --target-version py36 $(TEST) - black --target-version py36 *.py + . $(VENV)/bin/activate; \ + isort --line-length $(MAX_LINE_LENGTH) $(MRLN); \ + isort --line-length $(MAX_LINE_LENGTH) $(TEST); \ + isort --line-length $(MAX_LINE_LENGTH) *.py; \ + black --target-version py36 -l $(MAX_LINE_LENGTH) $(MRLN); \ + black --target-version py36 -l $(MAX_LINE_LENGTH) $(TEST); \ + black --target-version py36 -l $(MAX_LINE_LENGTH) *.py; \ # Increment the Merlin version. USE ONLY ON DEVELOP BEFORE MERGING TO MASTER. diff --git a/merlin/celery.py b/merlin/celery.py index 105e241c3..e8c0bc0ee 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -64,9 +64,7 @@ LOG.debug("broker_ssl = %s", BROKER_SSL) RESULTS_BACKEND_URI = results_backend.get_connection_string() RESULTS_SSL = results_backend.get_ssl_config(celery_check=True) - LOG.debug( - "results: %s", results_backend.get_connection_string(include_password=False) - ) + LOG.debug("results: %s", results_backend.get_connection_string(include_password=False)) LOG.debug("results: redis_backed_use_ssl = %s", RESULTS_SSL) except ValueError: # These variables won't be set if running with '--local'. @@ -141,11 +139,7 @@ def setup(**kwargs): # pylint: disable=W0613 process: psutil.Process = psutil.Process() # pylint is upset that typing accesses a protected class, ignoring W0212 # pylint is upset that billiard doesn't have a current_process() method - it does - current: billiard.process._MainProcess = ( - billiard.current_process() # pylint: disable=W0212, E1101 - ) - prefork_id: int = ( - current._identity[0] - 1 # pylint: disable=W0212 - ) # range 0:nworkers-1 + current: billiard.process._MainProcess = billiard.current_process() # pylint: disable=W0212, E1101 + prefork_id: int = current._identity[0] - 1 # pylint: disable=W0212 # range 0:nworkers-1 cpu_slot: int = (prefork_id * cpu_skip) % npu process.cpu_affinity(list(range(cpu_slot, cpu_slot + cpu_skip))) diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 1aa35c4a0..cc53e754f 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -106,12 +106,14 @@ def _get_npy_info2(f): hlen = hlen_char[0] + 256 * hlen_char[1] elif major == 2: hlen_char = list(map(ord, f.read(4))) + # fmt: off hlen = ( hlen_char[0] + 256 * hlen_char[1] + 65536 * hlen_char[2] + (1 << 24) * hlen_char[3] ) + # fmt: on else: raise Exception("unknown .npy format, e.g. not 1 or 2") hdr = eval(f.read(hlen)) # TODO remove eval @@ -135,12 +137,14 @@ def _get_npy_info3(f): hlen = hlen_char[0] + 256 * hlen_char[1] elif major == 2: hlen_char = list(f.read(4)) + # fmt: off hlen = ( hlen_char[0] + 256 * hlen_char[1] + 65536 * hlen_char[2] + (1 << 24) * hlen_char[3] ) + # fmt: on else: raise Exception("unknown .npy format, e.g. not 1 or 2") hdr = eval(f.read(hlen)) # TODO remove eval @@ -186,9 +190,7 @@ def read_rows(f, hdr, idx, n=-1, sep=""): if n < 0: n = hdr["shape"][0] - idx n = min(hdr["shape"][0] - idx, n) - a = np.fromfile( - f, dtype=hdr["dtype"], count=n * hdr["rowsize"] // hdr["itemsize"], sep=sep - ) + a = np.fromfile(f, dtype=hdr["dtype"], count=n * hdr["rowsize"] // hdr["itemsize"], sep=sep) return np.reshape(a, (n,) + hdr["shape"][1:]) @@ -216,9 +218,7 @@ def __init__(self, f): def _verify_open(self): if self.f is None: - self.f, self.hdr = _get_npy_info( - self.f if self.f is not None else self.fname - ) + self.f, self.hdr = _get_npy_info(self.f if self.f is not None else self.fname) @verify_open def load_header(self, close=True): @@ -261,10 +261,7 @@ def __getitem__(self, k): return read_rows(self.f, self.hdr, k.start, k.stop - k.start) else: return np.asarray( - [ - read_rows(self.f, self.hdr, _, 1)[0] - for _ in range(k.start, k.stop, 1 if k.step is None else k.step) - ] + [read_rows(self.f, self.hdr, _, 1)[0] for _ in range(k.start, k.stop, 1 if k.step is None else k.step)] ) @verify_open @@ -288,19 +285,13 @@ def __init__(self, filename_strs: List[str]): i: OpenNPY for i in self.files: i.load_header() - self.shapes: List[Tuple[int]] = [ - openNPY_obj.hdr["shape"] for openNPY_obj in self.files - ] + self.shapes: List[Tuple[int]] = [openNPY_obj.hdr["shape"] for openNPY_obj in self.files] k: Tuple[int] for k in self.shapes[1:]: # Match subsequent axes shapes. if k[1:] != self.shapes[0][1:]: - raise AttributeError( - f"Mismatch in subsequent axes shapes: {k[1:]} != {self.shapes[0][1:]}" - ) - self.tells: np.ndarray = np.cumsum( - [arr_shape[0] for arr_shape in self.shapes] - ) # Tell locations. + raise AttributeError(f"Mismatch in subsequent axes shapes: {k[1:]} != {self.shapes[0][1:]}") + self.tells: np.ndarray = np.cumsum([arr_shape[0] for arr_shape in self.shapes]) # Tell locations. self.tells = np.hstack(([0], self.tells)) def close(self): @@ -325,14 +316,7 @@ def __getitem__(self, k): return self.files[fno - 1][k - self.tells[fno - 1]] else: # Slice indexing. # TODO : Implement a faster version. - return np.asarray( - [ - self[_] - for _ in np.arange( - k.start, k.stop, k.step if k.step is not None else 1 - ) - ] - ) + return np.asarray([self[_] for _ in np.arange(k.start, k.stop, k.step if k.step is not None else 1)]) def to_array(self): return np.vstack([_.to_array() for _ in self.files]) diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index b932e4b24..27ee41c44 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -69,9 +69,7 @@ class SampleIndex: # Class variable to indicate depth (mostly used for pretty printing). depth = -1 - def __init__( - self, minid, maxid, children, name, leafid=-1, num_bundles=0, address="" - ): + def __init__(self, minid, maxid, children, name, leafid=-1, num_bundles=0, address=""): """The constructor.""" # The direct children of this node, generally also of type SampleIndex. @@ -140,9 +138,7 @@ def is_great_grandparent_of_leaf(self): return True return False - def traverse( - self, path=None, conditional=lambda c: True, bottom_up=True, top_level=True - ): + def traverse(self, path=None, conditional=lambda c: True, bottom_up=True, top_level=True): """ Yield the full path and associated node for each node that meets the conditional @@ -164,9 +160,7 @@ def traverse( for child_val in self.children.values(): child_path = os.path.join(path, child_val.name) - for node in child_val.traverse( - child_path, conditional, bottom_up=bottom_up, top_level=False - ): + for node in child_val.traverse(child_path, conditional, bottom_up=bottom_up, top_level=False): if node != "SKIP ME": yield node @@ -181,9 +175,7 @@ def traverse_all(self, bottom_up=True): """ Returns a generator that will traverse all nodes in the SampleIndex. """ - return self.traverse( - path=self.name, conditional=lambda c: True, bottom_up=bottom_up - ) + return self.traverse(path=self.name, conditional=lambda c: True, bottom_up=bottom_up) def traverse_bundles(self): """ @@ -197,9 +189,7 @@ def traverse_directories(self, bottom_up=False): Returns a generator that will traverse all Directories in the SampleIndex. """ - return self.traverse( - path=self.name, conditional=lambda c: c.is_directory, bottom_up=bottom_up - ) + return self.traverse(path=self.name, conditional=lambda c: c.is_directory, bottom_up=bottom_up) @staticmethod def check_valid_addresses_for_insertion(full_address, sub_tree): @@ -293,9 +283,7 @@ def write_multiple_sample_index_files(self, path="."): if filepath is not None: filepaths.append(filepath) for child_val in self.children.values(): - filepaths += child_val.write_multiple_sample_index_files( - os.path.join(path, self.name) - ) + filepaths += child_val.write_multiple_sample_index_files(os.path.join(path, self.name)) return filepaths def make_directory_string(self, delimiter=" ", just_leaf_directories=True): @@ -311,29 +299,25 @@ def make_directory_string(self, delimiter=" ", just_leaf_directories=True): "0/0 0/1 0/2 1/0 1/1 1/2" """ + # fmt: off if just_leaf_directories: return delimiter.join( [ - path - for path, node in self.traverse_directories() - if node.is_parent_of_leaf + path for path, node in self.traverse_directories() if node.is_parent_of_leaf ] ) + # fmt: on return delimiter.join([path for path, _ in self.traverse_directories()]) def __str__(self): """String representation.""" SampleIndex.depth = SampleIndex.depth + 1 if self.is_leaf: - result = ( - (" " * SampleIndex.depth) - + f"{self.address}: BUNDLE {self.leafid} MIN {self.min} MAX {self.max}\n" - ) + result = (" " * SampleIndex.depth) + f"{self.address}: BUNDLE {self.leafid} MIN {self.min} MAX {self.max}\n" else: result = ( - (" " * SampleIndex.depth) - + f"{self.address}: DIRECTORY MIN {self.min} MAX {self.max} NUM_BUNDLES {self.num_bundles}\n" - ) + " " * SampleIndex.depth + ) + f"{self.address}: DIRECTORY MIN {self.min} MAX {self.max} NUM_BUNDLES {self.num_bundles}\n" for child_val in self.children.values(): result += str(child_val) SampleIndex.depth = SampleIndex.depth - 1 diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 44a3c6b8c..46db3a537 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -150,9 +150,7 @@ def create_hierarchy_from_max_sample( bundle_id += 1 child_id += 1 num_bundles = bundle_id - start_bundle_id - return SampleIndex( - min_sample, max_sample, children, root, num_bundles=num_bundles, address=address - ) + return SampleIndex(min_sample, max_sample, children, root, num_bundles=num_bundles, address=address) def read_hierarchy(path): @@ -167,9 +165,7 @@ def read_hierarchy(path): with open("sample_index.txt", "r") as _file: token = _file.readline() while token: - parsed_token = parse( - "{type}:{ID}\tname:{name}\tsamples:[{min:d},{max:d})\n", token - ) + parsed_token = parse("{type}:{ID}\tname:{name}\tsamples:[{min:d},{max:d})\n", token) if parsed_token["type"] == "DIR": subhierarchy = read_hierarchy(parsed_token["name"]) subhierarchy.address = parsed_token["ID"] @@ -187,7 +183,5 @@ def read_hierarchy(path): min_sample = min(min_sample, parsed_token["min"]) max_sample = max(max_sample, parsed_token["max"]) token = _file.readline() - top_index = SampleIndex( - min_sample, max_sample, children, path, leafid=-1, num_bundles=num_bundles - ) + top_index = SampleIndex(min_sample, max_sample, children, path, leafid=-1, num_bundles=num_bundles) return top_index diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 714818736..c79c29049 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -42,17 +42,9 @@ from merlin.common.sample_index import uniform_directories from merlin.common.sample_index_factory import create_hierarchy from merlin.config.utils import Priority, get_priority -from merlin.exceptions import ( - HardFailException, - InvalidChainException, - RestartException, - RetryException, -) +from merlin.exceptions import HardFailException, InvalidChainException, RestartException, RetryException from merlin.router import stop_workers -from merlin.spec.expansion import ( - parameter_substitutions_for_cmd, - parameter_substitutions_for_sample, -) +from merlin.spec.expansion import parameter_substitutions_for_cmd, parameter_substitutions_for_sample from merlin.study.step import Step @@ -146,19 +138,13 @@ def merlin_step(self, *args: Any, **kwargs: Any) -> Optional[ReturnCode]: # noq ) result = ReturnCode.SOFT_FAIL elif result == ReturnCode.SOFT_FAIL: - LOG.warning( - f"*** Step '{step_name}' in '{step_dir}' soft failed. Continuing with workflow." - ) + LOG.warning(f"*** Step '{step_name}' in '{step_dir}' soft failed. Continuing with workflow.") elif result == ReturnCode.HARD_FAIL: # stop all workers attached to this queue step_queue = step.get_task_queue() - LOG.error( - f"*** Step '{step_name}' in '{step_dir}' hard failed. Quitting workflow." - ) - LOG.error( - f"*** Shutting down all workers connected to this queue ({step_queue}) in {STOP_COUNTDOWN} secs!" - ) + LOG.error(f"*** Step '{step_name}' in '{step_dir}' hard failed. Quitting workflow.") + LOG.error(f"*** Shutting down all workers connected to this queue ({step_queue}) in {STOP_COUNTDOWN} secs!") shutdown = shutdown_workers.s([step_queue]) shutdown.set(queue=step_queue) shutdown.apply_async(countdown=STOP_COUNTDOWN) @@ -170,9 +156,7 @@ def merlin_step(self, *args: Any, **kwargs: Any) -> Optional[ReturnCode]: # noq shutdown.set(queue=step.get_task_queue()) shutdown.apply_async(countdown=STOP_COUNTDOWN) else: - LOG.warning( - f"**** Step '{step_name}' in '{step_dir}' had unhandled exit code {result}. Continuing with workflow." - ) + LOG.warning(f"**** Step '{step_name}' in '{step_dir}' had unhandled exit code {result}. Continuing with workflow.") # queue off the next task in a chain while adding it to the current chord so that the chordfinisher actually # waits for the next task in the chain if next_in_chain is not None: @@ -271,8 +255,7 @@ def add_merlin_expanded_chain_to_chord( all_chains = [] LOG.debug(f"gathering up {len(samples)} relative paths") relative_paths = [ - os.path.dirname(sample_index.get_path_to_sample(sample_id + min_sample_id)) - for sample_id in range(len(samples)) + os.path.dirname(sample_index.get_path_to_sample(sample_id + min_sample_id)) for sample_id in range(len(samples)) ] LOG.debug(f"recursing grandparent with relative paths {relative_paths}") for step in chain_: @@ -286,9 +269,7 @@ def add_merlin_expanded_chain_to_chord( for sample_id, sample in enumerate(samples): new_step = task_type.s( step.clone_changing_workspace_and_cmd( - new_workspace=os.path.join( - workspace, relative_paths[sample_id] - ), + new_workspace=os.path.join(workspace, relative_paths[sample_id]), cmd_replacement_pairs=parameter_substitutions_for_sample( sample, labels, @@ -314,28 +295,20 @@ def add_merlin_expanded_chain_to_chord( next_step = add_merlin_expanded_chain_to_chord.s( task_type, chain_, - samples[ - next_index.min - min_sample_id : next_index.max - min_sample_id - ], + samples[next_index.min - min_sample_id : next_index.max - min_sample_id], labels, next_index, adapter_config, next_index.min, ) next_step.set(queue=chain_[0].get_task_queue()) - LOG.debug( - f"recursing with range {next_index.min}:{next_index.max}, {next_index.name} {signature(next_step)}" - ) - LOG.debug( - f"queuing samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}..." - ) + LOG.debug(f"recursing with range {next_index.min}:{next_index.max}, {next_index.name} {signature(next_step)}") + LOG.debug(f"queuing samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}...") if self.request.is_eager: next_step.delay() else: self.add_to_chord(next_step, lazy=False) - LOG.debug( - f"queued for samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}" - ) + LOG.debug(f"queued for samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}") return ReturnCode.OK @@ -356,11 +329,7 @@ def add_simple_chain_to_chord(self, task_type, chain_, adapter_config): # based off of the parameter substitutions and relative_path for # a given sample. - new_steps = [ - task_type.s(step, adapter_config=adapter_config).set( - queue=step.get_task_queue() - ) - ] + new_steps = [task_type.s(step, adapter_config=adapter_config).set(queue=step.get_task_queue())] all_chains.append(new_steps) add_chains_to_chord(self, all_chains) @@ -399,9 +368,11 @@ def add_chains_to_chord(self, all_chains): # kwargs. for g in reversed(range(len(all_chains))): if g < len(all_chains) - 1: + # fmt: off new_kwargs = signature(all_chains[g][i]).kwargs.update( {"next_in_chain": all_chains[g + 1][i]} ) + # fmt: on all_chains[g][i] = all_chains[g][i].replace(kwargs=new_kwargs) chain_steps.append(all_chains[0][i]) @@ -445,9 +416,7 @@ def expand_tasks_with_samples( """ LOG.debug(f"expand_tasks_with_samples called with chain,{chain_}\n") # Figure out how many directories there are, make a glob string - directory_sizes = uniform_directories( - len(samples), bundle_size=1, level_max_dirs=level_max_dirs - ) + directory_sizes = uniform_directories(len(samples), bundle_size=1, level_max_dirs=level_max_dirs) glob_path = "*/" * len(directory_sizes) @@ -471,11 +440,7 @@ def expand_tasks_with_samples( # sub in globs prior to expansion # sub the glob command steps = [ - step.clone_changing_workspace_and_cmd( - cmd_replacement_pairs=parameter_substitutions_for_cmd( - glob_path, sample_paths - ) - ) + step.clone_changing_workspace_and_cmd(cmd_replacement_pairs=parameter_substitutions_for_cmd(glob_path, sample_paths)) for step in steps ] @@ -499,9 +464,7 @@ def expand_tasks_with_samples( ] for condition in conditions: if not found_tasks: - for next_index_path, next_index in sample_index.traverse( - conditional=condition - ): + for next_index_path, next_index in sample_index.traverse(conditional=condition): LOG.info( f"generating next step for range {next_index.min}:{next_index.max} {next_index.max-next_index.min}" ) @@ -521,13 +484,9 @@ def expand_tasks_with_samples( if self.request.is_eager: sig.delay() else: - LOG.info( - f"queuing expansion task {next_index.min}:{next_index.max}" - ) + LOG.info(f"queuing expansion task {next_index.min}:{next_index.max}") self.add_to_chord(sig, lazy=False) - LOG.info( - f"merlin expansion task {next_index.min}:{next_index.max} queued" - ) + LOG.info(f"merlin expansion task {next_index.min}:{next_index.max} queued") found_tasks = True else: LOG.debug("queuing simple chain task") @@ -612,9 +571,7 @@ def queue_merlin_study(study, adapter): for gchain in chain_group ] ), - chordfinisher.s().set( - queue=egraph.step(chain_group[0][0]).get_task_queue() - ), + chordfinisher.s().set(queue=egraph.step(chain_group[0][0]).get_task_queue()), ) for chain_group in groups_of_chains[1:] ) diff --git a/merlin/config/broker.py b/merlin/config/broker.py index f841b0e1b..c3571d1df 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -92,9 +92,7 @@ def get_rabbit_connection(config_path, include_password, conn="amqps"): try: password = read_file(password_filepath) except IOError: - raise ValueError( - f"Broker: RabbitMQ password file {password_filepath} does not exist" - ) + raise ValueError(f"Broker: RabbitMQ password file {password_filepath} does not exist") try: port = CONFIG.broker.port diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 9ebad3007..34783443e 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -195,9 +195,7 @@ def get_cert_file(server_type, config, cert_name, cert_path): if os.path.exists(new_cert_file): cert_file = new_cert_file else: - LOG.error( - f"{server_type}: The file for {cert_name} does not exist, searched {cert_file} and {new_cert_file}" - ) + LOG.error(f"{server_type}: The file for {cert_name} does not exist, searched {cert_file} and {new_cert_file}") LOG.debug(f"{server_type}: {cert_name} = {cert_file}") except (AttributeError, KeyError): LOG.debug(f"{server_type}: {cert_name} not present") @@ -220,21 +218,15 @@ def get_ssl_entries( """ server_ssl: Dict[str, Union[str, ssl.VerifyMode]] = {} - keyfile: Optional[str] = get_cert_file( - server_type, server_config, "keyfile", cert_path - ) + keyfile: Optional[str] = get_cert_file(server_type, server_config, "keyfile", cert_path) if keyfile: server_ssl["keyfile"] = keyfile - certfile: Optional[str] = get_cert_file( - server_type, server_config, "certfile", cert_path - ) + certfile: Optional[str] = get_cert_file(server_type, server_config, "certfile", cert_path) if certfile: server_ssl["certfile"] = certfile - ca_certsfile: Optional[str] = get_cert_file( - server_type, server_config, "ca_certs", cert_path - ) + ca_certsfile: Optional[str] = get_cert_file(server_type, server_config, "ca_certs", cert_path) if ca_certsfile: server_ssl["ca_certs"] = ca_certsfile @@ -289,9 +281,7 @@ def process_ssl_map(server_name: str) -> Optional[Dict[str, str]]: return ssl_map -def merge_sslmap( - server_ssl: Dict[str, Union[str, ssl.VerifyMode]], ssl_map: Dict[str, str] -) -> Dict: +def merge_sslmap(server_ssl: Dict[str, Union[str, ssl.VerifyMode]], ssl_map: Dict[str, str]) -> Dict: """ The different servers have different key var expectations, this updates the keys of the ssl_server dict with keys from the ssl_map if using rediss or mysql. diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 6c3627e30..05c118a9f 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -59,12 +59,14 @@ } +# fmt: off MYSQL_CONNECTION_STRING = ( "db+mysql+mysqldb://{user}:{password}@{server}/mlsi" "?ssl_ca={ssl_ca}" "&ssl_cert={ssl_cert}" "&ssl_key={ssl_key}" ) +# fmt: on SQLITE_CONNECTION_STRING = "db+sqlite:///results.db" @@ -280,9 +282,7 @@ def _resolve_backend_string(backend, certs_path, include_password): return get_redis(certs_path=certs_path, include_password=include_password) elif backend == "rediss": - return get_redis( - certs_path=certs_path, include_password=include_password, ssl=True - ) + return get_redis(certs_path=certs_path, include_password=include_password, ssl=True) else: return None @@ -313,9 +313,7 @@ def get_ssl_config(celery_check=False): except AttributeError: certs_path = None - results_backend_ssl = get_ssl_entries( - "Results Backend", results_backend, CONFIG.results_backend, certs_path - ) + results_backend_ssl = get_ssl_entries("Results Backend", results_backend, CONFIG.results_backend, certs_path) if results_backend == "rediss": if not results_backend_ssl: diff --git a/merlin/config/utils.py b/merlin/config/utils.py index d26197025..12f7283bb 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -22,9 +22,7 @@ def get_priority(priority: Priority) -> int: broker: str = CONFIG.broker.name.lower() priorities: List[Priority] = [Priority.high, Priority.mid, Priority.low] if not isinstance(priority, Priority): - raise TypeError( - f"Unrecognized priority '{priority}'! Priority enum options: {[x.name for x in priorities]}" - ) + raise TypeError(f"Unrecognized priority '{priority}'! Priority enum options: {[x.name for x in priorities]}") if priority == Priority.mid: return 5 if is_rabbit_broker(broker): @@ -37,6 +35,4 @@ def get_priority(priority: Priority) -> int: return 10 if priority == Priority.high: return 1 - raise ValueError( - f"Function get_priority has reached unknown state! Maybe unsupported broker {broker}?" - ) + raise ValueError(f"Function get_priority has reached unknown state! Maybe unsupported broker {broker}?") diff --git a/merlin/display.py b/merlin/display.py index b8f080ab3..d2fabd364 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -97,9 +97,7 @@ def _examine_connection(s, sconf, excpts): counter += 1 if counter > connect_timeout: conn_check.kill() - raise Exception( - f"Connection was killed due to timeout ({connect_timeout}s)" - ) + raise Exception(f"Connection was killed due to timeout ({connect_timeout}s)") conn.release() if conn_check.exception: error, traceback = conn_check.exception @@ -131,9 +129,7 @@ def display_config_info(): excpts["broker server"] = e try: - conf["results server"] = results_backend.get_connection_string( - include_password=False - ) + conf["results server"] = results_backend.get_connection_string(include_password=False) sconf["results server"] = results_backend.get_connection_string() conf["results ssl"] = results_backend.get_ssl_config() except Exception as e: diff --git a/merlin/examples/workflows/feature_demo/scripts/hello_world.py b/merlin/examples/workflows/feature_demo/scripts/hello_world.py index efd275028..9d029bbf0 100644 --- a/merlin/examples/workflows/feature_demo/scripts/hello_world.py +++ b/merlin/examples/workflows/feature_demo/scripts/hello_world.py @@ -18,18 +18,10 @@ def process_args(args): def setup_argparse(): parser = argparse.ArgumentParser(description="Process some integers.") - parser.add_argument( - "X", metavar="X", type=float, help="The x dimension of the sample." - ) - parser.add_argument( - "Y", metavar="Y", type=float, help="The y dimension of the sample." - ) - parser.add_argument( - "Z", metavar="Z", type=float, help="The z dimension of the sample." - ) - parser.add_argument( - "-outfile", help="Output file name", default="hello_world_output.json" - ) + parser.add_argument("X", metavar="X", type=float, help="The x dimension of the sample.") + parser.add_argument("Y", metavar="Y", type=float, help="The y dimension of the sample.") + parser.add_argument("Z", metavar="Z", type=float, help="The z dimension of the sample.") + parser.add_argument("-outfile", help="Output file name", default="hello_world_output.json") return parser diff --git a/merlin/examples/workflows/hello/make_samples.py b/merlin/examples/workflows/hello/make_samples.py index bc2c7ca5e..3b7b5b398 100644 --- a/merlin/examples/workflows/hello/make_samples.py +++ b/merlin/examples/workflows/hello/make_samples.py @@ -6,9 +6,7 @@ # argument parsing parser = argparse.ArgumentParser(description="Make some samples (names of people).") -parser.add_argument( - "--number", type=int, action="store", help="the number of samples you want to make" -) +parser.add_argument("--number", type=int, action="store", help="the number of samples you want to make") parser.add_argument("--filepath", type=str, help="output file") args = parser.parse_args() diff --git a/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py b/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py index 6c7641edb..ea84f7d6f 100644 --- a/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py +++ b/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py @@ -37,29 +37,20 @@ def iter_df_from_json(json_file): def load_samples(sample_file_paths, nproc): """Loads all iterations' processed samples into a single pandas DataFrame in parallel""" with ProcessPoolExecutor(max_workers=nproc) as executor: - iter_dfs = [ - iter_frame - for iter_frame in executor.map(iter_df_from_json, sample_file_paths) - ] + iter_dfs = [iter_frame for iter_frame in executor.map(iter_df_from_json, sample_file_paths)] all_iter_df = pd.concat(iter_dfs) return all_iter_df def setup_argparse(): - parser = argparse.ArgumentParser( - description="Read in and analyze samples from plain text file" - ) + parser = argparse.ArgumentParser(description="Read in and analyze samples from plain text file") - parser.add_argument( - "sample_file_paths", help="paths to sample files", default="", nargs="+" - ) + parser.add_argument("sample_file_paths", help="paths to sample files", default="", nargs="+") parser.add_argument("--np", help="number of processors to use", type=int, default=1) - parser.add_argument( - "--hardcopy", help="Name of cumulative plot file", default="cum_results.png" - ) + parser.add_argument("--hardcopy", help="Name of cumulative plot file", default="cum_results.png") return parser @@ -89,9 +80,7 @@ def main(): min_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].min()) med_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].median()) - unique_names.append( - len(all_iter_df[all_iter_df["Iter"] <= it].index.value_counts()) - ) + unique_names.append(len(all_iter_df[all_iter_df["Iter"] <= it].index.value_counts())) ax[0].plot(iterations, min_counts, label="Minimum Occurances") ax[0].plot(iterations, max_counts, label="Maximum Occurances") diff --git a/merlin/examples/workflows/hpc_demo/faker_sample.py b/merlin/examples/workflows/hpc_demo/faker_sample.py index bea159f00..d6f7020e8 100644 --- a/merlin/examples/workflows/hpc_demo/faker_sample.py +++ b/merlin/examples/workflows/hpc_demo/faker_sample.py @@ -27,9 +27,7 @@ def setup_argparse(): parser = argparse.ArgumentParser("Generate some names!") parser.add_argument("-n", help="number of names", default=100, type=int) parser.add_argument("-b", help="number of batches of names", default=5, type=int) - parser.add_argument( - "-outfile", help="name of output .csv file", default="samples.csv" - ) + parser.add_argument("-outfile", help="name of output .csv file", default="samples.csv") return parser diff --git a/merlin/examples/workflows/hpc_demo/sample_collector.py b/merlin/examples/workflows/hpc_demo/sample_collector.py index 622aadc65..cfaac6e17 100644 --- a/merlin/examples/workflows/hpc_demo/sample_collector.py +++ b/merlin/examples/workflows/hpc_demo/sample_collector.py @@ -18,11 +18,7 @@ def load_samples(sample_file_path): def serialize_samples(sample_file_paths, outfile, nproc): """Writes out collection of samples, one entry per line""" with ProcessPoolExecutor(max_workers=nproc) as executor: - all_samples = [ - sample - for sample_list in executor.map(load_samples, sample_file_paths) - for sample in sample_list - ] + all_samples = [sample for sample_list in executor.map(load_samples, sample_file_paths) for sample in sample_list] with open(outfile, "w") as outfile: for sample in all_samples: @@ -30,19 +26,13 @@ def serialize_samples(sample_file_paths, outfile, nproc): def setup_argparse(): - parser = argparse.ArgumentParser( - description="Read in samples from list of output files" - ) + parser = argparse.ArgumentParser(description="Read in samples from list of output files") - parser.add_argument( - "sample_file_paths", help="paths to sample output files", default="", nargs="+" - ) + parser.add_argument("sample_file_paths", help="paths to sample output files", default="", nargs="+") parser.add_argument("--np", help="number of processors to use", type=int, default=1) - parser.add_argument( - "-outfile", help="Collected sample outputs", default="all_names.txt" - ) + parser.add_argument("-outfile", help="Collected sample outputs", default="all_names.txt") return parser diff --git a/merlin/examples/workflows/hpc_demo/sample_processor.py b/merlin/examples/workflows/hpc_demo/sample_processor.py index 41a0a53a2..ed4dd596b 100644 --- a/merlin/examples/workflows/hpc_demo/sample_processor.py +++ b/merlin/examples/workflows/hpc_demo/sample_processor.py @@ -19,17 +19,11 @@ def load_samples(sample_file_paths): def setup_argparse(): - parser = argparse.ArgumentParser( - description="Read in and analyze samples from plain text file" - ) + parser = argparse.ArgumentParser(description="Read in and analyze samples from plain text file") - parser.add_argument( - "sample_file_paths", help="paths to sample files", default="", nargs="+" - ) + parser.add_argument("sample_file_paths", help="paths to sample files", default="", nargs="+") - parser.add_argument( - "--results", help="Name of output json file", default="samples.json" - ) + parser.add_argument("--results", help="Name of output json file", default="samples.json") return parser diff --git a/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py b/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py index 6c7641edb..ea84f7d6f 100644 --- a/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py +++ b/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py @@ -37,29 +37,20 @@ def iter_df_from_json(json_file): def load_samples(sample_file_paths, nproc): """Loads all iterations' processed samples into a single pandas DataFrame in parallel""" with ProcessPoolExecutor(max_workers=nproc) as executor: - iter_dfs = [ - iter_frame - for iter_frame in executor.map(iter_df_from_json, sample_file_paths) - ] + iter_dfs = [iter_frame for iter_frame in executor.map(iter_df_from_json, sample_file_paths)] all_iter_df = pd.concat(iter_dfs) return all_iter_df def setup_argparse(): - parser = argparse.ArgumentParser( - description="Read in and analyze samples from plain text file" - ) + parser = argparse.ArgumentParser(description="Read in and analyze samples from plain text file") - parser.add_argument( - "sample_file_paths", help="paths to sample files", default="", nargs="+" - ) + parser.add_argument("sample_file_paths", help="paths to sample files", default="", nargs="+") parser.add_argument("--np", help="number of processors to use", type=int, default=1) - parser.add_argument( - "--hardcopy", help="Name of cumulative plot file", default="cum_results.png" - ) + parser.add_argument("--hardcopy", help="Name of cumulative plot file", default="cum_results.png") return parser @@ -89,9 +80,7 @@ def main(): min_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].min()) med_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].median()) - unique_names.append( - len(all_iter_df[all_iter_df["Iter"] <= it].index.value_counts()) - ) + unique_names.append(len(all_iter_df[all_iter_df["Iter"] <= it].index.value_counts())) ax[0].plot(iterations, min_counts, label="Minimum Occurances") ax[0].plot(iterations, max_counts, label="Maximum Occurances") diff --git a/merlin/examples/workflows/iterative_demo/faker_sample.py b/merlin/examples/workflows/iterative_demo/faker_sample.py index bea159f00..d6f7020e8 100644 --- a/merlin/examples/workflows/iterative_demo/faker_sample.py +++ b/merlin/examples/workflows/iterative_demo/faker_sample.py @@ -27,9 +27,7 @@ def setup_argparse(): parser = argparse.ArgumentParser("Generate some names!") parser.add_argument("-n", help="number of names", default=100, type=int) parser.add_argument("-b", help="number of batches of names", default=5, type=int) - parser.add_argument( - "-outfile", help="name of output .csv file", default="samples.csv" - ) + parser.add_argument("-outfile", help="name of output .csv file", default="samples.csv") return parser diff --git a/merlin/examples/workflows/iterative_demo/sample_collector.py b/merlin/examples/workflows/iterative_demo/sample_collector.py index 622aadc65..cfaac6e17 100644 --- a/merlin/examples/workflows/iterative_demo/sample_collector.py +++ b/merlin/examples/workflows/iterative_demo/sample_collector.py @@ -18,11 +18,7 @@ def load_samples(sample_file_path): def serialize_samples(sample_file_paths, outfile, nproc): """Writes out collection of samples, one entry per line""" with ProcessPoolExecutor(max_workers=nproc) as executor: - all_samples = [ - sample - for sample_list in executor.map(load_samples, sample_file_paths) - for sample in sample_list - ] + all_samples = [sample for sample_list in executor.map(load_samples, sample_file_paths) for sample in sample_list] with open(outfile, "w") as outfile: for sample in all_samples: @@ -30,19 +26,13 @@ def serialize_samples(sample_file_paths, outfile, nproc): def setup_argparse(): - parser = argparse.ArgumentParser( - description="Read in samples from list of output files" - ) + parser = argparse.ArgumentParser(description="Read in samples from list of output files") - parser.add_argument( - "sample_file_paths", help="paths to sample output files", default="", nargs="+" - ) + parser.add_argument("sample_file_paths", help="paths to sample output files", default="", nargs="+") parser.add_argument("--np", help="number of processors to use", type=int, default=1) - parser.add_argument( - "-outfile", help="Collected sample outputs", default="all_names.txt" - ) + parser.add_argument("-outfile", help="Collected sample outputs", default="all_names.txt") return parser diff --git a/merlin/examples/workflows/iterative_demo/sample_processor.py b/merlin/examples/workflows/iterative_demo/sample_processor.py index 41a0a53a2..ed4dd596b 100644 --- a/merlin/examples/workflows/iterative_demo/sample_processor.py +++ b/merlin/examples/workflows/iterative_demo/sample_processor.py @@ -19,17 +19,11 @@ def load_samples(sample_file_paths): def setup_argparse(): - parser = argparse.ArgumentParser( - description="Read in and analyze samples from plain text file" - ) + parser = argparse.ArgumentParser(description="Read in and analyze samples from plain text file") - parser.add_argument( - "sample_file_paths", help="paths to sample files", default="", nargs="+" - ) + parser.add_argument("sample_file_paths", help="paths to sample files", default="", nargs="+") - parser.add_argument( - "--results", help="Name of output json file", default="samples.json" - ) + parser.add_argument("--results", help="Name of output json file", default="samples.json") return parser diff --git a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py index 4b8671885..bb5ba8a4d 100644 --- a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py +++ b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py @@ -6,9 +6,7 @@ from typing import List -parser: argparse.ArgumentParser = argparse.ArgumentParser( - description="Launch 35 merlin workflow jobs" -) +parser: argparse.ArgumentParser = argparse.ArgumentParser(description="Launch 35 merlin workflow jobs") parser.add_argument("run_id", type=int, help="The ID of this run") parser.add_argument("output_path", type=str, help="the output path") parser.add_argument("spec_path", type=str, help="path to the spec to run") diff --git a/merlin/examples/workflows/null_spec/scripts/make_samples.py b/merlin/examples/workflows/null_spec/scripts/make_samples.py index b90ab0516..cf685c9d6 100644 --- a/merlin/examples/workflows/null_spec/scripts/make_samples.py +++ b/merlin/examples/workflows/null_spec/scripts/make_samples.py @@ -5,12 +5,8 @@ # argument parsing parser = argparse.ArgumentParser(description="Make some samples (names of people).") -parser.add_argument( - "--number", type=int, action="store", help="the number of samples you want to make" -) -parser.add_argument( - "--filepath", type=str, default="samples_file.npy", help="output file" -) +parser.add_argument("--number", type=int, action="store", help="the number of samples you want to make") +parser.add_argument("--filepath", type=str, default="samples_file.npy", help="output file") args = parser.parse_args() # sample making diff --git a/merlin/examples/workflows/null_spec/scripts/read_output.py b/merlin/examples/workflows/null_spec/scripts/read_output.py index 168667426..bc238d9d2 100644 --- a/merlin/examples/workflows/null_spec/scripts/read_output.py +++ b/merlin/examples/workflows/null_spec/scripts/read_output.py @@ -31,9 +31,7 @@ def single_task_times(): task_durations = [] for log in args.logfile: try: - pre_lines = subprocess.check_output( - f'grep " succeeded in " {log}', shell=True - ).decode("ascii") + pre_lines = subprocess.check_output(f'grep " succeeded in " {log}', shell=True).decode("ascii") pre_list = pre_lines.strip().split("\n") @@ -52,9 +50,7 @@ def single_task_times(): def merlin_run_time(): total = 0 for err in args.errfile: - pre_line = subprocess.check_output(f'grep "real" {err}', shell=True).decode( - "ascii" - ) + pre_line = subprocess.check_output(f'grep "real" {err}', shell=True).decode("ascii") pre_line = pre_line.strip() matches = re.search(r"\d\.\d\d\d", pre_line) match = matches[0] @@ -64,18 +60,14 @@ def merlin_run_time(): print(f"c{args.c}_s{args.s} merlin run : " + str(result)) except BaseException: result = None - print( - f"c{args.c}_s{args.s} merlin run : ERROR -- result={result}, args.errfile={args.errfile}" - ) + print(f"c{args.c}_s{args.s} merlin run : ERROR -- result={result}, args.errfile={args.errfile}") def start_verify_time(): all_timestamps = [] for log in args.logfile: try: - pre_line = subprocess.check_output( - f'grep -m1 "verify" {log}', shell=True - ).decode("ascii") + pre_line = subprocess.check_output(f'grep -m1 "verify" {log}', shell=True).decode("ascii") pre_line = pre_line.strip() matches = re.search(r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d", pre_line) match = matches[0] @@ -94,9 +86,7 @@ def start_run_workers_time(): all_timestamps = [] for log in args.logfile: try: - pre_line = subprocess.check_output(f'grep -m1 "" {log}', shell=True).decode( - "ascii" - ) + pre_line = subprocess.check_output(f'grep -m1 "" {log}', shell=True).decode("ascii") pre_line = pre_line.strip() matches = re.search(r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d", pre_line) match = matches[0] @@ -113,9 +103,7 @@ def start_sample1_time(): all_timestamps = [] for log in args.logfile: try: - pre_line = subprocess.check_output( - f"grep -m1 \"Executing step 'null_step'\" {log}", shell=True - ).decode("ascii") + pre_line = subprocess.check_output(f"grep -m1 \"Executing step 'null_step'\" {log}", shell=True).decode("ascii") pre_line = pre_line.strip() matches = re.search(r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d", pre_line) match = matches[0] diff --git a/merlin/examples/workflows/null_spec/scripts/read_output_chain.py b/merlin/examples/workflows/null_spec/scripts/read_output_chain.py index b7c22e7f2..f9fdfd521 100644 --- a/merlin/examples/workflows/null_spec/scripts/read_output_chain.py +++ b/merlin/examples/workflows/null_spec/scripts/read_output_chain.py @@ -43,9 +43,7 @@ def single_task_times(): for k, v in logmap.items(): task_durations = [] try: - pre_lines = subprocess.check_output( - f'grep " succeeded in " {v}', shell=True - ).decode("ascii") + pre_lines = subprocess.check_output(f'grep " succeeded in " {v}', shell=True).decode("ascii") pre_list = pre_lines.strip().split("\n") @@ -65,9 +63,7 @@ def single_task_times(): def merlin_run_time(): total = 0 for err in args.errfile: - pre_line = subprocess.check_output(f'grep "real" {err}', shell=True).decode( - "ascii" - ) + pre_line = subprocess.check_output(f'grep "real" {err}', shell=True).decode("ascii") pre_line = pre_line.strip() matches = re.search(r"\d\.\d\d\d", pre_line) match = matches[0] @@ -77,18 +73,14 @@ def merlin_run_time(): print(f"c{filled_c} merlin run : " + str(result)) except BaseException: result = None - print( - f"c{filled_c} merlin run : ERROR -- result={result}, args.errfile={args.errfile}" - ) + print(f"c{filled_c} merlin run : ERROR -- result={result}, args.errfile={args.errfile}") def start_verify_time(): for k, v in logmap.items(): all_timestamps = [] try: - pre_line = subprocess.check_output( - f'grep -m2 "verify" {v} | tail -n1', shell=True - ).decode("ascii") + pre_line = subprocess.check_output(f'grep -m2 "verify" {v} | tail -n1', shell=True).decode("ascii") pre_line = pre_line.strip() matches = re.search(r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d", pre_line) match = matches[0] @@ -108,9 +100,7 @@ def start_run_workers_time(): for k, v in logmap.items(): all_timestamps = [] try: - pre_line = subprocess.check_output(f'grep -m1 "" {v}', shell=True).decode( - "ascii" - ) + pre_line = subprocess.check_output(f'grep -m1 "" {v}', shell=True).decode("ascii") pre_line = pre_line.strip() matches = re.search(r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d", pre_line) match = matches[0] @@ -127,9 +117,7 @@ def start_sample1_time(): for k, v in logmap.items(): all_timestamps = [] try: - pre_line = subprocess.check_output( - f"grep -m1 \"Executing step 'null_step'\" {v}", shell=True - ).decode("ascii") + pre_line = subprocess.check_output(f"grep -m1 \"Executing step 'null_step'\" {v}", shell=True).decode("ascii") pre_line = pre_line.strip() matches = re.search(r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d", pre_line) match = matches[0] diff --git a/merlin/examples/workflows/openfoam_wf/scripts/combine_outputs.py b/merlin/examples/workflows/openfoam_wf/scripts/combine_outputs.py index 737071c49..2c23c840a 100644 --- a/merlin/examples/workflows/openfoam_wf/scripts/combine_outputs.py +++ b/merlin/examples/workflows/openfoam_wf/scripts/combine_outputs.py @@ -8,9 +8,7 @@ descript = """Using parameters to edit OpenFOAM parameters""" parser = argparse.ArgumentParser(description=descript) -parser.add_argument( - "-data", "--data_dir", help="The home directory of the data directories" -) +parser.add_argument("-data", "--data_dir", help="The home directory of the data directories") parser.add_argument("-merlin_paths", nargs="+", help="The path of all merlin runs") args = parser.parse_args() @@ -37,8 +35,6 @@ resolution = np.array(enstrophy).shape[-1] U = np.array(U).reshape(len(dir_names), num_of_timesteps, resolution, 3) -enstrophy = np.array(enstrophy).reshape( - len(dir_names), num_of_timesteps, resolution -) / float(resolution) +enstrophy = np.array(enstrophy).reshape(len(dir_names), num_of_timesteps, resolution) / float(resolution) np.savez("data.npz", U, enstrophy) diff --git a/merlin/examples/workflows/openfoam_wf/scripts/make_samples.py b/merlin/examples/workflows/openfoam_wf/scripts/make_samples.py index 075bba1a4..05b1f213f 100644 --- a/merlin/examples/workflows/openfoam_wf/scripts/make_samples.py +++ b/merlin/examples/workflows/openfoam_wf/scripts/make_samples.py @@ -22,8 +22,6 @@ def loguniform(low=-1, high=3, size=None, base=10): x[:, 0] = np.random.uniform(LIDSPEED_RANGE[0], LIDSPEED_RANGE[1], size=N_SAMPLES) vi_low = np.log10(x[:, 0] / REYNOLD_RANGE[1]) vi_high = np.log10(x[:, 0] / REYNOLD_RANGE[0]) -x[:, 1] = [ - loguniform(low=vi_low[i], high=vi_high[i], base=BASE) for i in range(N_SAMPLES) -] +x[:, 1] = [loguniform(low=vi_low[i], high=vi_high[i], base=BASE) for i in range(N_SAMPLES)] np.save(args.outfile, x) diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/combine_outputs.py b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/combine_outputs.py index 737071c49..2c23c840a 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/combine_outputs.py +++ b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/combine_outputs.py @@ -8,9 +8,7 @@ descript = """Using parameters to edit OpenFOAM parameters""" parser = argparse.ArgumentParser(description=descript) -parser.add_argument( - "-data", "--data_dir", help="The home directory of the data directories" -) +parser.add_argument("-data", "--data_dir", help="The home directory of the data directories") parser.add_argument("-merlin_paths", nargs="+", help="The path of all merlin runs") args = parser.parse_args() @@ -37,8 +35,6 @@ resolution = np.array(enstrophy).shape[-1] U = np.array(U).reshape(len(dir_names), num_of_timesteps, resolution, 3) -enstrophy = np.array(enstrophy).reshape( - len(dir_names), num_of_timesteps, resolution -) / float(resolution) +enstrophy = np.array(enstrophy).reshape(len(dir_names), num_of_timesteps, resolution) / float(resolution) np.savez("data.npz", U, enstrophy) diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/make_samples.py b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/make_samples.py index 075bba1a4..05b1f213f 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/make_samples.py +++ b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/make_samples.py @@ -22,8 +22,6 @@ def loguniform(low=-1, high=3, size=None, base=10): x[:, 0] = np.random.uniform(LIDSPEED_RANGE[0], LIDSPEED_RANGE[1], size=N_SAMPLES) vi_low = np.log10(x[:, 0] / REYNOLD_RANGE[1]) vi_high = np.log10(x[:, 0] / REYNOLD_RANGE[0]) -x[:, 1] = [ - loguniform(low=vi_low[i], high=vi_high[i], base=BASE) for i in range(N_SAMPLES) -] +x[:, 1] = [loguniform(low=vi_low[i], high=vi_high[i], base=BASE) for i in range(N_SAMPLES)] np.save(args.outfile, x) diff --git a/merlin/examples/workflows/optimization/scripts/optimizer.py b/merlin/examples/workflows/optimization/scripts/optimizer.py index c1bf9d797..74659c266 100644 --- a/merlin/examples/workflows/optimization/scripts/optimizer.py +++ b/merlin/examples/workflows/optimization/scripts/optimizer.py @@ -13,12 +13,8 @@ "-learner_dir", help="Learner directory (joblib file), usually '$(learner.workspace)'", ) -parser.add_argument( - "-bounds", help="ranges to scale results in form '[(min,max,type),(min, max,type)]'" -) -parser.add_argument( - "-input_uncerts", help="The standard deviation for each input dimension" -) +parser.add_argument("-bounds", help="ranges to scale results in form '[(min,max,type),(min, max,type)]'") +parser.add_argument("-input_uncerts", help="The standard deviation for each input dimension") parser.add_argument("-method", help="The optimizer method", default="trust-constr") args = parser.parse_args() @@ -66,9 +62,7 @@ def get_x_deltas(x_std=None, n_samples=500): return 0 else: x_cov = np.asarray(x_std) ** 2 - x_deltas = multivariate_normal.rvs( - np.zeros(x_cov.shape), cov=x_cov, size=n_samples - ) + x_deltas = multivariate_normal.rvs(np.zeros(x_cov.shape), cov=x_cov, size=n_samples) return x_deltas @@ -119,9 +113,7 @@ def prediction_percentile(x0, surrogate, x_deltas, percentile=50): best_f_x = existing_X[best_f_i] old_y_mean, old_y_std = mean_and_std(best_f_x, surrogate, x_deltas) -old_y_percentiles = prediction_percentile( - best_f_x, surrogate, x_deltas, eval_percents -).tolist() +old_y_percentiles = prediction_percentile(best_f_x, surrogate, x_deltas, eval_percents).tolist() delta_mults = [0.0, 0.5, 1.0, 1.5, 2.0] x_deltas_big = {} @@ -133,9 +125,7 @@ def prediction_percentile(x0, surrogate, x_deltas, percentile=50): x_samples_big = best_f_x + x_deltas_big[d] y_samples_big = surrogate.predict(x_samples_big) best_point_variations[d] = {} - best_point_variations[d]["percentiles"] = np.percentile( - y_samples_big, eval_percents - ).tolist() + best_point_variations[d]["percentiles"] = np.percentile(y_samples_big, eval_percents).tolist() counts, bins = np.histogram(y_samples_big, bins=20) best_point_variations[d]["histogram_counts"] = counts.tolist() @@ -156,9 +146,7 @@ def prediction_percentile(x0, surrogate, x_deltas, percentile=50): init_y = (start_f).tolist() opt_y_mean, opt_y_std = mean_and_std(optimum_func.x, surrogate, x_deltas) -opt_y_percentiles = prediction_percentile( - optimum_func.x, surrogate, x_deltas, eval_percents -).tolist() +opt_y_percentiles = prediction_percentile(optimum_func.x, surrogate, x_deltas, eval_percents).tolist() results = { "Results": { diff --git a/merlin/examples/workflows/optimization/scripts/visualizer.py b/merlin/examples/workflows/optimization/scripts/visualizer.py index 7fe44b317..3187ec643 100644 --- a/merlin/examples/workflows/optimization/scripts/visualizer.py +++ b/merlin/examples/workflows/optimization/scripts/visualizer.py @@ -8,9 +8,7 @@ plt.style.use("seaborn-white") parser = argparse.ArgumentParser("Learn surrogate model form simulation") -parser.add_argument( - "-study_dir", help="The study directory, usually '$(MERLIN_WORKSPACE)'" -) +parser.add_argument("-study_dir", help="The study directory, usually '$(MERLIN_WORKSPACE)'") args = parser.parse_args() study_dir = args.study_dir @@ -19,9 +17,7 @@ new_samples_path = f"{study_dir}/pick_new_inputs/new_samples.npy" new_exploit_samples_path = f"{study_dir}/pick_new_inputs/new_exploit_samples.npy" new_explore_samples_path = f"{study_dir}/pick_new_inputs/new_explore_samples.npy" -new_explore_star_samples_path = ( - f"{study_dir}/pick_new_inputs/new_explore_star_samples.npy" -) +new_explore_star_samples_path = f"{study_dir}/pick_new_inputs/new_explore_star_samples.npy" optimum_path = f"{study_dir}/optimizer/optimum.npy" old_best_path = f"{study_dir}/optimizer/old_best.npy" @@ -115,9 +111,7 @@ def Rosenbrock_mesh(): alpha=0.4, edgecolor="none", ) -ax.scatter( - existing_X[:, 0], existing_X[:, 1], np.clip(existing_y, -100, 100), marker="x" -) +ax.scatter(existing_X[:, 0], existing_X[:, 1], np.clip(existing_y, -100, 100), marker="x") ax.view_init(45, 45) ax.set_xlabel("DIM_1") ax.set_ylabel("DIM_2") @@ -134,9 +128,7 @@ def Rosenbrock_mesh(): alpha=0.4, edgecolor="none", ) -ax.scatter( - existing_X[:, 0], existing_X[:, 1], np.clip(existing_y, -100, 100), marker="x" -) +ax.scatter(existing_X[:, 0], existing_X[:, 1], np.clip(existing_y, -100, 100), marker="x") ax.view_init(45, 45) ax.set_xlabel("DIM_1") ax.set_ylabel("DIM_2") @@ -146,9 +138,7 @@ def Rosenbrock_mesh(): ax.scatter(existing_X[:, 0], existing_X[:, 1], label="Existing Inputs") ax.scatter(new_samples[:, 0], new_samples[:, 1], label="Suggested Inputs") ax.annotate("Predicted Optimum", xy=optimum, xytext=(1, 0), arrowprops=dict(width=0.01)) -ax.annotate( - "Current Best", xy=old_best, xytext=(-1.5, 1.5), arrowprops=dict(width=0.01) -) +ax.annotate("Current Best", xy=old_best, xytext=(-1.5, 1.5), arrowprops=dict(width=0.01)) ax.annotate("Actual Minimum", xy=(1, 1), xytext=(1.5, 2.5), arrowprops=dict(width=0.01)) ax.set_xlabel("DIM_1") @@ -158,12 +148,8 @@ def Rosenbrock_mesh(): ax.grid() ax = fig.add_subplot(3, 2, 6) -ax.scatter( - new_exploit_samples[:, 0], new_exploit_samples[:, 1], label="Exploit Samples" -) -ax.scatter( - new_explore_samples[:, 0], new_explore_samples[:, 1], label="Explore Samples" -) +ax.scatter(new_exploit_samples[:, 0], new_exploit_samples[:, 1], label="Exploit Samples") +ax.scatter(new_explore_samples[:, 0], new_explore_samples[:, 1], label="Explore Samples") ax.scatter( new_explore_star_samples[:, 0], new_explore_star_samples[:, 1], diff --git a/merlin/main.py b/merlin/main.py index fc6fa9804..03da04ffb 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -124,23 +124,15 @@ def parse_override_vars( for arg in variables_list: try: if "=" not in arg: - raise ValueError( - "--vars requires '=' operator. See 'merlin run --help' for an example." - ) + raise ValueError("--vars requires '=' operator. See 'merlin run --help' for an example.") entry: str = arg.split("=") if len(entry) != 2: - raise ValueError( - "--vars requires ONE '=' operator (without spaces) per variable assignment." - ) + raise ValueError("--vars requires ONE '=' operator (without spaces) per variable assignment.") key: str = entry[0] if key is None or key == "" or "$" in key: - raise ValueError( - "--vars requires valid variable names comprised of alphanumeric characters and underscores." - ) + raise ValueError("--vars requires valid variable names comprised of alphanumeric characters and underscores.") if key in RESERVED: - raise ValueError( - f"Cannot override reserved word '{key}'! Reserved words are: {RESERVED}." - ) + raise ValueError(f"Cannot override reserved word '{key}'! Reserved words are: {RESERVED}.") val: Union[str, int] = entry[1] with suppress(ValueError): @@ -182,9 +174,7 @@ def process_run(args: Namespace) -> None: # pgen checks if args.pargs and not args.pgen_file: - raise ValueError( - "Cannot use the 'pargs' parameter without specifying a 'pgen'!" - ) + raise ValueError("Cannot use the 'pargs' parameter without specifying a 'pgen'!") if args.pgen_file: verify_filepath(args.pgen_file) @@ -211,13 +201,9 @@ def process_restart(args: Namespace) -> None: filepath: str = os.path.join(args.restart_dir, "merlin_info", "*.expanded.yaml") possible_specs: Optional[List[str]] = glob.glob(filepath) if not possible_specs: # len == 0 - raise ValueError( - f"'{filepath}' does not match any provenance spec file to restart from." - ) + raise ValueError(f"'{filepath}' does not match any provenance spec file to restart from.") if len(possible_specs) > 1: - raise ValueError( - f"'{filepath}' matches more than one provenance spec file to restart from." - ) + raise ValueError(f"'{filepath}' matches more than one provenance spec file to restart from.") filepath: str = verify_filepath(possible_specs[0]) LOG.info(f"Restarting workflow at '{restart_dir}'") study: MerlinStudy = MerlinStudy(filepath, restart_dir=restart_dir) @@ -235,9 +221,7 @@ def launch_workers(args): spec, filepath = get_merlin_spec_with_override(args) if not args.worker_echo_only: LOG.info(f"Launching workers from '{filepath}'") - status = router.launch_workers( - spec, args.worker_steps, args.worker_args, args.worker_echo_only - ) + status = router.launch_workers(spec, args.worker_steps, args.worker_args, args.worker_echo_only) if args.worker_echo_only: print(status) else: @@ -301,9 +285,7 @@ def stop_workers(args): worker_names = spec.get_worker_names() for worker_name in worker_names: if "$" in worker_name: - LOG.warning( - f"Worker '{worker_name}' is unexpanded. Target provenance spec instead?" - ) + LOG.warning(f"Worker '{worker_name}' is unexpanded. Target provenance spec instead?") router.stop_workers(args.task_server, worker_names, args.queues, args.workers) @@ -381,8 +363,7 @@ def setup_argparse() -> None: dest="level", type=str, default=DEFAULT_LOG_LEVEL, - help="Set the log level. Options: DEBUG, INFO, WARNING, ERROR. " - "[Default: %(default)s]", + help="Set the log level. Options: DEBUG, INFO, WARNING, ERROR. [Default: %(default)s]", ) # merlin run @@ -392,9 +373,7 @@ def setup_argparse() -> None: formatter_class=ArgumentDefaultsHelpFormatter, ) run.set_defaults(func=process_run) - run.add_argument( - "specification", type=str, help="Path to a Merlin or Maestro YAML file" - ) + run.add_argument("specification", type=str, help="Path to a Merlin or Maestro YAML file") run.add_argument( "--local", action="store_const", @@ -461,9 +440,7 @@ def setup_argparse() -> None: formatter_class=ArgumentDefaultsHelpFormatter, ) restart.set_defaults(func=process_restart) - restart.add_argument( - "restart_dir", type=str, help="Path to an existing Merlin workspace directory" - ) + restart.add_argument("restart_dir", type=str, help="Path to an existing Merlin workspace directory") restart.add_argument( "--local", action="store_const", @@ -482,9 +459,7 @@ def setup_argparse() -> None: formatter_class=ArgumentDefaultsHelpFormatter, ) purge.set_defaults(func=purge_tasks) - purge.add_argument( - "specification", type=str, help="Path to a Merlin YAML spec file" - ) + purge.add_argument("specification", type=str, help="Path to a Merlin YAML spec file") purge.add_argument( "-f", "--force", @@ -560,8 +535,7 @@ def setup_argparse() -> None: action="store", type=str, default=None, - help="Specify a path to write the workflow to. Defaults to current " - "working directory", + help="Specify a path to write the workflow to. Defaults to current working directory", ) example.set_defaults(func=process_example) @@ -587,9 +561,7 @@ def generate_worker_touching_parsers(subparsers: ArgumentParser) -> None: formatter_class=ArgumentDefaultsHelpFormatter, ) run_workers.set_defaults(func=launch_workers) - run_workers.add_argument( - "specification", type=str, help="Path to a Merlin YAML spec file" - ) + run_workers.add_argument("specification", type=str, help="Path to a Merlin YAML spec file") run_workers.add_argument( "--worker-args", type=str, @@ -624,9 +596,7 @@ def generate_worker_touching_parsers(subparsers: ArgumentParser) -> None: ) # merlin query-workers - query: ArgumentParser = subparsers.add_parser( - "query-workers", help="List connected task server workers." - ) + query: ArgumentParser = subparsers.add_parser("query-workers", help="List connected task server workers.") query.set_defaults(func=query_workers) query.add_argument( "--task_server", @@ -637,9 +607,7 @@ def generate_worker_touching_parsers(subparsers: ArgumentParser) -> None: ) # merlin stop-workers - stop: ArgumentParser = subparsers.add_parser( - "stop-workers", help="Attempt to stop all task server workers." - ) + stop: ArgumentParser = subparsers.add_parser("stop-workers", help="Attempt to stop all task server workers.") stop.set_defaults(func=stop_workers) stop.add_argument( "--spec", @@ -654,9 +622,7 @@ def generate_worker_touching_parsers(subparsers: ArgumentParser) -> None: help="Task server type from which to stop workers.\ Default: %(default)s", ) - stop.add_argument( - "--queues", type=str, default=None, nargs="+", help="specific queues to stop" - ) + stop.add_argument("--queues", type=str, default=None, nargs="+", help="specific queues to stop") stop.add_argument( "--workers", type=str, @@ -670,9 +636,7 @@ def generate_worker_touching_parsers(subparsers: ArgumentParser) -> None: help="Check for active workers on an allocation.", formatter_class=RawTextHelpFormatter, ) - monitor.add_argument( - "specification", type=str, help="Path to a Merlin YAML spec file" - ) + monitor.add_argument("specification", type=str, help="Path to a Merlin YAML spec file") monitor.add_argument( "--steps", nargs="+", @@ -721,9 +685,7 @@ def generate_diagnostic_parsers(subparsers: ArgumentParser) -> None: number of connected workers) for a workflow spec.", ) status.set_defaults(func=query_status) - status.add_argument( - "specification", type=str, help="Path to a Merlin YAML spec file" - ) + status.add_argument("specification", type=str, help="Path to a Merlin YAML spec file") status.add_argument( "--steps", nargs="+", @@ -749,9 +711,7 @@ def generate_diagnostic_parsers(subparsers: ArgumentParser) -> None: help="Specify desired Merlin variable values to override those found in the specification. Space-delimited. " "Example: '--vars LEARN=path/to/new_learn.py EPOCHS=3'", ) - status.add_argument( - "--csv", type=str, help="csv file to dump status report to", default=None - ) + status.add_argument("--csv", type=str, help="csv file to dump status report to", default=None) # merlin info info: ArgumentParser = subparsers.add_parser( diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index d906796cd..6fc325184 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -44,9 +44,7 @@ def process_templates(args): - LOG.error( - "The command `merlin-templates` has been deprecated in favor of `merlin example`." - ) + LOG.error("The command `merlin-templates` has been deprecated in favor of `merlin example`.") def setup_argparse(): diff --git a/merlin/router.py b/merlin/router.py index ea2e9ee2c..b65f6cbe4 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -257,13 +257,9 @@ def check_merlin_status(args, spec): while count < max_count: # This list will include strings comprised of the worker name with the hostname e.g. worker_name@host. worker_status = get_workers(args.task_server) - LOG.info( - f"Monitor: checking for workers, running workers = {worker_status} ..." - ) + LOG.info(f"Monitor: checking for workers, running workers = {worker_status} ...") - check = any( - any(iwn in iws for iws in worker_status) for iwn in worker_names - ) + check = any(any(iwn in iws for iws in worker_status) for iwn in worker_names) if check: break diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 128eb679b..f03b17382 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -81,12 +81,14 @@ def expand_line(line, var_dict, env_vars=False): Expand one line of text by substituting user variables, optionally environment variables, as well as variables in 'var_dict'. """ + # fmt: off if ( (not contains_token(line)) and (not contains_shell_ref(line)) and ("~" not in line) ): return line + # fmt: on for key, val in var_dict.items(): if key in line: line = line.replace(var_ref(key), str(val)) @@ -159,25 +161,19 @@ def determine_user_variables(*user_var_dicts): determined_results = {} for key, val in all_var_dicts.items(): if key in RESERVED: - raise ValueError( - f"Cannot reassign value of reserved word '{key}'! Reserved words are: {RESERVED}." - ) + raise ValueError(f"Cannot reassign value of reserved word '{key}'! Reserved words are: {RESERVED}.") new_val = str(val) if contains_token(new_val): for determined_key in determined_results.keys(): var_determined_key = var_ref(determined_key) if var_determined_key in new_val: - new_val = new_val.replace( - var_determined_key, determined_results[determined_key] - ) + new_val = new_val.replace(var_determined_key, determined_results[determined_key]) new_val = expandvars(expanduser(new_val)) determined_results[key.upper()] = new_val return determined_results -def parameter_substitutions_for_sample( - sample, labels, sample_id, relative_path_to_sample -): +def parameter_substitutions_for_sample(sample, labels, sample_id, relative_path_to_sample): """ :param sample : The sample to do substitution for. :param labels : The column labels of the sample. diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 01bd3e7ea..538f895e8 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -14,9 +14,7 @@ def error_override_vars(override_vars, spec_filepath): original_text = open(spec_filepath, "r").read() for variable in override_vars.keys(): if variable not in original_text: - raise ValueError( - f"Command line override variable '{variable}' not found in spec file '{spec_filepath}'." - ) + raise ValueError(f"Command line override variable '{variable}' not found in spec file '{spec_filepath}'.") def replace_override_vars(env, override_vars): diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 59446b2ec..714a7d30a 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -144,9 +144,7 @@ def process_spec_defaults(self): MerlinSpec.fill_missing_defaults(self.environment, defaults.ENV["env"]) # fill in missing global parameter section defaults - MerlinSpec.fill_missing_defaults( - self.globals, defaults.PARAMETER["global.parameters"] - ) + MerlinSpec.fill_missing_defaults(self.globals, defaults.PARAMETER["global.parameters"]) # fill in missing step section defaults within 'run' defaults.STUDY_STEP_RUN["shell"] = self.batch["shell"] @@ -176,12 +174,14 @@ def recurse(result, defaults): if not isinstance(defaults, dict): return for key, val in defaults.items(): + # fmt: off if (key not in result) or ( (result[key] is None) and (defaults[key] is not None) ): result[key] = val else: recurse(result[key], val) + # fmt: on recurse(object_to_update, default_dict) @@ -202,31 +202,21 @@ def warn_unrecognized_keys(self): # check steps for step in self.study: MerlinSpec.check_section(step["name"], step, all_keys.STUDY_STEP) - MerlinSpec.check_section( - step["name"] + ".run", step["run"], all_keys.STUDY_STEP_RUN - ) + MerlinSpec.check_section(step["name"] + ".run", step["run"], all_keys.STUDY_STEP_RUN) # check merlin MerlinSpec.check_section("merlin", self.merlin, all_keys.MERLIN) - MerlinSpec.check_section( - "merlin.resources", self.merlin["resources"], all_keys.MERLIN_RESOURCES - ) + MerlinSpec.check_section("merlin.resources", self.merlin["resources"], all_keys.MERLIN_RESOURCES) for worker, contents in self.merlin["resources"]["workers"].items(): - MerlinSpec.check_section( - "merlin.resources.workers " + worker, contents, all_keys.WORKER - ) + MerlinSpec.check_section("merlin.resources.workers " + worker, contents, all_keys.WORKER) if self.merlin["samples"]: - MerlinSpec.check_section( - "merlin.samples", self.merlin["samples"], all_keys.SAMPLES - ) + MerlinSpec.check_section("merlin.samples", self.merlin["samples"], all_keys.SAMPLES) @staticmethod def check_section(section_name, section, all_keys): diff = set(section.keys()).difference(all_keys) for extra in diff: - LOG.warn( - f"Unrecognized key '{extra}' found in spec section '{section_name}'." - ) + LOG.warn(f"Unrecognized key '{extra}' found in spec section '{section_name}'.") def dump(self): """ @@ -287,16 +277,9 @@ def _process_dict_or_list(self, obj, string, key_stack, lvl, tab): key_stack = deepcopy(key_stack) key_stack.append("elem") if use_hyphens: - string += ( - (lvl + 1) * tab - + "- " - + str(self._dict_to_yaml(elem, "", key_stack, tab)) - + "\n" - ) + string += (lvl + 1) * tab + "- " + str(self._dict_to_yaml(elem, "", key_stack, tab)) + "\n" else: - string += str( - self._dict_to_yaml(elem, "", key_stack, tab, newline=(i != 0)) - ) + string += str(self._dict_to_yaml(elem, "", key_stack, tab, newline=(i != 0))) if n > 1 and i != len(obj) - 1: string += ", " key_stack.pop() @@ -317,12 +300,7 @@ def _process_dict_or_list(self, obj, string, key_stack, lvl, tab): string += list_offset + (tab * lvl) else: string += tab * (lvl + 1) - string += ( - str(k) - + ": " - + str(self._dict_to_yaml(v, "", key_stack, tab)) - + "\n" - ) + string += str(k) + ": " + str(self._dict_to_yaml(v, "", key_stack, tab)) + "\n" key_stack.pop() i += 1 return string @@ -357,9 +335,7 @@ def get_queue_list(self, steps): task_queues = [queues[steps]] except KeyError: nl = "\n" - LOG.error( - f"Invalid steps '{steps}'! Try one of these (or 'all'):\n{nl.join(queues.keys())}" - ) + LOG.error(f"Invalid steps '{steps}'! Try one of these (or 'all'):\n{nl.join(queues.keys())}") raise return sorted(set(task_queues)) diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 00736c696..8e6648638 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -149,9 +149,7 @@ def batch_worker_launch( if nodes is None or nodes == "all": nodes = get_node_count(default=1) elif not isinstance(nodes, int): - raise TypeError( - "Nodes was passed into batch_worker_launch with an invalid type (likely a string other than 'all')." - ) + raise TypeError("Nodes was passed into batch_worker_launch with an invalid type (likely a string other than 'all').") shell: str = get_yaml_var(batch, "shell", "bash") @@ -173,9 +171,7 @@ def batch_worker_launch( if btype == "flux": flux_path: str = get_yaml_var(batch, "flux_path", "") flux_opts: Union[str, Dict] = get_yaml_var(batch, "flux_start_opts", "") - flux_exec_workers: Union[str, Dict, bool] = get_yaml_var( - batch, "flux_exec_workers", True - ) + flux_exec_workers: Union[str, Dict, bool] = get_yaml_var(batch, "flux_exec_workers", True) flux_exec: str = "" if flux_exec_workers: @@ -194,9 +190,7 @@ def batch_worker_launch( return worker_cmd -def construct_worker_launch_command( - batch: Optional[Dict], btype: str, nodes: int -) -> str: +def construct_worker_launch_command(batch: Optional[Dict], btype: str, nodes: int) -> str: """ If no 'worker_launch' is found in the batch yaml, this method constructs the needed launch command. diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 12ce3911f..ead61ebe8 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -39,13 +39,7 @@ from contextlib import suppress from merlin.study.batch import batch_check_parallel, batch_worker_launch -from merlin.utils import ( - check_machines, - get_procs, - get_yaml_var, - is_running, - regex_list_filter, -) +from merlin.utils import check_machines, get_procs, get_yaml_var, is_running, regex_list_filter LOG = logging.getLogger(__name__) @@ -248,15 +242,11 @@ def start_celery_workers(spec, steps, celery_args, just_return_command): worker_args += " --logfile %p.%i" # Get the celery command - celery_com = launch_celery_workers( - spec, steps=wsteps, worker_args=worker_args, just_return_command=True - ) + celery_com = launch_celery_workers(spec, steps=wsteps, worker_args=worker_args, just_return_command=True) celery_cmd = os.path.expandvars(celery_com) - worker_cmd = batch_worker_launch( - spec, celery_cmd, nodes=worker_nodes, batch=worker_batch - ) + worker_cmd = batch_worker_launch(spec, celery_cmd, nodes=worker_nodes, batch=worker_batch) worker_cmd = os.path.expandvars(worker_cmd) @@ -324,13 +314,10 @@ def examine_and_log_machines(worker_val, yenv) -> bool: output_path = get_yaml_var(yenv, "OUTPUT_PATH", None) if output_path and not os.path.exists(output_path): hostname = socket.gethostname() - LOG.error( - f"The output path, {output_path}, is not accessible on this host, {hostname}" - ) + LOG.error(f"The output path, {output_path}, is not accessible on this host, {hostname}") else: LOG.warning( - "The env:variables section does not have an OUTPUT_PATH" - "specified, multi-machine checks cannot be performed." + "The env:variables section does not have an OUTPUT_PATH specified, multi-machine checks cannot be performed." ) return False @@ -340,19 +327,11 @@ def verify_args(spec, worker_args, worker_name, overlap): parallel = batch_check_parallel(spec) if parallel: if "--concurrency" not in worker_args: - LOG.warning( - "The worker arg --concurrency [1-4] is recommended " - "when running parallel tasks" - ) + LOG.warning("The worker arg --concurrency [1-4] is recommended when running parallel tasks") if "--prefetch-multiplier" not in worker_args: - LOG.warning( - "The worker arg --prefetch-multiplier 1 is " - "recommended when running parallel tasks" - ) + LOG.warning("The worker arg --prefetch-multiplier 1 is recommended when running parallel tasks") if "fair" not in worker_args: - LOG.warning( - "The worker arg -O fair is recommended when running parallel tasks" - ) + LOG.warning("The worker arg -O fair is recommended when running parallel tasks") if "-n" not in worker_args: nhash = "" @@ -423,9 +402,7 @@ def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): """ from merlin.celery import app - LOG.debug( - f"Sending stop to queues: {queues}, worker_regex: {worker_regex}, spec_worker_names: {spec_worker_names}" - ) + LOG.debug(f"Sending stop to queues: {queues}, worker_regex: {worker_regex}, spec_worker_names: {spec_worker_names}") active_queues, _ = get_queues(app) # If not specified, get all the queues @@ -447,20 +424,14 @@ def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): print(f"all_workers: {all_workers}") print(f"spec_worker_names: {spec_worker_names}") - if ( - spec_worker_names is None or len(spec_worker_names) == 0 - ) and worker_regex is None: + if (spec_worker_names is None or len(spec_worker_names) == 0) and worker_regex is None: workers_to_stop = list(all_workers) else: workers_to_stop = [] if (spec_worker_names is not None) and len(spec_worker_names) > 0: for worker_name in spec_worker_names: - print( - f"Result of regex_list_filter: {regex_list_filter(worker_name, all_workers)}" - ) - workers_to_stop += regex_list_filter( - worker_name, all_workers, match=False - ) + print(f"Result of regex_list_filter: {regex_list_filter(worker_name, all_workers)}") + workers_to_stop += regex_list_filter(worker_name, all_workers, match=False) if worker_regex is not None: workers_to_stop += regex_list_filter(worker_regex, workers_to_stop) diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 22f3d7629..067334fcb 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -169,9 +169,7 @@ def compatible_merlin_expansion(self, task1, task2): """ step1 = self.step(task1) step2 = self.step(task2) - return step1.needs_merlin_expansion( - self.labels - ) == step2.needs_merlin_expansion(self.labels) + return step1.needs_merlin_expansion(self.labels) == step2.needs_merlin_expansion(self.labels) def find_independent_chains(self, list_of_groups_of_chains): """ @@ -206,16 +204,11 @@ def find_independent_chains(self, list_of_groups_of_chains): if self.compatible_merlin_expansion(child, task_name): - self.find_chain(child, list_of_groups_of_chains).remove( - child - ) + self.find_chain(child, list_of_groups_of_chains).remove(child) chain.append(child) - new_list = [ - [chain for chain in group if len(chain) > 0] - for group in list_of_groups_of_chains - ] + new_list = [[chain for chain in group if len(chain) > 0] for group in list_of_groups_of_chains] new_list_2 = [group for group in new_list if len(group) > 0] return new_list_2 diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 0a8d367ea..c4f47e8c7 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -357,9 +357,7 @@ def __init__(self, **kwargs): # Using super prevents recursion. self.batch_adapter = super(MerlinScriptAdapter, self) if self.batch_type != "merlin-local": - self.batch_adapter = MerlinScriptAdapterFactory.get_adapter( - self.batch_type - )(**kwargs) + self.batch_adapter = MerlinScriptAdapterFactory.get_adapter(self.batch_type)(**kwargs) def write_script(self, *args, **kwargs): """ @@ -404,9 +402,7 @@ def submit(self, step, path, cwd, job_map=None, env=None): elif retcode == ReturnCode.STOP_WORKERS: LOG.debug("Execution returned status STOP_WORKERS") else: - LOG.warning( - f"Unrecognized Merlin Return code: {retcode}, returning SOFT_FAIL" - ) + LOG.warning(f"Unrecognized Merlin Return code: {retcode}, returning SOFT_FAIL") submission_record._info["retcode"] = retcode retcode = ReturnCode.SOFT_FAIL @@ -417,9 +413,7 @@ def submit(self, step, path, cwd, job_map=None, env=None): return submission_record - def _execute_subprocess( - self, output_name, script_path, cwd, env=None, join_output=False - ): + def _execute_subprocess(self, output_name, script_path, cwd, env=None, join_output=False): """ Execute the subprocess script locally. If cwd is specified, the submit method will operate outside of the path @@ -437,9 +431,7 @@ def _execute_subprocess( """ script_bn = os.path.basename(script_path) new_output_name = os.path.splitext(script_bn)[0] - LOG.debug( - f"script_path={script_path}, output_name={output_name}, new_output_name={new_output_name}" - ) + LOG.debug(f"script_path={script_path}, output_name={output_name}, new_output_name={new_output_name}") p = start_process(script_path, shell=False, cwd=cwd, env=env) pid = p.pid output, err = p.communicate() diff --git a/merlin/study/step.py b/merlin/study/step.py index bdde0a9b4..adac133ca 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -56,16 +56,13 @@ def __init__(self, workspace, step, **kwargs): def mark_submitted(self): """Mark the submission time of the record.""" - LOG.debug( - "Marking %s as submitted (PENDING) -- previously %s", self.name, self.status - ) + LOG.debug("Marking %s as submitted (PENDING) -- previously %s", self.name, self.status) self.status = State.PENDING if not self._submit_time: self._submit_time = datetime.now() else: LOG.debug( - "Merlin: Cannot set the submission time of '%s' because it has " - "already been set.", + "Merlin: Cannot set the submission time of '%s' because it has already been set.", self.name, ) @@ -95,9 +92,7 @@ def get_restart_cmd(self): """ return self.mstep.step.__dict__["run"]["restart"] - def clone_changing_workspace_and_cmd( - self, new_cmd=None, cmd_replacement_pairs=None, new_workspace=None - ): + def clone_changing_workspace_and_cmd(self, new_cmd=None, cmd_replacement_pairs=None, new_workspace=None): """ Produces a deep copy of the current step, performing variable substitutions as we go @@ -120,9 +115,7 @@ def clone_changing_workspace_and_cmd( restart_cmd = step_dict["run"]["restart"] if restart_cmd: - step_dict["run"]["restart"] = re.sub( - re.escape(str1), str2, restart_cmd, flags=re.I - ) + step_dict["run"]["restart"] = re.sub(re.escape(str1), str2, restart_cmd, flags=re.I) if new_workspace is None: new_workspace = self.get_workspace() diff --git a/merlin/study/study.py b/merlin/study/study.py index ad590d79e..487b69d37 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -43,21 +43,11 @@ from merlin.common.abstracts.enums import ReturnCode from merlin.spec import defaults -from merlin.spec.expansion import ( - determine_user_variables, - expand_by_line, - expand_env_vars, - expand_line, -) +from merlin.spec.expansion import determine_user_variables, expand_by_line, expand_env_vars, expand_line from merlin.spec.override import error_override_vars, replace_override_vars from merlin.spec.specification import MerlinSpec from merlin.study.dag import DAG -from merlin.utils import ( - contains_shell_ref, - contains_token, - get_flux_cmd, - load_array_file, -) +from merlin.utils import contains_shell_ref, contains_token, get_flux_cmd, load_array_file LOG = logging.getLogger(__name__) @@ -142,9 +132,7 @@ def label_clash_error(self): if self.original_spec.merlin["samples"]: for label in self.original_spec.merlin["samples"]["column_labels"]: if label in self.original_spec.globals: - raise ValueError( - f"column_label {label} cannot also be " "in global.parameters!" - ) + raise ValueError(f"column_label {label} cannot also be in global.parameters!") @staticmethod def get_user_vars(spec): @@ -169,16 +157,12 @@ def get_expanded_spec(self): Useful for provenance. """ # get specification including defaults and cli-overridden user variables - new_env = replace_override_vars( - self.original_spec.environment, self.override_vars - ) + new_env = replace_override_vars(self.original_spec.environment, self.override_vars) new_spec = deepcopy(self.original_spec) new_spec.environment = new_env # expand user variables - new_spec_text = expand_by_line( - new_spec.dump(), MerlinStudy.get_user_vars(new_spec) - ) + new_spec_text = expand_by_line(new_spec.dump(), MerlinStudy.get_user_vars(new_spec)) # expand reserved words new_spec_text = expand_by_line(new_spec_text, self.special_vars) @@ -288,9 +272,7 @@ def output_path(self): else: output_path = str(self.original_spec.output_path) - if (self.override_vars is not None) and ( - "OUTPUT_PATH" in self.override_vars - ): + if (self.override_vars is not None) and ("OUTPUT_PATH" in self.override_vars): output_path = str(self.override_vars["OUTPUT_PATH"]) output_path = expand_line(output_path, self.user_vars, env_vars=True) @@ -321,9 +303,7 @@ def workspace(self): """ if self.restart_dir is not None: if not os.path.isdir(self.restart_dir): - raise ValueError( - f"Restart directory '{self.restart_dir}' does not exist!" - ) + raise ValueError(f"Restart directory '{self.restart_dir}' does not exist!") return os.path.abspath(self.restart_dir) workspace_name = f'{self.original_spec.name.replace(" ", "_")}_{self.timestamp}' @@ -362,26 +342,20 @@ def expanded_spec(self): expanded_filepath = os.path.join(self.info, expanded_name) # expand provenance spec filename - if contains_token(self.original_spec.name) or contains_shell_ref( - self.original_spec.name - ): + if contains_token(self.original_spec.name) or contains_shell_ref(self.original_spec.name): name = f"{result.description['name'].replace(' ', '_')}_{self.timestamp}" name = expand_line(name, {}, env_vars=True) if "/" in name: - raise ValueError( - f"Expanded value '{name}' for field 'name' in section 'description' is not a valid filename." - ) + raise ValueError(f"Expanded value '{name}' for field 'name' in section 'description' is not a valid filename.") expanded_workspace = os.path.join(self.output_path, name) if result.merlin["samples"]: sample_file = result.merlin["samples"]["file"] if sample_file.startswith(self.workspace): - new_samples_file = sample_file.replace( + new_samples_file = sample_file.replace(self.workspace, expanded_workspace) + result.merlin["samples"]["generate"]["cmd"] = result.merlin["samples"]["generate"]["cmd"].replace( self.workspace, expanded_workspace ) - result.merlin["samples"]["generate"]["cmd"] = result.merlin[ - "samples" - ]["generate"]["cmd"].replace(self.workspace, expanded_workspace) result.merlin["samples"]["file"] = new_samples_file shutil.move(self.workspace, expanded_workspace) @@ -390,9 +364,7 @@ def expanded_spec(self): self.special_vars["MERLIN_INFO"] = self.info expanded_filepath = os.path.join(self.info, expanded_name) - new_spec_text = expand_by_line( - result.dump(), MerlinStudy.get_user_vars(result) - ) + new_spec_text = expand_by_line(result.dump(), MerlinStudy.get_user_vars(result)) result = MerlinSpec.load_spec_from_string(new_spec_text) result = expand_env_vars(result) @@ -458,9 +430,7 @@ def generate_samples(self): """ try: if not os.path.exists(self.samples_file): - sample_generate = self.expanded_spec.merlin["samples"]["generate"][ - "cmd" - ] + sample_generate = self.expanded_spec.merlin["samples"]["generate"]["cmd"] LOG.info("Generating samples...") sample_process = subprocess.Popen( sample_generate, diff --git a/merlin/utils.py b/merlin/utils.py index c1914735c..19c840913 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -79,11 +79,7 @@ def get_user_process_info(user=None, attrs=None): if user == "all_users": return [p.info for p in psutil.process_iter(attrs=attrs)] else: - return [ - p.info - for p in psutil.process_iter(attrs=attrs) - if user in p.info["username"] - ] + return [p.info for p in psutil.process_iter(attrs=attrs) if user in p.info["username"]] def check_pid(pid, user=None): @@ -154,9 +150,7 @@ def is_running(name, all_users=False): cmd[1] = "aux" try: - ps = subprocess.Popen( - cmd, stdout=subprocess.PIPE, encoding="utf8" - ).communicate()[0] + ps = subprocess.Popen(cmd, stdout=subprocess.PIPE, encoding="utf8").communicate()[0] except TypeError: ps = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] @@ -380,9 +374,7 @@ def get_flux_version(flux_path, no_errors=False): ps = None try: - ps = subprocess.Popen( - cmd, stdout=subprocess.PIPE, encoding="utf8" - ).communicate() + ps = subprocess.Popen(cmd, stdout=subprocess.PIPE, encoding="utf8").communicate() except FileNotFoundError as e: if not no_errors: LOG.error(f"The flux path {flux_path} canot be found") @@ -472,9 +464,7 @@ def convert_to_timedelta(timestr: Union[str, int]) -> timedelta: timestr = str(timestr) nfields = len(timestr.split(":")) if nfields > 4: - raise ValueError( - f"Cannot convert {timestr} to a timedelta. Valid format: days:hours:minutes:seconds." - ) + raise ValueError(f"Cannot convert {timestr} to a timedelta. Valid format: days:hours:minutes:seconds.") _, d, h, m, s = (":0" * 10 + timestr).rsplit(":", 4) tdelta = timedelta(days=int(d), hours=int(h), minutes=int(m), seconds=int(s)) return tdelta @@ -508,9 +498,7 @@ def repr_timedelta(td: timedelta, method: str = "HMS") -> str: elif method == "FSD": return _repr_timedelta_FSD(td) else: - raise ValueError( - "Invalid method for formatting timedelta! Valid choices: HMS, FSD" - ) + raise ValueError("Invalid method for formatting timedelta! Valid choices: HMS, FSD") def convert_timestring(timestring: Union[str, int], format_method: str = "HMS") -> str: diff --git a/setup.py b/setup.py index db5ed9a74..40c172216 100644 --- a/setup.py +++ b/setup.py @@ -59,10 +59,7 @@ def _pip_requirement(req): def _reqs(*f): return [ _pip_requirement(r) - for r in ( - _strip_comments(line) - for line in open(os.path.join(os.getcwd(), "requirements", *f)).readlines() - ) + for r in (_strip_comments(line) for line in open(os.path.join(os.getcwd(), "requirements", *f)).readlines()) if r ] diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index cba5aba84..afafa0d99 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -219,10 +219,7 @@ def __str__(self): @property def glob_string(self): - return ( - f"{self.output_path}/{self.name}" - f"_[0-9]*-[0-9]*/merlin_info/{self.name}.{self.prov_type}.yaml" - ) + return f"{self.output_path}/{self.name}" f"_[0-9]*-[0-9]*/merlin_info/{self.name}.{self.prov_type}.yaml" def is_within(self): """ diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index a43a3f51e..f058efd0d 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -175,9 +175,7 @@ def run_tests(args, tests): if failures == 0: print(f"Done. {n_to_run} tests passed in {round(total_time, 2)} s.") return 0 - print( - f"Done. {failures} tests out of {n_to_run} failed after {round(total_time, 2)} s.\n" - ) + print(f"Done. {failures} tests out of {n_to_run} failed after {round(total_time, 2)} s.\n") return 1 @@ -188,12 +186,8 @@ def setup_argparse(): action="store_true", help="Flag for stopping all testing upon first failure", ) - parser.add_argument( - "--verbose", action="store_true", help="Flag for more detailed output messages" - ) - parser.add_argument( - "--local", action="store_true", default=None, help="Run only local tests" - ) + parser.add_argument("--verbose", action="store_true", help="Flag for more detailed output messages") + parser.add_argument("--local", action="store_true", default=None, help="Run only local tests") parser.add_argument( "--ids", action="store", @@ -201,8 +195,7 @@ def setup_argparse(): type=int, nargs="+", default=None, - help="Provide space-delimited ids of tests you want to run." - "Example: '--ids 1 5 8 13'", + help="Provide space-delimited ids of tests you want to run. Example: '--ids 1 5 8 13'", ) return parser diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index f9c4e9620..5c07f62ac 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -1,10 +1,4 @@ -from conditions import ( - HasRegex, - HasReturnCode, - ProvenanceYAMLFileHasRegex, - StepFileExists, - StepFileHasRegex, -) +from conditions import HasRegex, HasReturnCode, ProvenanceYAMLFileHasRegex, StepFileExists, StepFileHasRegex from merlin.utils import get_flux_cmd @@ -161,9 +155,7 @@ def define_tests(): ), "dry launch slurm": ( f"{run} {slurm} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex( - "runs", "*/runs.slurm.sh", "slurm_test", OUTPUT_DIR, "srun " - ), + StepFileHasRegex("runs", "*/runs.slurm.sh", "slurm_test", OUTPUT_DIR, "srun "), "local", ), "dry launch flux": ( @@ -179,9 +171,7 @@ def define_tests(): ), "dry launch lsf": ( f"{run} {lsf} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex( - "runs", "*/runs.slurm.sh", "lsf_par", OUTPUT_DIR, "jsrun " - ), + StepFileHasRegex("runs", "*/runs.slurm.sh", "lsf_par", OUTPUT_DIR, "jsrun "), "local", ), "dry launch slurm restart": ( diff --git a/tests/unit/common/test_sample_index.py b/tests/unit/common/test_sample_index.py index eddeb1524..c693827f0 100644 --- a/tests/unit/common/test_sample_index.py +++ b/tests/unit/common/test_sample_index.py @@ -24,9 +24,7 @@ def wrapper(): @clear def test_index_file_writing(): - indx = create_hierarchy( - 1000000000, 10000, [100000000, 10000000, 1000000], root=TEST_DIR - ) + indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=TEST_DIR) indx.write_directories() indx.write_multiple_sample_index_files() indx2 = read_hierarchy(TEST_DIR) @@ -34,9 +32,7 @@ def test_index_file_writing(): def test_bundle_retrieval(): - indx = create_hierarchy( - 1000000000, 10000, [100000000, 10000000, 1000000], root=TEST_DIR - ) + indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=TEST_DIR) expected = f"{TEST_DIR}/0/0/0/samples0-10000.ext" result = indx.get_path_to_sample(123) assert expected == result @@ -97,9 +93,7 @@ def test_directory_writing(): clear_test_tree() path = os.path.join(TEST_DIR) - indx = create_hierarchy( - 1000000000, 10000, [100000000, 10000000, 1000000], root=path - ) + indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=path) indx.write_directories() diff --git a/tests/unit/spec/test_specification.py b/tests/unit/spec/test_specification.py index 4bd5c6218..6b3503fb5 100644 --- a/tests/unit/spec/test_specification.py +++ b/tests/unit/spec/test_specification.py @@ -166,13 +166,7 @@ def tearDown(self): def test_default_merlin_block(self): self.assertEqual(self.spec.merlin["resources"]["task_server"], "celery") self.assertEqual(self.spec.merlin["resources"]["overlap"], False) - self.assertEqual( - self.spec.merlin["resources"]["workers"]["default_worker"]["steps"], ["all"] - ) - self.assertEqual( - self.spec.merlin["resources"]["workers"]["default_worker"]["batch"], None - ) - self.assertEqual( - self.spec.merlin["resources"]["workers"]["default_worker"]["nodes"], None - ) + self.assertEqual(self.spec.merlin["resources"]["workers"]["default_worker"]["steps"], ["all"]) + self.assertEqual(self.spec.merlin["resources"]["workers"]["default_worker"]["batch"], None) + self.assertEqual(self.spec.merlin["resources"]["workers"]["default_worker"]["nodes"], None) self.assertEqual(self.spec.merlin["samples"], None) diff --git a/tests/unit/study/test_study.py b/tests/unit/study/test_study.py index 126a8f802..a00995d55 100644 --- a/tests/unit/study/test_study.py +++ b/tests/unit/study/test_study.py @@ -254,43 +254,27 @@ def test_expanded_spec(self): object, the MerlinStudy should produce a new spec with all instances of $(OUTPUT_PATH), $(SPECROOT), and env labels and variables expanded. """ - assert TestMerlinStudy.file_contains_string( - self.merlin_spec_filepath, "$(SPECROOT)" - ) - assert TestMerlinStudy.file_contains_string( - self.merlin_spec_filepath, "$(OUTPUT_PATH)" - ) + assert TestMerlinStudy.file_contains_string(self.merlin_spec_filepath, "$(SPECROOT)") + assert TestMerlinStudy.file_contains_string(self.merlin_spec_filepath, "$(OUTPUT_PATH)") assert TestMerlinStudy.file_contains_string(self.merlin_spec_filepath, "$PATH") - assert not TestMerlinStudy.file_contains_string( - self.study.expanded_spec.path, "$(SPECROOT)" - ) - assert not TestMerlinStudy.file_contains_string( - self.study.expanded_spec.path, "$(OUTPUT_PATH)" - ) - assert TestMerlinStudy.file_contains_string( - self.study.expanded_spec.path, "$PATH" - ) - assert not TestMerlinStudy.file_contains_string( - self.study.expanded_spec.path, "PATH_VAR: $PATH" - ) + assert not TestMerlinStudy.file_contains_string(self.study.expanded_spec.path, "$(SPECROOT)") + assert not TestMerlinStudy.file_contains_string(self.study.expanded_spec.path, "$(OUTPUT_PATH)") + assert TestMerlinStudy.file_contains_string(self.study.expanded_spec.path, "$PATH") + assert not TestMerlinStudy.file_contains_string(self.study.expanded_spec.path, "PATH_VAR: $PATH") def test_column_label_conflict(self): """ If there is a common key between Maestro's global.parameters and Merlin's sample/column_labels, an error should be raised. """ - merlin_spec_conflict: str = os.path.join( - self.tmpdir, "basic_ensemble_conflict.yaml" - ) + merlin_spec_conflict: str = os.path.join(self.tmpdir, "basic_ensemble_conflict.yaml") with open(merlin_spec_conflict, "w+") as _file: _file.write(MERLIN_SPEC_CONFLICT) # for some reason flake8 doesn't believe variables instantiated inside the try/with context are assigned with pytest.raises(ValueError): study_conflict: MerlinStudy = MerlinStudy(merlin_spec_conflict) - assert ( - not study_conflict - ), "study_conflict completed construction without raising a ValueError." + assert not study_conflict, "study_conflict completed construction without raising a ValueError." # TODO the pertinent attribute for study_no_env should be examined and asserted to be empty def test_no_env(self): @@ -298,18 +282,12 @@ def test_no_env(self): A MerlinStudy should be able to support a MerlinSpec that does not contain the optional `env` section. """ - merlin_spec_no_env_filepath: str = os.path.join( - self.tmpdir, "basic_ensemble_no_env.yaml" - ) + merlin_spec_no_env_filepath: str = os.path.join(self.tmpdir, "basic_ensemble_no_env.yaml") with open(merlin_spec_no_env_filepath, "w+") as _file: _file.write(MERLIN_SPEC_NO_ENV) try: study_no_env: MerlinStudy = MerlinStudy(merlin_spec_no_env_filepath) - bad_type_err: str = ( - f"study_no_env failed construction, is type {type(study_no_env)}." - ) + bad_type_err: str = f"study_no_env failed construction, is type {type(study_no_env)}." assert isinstance(study_no_env, MerlinStudy), bad_type_err except Exception as e: - assert ( - False - ), f"Encountered unexpected exception, {e}, for viable MerlinSpec without optional 'env' section." + assert False, f"Encountered unexpected exception, {e}, for viable MerlinSpec without optional 'env' section." diff --git a/tests/unit/utils/test_time_formats.py b/tests/unit/utils/test_time_formats.py index 35feb5692..93fe819d9 100644 --- a/tests/unit/utils/test_time_formats.py +++ b/tests/unit/utils/test_time_formats.py @@ -39,17 +39,11 @@ def test_convert_explicit(time_string: str, expected_result: str, method: str) - ], ) @pytest.mark.parametrize("method", ["HMS", "FSD", None]) -def test_convert_timestring_same( - test_case: List[Union[str, int]], expected_bool: bool, method: Optional[str] -) -> None: +def test_convert_timestring_same(test_case: List[Union[str, int]], expected_bool: bool, method: Optional[str]) -> None: """Test that HMS formatted all the same""" err_msg: str = f"Failed on test case '{test_case}', expected {expected_bool}, not '{not expected_bool}'" - converted_times: List[str] = [ - convert_timestring(time_strings) for time_strings in test_case - ] - all_equal: bool = all( - time_string == converted_times[0] for time_string in converted_times - ) + converted_times: List[str] = [convert_timestring(time_strings) for time_strings in test_case] + all_equal: bool = all(time_string == converted_times[0] for time_string in converted_times) assert all_equal == expected_bool, err_msg @@ -57,9 +51,5 @@ def test_invalid_time_format() -> None: """Test that if not provided an appropriate format (HMS, FSD), the appropriate error is thrown.""" with pytest.raises(ValueError) as invalid_format: repr_timedelta(datetime.timedelta(1), "HMD") - examination_err_msg: str = ( - "Did not raise correct ValueError for failed repr_timedelta()." - ) - assert "Invalid method for formatting timedelta! Valid choices: HMS, FSD" in str( - invalid_format.value - ), examination_err_msg + examination_err_msg: str = "Did not raise correct ValueError for failed repr_timedelta()." + assert "Invalid method for formatting timedelta! Valid choices: HMS, FSD" in str(invalid_format.value), examination_err_msg From b03640006e156059b801fe2179f28760fb38029a Mon Sep 17 00:00:00 2001 From: Alexander Winter <80929291+AlexanderWinterLLNL@users.noreply.github.com> Date: Mon, 11 Oct 2021 10:18:41 -0700 Subject: [PATCH 009/126] Patch to suppress console output about unsupported shell, minor fix to Makefile target. (#335) Updated self._unsupported and new_unsupported in script_adapter.py to include 'shell', overcoming console arning about shell being unsupported. Typing added to touched class __inits__. Makefile had an error in install-merlin that wasn't activating the venv, now activates to enable that target as well as install-dev. Co-authored-by: Alexander Cameron Winter --- CHANGELOG.md | 6 ++++- Makefile | 1 + merlin/study/script_adapter.py | 47 ++++++++++++++++++---------------- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffde16acb..1136650b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added PyLint pipeline to Github Actions CI (currently no-fail-exit). - Corrected integration test for dependency to only examine release dependencies. - PyLint adherence for: celery.py, opennplib.py, config/__init__.py, broker.py, -configfile.py, formatter.py, main.py, router.py + configfile.py, formatter.py, main.py, router.py - Integrated Black and isort into CI +### Fixed +- 'shell' added to unsupported and new_unsupported lists in script_adapter.py, prevents + `'shell' is not supported -- ommitted` message. +- Makefile target for install-merlin fixed so venv is properly activated to install merlin ## [1.8.1] diff --git a/Makefile b/Makefile index d9aa80e18..734b2dff3 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,7 @@ virtualenv: # install merlin into the virtual environment install-merlin: virtualenv $(PIP) install -e .; \ + . $(VENV)/bin/activate; \ merlin config; \ diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index c4f47e8c7..99e45fdb3 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -34,6 +34,7 @@ import logging import os +from typing import Dict, List, Set from maestrowf.interfaces.script import SubmissionRecord from maestrowf.interfaces.script.localscriptadapter import LocalScriptAdapter @@ -66,7 +67,7 @@ def __init__(self, **kwargs): """ super(MerlinLSFScriptAdapter, self).__init__(**kwargs) - self._cmd_flags = { + self._cmd_flags: Dict[str, str] = { "cmd": "jsrun", "ntasks": "--np", "nodes": "--nrs", @@ -79,22 +80,23 @@ def __init__(self, **kwargs): "lsf": "", } - self._unsupported = { + self._unsupported: Set[str] = { "cmd", - "ntasks", - "nodes", + "depends", + "flux", "gpus", - "walltime", + "max_retries", + "nodes", + "ntasks", + "post", + "pre", "reservation", "restart", - "task_queue", - "max_retries", "retry_delay", - "pre", - "post", - "depends", + "shell", "slurm", - "flux", + "task_queue", + "walltime", } def get_header(self, step): @@ -158,7 +160,7 @@ class MerlinSlurmScriptAdapter(SlurmScriptAdapter): the SlurmScriptAdapter uses non-blocking submits. """ - key = "merlin-slurm" + key: str = "merlin-slurm" def __init__(self, **kwargs): """ @@ -174,20 +176,21 @@ def __init__(self, **kwargs): self._cmd_flags["slurm"] = "" self._cmd_flags["walltime"] = "-t" - new_unsupported = [ - "task_queue", - "max_retries", - "retry_delay", - "pre", - "post", + new_unsupported: List[str] = [ + "bind", + "flux", "gpus per task", "gpus", - "restart", - "bind", "lsf", - "flux", + "max_retries", + "post", + "pre", + "restart", + "retry_delay", + "shell", + "task_queue", ] - self._unsupported = set(list(self._unsupported) + new_unsupported) + self._unsupported: Set[str] = set(list(self._unsupported) + new_unsupported) def get_header(self, step): """ From 6b2737e9b239cd2bf69521ff6e2db8d5879d3965 Mon Sep 17 00:00:00 2001 From: Alexander Winter <80929291+AlexanderWinterLLNL@users.noreply.github.com> Date: Thu, 21 Oct 2021 15:48:19 -0700 Subject: [PATCH 010/126] Enabled distributed testing suite. (#333) * Enabled distributed testing suite. Many changes. This brings the distributed testing online and adds them to the CI, adding Makefile targets for both local and distributed tests. The previous distributed test hung forever because the workers were simply left standing, awaiting additional tasks, so the test never 'completed'. Remote_feature_demo fixes this - it is very nearly the feature_demo workflow, with an additional step in the study, 'stop-workers', which 'exits' at the end of the workflow. This required standing up a docker image of redis to run the distributed tests, so that change has been made in the CI. Embedded in and commented out is another test, which adds a `sleep` command and `stop-workers` so that the process could more gracefully reach completion () rather than relying on the exit command, which shuts down all workers. This is unplugged this way as `stop-workers` is not currently functional, but, when that is online, this should be activated as well. In a similar vein, there is embedded code for running against rabbitmq - this is not currently configured, but should be in the future for total test coverage of all supported configurations. Lastly, upgraded the merlin config args to utilize the `test` arg. This arg had been in the source, but was unused (it read through the option and performed the exact same operations regardless). It now is invoked when running the distributed tests to examine the app_test.yaml. Co-authored-by: Alexander Cameron Winter --- .github/workflows/push-pr_workflow.yml | 80 +++++++++- CHANGELOG.md | 3 +- Makefile | 17 ++- merlin/data/celery/app.yaml | 4 +- merlin/data/celery/app_test.yaml | 44 ++++++ .../workflows/remote_feature_demo/.gitignore | 1 + .../remote_feature_demo.yaml | 141 ++++++++++++++++++ .../remote_feature_demo/requirements.txt | 2 + .../remote_feature_demo/scripts/features.json | 11 ++ .../scripts/hello_world.py | 52 +++++++ .../remote_feature_demo/scripts/pgen.py | 20 +++ merlin/main.py | 10 +- merlin/router.py | 10 +- requirements/release.txt | 1 + tests/integration/test_definitions.py | 28 +++- 15 files changed, 410 insertions(+), 14 deletions(-) create mode 100644 merlin/data/celery/app_test.yaml create mode 100644 merlin/examples/workflows/remote_feature_demo/.gitignore create mode 100644 merlin/examples/workflows/remote_feature_demo/remote_feature_demo.yaml create mode 100644 merlin/examples/workflows/remote_feature_demo/requirements.txt create mode 100644 merlin/examples/workflows/remote_feature_demo/scripts/features.json create mode 100644 merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py create mode 100644 merlin/examples/workflows/remote_feature_demo/scripts/pgen.py diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 8a9057616..479b18cc9 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -65,8 +65,9 @@ jobs: python3 -m pylint merlin --rcfile=setup.cfg --exit-zero python3 -m pylint tests --rcfile=setup.cfg --exit-zero - Test-suite: + Local-test-suite: runs-on: ubuntu-latest + strategy: matrix: python-version: [3.6, 3.7, 3.8, 3.9] @@ -104,6 +105,81 @@ jobs: run: | python3 -m pytest tests/unit/ - - name: Run integration test suite, locally + - name: Run integration test suite run: | python3 tests/integration/run_tests.py --verbose --local + + Distributed-test-suite: + runs-on: ubuntu-latest + services: + # rabbitmq: + # image: rabbitmq:latest + # ports: + # - 5672:5672 + # options: --health-cmd "rabbitmqctl node_health_check" --health-interval 10s --health-timeout 5s --health-retries 5 + # Label used to access the service container + redis: + # Docker Hub image + image: redis + # Set health checks to wait until redis has started + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Check cache + uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip3 install -r requirements/dev.txt + + - name: Install merlin to run unit tests + run: | + pip3 install -e . + merlin config --broker redis + + - name: Install CLI task dependencies generated from the 'feature demo' workflow + run: | + merlin example feature_demo + pip3 install -r feature_demo/requirements.txt + + - name: Run integration test suite for Redis + env: + REDIS_HOST: redis + REDIS_PORT: 6379 + run: | + python3 tests/integration/run_tests.py --verbose --ids 31 32 + + # - name: Setup rabbitmq config + # run: | + # merlin config --test rabbitmq + + # - name: Run integration test suite for rabbitmq + # env: + # AMQP_URL: amqp://localhost:${{ job.services.rabbitmq.ports[5672] }} + # RABBITMQ_USER: Jimmy_Space + # RABBITMQ_PASS: Alexander_Rules + # ports: + # - ${{ job.services.rabbitmq.ports['5672'] }} + # run: | + # python3 tests/integration/run_tests.py --verbose --ids 31 32 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1136650b6..88dc79493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +### Fixed +- Re-enabled distributed integration testing. Added additional examination to distributed testing. ### Changed - CI now splits linting and testing into different tasks for better utilization of diff --git a/Makefile b/Makefile index 734b2dff3..d3036deb9 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,9 @@ include config.mk .PHONY : install-dev .PHONY : unit-tests .PHONY : e2e-tests +.PHONY : e2e-tests-diagnostic +.PHONY : e2e-tests-local +.PHONY : e2e-tests-local-diagnostic .PHONY : tests .PHONY : check-flake8 .PHONY : check-black @@ -88,12 +91,22 @@ unit-tests: # run CLI tests - these require an active install of merlin in a venv e2e-tests: . $(VENV)/bin/activate; \ - $(PYTHON) $(TEST)/integration/run_tests.py --local; \ + $(PYTHON) $(TEST)/integration/run_tests.py; \ e2e-tests-diagnostic: . $(VENV)/bin/activate; \ - $(PYTHON) $(TEST)/integration/run_tests.py --local --verbose; \ + $(PYTHON) $(TEST)/integration/run_tests.py --verbose + + +e2e-tests-local: + . $(VENV)/bin/activate; \ + $(PYTHON) $(TEST)/integration/run_tests.py --local; \ + + +e2e-tests-local-diagnostic: + . $(VENV)/bin/activate; \ + $(PYTHON) $(TEST)/integration/run_tests.py --local --verbose # run unit and CLI tests diff --git a/merlin/data/celery/app.yaml b/merlin/data/celery/app.yaml index ae28fcb9c..4dd05515b 100644 --- a/merlin/data/celery/app.yaml +++ b/merlin/data/celery/app.yaml @@ -37,6 +37,4 @@ results_backend: # your redis server. encryption_key: ~/.merlin/encrypt_data_key port: 6379 - db_num: 0 - - + db_num: 0 \ No newline at end of file diff --git a/merlin/data/celery/app_test.yaml b/merlin/data/celery/app_test.yaml new file mode 100644 index 000000000..798a863b1 --- /dev/null +++ b/merlin/data/celery/app_test.yaml @@ -0,0 +1,44 @@ +celery: + # see Celery configuration options + # https://docs.celeryproject.org/en/stable/userguide/configuration.html + override: + visibility_timeout: 86400 + +broker: + # can be redis, redis+sock, or rabbitmq + name: rabbitmq + #username: # defaults to your username unless changed here + username: $(RABBITMQ_USER) + # password: + password: $(RABBITMQ_PASS) + # server URL + server: $(RABBITMQ_PORT) + + + + ### for rabbitmq, redis+sock connections ### + #vhost: # defaults to your username unless changed here + + ### for redis+sock connections ### + #socketname: the socket name your redis connection can be found on. + #path: The path to the socket. + + ### for redis connections ### + #port: The port number redis is listening on (default 6379) + #db_num: The data base number to connect to. + + +results_backend: + # must be redis + name: redis + dbname: mlsi + username: mlsi + # name of file where redis password is stored. + password: redis.pass + server: jackalope.llnl.gov + # merlin will generate this key if it does not exist yet, + # and will use it to encrypt all data over the wire to + # your redis server. + encryption_key: ~/.merlin/encrypt_data_key + port: 6379 + db_num: 0 \ No newline at end of file diff --git a/merlin/examples/workflows/remote_feature_demo/.gitignore b/merlin/examples/workflows/remote_feature_demo/.gitignore new file mode 100644 index 000000000..0806ddf1f --- /dev/null +++ b/merlin/examples/workflows/remote_feature_demo/.gitignore @@ -0,0 +1 @@ +studies/ diff --git a/merlin/examples/workflows/remote_feature_demo/remote_feature_demo.yaml b/merlin/examples/workflows/remote_feature_demo/remote_feature_demo.yaml new file mode 100644 index 000000000..014e3e1da --- /dev/null +++ b/merlin/examples/workflows/remote_feature_demo/remote_feature_demo.yaml @@ -0,0 +1,141 @@ +description: + name: $(NAME) + description: Run 10 hello worlds. + +batch: + type: local + +env: + variables: + OUTPUT_PATH: ./studies + N_SAMPLES: 10 + WORKER_NAME: demo_worker + VERIFY_QUEUE: default_verify_queue + NAME: feature_demo + + SCRIPTS: $(MERLIN_INFO)/scripts + HELLO: $(SCRIPTS)/hello_world.py + FEATURES: $(SCRIPTS)/features.json + +study: + - name: hello + description: | + process a sample with hello world + run: + cmd: | + python3 $(HELLO) -outfile hello_world_output_$(MERLIN_SAMPLE_ID).json $(X0) $(X1) $(X2) + task_queue: hello_queue + max_retries: 1 + + - name: collect + description: | + process the output of the hello world samples, extracting specific features; + run: + cmd: | + echo $(MERLIN_GLOB_PATH) + echo $(hello.workspace) + ls $(hello.workspace)/X2.$(X2)/$(MERLIN_GLOB_PATH)/hello_world_output_*.json > files_to_collect.txt + spellbook collect -outfile results.json -instring "$(cat files_to_collect.txt)" + depends: [hello_*] + task_queue: collect_queue + + - name: translate + description: | + process the output of the hello world samples some more + run: + cmd: spellbook translate -input $(collect.workspace)/results.json -output results.npz -schema $(FEATURES) + depends: [collect] + task_queue: translate_queue + + - name: learn + description: | + train a learner on the results + run: + cmd: spellbook learn -infile $(translate.workspace)/results.npz + depends: [translate] + task_queue: learn_queue + + - name: make_new_samples + description: | + make a grid of new samples to pass to the predictor + run: + cmd: spellbook make-samples -n $(N_NEW) -sample_type grid -outfile grid_$(N_NEW).npy + task_queue: make_samples_queue + + - name: predict + description: | + make a new prediction from new samples + run: + cmd: spellbook predict -infile $(make_new_samples.workspace)/grid_$(N_NEW).npy -outfile prediction_$(N_NEW).npy -reg $(learn.workspace)/random_forest_reg.pkl + depends: [learn, make_new_samples] + task_queue: predict_queue + + - name: verify + description: | + if learn and predict succeeded, output a dir to signal study completion + run: + cmd: | + if [[ -f $(learn.workspace)/random_forest_reg.pkl && -f $(predict.workspace)/prediction_$(N_NEW).npy ]] + then + touch FINISHED + exit $(MERLIN_SUCCESS) + else + exit $(MERLIN_SOFT_FAIL) + fi + depends: [learn, predict] + task_queue: $(VERIFY_QUEUE) + + - name: python3_hello + description: | + do something in python + run: + cmd: | + print("OMG is this in python?") + print("Variable X2 is $(X2)") + shell: /usr/bin/env python3 + task_queue: pyth3_q + + - name: python2_hello + description: | + do something in python2, because change is bad + run: + cmd: | + print "OMG is this in python2? Change is bad." + print "Variable X2 is $(X2)" + shell: /usr/bin/env python2 + task_queue: pyth2_hello + + - name: stop_workers + description: Stop workers + run: + cmd: | + exit $(MERLIN_STOP_WORKERS) + depends: [verify] + task_queue: $(VERIFY_QUEUE) + +global.parameters: + X2: + values : [0.5] + label : X2.%% + N_NEW: + values : [10] + label : N_NEW.%% + +merlin: + resources: + task_server: celery + overlap: False + workers: + $(WORKER_NAME): + args: -l INFO --concurrency 3 --prefetch-multiplier 1 -Ofair + samples: + generate: + cmd: | + cp -r $(SPECROOT)/scripts $(SCRIPTS) + + spellbook make-samples -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy + # can be a file glob of numpy sample files. + file: $(MERLIN_INFO)/samples.npy + column_labels: [X0, X1] + level_max_dirs: 25 + diff --git a/merlin/examples/workflows/remote_feature_demo/requirements.txt b/merlin/examples/workflows/remote_feature_demo/requirements.txt new file mode 100644 index 000000000..93909e8aa --- /dev/null +++ b/merlin/examples/workflows/remote_feature_demo/requirements.txt @@ -0,0 +1,2 @@ +sklearn +merlin-spellbook diff --git a/merlin/examples/workflows/remote_feature_demo/scripts/features.json b/merlin/examples/workflows/remote_feature_demo/scripts/features.json new file mode 100644 index 000000000..06e2f5191 --- /dev/null +++ b/merlin/examples/workflows/remote_feature_demo/scripts/features.json @@ -0,0 +1,11 @@ +{ + "inputs": + { + "X":0.0, + "Z":0.0 + }, + "outputs": + { + "X+Y+Z": 0.0 + } +} diff --git a/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py b/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py new file mode 100644 index 000000000..565806ce4 --- /dev/null +++ b/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py @@ -0,0 +1,52 @@ +import argparse +import json +import sys +from typing import Dict + +def process_args(args: argparse.Namespace) -> None: + """ + Writes a json file of the parsed args after doing some trivial math. + """ + results: Dict[str, Dict[str, float]] = { + "inputs": {"X": args.X, "Y": args.Y, "Z": args.Z}, + "outputs": { + "X+Y+Z": args.X + args.Y + args.Z, + "X*Y*Z": args.X * args.Y * args.Z, + }, + } + + with open(args.outfile, "w") as f: + json.dump(results, f) + + +def setup_argparse() -> argparse.ArgumentParser: + """ + This method sets up the argparser. + """ + parser: argparse.ArgumentParser = argparse.ArgumentParser(description="Process some integers.") + parser.add_argument( + "X", metavar="X", type=float, help="The x dimension of the sample." + ) + parser.add_argument( + "Y", metavar="Y", type=float, help="The y dimension of the sample." + ) + parser.add_argument( + "Z", metavar="Z", type=float, help="The z dimension of the sample." + ) + parser.add_argument( + "-outfile", help="Output file name", default="hello_world_output.json" + ) + return parser + + +def main(): + """ + Primary coordinating method for collecting args and dumping them to a json file for later examination. + """ + parser: argparse.ArgumentParser = setup_argparse() + args: argparse.Namespace = parser.parse_args() + process_args(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/merlin/examples/workflows/remote_feature_demo/scripts/pgen.py b/merlin/examples/workflows/remote_feature_demo/scripts/pgen.py new file mode 100644 index 000000000..ad13cc50b --- /dev/null +++ b/merlin/examples/workflows/remote_feature_demo/scripts/pgen.py @@ -0,0 +1,20 @@ +from typing import Dict, List, Union + +from maestrowf.datastructures.core import ParameterGenerator + + +def get_custom_generator(env, **kwargs) -> ParameterGenerator: + """ + Generates parameters to feed at the CL to hello_world.py. + """ + p_gen: ParameterGenerator = ParameterGenerator() + params: Dict[str, Union[List[float], str]] = { + "X2": {"values": [1 / i for i in range(3, 6)], "label": "X2.%%"}, + "N_NEW": {"values": [2 ** i for i in range(1, 4)], "label": "N_NEW.%%"}, + } + key: str + value: Union[List[float], str] + for key, value in params.items(): + p_gen.add_parameter(key, value["values"], value["label"]) + + return p_gen diff --git a/merlin/main.py b/merlin/main.py index 03da04ffb..9ef8e96ef 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -311,7 +311,8 @@ def config_merlin(args: Namespace) -> None: if output_dir is None: user_home: str = os.path.expanduser("~") output_dir: str = os.path.join(user_home, ".merlin") - router.create_config(args.task_server, output_dir, args.broker) + + router.create_config(args.task_server, output_dir, args.broker, args.test) def process_example(args: Namespace) -> None: @@ -516,6 +517,13 @@ def setup_argparse() -> None: help="Optional broker type, backend will be redis\ Default: rabbitmq", ) + mconfig.add_argument( + "--test", + type=str, + default=None, + help="A config used in the testing suite (or for exemplative purposes).\ + Default: rabbitmq", + ) # merlin example example: ArgumentParser = subparsers.add_parser( diff --git a/merlin/router.py b/merlin/router.py index b65f6cbe4..66742b56d 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -207,15 +207,19 @@ def route_for_task(name, args, kwargs, options, task=None, **kw): return {"queue": queue} -def create_config(task_server: str, config_dir: str, broker: str) -> None: +def create_config(task_server: str, config_dir: str, broker: str, test: str) -> None: """ Create a config for the given task server. :param [str] `task_server`: The task server from which to stop workers. :param [str] `config_dir`: Optional directory to install the config. :param [str] `broker`: string indicated the broker, used to check for redis. + :param [str] `test`: string indicating if the app.yaml is used for testing. """ - LOG.info("Creating config ...") + if test: + LOG.info("Creating test config ...") + else: + LOG.info("Creating config ...") if not os.path.isdir(config_dir): os.makedirs(config_dir) @@ -225,6 +229,8 @@ def create_config(task_server: str, config_dir: str, broker: str) -> None: data_config_file = "app.yaml" if broker == "redis": data_config_file = "app_redis.yaml" + elif test: + data_config_file = "app_test.yaml" with resources.path("merlin.data.celery", data_config_file) as data_file: create_celery_config(config_dir, config_file, data_file) else: diff --git a/requirements/release.txt b/requirements/release.txt index 4771b7a4c..48e468993 100644 --- a/requirements/release.txt +++ b/requirements/release.txt @@ -8,4 +8,5 @@ numpy parse psutil>=5.1.0 pyyaml>=5.1.2 +redis tabulate diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 5c07f62ac..ef3b7ac1b 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -23,6 +23,7 @@ def define_tests(): examples = "merlin/examples/workflows" dev_examples = "merlin/examples/dev_workflows" demo = f"{examples}/feature_demo/feature_demo.yaml" + remote_demo = f"{examples}/remote_feature_demo/remote_feature_demo.yaml" demo_pgen = f"{examples}/feature_demo/scripts/pgen.py" simple = f"{examples}/simple_chain/simple_chain.yaml" slurm = f"{examples}/slurm/slurm_test.yaml" @@ -318,8 +319,8 @@ def define_tests(): f"{run} {demo} ; {purge} {demo} -f", HasReturnCode(), ), - "distributed feature_demo": ( - f"{run} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; {workers} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers", + "remote feature_demo": ( + f"{run} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; {workers} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers", [ HasReturnCode(), ProvenanceYAMLFileHasRegex( @@ -337,6 +338,27 @@ def define_tests(): ), ], ), + # this test is deactivated until the --spec option for stop-workers is active again + + # "stop workers for distributed feature_demo": ( + # f"{run} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; {workers} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; sleep 20 ; merlin stop-workers --spec {demo}", + # [ + # HasReturnCode(), + # ProvenanceYAMLFileHasRegex( + # regex="cli_test_demo_workers:", + # name="feature_demo", + # output_path=OUTPUT_DIR, + # provenance_type="expanded", + # ), + # StepFileExists( + # "verify", + # "MERLIN_FINISHED", + # "feature_demo", + # OUTPUT_DIR, + # params=True, + # ), + # ], + # ), } # combine and return test dictionaries @@ -353,7 +375,7 @@ def define_tests(): # provenence_equality_checks, # omitting provenance equality check because it is broken # style_checks, # omitting style checks due to different results on different machines dependency_checks, - # distributed_tests, # omitting distributed tests as they are not yet ready + distributed_tests ]: all_tests.update(test_dict) From 1e578aabc2eaab6fdec85ad60da2216bdb14e52a Mon Sep 17 00:00:00 2001 From: Yamen Mubarka Date: Fri, 22 Oct 2021 13:30:27 -0700 Subject: [PATCH 011/126] adding old changes (#337) * Update to the example optimization workflow. Originally merged in an incomplete state, this updates the example example optimization workflow. It renames the workflow to template_optimization, integrates the merlin spellbook to some of the scripts (not all that are possible), adds some test functions for examination, implements Jinja to develop the environment. This update will be released with 1.8.2 although additional work is desired for a future release - these future changes are recorded in github https://github.com/LLNL/merlin/pull/337. --- CHANGELOG.md | 1 + .../workflows/optimization/optimization.yaml | 175 ---------------- .../workflows/optimization/requirements.txt | 4 +- .../optimization/scripts/collector.py | 19 +- .../workflows/optimization/scripts/learner.py | 40 ---- .../optimization/scripts/optimizer.py | 19 +- .../optimization/scripts/test_functions.py | 44 +++- .../optimization/scripts/visualizer.py | 55 +++-- .../workflows/optimization/template_config.py | 67 ++++++ .../optimization/template_optimization.yaml | 195 ++++++++++++++++++ 10 files changed, 365 insertions(+), 254 deletions(-) delete mode 100644 merlin/examples/workflows/optimization/optimization.yaml delete mode 100644 merlin/examples/workflows/optimization/scripts/learner.py create mode 100644 merlin/examples/workflows/optimization/template_config.py create mode 100644 merlin/examples/workflows/optimization/template_optimization.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 88dc79493..a05ad88e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Re-enabled distributed integration testing. Added additional examination to distributed testing. ### Changed +- Updated the optimization workflow example with a new python template editor script - CI now splits linting and testing into different tasks for better utilization of parallel runners, significant and scalable speed gain over previous setup - CI now uses caching to restore environment of dependencies, reducing CI runtime diff --git a/merlin/examples/workflows/optimization/optimization.yaml b/merlin/examples/workflows/optimization/optimization.yaml deleted file mode 100644 index 6f61d5932..000000000 --- a/merlin/examples/workflows/optimization/optimization.yaml +++ /dev/null @@ -1,175 +0,0 @@ -description: - name: optimization - description: Design Optimization Template - -##TODO -# DONE Step to delete run_sim files -# DONE: Removing the make-samples.py script -# Ability to change the test function - # Need to make sure visualizer step doesn't get affected -# Ability to change the learner -# Ability to change optimizer - -env: - variables: - EMAIL_ADDRESS: "enter your email address here, or override this with --vars" - N_DIMS: 2 - METHOD: 'trust-constr' - - ITER: 1 - MAX_ITER: 3 - - SCRIPTS: $(SPECROOT)/scripts - - DIM_1_MIN: -2.0 - DIM_1_MAX: 2.0 - DIM_2_MIN: -1.0 - DIM_2_MAX: 3.0 - DIM_1_UNCERT: 0.1 - DIM_2_UNCERT: 0.1 - - BOUNDS_X: "[[$(DIM_1_MIN),$(DIM_1_MAX)],[$(DIM_2_MIN),$(DIM_2_MAX)]]" - UNCERTS_X: "[$(DIM_1_UNCERT),$(DIM_2_UNCERT)]" - - # pick_new_inputs step - SEARCH_SCALE: 0.30 - N_SAMPLES: 5 # Number of new samples per iteration around the predicted new point - N_EXPLOIT: 5 # Number of new samples per iteration around the current best - N_SAMPLES_LINE: 5 # Number between predicted best and current best - N_SAMPLES_START: 20 # Number to initialize - - -study: - - name: run_simulation - description: Run the desired simulation - run: - cmd: |- - python3 $(SCRIPTS)/test_functions.py -ID "$(ITER)/$(MERLIN_SAMPLE_ID)" -inputs $(DIM_1) $(DIM_2) - - cores per task: 1 - nodes: 1 - procs: 1 - task_queue: simulation - - - name: collector - description: Collect the results into a single file and make an npz of some features - run: - cmd: |- - python3 $(SCRIPTS)/collector.py -sim_dirs "$(run_simulation.workspace)/$(MERLIN_GLOB_PATH)" - - cores per task: 1 - nodes: 1 - procs: 1 - depends: [run_simulation_*] - task_queue: simulation_postprocess - - - name: clean_up_simulation - description: Cleans up the merlin sample paths of the run_simulation step - run: - cmd: |- - rm -rf $(run_simulation.workspace)/$(MERLIN_SAMPLE_PATH) - - cores per task: 1 - nodes: 1 - procs: 1 - depends: [collector] - task_queue: simulation_postprocess - - - name: learner - description: Train an ML model on the simulation - run: - cmd: |- - temp=1 - if [ $(ITER) -ge "$temp" ] ; then - echo "Copying the npz file from previous iteration" - cp ../../../learner/all_iter_results.npz . - fi - - python3 $(SCRIPTS)/learner.py -collector_dir "$(collector.workspace)" - - cores per task: 1 - nodes: 1 - procs: 1 - depends: [clean_up_simulation] - task_queue: learner - - - name: optimizer - description: Optimizer - run: - cmd: |- - python3 $(SCRIPTS)/optimizer.py -learner_dir "$(learner.workspace)" -bounds "$(BOUNDS_X)" -input_uncerts "$(UNCERTS_X)" -method "$(METHOD)" - - cores per task: 1 - nodes: 1 - procs: 1 - depends: [collector, learner] - task_queue: learner - - - name: pick_new_inputs - description: Picking new simulations to run in the next iteration - run: - cmd: |- - spellbook make-samples -dims $(N_DIMS) -x0 $(optimizer.workspace)/optimum.npy -x1 $(optimizer.workspace)/old_best.npy -n_line $(N_SAMPLES_LINE) -scale_factor $(SEARCH_SCALE) -scale "$(BOUNDS_X)" -n $(N_SAMPLES) -sample_type lhd -outfile new_explore_samples.npy --hard-bounds - spellbook make-samples -dims $(N_DIMS) -x0 $(optimizer.workspace)/optimum.npy -scale_factor 0.05 -scale "$(BOUNDS_X)" -sample_type star -outfile new_explore_star_samples.npy --hard-bounds - # Add points near current best too - spellbook make-samples -dims $(N_DIMS) -x0 $(optimizer.workspace)/old_best.npy -scale_factor $(SEARCH_SCALE) -scale "$(BOUNDS_X)" -n $(N_EXPLOIT) -sample_type lhd -outfile new_exploit_samples.npy --hard-bounds - - # combine them - python3 -c "import numpy as np; np.save('new_samples.npy',np.vstack((np.load('new_explore_samples.npy'),np.load('new_exploit_samples.npy'),np.load('new_explore_star_samples.npy'))))" - - cores per task: 1 - nodes: 1 - procs: 1 - depends: [optimizer] - task_queue: learner - - - name: visualizer - description: Either launches new simulations or iterated with inputs from previous step - run: - cmd: |- - # Add an if statment to make sure it is rosenbrock function and 2D - python3 $(SCRIPTS)/visualizer.py -study_dir $(MERLIN_WORKSPACE) - cores per task: 1 - nodes: 1 - procs: 1 - depends: [pick_new_inputs] - task_queue: learner - - - name: iterate - description: Either launches new simulations or iterated with inputs from previous step - run: - cmd: |- - cp $(optimizer.workspace)/optimization_results.json optimization_results_iter_$(ITER).json - cp $(visualizer.workspace)/results.png results_iter_$(ITER).png - echo "Done iteration $(ITER) in $(MERLIN_WORKSPACE)" | mail -s "Status $(MERLIN_WORKSPACE)" -a optimization_results_iter_$(ITER).json -a results_iter_$(ITER).png $(EMAIL_ADDRESS) - - if [ $(ITER) -ge $(MAX_ITER) ] ; then - echo "done" - else - next_iter=$(ITER) - ((next_iter=next_iter+1)) - echo "Starting iteration " $next_iter - merlin run --local $(MERLIN_INFO)/*partial.yaml --samplesfile $(pick_new_inputs.workspace)/new_samples.npy --vars ITER=$next_iter - fi - cores per task: 1 - nodes: 1 - procs: 1 - depends: [visualizer] - task_queue: learner - -merlin: - resources: - overlap: true - task_server: celery - workers: - all_workers: - args: -O fair --prefetch-multiplier 1 -E -l info --concurrency 20 - steps: [all] - samples: - column_labels: - - DIM_1 - - DIM_2 - file: $(MERLIN_INFO)/samples.npy - generate: - cmd: |- - spellbook make-samples -dims $(N_DIMS) -n $(N_SAMPLES_START) -sample_type lhs -outfile=$(MERLIN_INFO)/samples.npy -scale "$(BOUNDS_X)" diff --git a/merlin/examples/workflows/optimization/requirements.txt b/merlin/examples/workflows/optimization/requirements.txt index 8caf85646..a487c9f39 100644 --- a/merlin/examples/workflows/optimization/requirements.txt +++ b/merlin/examples/workflows/optimization/requirements.txt @@ -1,6 +1,6 @@ -merlin merlin-spellbook numpy -joblib scikit-learn matplotlib +Jinja2 +pyDOE diff --git a/merlin/examples/workflows/optimization/scripts/collector.py b/merlin/examples/workflows/optimization/scripts/collector.py index 0d926d9ee..4ad8f1bd7 100644 --- a/merlin/examples/workflows/optimization/scripts/collector.py +++ b/merlin/examples/workflows/optimization/scripts/collector.py @@ -22,4 +22,21 @@ all_results.update(data) -np.savez("current_results.npz", all_results) +with open("current_results.json", "w") as outfile: + json.dump(all_results, outfile) + +X = [] +y = [] +for key in all_results.keys(): + X.append(all_results[key]["Inputs"]) + y.append(all_results[key]["Outputs"]) + +X = np.array(X) +y = np.array(y) + +if len(X.shape) == 1: + X = X.reshape(-1, 1) +if len(y.shape) == 1: + y = y.reshape(-1, 1) + +np.savez("current_results.npz", X=X, y=y) diff --git a/merlin/examples/workflows/optimization/scripts/learner.py b/merlin/examples/workflows/optimization/scripts/learner.py deleted file mode 100644 index 6ead50b66..000000000 --- a/merlin/examples/workflows/optimization/scripts/learner.py +++ /dev/null @@ -1,40 +0,0 @@ -import argparse - -import numpy as np -from joblib import dump -from sklearn.ensemble import RandomForestRegressor - - -parser = argparse.ArgumentParser("Learn surrogate model form simulation") -parser.add_argument( - "-collector_dir", - help="Collector directory (.npz file), usually '$(collector.workspace)'", -) -args = parser.parse_args() - -collector_dir = args.collector_dir -current_iter_npz = np.load(f"{collector_dir}/current_results.npz", allow_pickle=True) -current_iter_data = current_iter_npz["arr_0"].item() - -try: - prev_iter_npz = np.load("all_iter_results.npz", allow_pickle=True) - prev_iter_data = prev_iter_npz["arr_0"].item() - data = dict(prev_iter_data, **current_iter_data) -except BaseException: - data = current_iter_data - -X = [] -y = [] - -for i in data.keys(): - X.append(data[i]["Inputs"]) - y.append(data[i]["Outputs"]) - -X = np.array(X) -y = np.array(y) - -surrogate = RandomForestRegressor(max_depth=4, random_state=0, n_estimators=100) -surrogate.fit(X, y) - -dump(surrogate, "surrogate.joblib") -np.savez("all_iter_results.npz", data) diff --git a/merlin/examples/workflows/optimization/scripts/optimizer.py b/merlin/examples/workflows/optimization/scripts/optimizer.py index 74659c266..39735dc29 100644 --- a/merlin/examples/workflows/optimization/scripts/optimizer.py +++ b/merlin/examples/workflows/optimization/scripts/optimizer.py @@ -1,9 +1,9 @@ import argparse import ast import json +import pickle import numpy as np -from joblib import load from scipy.optimize import minimize from scipy.stats import multivariate_normal @@ -22,20 +22,11 @@ method = args.method learner_dir = args.learner_dir -surrogate = load(f"{learner_dir}/surrogate.joblib") -from_file = np.load(f"{learner_dir}/all_iter_results.npz", allow_pickle=True) +surrogate = pickle.load(open(f"{learner_dir}/surrogate.pkl", "rb")) +all_iter_results = np.load(f"{learner_dir}/all_iter_results.npz", allow_pickle=True) -data = from_file["arr_0"].item() - -X = [] -y = [] - -for i in data.keys(): - X.append(data[i]["Inputs"]) - y.append(data[i]["Outputs"]) - -existing_X = np.array(X) -existing_y = np.array(y) +existing_X = all_iter_results["X"] +existing_y = all_iter_results["y"] def process_bounds(args): diff --git a/merlin/examples/workflows/optimization/scripts/test_functions.py b/merlin/examples/workflows/optimization/scripts/test_functions.py index 5d977c1ea..d6f391a0a 100644 --- a/merlin/examples/workflows/optimization/scripts/test_functions.py +++ b/merlin/examples/workflows/optimization/scripts/test_functions.py @@ -1,10 +1,11 @@ import argparse import json +import math import numpy as np -def N_Rosenbrock(X): +def rosenbrock(X): X = X.T total = 0 for i in range(X.shape[0] - 1): @@ -13,15 +14,50 @@ def N_Rosenbrock(X): return total +def rastrigin(X, A=10): + first_term = A * len(inputs) + + return first_term + sum([(x ** 2 - A * np.cos(2 * math.pi * x)) for x in X]) + + +def ackley(X): + firstSum = 0.0 + secondSum = 0.0 + for x in X: + firstSum += x ** 2.0 + secondSum += np.cos(2.0 * np.pi * x) + n = float(len(X)) + + return -20.0 * np.exp(-0.2 * np.sqrt(firstSum / n)) - np.exp(secondSum / n) + 20 + np.e + + +def griewank(X): + term_1 = (1.0 / 4000.0) * sum(X ** 2) + term_2 = 1.0 + for i, x in enumerate(X): + term_2 *= np.cos(x) / np.sqrt(i + 1) + return 1.0 + term_1 - term_2 + + parser = argparse.ArgumentParser("Generate some samples!") -parser.add_argument("-ID") -parser.add_argument("-inputs", nargs="+") +parser.add_argument( + "-function", + help="Which test function do you want to use?", + choices=["ackley", "griewank", "rastrigin", "rosenbrock"], + default="rosenbrock", +) +parser.add_argument("-ID", help="Insert run_id here") +parser.add_argument("-inputs", help="Takes one input at a time", nargs="+") args = parser.parse_args() run_id = args.ID inputs = args.inputs +function_name = args.function + inputs = np.array(inputs).astype(np.float) -outputs = N_Rosenbrock(inputs) + +test_function = locals()[function_name] +outputs = test_function(inputs) results = {run_id: {"Inputs": inputs.tolist(), "Outputs": outputs.tolist()}} diff --git a/merlin/examples/workflows/optimization/scripts/visualizer.py b/merlin/examples/workflows/optimization/scripts/visualizer.py index 3187ec643..215ec0d46 100644 --- a/merlin/examples/workflows/optimization/scripts/visualizer.py +++ b/merlin/examples/workflows/optimization/scripts/visualizer.py @@ -1,19 +1,31 @@ import argparse +import matplotlib + + +matplotlib.use("pdf") +import ast +import pickle + import matplotlib.pyplot as plt import numpy as np -from joblib import load +from mpl_toolkits.mplot3d import Axes3D plt.style.use("seaborn-white") + parser = argparse.ArgumentParser("Learn surrogate model form simulation") parser.add_argument("-study_dir", help="The study directory, usually '$(MERLIN_WORKSPACE)'") +parser.add_argument( + "-scale", + help="ranges to scale results in form '[(min,max),(min, max)]'", +) args = parser.parse_args() study_dir = args.study_dir npz_path = f"{study_dir}/learner/all_iter_results.npz" -learner_path = f"{study_dir}/learner/surrogate.joblib" +learner_path = f"{study_dir}/learner/surrogate.pkl" new_samples_path = f"{study_dir}/pick_new_inputs/new_samples.npy" new_exploit_samples_path = f"{study_dir}/pick_new_inputs/new_exploit_samples.npy" new_explore_samples_path = f"{study_dir}/pick_new_inputs/new_explore_samples.npy" @@ -21,21 +33,12 @@ optimum_path = f"{study_dir}/optimizer/optimum.npy" old_best_path = f"{study_dir}/optimizer/old_best.npy" -from_file = np.load(npz_path, allow_pickle=True) - -data = from_file["arr_0"].item() +all_iter_results = np.load(npz_path, allow_pickle=True) -X = [] -y = [] +existing_X = all_iter_results["X"] +existing_y = all_iter_results["y"] -for i in data.keys(): - X.append(data[i]["Inputs"]) - y.append(data[i]["Outputs"]) - -existing_X = np.array(X) -existing_y = np.array(y) - -surrogate = load(learner_path) +surrogate = pickle.load(open(learner_path, "rb")) new_samples = np.load(new_samples_path) new_exploit_samples = np.load(new_exploit_samples_path) @@ -45,8 +48,24 @@ old_best = np.load(old_best_path) -def Rosenbrock_mesh(): - X_mesh_plot = np.array([np.linspace(-2, 2, n_points), np.linspace(-1, 3, n_points)]) +def process_scale(args): + if args.scale is not None: + raw = ast.literal_eval(args.scale) + processed = np.array(raw, dtype=float).tolist() + return processed + + +def rosenbrock_mesh(): + scales = process_scale(args) + print("args.scale", args.scale) + print("scales", scales) + limits = [] + for scale in scales: + limits.append((scale[0], scale[1])) + + X_mesh_plot = np.array( + [np.linspace(limits[0][0], limits[0][1], n_points), np.linspace(limits[1][0], limits[1][1], n_points)] + ) X_mesh = np.meshgrid(X_mesh_plot[0], X_mesh_plot[1]) Z_mesh = (1 - X_mesh[0]) ** 2 + 100 * (X_mesh[1] - X_mesh[0] ** 2) ** 2 @@ -57,7 +76,7 @@ def Rosenbrock_mesh(): # Script for N_dim Rosenbrock function n_points = 250 -X_mesh, Z_mesh = Rosenbrock_mesh() +X_mesh, Z_mesh = rosenbrock_mesh() Z_pred = surrogate.predict(np.c_[X_mesh[0].ravel(), X_mesh[1].ravel()]) Z_pred = Z_pred.reshape(X_mesh[0].shape) diff --git a/merlin/examples/workflows/optimization/template_config.py b/merlin/examples/workflows/optimization/template_config.py new file mode 100644 index 000000000..abbf45687 --- /dev/null +++ b/merlin/examples/workflows/optimization/template_config.py @@ -0,0 +1,67 @@ +import yaml +from jinja2 import Environment, FileSystemLoader, meta + + +# get all variable in template file +def get_variables(filename): + env = Environment(loader=FileSystemLoader("./")) + template_source = env.loader.get_source(env, filename)[0] + parsed_content = env.parse(template_source) + + return meta.find_undeclared_variables(parsed_content) + + +def get_dict_from_yaml(filename, get_template=True): + env = Environment(loader=FileSystemLoader("./")) + template = env.get_template(filename) + outputText = template.render() + + if get_template: + return yaml.safe_load(outputText), template + return yaml.safe_load(outputText) + + +def get_bounds_X(test_function): + return { + "rosenbrock": str([[-2, 2] for i in range(N_DIMS)]).replace(" ", ""), + "ackley": str([[-5, 5] for i in range(2)]).replace(" ", ""), + "rastrigin": str([[-5.12, 5.12] for i in range(N_DIMS)]).replace(" ", ""), + "griewank": str([[-10, 10] for i in range(N_DIMS)]).replace(" ", ""), + }[test_function] + + +filename = "template_optimization.yaml" + +workflow_dict, template = get_dict_from_yaml(filename) +undefined_variables = get_variables(filename) + +TEST_FUNCTION = workflow_dict["env"]["variables"]["TEST_FUNCTION"] +DEBUG = workflow_dict["env"]["variables"]["DEBUG"] +N_DIMS = workflow_dict["env"]["variables"]["N_DIMS"] + +if (TEST_FUNCTION == "ackley") and (N_DIMS != 2): + raise Exception("The ackley function only accepts 2 dims, change the N_DIMS variable") + +bounds_x = get_bounds_X(TEST_FUNCTION) +uncerts_x = [0.1 for i in range(N_DIMS)] +column_labels = [f"INPUT_{i + 1}" for i in range(N_DIMS)] + +if DEBUG: + # Reduce the number of iterations and the number of samples per iteration + max_iter = 3 + + n_samples = 2 + n_exploit = 2 + n_samples_line = 2 + n_samples_start = 8 + +defined_variables = {} +for undefined_variable in undefined_variables: + if undefined_variable in locals(): + defined_variables[undefined_variable] = globals()[undefined_variable] + else: + print(f"Variable '{undefined_variable}' is not defined, using default specified in yaml file") + +# to save the results +with open("optimization.yaml", "w") as fh: + fh.write(template.render(defined_variables)) diff --git a/merlin/examples/workflows/optimization/template_optimization.yaml b/merlin/examples/workflows/optimization/template_optimization.yaml new file mode 100644 index 000000000..3aa00900f --- /dev/null +++ b/merlin/examples/workflows/optimization/template_optimization.yaml @@ -0,0 +1,195 @@ +description: + name: $(WORKFLOW_NAME)_ITER_$(ITER) + description: |- + Design Optimization Template + To use, + 1. Specify the first three variables here (N_DIMS, TEST_FUNCTION, DEBUG) + 2. Run the template_config file in current directory using `python template_config.py` + 3. Merlin run as usual (merlin run optimization.yaml) + * MAX_ITER and the N_SAMPLES options use default values unless using DEBUG mode + * BOUNDS_X and UNCERTS_X are configured using the template_config.py scripts + + +env: + variables: + # These three are necessary for template_config.py + N_DIMS: 2 + TEST_FUNCTION: "rosenbrock" + DEBUG: 0 + SEED: 4321 + + METHOD: 'trust-constr' + EMAIL_ADDRESS: "NONE" #enter your email address here, or override this with --vars + RUN_TYPE: "run --local" + + ITER: 1 + MAX_ITER: "{{ max_iter|default(2) }}" + PREV_TIMESTAMP: 0 + PREV_ITER: 0 + + BOUNDS_X: "{{ bounds_x }}" + UNCERTS_X: "{{ uncerts_x }}" + + # pick_new_inputs step + SEARCH_SCALE: 0.30 + N_SAMPLES: "{{ n_samples|default(3) }}" # Number of new samples per iteration around the predicted new point + N_EXPLOIT: "{{ n_exploit|default(3) }}" # Number of new samples per iteration around the current best + N_SAMPLES_LINE: "{{ n_samples_line|default(3) }}" # Number between predicted best and current best + N_SAMPLES_START: "{{ n_samples_start|default(12) }}" # Number to initialize + + WORKFLOW_NAME: $(TEST_FUNCTION)_optimization + SCRIPTS: $(SPECROOT)/scripts + OUTPUT_PATH: ~/optimization_runs + PREV_WORKSPACE_DIR: 0 + + PYTHON: /usr/WS2/mubarka1/venvs/forked_merlin/merlin/venv_merlin_py_3_7/bin/python + + +study: + - name: run_simulation + description: Run the desired simulation + run: + cmd: |- + $(PYTHON) $(SCRIPTS)/test_functions.py -function $(TEST_FUNCTION) -ID "$(ITER)_$(MERLIN_SAMPLE_ID)" -inputs {% for column_label in column_labels %} $({{ column_label }}) {% endfor %} + + cores per task: 1 + nodes: 1 + procs: 1 + task_queue: simulation + + - name: collector + description: Collect the results into a single file and make an npz of some features + run: + cmd: |- + $(PYTHON) $(SCRIPTS)/collector.py -sim_dirs "$(run_simulation.workspace)/$(MERLIN_GLOB_PATH)" + nodes: 1 + procs: 1 + depends: [run_simulation_*] + task_queue: simulation_postprocess + + - name: clean_up_simulation + description: Cleans up the merlin sample paths of the run_simulation step + run: + cmd: |- + #rm -rf $(run_simulation.workspace)/$(MERLIN_SAMPLE_PATH) + echo "skipped" + + cores per task: 1 + nodes: 1 + procs: 1 + depends: [collector] + task_queue: simulation_postprocess + + - name: learner + description: Train an ML model on the simulation + run: + cmd: |- + cp $(collector.workspace)/current_results.npz current_results.npz + if [ $(ITER) -ge "2" ] ; then + echo "Copying the npz file from previous iteration" + spellbook stack-npz all_iter_results.npz $(PREV_WORKSPACE_DIR)/collector/*.npz current_results.npz + else + cp current_results.npz all_iter_results.npz + fi + $(PYTHON) -c "import numpy as np; all_results_npz=np.load('all_iter_results.npz');np.savez('learner.npz',X=all_results_npz['X'],y=all_results_npz['y'].ravel())" + + spellbook learn -regressor RandomForestRegressor -infile learner.npz -outfile surrogate.pkl + + cores per task: 1 + nodes: 1 + procs: 1 + depends: [clean_up_simulation] + task_queue: learner + + - name: optimizer + description: Optimizer + run: + cmd: |- + $(PYTHON) $(SCRIPTS)/optimizer.py -learner_dir "$(learner.workspace)" -bounds "$(BOUNDS_X)" -input_uncerts "$(UNCERTS_X)" -method "$(METHOD)" + + cores per task: 1 + nodes: 1 + procs: 1 + depends: [collector, learner] + task_queue: learner + + - name: pick_new_inputs + description: Picking new simulations to run in the next iteration + run: + cmd: |- + spellbook make-samples -dims $(N_DIMS) -x0 $(optimizer.workspace)/optimum.npy -x1 $(optimizer.workspace)/old_best.npy -n_line $(N_SAMPLES_LINE) -scale_factor $(SEARCH_SCALE) -scale "$(BOUNDS_X)" -n $(N_SAMPLES) -sample_type lhd -outfile new_explore_samples.npy -seed $(SEED) --hard-bounds + spellbook make-samples -dims $(N_DIMS) -x0 $(optimizer.workspace)/optimum.npy -scale_factor 0.05 -scale "$(BOUNDS_X)" -sample_type star -outfile new_explore_star_samples.npy -seed $(SEED) --hard-bounds + # Add points near current best too + spellbook make-samples -dims $(N_DIMS) -x0 $(optimizer.workspace)/old_best.npy -scale_factor $(SEARCH_SCALE) -scale "$(BOUNDS_X)" -n $(N_EXPLOIT) -sample_type lhd -outfile new_exploit_samples.npy -seed $(SEED) --hard-bounds + + # combine them + $(PYTHON) -c "import numpy as np; np.save('new_samples.npy',np.vstack((np.load('new_explore_samples.npy'),np.load('new_exploit_samples.npy'),np.load('new_explore_star_samples.npy'))))" + + cores per task: 1 + nodes: 1 + procs: 1 + depends: [optimizer] + task_queue: learner + + - name: visualizer + description: Either launches new simulations or iterated with inputs from previous step + run: + cmd: |- + # Add an if statment to make sure it is rosenbrock function and 2D + if [ $(N_DIMS) -ne "2" ] ; then + echo "We currently don't have a way of visualizing anything other than 2D" + else + $(PYTHON) $(SCRIPTS)/visualizer.py -study_dir $(MERLIN_WORKSPACE) -scale "$(BOUNDS_X)" + fi + + cores per task: 1 + nodes: 1 + procs: 1 + depends: [pick_new_inputs] + task_queue: learner + + - name: iterate + description: Either launches new simulations or iterated with inputs from previous step + run: + cmd: |- + mv $(optimizer.workspace)/optimization_results.json optimization_results_iter_$(ITER).json + mv $(visualizer.workspace)/results.png results_iter_$(ITER).png + + # Checking if e-mail address is present + if [ "$(EMAIL_ADDRESS)" = "NONE" ]; then + echo "Done iteration $(ITER) in $(MERLIN_WORKSPACE)" + else + echo "Done iteration $(ITER) in $(MERLIN_WORKSPACE)" | mail -s "Merlin Status for $(WORKFLOW_NAME)" -a optimization_results_iter_$(ITER).json -a results_iter_$(ITER).png $(EMAIL_ADDRESS) + fi + + if [ $(ITER) -ge $(MAX_ITER) ] ; then + echo "Max iterations reached" + else + next_iter=$(ITER) + ((next_iter=next_iter+1)) + echo "Starting iteration " $next_iter + merlin $(RUN_TYPE) $(MERLIN_INFO)/*partial.yaml --samplesfile $(pick_new_inputs.workspace)/new_samples.npy --vars ITER=$next_iter PREV_TIMESTAMP=$(MERLIN_TIMESTAMP) PREV_ITER=$(ITER) PREV_WORKSPACE_DIR=$(MERLIN_WORKSPACE) + fi + cores per task: 1 + nodes: 1 + procs: 1 + depends: [visualizer] + task_queue: learner + +merlin: + resources: + overlap: true + task_server: celery + workers: + all_workers: + args: -O fair --prefetch-multiplier 1 -E -l info --concurrency 20 + steps: [all] + samples: + column_labels: + {% for column_label in column_labels %} + - {{ column_label }} + {% endfor %} + file: $(MERLIN_INFO)/samples.npy + generate: + cmd: |- + spellbook make-samples -dims $(N_DIMS) -n $(N_SAMPLES_START) -sample_type lhs -outfile=$(MERLIN_INFO)/samples.npy -scale "$(BOUNDS_X)" -seed $(SEED) From c4d2524e5fe3c9d3d817c9f2bae249f208456ab5 Mon Sep 17 00:00:00 2001 From: Alexander Winter <80929291+AlexanderWinterLLNL@users.noreply.github.com> Date: Fri, 22 Oct 2021 14:55:54 -0700 Subject: [PATCH 012/126] Release 182 (#338) * Used make fix-style to make the release style compliant, updated version number from 1.8.1 to 1.8.2, updated CHANGELOG.md with 1.8.2 and did minor correction for categorization of a patch note. Co-authored-by: Alexander Cameron Winter --- CHANGELOG.md | 13 ++++++++----- Makefile | 2 +- merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- .../common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- .../remote_feature_demo/scripts/hello_world.py | 17 +++++------------ merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 3 +-- 48 files changed, 60 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a05ad88e4..56117b8c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,16 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -### Fixed +## [1.8.2] + +### Added - Re-enabled distributed integration testing. Added additional examination to distributed testing. +### Fixed +- 'shell' added to unsupported and new_unsupported lists in script_adapter.py, prevents + `'shell' is not supported -- ommitted` message. +- Makefile target for install-merlin fixed so venv is properly activated to install merlin + ### Changed - Updated the optimization workflow example with a new python template editor script - CI now splits linting and testing into different tasks for better utilization of @@ -20,10 +27,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - PyLint adherence for: celery.py, opennplib.py, config/__init__.py, broker.py, configfile.py, formatter.py, main.py, router.py - Integrated Black and isort into CI -### Fixed -- 'shell' added to unsupported and new_unsupported lists in script_adapter.py, prevents - `'shell' is not supported -- ommitted` message. -- Makefile target for install-merlin fixed so venv is properly activated to install merlin ## [1.8.1] diff --git a/Makefile b/Makefile index d3036deb9..1c11d8a91 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index b94a89202..34a8c8567 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.8.1" +__version__ = "1.8.2" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 5b73d4a38..5a542bb30 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index e8c0bc0ee..acb552d7a 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index b0ab8ba53..94aba9638 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index b0ab8ba53..94aba9638 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index a3becf73a..1afb968ba 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index b1d66645e..40f04d6e5 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index cc53e754f..e5cf457a1 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 27ee41c44..debfb2286 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 46db3a537..0ab81b45c 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index b0ab8ba53..94aba9638 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 554fd87e4..81ba97f8e 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index ec540bf23..756098fb3 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index c79c29049..a9665b85b 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index acf06b74b..b4dba2464 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 7ea6493a9..a529bdf68 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index c3571d1df..e33e647fd 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 34783443e..553e9b90a 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 05c118a9f..e63c51cb5 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index b0ab8ba53..94aba9638 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index d2fabd364..dfbcfd18f 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index b0ab8ba53..94aba9638 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 152fe8763..6f4f4b1e6 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 460f31e55..ead50d579 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py b/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py index 565806ce4..0e2b624d6 100644 --- a/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py +++ b/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py @@ -3,6 +3,7 @@ import sys from typing import Dict + def process_args(args: argparse.Namespace) -> None: """ Writes a json file of the parsed args after doing some trivial math. @@ -24,18 +25,10 @@ def setup_argparse() -> argparse.ArgumentParser: This method sets up the argparser. """ parser: argparse.ArgumentParser = argparse.ArgumentParser(description="Process some integers.") - parser.add_argument( - "X", metavar="X", type=float, help="The x dimension of the sample." - ) - parser.add_argument( - "Y", metavar="Y", type=float, help="The y dimension of the sample." - ) - parser.add_argument( - "Z", metavar="Z", type=float, help="The z dimension of the sample." - ) - parser.add_argument( - "-outfile", help="Output file name", default="hello_world_output.json" - ) + parser.add_argument("X", metavar="X", type=float, help="The x dimension of the sample.") + parser.add_argument("Y", metavar="Y", type=float, help="The y dimension of the sample.") + parser.add_argument("Z", metavar="Z", type=float, help="The z dimension of the sample.") + parser.add_argument("-outfile", help="Output file name", default="hello_world_output.json") return parser diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 2481247df..b37a730bf 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index b336e7715..0ed5cb2e7 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 9ef8e96ef..cbf4c47e2 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 6fc325184..cdc1728b6 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 66742b56d..28a1d0e12 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index b0ab8ba53..94aba9638 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index ea8483682..cab03ffa5 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 50506dfec..b8b5aecd2 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index f03b17382..e5f5f277d 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 714a7d30a..1eac39044 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index b0ab8ba53..94aba9638 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 8e6648638..8bb8c4a44 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index ead61ebe8..42137357a 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 067334fcb..4e580918c 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 99e45fdb3..cc4f8b658 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index adac133ca..74136e57d 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 487b69d37..b31ddab31 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index 19c840913..e30ce64e6 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index 40c172216..179ed2888 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index f058efd0d..c0ffef5f1 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.1. +# This file is part of Merlin, Version: 1.8.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index ef3b7ac1b..ce525aafc 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -339,7 +339,6 @@ def define_tests(): ], ), # this test is deactivated until the --spec option for stop-workers is active again - # "stop workers for distributed feature_demo": ( # f"{run} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; {workers} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; sleep 20 ; merlin stop-workers --spec {demo}", # [ @@ -375,7 +374,7 @@ def define_tests(): # provenence_equality_checks, # omitting provenance equality check because it is broken # style_checks, # omitting style checks due to different results on different machines dependency_checks, - distributed_tests + distributed_tests, ]: all_tests.update(test_dict) From cb67c5cde06604d49ab1ef07972c88a82d2b66f7 Mon Sep 17 00:00:00 2001 From: Alexander Cameron Winter Date: Fri, 22 Oct 2021 20:18:22 -0700 Subject: [PATCH 013/126] merge-conflict resolution introduced a few errors/tidiness problems, this resolves them: duplicate imports removed from merlin/common/opennpylib.py, removed duplicated Makefile targets, re-ordered for a better Makefile target flow after merge disordered. --- Makefile | 11 ----------- merlin/common/opennpylib.py | 2 -- 2 files changed, 13 deletions(-) diff --git a/Makefile b/Makefile index c96549d64..1c11d8a91 100644 --- a/Makefile +++ b/Makefile @@ -121,14 +121,6 @@ check-flake8: $(PYTHON) -m flake8 . --count --max-complexity=15 --statistics --max-line-length=127; \ -# run code style checks -check-style: - . $(VENV)/bin/activate - -$(PYTHON) -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics - -$(PYTHON) -m flake8 . --count --max-complexity=15 --statistics --max-line-length=127 - -black --check --target-version py36 $(MRLN) - - check-black: . $(VENV)/bin/activate; \ $(PYTHON) -m black --check --line-length $(MAX_LINE_LENGTH) --target-version py36 $(MRLN); \ @@ -136,9 +128,6 @@ check-black: $(PYTHON) -m black --check --line-length $(MAX_LINE_LENGTH) --target-version py36 *.py; \ -check-push: tests check-style - - check-isort: . $(VENV)/bin/activate; \ $(PYTHON) -m isort --check --line-length $(MAX_LINE_LENGTH) merlin; \ diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 53e07d19c..f14061da1 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -83,8 +83,6 @@ """ from typing import List, Tuple -from typing import List, Tuple - import numpy as np From 05bc6bc9ab41d13c9347d3255edf10a815903883 Mon Sep 17 00:00:00 2001 From: Yamen Mubarka Date: Mon, 22 Nov 2021 11:05:01 -0800 Subject: [PATCH 014/126] Fixing the 'merlin example list' error (#348) * changed the template extension, added 'optimization.yaml' for the merlin example list function * Updated changelog and added a basic optimization workflow that does not get changed with jinja * added a test for merlin example list * complying with lint * complying with lint * removed specific python references in favor of 'python3' * Fixed the discription for the visualize step --- CHANGELOG.md | 9 +- .../optimization/optimization_basic.yaml | 195 ++++++++++++++++++ .../optimization/scripts/visualizer.py | 2 +- .../workflows/optimization/template_config.py | 2 +- ...zation.yaml => template_optimization.temp} | 16 +- tests/integration/test_definitions.py | 8 + 6 files changed, 220 insertions(+), 12 deletions(-) create mode 100644 merlin/examples/workflows/optimization/optimization_basic.yaml rename merlin/examples/workflows/optimization/{template_optimization.yaml => template_optimization.temp} (86%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56117b8c7..18be89a98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,15 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.8.2] +## [Unreleased] +### Added +- Unreleased section in the changelog +- Test for `merlin example list` +### Fixed +- The Optimization workflow example now has a ready to use workflow (`optimization_basic.yaml`). This solves the issue faced before with `merlin example list`. + +## [1.8.2] ### Added - Re-enabled distributed integration testing. Added additional examination to distributed testing. diff --git a/merlin/examples/workflows/optimization/optimization_basic.yaml b/merlin/examples/workflows/optimization/optimization_basic.yaml new file mode 100644 index 000000000..2570b113b --- /dev/null +++ b/merlin/examples/workflows/optimization/optimization_basic.yaml @@ -0,0 +1,195 @@ +description: + name: $(WORKFLOW_NAME)_ITER_$(ITER) + description: |- + Design Optimization Template + To use, + 1. Specify the first three variables here (N_DIMS, TEST_FUNCTION, DEBUG) + 2. Run the template_config file in current directory using `python template_config.py` + 3. Merlin run as usual (merlin run optimization.yaml) + * MAX_ITER and the N_SAMPLES options use default values unless using DEBUG mode + * BOUNDS_X and UNCERTS_X are configured using the template_config.py scripts + + +env: + variables: + # These three are necessary for template_config.py + N_DIMS: 2 + TEST_FUNCTION: "rosenbrock" + DEBUG: 0 + SEED: 4321 + + METHOD: 'trust-constr' + EMAIL_ADDRESS: "NONE" #enter your email address here, or override this with --vars + RUN_TYPE: "run --local" + + ITER: 1 + MAX_ITER: "2" + PREV_TIMESTAMP: 0 + PREV_ITER: 0 + + BOUNDS_X: "[[-2,2],[-2,2]]" + UNCERTS_X: "[0.1, 0.1]" + + # pick_new_inputs step + SEARCH_SCALE: 0.30 + N_SAMPLES: "3" # Number of new samples per iteration around the predicted new point + N_EXPLOIT: "3" # Number of new samples per iteration around the current best + N_SAMPLES_LINE: "3" # Number between predicted best and current best + N_SAMPLES_START: "12" # Number to initialize + + WORKFLOW_NAME: $(TEST_FUNCTION)_optimization + SCRIPTS: $(SPECROOT)/scripts + OUTPUT_PATH: ~/optimization_runs + PREV_WORKSPACE_DIR: 0 + + +study: + - name: run_simulation + description: Run the desired simulation + run: + cmd: |- + python3 $(SCRIPTS)/test_functions.py -function $(TEST_FUNCTION) -ID "$(ITER)_$(MERLIN_SAMPLE_ID)" -inputs $(INPUT_1) $(INPUT_2) + + cores per task: 1 + nodes: 1 + procs: 1 + task_queue: simulation + + - name: collector + description: Collect the results into a single file and make an npz of some features + run: + cmd: |- + python3 $(SCRIPTS)/collector.py -sim_dirs "$(run_simulation.workspace)/$(MERLIN_GLOB_PATH)" + nodes: 1 + procs: 1 + depends: [run_simulation_*] + task_queue: simulation_postprocess + + - name: clean_up_simulation + description: Cleans up the merlin sample paths of the run_simulation step + run: + cmd: |- + #rm -rf $(run_simulation.workspace)/$(MERLIN_SAMPLE_PATH) + echo "skipped" + + cores per task: 1 + nodes: 1 + procs: 1 + depends: [collector] + task_queue: simulation_postprocess + + - name: learner + description: Train an ML model on the simulation + run: + cmd: |- + cp $(collector.workspace)/current_results.npz current_results.npz + if [ $(ITER) -ge "2" ] ; then + echo "Copying the npz file from previous iteration" + spellbook stack-npz all_iter_results.npz $(PREV_WORKSPACE_DIR)/collector/*.npz current_results.npz + else + cp current_results.npz all_iter_results.npz + fi + python3 -c "import numpy as np; all_results_npz=np.load('all_iter_results.npz');np.savez('learner.npz',X=all_results_npz['X'],y=all_results_npz['y'].ravel())" + + spellbook learn -regressor RandomForestRegressor -infile learner.npz -outfile surrogate.pkl + + cores per task: 1 + nodes: 1 + procs: 1 + depends: [clean_up_simulation] + task_queue: learner + + - name: optimizer + description: Optimizer + run: + cmd: |- + python3 $(SCRIPTS)/optimizer.py -learner_dir "$(learner.workspace)" -bounds "$(BOUNDS_X)" -input_uncerts "$(UNCERTS_X)" -method "$(METHOD)" + + cores per task: 1 + nodes: 1 + procs: 1 + depends: [collector, learner] + task_queue: learner + + - name: pick_new_inputs + description: Picking new simulations to run in the next iteration + run: + cmd: |- + spellbook make-samples -dims $(N_DIMS) -x0 $(optimizer.workspace)/optimum.npy -x1 $(optimizer.workspace)/old_best.npy -n_line $(N_SAMPLES_LINE) -scale_factor $(SEARCH_SCALE) -scale "$(BOUNDS_X)" -n $(N_SAMPLES) -sample_type lhd -outfile new_explore_samples.npy -seed $(SEED) --hard-bounds + spellbook make-samples -dims $(N_DIMS) -x0 $(optimizer.workspace)/optimum.npy -scale_factor 0.05 -scale "$(BOUNDS_X)" -sample_type star -outfile new_explore_star_samples.npy -seed $(SEED) --hard-bounds + # Add points near current best too + spellbook make-samples -dims $(N_DIMS) -x0 $(optimizer.workspace)/old_best.npy -scale_factor $(SEARCH_SCALE) -scale "$(BOUNDS_X)" -n $(N_EXPLOIT) -sample_type lhd -outfile new_exploit_samples.npy -seed $(SEED) --hard-bounds + + # combine them + python3 -c "import numpy as np; np.save('new_samples.npy',np.vstack((np.load('new_explore_samples.npy'),np.load('new_exploit_samples.npy'),np.load('new_explore_star_samples.npy'))))" + + cores per task: 1 + nodes: 1 + procs: 1 + depends: [optimizer] + task_queue: learner + + - name: visualizer + description: Either launches new simulations or iterated with inputs from previous step + run: + cmd: |- + # Add an if statment to make sure it is rosenbrock function and 2D + if [ $(N_DIMS) -ne "2" ] ; then + echo "We currently don't have a way of visualizing anything other than 2D" + else + python3 $(SCRIPTS)/visualizer.py -study_dir $(MERLIN_WORKSPACE) -scale "$(BOUNDS_X)" + fi + + cores per task: 1 + nodes: 1 + procs: 1 + depends: [pick_new_inputs] + task_queue: learner + + - name: iterate + description: Either launches new simulations or iterated with inputs from previous step + run: + cmd: |- + mv $(optimizer.workspace)/optimization_results.json optimization_results_iter_$(ITER).json + mv $(visualizer.workspace)/results.png results_iter_$(ITER).png + + # Checking if e-mail address is present + if [ "$(EMAIL_ADDRESS)" = "NONE" ]; then + echo "Done iteration $(ITER) in $(MERLIN_WORKSPACE)" + else + echo "Done iteration $(ITER) in $(MERLIN_WORKSPACE)" | mail -s "Merlin Status for $(WORKFLOW_NAME)" -a optimization_results_iter_$(ITER).json -a results_iter_$(ITER).png $(EMAIL_ADDRESS) + fi + + if [ $(ITER) -ge $(MAX_ITER) ] ; then + echo "Max iterations reached" + else + next_iter=$(ITER) + ((next_iter=next_iter+1)) + echo "Starting iteration " $next_iter + merlin $(RUN_TYPE) $(MERLIN_INFO)/*partial.yaml --samplesfile $(pick_new_inputs.workspace)/new_samples.npy --vars ITER=$next_iter PREV_TIMESTAMP=$(MERLIN_TIMESTAMP) PREV_ITER=$(ITER) PREV_WORKSPACE_DIR=$(MERLIN_WORKSPACE) + fi + cores per task: 1 + nodes: 1 + procs: 1 + depends: [visualizer] + task_queue: learner + +merlin: + resources: + overlap: true + task_server: celery + workers: + all_workers: + args: -O fair --prefetch-multiplier 1 -E -l info --concurrency 20 + steps: [all] + samples: + column_labels: + + - INPUT_1 + + - INPUT_2 + + file: $(MERLIN_INFO)/samples.npy + generate: + cmd: |- + spellbook make-samples -dims $(N_DIMS) -n $(N_SAMPLES_START) -sample_type lhs -outfile=$(MERLIN_INFO)/samples.npy -scale "$(BOUNDS_X)" -seed $(SEED) \ No newline at end of file diff --git a/merlin/examples/workflows/optimization/scripts/visualizer.py b/merlin/examples/workflows/optimization/scripts/visualizer.py index 215ec0d46..f41885bf9 100644 --- a/merlin/examples/workflows/optimization/scripts/visualizer.py +++ b/merlin/examples/workflows/optimization/scripts/visualizer.py @@ -15,7 +15,7 @@ plt.style.use("seaborn-white") -parser = argparse.ArgumentParser("Learn surrogate model form simulation") +parser = argparse.ArgumentParser("Visualize the surrogate response surface in comparison to the analytic function") parser.add_argument("-study_dir", help="The study directory, usually '$(MERLIN_WORKSPACE)'") parser.add_argument( "-scale", diff --git a/merlin/examples/workflows/optimization/template_config.py b/merlin/examples/workflows/optimization/template_config.py index abbf45687..201ecb5d4 100644 --- a/merlin/examples/workflows/optimization/template_config.py +++ b/merlin/examples/workflows/optimization/template_config.py @@ -30,7 +30,7 @@ def get_bounds_X(test_function): }[test_function] -filename = "template_optimization.yaml" +filename = "template_optimization.temp" workflow_dict, template = get_dict_from_yaml(filename) undefined_variables = get_variables(filename) diff --git a/merlin/examples/workflows/optimization/template_optimization.yaml b/merlin/examples/workflows/optimization/template_optimization.temp similarity index 86% rename from merlin/examples/workflows/optimization/template_optimization.yaml rename to merlin/examples/workflows/optimization/template_optimization.temp index 3aa00900f..df9b56e46 100644 --- a/merlin/examples/workflows/optimization/template_optimization.yaml +++ b/merlin/examples/workflows/optimization/template_optimization.temp @@ -41,16 +41,14 @@ env: SCRIPTS: $(SPECROOT)/scripts OUTPUT_PATH: ~/optimization_runs PREV_WORKSPACE_DIR: 0 - - PYTHON: /usr/WS2/mubarka1/venvs/forked_merlin/merlin/venv_merlin_py_3_7/bin/python - + study: - name: run_simulation description: Run the desired simulation run: cmd: |- - $(PYTHON) $(SCRIPTS)/test_functions.py -function $(TEST_FUNCTION) -ID "$(ITER)_$(MERLIN_SAMPLE_ID)" -inputs {% for column_label in column_labels %} $({{ column_label }}) {% endfor %} + python3 $(SCRIPTS)/test_functions.py -function $(TEST_FUNCTION) -ID "$(ITER)_$(MERLIN_SAMPLE_ID)" -inputs {% for column_label in column_labels %} $({{ column_label }}) {% endfor %} cores per task: 1 nodes: 1 @@ -61,7 +59,7 @@ study: description: Collect the results into a single file and make an npz of some features run: cmd: |- - $(PYTHON) $(SCRIPTS)/collector.py -sim_dirs "$(run_simulation.workspace)/$(MERLIN_GLOB_PATH)" + python3 $(SCRIPTS)/collector.py -sim_dirs "$(run_simulation.workspace)/$(MERLIN_GLOB_PATH)" nodes: 1 procs: 1 depends: [run_simulation_*] @@ -91,7 +89,7 @@ study: else cp current_results.npz all_iter_results.npz fi - $(PYTHON) -c "import numpy as np; all_results_npz=np.load('all_iter_results.npz');np.savez('learner.npz',X=all_results_npz['X'],y=all_results_npz['y'].ravel())" + python3 -c "import numpy as np; all_results_npz=np.load('all_iter_results.npz');np.savez('learner.npz',X=all_results_npz['X'],y=all_results_npz['y'].ravel())" spellbook learn -regressor RandomForestRegressor -infile learner.npz -outfile surrogate.pkl @@ -105,7 +103,7 @@ study: description: Optimizer run: cmd: |- - $(PYTHON) $(SCRIPTS)/optimizer.py -learner_dir "$(learner.workspace)" -bounds "$(BOUNDS_X)" -input_uncerts "$(UNCERTS_X)" -method "$(METHOD)" + python3 $(SCRIPTS)/optimizer.py -learner_dir "$(learner.workspace)" -bounds "$(BOUNDS_X)" -input_uncerts "$(UNCERTS_X)" -method "$(METHOD)" cores per task: 1 nodes: 1 @@ -123,7 +121,7 @@ study: spellbook make-samples -dims $(N_DIMS) -x0 $(optimizer.workspace)/old_best.npy -scale_factor $(SEARCH_SCALE) -scale "$(BOUNDS_X)" -n $(N_EXPLOIT) -sample_type lhd -outfile new_exploit_samples.npy -seed $(SEED) --hard-bounds # combine them - $(PYTHON) -c "import numpy as np; np.save('new_samples.npy',np.vstack((np.load('new_explore_samples.npy'),np.load('new_exploit_samples.npy'),np.load('new_explore_star_samples.npy'))))" + python3 -c "import numpy as np; np.save('new_samples.npy',np.vstack((np.load('new_explore_samples.npy'),np.load('new_exploit_samples.npy'),np.load('new_explore_star_samples.npy'))))" cores per task: 1 nodes: 1 @@ -139,7 +137,7 @@ study: if [ $(N_DIMS) -ne "2" ] ; then echo "We currently don't have a way of visualizing anything other than 2D" else - $(PYTHON) $(SCRIPTS)/visualizer.py -study_dir $(MERLIN_WORKSPACE) -scale "$(BOUNDS_X)" + python3 $(SCRIPTS)/visualizer.py -study_dir $(MERLIN_WORKSPACE) -scale "$(BOUNDS_X)" fi cores per task: 1 diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index ce525aafc..dbdffd2cd 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -45,6 +45,13 @@ def define_tests(): "local", ), } + examples_check = { + "example list": ( + "merlin example list", + HasReturnCode(), + "local", + ), + } run_workers_echo_tests = { "run-workers echo simple_chain": ( f"{workers} {simple} --echo", @@ -364,6 +371,7 @@ def define_tests(): all_tests = {} for test_dict in [ basic_checks, + examples_check, run_workers_echo_tests, wf_format_tests, example_tests, From cc17d69f780de1e39f337b096fd47be5d3535b27 Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Wed, 12 Jan 2022 10:56:37 -0800 Subject: [PATCH 015/126] Bump github testing to include python 3.10 (#350) * Bump github testing to include python 3.10 * Update CHANGELOG * Python version need to be strings * Add to distributed test suite too --- .github/workflows/push-pr_workflow.yml | 4 ++-- CHANGELOG.md | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 479b18cc9..3b2f809eb 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -70,7 +70,7 @@ jobs: strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v2 @@ -132,7 +132,7 @@ jobs: strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 18be89a98..5c13b2f5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Unreleased section in the changelog - Test for `merlin example list` +- Python 3.10 to testing ### Fixed - The Optimization workflow example now has a ready to use workflow (`optimization_basic.yaml`). This solves the issue faced before with `merlin example list`. From 530312110e5731eae35cd4e167c0ec0ee40b114d Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Wed, 12 Jan 2022 15:32:20 -0800 Subject: [PATCH 016/126] Bugfix/redis version (#349) * Don't need to add redis manually b/c celery will get the right version * Update changelog Co-authored-by: Kasey Nagle <88013612+KaseyNagleLLNL@users.noreply.github.com> --- CHANGELOG.md | 1 + requirements/release.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c13b2f5c..6ceb069ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - The Optimization workflow example now has a ready to use workflow (`optimization_basic.yaml`). This solves the issue faced before with `merlin example list`. +- Redis dependency handled implictly by celery for cross-compatibility ## [1.8.2] ### Added diff --git a/requirements/release.txt b/requirements/release.txt index 48e468993..4771b7a4c 100644 --- a/requirements/release.txt +++ b/requirements/release.txt @@ -8,5 +8,4 @@ numpy parse psutil>=5.1.0 pyyaml>=5.1.2 -redis tabulate From 7d9505fffcc055dbe077c78e9e0ee6ab0d0270e3 Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Wed, 12 Jan 2022 16:08:36 -0800 Subject: [PATCH 017/126] Bump to version 1.8.3 (#351) --- CHANGELOG.md | 3 ++- Makefile | 2 +- merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/run_tests.py | 2 +- 46 files changed, 48 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ceb069ef..010dbd254 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## [1.8.3] ### Added -- Unreleased section in the changelog - Test for `merlin example list` - Python 3.10 to testing diff --git a/Makefile b/Makefile index 1c11d8a91..2f5086ce9 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index 34a8c8567..75513329f 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.8.2" +__version__ = "1.8.3" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 5a542bb30..8acd698b4 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index acb552d7a..c2e98f3a3 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 94aba9638..f78f9c91a 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 94aba9638..f78f9c91a 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index 1afb968ba..e16c45498 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 40f04d6e5..4c87bb978 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index f14061da1..bcb298035 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index debfb2286..6d4511307 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 0ab81b45c..e23fb9081 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 94aba9638..f78f9c91a 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 81ba97f8e..7c12a70d4 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 756098fb3..1c566a377 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index a9665b85b..0737bd887 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index b4dba2464..2c71cee2d 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index a529bdf68..47945b01b 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index e33e647fd..4084d13c2 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 553e9b90a..2d5141ae3 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index e63c51cb5..01c210a7f 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 94aba9638..f78f9c91a 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index dfbcfd18f..2ad038034 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 94aba9638..f78f9c91a 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 6f4f4b1e6..bbbad64d9 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index ead50d579..d1ce08b4e 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index b37a730bf..8fac45395 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 0ed5cb2e7..cbb285618 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index cbf4c47e2..8a7b5d302 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index cdc1728b6..f9885b242 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 28a1d0e12..85e298510 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 94aba9638..f78f9c91a 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index cab03ffa5..cd4bf238c 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index b8b5aecd2..ef3f6da3e 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index e5f5f277d..6d4ec0329 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 1eac39044..eb77c3157 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 94aba9638..f78f9c91a 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 8bb8c4a44..dec4cfac4 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 42137357a..eef29aa63 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 4e580918c..31ef0f08b 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index cc4f8b658..07c4622b9 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 74136e57d..af331c3cf 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index b31ddab31..1cf1e2adc 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index e30ce64e6..b35ef2b77 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index 179ed2888..e9ed37693 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index c0ffef5f1..f92f20354 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.2. +# This file is part of Merlin, Version: 1.8.3. # # For details, see https://github.com/LLNL/merlin. # From 75dde86f138e3b3a985d862cc4f01b0e8e53ff83 Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Wed, 26 Jan 2022 17:06:18 -0800 Subject: [PATCH 018/126] Create python-publish.yml (#353) Adding GitHub action for publishing to pypi --- .github/workflows/python-publish.yml | 36 ++++++++++++++++++++++++++++ CHANGELOG.md | 2 ++ Makefile | 2 +- requirements/dev.txt | 1 + setup.py | 1 + 5 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/python-publish.yml diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 000000000..3bfabfc12 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,36 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 010dbd254..e958caf51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Auto-release of pypi packages ## [1.8.3] ### Added diff --git a/Makefile b/Makefile index 2f5086ce9..5607e32fc 100644 --- a/Makefile +++ b/Makefile @@ -191,7 +191,7 @@ reqlist: release: - $(PYTHON) setup.py sdist bdist_wheel + $(PYTHON) -m build . clean-release: diff --git a/requirements/dev.txt b/requirements/dev.txt index c66854a61..9321694f8 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,4 +1,5 @@ # Development dependencies. +build black dep-license flake8 diff --git a/setup.py b/setup.py index e9ed37693..e44af0d6d 100644 --- a/setup.py +++ b/setup.py @@ -100,6 +100,7 @@ def extras_require(): "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], keywords="machine learning workflow", url="https://github.com/LLNL/merlin", From 05cdecba6f9292d2f38cec328374f987463a5f4d Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Thu, 3 Feb 2022 13:56:08 -0800 Subject: [PATCH 019/126] Workflows Community Initiative Metadata (#355) * Added Workflows Community Initiative metadata info; fixed some old links * Run black --- .wci.yml | 12 ++++++++++++ CHANGELOG.md | 4 ++++ README.md | 4 ++-- docs/images/merlin_icon.png | Bin 0 -> 75969 bytes docs/source/modules/contribute.rst | 2 +- .../workflows/feature_demo/scripts/pgen.py | 2 +- .../workflows/null_spec/scripts/launch_jobs.py | 4 ++-- .../workflows/openfoam_wf/scripts/learn.py | 2 +- .../openfoam_wf_no_docker/scripts/learn.py | 2 +- .../optimization/scripts/test_functions.py | 6 +++--- .../remote_feature_demo/scripts/pgen.py | 2 +- 11 files changed, 28 insertions(+), 12 deletions(-) create mode 100644 .wci.yml create mode 100644 docs/images/merlin_icon.png diff --git a/.wci.yml b/.wci.yml new file mode 100644 index 000000000..0ced2c93b --- /dev/null +++ b/.wci.yml @@ -0,0 +1,12 @@ +name: Merlin + +icon: https://raw.githubusercontent.com/LLNL/merlin/main/docs/images/merlin_icon.png + +headline: Enabling Machine Learning HPC Workflows + +description: The Merlin workflow framework targets large-scale scientific machine learning (ML) workflows in High Performance Computing (HPC) environments. Merlin is a producer-consumer workflow model that enables multi-machine, cross-batch job, dynamically allocated yet persistent workflows capable of utilizing surge-compute resources. Key features are a flexible and intuitive HPC-centric interface, low per-task overhead, multi-tiered fault recovery, and a hierarchical sampling algorithm that allows for highly scalable task execution and queuing to ensembles of millions of tasks. + +documentation: + general: https://merlin.readthedocs.io/ + installation: https://merlin.readthedocs.io/en/latest/modules/installation/installation.html + tutorial: https://merlin.readthedocs.io/en/latest/tutorial.html diff --git a/CHANGELOG.md b/CHANGELOG.md index e958caf51..6a3af59a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - Auto-release of pypi packages +- Workflows Community Initiative metadata file + +### Fixed +- Old references to stale branches ## [1.8.3] ### Added diff --git a/README.md b/README.md index dbbd57707..fad1de93f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Issues](https://img.shields.io/github/issues/LLNL/merlin)](https://github.com/LLNL/merlin/issues) [![Pull requests](https://img.shields.io/github/issues-pr/LLNL/merlin)](https://github.com/LLNL/merlin/pulls) -![Merlin](https://raw.githubusercontent.com/LLNL/merlin/master/docs/images/merlin.png) +![Merlin](https://raw.githubusercontent.com/LLNL/merlin/main/docs/images/merlin.png) ## A brief introduction to Merlin Merlin is a tool for running machine learning based workflows. The goal of @@ -133,6 +133,6 @@ the Merlin community, you agree to abide by its rules. ## License -Merlin is distributed under the terms of the [MIT LICENSE](https://github.com/LLNL/merlin/blob/master/LICENSE). +Merlin is distributed under the terms of the [MIT LICENSE](https://github.com/LLNL/merlin/blob/main/LICENSE). LLNL-CODE-797170 diff --git a/docs/images/merlin_icon.png b/docs/images/merlin_icon.png new file mode 100644 index 0000000000000000000000000000000000000000..664ed465570859ce4a609e658757dc243f56a3e4 GIT binary patch literal 75969 zcmY)U1C%H|ur7d(ZQC~X*tTukwrBR(Huu=JZQHiZH~&57-uGVj%1U+hS5>KWR&|m} zxV)?wEEEQAZ7wG0ssJYvCy9epuf)qM&gPx z005q30092M008g5EdNsg02c-TfOCBS0M0Z30CfATb_MR=7w#tN5~eaT093y;1OO-? z5&-Zo1^9ad0Ad0Br|p*lkOaj3f3zYX#s6Y}002TQ06_j1qxoC^ccuK6|78BB2Fe5a zzX(9`JmCLF8~z6^huLrXtsw2iHJpCig!EqlG|Qd;&r1ypWp!tD8EH-?3= zI(HlU|D*u8-8p|r8xv;(0(Tp0TPIF;9-{w);QXckW7886{2z$36%UcRj68vmoudf> z3*8?&1|nW40s;bVM`Ke?MPbqZ75{zWAu@M%w&$d$cXM;2b7Q8nb2Ou8y*}~42;6J?vhITH_JVZqQ8TxE~tS{y)h7Ki>b^;imu3 z`2SC1{>Rh*!~J!Y7mAzy|8*NLlypD21^@s*fP}DsvOD0V54gXMO4D<<$-A`aycap) zVlWy6f+a=1Br1YDhysd2?iTs$;>kmJ_X`aLA4P$<*gq%;$s5vuUs&9>X@WF;`m^7T zrdM{u%~ZC8ArR$ec2>iSu1{){X|B&S5Bc6bXU91do5-BK}B?{o)?P0*$ilH^s9=_tYES=u5BaH6TcCI6GP}BBj`ynbz85 z=esxjq1SHVA~1yD5c}~FglHRo?iaO;cn!x z`Fvai29!^Tg?KT-gh?%dm_BAvjI5RFH?xd=A#0ueYmYnqOu9 zU-=%#^%!tF>_0r(kS0lDS@y$m#ui6=bSm!BMDg7Hduzoz+^^m*FVmc!Mwto?-f#DQ zfoYBp)TPl^O;>4j_*u;deH&oXGZ6d%JmG-k*(|M!AtSHtVj3{)nvJb*?r}!Mt|F# zFxY!AjiIip>*1~j)khJ}&a}40BOf#u2UQt`bfK6WrVviFxA*=xfrKJIu9xu0R!uYR zoRMKlSMkR8_Qh>Wr$(iOirN)ZIa!>S=Qb@7uI&4R2fB;4*|B~z=wU5p7Wi$qkvM8U z#R^y`>|azpNZ~whs0%s@{#Lg5BtHYzrsir-Q(IVHsh^=vJCp!_`IrI z(S#yS-7)lr1$;2FJ(rE(_T|)Rfc+=)cu)+t7r#PJ{XoK5L7@W&EAiYCOKVf9o|V&{ z{59sAzVMW>kv6|=s#UjSB}soekfQ3xuA=SQ3%rYo$lcDUQgvo@azalyI%tEszQ{!2 zv@fiam6nwnZuk8+*LENV7zNU5fFbV{1P{VY1a3ad8>@9GI1`XOe-;MQLzHjGKh#ii zD5;o!sMAtmx0Sf%e%~ru5uuMp;l5G5Vn`z#hCi(N)B61p7Gt_B@$`V)8u7dwF6yHC znu;6e*FHS%?Y)7TB(rUQ6Bx^EQ%^T`raTN+CmUkapL?rn1eAS2NqO3hvhVvWMzChI ztLo$pI4qC`WVt)++Djkp)#RN~Y_IU6J1m5>pg)=kFjN{*;H1w@G%=#2{}4lq+}$dW z3D(67sB$A!5*bTh)4pv6q^CNsa5t>Fn1(Cc0vY*4lNKg!jVb2_Y&GCxJ8Wu*(YO69 zYY>crQ(TI^u76tdSouBW02?EBZ@ zwN+(UKOuwC$OnGiX1}+Kb=T?*E^n_LiT><5v~&W>K?`#iU1_LUT-;v^JkhgMTUBPZ zUFKx!g1D4Orj}Yn#m9IXe4Gb-j$3?S)o3qkBGhO7fdngbR@oV7hmLeePvPWT+hWlU zLS(B2C)=3!!3?`SmZ+78y6eUP4YF1l42pZDm<9Q^v#;0Ca0H5qWv}x6&YM(O&^xJg zp14?vXO|_)8D{hlO$@hmx|jo>;>UM;ysgNG~ttp8OYS>P(J2vTM5@w%kwECLI|R?>8NmwY@^Rf;{97 zQlB;Ey>;r17dV!Rm{o7?kGEbjOWV1RpcZ%k4B&cNqiZkM*}^&j!mJp4pTwNpSnV8i zu4jF2S<2&b*Tu#IR`3Z$WhQ5%*ZbQG8~F5|Y>7vT)!g4%yej{OAVi0l+mR_j3S;6| zd#7PU$CM46H4cq}hw<@cXiCF`5g1AjCDE)wPCz9#dntjzfhs~d-hjT)<*?%s)3(*; zPIKIQREsKL0U3cyP8FS<;SO6+;2o(G!INNJXSO*$i=1Y(J!hifgXnq!BRH)-XzbAr z7n+xVS;bF-h-|Ty`53_z@u45mONl zyj6Bd&M7^3Ks7j-fuCLRt>@p)NAi8xTsO z1S+;SjiM%6NQ!|9M{7rml{U{q{CzmURdWTb(PO*ia1fXJRQ; z*~e@LQ4f(G-q2R#xhP>K7wdu!qe;%qbvp_y-I$XoMH|qs-b5M{@;|m?gn<<(Qe<=b zs^q%*qH@CUF|wpZCI|%SqycXceG-Y;5a$9vq#D?JY~7uk;KlL~AZgX|oomLfQk94% zh|KSkRi|F|X9zEkPleO1wdQlUD_da3-$?Z5`BxPeg8v37P^u#ZX{fPl#)CI^TFxd#Uj!vh!?6r$sPgg;GhNA3j zZ{sCfSx2yORt$(Z7iVQj`C?bmGJR0x4Sld0AIeSzRYy|LKi*rg2#IDcN`}=Zk0?00vsL9`_vNTi`c&$C%+T;4n$y7y+R`zT) zo%U+)m-_`MtxX5`Jkp1Yrpemgs7NK&fG3fs=9sfk8;dytf%Q$wPg&A^5isRQVxxL9 zbrXf!;#hG<-0$cL#6+Edoer)H6sRu``kgr@URK0A)AX5@>dDq zJ?AGQ1H*BaHCn&sowXtjRDCTCzC6g^>z0|l z$7UZy=jj3~%m;=Wo)JKZf-)uRSa+M(4~D%oH#uy1EzlRXePBwN67aF5D60<)Og0JU zca;-nk0;8sUuFhD@FxLUrlum`4obZwLiaFtQYTXY%RV8bnZMZ-0f%wx`Fgzuku0~> zJ`wf)VQi@m3SI-^Rwx}=M{QKl+(0%YIyeNIiZD5i=QJGh;#vX~dnsESJ1}nE-ySI0!?bULr6%fKrjXW&EUABI zGB%kxxX>HASs1cmeww!*oClA)qDp`Nz%*_MRMk7c^ycwOB9g-`hXMeARJ$gUz;C9` z%2_uxkqi^Y|3eBM4HbXgGD9~FC7p+^kwashn@q~qHYtH!nPC_Qk?%QvDV^Mk`UOXc zTYp?;7tQ<7AOdaIa~#@x+v5K0#Vvod(7X)H)VYzNpf!L8lKGP7c3FJt`_%Ohw;^?r z%#7AbqPw$vqpQ1qX{pJTKlmjgtA7$c3apJ?N7gWwxJs_b1Zs7t%qvg?&(1T1>q0o| zthTG#8nj=W zNBn>U2*o36(lK5r3n)RyW|PEDvWZQv;kMbT8~Bf!JdAvjWrD(FiA_GZv2+WlP;)cv zkEH#0GbYpKhn415B`NInl1a2*Rm1*Hy_=)u=s#=rw3&llf~|i;jc=g=IC6 z6@d`F@{WIC%^EL&K?{R@Ux~jofI8J@5X_3q?k_R>l;ww4uijHp+evr022l@OI#Wz_ z9}>K7tFZJ!;>3FtUN_gT?l8}1<-zdDz7^t-^ld3_?j8(hX-maTv%wNg0 z&~;v&(*q>I%R9VSGm~Q9*bL`YnLv=e7Tbo|rIg`9NZS$dGwM4evtR>Wpn_Cq6ua1a z!bEbAHMyTe;mmNy+eLjE;o^PVo;jvjwm{@&K&nhqjV{*8niI-Pe>y z-ripWUwEp9hGtEhFNY1wVr!Qn3Gc?bpEyDxjJ$o=1e%6*&i2i_p6m-(T*ZA}r(TAZ zk=c8ih)F6ozr?Yri{q4MSvK^FiZ<#Ddk^)S*4QU3xGx^8M}0hC3u{SKE$siw8 zg!B{H5Q=tcY#mKgSMIqA@z@(ZB$>vE=h|Hk4__*k_3jev;C&IPq0p^wvfzyv`Z9sUFcUwva2?65-xt;24x?Mv`)Z)cqY$(KXt(c zh^BHOV(1vB;Ow%tlZkoMWX8!Kb)!>3w2yg}-PA&{wOJiv*6 zr+)7?7br|4wim0lO-=Ri?hNez#B$u9>prd;erWyPwo?n-JUatWC%S)L>aEUpn?WRm ze-K`cicurtBTR~8^rv)EkdjlVz0Fhi(zW)6GK&6b-+_it#K+2Ww@LkzLrd2 zzhm;Rjk>~LNSjPMv#_5VAeT{&Q|f`0P}unq?u}mpgyUQFL|pHsYfA&t4x1ZC4h(4}; zx`wlzd3f&)Vz3Mx(Tk(64|ut3(zu)?Zz?ElBX&N9E#0mfq_%_}R3m|}nMxOBOl$j( z_LKIGkQ%{Mc5T`-<4aGfRHhtL;{qbxpwrdZ_I$Qd4DDVJfYY3SX|BZF>PBE>&M$od zzeh%7>~}H{>3Pj)tKB|6;kOHHcnF7B5%^rmyB*;_-P`7(mP0MGbGy|UNFyj zgZP{ByC|!ROHba>Q`<~j`f$0zaJ6+$pRw`-EXNrRA#{;%F>jo2Gd5_XSj)$!+;oxd z=WkUyF?Uff&AOBph*I+$f+2(@7Wz*ScLRxfXu4m$LfZsT-~03m=-OSXJFegUo$trv z5KJD?61`nd=t6?-H?-@49>PiYBFd!mHV}@(ITz;`vFCj!rxdtU=+7W2gp8ZAW?o}Pu)BAtGF?2tDbqAYAVzGeg3-I{5yVdUY zfK=xYcv-~I+yb5cBNb#)y$!~NCB z0wjUZ=%6tHsHIqM&?Pn-m%pfWbbcc`dVmz`4)Is*K1sCcK9=}rE1@x%U(+)8uf zL5O2)`>{)_sWZ0xkn2O7Pch+0PP>@GPJS#suCuj- zSox-o*Rb#!7ZVrXPpZfDx!G$s2U)#`SpsJT>jfl#y17HSzpQ>co69z@(B{x9;U%6r z=u=9;(jv6F%v!31^3T&Y9gO3aJHPSHP#nQj0 zz8N_t5$>mivfUC=jQdmvtE?I+ywa#`Os{5(>sxN zRqigMH}V$2Q4?+S7}grzN*3tQe{VE~r3Kie;`1FHt2ljtQJ%v;tLV4bMeqw4^(B2u zW3j8>yF9WCgjGj-^Al<&^p)BA_$nMNMQY#1$r_gdmLt@b53_BIYogQJdi(7eROEKq zp^CRj*kT_Q(E7dWd-@CADM(o}2O-A>Dwz3RvsIZ4%&3Q`b*vBi-A#P7`iMP;zaRDL zbzkP_sN*jk1WG$82W>_$DqJVHVR`D9D9mTK+|0_T%h0;4gjSTnz?uR$k==x7hEI3a zW(9WyB_cLD<4Xmt?CR%I5N!to;kW7g*Z;AS$jRrA zi69DQX&_za4T7WvMYQGyFx@YU99Jd}u~Yy~tQv-B7<3dl=yXA)g-%^jqvtff;=9&? zivy)dC?ljfT0pxBu*YdG-L+B1*q9KaJNcXY8IG)Y^!3x~zU%OBOTu~^DQK+YFamh% zcHEuW?b46;0!_Fl&un54SpAl=QVX*URf2RH0#rAY4iqLP%RZ|(y4)V!$nJcSGKj5} zt?An`f{DiD0$kgf03AG#i~D8h zq=%#rr*70*2+uVaQvyqeF<#25AAc#3->5(!g?SZHyNyy}C|&i?6j~lt=3(W@!ndW^ z~$lv-&7_0{JpZB^U4}nqrP?60>;HoOud@}e9#Lo3-8BxBhg( zTP&z4vB;v4noj?HR3cBD^I`^#n^295IXdRLMjD5L_R<)+Vwy$Wr`ta!`(}S#GEp~q z^B-D5vx75lDK#*yK27+Jb<0&B$Ltip{MczH@XvEX6U|kbb-Kc)qKemf z2-vL?IK$h2K4yLY#9q3&_0~q5U}H=O4;#bdJw)${2J^Pc1UQCjAr@*7uWclbiS6V< ziFjd0bM4Tz?LMTGE-`bOit-rw4z0);uR?VG2ul3V$Zk=-_(1~%VJ$@1N%6e8%$@8` zSakaWdR6iH=SPur&Kmx@)qAw4wIFF@JVOI5l~f-tYj>{K<9zFmdzraLrqjB<9AIDmDuW5h_eqX4_yHnt$p%sx2q=#`cYY^adS@N?OLBLA3 zalYKio3E{?^k)t3dj*ax?O6(!phH`^vR)E1*mH%!vXKao-bQPEsbw$vuS;}Ath(!s z&1auJFTcf&bsh7FzL*@0q_tVO;e1g!V8+>47xuHaTKAl#YQ=+*|RF?j5v-TPz z_Boz(JG@f7Bq9$}W1akXdB=aWcx^|YYs!~Iy*12MCI}CDYC65Dt4!uVr{ER|X27__ z6ZNn)g;!8^#wXyBK8tel^je#;2>F?Q(DQkAe)LU379~8AsWNwbze?Qthv!Tgo5T;O zNU=r%Sa%!EM5G3MhQk4)zcfAz&z4g~^HKgw%5*(G{;*p0PW5kUoSqtD1ozT-s_MIe zp8Ga;0ll*<4-N0@rWHtx&{A`GvGQ(m&>2`iNZIm&uK5`NXQ?V=om%B?dh~aGgIh

6|5QEpQ&^#d~<7^c=x<^CRjF! zQ&{!RC31s7tbDabi>dpPGD#Lf{IT@(b4diLRKu*bw-`lT30#^owE8O@Wzkv?5!$nX zT2#$1v2ETghO=QJ z=DABK7EOiCxOa5)1y?gW;|)BtmQ0MX>3CgzUD*XDX2bh)%(j`Us~gUFMyuf5Yex~7 z?g=Fu5xta=Ce8`~#2lbsiyC@YoeJK##=gpEBfV67Bh#gFMITU#A6g3Wgu$>UyJnR4 zaZ&OT^M|d;rPV-<`}};x1tORI52!FxEo&X?v3aKC@>0Z>LuBxW@BaDU)`)_}%F0&1 zAu-(Is8Z(+__Ec@DYxq4*8YrvB~&=x{aa2t__qQ9vZh>6oNgVF3)O=c_nJR0EQ>ks zv&lO?UMJaZ-Ox6<((eTE8G1>)c`oc?gnRv{n~HzbkreTU9~!; zhdmlKZrL*3C`O;Y54S=2t+P6}GU*CGGq5eePE1-GutjFJ8o3rgSpXWp+l$U#uIi7h z(RNvo9&APtE^v_#z);eHvzh!lZ|hxRN35!(E-^GCla4ec1&wrV)B*4Mu}k@Gbobg@ z-3rs-467=gJ1|bklulDB9v2~vtS=gAsDznj=(;D`6VE5PT!M#?4{&l(NCrojDkY$c zwoYXwp6hOcKDHJN#I|!%H5i*`;-W$=5SjYdyEQ4C6{S+^pg4ur%Hy?UM`n-GgKox4 zveTy>eLd+ZXdq{lFXL8!&@By4KhvFo;h$Kx?;IPKB$%(uT5q8t{Q18!J6|Iz-(>|f zh0D7I^i3;wjm)sdT_l5`z;~NV6+tOxe*~QIV`V%XkCInQ0)l7d8iHO(k`n)33^ZUo z%Hel-45e^35_>z#QBm#?-K%OJ)4I*G!pRD`ohcuzSf-+&)X|WL674q>-JbsM0>#`} zTUc4B!Y76LV*hPb;pQ9UIgIHSKd>G|4L|VqsF;e%fFqH2CEUvTXXDv^Nh_qlRa3q< z-JNo3nl<+tryr+vJX`)ZQP6&@N64J-sgfMe-r*YZ&u1_*FZWU2g;^unotoO8xE^{; zcWVi=2(Mk?K7nN?U;lK5Ix9bC@nI`5Fx~H>ioOb$LKMz+baHk|@Oq~+2G7Fi;DWeF z-P;0&yhctZk_t&o)I_5;gtx@ACw3oOC@&=lWiS1MwwA#zN?X+z*N@x}(bS!v=8w67 z8xhk%FeZKpbmA9Kh0?1qV5P}?^X4%cC0-+P_V!2SJo4xY7|-;vS12XQHSZnd)S%6Q zbt4ScUBp&aw+9|2t(vBq0{ZV3wTM^-Ca=xrlWKjfOY!8`ordzC#j^&h_iMh!GGh`p z%D6>DPfKG5&OJ?ENA+Oa5p0sk?mPEMQon5C$9K@a4?2@{%1eU)4*=!_@tg@dA;mDc z%yK{d&0WiJ=`%_*|NRShJ?%j?0>Z;I&qe12u0*9E@m5R4&RI{> zGonC3ty2YI#Sl1u6l;dpxvY{ zdenWs`J+k3_3n;9o4tr|vA)sp3b@-?*w^8+$tTp@@CEWsjLw(SuN&D& zd#T$x|9C!c(F=vjxb&DcIR@GM&Zzo=WDa(%SgAaSU7Ek2h~p@cf=GL}M-U-E9;Ep= zeLNKU;hmf%zTd_4SJOw>NX5~?LC@V8_wy?gt@I5iat;d-uDizPL;byz)Dlnfmb!~NBZL9ru}Q4kA6*J%J@*i(oUQ3_nKpbb=?VaTWwAX`t_1d1A%cte_M(ZsUl z{T!+A*VbBtwNzsG@~SN^{WatzaqH9}!eIn729OA)=Z>--DKkw1q%un#JEcuV0Yp-r zE}pC^Riu%06Cn>evki;GR#5T#m`oW1vkDTgJhA0+9_Y4l8&X=oMjo9)Wmk)MuHT4g z{KnUSolkKI9iYaJw9iT%T6W_D`dvc48g)3x&fa0kI(}pF1Kot9#}|YdX~Iln6qJVZ zF_bt&(RGC+#%%z3{SdTPY>u>L%9f+~r#HsbL zltJ-fId{?ecD3UxB%}ikIX-t7vv-LBlFw10F9D9Zwd?t88M>{dz)nmLCouspC2D;g zf7D&4tq>i$HHH_|&gI>>?9i~uXm-ZDKUD+Zi^Lra{PQB#_&S#HIl%rwIM*?0uSo}UutA>dVWMCDjeZ0{-N5vfp3aCk(I{Wi-Q!qoMg8FYjKE>?8+B=GbzCt9A3oM1F5e z?|7UcC_A{RrU=wb`!rr=qT*!sp_|2_BJ3q=J1h+~^HsTb?L^8$mJ1|badmNCbZ-8m;>J>?vo@!$w-8Zi<02!Y z^~Ch;*m7~cp>%zs_j-u0Sf+ejjPrCPKGP}3O7a{lH?P8wC2J=`Aemu+utR21>j=65*b$-ASr}LI?SnIl+8{CE!fx}Gzb9eNfIW528a0XK5E$Ng~=k*=>R;b z9lVEqEBX@K+96^XPfiM*M%~^%c9mPuQ{M|#HjB%ZWWx0~1Dv@I+^)xD-X$grjf}{JV4um!u8uv8(4X{p{dCwHjuwa(hv6T|cTXF^ zKRg~XV+};cX_F(ld(I6Z>*H18h%SUn(P><%1DsQxnp^josGGGnkS3qoQumdYv{f{C z*{FGGxu`jPjZY81TY}`APybzCs`?EN8`$TsDq2u5@a1*UngMREuC9EKvqEgsSsV`w z!p|w~w#efRG@{|B!xYDWxI81-+p@kE!=&o9jEkW{cEx%w@xv$iECkHW83-sy7Bv&P}B7= zfnJ>wW+e0*zXZgskzRe4FWk~*yraN)yOa7})MWKEby@l*t%J7C!%PeRx|Mo4xnm5< zzD=!Zah_t9+B&)QD5U_MCVrr<%=8;=d|=kN1;C&|G6rQr94q=wLw+YjrfPYPEye8q z4hOEKCB?MUYnsz@*(Dbj*UD>4tFmkBN}aW}MfrtoWwkEbtMet=iWHTOXV=eO8Ke-~ z$-X>c&I#smyPmISH8SZm24r_;w4tX3(w3g%Y7f(Rf}$~sAsCc0Ng|97aMTWMrdEHA zJNINDO~gGl;a;&m;l@&L17L$@X$3kk5Kclp=M1{4@MJ3TM!mEllI?#TKhWR@;kT17 zid*;lL~e7@^oA4F8_}5Z*Td-5*A&`ZKg-}X?_+M7nwqb#+?4W0hAgz^?djrni?zP%^DnPwlQ(piLAkOICx|eGFj|*p>EqA zCrV=YUYBfAk&z$%F;%+sUzdjxuRMlLUhq5Z#0@Ok6XQiu!D$d(+{liunIJF&=@7|z zTQ~G401;(jvA_wV1)v~_VH|P@`qWGwn?d?D3+Tx)s1j5J-9=8e=cnD*IWzc&@4wm8 zv+un%(Cdem?#p(>Gzw<{nZyd)-we72M-6H~xT^g&(@DHs*JJN9T9&muNADQnf?13d zTB=S{=gnjNHHe>NC<-W8)iASC9A4Ot=APq~w)du^(SjNr=D)shH1S;lx+8)T2HDSQ zI1t>Pxk|uN=B=gGKovQ&^1?ELr7c|$bjxAjQ>*rM_c4{0f(FPBZg%@2m4$Xo6Bjcl z;M6batfMx#eVS5??ei35Ta)8GgVvl|QLFm3^+U>%toz>4SikRuF+^}xI85Hoa6DkJR)R#_0GM`vFkWJ9`Xusz=8(j8ZqkcN?eKnLE=yt{P;ohA4Q%H7C7BY z-+fAjj=v}wg0zaJamWU}nex39OT`hpU6T-}W3&Vc?cp5e+J0^;CveWF?sIx+)xP^W zFLr)0I~nEoWt8d8V?equ4%^2IqR;>J(<;q$d=WN7UakFLtJaF@rD|if|A$Bx$fS2Q zOO}l!Vod#{wRa#S-wK7e&8IepyY;_=2d+;S&-xBpqou@aYi;sWZ3ywQ=@g>RHR(Q5 z`?00A_KGGC6?D2b&J1kJjM}+QTNxRdiSnZgofEWNw2`r&NY(?tLPws&9wK7&u^TD-?;~H2 zh68WnqRUNeOF900#Clxs+<3^W=a&$qky$K!j`j-l@9Q^ zdUy#ewZ6|wme9pBmIXm~X6gUzPo7ZnjRH2f>wXN3n$t{T_gp+AK>1Kad)`>!__}Hx2yQxTrY9 z-Ra&S<(DTY{M(h7SxQD1KomhYv;0uty@YW2C7zQmtRU#5}hX!4Y4OJsov+m?1l_XYmRMI%VB ziK_|xt`9!!R9elYwe>+*8M5?s4##~A6v56VuL&;etKEX#fGiGmR{|c9ZJNk18q;9?0H$rmuxIrA=NwpK-y`EVqBXJNi^27m6uCY9tRmKVi7 z-*w%AEquPV9nXrZn@ygTuN}Va=Y0`ORHAe`7!YUjLaAxgF=7@0QAP&fXrd5ZZyFTjU%TnLeD_f=Tru-k@BQ+bUpeFN+q(qX@Q(ZLbF% zdnd5Jp`d{M^%JUQczB{#wU?fDw{a;Z6({4cx!FOqg|MpdPATcH-~yht_OqpaHME(^ z3B70$VBWF*&aSk49KMXa!j2~)A5g?{9k5mQYKB3d)Nb8~cj{S)`ItsQ2p}9F>3#78 z07nT307iteYMe!84E(ioCg{IeYlr{{I8q*a-@tQ`DAlf?TS9G+CIij zy86U_mU=qRTW3HFu}T8twepCoEcd9G`*xG$GG~D|gzOES*oU1H9{tCZYHX-*b>^Z+|x&44Dd((vl1St+|21-CM zXCR=bIPJljwv9D-rZsft1Ct;|Cp#SlGb-eex@Nqi-S>%Qv6IG2->k`V_tyvjz?3$H zc5H1qskQI$L~5Nah^FJb3<(WGeAd0SZeX(4)L%da86kl}wY$ zKS5YEhsiss(aaffkqm3czFd_>4R0WLDmRSY-_PU~D^_kl8%Aw8L5?=2i~JtXPv+`y zc`Nz3)Uj>PTu>i3+Be&P!3(kD?t&$v*=l zNkw2D;V%zfGe#oA{aK{aZJ#U@E?Evaam`PH1kKFn=y1S&Y;LoQA+ z3JQIPfd1rTIg2=QlOkRGY)6BaJPGr9etRhm$=+>;z0SE+V(ZJ|9@Wdy-FvlSb`1!H zFPj%l4MTSC2y=uK>7q<~3K1Pw*5LH$43aGN2O7_wuaOD>6+#hrk|!5pS0bILbM(m` z!SAXx`~tMt<+Zo3ZKWqKApCsDmf;;`4c;9GO#-wnJh&=LTt?)rF#vD0^QsnbA0BfG z&{$ux&G#$fLoWSW-)1N$NGl?=24B8V15;Dn6~}j9lzC<4E~yE-{dn%U*yKU)`z@9J z?dATyUX7ZX8k<4m;ubl`EtN*&zL(zjE^=aq`(g59edBk4)99<$j!t88_Y?GcVhC4G z5*P-M&}hkzORxu^PDjZ6IzN5du+ z9?YZxYRuY_IaI?$(O>^Q2-ewpiP=#(Tn(Rb%6{uO+1%09f{Qyx*FC=zrjjzrFkVl- z)C{EusWGqgx*FmOSUd(DNe8R4OGwGlJ0rtaRErTm2N1&8 zz!a&e%3Fp#dQp^@>unAH@{gDdL(|ds`n=I9*CDn?3HNlIHWUV{8buB}0avElZwq4DFiTHuAOOZ`90u((kOzPtkX_WHXsy6#qbVun7@xfMqX7NDbkqJVnuL-EFo9Sa-xW%|t&0uX07a9-MByxY-6p-oU#H3_25s6~0AmFz? zgG)2_t zu9x2TRPn9_zgcx`?&GrW#F8iaN*s6OZD8HN=-bV9;!r@%@j2_pwQF*few9DdH#m*Y z>BH-5a$ovWQgXb2aA4SA_+UiP(lp4!1WCBw82+sZ^{C09Sqv?^NXLK1;exbyxWLa; zrH*Z~(8~mr;PEr1$zyFO@KRMrO3E%(6Y?)<*-{(7unB=;oIgQc-L8F<_+2yFF7(~T z`UQe9RNebo`1ywe8_(@x^i&P>ws?B_-2-Iv1cCfjy=4e{OX z`VK%$OIXy8raI3dUnadOjJ~>c1P`vBy7sQf!Gv>U9p)443m26SA9OwSwV-AHBp*=v zHBFGov{ly`OKV8o$xjz-PBDlf%4WL?Qix=eWKn)ym^D3BWp}l|hkx`ayewy}2n_5l zXvx(*K527|Hm?opM!xr$PX^m9rN+p(ka}>4WroiaTmg%DwD)Ask37d!&3K zJ)mKNWa3^E$0&U2U@s!TWI(99)1C!on{tJHbq^%{P3yy-?7tBL=K-{cO#vZDWzZ0u z$zuTD>(j-hI4_nw0;B=NTX|!Nt5f&7z1nu!mvi~tApTAAwy=rJyo)5?w3?`OfDda1 zZw7rA{3{-PhR9h1K1wGpQ1QA7Ct-9iFW%j$3vW%`HwGqyV>G)*E(`RxY zC2sEeXf~-0EKYeom=b~ijj|2FVEx3Gkyx?gV= zXqH+q!iNjCLYI360(Df;-g@UyhUd^-5QJx z*B2pD01~^KhPHDo^*OG855U`f`%LAM4huavh3-JZFbNIgnO=lQid1r$<S04;DUUuMw~`Cu8H~ zOd1N42nqacxBA?p>`5;R&vtwRtbIpRVe0L4CMc;SWN<+NW>C3=(!0UZZ@&RzFS*=x6Uny!ZSMNI#{&2mv` zaJ!z}GH8h<3$8+6`9KIcUPUB3n}3+=zRlEY>2%c5HSrdK zF2l(F2@WEOg-wdx`FbC&zx!7ulKuf;X!GaIMw$^5WEe-#U939A?eMs(=P^L5=lW)p z5n@e}vNNh7+79fW{ehNQ8pY>m)?%)=KGm8gi@XoZ@dIg(h!J02z0+Z zsrF+!V9`V4pb|c1s2W;_pYNG`%Y}URGqNpD2D512Am3#qr zA{AJzq|Y|p9N80wZL^UQj-L~Bn`3>^er3wc%0)mGY8E>K_hHRb=>pxSv|0P3{}QU# z-yn2ss{sn+89A;1Qtk2zmTqFVm75xf>roB+^)xO=ly?Q0Id+&d?k^`=k)T=*MZBkw z8`HAP!tq%^!U_;hvRbA;qanQe)0e7XP|##kucyZ)-+!cZy{r$3%Oy8e%w$u_L7-EF zWB)%yy#sq@TN5oB+vXdao$lDSZ6_VGW81cE+qP}nHc$3H0qintk3Mfk@FAGR)3i`d;dm+AO8|MEidEkXo7KlVb!gmkzcYZEUj z#~W^kgkt6fW7%h{ty)*}Be%nJoDK@pBw@3ob(M3rUqPn0ile0cx-Ii!2jJ#BGW9Th zaGYCqkuX^5o(HLZcNQ9+c6{KoKP?s&+Ttk~gAOMYhDp$u)2+8IfP{ddj*t??gMm1x z9bi}vgz_69P#cOMHrIkwbF_)0*j=rHTWossel)3kyjFOMv_xND+FZwP!!GBb-77Ya zB)=k4qF`vBO5rITefgr#pLlf$Mb3LVB#+u0Y9JAg(i2l^z)XCcMH7D9?^0YrhM{Qa z#o}^1h)>YvXMy*~w0+c@iql(>f&`R2;f z0YlK!39@Dc9x5xm*zuhB&h^BG-KdNY5d`N0Kr6*ZWQKO1dR8~dF#UL@Bc83<=Jk#s zgC*5Twxz6U$nUUz-(ff1mCpIleh_}V;Nc*#zwkTwi)90)`HBj7QeF=`>();wUHarQ z-CTyUtY`$Bg&>-#jI8M+^uBSlMH}^si2l&qJ+D6_$9qAB%os(y(;9nIee2*a7z;B^ zJ)~yIr*l0AWNH_=2#n$MN3=*qscQ*~UR>47ne<|swoGFlTat(#E4|{!QB`Sk`A@t@ z6jp)$L3_2nw48n2Tr9EBhLSh9t>phRXQ?COp4Gl-I-&FSQ@)ENnH#*%POPohUowvD zP`8{Wos_q#X(xWPz|bJ#f&1DU<PaJCrMT+ zmB$PA$~k*Q^9By(4zYCv4qi_P8GNjiqkU7y#rkY_9Kp_?+Rquq??85%!7=+|AGRu< zDROZgf*-|}GL$d~z(1-k`G-)Ynn_F=H!)N#sk5>oR(U2SC2T7QT93!71hUSrFKC1n z)@ENKvs8aLn;G<4l?ze#^f4MfpicZO9*$ntN59~Gxs%rUpp2mV`5fNn_#;y5D=Onr z`?@t!)a~GFIAcAjN}{@7u(}NZg*ABV@$D{3DyD*;MXNCgq+br@VB|*rU8+eEet=xC zwxzXt(&hP#tCMCTDvfhM&u!VE(!nDUt0#<&LQdI7OFq$RX81n75As=;<@|A}c-=a^ z@e+4oXS~ADnR7p>B1sHID4<-%cVP4;TwNCmF95w)!obD5dxvP#FI8}0_pX*wcB3F* z+cg1rsqsUF?1@rZ)DeZFFS+<0R`x{fGXzyfoE#g-H}lePVG!H7-w<4S;k19={AfHn z3{UnOVd=z<=^JnPaXKgcE2DivZ&EE`Vg0&_Jxw*{#QNb=G3>3q_iQX^P(GA`{)6F> zj?527xl#0Fe69gfn3&r~oZg3XitT<2@iK6$VuIa58_s=fN`>F0R1JMErLvjXHG&jy zJ@MO+tvoI>Oj6w*5BScjm(^lBgr)^qEFsW#>6M=ek_Bf9W(JTlmhli&)xtQHEK~%S zOaKO8-9WK|gE|>0Ae8}E=vP`%L=Uyz&@y%`4@<&L)42lNvvDWt(bpB92|U>=3Md|> zP-jN&jSP+Zt8mGywl5eD{ADi^(_SiX(J1Yv+fh7H=DU!1o-X>$;)2SWYBqzQ;g>JW z+P&ek0Y^S;@~jD1@T1n)S+dg;WS)k|245E3J6M~n%d?v=QppdMoyX%n3#FHoQnt?1 zxR5?V{RC)dIGuib)~>#B0<_@MvEVKH?G}fqn{c>a>LkHc_8w@!sE6u|d4H9UZ%h8} zC};+#gmkbpjp7Q`E^r!t0fBHGem@6hlfaekh2xA_3~A{pizkhWgegi)39*yH?&+z|cTm2eal z8N-6rgFz`&{?<|Ls;^3`8xqKbr^UjcD$f>QRe?&dmRa4rY)|hG^FBHe=tl)7irso4 zN#vu`nZM|Ac`p|*EcO)LAMX84A2y7^oW*nm%iPF z>tMt}UWz=N{mQDLY^ev7>KM0_9#*+I zcAeyQj69{FyEVkQK7lQf!vq2~QN;B^1dOT*M>l$NY)pAVs$kp(Qg6bKmeo zP&n}*o{2?s?W|<$N1MqRj;dAv#yTh&NnIFhR%udM>{3v?fAM_fw7*re3XO{tHD9qa z#a~XPzS{Q7hIQ3_xch~5^9Pp6NH6c+?$7=>3H~47er&3K`s9dUGz}G3dliU9vw*(o zo&mm{jkP1EXLYa?Mrk~yT`#vYfC1)iG9J^^XLtSckZBhQkv+wgz(#EQ52ouRHOIn{ zqAC<(y2A@~Akr#45L*R`sF}PIEjy<*X5WTfDzX!k@c>64TI?bAxzSKmQ;|<5?*T?O zxFdGcH4Tc_V{z=}*oVns!luir(r~!SB>zyza`AMKu)18%Po54LhbQs4yhqpEP8Q#{ zbPxEKlLAzZ%|pAnNB-Y!sYthIzIEbk{j8~5J(13>6J+3P2mX27)|PyM93(+L)Azi!hfeP zqoMpIwaSPLsV8}-6v9XD0s-zj{%sZeKqI;*V|MM^YN{;2(eQ9eBp21uaMd!zpOJW1 zBXpV0%X9D6ABEl)WS}U|i5FFP4t;md2Q$#=F>Dx4BVG3tZ zdU>QgKpP+71s8?l(+CGY*Ij8cU}dg%ltcSVAe!T|Z=3^`n*66d4B$1R%9jpLP}vz-lpE8nXjx9iMBwtQ4GGhWh* zH+i=;uzL+(!BTY^1t)_AUS%a+Ukdd2(1z-nm&J&EW^l-Abr2=`E^p=U#r@jnU0~vS z#l@h;inrYLBz2Z*=Uqck;j-+`j4pm_M-4;VjE`U3_89ek^Gselsw{S{)tNjgey=jx z(3=vf3h!>kUH%FzFZ?hBIVp&gVj*XtKkf4std3zt)ORL0j^yLc%?c+)q zAVo)lIS?H}kq1;VQNns0sHTol2<0s56?BwPuYiRJ32@{6jA5qoZjnVX@w^UWZq8pX zLFER(^t?o~gl#TdU$WkYDfA~IGyqZa1jWXw}~gZjk}gv4q=WJh??CI!p7D4LKqc)NHu#{nz(4Ss7eZ@n#S+dQJ}t zGsrDLYte$fkq25HNy^3;j}e9(snw;hjOWGR>-lll0k zdRo;<#X?<=TGn>$a-|eN@yEehqrq^xLhfBqV508OR8k~b(s{WTJ@F9LpArz#*+x8X zYcdzK>UF{SGBjY#`l_CAC+w>U#O5S@h$TXqr*|-Ssb!~}4D`ZO&4?jOT09Hx)|-4 zZ=Dv?Z1Ks2-y}zD(sxIG{Kx|o=xczs!&t~onG9GG8tG;>uY(UKHm@Y~Jn=`zA-2vC z_$egC)sYi+9^#`y7U_I+*jj&vNjl1Srw&9u^5W*xUJepBBQ^B1T6^@s6lC+lq# zo_}0risj-!#*J-sXf#YfG;B_qEbc5Q3kyny1O$9{UYDf+k+(3>l#pm*u6~s)0^;^m{+7i&%WPne+$u=Hv18@y$5%549=X-j@zuIrdGP1riuUs$Eb$mc3O75orO4|AW9fQ020J&i38si32BZr%9$^JY?<8kJ=~@H~}jCnvYIR z5GyWXO^A>hJ@cKCgpn;Q$Gj!H-Vu5wB)*a~rKsgdZ_8ODF(3{G2aJY4f*<}uIm5(_ zpeJ^6@uIrC8m5u`d4h5WE2ov{k9V%NRT(d(Hw>FR>vD0j*sIIhb{|zJa8ihojI(e= z=DZ_=c)FN4R>c|!9j3&cW5muQ)tAVFgoLdS|9t;yhYP)+6jHc?5-r=DM~t-GwO;#9 zLd!7G&z(cbkjS*(l;fZBv@a!HpCwCiY_yOp|N49R8{Wc%_j7ZRf#>$m2_0h~f2ePm zZF|BvB$?xo2`Mh_*Q{6~JpTLjqw)Scy>Bq4Z|?L;m=6c3S_sKtR58C4zN~gk0vawk zzTq59WX--1lYGv{5I?$5Dc2!$+n3Ax;Wh*JiZt%>_6S*aeZ$h^%OR?@t7ka`FjNa@ z^;+j+p)j`NH;bWZGjEruXk(eqp(5saghbpum|%rx0BkgquoG^U?=WeJ(IG|lnvB;7 z1n(h~E@8MW`NvK2E!F+0yzt>P_d`!NPR7b1*GM*w(?Y5H{^84ZAM5d$aed+#qXOue zj(MfV_)jkB-MU+;x(%`bf~qRMiwr^jg-7O%9W3*L8Se~0xTvQyF<+hN@|>FE*kK^yqIYmbl~YP~)IY!97mpbe>Bm~-p#j~& zYK^aXT~ePjULvi5^WC;G}l(+57ID5A36- zyFnU98On>|V)wfi&n#*F-;-( z2K?qo7Gr@QZ0lx%^b>`zvR_sXuUi#6Qyb~dHwIZd(bnOP z>_Epir)B14C4beFWKt`?c;+B$>On@vdgVk8K~|co;kRAK0)qddUc8BP0tmn|g2@1M}U0gRE0&UcAMJA*j75 z!glPr*6UW~_*K{Lx2!dDa?$1bSY?n}YAsO*cI;q8eL#Z%eSCl+9?gCdM)gl=Z$U$dbh%Z+=YVTV8Dg;y|OF&AHIKA0Yq zT17~=k+3zOu-!PQ^_@dDv?;AzY5c3L@BJZibs6PrkzHAydB0)IJk9%t_P4Lc2zR^5 zVDIavLU731HH)oDX=mwTD>a|Ud3kFiO-6#E?eUuT{L3u|937t2Q9?K_)>vCx3?4~R zfmhpyR7MS9~kRp72QlRQ}ps*`$j69|JiRfv^LEp+mkkECN z`i#G=H`=WD+)G}YEsUpC_<^RY?yBEw{u5&7=Zjj9dojJ05vGP1fNW@5 z)Cwy`Oe756fl=@r@aD*ttmlxZCXKR1&<%(=I|F_iY&=hnbvyUPS?+p#Rbu}mFQyR7!oh49y1wHLnY(>CfaQxY?ll#IyP-&8RFIp8eT7cxGmI=>pm7NI<4cDWT32gpo|%C2pSu!r-!%m zHztC8qWX5ie$W%9b!KGq?{MdEd=!_CN)*}(+Q`|_5{!A1!+p+d`hi3I?Q5gZ?do@Y z07FkC6Z7CWaD2fC*TTO+A;tsO0I{jRyz&?eh8*a{pQ2o^9WVqHB|bMsrfn4qfB`hz zyZ_7+><#s_6pf39M|;BcaF;&p8ibQN9V$d*_*uS^?6^>Q1QS~C=PtfGt)PaWD)rhE zfUw#}g+nkRofxs*Nv*EsVW)j6Lw01eAWB?*eg?z{1dSO?;?;Qoim`)&Nj$9NQ-R_` z*i9CXy^I>Ad)*1Soj^1FR2le)B|`^P_gtLWhc8< zI^5qv*rGhq|5-IkX}q{#vu)OGMh8$>?6y{sRNV|KPyRS;yKZ;dPzeP^{mHQPH&uZ+^q+~Ysxf69KKM*hyl$FVHx6H_GO&P6*38=|!wrg6Q za}d5rHbDGSoi-J(+k&^$kjw0o37Xs2C$sYN0JZY2RKXpo`oi!?=_RniGb`dRi?p?o z8|D=&!c1e2OL=`vig`5ZN`HIBfk39R?VtE;dc}i4w3tpsGv7+*QAY7&qP!oZ3 zvj1xDBD2`%V99<-=`cDB3?iq_QgKK2c&`4=tbMk1rqOYa`;>0A*@Nx)etd4G+~Q=u ztRp1STa--Y`Q<3z3K2hI{#$sGn9fnxem>k<8ugproVqXC-`n}?s|qMP#Q<`KOQObG z0-Xl+{t!%Kw3F~jUWHejV1^oz)qYaCXf1r4qob|!r@40A?jX1-0$vO(3SgyHR`L+c zYcH=q{d6^@{XX7|cI$^8o60DT3W^R5pGy`(6)Dey_ta;IQG*=KsmMXyvL1O3iJ=%%&#tIk?iU-+y)vA7z z;is&l52ob;RE#hdH{eJ4%V%$4kUP>iE8rku(8E<5_MyWPE$*5g5EhDQQm60yG7i8B&^ zRkau=Pb$ry{~aS&i~h4D4PuEin_@=lFMM7yPaYo@7j!53-GD1FYe!58BFyd}Wc0t~ zta{ZU*2A_M{m;~N9j9fU&o0$Gzki3jhng5d)EI|AGM5zDHnFF3C400k&~$7m!=vj%a8MT6BLn^^r+y9K#VKUE zKRD0{PI3CqFFJv@v^je_7iJ1r&9{Dn?I0N*K#%7hAJ`cnfs-0=aJk_7mCfS=ko(@5 ze}lwx|7!E|Dt&S+LGV=h*ykk=R z1Oduu^0WIYxr9EA%xZQW#5*lwpo$k`n-Hb&pYJ zE*u>X=Pim-YV=hXwb_JB<&dm^y;8|_z+gZBey@4HmGVeHe;LRSBS|pF^r=wHdWE<{UmcUx{k-G6+zZEJ2IH&x^I{;NNmt`Ci@`yzF9B{wuQ72P~LqE7^9K? z*%J%IH4_+vhI&OeMi9fxVx0@0mua@+nrHphx|!ngIbrR1yTQjh1Uf`@@9~qa+Jbsd za&gM5Oj)UIQZQhLlX{J`aU2PbVdLSNBSGMOt%!0ozHM_v7lG~vEBRnHofVPqIopHC zi^NmP3XNU#`3hBK?+J)91r{!VB9Y7sHb;Bguyb<;<( zid)R!%T=}{Pr4Jz=?f6GQW4&=A7wHt%-m5{r_Qd+axqPOUnH>!C+Ey$waVc1ir&qO zUK`2($dZ-1z3Y@nBma|w*p<}~l zZ%+gc_6?2A@}*SOBj)*uh0p}Hb>eh1nkN#)KXODxO&;XyIbsRZPBbj*1ywbW5Ydc; zwKJTY3E)LgL(}4%k=sr@c&*JVZ<+OZ&;622>6n8^EBvd1@@yx>b=Jcl<<0^XORgSp zuU{s(Kz;aw$#2&RS`fm~2V8D&5#uJVZ)*5nW$Ff!O6xM&%FQQq!Ij3Pjk zd&mh;c#MpCkTCBJ)XScWC95=`@4&~M^)mf!2#yjFfQCVVwpW7&D2(dGXM>aXjmq-$ z5En;|NxY=j7qv(t;#YHhX$3no;yW)YUX40`PK!r~swq{mDqAWts2)dymujnE-gC#V z3F&8v;!?51^B~Wf0hyhj49xFOw?6F*_kz;g?C&@fnXiw4*`bI9sKP0m?D<5jMW;QM z>IEdk#ORjo09WQj5VY&G9T$MTI#hGl+4Y~A0IbAoPeMHn?;k>nMy)w`-m3~!&iTPljc^EI z1U{MpA~hhgP;8?-(pjQ1XcfE&!Z>LExo|}k%Zm|psr|EqBiJX0;|o@pMz+cU_pazZ zi+`pJRp^Amv%dxbFv$Gm5n}$(>Nz|GM5#;gIIHKvjHoHIsaX|I-yZ}vt;bvC~f z6zAwq42ZCDKY@!3Pd_Nxj*mN=X9ch$)Fh}Xo%l&#O_NHni9gT+PpLuNrlYsj5dK3Ozs(w9Y}57D4y5F}N_{Z*)uK%d3N@$Y(SKi68|iJzoQR3uvh8J$ z!^Rj%wJ~jSSMoV##k9bJl=?|Lq>{1#{p+I`qyuBJO&ew(!}F09C6zy+T8Mi_(e_S| zRB+JVoH!wjo$8%L!&}a+eefauHd7&j&%Rhs|!a6*tcVXwos#5s2Gbz2l zV`Cp0!T!Z}v5KqFW4+YP6$OWfbTW7dQGbVJUG+7@gD*T!StzVMTBgs=+n%{iM3R4X zw5$4>&u?SkyZ!y$nQ|?EEMVjt>=S)75?WtE5h!+X(z`|Vyeum#t{G7~ID@9(c2KCU zmSQa?+c^cPh(thl1bQ3W2U*3q_ixjOu{n^*4HWoRp~lPPnbg$(k)&ckUe$KZ{0Qwp zw7-;N?4U_0nm;Aes)<{qZ9mbFttQP=MqObmC?FwU) zAe}9#p8H8nX1MTk(vTy2)?mYI$rhPA)#v?ds$22K{ck&eAE?YY<2YhdAWvdnAqtEMg(?zv6B)c5bPBlDo<0#`hXo1^QPjAp07r z6d%)flr`M~hW;Rr1O8_;^_T_IpQlk3qkSvBoAb4t)#=7fcy(jwlY=A5MpJ=^u1(Vr zx^U9%HNza++QFVf;pIa59E+ECA~|hrbT&pjYW6ZFSLJP8npaK0UaI(<#jY_+VBjoA zZJwF>Y{Fu~dU;hjv~(I;q+>8=+c;=R=-mILAnikBmE(-a%tcwpA zBy!prTL!i~gwn_g9OmX%zjsnnt@3X_v}GJ2GNy0w&{*!nt&l+#^mDLd^mf%F!THu! z;0Dv>U+Tu0-<->_g3bZ1WS2h9W0QU8I}#vRTj5B_7y_cAnk?5Gf#jMoIh5K~NIUYX zhU_oW(>H+`H9eg9F&ib(cmX>?UJI?xMa4S%=V)|xT;9$*2@932F5kvSU@GCr!|-Q! zk+?;7FOma+rA%hn^(4h(LFSe7$R+eLnF!;MB3O30`9;8*7aa&$f^}ONN4+KYHtpcB z^84!z=N(ss0<$cp>V10hxl6=r_p13B9qnzpr|kjIVtg&=De5 zHhbnz{j%-Mx3ToGK@XOayq_a)B{*9Xkg;asxF0MgzKvS@gv<-KydQ@IXi6?kQBeuJ zA{v`25q$sb%CMpkwaCaBpA|g?v-w54SDVaHtCD*sp1j$2i1{p+h6N0PPx(jn3VH0u z)Rm^J>!DY1K7XEv^qgErWOD|6$Fb|?z<4p7*edQ|wkKn5Y7TG1E&=oD3NYExRXqP) z(lBmNF#c;WdD-2*wh}8Lqg`ky2&sfY?nG5C-H;GhE^J^rfjj%>-5Dg@#kwa8UaAv- zcVH+a<0#>b#Y-um(6Z-@ezGa-2uF(%DxcZRH5mm-L%Q(4qc7}z@j>VIxM(i&!j;wp zeKxk@DmG5(w1-D7GK3H*D+W^26Pc;QCol;?J7df#A)$0&Zld6!yBnLfj@V9nsTYf# zYP(Nun5J?m&WNJl2>S~!o~H&EF^0j5!^~NP3HT*YFZ;(Z>VVI55d+jF-{&a?>*C0F z-NgSU0oUe5AcVEFN98V##8M&o)IE74{mT0aKGE==_gzZDvyH_y*+5^oeE$&m=1OB1 z)2I$uc|SPH2(CHom_4>ME$KJN;-DEj=shr!!=5<{NwozI*9ESNyAAi1Wpn6NE>^CD zH#TY7Wg%7dN;m{WoVR_9U@|bL-QVbk=4s0UJyO*IMVn96w-qyIG+lSGkZ8YXHA)de z41^Xgv66fy&{YpP=6!j&0E_)LY@;#}y{~p6%vR7LZCal|={H=Nvn7{PaBR!7!%t#eo0L zo?yZZ2{qYO`&wMHA}qhC(xe%#+>}y{O^r}fX;}9ktcTxT2q6ZtlcFfe821R)!3(dk z(6l2vMiIaTNt6BaDTrc@;`;&dH!?gaWa-Nxh?Rs9P6eSEvXIPsfEv^EVZ3CCwDuV7 z4H01v&F=5d0yZ~9@C7d9X_XKddW&h7KWW;Ay9y|yW6Y;t$94`k!im*MYfhE?r!l4F z;>>RfzrG&gHyWnnJZdxktu-1IzBO|$9c9AF8xLTMp7Qd+SW$?hd?YfG9I|2;5O<&EpQNp6?F zBYcXmB1G{8*$ykh*cI8AXh@Z$6dJUjx5w6?bCTw5Vqfu%^umN%MnT3ThzLnHG5KV$ zguwjNYcC(F^Xnl0yi0?eql+G#FNyqqZ?8@kc^IOL;6Dtk5vF?+@fbOg6n$s;8(Iw9 z-+*0W&57@OwzqIv(^1`*(()#;Y`AA4Xn}L)l(ET?*K3gfLR0yd|KT?!8bHHm!-0lbN}@8d=|s4?+k4gj)B^LXz9)y@R@ZER%+#YdW*F%pQ7rOq zoMWFSZN;U&>Ya)WU2D`zV_HiTA+9{3r~eB!vOP@PxoJUak$~fJ(ssLqK_^-yOQ@XhSP~m<&=!@uB54w_X)QzZ98=YD+^aF?!W7T zVb_1%?JB+IFnCZkVMQ$0)|iLVpD>2qf$Cu+)?=h3Jn>SA)H9fp1A+N|9#90I`g#cp zy>_H2uGV$YeY-M7WZ!UH1vIq)jRIbY8UiIMS6e=86;@Gk6Q#vc!OP#lWSWF$+S6VN zC9xdqHd;$_cw9ACs~ktqH*zlvLR&)TZ$`M=lR-a5A|@qM?c zuwdX=?&s04+mP7C42MAaK!et_q!x}=JeLXLzFcPBp5nyw|1lkHuxYxL*}KgGW>hwM{Tn zZXS>NiSimwQd_d(QW{suZvexk*S(CSUKdou(>bi{IXEDhe_!D#^{Wp58w0co_k2Ig=LC?hXtNrO14e*K0L8 zw?4?!S&-`A!og1S=bVZtAze#>teM3CpV7?+$qAd`RIlodbqW#U?maAm-G4F6%6bFOqg zTcHOW4?UXefJqqxi|NX;kT`t!Es$;+>2qP_=(t>LBg z8wCTm85Pzarw?DyzZYDKS&0^{kH?O-o5x-MZQ^uZ7&ZMDXj}31MN^gsWz5#tS76?| zLhoo(T3TUgCEe6YpcC`8(SMJg!vBsZ46Ck*EfVnD>N*KJZ?H+B@dOnfC*b3qSUU9z zxP;PFrWH~%iKuy{(X3ALYI|>yQ<^I8eC9lia(v31&efrH%N5c>*3v?W7@sulpL2*Y zJ47P$dpNdhyJa`qSO*PMPa)9u+fl8_;4m zIN7U^NOYVCy$}7~WV=jTYGV=ke(qP7=FId1uch()uGvBv{Tdl>742WeN32iMl4lyo zj4;@(1wi*@2GnU>A!?D`OP0ky+k`b~Zlya8|3J#3tvtSH&Y>gTxB;k%Lo&zf@k#)7 zIt+f99s_c`kiP?Cz>;0a?%^-7;?GkXjc>@LUIb$wv%3ja;EJZvbe zCWM!%|3}KgVqL!NW#qgX7#k6==NpS^AAh@~WL$MD`W?h)=MH$o&Ed`bOp7$c#77+J zdK!7WeH@jmJVZF}Uf-mr&o#O{g!T!)2ncuV7*??fS~H#btgfq(Bzy(xR*0WO!g(3| zR((j|BTm#jHXbh!Iu??)hmp81<4AI$8lg)x4(E6g)lM`THjf+WunXvVL|?m#&k-6I zac}z!Rh^&yR(UlquApJ^Y(nP^VCT|XxmMxb84m$5-k z`YYQWvMjW&UaOXf0fD*`ed`mObm1_b~&^V^Q|yZ}!LQ zEd$SZot{bqk0MLYfzjomptEHTg6`(AvhaD$$f@@2q1N$ejG~2R8w+WNbxhNyUF43s zOW?q|FBHL(Uq;|cv^#W!D@#6-U^1Mx1{`|*xdtKd1jX&}I!>;bI`pFhSc^%7I%mBHaZij$=8J(gx7+UG~F@(BfieI|Rw7`*key0YOj^G#6fjB ze{c|8M<>uORbWL2E~N{gD1IaK^>yi&4iz)uIgLe{0ee-4FXUHJeIEHlhU5D{Jf*wb z#csEh2oPAT-|9@V*@qS7lpp>7JlQ2XEF(uFBSSzp(!7X6J~l#Uwr!utku3c7`4T)PJ`o@qsx z4GfA+DJ4+iD|IYob{;Dl@m>y>pr=EkI*iKOUO&HxF>CblD5!l$nPO|lON5J7ttxjZ zQUsT6RQS)wsfYGHDeccoxfXpRj@+-)z_6CQ*Y77L%*Ay{Nia1f9 zCYlw!qRL*7sV!ikX>M@@_1JqH3i_F`h2$ zRI)Y?t0B+I7#L9O3}_@Ch6gQL$6j!x!EW6NG-k;7=;Mh(ukn|0ha;T+ms)!ShER|w z_gnE-IsV%$JUSktPErLl8HcY1HUnFs9MgAgqRt6w{{bwrq&Wro5CMk?n7Xvv)8pzX zjj6H@H;+D>4T8?_;>hVfCL8wgUP1N0GX7Cb`Ld$0|F{Ku|816vf&CKvmg_b*=5k{r zo#jPhF*Pn750|Z4uc+^(U7z#VCp<&@hF;5N1kTp>=LB5G#of@uUd1aME0b$YVvC-S zujyq}K}4;kMqgT@H68!TdS4d#B9=|wx z!*58Pul$M(#-x16KY2#4nybY$e&UN4*`DvHr+w{T#%NuKdw=v(K64o(PJt z)5&Od7A4I7V-;qMTlvJrth{y$O1kSy`{}>iBbMajUwd-=;y{2=l!%eAiD;h(>np4K z?z@H&sk$Ig+~2!(mJdOyC)I&So}%q{xn(Nw`1hUU8Uw|^1&p?}ZiM9(l48+ZljdQ{ zpSVb;{D#@YCo@L9K=5FoG#?h#Z2bR_bogv1kBf_HL8-T?ocr^u z3(UH^u)6~X{O59{I#66AqJgdWEi!URu$72|#p};^Ex?9E@R?{ZvlHoNq=TIb`pYJU zoeqCLQ(5tl{nT062QwIhQC3l*_)TA{O7I~d?}kxiQheXe;#H=*gQ zJ=)kOqchOgbrY00qCXdJ!&Q{8c|_srw@RsX<)JkGrtpxJcvGb@wl=T)%$qLLwVN9U z7hSaKXt{=xj75n&{e4hTTG{`7R|sZZ0HRZ=_w%ae-Fe~JupbkxcN#jI1`qo_k63SL z(+LK13UY!}YDeUf=FUFbc0$bJ|5WsI9@2MR2R6Ki&fD4PX{8Uy%tjDTgq0`|TZ4zK z=g*K(V4$ALo6nNsH5QchhtDBH7r?lJ1O5d3+JyO@Y}D)E*{p+qLQqA-3Bcx z`WaI~I~yxb)8LvmOC>3mLSHf0`6(ifv*)~yvWFpJ7Qek+dET1e{?fJwV-n+PAsOlK zZM;FhqV52AAXz6;7CT(Iv0Qr2S(zaNy!(7c`Ej|Fo|x8O^kXPbsD`~IZy(3Ngm2I} z%!#tT`S{qN(bP9m_%^_|RwiTY6b-^j&jV?1M0V__8^HXsznOH5m!ylj#(5otz0fPk z(b72i5!}lkiH1a}w=h+mV^=BST%6$>ojSl>?cNTkmW#Yc@i% zyV)0)?gxE#b*y!xDmJ_VkM2H4k{lq#)&k* z_>gSTOO7x1oFJ353rY_8uVA%~p#5&JWd;m{5D&VWW`RLsgph>qHx7E<4V0HgpK^`* zW87gIdq_-c55iw_j1@*IPVryfpUv9zA}dAo;y; zyJZk?GXZS)09h&*0%mW9t2`mOt)qxlL)LhLy!`iJGP59`@o+h?F*z!(s;wsKy0qv& z%CDE|Z!u@y$1>a6*U>G-8w_I}Bar>+PkB|`Yc~pJ-o=GcC?&|EpG*Q%2+tJ%7m*EC z%_2Q6>jSRa7A!aGFlajA(#KE`DkcmJ5E{#1V4}xAMNP#+!(wmOOT*Lg%Rz7ZLe`;R&%X;3aW@hl5) zhhM?>WL)nVsr@|QQWe86IIYZo-&LNZD|;+HdYU(WkrRISd$!e>Z(Eln z5_s`Qdp`vJF;=QxP-s3ld{X}%mC0texBB;3aLl!o{hIs;=vGl0%XcMnvDf8U#K&+V zIw~cG6@K$wF?s5O!)b0KiKwv&DX|Y6HS@ON66f>gtO#DgAy}35b0;i#uKD}zXA>E_ zC1V_PgBCLm-~!*r;!@GkGk?kK+#bY>x4(&hkcZlTr;Ut z`F&IMsj>1=`B{hQ^Hda$w(!0*8+%%)#*MZxz#nnVf3AX_-QU-BCFfm>a^a*Yv^WZ5 zw?|m!pX`GCb8M(?LE!gUM#9P!IIA(5mQtk(K zxx3dw$>gmW@pTD?vi}42KncGtPVfE{;7sR-V_o0aXYes^wyla^wo&C|=ZwdP0l^8r z_j-fV8F;5$Bc3u)PG36j-t?m@`JnxX5P=OJiXJ_-P#7OY3N_{)QIQ)1mn@db!50|F zoKBsYR6$uu>1jyV7}^*M8L0Z?kD@)YR8bncX}D&&_~6d}5*COh0(i zuypDH>**GV@PWsrR(*znSqzlO}ZP~g8v8aKyFEWEc1UySRxA-&RwzN-ZE-{!Towch}L<>}ae zi(0W2s*BURyMH-zImZ0dHT4~K^sKf~^*CK)frbgqe@vg6Ix-!<&-heZdf6o}5jf=Z zJJauPe$urfP?!TLC^iuo3>rm^RB;GNw^*aHq=(^R#>D}AIrASD1 zy*J)wH5@fkF%62J|5Mou>8PDPTp`FV1zY$h~}j`o=lR5C0<8wGJHqwb?Br>f7sl$T`80zPa6) zbo8|15wDc-8P$8Ae0#d|{ueDKDMYd$Rs@&;9K3FZ!J@y~&Z$^a83K;|E*!8UN?!m>Tw%jCg|2D0qXo*8oFFeCfoEg*%Cfpod!~@p=%;mnS_V;xW$TO6 zKnh*QyNYDcWKYyzQAHyvHS&*YavH@mT1}2U%j^Lng5;0#s8q#K^sy zY6xs#D2&BMCLFKTC$YA)DdskcO$NkS}|UN*n*;d0?+nJ`=lkSf-e7>{3o)et6VW-*w0uU!B!F6tCT| z;pQa%#U5*?Pi-=?!uqWFi_$K~-{YH}Dc&u{#-s`4=Cl9Nz{KnjcX~*B(lK%cQ-YJI`PJ1RUATjx^0TMg*W0NrUcC4_Lxv1l@OE@p7M<$i^q=3;oag*-ajI_{wdRp? zS_aoFt!Yekke+hDL|w~NHv05;9*z<7Nd=GN4n!X)FQI9y zqRKOIapf+!`n#N7m^mL$ue>Qe_#$52ga*z?Tas92ura`Fo?5*4UvKKUHR%ZmfiDE& zg3MLiBx=s3zy3}@JpZdI4E0E5LCPPkgC}y?0f2RyYzJ?~JFn?b!OUB`Edmq;7xC(2 z2r_Rh5{4%r{{XIjcChcJ>`03h61yGGIii%SZ`lL#tHpAtC_#@t%YFHTlK!`+4Mz4$ z-`QhB<*r?k@oZ}Y9(oz6Jl-K&xOhqW{<+sJJ?+AChqtY{e)l=|-|;%yWUiEl`bw7< zqqov^{ax!Yr?wWSZezxOVQ%w)n#ShVbRKR;>9{aG{rX(JA*EQDI4^B(+QLL16%8yX zpkk8isG?A3bW=fkbbux1OeXab0}e&Qmk0#%dM*dps15oND&Hv=3kZ({Sc|wqnbCnvQsxvzkV#0Hgoa~T$-=4}ftYJLz zj{kJSDnrsGpW8A`9)VXrl@gDXrv2)s^un9-3n_!eqQL@B(^3(rDXd^ZyXA~I=<5d~ z3$h4D${CzCtI6pmoWIOd7-z{C3?G;XACkBQ@{<&~B+ehaa^B&gAT;fZCHxH^&4%oH z{|FZ?O|fJIMQ(!dNY>0g4fw)IDSdUk+9|?6ax%9v6{!k&*g*>`l(piqCr50JGjLmv zTkaXSEAD+Nee2xY(u{cv(v9ETv*IfM7iP>!pFH`RWv+Xw=MRvy!-U7?%xM|krEB}6 z-|9bYujwyd_7}bokBuw!?@!gmiIe01o;cI^(ZbX+cyza(J$fH;(ou)CP8eFbFx~sy zOuZrH)!BS?lg&T|i7N5~q2s>`F}S#3`4qMQk`C2@b^-%6ueZ7cKIpG`XN2+Mc|7r^kv@s4?MjzbfvFBFTOCRyb^B%;r7L49f7!GGp;0v5vpn2&PyuFc%r9 zoaiiQVe-f|8RXt3w@Xtt+Kp98t^y_J3NseWVb@5)R^TI-L)j@`7~~T)Xf(KKO>&vd z@>n?NQIY5qYS9<)h#7c^MQi;hXQ&b{2qpaQcst@mqwKpj%F!n(7h`NkP2sl|R`3|F zacsZN(Dd!yHdyW%xF_%;gYRB&2VSW8g!~}*YdcR&Q^yRXi#h$^qQ9q`A9-H&^0UP$ zz$;ixk3P$OO-L4={rmXrLkrT)kH407`Q0sP*8IgLz`2kWmE9bMset41`Kbkbj<9~29g#Jv%v*Hc=z5a z=7U5ivR${O$S3CnO)Lb)sV#J&j1+BV7aS6*NoCf#Owl8?8O)t2+s7!NjeR9&IT1~m z;f=uTqXnw#kL;Viv-|pK(yIN-uv~r>M?gQpV|Y)w;eoU`USt3oE=phAZr!ZM!gUWm zn|^WmKV*wukbfy|c0XZynumMlm>3ojXcD+ABF`o& z7AMST&PIh13W{UM?1+F!3o7PZn5b+u!Slr?P zI`~i|g=}&oa>o-R{(?&jf%93T^$$o3RJ6%%2OV0lz8QDemtndmf{ymx9q(j}P4?HE zBQlM`8MtG1otSo>Fhcc?8{SfM{MGkivAHMBp1;tYHLX5fo6`x0Y>^ioO8Ew0{?4>i zHj4f)!TShn^?SEr>oG=)e39e6<+yVo<}?rE?@wPIxcxpiKY!_;ZiF`7Ug>^+sxD48 zPu}(x6KFV;Z^!3tJ$r{Y@A4*I-pm`x0z@I@B1)LNMS+b;T~<-EwcFw$fNxB(Lft2N z9GUP?NH~iE4=}UgG2~nfCJPkV$0m54Qf>yX+Sm|X09FenWQGegauLu>xo^JUP^<)^ z@DcUGy6LHjayeJ28+clzDW(F?{u{NNg%TH*ukNsJ`q;*+FY}uavHU7_<}Ht=|Gwys z^wOL2eEir#BIEkq$G52XO^QcHhvRpqm*1SbO#c@zX~!i7#9OMvkK%Zvmv}`8F?~<9 z4QX!Ekk-~!+Ry4fX|oA4reDYJhTbD8RFoxH5jWMv>7DP@D*$<03)9Vyznb>KO}vXS zVIyhsp~Ao9{BJR41qK}D5kxtb1tteV%?T{{xBwJE8#YMLIhTV;Vg#Pa8*#)x5RCp5y5uPnL$UCTy*h|UkBCa zkW)qfXhRkd|=pr6D&36_1!+{3W>GzWQu4!Ro&(`k4*Z4CQr!B2TM^BZc1^o(F zsxD6NV3%S7aVAziM&K>*;|8sg7p4dhPCWisAm$%f(8{2~WYuCr1(xDyhc*}))X~(m zMaja>xr8x^eke8uNDZUFBGw2h0~P@4$uVr1=r2>Y=N<+GL~U^hS!To0w2bq-A0Q zQeKompfconfpy=gSHY9*h=9$qUko3#fsFtc4S9j1d49;p3%2*4Jo;^%fx9hD$Ay%^ zNicmBlgRPPuF(UrI8A`8NH=0pI_}atrD^KcMw;H0@sx4-rI&(>0W&e8 z1{1#GuRhupXf)=F5(6cQ#R9ghGZye36yS+PM9$K#2Y_c5ZxB`lUtTyMWjR3P%RXE8 z6|0U9J(|geN)Cki=xeAEm+i||VN-qeRY<%>p6QJPPSFmWv2pYMqcPOCAICFrya%rH zN9XDc+=YKn7u@-TFwsqozkKo$17WW&E$KvjovYJ@l=2|Gg0FqNY=FNrsV_D#HvXoP z_?uHX=`r#28AAHG%=n>C^O}ct>C*n#X9n%O_YKco^k@FgN%bIdryyAYQ+08AXFHWY zgum^?i9N7eQ+j5`{L;j$TsVr^#l^=0j0R$|#?b*~obOq#lBC=ysSO;=xrsnkV<}jI z$H*8=Ozw%}<1{^`=9cS~eUuG>g)jS0sq91tP%*Az;V}EwC)zjaVwoa9_w0!)>^m=H z;TLVuSjjc-8t;L_8Mr+*NRviYUZF1xHpLmZ2h(D5cms2dUkI!-YCZno&Go8rSJhd# zPo5mVGX;PI_(Tr2BLjnA>wX16xsTp`>8c+t zi2l)|GG4wyM>z=P7wmHcn1ZjswoXluU(EiI+fPVat~sQF?x&aDrAr42kI>A0VeX-9TpfRN;*pnOsEE=sV`gTS)U~BW zjeS~LhqnJEJ=$lJ^vWwZWtiV{XG$yJwz@dIvz>Y#)Tg~&2oLdJ!wsJC7cu6l&33XB zi-rQuLOLx0=437me(y~AW*xU+y^Ka56^E~yL1U;T{3UT@S>jk;`*`zhv*MQfj4<3P{BMhuJE4? zxj!X5du#gPW{vKF`_lI7qjD#IkwIQBT z=CY%pfPjhE4tO#tWnWyv3?JEd74bnPA892pf|x(n$FO5~5=n&{eYo-QLy0%g9<=f3 zbk}$GNZ-JHaGgCKm-oPJf7~T$-``%D9(~Cl=*O{f-y*ZR?H9>PI=?w+^E9YO@sdpT zjTc(Jhzl-RrcdS|{Rb2NckF-1uXVOKQPDlHLAS9FJo23InCE%8{;94@*He3Kw$pIP zxj<1PJ}9m@srt3IT-IVWk9Wh5p(axZEjp$>7LB1ja@fmd!IYc~l0Dp4BA%)3P9BY0hgEEY8JmvzF2 zuoeDGUWu1TK?uvx4>!pvehSd8z|MGbpyJBLf$!IA4@t+qI>VAPur|LG?bM89Ji)ieg88y{Bnq5;?4i?)j;eg&)VAWk0!%mkIwe7 zZ~PSf6RG;@W8wgg$&$Jj{QlItsmp-&b6@K*a!PvRiFs(|(L703s8l_2dS|~PKYP*jHbu%yry#Eg?}gPKQaB`pbb(R9)!=iq#yx9Aa*>lfp>lK+Jb_D{m)#5f%rmL=nakF=+Ls02mY2m*0% z0gzI*uNs9N@aAWMH*V6C!0bAc?t#1P$nDdapW3R^O9OZY?yG;kHEnq8Il44}xbTa- zTBK#)tk~8at?)mk4My}!U*CECFs*$0V=ts1;;Qk$mAwl)R3ugYgMf0!vvm#rb>A>% zzhqu`>%Y$xa^ka~p=ZO=y0&4%w%+2WsIIVI@7S?w`u8KiIl&LdtA4kVpZ3IfwD^U&##+G4_1Gl!C6rBUL|Il_AZqGC~rZ0RjLq7LlCaouUq>0zEjfo_uL>2?pb`ravGc&U7Y^s zZ}+ByPq;SS`ov2(#qa#!h$Aucn@1$M`%=m_S@TNwmd12G9)r>yzc)pCefIpc`_C^) zGx6-EqW=X(*icrE-nsn)r>>d$cW=wBr{DO1?!%KH9AYl-LXD$iM^H8lw*ME^wly{` zowa(`k!w8q=EL_q+yNBe4|QH$oc{gpXk5JUkO%f>DF55?07?TAxWqrCd;b{RA)2|NLIE@{&?3;A?b?JT5 z2z<*CSNIEy$F=GP8`k*2N zmG<}Bc#Sl$^t;vRH$9ME#KY+Ff(E*b7h%*({<{MGTT^| z`On}PZ^R_9z%r}_wTGgy6BxdEP?=xE5pFqo_^+H)cX{X$Lq6GK8SSmO!v4$C#^DUy zo^O2y?z=bxcgD>)12-2q94smTQDmPwm&+`?J zVjVl6TN=}^tJojG*S@L8U6+o(;qhgxOdu>T1x$XD|?`|Hi{cwi%Vn7+6_kaD@zg>Ue};L8Jvzn$m#KDo%?E z1%@x|d#ijid%XUKXW;&E?C$C7Z+QmpuW6s(UWJ=;-=qU>x#Tls*)|FoJ=v!$eAPHo zTENr9;eFFL@w%sSy5sQ|)AugCjhYe%E(~Zau6+5Pt4pfv0aWK>}-b zfh}B+psj1(fQ)YG< z-PB%Zzd_hk=ag?xZ_HhUSBSq{!S;~}gKXA=g^x`6uXFBBA3gCF=<)*JA`M)nC0+j6 zP0}~EAFFrUSpfLlV$`5W8xc@MDwyN|HVgvk7^h8QX#rJAt|Vq*2{n9~Kqg&@YAPx( z8!_I%(>Io>OT2+tC?{hIKDGv)_KDkm+>kW=o4cl8?6*zU#= zEK!i0Nq8H7l_g>w{b4!mBVYD9CxlmboYZly|}(@=-j5kwJ~|9z`v8G*uKnHyFDii(Ik}) zl!&|N-WSq_KfDq@h`(B4nNLRi#FVe?C^_u5n1!6&1WQ z>OU^fY#P{B)7bjUfm`f87A-5p0##j{Vg^>`oD=+LVd^z`o1rz$E#G>h%i3LPu!yjM zQ^_VYwmvg2zVC&^mnlZ#*SfU^b~8;*EF$s$S^QYO^)IeVKf2;U-835d%r{!KPrB`> ztOM9B>cZIBP|qKv#;~MA*pz$pfwc2A)`MEQVPp-Wb%zcFG{4TrF%6 z)-^KT&o~s!H>ZtCiPr+qtpFQ%S|cwZbWNKu96TOkvQHJq^F46K?L8^o zcJ!`kmricZ#@1i@?oCSQ8QGqWy2%+05`^ENblPUz$7uYZRNjVp_ z@x%J0V|Lq6TCutF$(Pcx=f_)`yko+H!)*p$7_^yKqd{h~MGVS$VRZxbxCoQBGlqEg zlq#}Fcf%YD_%I8#Sdgi|sJ>h6;`%OqdX8W3cvL_1zduzMC+^kCr1=Js7N#LP?YrmF zx~BDJVR6!AL4!>o3t7#+m@LTua@R8zYmPX~ZGf+Ne|zA?3LGn+ z-fXpW{g-f|VZQanRXCfCz|F^`rcc3ay0YoBoG|+k?*u;Kr9w37EHh_fU!zSqHz0|2 zI#4VMRu-^?R~>qUw&4uiS9icYaQLzN&`mpkJnrSV9ANTy&&NG*H{)@)^WDC&L?67H zP7Q8zvV;F-)0?1+5-a?;e;lu_%?;_4Pi%$DJ$SHTX(n#i{`iSkl^zi1<0U*cj;9}a z%m6LQ)OWv#LF)LM6Ccw{eZ!y(9r`2pyq5b*0`H{g!hhw4A$op&pVn00wB4}X4mlWz z^3jufL81>wsxD5I`-wY{AC601&$ZU?RoA8aFW%@nt_xp$$jQS-P9~-ZTM1c5<0b!m zq2hVrtKw_R_`%)L8k3Mn4;oE2Jh=exu-o;wH>R&&aBs!eLTau9?9E@@I_*7iC=HvD zun&9YiE)-ZqConb>|V^u*?*txYR5)@#idOq!ijYY$XKGBelXX<2VuxK04h_533_24 z-(YYCZZ$gtx8u6Yxd-l*H0`)c{S4ep$C~|)zR-{jU$k32JfI)eB1by(#hecRe|g}h zX;^Q5{VvjH|8QM;`c<3-3>J=s#skdC&0b-nY~Q@7^trRIb>;B$F?AQ@Kgngl=8o*L zK|A=@VP5iDy2ItZ?bKSKXT~xf@T!EQpJ;`ej%H z(Kj!Z?W1gX4ZL(Z{vaMtC9wHW_Mc@Rb8@Tf1&)CYfGN>V%s^h_MIZ2jg!v79@q*5N z=eDi2A^$Q~7bn|AAKESVK0jKR25%yP1t^Ep>D zENNn^*@A@vevs4#p72e5FjZ%#t_he$P)nc65X!#ug{FCCXW%YBa$5S+C$~te^{c#% zhq|xM!Wp=;Z%Z2-dmhff{j0EPMB37p^-RWOQ?eL$7%TjbQ89!QIGZmP;0yoP9@0A< zyZdBsx9skxUP;HEH@)yhgJfU1%4-70BJ9c^z-c}DUS8NJWG^}{PJGxL5jG!y2dCjn z^FsCkc0YUo(Tr7 zGrFwV+K$Ima1v-Da58X~MJks$$x!D6{_B0ur+Eu`qqwEvz1q^mVLcpb3t6;9P6uqV z*Q7V`!}`ZgnVya~^A6oF7COxNHSyXbw@RPfbhW~#tY8jBuux2YRHt+Nc%q!0v3{8_ zS_7|8F~@{0XO)#zm8Kt=zBM@39%x148;+tUI(JrmdDO&zw!AxA*)q5FDQ zU7WDjJaNBr2l5t`V-_~{>NdZzPo4z!Hw{~AF>#`x997oU1~DJad)fUjn4WL<;x$hw z7k?XV_6ZNH0@DeUq2bh9o=8)Fa#gzjd3y#?5Hg^Cy4Qqs_9v$5A!rey#$EUolU+8< zcu)Xh6FK~0gY@7@^f7eQ2BH44o09N002EkX`gUzeKmF(i>GorGNxQ5&jIfHtGjM;< z8Mt$B2JWsj8)x9mmiOO1piE{s#vCzLLS}%%e<{+8^@k-pYC(hW5>O~kho_(KzeyU= zr}Aj-^QT>(p2F`;p%ddrPT8jqYJ|T&ej=4Z*}epYeaCk^a^l%KhaxM+Js$zlfA!Bg zX|@AmW3CqHX{6|zAhxe4o8Q#GOHE_bu>-c(}z#Y3=I&67o;O^rYxC_(gaR%-se4UFxi~g$r zvSHT9tB%k&SF-+D5L+Cw&GnU;k5&vkYQl&ADSdpiv1zx7tCoT3#B1+M7u|*1i-NCg zpPDoUnZ^~>!hQg<5(~z|fsO>V#ieWDDCTWC*xYBx$!~TU)5M!9Db!A8 zfi>nP)Fe=Vb1q0`Itn^WlFRX?-?%o1Ekk;?rb#0>H<)Y^35Rk7y6Cy_WL&uQ1+Rtu z{8@LV{eORRn$h9bmq9&R)1{x^A{~XR%W*={d=cJ=@?_7vn#OtZQdJY3lbU673MC7I za*)VB4uD0Gg6m+MfxDac!0kBxEziL1{oAW?58O=IAU=Y}?2y@mF8fJUw`C;?l-zvw zyMUl4#m*qwd_@*IYYpz1zPo4T?@agNDFerxbBnmMqA;}YVkclG5%;?*2e&ly__<9w zi$rn9#mU~j!u>B5v6cOoAn>L_CfM&de<03&f()4OiJEvpLe1iau3g(kuXP4AKQzBT zt?Wlmu@5;8yhY`R#r0i>%xN0vf76PIlXp0GJZBRoLlI!C`t0z@G!OUE{S}uLloM}f z2{rO%43xBp&>PY?M=UH7Lw3R4&!)*gydvHHWaX_d{9*N*JB&-0es0UuA7{hlmw{}O zMwvMbL{PcM6oyIjiT2UQ;8Ay}41D;kGjK=mj5BbPmwyKC_t&MZe)89J>*KiRP@FR5 zn@;*4EurgvlzlP(skl5m91UNcC;acyT%S(<#MY^~f$?c+*8GL~ooT_M;>!N;HTH?& zZNJm1hVw1$(*btb^1$KpB z$Q*TUhp%?K;;S9$2qD=1Id91ho5P0+S1{?Gd3{dW`sY`tpZ)Ct^GC|InYvovbnDlq zrR~>53z_!x^~EmwyKCES!P+&INc1 z<-df@8@F$LD)k=@mit+@ZDs3#!6C8#h5s4v9nA)*TCn|WzfJJKI-XfC(ii@CU3&7B znc04!Z91yE-`xMfM|swjyx>&H-@c-Lxm(K|Ir(o+#!D0wL7I6XDIdvl0g6V-?W5f% zlWG$_7)s-TKezK6`{MVfhNA}V@R5xG_`v@_zA`UP+@aiy9E19{fum2D)iSbSab2r~ znh0LX!OIB&Wlkn7RM_{*A+@6^%8itGUiCd|{vseO4eH*4htlxdXGWQS1zu*r7jorcAM~t; zUCT|zd*DtxY}0flZq8l)8Mu4XI!B+APPqO-yO^}xCw0)rhMlGl^21~?YH%PI!w;y* zC(;=uk(R6d4-Cou`QWq+unn)x73^7==<2~U|TIzDrN zfoGoQH}q+3?Ar6J9_wwz)2N*Lf(Pn@l&`FdQ|u5eOhb0w?;|+m9Y3dKxZk1@OF`b# zw$EZR04)vXnprkcNE(Ul{H^Gcd!G-H^8BO%VTt!<&ML1IGPe^9#mr+vy7r+L)8rpq zp00kd^6MkjKeOfP>FOi5!D)SeoNd@a8;j;QymYzD6R1=u_wCx8j^i0Pe1&_fGjQXN zJ~w^u;(z$>Y%zW$GcJS%$_!IVtlj*e4wH{Hgkjw!)+`O;El@)z`jPR_i#Ezw9y_RK z`u<0zWF8jofA+QX&9iSU_1`>WTY;}EoA++tmCQasr2))%bs%bd<8SG!BOv_EDf?f> z2ZYWrsg4bFVlm@>vkx0#h-B-@cYyLlSZaz z-k6i_e{P0*RQM*n3iB3t6!9LoFK)ADI_2Y=q{*w|`RFCy;J+Ma;P(67wdwR*9>e7m zOLE|#+vc5Be+&6b*p&VF0P>9TVs-Du3Ey67-HcNcBXrkLh;J%Hy%Urur1x#DzE_0|R zv(27#)x$60Vfe*YyZ*Qvf0Na4{ggQduKPj~|+7-#=d6@l>9SjIA3_N5i&cJ_W($L3$$@_nXRbv}S1^ffxv;>N?AN5@@H^A! z{*})iKk|(0)8jAOnZv@q$EPT5VM0-MWrp^_NtsggISZmeUb%fM8_%+5>-cJ@7`u*3 zL-@~r2VVA3iCl8{!JH{_Reby@l$;xvu@|^NlP~D}+~6Je-5f0+$lsqoYQXG0vkdMSKt*O%!4C=dfJG0-$Tk#AEV}hvop!oi#m1%K`L1$jub>Qf> z`c-i$Jh_h%Ho#MTy#mi4dgzxg8dZopAA_{%TU2 zc!Ovbu^3gdDRE-DRFjfVStQP7c~ZmuSqm1W15dg!eg03k;}#lw22#+B#;cXC{qlC{ z(5W7gNA4zd7-29hH0lV_c2dnrO@VRs2B;J@?eTAgb^* zMqe^;6h1m1w3pO1r^WTFv@{PI^_SGAPa{M;f(lv`-`}({ElwP14m@{2efKGweS1M; zudX<})y`+55@#3^CjfVB5oD-V>^7eW$Sio+$dcndC;l3IwJSe2ybpe4-h6Z)=$jpy zD$vARjqNgtjY%z#$!TsNV5BZv977o~ z?!P#tUxXR8nF4Nj2#6su1jO@-^^@@&@Ycroqr>0k7U_DX`Fd@X*^+6&EAgZv5BXRY zHFBj8xbN&`_#F0y4|OLAB;;h` zuExgZXKAu%$E%eZdD5ALdgY0OD)jk}r~WU*1(fBq`-GvTiDxaGhy_>R9pzlC<%!0L zYwjt}^Vt1(;Ds4!lOJ7*``Pd{rbF6s{LpmWSGP|)Bk!2(9=J!-cq~fa#TmHS2yE+% zjZxT-z>&}JBiF4Hptb(9sX`8CsCQQTf%eoOyhJ{JKi+4f zwEDoFlE>!DXWW1*)cqdCj0Zs;zX*Oql&GB_iqQ$r)!%RlTE+gO|5RtADX_fEtD$s~ z!3l`RheTY82Rb=&$*}dT`muQcbfB9P=7_wbkO5t-(9*I}PFw z%mTHtElwQx#+F`qi%Qqkaf?bL_g_xhks;tbusw)&PF&yXXfMh!QHnN_spfiCPWWo~ zOt!yx5B33Y?$1pcQOZp+%o%YVwFBdyK#m0KAuqGO97Nv zCdO*m7CID_eX%;X#J8utCyz<{PFdXqzCHcs2h&-%JwgR)Ii8J0T_v>hU$n$7C$2nJ zRyIB3GX(8tsW76FU1B=k_{%H%g_oL-WkDdD8aK2PuO{;1pDaNTV@e+NFYu@rK6fY+ z4<07wHMRWxsaI=#TenktZ@fMKS}{&I{P98Yy-zE{;uLe7eXSd|$HAY)Eh+=?whHxK z6VP{5u`_M2Xm_wpGlVh?}XJV4C9qmlKU9JvgD>_`$Rrc?0yD?`~-)3mGRssHg4qCqK7s-iW33K}Lr2z4KbDbbYHp~-eRJF3`mEx2rnw94ccyv%wKO{m9y4%^;jAu{Xf8y{@dm)^no^?oumVKU65q|6sVWwVicC3rYU0Ws-0eK8B z+vbtHl+2UxHgyVxq;UYhqc1VNXEH2JAT7!R9N=IVdy{O-?=TcsuekjHqIgxLU`rc01y% zT?5~);_W%h=BpOz!|tNZmlzLqdR#DMf>u{7_3?>kfFv_VNA^`fHE+s6Jw^a_8#Ou#>jZb z3jH$Ph<~)tR9y)kW|)8Fj2qL#FT7zQ`pSReLeH{&HmgHwXD$27PDKJvez`9h?>;lO9x!wq5QSxSw2pUmE+3bJK)xUy!bQsPfhq@_ugXwe*6^ z5xwo!7aN04?4yBWMh|oU-5E-A|5;&@ss$nKNNnWZ9HUF!02O8JAILc_{Mfi8-7~lU zMwP!aoptNO=}*%i7O!p0*9gkOeyFGi-XFJF?f~3TUy)Os@$7~BN`JE3MS)~8vh1Y< zwwSYB)-%Rz$vk9+CPh(lfXmEL+ZUy=k=2ra~jZkhcIIjh?~+5jEdTz zVT+P02*t?|lg!G-1s&&C9R!>Q!+PS8y~E?VPB^^`;YIfpXY%BeJ792|xYTF7vD(x|N7Fjh>ykPpwlnGY0LH%t43wAyq}{}0%4Q!BA^t>IlUD7 zOq^vkbcsikS#8M9wKHO-i@BzHO#A$Ygbihkt8{?)Cr+PI(N z7L~?D4R|bT>^bDIlep{Rua33bj*fOti}?Fh=yljNfM{nsFFC_jo)8e7)8T^59o8O{ znt9p|xUfSPgG&?N2#U#6Xr{Kty7bj)Yo&X>y>mKjlQFuPw1a2qviqJ*8}ba?nbXsY zZ_Y(C0~ATVrgR*Rq^AAsay*Wu^41sr4t4U!H%usWko-a^j(kPl0DJ5slNX_)5&@Gj+i&I+1wc=^?+nBF)9~w4`Nl z$Ftw1arSbhFJRUVFgT5AkzoJaZ)IvWF51pr5uDzIFguWzeV!Air|HThiBV8r9nVEY z1L%6$5Mge{{0W|-fh8?U*v44LIBGs;r%B~x%pu3i5$cxIH4PfR!Pfr+ntFWbY&e7B zyPs%qMN2V&>|5Qy9roU`y`f>-H``XTUxPW>(7=gifzN&6lPnt89;4y~%VL6>g49ur z9WMxkvJf~L(tfuIGX05H(hD<+H~jM6x2?wx$Oau2)-Cr*WSI=$G6dWMxACf2ly*tq z*m0eT01faRPrj12``O?83>@CUrhXc}Y=;x`cIZz1?~BvLcRv;IoPTVi(dpW+?VQFA zup=@HQgfL+n%E^mYli^+E{Q9~NkId}zz3&BFF>(hYycf^jokhSSAegDH>i};QGdQM zJ^0+~CUW~4en|3u0$BFLJK71GWir^MIB=0ug>v)W+PUm0iQ5&3Hv%;+TEYdhNPJ`p z|CM}?yc?hje88= z?PDKB!+ZYqZbe(1IIQuh?Xv2sT^hUfJaI-F-lBp-RVFraCvks9u6@BhZ9?gBT9~fs zNm$;Hga42dYLn1)9cYxMM^W&d_m8K`Olgnxa6zV84Iem>(kD)Qf{~c( z$6ub64*dNMX{(=Jk#2eHCHZK0pu-7rDp^kZd<5|Rc*?+$f1Zx(qbhHGSse@0HAn4; z=OT^KFwv*@M@oK}I3B7ohfsBm(dD7p002M$Nkln>BT84c&kXxkdN`-ysQG-c%y%XNd3ayHUM z>an0)2%)TzdH}INPQO&*59D<{+EP=?ulj7Y^BR!8rwi1Iwm5NEIm#Tw`d;g9aqRr2 z-d%W$igpZl20onhtaFF*qY8J1X_*W)rqEM~Dv`v&V4((*!-GyD$h3; zN=_uY%|9=NB_Y!fx~VnSLy z>=?IAQuqt;NjE&4w*1jwiz}*~fRDH3i!o0-Z1c1VzVZ=i=&)dT)r`I*U|%@?9FqbS zYBqpSfj}}i5Rgk|6r4?y$;+V##Zs68< z6>~$wK{H#|(4#Fm(epwAMePc1xU;px3AG(m?0*K6$x<- zX!fg5+s`RRlk z9?=incGN+uv26NaZ7zp-nYI08@g z#6R*dmHn3u@?}oikQLL_Rvr*(wX7Rs>`K%?8_|te#Cbha*Jj?QYKcdAeBaz`UK#-< z>B>~uH|t|y+#;3jll22n0h4K#_oJjvpgxmBb_Eb$G1sU_G%j@X{<#(|j^&d|tyTNW zZG}zgiCfNj2a5eju#Guo#bRz_#Um`b5@U%RlahYW*CLsJ*%md3kx91Uh_AM_7Qa9B z?Ky7h@u_1yY{QYo?3Ov7tT#5(EJ^s?nwD}r5 zx+|v3D!2?{-$|=!A)L+^7?3MAEU>Rg;g3W;`uw%#%a5ChCAA_ zul(u8bmc$&tFrJm{$hj!F$A$sB@c#Zj5>0Pxc|&M;?2t(w5~%)j32wf(C4G3F0)`p6XaLx;JIALj!2sHl(iDf9%^ zrh`NTXy`EhOKN!BykJ`&d$Eq4DrF2x_9TU0Q} z9L4g)(rnPsV&gl3;Mn0diDRdg_5pV`^{5=s5>HH&SeR`8QNcM%_;ddL6j^fGcFh5L z`Xp6b@#KJ`a0c$)Z||hXo7gLavZ@MPj5hqHdU|Ol&cJEH#$QXDpi*T2%6){8 z{0WIANiBDjgV;O~#6KiqJ?bTs}_FKijU6Z_7a{Xo6g ze;lJcN^U?|!Eyb!-E|zyl)v7I8nXXr zD*hDyM`_vOeS6_{t%ubz-KG*aVFDn+eu^TNJUE6 z+=16Z9dhzb=@(btpI)1@K)#zTvmLHP9=QM4-gf}#RaN=lOvz+YAqfFONkVT4C4dw` ziUJ}cWd%{OP;BeE_O-4ch-K||L^xZ=e&2n zFOvY8#RT3>-n-|Xe$T!4%==!o+WNvEhz{S)efLuh9#$H92z z3x8Eu;a5|R`RSC@nZ29a#ihsOYwdHT5mVY%!!;65(EKZB1{bz#vksK{fesl{Vtm{S zGV|hDZ>^!}J$p?wHtc6ESe(wjWvc6T#)6sypvg$^l>W$$ce=3Y{^t(J7guS?3ynZh zKJ$}o8b9Y(8#~^hiE~hmg9bJq(%sN`X6LL~e8ix1D&t=j;A4*sdX&$}XoBxbQ zR5rp_Cio82GRQu1R&gIOwJ;jSaD=1SB(|x^J;#xoiG!aFF18Q=fqeFttQ>!m#e`S zDq;rATE*;!YeXB#_o!;*UB+a&4@uLn_~A2AlxA{z@7_GDi8!;z4t4#IeYCcfR;(==q{x|0?@ zbkBJ}nT(mA z^5Qi;(!tTNS)hSOqZkmX%Lg;35#~QSOcCigt@+ovg?eS6wy z)KJMptV%B-;XaxFra`d@)2Inb2E3=dbN964cz(&KEMNKaEothr^9o&Lw*DB8$y0+_ zHa`f1q5-fm2wzN|vXtaTwmNxJ<@jK8WHcdb(o!H^wyCQA@s;?frdtCSN(j49;R6)7 z+h8)AV##C}&xnE1z^6V?f-)u-Re$qG>L@HnC}lGAga1gv1H!GTsd=AqM}FWVXz~Du zd4wcMqFyCsO-@b`3>`J$5tVfs1`qk}f?*T!QB0r^Ob!$WK!d80;9!*lXai>fC&4Cy z$2H^`?_36Cn*cFvY>P$mhP43|LF&vG>R};2pMm?_fwkvPjB5LK-0A#3mp_;$eeFVg z2kuck<^p^sZ@wE&{(x3a<_xR_2lP7#8Y&>Fem=!s!P+k(@pC~?N@Pg>zA!m#~VHRssrTIf$ z6e3=}xqNs1O_(?$G#NP{85xtj>vdAtw0@ag?P-SqYU&Cf#Ai;6ElZhD?EHnDQl89E z!@bzdyvIU#h~p~b(#8=(1`KRHb?ia!+!@SVgWSk&A^`r`LWOH`VIbIt#^GB``ceC! z4F`5L4a=g#!9uji#lSki0W45}IFKwcmWpo9D+C7!OjIbP1{*2{GNyn8s8YO!HTiY$ z(aY~2H96h=mA9pL?68sXWPAGdyIUStQaa`0`}Iz68*g@m`lyn+{LmdZn~?P1?PlgF zklUC0#F1Be%tyK4@}Y(}O>#FZwNhZd@ZE6VnQ~ux`}Z$RGqPJFD98Jte*2;Q)A!!} zM*Y+yP632y#IL3wBA5OX+Gy0U^pp4R1E?%d&s~&0@w@933%XLo0W2m>0~t&vr+7DH z-h5$f=#KHk=uzHeIDZ%`!-R}0uzr9Puh=Q#AL61xlZ@DNl&(ot6BfDYHlNi1Zb${| z_u^|utK@_rurQ9?l$gdIeDs@fnE94Pt(&&kKr*aW6mSI}xa9^*;r9WJq4B{IuMN0~ zQ$DnolORKZ=qZC+Vqt6frrd(xC?!+Q_oMA-Z!19YRU)q z-a1|Vg~RaBE1qi%FQ7EK5DS9{!e*~m@i_zU0Ut4#?>)29iJKUY$Nfxv)k`TKRMXA` zQ|KfK9aEcqQ~u!!5=iu;kf)lP?skNIqkueg5nJHN8~e%(TIlYWSi- zDciCQTJ$}{Lf*y8{)>@1W0QIhzT;d->&E$tWJ`AFVB1_W|D+bX?uM2D_(a-gln$xDW8(8-4_ zCb?v&kbo-))kIsN5^qo*|0vJDj>o-m(=_$ux2G?^ZBoA{yF0Q+MRXvP55<#NNv4;KV6Pkv}<;7Bn6#9g=$Y?RJEw!dWFuUsZVgb#^mBV9CzdC*4 zoLdu5R_BFZ+`aaTN2FuJUgWq}ZKw#BOQwgmrT@>1*fRhLiEQrz-jn zY9UWp+#VN)SqaabHtuNOUTEhg%UF>OEaXbTLs+t;nY{Y%X!(PkQD~p@hu!3u6r%WP zUi%U~et}1Y@uU)0EgTd?jMyeg8}a-HU9?dY*s`|yA-weFOu&+fc=`s75p#3ZdBH{o%d@X) z7i=caCL!;>^8iCxCumX^@@33UzwV**mTzB}9>?P(d1=KLQGRsn-sy}J_D{oG?Vc#A zlRjsZe?S~GXplj)b&;NCyTSpRL2%BP~o>`$RA0IyNW|A-%Z2WA(r zRv8bWX(&qg8vITQbCM4pY6cwA!PDK(<13+FwRF=zNV=Va>^!F2-7Evg!z|GGP1jfg+L8i332aJXrOsp%;KUG+<+_bM1^-oe(;x&bH4cN}M5J?6NDH0upV4og4%z`iw7&(3SZd8X?P2PQ2N z`DJ}IdDG)va(OZrVvdvh?1KCz+2ozi8-Y{&ND?BCOUPM#j6#@<&IeF&GS&S&_{-%7 z-dqIoX4Co;7RjQ`byB5lQJgkQlO%Z&Ga1T0QVDqF7hx2eGp0c9yE3sdP+~G2wwD6Q zNf|znu&i-tYx7#8&K4yZnH7N(jQLPGhOqU zH`m<1L^2Dnz`KFk-odEpJ2hvI&x=oBYk*e`0EVI~nBmjrzB zerw@_rpXsL*4EL2FWTaRUfFP8Y_UG3%_cPq1 z4Drz?)$|WI>r2SfqW5VzU6jX087kyX2Gk-3rB+_NZl~}oXwV;{2O_Rc)>oiXJPdofa7P{vC8;rS1oP3Mge5Qt;0nK;^ z)S#imHre`*Xs(iYTqt-Ey4`UCPoA_tkG6pKrUn3%>Be9Oxh zlwv_pT!ZEYjKlV0a0TuM@7g)tjh}G+U4i3UD9_XtxQo)Mmpp(sP2I0mQ$=jVZwdd7wmqUJTUgZ&>?&w_jad z$9D~AZRu-l-eKI4$A1ay?&yv>?W7z=JpOikS4|hDNqY1Mvr<|1oJ^)weI?{*t*|HGd7_aj^8HTb2F7V9$vx$8x#6%UKF4a&t#g=Hz_fpfa#6&T3VSCGODVcZlic6{RC-3n#A`uLM6 zx2r92nW~jnigS7Lu=Lx1(ZEO@?pNM_;Yb24<}`H$Kl^XCtp64V@){FW$^@I_^f(2d z5qSrB)4=A2?p=mW*y6rLf1mnuo`|F8E}zNLjr(T;0C1Qc?IsILvRIdVj7x#j!3-N@fo-Se{fYg z`WN`p?5suZ2Z!G#pKMF5S$u5vqV&MCFNU}t3Ox)z?P5Zb5R)hGmGz?38y${*v27Mi zgU>eEF(0@$c_IsYA>?&584|*@8#U#qLq&|(sRqVt{yh8QIk%)^es)=UadG*P2gne`(`XmXM*lW87Ia^dZNQV)-|%~Qm(U% z6NPhm%^$}xh$dwogo*k)#5s>*oFc9@KC7Q4MmE@g*^zbFaZB!hG_8&WU@@a6V>20P zz>amqHLQt4;y$kA(HL7h{Ej+?zmE^M3_yMwJnWMY$&a2w^~hs@_fMSf7%_6pLWqT0>wmRUsbFR1g-nlN9zZW z6jq-cCL@Y17LyGPArAqUVL-s_^73DHffR{EvWN*#9MKTzv6g*unHu_k#RE^LH=X*I zbnD-rH-hiWc-vzhC;aLvoM+-kNZn58jJDggzLJMdp;iU^>?GU4U&x!m@LTekoT%P5 zS+_4e>NrhEH)U;uS+{E>lc9V-gnR35C1@<}5SQBpH)Uvp($uf<@;daHX0~|V(j^LF z#GfQXs-I|*SN;nkxwIi$po*9M(LeKrk{mx*XL{LSpWHm*fxU~{8+(>M@${AFz8@m| zWyXyr!z{p;S627r6rG}y?Vc(fQbi1Jg^j8NxjxO66sMkbAplv}rlk+H+ z5R`JSDQH~S1E&EZj&z`Q;}y71?T0IHd#&^e+|H+5nl8Y{GfR2%Q3IPz46{X>0i|@o zJx|k(qVS6->yE(5VX?p`%>s~%8A9n&-Wla{*r)UK-*%>&;vpF|DTO8ioOGzI|E42i z8g-3KoqX7*?bzr)RX;y}aeBx1FHYaN_?I3{U~nVXR|*|w2aTL!`q>gn?qr+c5Cmv$OwpN`@=tiyL4mrlQ)-(T|y<^1%S zMSB?G>nvoG82k_W^v5{GUf=_(lm{O1hY$q_*UK_`{LffHt4&tMob2VKmL+2`nsHm~ z9nYhPyPkMKto1wI6ZN}~zBQe3-PH6|JmAZ7GjgdIJ(DOFi+3=_Ki9{8>cSh4G+h++ zjXv9)5zT#*)6Qsdasn#xVNcsEMu8r})&OL&AYFsuRkzCsjU8{$HB^li#a-c-oarK; z=Co{tA&ZJ=K&dZ`l%b`kG)biSE49vNKD&Jx1q`8b{g|9&X;uY*6uYgyo;0Ytt)pxH z>=oqFj5`cP&p)KEaj zZ4H}2wdI2QNki%YkRrls)ED5YT_h*6^QYbE*H0s;qB(*7aq_UA6s1iq7B!4dd+D`y z^hZ;H?&>iwbGe%oCt-+9J+kQTQY&;Jp?TV9C2z#!Cg%WsJnh zS2PJ&+wHr2bWCGG9;jHZ1djS}D?u?&z{o#7G{=&$EeBh_$V4Cc-^O2p*oFU^Z;n(q zicT0mcZRP&V)?tveVU1_nDA6doeG9Eq?cT163tSEkIX0(l1#w1 zz)de1Op-kzWI8G9TOkjx)ED=IYGEwwh9=(6^s5uzl12|J`^9Ac%WEIdQ4;y0Y`fv; zbnTaqO2_Q6l~87%Ae7U*Y8o;)9!sGgq}AJI-xYt6jmP>=PP8RgRFJ0tbz5K5M4S3G z2CT6ei{r>FJ@%LTzNozUIXyR5sVtF3MReYc;C#B@DKytEt}f7WrML$LvT z`DJxYPV50g!b8KRQM>GM>avzGgFBn?@H4PBa12sJjqDb*8jOvT5fM!OBVFT1pco|> zLffvuy$x63w(I{2-1K?40(VV1idW!pbuG-B&2l0Ho?{Ze(zcCJS(SXi(KjHTeQ`;; z8{d$NLfm2apn%Dm|562v*^L_d!5g@c&+X^^H$LvmAele84*GBUbN@qU$?4imL70YS z%A5XN{)>n1lJ?kQY{5t18yDV@zJ%Kb4*J&l>B-p(YiNUUHS{MR*gu_q!T~leuCo{b zasyZIKP_`pFD2u?5awYH1%$FX=9BdX1n`;P1g$Tji+wNor9_rYmZ0gR4i%U0MiS0T zgPbN0$2Qk#HqAd$%^#CBorNtSqyJ_)_=@;@UziH%63v=v(?x#aOi;d&p|o z0(ZtOY2W|3GTn-o`OBW`(xABm?v6PF2Q=B}7dFBsf_*qlL+%@8`PYg(=hdAUKXT{s zk|#U0<$cNR>m;^YAt%i+kcEt3Hy_T@Aem&J4FRfY$4KmpDA-dTEwVW;o9u^tAp_N5 zSKoi)_rZq}K6%hiWGKqD4?UB9blKk|fB!SH(|)I%gPS*bo~^bVxhp>I`PFx&9meBc zDYrqp@&~MDp9B%*bKyUnj4p@&#s^+9K_-qCLN)tkq|ydCDIa}TEv}F^+cHLBZkdn$ z-cA4}N@@J%q-#}RK>*c2*G;A1Ge4b%M#MkjVDVQEiZP=$>Ixp}2;IcFR?$oI(SL9w zF}3tAUXt1uJpSC2vp=_N=CjX1MTc*AnY~~pDJCc$MMZL=DQ+*jtZvDPk&LY&t+iH5 z%c%8#TRftoXrX@`Y%sE?s=XbHLA?h-(ICy`SfS~H1JWDDk4#td3fzfrT6ot;vs%nEtiKdpiSdB+CDB`3b5B=mc=c-DVfEl+~;>7pD_MZ8(ZhZEyzdN~S$r7FjWMW!^ZL!x(Q2iw*pkG;8-I5c%XN+{z zw9%W7{7hH#@G%`N>-wtnK?p9^C={^b^eosUGWbG?%JsSEYHlDbwU@gVX#r%r*8HF5b8qDNH@mRzWtXRn94p4i4$w{)o z2O&Fx1Gg(MuPN>GxDXHuJERoyy8W=1+b1dX7~Yl?u?3&^CypRx21M#>=e4-w9legDy-y0 z2_LP6efSS1(##AQ%~ruDQi*duYNZWYV*+w3Ze7}?A;wfTfNARO?8S-i_5}~z^_|%_ zUwIi&JTi*J#3QCSYGT6TowNP@qyzeumDMRZ(R0R&J~g$jxA7T^2X8hIA1pvq;Z(2* zmq8}pgamMhK_fk%7=s7$3fwkn>emil`4zZtT#oO+*%i1L7}+*6(y0mWlaFrs{3&dP zA2!*&RmhkAX_62h3dowlf4z4)e%`wvNIQ-joyLtCA|Eu3=tD`Su+zN%9w)<7ISTc= z4O<8y1C3NCeWsLq#!*ECm#vYf3^~h~y9B)fsF6r}ZLvZ6?7@?Rr22+Oo=xAM!kaJ+ zElxB-j&<1Y?CMQl_~Q-feLuTYZzsqV@rx$sfBJ~@g?H?xK7_(@{OO(fC>jkyaV$Xd zKRg8u^zel-p|m#oCjZQOk;u?AKTTHbWTBn1k`MHw{>UjlazGoVA97G=(o#`2sMGb< zkDqKws`TFjQpiX@IHzru=J1I^W&3>wY7?aP|G+g_Y?%hg|4@X$T?!cP7(E|O9l zF)=amclbYb#KgX^tS-rk}X0$kffv+%I^|CviK@ zCb%$CKnQ}8#|}95AQ91$^pBO*AvrM|+~`Z= zh+W>eUTPis%>~1@8){v$h{Ysf60pg{LAQYyAA-$7I(gr|{n&K(mk!VsxOIo)5s8Mr6raz2zL)5XEE`2q$2 ztNy3m5lBv=OJ^xxE-k9G{<~S9Y;Ks&P=U*=Vzs#(#mt#0Pt$@Mw@Hz~{8J%h$luX1%<8B9i)Hmy7XvJP>^7cg{=Szx2+UBTMSp z9Y<)_f9*Z^T=|5;Z!KWz@Sm~fDYcQy^ zo%mif@2+PrJo~F%Z42-Xu9rMwVp8G}lU(o`)jwWVhvY=>IRuuRHrehc9jzNTcQ@N# zKn+e7cr73oprDG=sf5qKU4tud=dA1s+@J4BlTWz}SH*Z0l%<>mnhV&ews8}OwQaHr z<`6qyF|gubl&6R7bYG$+b`z-&5- zqFFINC?0gHZBKcJaL0E~YVBFDbm}ep`^(*;P>f)yau5 z;ikm2!6EN@Yib&}&!W}|dPIfMw~1rHvjDQ!Tqvwj6!8IRbG(J}tmAi2*M53GyoGYK z(Xu_Cf%}%Oz+IYtal>P&OP6#xU9805#6Y=Ph1mw*I5;89HhnO8Qj7|~h>|m2SO6{- za^NH>Uh{x>7UBs~H?Lyd^!M85&yLu6g7k?qY=Ph8DP79jA_pe;s2E$ci8g*LCMgJB zFGChVD{YEf56ByZR@8!V{tG+ozc7BHn0_cG z!RP)-j~u5K%A##uMw5tR``H{0s61;Zwdz{idYpGYJNYfc_(dq9EXPquFywtf1= zdv;FSjO9eEE&NM$=B-brV}5;0;$tVBc;cPf3Oxn8O(qgZ6WxIVE_e+n_|)W>029or zi83H2KU=wdiE8KeogS)MaM}E=lQt^YMh_j7&bV=!+NR4)Mr>-$x^3ztXtUW|Bu<+M zfUIbd5j4@Fw#DZ>AewAJ2Epbe?Tmtt=YQ%@eaOHhQkt`Q7xl(1H`IAX)5P~g4N33XbK5i%Z$Wr?2H(W#V?+iw z!G2LsEM~$qrkXH{!PrY2J9g3;2u57R%sDEgu{fJ>wDG=uc1&YN_!$cb{Qic&AvRq1 zR*WC}LeTMQTmN%jdP-7^KO1a|-C^?6C+oe%!pX z23&#L8du=nnm)AGrYn2~ZWmsG`{Ui|g@w46rbdG+(}J}D(*y-te^qFVG|>q=KmBk= zh@yW+#NZda?yJj-9dJyLS~pCdzJ+}FLE9JMr(J#$b(;;=#y8;Bk~3l7VnliK(R_kc zw@<9~$sIR`8wDDdEu1;y+6zCkaqo6T5WIqHbGq07;O-8KiUiV`R&Wn z4Uf(wLs1ULvv#gh_6b{`Z0gA;21`t6p;kYEBg#mn=O%CLQdKfZ&g9Lf&`lQGLfUM9 z>a9Da{kGj$YGU_;%kD`xJn|f(=QiY;$%Qm|3zz_%Bo4-9pQwmG@!HTI0Nhyan?5V; z`}MQZ`FB2Qh-`naDs(iHNn zV}1p${@FUdHYG~*LmhG(MgL5JSdB!4^+5pqjzXDojv60nfdk}%eQ8i{+mgN|^QS&{ z(b-?=A!VH z0;!C>I!V{uGqy`7&5H97ZzS5bV*Q{DDJYT$jG8ny)hE(|g=~d=5>k4@=IbRs)SH)E z|2`vq`=UF=DbYgS^pdy6KAn<%(^A+kyv_Xg4oF@y1JdG-&h+75UYb66=9PH=I!}p| zWz%)nPM3fA-RZ>rca+VLQ$HXdiG*siQFaBu!gsoFVMZG=;X8FiA`=WI>M^?2l+BNV zshF?o4IQRkLI#=%qqBb~v`szXv(<8?@Jpc(w>)sDoS#Gt=1V(_){*<8nkqo$4bKjOde6_wFT@rVisDb@!$ zY!EQ%CyY$jd~#o0f!h<$z{S<7G93I4+^5gIC+&2~<>|uvXNbUwLU|j;a^kR!(FWcO zh|d_s?dXCo{(Ry=R(_JhomEA(CS`g=C&=joN_0%N$!VpqU!vS^(B)TOy7d01^_lXJ z;m^8l$F5!Q(GS_69mh&E`c-4z{2?jG>WA!?PKX{pL$BFj=<4Os12B&q(vp6S*Pz%P z&*tgewvKe-udc+GZW(`%L{0pRFSj2ii@2yy<@m|E^qF01kURn4qf=+!@=$v7sb{5! zpIiRXm(z~fFa6;^4zo{C!LjID$Xi;WLKKFCdZ`!{K_wG*h*-+IQTpa+qiGK;{q>I3 zf0CS6>!QyxAQ@Tn9c?>^TJev&yCGN;r>y_s2&K(0_Y8=Jo@KaIU{Oc=Q;+;|=2aK` z0WA5srTmDA_()9k_b-`aVEDf(uXb``gko!4XYWI|958Uue=ZoZHO>%P@HCKMV!avO zLiwi?_DI*Q^a|X=X}gmyO{d@V1U_0AS0PNELF1%w&1_oHY(Q)kUQQnJfkYq%Sg^^P zkWD-}AM#RAD`OMU?MtuQ2OJZ|03ujo@|q0g#mAZjwidK^rfVOzq-#jLgl^~Y<~|S> zGk8W3mzN&Ea+N|<3l{m$wu;**Yyz3dM_+<9Vre~cSt^Fi&p)_-8oPFRqL;EC{ms?7 znbd62wn>&g)!VmNke~8$rDQIA7m7q~&MTrdu(zX;$7a5m4m|bjbmsNdXX4UeewHu5G(TS%O~V}{`wxTAJT_kQ_+^bWkvq`vS9 z+;8{{+{shYDVIEmXW*FBq1Z<<2C*DK5Pc$x(cyajawUt4Ld7Jrp*4Il@nX7!UDRP) z6Qr0xl;dPF`2Y%T7t%xy1rni*30-i{Qz5KccZ)m5%!7|WE)P1zi7I1}9~T%UWTZ!Z z2qix8Yvf&*+h+9%yaMGd6USwklzw>Gy@{_vVXO-K9)O6S#TyQKDaIqr)9a8&ZJbOz z5Fig7b~=gjU%Ff;XiK}h)93zhb^74XFH8&DdD^5bV{te61)n=AeeTdbOcv@2(`1vO zQkU9faFqG$4WK!m;e)vbnsl%-xc+CIjx^n<^wl^}h|4G=AoI}0TDDw2?XVr62qC-kd8z27u z<9eC~Z{9v=!$uoOzJ;0T+`(lf&dW8T$A*fhL9N3;SlZl&1zWK@NJw1VOmg zzEQ;@THz;Jmp$-ITGoY^$m30&EWCebB5ulj;F)>mC-w0zXciiaifH@`U9>DJC$;x4 ze4~98V=%oT4}$c#|Livv?Xtns=DLxndzGs-C=GN^44dnwz!(S=>eAU3?%~S zC%K6){U=J@bN|U2ZKBFzGA5tWrT0Fr6Xw7C@FDnc=h%Xc$>HXI~T z0e{t?xPHp>f5eaYh?idLtG17)+Yh?>Df!a>sb86b61}K?Vcrz@SK+JVTz-sNK(w+{29k{ORM~jZ|HnQ z<<+{+iLvFT#57`${l>PA9{ZR1!*?9o*Vv-VsVCrF-RGRRM>=@Bv3MoM@&|Nx;VqOO zIP;G5y{o3-=EP+_A*mH=eOA%{3sC_Z#`ZCY8=g~37&GX`Jo*a06t*kV1qjew+uaV3xar>rYbKiR~lN$QZP z_a2GY9QIrL9Zg}qEvwzUnwT#&}k`jNy#m~6<#FQT` z*#J@Bt5}xL0;||K0yq*IZ+c;DX|Hj_MqB=@qjjU^?xvyX?UTl)d-V+5PAj+qw-{I8 zZcT6c?lpJ@jwd(sI04Qkwk8WX99K*cwK+{pmX5I0v~A=KAM%(bLQ9UakT4a(vytl{ z&D0ogBn!%!HPp`IhJ+7)q&)K@CaGD$OX%u;+8wsj1j$pEbhraW|AAx=!bkF%|Lz|v zFJ{&kotVr~;oX1r!4yd9XCK%X58BqAXZm-%Any73{Dpub`?Oga%0T0B68lP;{LPq( zw)3gt5vzPEtD;Sm%hPxIz_xIxdF1kq^KMQ@{opTY_Cl`9vW&z-%4hxOk?EA9_f5?> zqLMtxuFooWq!cS4(kT=6*(m%M7wy-|>zos4vLyrjV@c*RDX|@MK={HjvAJ|P&SAd+ zSLFC83Le@$tDko;`X^pDWgR{B@5BQ!HSg82O+Q zR?%Q`n&2N{4mL)-&_yQ72N6xN*(|Y%qrJQB-ZScz)3{} zK@v@dI>bX73fuf?x3tU7uMszXo#FUW%VB^~3L)N0A7*SW|fM6O4`=W*a>WOU2zD365T}Sk>h*PecPEZCo z$-_pFWtVt&;QEKAr+rU4GhO%KQajI7yCuK{3^C(;P6o z_TpIn**e|lWXUspGfqo3vWH~MS{sZouEpFC1MS!LwXt_uFCIl{U+|#b!88R7Og?db zDNjss;yZ2=h!fwY6hi;1vdVwfA!zSJXRg4u3pT!tDaxp(rlx%l+I`n|>^yO&meY=y zoHoR(XX^`}f#XlNV}Em7y6vfXHU|+V4!q50YZQx23OF$3Uk;5|u9#sak~Ier$|UcT zidZXRa9t)Me2u*M67rgmuCrK-p{~CgHcR_pfL}^a(LtivS94arPz~MZC%q?aI0|=& zuT7K+z803QdjwbV3VG8{dCIUD%oaN0eF^`A$;6}oxeQ>SNmoH*GR`yo5^o#GZyVq@ z=st|M4Tx8kGW+VYkfsB=3j4x%GLv)G*BbxBK9#f37dhnZ=#=-b2n7Je)5<}g1hXge(5VEWxNtsMn~`JRisS)4 z;-Xh?o4i%pYQr(Ya(C%HkEBOu;1@yRG~-q`mmyw|rQh3x@sEXj^$$m81N%CbHFYgs zIQzD1{(JFb4^0CpPfYPME|Su#@+b;@e5GZTCMOKY9&q3iY~@LaI*-;GI;0_ATtVjH z?wPkfg=gSyO_x466Q2m;CvvhuM)WldKJ z+K0>zRVqG44OL}K$bik)O7LbOHF=49RNlrp8CH4=b8BQf0PAFmZrmhz^h?6 z&qVz&po&l-4-ApqioP0`P+82up%Sd(GS<@n2vWi6GQme5;0*0kh<=e|o14-hlebJ8jvAhB!jTsLI8Y+SMEa>yNS}XfFqr_)61;(V>rPvy zZ8lohh}d6t@3iy?lau_U4zoZ8Q)|iyGi3$U+ScNp1vd(&L49}^O-I|aGq1kjzn9IL z!53fkcYcXSQ2G5!JX&H4uOqu&S#ei>yV?cQ{#Uy zhO?~Ap~5F)Df?H-v#+ke<$vrh6jpKK zyPGGzm-H@}clWaw{^6@#3+C|=iAYTO&p0L~9!W7d{mZY&!G{%=RhFC(&~0I5rxjZ2 zhj1sZz`gm0*QKL=c}sc}&%m*0qIvD(F1&V*AWv+>$Alp)dF)hYV`pt&fNmugUQC9N z7cU!_0LA12ayhD4_od73f2u@;eq!qM zMmu6^_Ujij=7BmNunruYr^nlc%;^ zj|M-OLOC^PQqTbP8iU~gi)MBRi;A6z`+{;mT!AV|u#kf?;*>55iv%PEvkhqJt&z*w zO=rlTf7g?)$f|eXs-3y-ez_jgKs(N!`A?i&a6gOw2eW*3Ilw!NACtazYk@& zrIXLURc_gmg#D0XZNV}_>N^*8EJov{Yb04{3`dxHS@3hYi1S8izWllbvfK|C_9Jw{DZZGJcuLA!dSXhPg3FcN zXX_F2yY)D+E8zE566B`lza1+p+;O?=-3*F>?`Vax>UeE~e*2w9FpgxHNRUIYRwaiO z#kXGm?B#hPnh-GRrHIuJE_H6aB5?PYUWguT0&hqiXHE$pF(08|(k*e!a+B^b6GMvb z(_-@K8ywar0%&E;@PbyU2Xx)F( z!Wv&Icim2+AQ^z$x*l<}FT2|5E(wi+`<#B=EmW1uoxJ_n75Qh(;Qp=;_}poBjzTo- zo_i8DmzC?_`X6bEra9SR&r)SE44%Fiq@NQ@#=GZpy8KE2Dd=@8J%SD{53I~3hNpW< zH)BqwPOfjEfd^>12axJ|D8TXRPB0v`Hu@}(Y?E6&@$4+HCiPg>BjY&cDP~CB9^)NA z^Zti9B}n=HZQ7baiykx6O{<~)PYREs~PU|ME%VuDK{hh?81LAcVvX+XHDg#ip`M)tu>k1y+XpPyLXEQr4dZ; zJ?lPFk&|#Omg_aKsFFTe)RdK@h5r8gN*iFJuadOiZAT8dT}gK7eKLwCo)tsZ^$f%J zdpRkx#65>|f2s(TP}f_fj(-t>-q8V3gN^mN4DvO9wA{qCA zXW~jMUG?|*W2H(grbp0m5x^!mw?WEq7G3SNx_5Ee_8F0Rz zHqQX*D2N-L9s|r^Q=)%1P$TBD;)7&<0cR)&skrGHVk$pLfJn@()EMla>Aa?C674US z06~9wivGbSF7VF+=!t&I79M`NX8s~Ww~Q9wjWBIq^uv#h(ws5W@y@}5l1G+M)r0(3 zPa=y}b9u@>x{0R)f(o5a3g3#nAyvLNnauTF9{u9`6A)kgRePa+j!-Fy4}mDRA9)9h zp05USS8%Y@lVZe)9&sfxZ~Fb{YdFMi#yTw?r6&ws=KMKwL=G|DTtJ+h8VDml8l z;cRZRbG)1GP!WsZ!jwwF1do(3%0&fh87H%+{}2oi|HgE?IkBUf7cw3LgH+3l+~q00QQt)0fd zt3uTufonH{eb9@g2)bYVdZ7|uuD)hL&aEL!kjBj7xuz-8P_|OUN2~UNe>CR3Q;6=* z(m1|W1?KUMuV(`3&855S$_`IEC({$}+bGTIR4r3Fnlj>R?vRnzU6EP&c3#o1C*{^Z zBCdx=JPEnKe-uG-YJQIN$Gh1SEAtj#>v7qx_kVbR8f{Sicdn03-*d5jq4RVe#d%j- zr9?V8O`vN86LgaG$#Z|$5$F=K8%<`t=b35PDj&-2y?V7CO@@W{{cY!Vk)8ff?3&YQ zM1sg`(n-N8@cMA>*YQHki^~pQJi%%$fvZe4QUw*#t|aZGxYt$R!CP^}_Y3UK4 zRYjSDN`4&{zZwwA_)48Oa%aIz{W7snTvL_(MB=5f^Ig%e$<7h|wn3qTl;%sW$ASyF zv$5l>+||ZA;`brDO>V0UEBEfJf6X2*|5#tQyjTf5HEx&ueNV+-CiJk=heeD}mgGPH zC=S)6f2^ui^K*uFOlKrwIOn9eMvht zx?ZIA#L3#c9)A(S%XiY zpWUeaW|jK!H2moj@^lNi4jjxo+o@eZB3s+qQel}~^@y|Z$EMSsHcn&VZ#x|09h6`_ zauM<{w{e5EMjJPf^cwe#XkI?lltxC~))IsfqUHnRC4T)A@e?Ww1O`lh;>}7CXA2l} z0VLy;bCp?rI|tC>yAZriG}%zMrA_a2Mv?EgpkdJYzNM5^GUg@e=~o5MX*;iNBwD@r zS6@l>b8F8R=Wa$y*Y%m3#Vh)XTa_&Rk>uptUqg@?1{jIGI?*rEQ(jn7MdP={its)s zfB`=e9FX!T62gAMn|>F>P`Un?*(s{7*r9X<($-Gm|{g55F#x#WC*x(1}i-2 zgbGz49W)QOb68Vun{!H=bF3$GrTA^OO$o7i_EM<&5m}n%zok#8^kbCT9E5UJQuL{F za_zIOKHEV){$iQr^z$*XcISa{aWtPaIkbB`<+!5^p!XN`hx9|7s_50v9ED?3&->B; zWol{fCji@ygU@Uj%6TOI<>MpfmY3Tx5;Z1V=MAs-oQ5cez#1G2>0ca4i(^T#@@eG@nZ1ITQld54TK%HCSgnzT6}m@ z*_A@Jfq4@;DBBHjBW2cBJg+G->%~<@#$GBGI7x=7q;%iW(W6CGm|^8+VwfOt3eY-7bL=G6FVpV*I##zk$L2I6>C6}Iu0-rKsc9jwi zgGda&<*cRw^##kL=LY06{A_>2JNHx8$~)iJ_;~X=H}cxL&d!b_bn%JvuYX#*Vpuin zC*tqYY+}jDZeVj&8h#^1ffaZSWM2CI{j@ zdpo{P0vtX9fM=4?350RQ}49_nX*JaEDs>Zhg58Yy&3{18$c@CrlHF@IXx zXcVlTA;;2KU>S+8)%ciIO-eS?WwZ%--!5k;0o_%T|)VLP* zC+oU;`(^>(+p4jD_f&R1=Lqj&PO29F{0%IUmH1BI$Ht%VUuXmOeU++Sr)Pn&vT9=f z+ZbzlnyWpZU2KV77~DLO6#Ko(7%<vW3n+oYxo=qKc+kCi6&g$?q8H`{Bb{4*FuMy_%?<28JLRv%q^er9uowpn(eML9 zP8m?h+69D##uWOr8|NS&v)BTdaaR9D zkW5l`EbV-Ef^t!D+}_S=YI2e={B6)#0O%O{_cLr;v%q-Hyus#SY4{dUxVIz7@2P+n z{{(FU@lCY{dpmy^FZ8K%3(fAl=fJJd!FgOLTClaNMw`iuAj{z2ALTT2%$_%i4kar@ zWwaNt!HA!fZW!QY6q)%d1x2GDySJXW(?U?uc>XW$7Jto-goUItdHo(-PPeM3?c!9` zC~ww#LHK{sdbGAg+qUWpK931D>gd$chzOC_YCf0YwxqlcS(4gS> zrIHR(dcIw_{R3!gluJWe)sDgzNVDXq5I?I6bE{$t3(B7R&wNn9jVQO7EoKU5btV7J zc-mk&TP`QFLnP46maKE`we`o%JXoL&bM|Kkgl)euRIWlc^2OmTByGijTi;&?Hpjfecku8_ zsEyAAoI+wXaz}rMuH(fxr1A=9P`JZEFO zD#H=7!M)={0cqZ;EjOIB+tk<_KVwkWyw7UL&z9N+yZY&i7t-_SxSa6%$qGR1>iRZ# zt6iBRtoRgYr1}h%*v`4O=N5#gN)PvASU+05+Lnmf_0ErgYV2NH4gwq=TY*X%?sq^A zyfI>E$v_{byZHRna=iy@=Rg!BKt*tW6s(o)d`k*+qxhILyq!QC(VG#$Ba=u=5Mn@Y zx>%+!s)tt~iQ1rxZE7f9-Xn{{p}Ky(7`y1JM5X~(Yl9g2cDcMUjfVb%z_zZSgE}Gp z*MFcTmR56fV5Y+fy4rhd~1x!^U^fHaAMfUzEFErQ25!6l3$~eN4naKi7ZCVf||bv(ry} z^q5}zw|Ex0Ip-dhRx?N4G8fR;w)BK7g*@#kIu8BO$W8Wm21%FeqGhgGO|#k8mIKcG z8%2oR?kbKFR0nR2}DeDDOkXB_J1_ks|}hnRwtAw4|E0>9dKCeabuksWoGi%@a}dVpf*@Hn>g zCi10Ztou8Od&KodXV&?O-wAR**@bA;=+kJL^G+CJp!WUm<{h?^{h~um%S+rMw^9xAar;PuT zHoEa`!kym7UdC}u>Eeba3kWH_;V0HrGvqLAHF>8+mwxCo{qp3s>d&$-7HELkyc$a?E%jFemPINS9!0lG z4G}JXn1!&L76SWJKytKQV1T4OU*m6O<^(g$z=EwnI&)~KXU|IG$bKGT=#Y)xaOJb5 zGdz7O&==ocQc(C#2>UbgK?T3GYx&lFx;i$Uh)3~ty;AiN?w4Nw!YKhMx%$V<2@5ws zS$_Bf?^Ajv19QblxZx-(N&*UA+cza$u1q5bk+>PuuV*rp*cBg8oab(g5BQDy--r~@ zFte$nkv!n1z?eiKE#9GDyHT(xCVy-Z3g~dNZbwG?H(1j(>6H16n_ka0>IwbMtM4bB z@5@HJAL|K`N#9M^1@-Ew7nm7ea7Yy3hyvf5&c%o!>q=6DC^-{`Dj`EhjtY0E?+K&X$@og$i2=O-wM@WSb<&)oYTiv1{&2_HCmPHl6!Ly zuNO|MleaPRrNs@YG4f`zx%!z(x{(n)(PVkgTfaUu#C1yw)i z50Il6Ij6k}n^`*B=*7rZ+SW>YqpR?+R~RRb&mEV2pyo5ACZyO`Iqzn=K9>v#?h$Uf zua$pAaVYpNhRw@~I|mVkNSCT(vYrq3EWt4%IYRWcD(_W@K0Wu8-OjAVb#{X*RKCzM;q1FQQ(j zN3`CH^Y@MM&lS(!)~(a2wD+CY3Tff-OAZ5AN)vZm`PL6d7jPvts!`Th=;m4k2&rKHD>JTqTfDe~n^>8Ws5| zlS}u@phe2jur!DVs+d8+jmbpiXJ4LQ%LA`p^;Pkuq(t*hby*Km)B&sMn}e?ti}6+v zoJCg{YFF>GU5$6M-?~??ET3+ncq4wP?2IZ(iX!=TJ5GF0+%?jur_qx z0#|Nc3YOJdduZavR-6jbEU5qd*i5q#L8DDX88a}n_>B_fGd~OByrK#LPPuQ}2nDD4 zJ~|F<=2sRQT8~ZKV?ws%YVgE$2x29h5|q1E&N1~9Ve#cN^`3a7AH;hdWWSJb^O8xRbWkeXQ>+cxZe7K;L){Fsg*j}3crZ~2-wLQ|Wvl!vj*V7xV%@@}6pZ%S|(OeeW2=-H8J2Ctc?&go`n$({@Snf@JP;Lg0fStqVm z<<1vJ8dVukPU2n#KHGYUR$ahK$|fmlg{pcA(IXn9Qr3Blf^s;^}t60a@+fdx&*YxaFT; zWZptKqqbre&2g10?(4upOZ|MiAAH%2RG1C60q0V>&wPtHYJ}XxZrw|of#5Izjt$A zR$$P8HpTkXp@3C>3-dX-1AcrG_Eu~u+af) zr>>$oXXAi1`nfFKC_Jz7@pi($laearN&L}1<=UGtr>7;Xrbhqt=Y2wHj@w_lJWoa1 zd4-PJHUN0LarLYnf&v;xmlABs>_-&^zE09)$d&)hCR~8CS|+U;iQ`2U_G8U?Gk=qR z#czENk4fOQZMF&X+&ooKAzM2RCYU7m)F7I&dZC@&AID4ocn3VUJ^~ypV(W5b_^k`a zqL*;L!h2@tS(dE%A5~i);*IXMdR*-2tKK~~DG%$EBR<52!WdX$a?xjC0My`=wEMCV zmjpC2?zfxFuOGk<`QJP}NwDfYHW+wdw|f>~V|+h*S6ALE*Gss;o52ETaV^vVC2?YT z6?U~K7QJNY%4waow9@2wRY$V>8<9S?6|BEoEi?Z93My+>Ry(3(V0($kmhg!3TJH|* zm@ZMdBlT;d41v1-kb$%%BvbVyOkFpvdL7#23&2vsa!>J!=8Yp;&b$si|A~CRf;e`{ zPDK>~6qpX=#S^N|h1CqU@K@tG04AGxv>biVzxE>SCJx~{aGOmGZ#-$o&$H2m6+NOA zy$D?0wZ();;x^{@?hCQmd%`>1d()kFD9iC*#D!6$xLm5pbGs0z5jzRu-79MFhF-$dC}`p9-_!iOT;cr}8tdskm5^L$%7W z+{z!fB4zr8-*<-8oDvkxJHUNBJJLikQFi&@NExCL#zYl{JWC`*t+g|x+0rYZn8LcW2gxF^p_T)pkf4wP?&W#EyfN|K2%&+hxH3lOXqoY0 zmFiCr;rznV(zTg`4oR$!@bl+23ZRD!fLn<<(%WoT*_rY}XY`=?<)NygN(^D^(Qf{( z>**B36K+KLMQ0~2$wXtQt_9vW}|T>=l!zH@z1$Bf4B6 z+bZF|KHODWXKub}cr@8SrFNW2#`TbO^%oU-zy5eY0TNuBOtza7>ZE!td^qoWW_{Bd z*Efp#hU?(J?Dt<808nLkmS5_3%l&zfq&G~`nl(=)hRQq#buh%AKzU6iN-%+Pb;*n8 zFsMG;ox787jf_+67s)VMeekPy?AvLA+;JFuG@XiEMR`Br5QzW@a(`LR?;4>3R8|Hi-p3dkhR`^Fl+va{&!@=#`Ejf(NC} zH$b4^#k@~N(^K?iRfq7_O@au8q0Es8#_-5Zx-lRTTwa$mrHqdCSB;up zLYcfxzhsh_d(hw*hdFtg5#QYotH1+~0BJAr-I6hPnHrg=x2)sd=;e01MQ&%0GkAFMS@mIOLXIW34Pn^LVki3wL5L8flC@u#;i=n)XUR`#43pf(# z=^h^8* z%(dEe<_sO%Cu+}>qahyrez1BtSWOyUw&>J*OAONL=i(Y5&|+Yt{4$=-wb^y%(SK)1 zJbR82r<5&b+8gqO5*isE-6@VDD^iNFY^-7s3#y+~5!_S9ise&^75D*1jm`AAES4}3 zQ=2tHqmSnd#llHdb z*2Cl#wEug~rwFmQ7=uwe(WQaAJ ze0?rC^?J3C3ywr>l{)0gi64~KZwC=mG~+(4BCvA#_`Dmu9hOBk&7r2QbeTSG$?F#-1y)-`ynouTCIVc=xvlm=nX0G+!hu^8Zud zXjJ0ws8d>`_ from your fork. Before you send it, summarize your change in the ``[Unreleased]`` section of ``CHANGELOG.md`` and make sure develop is the destination branch. We also appreciate `squash commits `_ before pull requests are merged. -Merlin uses a rough approximation of the Git Flow branching model. The develop branch contains the latest contributions, and master is always tagged and points to the latest stable release. +Merlin uses a rough approximation of the Git Flow branching model. The develop branch contains the latest contributions, and main is always tagged and points to the latest stable release. If you're a contributor, try to test and run on develop. That's where all the magic is happening (and where we hope bugs stop). diff --git a/merlin/examples/workflows/feature_demo/scripts/pgen.py b/merlin/examples/workflows/feature_demo/scripts/pgen.py index 99b9b105d..6ffb3ed37 100644 --- a/merlin/examples/workflows/feature_demo/scripts/pgen.py +++ b/merlin/examples/workflows/feature_demo/scripts/pgen.py @@ -5,7 +5,7 @@ def get_custom_generator(env, **kwargs): p_gen = ParameterGenerator() params = { "X2": {"values": [1 / i for i in range(3, 6)], "label": "X2.%%"}, - "N_NEW": {"values": [2 ** i for i in range(1, 4)], "label": "N_NEW.%%"}, + "N_NEW": {"values": [2**i for i in range(1, 4)], "label": "N_NEW.%%"}, } for key, value in params.items(): diff --git a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py index bb5ba8a4d..99c3c3d6a 100644 --- a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py +++ b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py @@ -21,8 +21,8 @@ # launch n_samples * n_conc merlin workflow jobs submit_path: str = os.path.abspath(os.path.dirname(os.path.abspath(__file__))) -concurrencies: List[int] = [2 ** 0, 2 ** 1, 2 ** 2, 2 ** 3, 2 ** 4, 2 ** 5, 2 ** 6] -samples: List[int] = [10 ** 1, 10 ** 2, 10 ** 3, 10 ** 4, 10 ** 5] +concurrencies: List[int] = [2**0, 2**1, 2**2, 2**3, 2**4, 2**5, 2**6] +samples: List[int] = [10**1, 10**2, 10**3, 10**4, 10**5] nodes: List = [] c: int for c in concurrencies: diff --git a/merlin/examples/workflows/openfoam_wf/scripts/learn.py b/merlin/examples/workflows/openfoam_wf/scripts/learn.py index 1a4ba7c00..b7c40f6d3 100644 --- a/merlin/examples/workflows/openfoam_wf/scripts/learn.py +++ b/merlin/examples/workflows/openfoam_wf/scripts/learn.py @@ -26,7 +26,7 @@ U = outputs["arr_0"] enstrophy = outputs["arr_1"] -energy_byhand = np.sum(np.sum(U ** 2, axis=3), axis=2) / U.shape[2] / 2 +energy_byhand = np.sum(np.sum(U**2, axis=3), axis=2) / U.shape[2] / 2 enstrophy_all = np.sum(enstrophy, axis=2) X = np.load(inputs_dir + "/samples.npy") diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py index 1a4ba7c00..b7c40f6d3 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py +++ b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py @@ -26,7 +26,7 @@ U = outputs["arr_0"] enstrophy = outputs["arr_1"] -energy_byhand = np.sum(np.sum(U ** 2, axis=3), axis=2) / U.shape[2] / 2 +energy_byhand = np.sum(np.sum(U**2, axis=3), axis=2) / U.shape[2] / 2 enstrophy_all = np.sum(enstrophy, axis=2) X = np.load(inputs_dir + "/samples.npy") diff --git a/merlin/examples/workflows/optimization/scripts/test_functions.py b/merlin/examples/workflows/optimization/scripts/test_functions.py index d6f391a0a..ab1a3ae4a 100644 --- a/merlin/examples/workflows/optimization/scripts/test_functions.py +++ b/merlin/examples/workflows/optimization/scripts/test_functions.py @@ -17,14 +17,14 @@ def rosenbrock(X): def rastrigin(X, A=10): first_term = A * len(inputs) - return first_term + sum([(x ** 2 - A * np.cos(2 * math.pi * x)) for x in X]) + return first_term + sum([(x**2 - A * np.cos(2 * math.pi * x)) for x in X]) def ackley(X): firstSum = 0.0 secondSum = 0.0 for x in X: - firstSum += x ** 2.0 + firstSum += x**2.0 secondSum += np.cos(2.0 * np.pi * x) n = float(len(X)) @@ -32,7 +32,7 @@ def ackley(X): def griewank(X): - term_1 = (1.0 / 4000.0) * sum(X ** 2) + term_1 = (1.0 / 4000.0) * sum(X**2) term_2 = 1.0 for i, x in enumerate(X): term_2 *= np.cos(x) / np.sqrt(i + 1) diff --git a/merlin/examples/workflows/remote_feature_demo/scripts/pgen.py b/merlin/examples/workflows/remote_feature_demo/scripts/pgen.py index ad13cc50b..5332d5b93 100644 --- a/merlin/examples/workflows/remote_feature_demo/scripts/pgen.py +++ b/merlin/examples/workflows/remote_feature_demo/scripts/pgen.py @@ -10,7 +10,7 @@ def get_custom_generator(env, **kwargs) -> ParameterGenerator: p_gen: ParameterGenerator = ParameterGenerator() params: Dict[str, Union[List[float], str]] = { "X2": {"values": [1 / i for i in range(3, 6)], "label": "X2.%%"}, - "N_NEW": {"values": [2 ** i for i in range(1, 4)], "label": "N_NEW.%%"}, + "N_NEW": {"values": [2**i for i in range(1, 4)], "label": "N_NEW.%%"}, } key: str value: Union[List[float], str] From ed79780d7ac34e2c941974d4a8b77c6d5fbb8323 Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Fri, 4 Feb 2022 10:19:44 -0800 Subject: [PATCH 020/126] Release Version 1.8.4 (#356) * Create python-publish.yml (#353) * Workflows Community Initiative Metadata (#355) --- .github/workflows/python-publish.yml | 36 ++++++++++++++++++ .wci.yml | 12 ++++++ CHANGELOG.md | 8 +++- LICENSE | 2 +- Makefile | 6 +-- README.md | 4 +- docs/images/merlin_icon.png | Bin 0 -> 75969 bytes docs/source/modules/contribute.rst | 2 +- merlin/__init__.py | 6 +-- merlin/ascii_art.py | 4 +- merlin/celery.py | 4 +- merlin/common/__init__.py | 4 +- merlin/common/abstracts/__init__.py | 4 +- merlin/common/abstracts/enums/__init__.py | 4 +- merlin/common/openfilelist.py | 4 +- merlin/common/opennpylib.py | 4 +- merlin/common/sample_index.py | 4 +- merlin/common/sample_index_factory.py | 4 +- merlin/common/security/__init__.py | 4 +- merlin/common/security/encrypt.py | 4 +- .../security/encrypt_backend_traffic.py | 4 +- merlin/common/tasks.py | 4 +- merlin/common/util_sampling.py | 4 +- merlin/config/__init__.py | 4 +- merlin/config/broker.py | 4 +- merlin/config/configfile.py | 4 +- merlin/config/results_backend.py | 4 +- merlin/data/celery/__init__.py | 4 +- merlin/display.py | 4 +- merlin/examples/__init__.py | 4 +- merlin/examples/examples.py | 4 +- merlin/examples/generator.py | 4 +- .../workflows/feature_demo/scripts/pgen.py | 2 +- .../null_spec/scripts/launch_jobs.py | 4 +- .../workflows/openfoam_wf/scripts/learn.py | 2 +- .../openfoam_wf_no_docker/scripts/learn.py | 2 +- .../optimization/scripts/test_functions.py | 6 +-- .../remote_feature_demo/scripts/pgen.py | 2 +- merlin/exceptions/__init__.py | 4 +- merlin/log_formatter.py | 4 +- merlin/main.py | 4 +- merlin/merlin_templates.py | 4 +- merlin/router.py | 4 +- merlin/spec/__init__.py | 4 +- merlin/spec/all_keys.py | 4 +- merlin/spec/defaults.py | 4 +- merlin/spec/expansion.py | 4 +- merlin/spec/specification.py | 4 +- merlin/study/__init__.py | 4 +- merlin/study/batch.py | 4 +- merlin/study/celeryadapter.py | 4 +- merlin/study/dag.py | 4 +- merlin/study/script_adapter.py | 4 +- merlin/study/step.py | 4 +- merlin/study/study.py | 4 +- merlin/utils.py | 4 +- requirements/dev.txt | 1 + setup.py | 5 ++- tests/integration/run_tests.py | 4 +- 59 files changed, 162 insertions(+), 106 deletions(-) create mode 100644 .github/workflows/python-publish.yml create mode 100644 .wci.yml create mode 100644 docs/images/merlin_icon.png diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 000000000..3bfabfc12 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,36 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.wci.yml b/.wci.yml new file mode 100644 index 000000000..0ced2c93b --- /dev/null +++ b/.wci.yml @@ -0,0 +1,12 @@ +name: Merlin + +icon: https://raw.githubusercontent.com/LLNL/merlin/main/docs/images/merlin_icon.png + +headline: Enabling Machine Learning HPC Workflows + +description: The Merlin workflow framework targets large-scale scientific machine learning (ML) workflows in High Performance Computing (HPC) environments. Merlin is a producer-consumer workflow model that enables multi-machine, cross-batch job, dynamically allocated yet persistent workflows capable of utilizing surge-compute resources. Key features are a flexible and intuitive HPC-centric interface, low per-task overhead, multi-tiered fault recovery, and a hierarchical sampling algorithm that allows for highly scalable task execution and queuing to ensembles of millions of tasks. + +documentation: + general: https://merlin.readthedocs.io/ + installation: https://merlin.readthedocs.io/en/latest/modules/installation/installation.html + tutorial: https://merlin.readthedocs.io/en/latest/tutorial.html diff --git a/CHANGELOG.md b/CHANGELOG.md index 010dbd254..b44333f52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,13 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.8.4] +### Added +- Auto-release of pypi packages +- Workflows Community Initiative metadata file + +### Fixed +- Old references to stale branches ## [1.8.3] ### Added diff --git a/LICENSE b/LICENSE index 494069b61..746aac58b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2019 Lawrence Livermore National Laboratory +Copyright (c) 2022 Lawrence Livermore National Laboratory Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index 2f5086ce9..d1e441a87 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # @@ -191,7 +191,7 @@ reqlist: release: - $(PYTHON) setup.py sdist bdist_wheel + $(PYTHON) -m build . clean-release: diff --git a/README.md b/README.md index dbbd57707..fad1de93f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Issues](https://img.shields.io/github/issues/LLNL/merlin)](https://github.com/LLNL/merlin/issues) [![Pull requests](https://img.shields.io/github/issues-pr/LLNL/merlin)](https://github.com/LLNL/merlin/pulls) -![Merlin](https://raw.githubusercontent.com/LLNL/merlin/master/docs/images/merlin.png) +![Merlin](https://raw.githubusercontent.com/LLNL/merlin/main/docs/images/merlin.png) ## A brief introduction to Merlin Merlin is a tool for running machine learning based workflows. The goal of @@ -133,6 +133,6 @@ the Merlin community, you agree to abide by its rules. ## License -Merlin is distributed under the terms of the [MIT LICENSE](https://github.com/LLNL/merlin/blob/master/LICENSE). +Merlin is distributed under the terms of the [MIT LICENSE](https://github.com/LLNL/merlin/blob/main/LICENSE). LLNL-CODE-797170 diff --git a/docs/images/merlin_icon.png b/docs/images/merlin_icon.png new file mode 100644 index 0000000000000000000000000000000000000000..664ed465570859ce4a609e658757dc243f56a3e4 GIT binary patch literal 75969 zcmY)U1C%H|ur7d(ZQC~X*tTukwrBR(Huu=JZQHiZH~&57-uGVj%1U+hS5>KWR&|m} zxV)?wEEEQAZ7wG0ssJYvCy9epuf)qM&gPx z005q30092M008g5EdNsg02c-TfOCBS0M0Z30CfATb_MR=7w#tN5~eaT093y;1OO-? z5&-Zo1^9ad0Ad0Br|p*lkOaj3f3zYX#s6Y}002TQ06_j1qxoC^ccuK6|78BB2Fe5a zzX(9`JmCLF8~z6^huLrXtsw2iHJpCig!EqlG|Qd;&r1ypWp!tD8EH-?3= zI(HlU|D*u8-8p|r8xv;(0(Tp0TPIF;9-{w);QXckW7886{2z$36%UcRj68vmoudf> z3*8?&1|nW40s;bVM`Ke?MPbqZ75{zWAu@M%w&$d$cXM;2b7Q8nb2Ou8y*}~42;6J?vhITH_JVZqQ8TxE~tS{y)h7Ki>b^;imu3 z`2SC1{>Rh*!~J!Y7mAzy|8*NLlypD21^@s*fP}DsvOD0V54gXMO4D<<$-A`aycap) zVlWy6f+a=1Br1YDhysd2?iTs$;>kmJ_X`aLA4P$<*gq%;$s5vuUs&9>X@WF;`m^7T zrdM{u%~ZC8ArR$ec2>iSu1{){X|B&S5Bc6bXU91do5-BK}B?{o)?P0*$ilH^s9=_tYES=u5BaH6TcCI6GP}BBj`ynbz85 z=esxjq1SHVA~1yD5c}~FglHRo?iaO;cn!x z`Fvai29!^Tg?KT-gh?%dm_BAvjI5RFH?xd=A#0ueYmYnqOu9 zU-=%#^%!tF>_0r(kS0lDS@y$m#ui6=bSm!BMDg7Hduzoz+^^m*FVmc!Mwto?-f#DQ zfoYBp)TPl^O;>4j_*u;deH&oXGZ6d%JmG-k*(|M!AtSHtVj3{)nvJb*?r}!Mt|F# zFxY!AjiIip>*1~j)khJ}&a}40BOf#u2UQt`bfK6WrVviFxA*=xfrKJIu9xu0R!uYR zoRMKlSMkR8_Qh>Wr$(iOirN)ZIa!>S=Qb@7uI&4R2fB;4*|B~z=wU5p7Wi$qkvM8U z#R^y`>|azpNZ~whs0%s@{#Lg5BtHYzrsir-Q(IVHsh^=vJCp!_`IrI z(S#yS-7)lr1$;2FJ(rE(_T|)Rfc+=)cu)+t7r#PJ{XoK5L7@W&EAiYCOKVf9o|V&{ z{59sAzVMW>kv6|=s#UjSB}soekfQ3xuA=SQ3%rYo$lcDUQgvo@azalyI%tEszQ{!2 zv@fiam6nwnZuk8+*LENV7zNU5fFbV{1P{VY1a3ad8>@9GI1`XOe-;MQLzHjGKh#ii zD5;o!sMAtmx0Sf%e%~ru5uuMp;l5G5Vn`z#hCi(N)B61p7Gt_B@$`V)8u7dwF6yHC znu;6e*FHS%?Y)7TB(rUQ6Bx^EQ%^T`raTN+CmUkapL?rn1eAS2NqO3hvhVvWMzChI ztLo$pI4qC`WVt)++Djkp)#RN~Y_IU6J1m5>pg)=kFjN{*;H1w@G%=#2{}4lq+}$dW z3D(67sB$A!5*bTh)4pv6q^CNsa5t>Fn1(Cc0vY*4lNKg!jVb2_Y&GCxJ8Wu*(YO69 zYY>crQ(TI^u76tdSouBW02?EBZ@ zwN+(UKOuwC$OnGiX1}+Kb=T?*E^n_LiT><5v~&W>K?`#iU1_LUT-;v^JkhgMTUBPZ zUFKx!g1D4Orj}Yn#m9IXe4Gb-j$3?S)o3qkBGhO7fdngbR@oV7hmLeePvPWT+hWlU zLS(B2C)=3!!3?`SmZ+78y6eUP4YF1l42pZDm<9Q^v#;0Ca0H5qWv}x6&YM(O&^xJg zp14?vXO|_)8D{hlO$@hmx|jo>;>UM;ysgNG~ttp8OYS>P(J2vTM5@w%kwECLI|R?>8NmwY@^Rf;{97 zQlB;Ey>;r17dV!Rm{o7?kGEbjOWV1RpcZ%k4B&cNqiZkM*}^&j!mJp4pTwNpSnV8i zu4jF2S<2&b*Tu#IR`3Z$WhQ5%*ZbQG8~F5|Y>7vT)!g4%yej{OAVi0l+mR_j3S;6| zd#7PU$CM46H4cq}hw<@cXiCF`5g1AjCDE)wPCz9#dntjzfhs~d-hjT)<*?%s)3(*; zPIKIQREsKL0U3cyP8FS<;SO6+;2o(G!INNJXSO*$i=1Y(J!hifgXnq!BRH)-XzbAr z7n+xVS;bF-h-|Ty`53_z@u45mONl zyj6Bd&M7^3Ks7j-fuCLRt>@p)NAi8xTsO z1S+;SjiM%6NQ!|9M{7rml{U{q{CzmURdWTb(PO*ia1fXJRQ; z*~e@LQ4f(G-q2R#xhP>K7wdu!qe;%qbvp_y-I$XoMH|qs-b5M{@;|m?gn<<(Qe<=b zs^q%*qH@CUF|wpZCI|%SqycXceG-Y;5a$9vq#D?JY~7uk;KlL~AZgX|oomLfQk94% zh|KSkRi|F|X9zEkPleO1wdQlUD_da3-$?Z5`BxPeg8v37P^u#ZX{fPl#)CI^TFxd#Uj!vh!?6r$sPgg;GhNA3j zZ{sCfSx2yORt$(Z7iVQj`C?bmGJR0x4Sld0AIeSzRYy|LKi*rg2#IDcN`}=Zk0?00vsL9`_vNTi`c&$C%+T;4n$y7y+R`zT) zo%U+)m-_`MtxX5`Jkp1Yrpemgs7NK&fG3fs=9sfk8;dytf%Q$wPg&A^5isRQVxxL9 zbrXf!;#hG<-0$cL#6+Edoer)H6sRu``kgr@URK0A)AX5@>dDq zJ?AGQ1H*BaHCn&sowXtjRDCTCzC6g^>z0|l z$7UZy=jj3~%m;=Wo)JKZf-)uRSa+M(4~D%oH#uy1EzlRXePBwN67aF5D60<)Og0JU zca;-nk0;8sUuFhD@FxLUrlum`4obZwLiaFtQYTXY%RV8bnZMZ-0f%wx`Fgzuku0~> zJ`wf)VQi@m3SI-^Rwx}=M{QKl+(0%YIyeNIiZD5i=QJGh;#vX~dnsESJ1}nE-ySI0!?bULr6%fKrjXW&EUABI zGB%kxxX>HASs1cmeww!*oClA)qDp`Nz%*_MRMk7c^ycwOB9g-`hXMeARJ$gUz;C9` z%2_uxkqi^Y|3eBM4HbXgGD9~FC7p+^kwashn@q~qHYtH!nPC_Qk?%QvDV^Mk`UOXc zTYp?;7tQ<7AOdaIa~#@x+v5K0#Vvod(7X)H)VYzNpf!L8lKGP7c3FJt`_%Ohw;^?r z%#7AbqPw$vqpQ1qX{pJTKlmjgtA7$c3apJ?N7gWwxJs_b1Zs7t%qvg?&(1T1>q0o| zthTG#8nj=W zNBn>U2*o36(lK5r3n)RyW|PEDvWZQv;kMbT8~Bf!JdAvjWrD(FiA_GZv2+WlP;)cv zkEH#0GbYpKhn415B`NInl1a2*Rm1*Hy_=)u=s#=rw3&llf~|i;jc=g=IC6 z6@d`F@{WIC%^EL&K?{R@Ux~jofI8J@5X_3q?k_R>l;ww4uijHp+evr022l@OI#Wz_ z9}>K7tFZJ!;>3FtUN_gT?l8}1<-zdDz7^t-^ld3_?j8(hX-maTv%wNg0 z&~;v&(*q>I%R9VSGm~Q9*bL`YnLv=e7Tbo|rIg`9NZS$dGwM4evtR>Wpn_Cq6ua1a z!bEbAHMyTe;mmNy+eLjE;o^PVo;jvjwm{@&K&nhqjV{*8niI-Pe>y z-ripWUwEp9hGtEhFNY1wVr!Qn3Gc?bpEyDxjJ$o=1e%6*&i2i_p6m-(T*ZA}r(TAZ zk=c8ih)F6ozr?Yri{q4MSvK^FiZ<#Ddk^)S*4QU3xGx^8M}0hC3u{SKE$siw8 zg!B{H5Q=tcY#mKgSMIqA@z@(ZB$>vE=h|Hk4__*k_3jev;C&IPq0p^wvfzyv`Z9sUFcUwva2?65-xt;24x?Mv`)Z)cqY$(KXt(c zh^BHOV(1vB;Ow%tlZkoMWX8!Kb)!>3w2yg}-PA&{wOJiv*6 zr+)7?7br|4wim0lO-=Ri?hNez#B$u9>prd;erWyPwo?n-JUatWC%S)L>aEUpn?WRm ze-K`cicurtBTR~8^rv)EkdjlVz0Fhi(zW)6GK&6b-+_it#K+2Ww@LkzLrd2 zzhm;Rjk>~LNSjPMv#_5VAeT{&Q|f`0P}unq?u}mpgyUQFL|pHsYfA&t4x1ZC4h(4}; zx`wlzd3f&)Vz3Mx(Tk(64|ut3(zu)?Zz?ElBX&N9E#0mfq_%_}R3m|}nMxOBOl$j( z_LKIGkQ%{Mc5T`-<4aGfRHhtL;{qbxpwrdZ_I$Qd4DDVJfYY3SX|BZF>PBE>&M$od zzeh%7>~}H{>3Pj)tKB|6;kOHHcnF7B5%^rmyB*;_-P`7(mP0MGbGy|UNFyj zgZP{ByC|!ROHba>Q`<~j`f$0zaJ6+$pRw`-EXNrRA#{;%F>jo2Gd5_XSj)$!+;oxd z=WkUyF?Uff&AOBph*I+$f+2(@7Wz*ScLRxfXu4m$LfZsT-~03m=-OSXJFegUo$trv z5KJD?61`nd=t6?-H?-@49>PiYBFd!mHV}@(ITz;`vFCj!rxdtU=+7W2gp8ZAW?o}Pu)BAtGF?2tDbqAYAVzGeg3-I{5yVdUY zfK=xYcv-~I+yb5cBNb#)y$!~NCB z0wjUZ=%6tHsHIqM&?Pn-m%pfWbbcc`dVmz`4)Is*K1sCcK9=}rE1@x%U(+)8uf zL5O2)`>{)_sWZ0xkn2O7Pch+0PP>@GPJS#suCuj- zSox-o*Rb#!7ZVrXPpZfDx!G$s2U)#`SpsJT>jfl#y17HSzpQ>co69z@(B{x9;U%6r z=u=9;(jv6F%v!31^3T&Y9gO3aJHPSHP#nQj0 zz8N_t5$>mivfUC=jQdmvtE?I+ywa#`Os{5(>sxN zRqigMH}V$2Q4?+S7}grzN*3tQe{VE~r3Kie;`1FHt2ljtQJ%v;tLV4bMeqw4^(B2u zW3j8>yF9WCgjGj-^Al<&^p)BA_$nMNMQY#1$r_gdmLt@b53_BIYogQJdi(7eROEKq zp^CRj*kT_Q(E7dWd-@CADM(o}2O-A>Dwz3RvsIZ4%&3Q`b*vBi-A#P7`iMP;zaRDL zbzkP_sN*jk1WG$82W>_$DqJVHVR`D9D9mTK+|0_T%h0;4gjSTnz?uR$k==x7hEI3a zW(9WyB_cLD<4Xmt?CR%I5N!to;kW7g*Z;AS$jRrA zi69DQX&_za4T7WvMYQGyFx@YU99Jd}u~Yy~tQv-B7<3dl=yXA)g-%^jqvtff;=9&? zivy)dC?ljfT0pxBu*YdG-L+B1*q9KaJNcXY8IG)Y^!3x~zU%OBOTu~^DQK+YFamh% zcHEuW?b46;0!_Fl&un54SpAl=QVX*URf2RH0#rAY4iqLP%RZ|(y4)V!$nJcSGKj5} zt?An`f{DiD0$kgf03AG#i~D8h zq=%#rr*70*2+uVaQvyqeF<#25AAc#3->5(!g?SZHyNyy}C|&i?6j~lt=3(W@!ndW^ z~$lv-&7_0{JpZB^U4}nqrP?60>;HoOud@}e9#Lo3-8BxBhg( zTP&z4vB;v4noj?HR3cBD^I`^#n^295IXdRLMjD5L_R<)+Vwy$Wr`ta!`(}S#GEp~q z^B-D5vx75lDK#*yK27+Jb<0&B$Ltip{MczH@XvEX6U|kbb-Kc)qKemf z2-vL?IK$h2K4yLY#9q3&_0~q5U}H=O4;#bdJw)${2J^Pc1UQCjAr@*7uWclbiS6V< ziFjd0bM4Tz?LMTGE-`bOit-rw4z0);uR?VG2ul3V$Zk=-_(1~%VJ$@1N%6e8%$@8` zSakaWdR6iH=SPur&Kmx@)qAw4wIFF@JVOI5l~f-tYj>{K<9zFmdzraLrqjB<9AIDmDuW5h_eqX4_yHnt$p%sx2q=#`cYY^adS@N?OLBLA3 zalYKio3E{?^k)t3dj*ax?O6(!phH`^vR)E1*mH%!vXKao-bQPEsbw$vuS;}Ath(!s z&1auJFTcf&bsh7FzL*@0q_tVO;e1g!V8+>47xuHaTKAl#YQ=+*|RF?j5v-TPz z_Boz(JG@f7Bq9$}W1akXdB=aWcx^|YYs!~Iy*12MCI}CDYC65Dt4!uVr{ER|X27__ z6ZNn)g;!8^#wXyBK8tel^je#;2>F?Q(DQkAe)LU379~8AsWNwbze?Qthv!Tgo5T;O zNU=r%Sa%!EM5G3MhQk4)zcfAz&z4g~^HKgw%5*(G{;*p0PW5kUoSqtD1ozT-s_MIe zp8Ga;0ll*<4-N0@rWHtx&{A`GvGQ(m&>2`iNZIm&uK5`NXQ?V=om%B?dh~aGgIh

6|5QEpQ&^#d~<7^c=x<^CRjF! zQ&{!RC31s7tbDabi>dpPGD#Lf{IT@(b4diLRKu*bw-`lT30#^owE8O@Wzkv?5!$nX zT2#$1v2ETghO=QJ z=DABK7EOiCxOa5)1y?gW;|)BtmQ0MX>3CgzUD*XDX2bh)%(j`Us~gUFMyuf5Yex~7 z?g=Fu5xta=Ce8`~#2lbsiyC@YoeJK##=gpEBfV67Bh#gFMITU#A6g3Wgu$>UyJnR4 zaZ&OT^M|d;rPV-<`}};x1tORI52!FxEo&X?v3aKC@>0Z>LuBxW@BaDU)`)_}%F0&1 zAu-(Is8Z(+__Ec@DYxq4*8YrvB~&=x{aa2t__qQ9vZh>6oNgVF3)O=c_nJR0EQ>ks zv&lO?UMJaZ-Ox6<((eTE8G1>)c`oc?gnRv{n~HzbkreTU9~!; zhdmlKZrL*3C`O;Y54S=2t+P6}GU*CGGq5eePE1-GutjFJ8o3rgSpXWp+l$U#uIi7h z(RNvo9&APtE^v_#z);eHvzh!lZ|hxRN35!(E-^GCla4ec1&wrV)B*4Mu}k@Gbobg@ z-3rs-467=gJ1|bklulDB9v2~vtS=gAsDznj=(;D`6VE5PT!M#?4{&l(NCrojDkY$c zwoYXwp6hOcKDHJN#I|!%H5i*`;-W$=5SjYdyEQ4C6{S+^pg4ur%Hy?UM`n-GgKox4 zveTy>eLd+ZXdq{lFXL8!&@By4KhvFo;h$Kx?;IPKB$%(uT5q8t{Q18!J6|Iz-(>|f zh0D7I^i3;wjm)sdT_l5`z;~NV6+tOxe*~QIV`V%XkCInQ0)l7d8iHO(k`n)33^ZUo z%Hel-45e^35_>z#QBm#?-K%OJ)4I*G!pRD`ohcuzSf-+&)X|WL674q>-JbsM0>#`} zTUc4B!Y76LV*hPb;pQ9UIgIHSKd>G|4L|VqsF;e%fFqH2CEUvTXXDv^Nh_qlRa3q< z-JNo3nl<+tryr+vJX`)ZQP6&@N64J-sgfMe-r*YZ&u1_*FZWU2g;^unotoO8xE^{; zcWVi=2(Mk?K7nN?U;lK5Ix9bC@nI`5Fx~H>ioOb$LKMz+baHk|@Oq~+2G7Fi;DWeF z-P;0&yhctZk_t&o)I_5;gtx@ACw3oOC@&=lWiS1MwwA#zN?X+z*N@x}(bS!v=8w67 z8xhk%FeZKpbmA9Kh0?1qV5P}?^X4%cC0-+P_V!2SJo4xY7|-;vS12XQHSZnd)S%6Q zbt4ScUBp&aw+9|2t(vBq0{ZV3wTM^-Ca=xrlWKjfOY!8`ordzC#j^&h_iMh!GGh`p z%D6>DPfKG5&OJ?ENA+Oa5p0sk?mPEMQon5C$9K@a4?2@{%1eU)4*=!_@tg@dA;mDc z%yK{d&0WiJ=`%_*|NRShJ?%j?0>Z;I&qe12u0*9E@m5R4&RI{> zGonC3ty2YI#Sl1u6l;dpxvY{ zdenWs`J+k3_3n;9o4tr|vA)sp3b@-?*w^8+$tTp@@CEWsjLw(SuN&D& zd#T$x|9C!c(F=vjxb&DcIR@GM&Zzo=WDa(%SgAaSU7Ek2h~p@cf=GL}M-U-E9;Ep= zeLNKU;hmf%zTd_4SJOw>NX5~?LC@V8_wy?gt@I5iat;d-uDizPL;byz)Dlnfmb!~NBZL9ru}Q4kA6*J%J@*i(oUQ3_nKpbb=?VaTWwAX`t_1d1A%cte_M(ZsUl z{T!+A*VbBtwNzsG@~SN^{WatzaqH9}!eIn729OA)=Z>--DKkw1q%un#JEcuV0Yp-r zE}pC^Riu%06Cn>evki;GR#5T#m`oW1vkDTgJhA0+9_Y4l8&X=oMjo9)Wmk)MuHT4g z{KnUSolkKI9iYaJw9iT%T6W_D`dvc48g)3x&fa0kI(}pF1Kot9#}|YdX~Iln6qJVZ zF_bt&(RGC+#%%z3{SdTPY>u>L%9f+~r#HsbL zltJ-fId{?ecD3UxB%}ikIX-t7vv-LBlFw10F9D9Zwd?t88M>{dz)nmLCouspC2D;g zf7D&4tq>i$HHH_|&gI>>?9i~uXm-ZDKUD+Zi^Lra{PQB#_&S#HIl%rwIM*?0uSo}UutA>dVWMCDjeZ0{-N5vfp3aCk(I{Wi-Q!qoMg8FYjKE>?8+B=GbzCt9A3oM1F5e z?|7UcC_A{RrU=wb`!rr=qT*!sp_|2_BJ3q=J1h+~^HsTb?L^8$mJ1|badmNCbZ-8m;>J>?vo@!$w-8Zi<02!Y z^~Ch;*m7~cp>%zs_j-u0Sf+ejjPrCPKGP}3O7a{lH?P8wC2J=`Aemu+utR21>j=65*b$-ASr}LI?SnIl+8{CE!fx}Gzb9eNfIW528a0XK5E$Ng~=k*=>R;b z9lVEqEBX@K+96^XPfiM*M%~^%c9mPuQ{M|#HjB%ZWWx0~1Dv@I+^)xD-X$grjf}{JV4um!u8uv8(4X{p{dCwHjuwa(hv6T|cTXF^ zKRg~XV+};cX_F(ld(I6Z>*H18h%SUn(P><%1DsQxnp^josGGGnkS3qoQumdYv{f{C z*{FGGxu`jPjZY81TY}`APybzCs`?EN8`$TsDq2u5@a1*UngMREuC9EKvqEgsSsV`w z!p|w~w#efRG@{|B!xYDWxI81-+p@kE!=&o9jEkW{cEx%w@xv$iECkHW83-sy7Bv&P}B7= zfnJ>wW+e0*zXZgskzRe4FWk~*yraN)yOa7})MWKEby@l*t%J7C!%PeRx|Mo4xnm5< zzD=!Zah_t9+B&)QD5U_MCVrr<%=8;=d|=kN1;C&|G6rQr94q=wLw+YjrfPYPEye8q z4hOEKCB?MUYnsz@*(Dbj*UD>4tFmkBN}aW}MfrtoWwkEbtMet=iWHTOXV=eO8Ke-~ z$-X>c&I#smyPmISH8SZm24r_;w4tX3(w3g%Y7f(Rf}$~sAsCc0Ng|97aMTWMrdEHA zJNINDO~gGl;a;&m;l@&L17L$@X$3kk5Kclp=M1{4@MJ3TM!mEllI?#TKhWR@;kT17 zid*;lL~e7@^oA4F8_}5Z*Td-5*A&`ZKg-}X?_+M7nwqb#+?4W0hAgz^?djrni?zP%^DnPwlQ(piLAkOICx|eGFj|*p>EqA zCrV=YUYBfAk&z$%F;%+sUzdjxuRMlLUhq5Z#0@Ok6XQiu!D$d(+{liunIJF&=@7|z zTQ~G401;(jvA_wV1)v~_VH|P@`qWGwn?d?D3+Tx)s1j5J-9=8e=cnD*IWzc&@4wm8 zv+un%(Cdem?#p(>Gzw<{nZyd)-we72M-6H~xT^g&(@DHs*JJN9T9&muNADQnf?13d zTB=S{=gnjNHHe>NC<-W8)iASC9A4Ot=APq~w)du^(SjNr=D)shH1S;lx+8)T2HDSQ zI1t>Pxk|uN=B=gGKovQ&^1?ELr7c|$bjxAjQ>*rM_c4{0f(FPBZg%@2m4$Xo6Bjcl z;M6batfMx#eVS5??ei35Ta)8GgVvl|QLFm3^+U>%toz>4SikRuF+^}xI85Hoa6DkJR)R#_0GM`vFkWJ9`Xusz=8(j8ZqkcN?eKnLE=yt{P;ohA4Q%H7C7BY z-+fAjj=v}wg0zaJamWU}nex39OT`hpU6T-}W3&Vc?cp5e+J0^;CveWF?sIx+)xP^W zFLr)0I~nEoWt8d8V?equ4%^2IqR;>J(<;q$d=WN7UakFLtJaF@rD|if|A$Bx$fS2Q zOO}l!Vod#{wRa#S-wK7e&8IepyY;_=2d+;S&-xBpqou@aYi;sWZ3ywQ=@g>RHR(Q5 z`?00A_KGGC6?D2b&J1kJjM}+QTNxRdiSnZgofEWNw2`r&NY(?tLPws&9wK7&u^TD-?;~H2 zh68WnqRUNeOF900#Clxs+<3^W=a&$qky$K!j`j-l@9Q^ zdUy#ewZ6|wme9pBmIXm~X6gUzPo7ZnjRH2f>wXN3n$t{T_gp+AK>1Kad)`>!__}Hx2yQxTrY9 z-Ra&S<(DTY{M(h7SxQD1KomhYv;0uty@YW2C7zQmtRU#5}hX!4Y4OJsov+m?1l_XYmRMI%VB ziK_|xt`9!!R9elYwe>+*8M5?s4##~A6v56VuL&;etKEX#fGiGmR{|c9ZJNk18q;9?0H$rmuxIrA=NwpK-y`EVqBXJNi^27m6uCY9tRmKVi7 z-*w%AEquPV9nXrZn@ygTuN}Va=Y0`ORHAe`7!YUjLaAxgF=7@0QAP&fXrd5ZZyFTjU%TnLeD_f=Tru-k@BQ+bUpeFN+q(qX@Q(ZLbF% zdnd5Jp`d{M^%JUQczB{#wU?fDw{a;Z6({4cx!FOqg|MpdPATcH-~yht_OqpaHME(^ z3B70$VBWF*&aSk49KMXa!j2~)A5g?{9k5mQYKB3d)Nb8~cj{S)`ItsQ2p}9F>3#78 z07nT307iteYMe!84E(ioCg{IeYlr{{I8q*a-@tQ`DAlf?TS9G+CIij zy86U_mU=qRTW3HFu}T8twepCoEcd9G`*xG$GG~D|gzOES*oU1H9{tCZYHX-*b>^Z+|x&44Dd((vl1St+|21-CM zXCR=bIPJljwv9D-rZsft1Ct;|Cp#SlGb-eex@Nqi-S>%Qv6IG2->k`V_tyvjz?3$H zc5H1qskQI$L~5Nah^FJb3<(WGeAd0SZeX(4)L%da86kl}wY$ zKS5YEhsiss(aaffkqm3czFd_>4R0WLDmRSY-_PU~D^_kl8%Aw8L5?=2i~JtXPv+`y zc`Nz3)Uj>PTu>i3+Be&P!3(kD?t&$v*=l zNkw2D;V%zfGe#oA{aK{aZJ#U@E?Evaam`PH1kKFn=y1S&Y;LoQA+ z3JQIPfd1rTIg2=QlOkRGY)6BaJPGr9etRhm$=+>;z0SE+V(ZJ|9@Wdy-FvlSb`1!H zFPj%l4MTSC2y=uK>7q<~3K1Pw*5LH$43aGN2O7_wuaOD>6+#hrk|!5pS0bILbM(m` z!SAXx`~tMt<+Zo3ZKWqKApCsDmf;;`4c;9GO#-wnJh&=LTt?)rF#vD0^QsnbA0BfG z&{$ux&G#$fLoWSW-)1N$NGl?=24B8V15;Dn6~}j9lzC<4E~yE-{dn%U*yKU)`z@9J z?dATyUX7ZX8k<4m;ubl`EtN*&zL(zjE^=aq`(g59edBk4)99<$j!t88_Y?GcVhC4G z5*P-M&}hkzORxu^PDjZ6IzN5du+ z9?YZxYRuY_IaI?$(O>^Q2-ewpiP=#(Tn(Rb%6{uO+1%09f{Qyx*FC=zrjjzrFkVl- z)C{EusWGqgx*FmOSUd(DNe8R4OGwGlJ0rtaRErTm2N1&8 zz!a&e%3Fp#dQp^@>unAH@{gDdL(|ds`n=I9*CDn?3HNlIHWUV{8buB}0avElZwq4DFiTHuAOOZ`90u((kOzPtkX_WHXsy6#qbVun7@xfMqX7NDbkqJVnuL-EFo9Sa-xW%|t&0uX07a9-MByxY-6p-oU#H3_25s6~0AmFz? zgG)2_t zu9x2TRPn9_zgcx`?&GrW#F8iaN*s6OZD8HN=-bV9;!r@%@j2_pwQF*few9DdH#m*Y z>BH-5a$ovWQgXb2aA4SA_+UiP(lp4!1WCBw82+sZ^{C09Sqv?^NXLK1;exbyxWLa; zrH*Z~(8~mr;PEr1$zyFO@KRMrO3E%(6Y?)<*-{(7unB=;oIgQc-L8F<_+2yFF7(~T z`UQe9RNebo`1ywe8_(@x^i&P>ws?B_-2-Iv1cCfjy=4e{OX z`VK%$OIXy8raI3dUnadOjJ~>c1P`vBy7sQf!Gv>U9p)443m26SA9OwSwV-AHBp*=v zHBFGov{ly`OKV8o$xjz-PBDlf%4WL?Qix=eWKn)ym^D3BWp}l|hkx`ayewy}2n_5l zXvx(*K527|Hm?opM!xr$PX^m9rN+p(ka}>4WroiaTmg%DwD)Ask37d!&3K zJ)mKNWa3^E$0&U2U@s!TWI(99)1C!on{tJHbq^%{P3yy-?7tBL=K-{cO#vZDWzZ0u z$zuTD>(j-hI4_nw0;B=NTX|!Nt5f&7z1nu!mvi~tApTAAwy=rJyo)5?w3?`OfDda1 zZw7rA{3{-PhR9h1K1wGpQ1QA7Ct-9iFW%j$3vW%`HwGqyV>G)*E(`RxY zC2sEeXf~-0EKYeom=b~ijj|2FVEx3Gkyx?gV= zXqH+q!iNjCLYI360(Df;-g@UyhUd^-5QJx z*B2pD01~^KhPHDo^*OG855U`f`%LAM4huavh3-JZFbNIgnO=lQid1r$<S04;DUUuMw~`Cu8H~ zOd1N42nqacxBA?p>`5;R&vtwRtbIpRVe0L4CMc;SWN<+NW>C3=(!0UZZ@&RzFS*=x6Uny!ZSMNI#{&2mv` zaJ!z}GH8h<3$8+6`9KIcUPUB3n}3+=zRlEY>2%c5HSrdK zF2l(F2@WEOg-wdx`FbC&zx!7ulKuf;X!GaIMw$^5WEe-#U939A?eMs(=P^L5=lW)p z5n@e}vNNh7+79fW{ehNQ8pY>m)?%)=KGm8gi@XoZ@dIg(h!J02z0+Z zsrF+!V9`V4pb|c1s2W;_pYNG`%Y}URGqNpD2D512Am3#qr zA{AJzq|Y|p9N80wZL^UQj-L~Bn`3>^er3wc%0)mGY8E>K_hHRb=>pxSv|0P3{}QU# z-yn2ss{sn+89A;1Qtk2zmTqFVm75xf>roB+^)xO=ly?Q0Id+&d?k^`=k)T=*MZBkw z8`HAP!tq%^!U_;hvRbA;qanQe)0e7XP|##kucyZ)-+!cZy{r$3%Oy8e%w$u_L7-EF zWB)%yy#sq@TN5oB+vXdao$lDSZ6_VGW81cE+qP}nHc$3H0qintk3Mfk@FAGR)3i`d;dm+AO8|MEidEkXo7KlVb!gmkzcYZEUj z#~W^kgkt6fW7%h{ty)*}Be%nJoDK@pBw@3ob(M3rUqPn0ile0cx-Ii!2jJ#BGW9Th zaGYCqkuX^5o(HLZcNQ9+c6{KoKP?s&+Ttk~gAOMYhDp$u)2+8IfP{ddj*t??gMm1x z9bi}vgz_69P#cOMHrIkwbF_)0*j=rHTWossel)3kyjFOMv_xND+FZwP!!GBb-77Ya zB)=k4qF`vBO5rITefgr#pLlf$Mb3LVB#+u0Y9JAg(i2l^z)XCcMH7D9?^0YrhM{Qa z#o}^1h)>YvXMy*~w0+c@iql(>f&`R2;f z0YlK!39@Dc9x5xm*zuhB&h^BG-KdNY5d`N0Kr6*ZWQKO1dR8~dF#UL@Bc83<=Jk#s zgC*5Twxz6U$nUUz-(ff1mCpIleh_}V;Nc*#zwkTwi)90)`HBj7QeF=`>();wUHarQ z-CTyUtY`$Bg&>-#jI8M+^uBSlMH}^si2l&qJ+D6_$9qAB%os(y(;9nIee2*a7z;B^ zJ)~yIr*l0AWNH_=2#n$MN3=*qscQ*~UR>47ne<|swoGFlTat(#E4|{!QB`Sk`A@t@ z6jp)$L3_2nw48n2Tr9EBhLSh9t>phRXQ?COp4Gl-I-&FSQ@)ENnH#*%POPohUowvD zP`8{Wos_q#X(xWPz|bJ#f&1DU<PaJCrMT+ zmB$PA$~k*Q^9By(4zYCv4qi_P8GNjiqkU7y#rkY_9Kp_?+Rquq??85%!7=+|AGRu< zDROZgf*-|}GL$d~z(1-k`G-)Ynn_F=H!)N#sk5>oR(U2SC2T7QT93!71hUSrFKC1n z)@ENKvs8aLn;G<4l?ze#^f4MfpicZO9*$ntN59~Gxs%rUpp2mV`5fNn_#;y5D=Onr z`?@t!)a~GFIAcAjN}{@7u(}NZg*ABV@$D{3DyD*;MXNCgq+br@VB|*rU8+eEet=xC zwxzXt(&hP#tCMCTDvfhM&u!VE(!nDUt0#<&LQdI7OFq$RX81n75As=;<@|A}c-=a^ z@e+4oXS~ADnR7p>B1sHID4<-%cVP4;TwNCmF95w)!obD5dxvP#FI8}0_pX*wcB3F* z+cg1rsqsUF?1@rZ)DeZFFS+<0R`x{fGXzyfoE#g-H}lePVG!H7-w<4S;k19={AfHn z3{UnOVd=z<=^JnPaXKgcE2DivZ&EE`Vg0&_Jxw*{#QNb=G3>3q_iQX^P(GA`{)6F> zj?527xl#0Fe69gfn3&r~oZg3XitT<2@iK6$VuIa58_s=fN`>F0R1JMErLvjXHG&jy zJ@MO+tvoI>Oj6w*5BScjm(^lBgr)^qEFsW#>6M=ek_Bf9W(JTlmhli&)xtQHEK~%S zOaKO8-9WK|gE|>0Ae8}E=vP`%L=Uyz&@y%`4@<&L)42lNvvDWt(bpB92|U>=3Md|> zP-jN&jSP+Zt8mGywl5eD{ADi^(_SiX(J1Yv+fh7H=DU!1o-X>$;)2SWYBqzQ;g>JW z+P&ek0Y^S;@~jD1@T1n)S+dg;WS)k|245E3J6M~n%d?v=QppdMoyX%n3#FHoQnt?1 zxR5?V{RC)dIGuib)~>#B0<_@MvEVKH?G}fqn{c>a>LkHc_8w@!sE6u|d4H9UZ%h8} zC};+#gmkbpjp7Q`E^r!t0fBHGem@6hlfaekh2xA_3~A{pizkhWgegi)39*yH?&+z|cTm2eal z8N-6rgFz`&{?<|Ls;^3`8xqKbr^UjcD$f>QRe?&dmRa4rY)|hG^FBHe=tl)7irso4 zN#vu`nZM|Ac`p|*EcO)LAMX84A2y7^oW*nm%iPF z>tMt}UWz=N{mQDLY^ev7>KM0_9#*+I zcAeyQj69{FyEVkQK7lQf!vq2~QN;B^1dOT*M>l$NY)pAVs$kp(Qg6bKmeo zP&n}*o{2?s?W|<$N1MqRj;dAv#yTh&NnIFhR%udM>{3v?fAM_fw7*re3XO{tHD9qa z#a~XPzS{Q7hIQ3_xch~5^9Pp6NH6c+?$7=>3H~47er&3K`s9dUGz}G3dliU9vw*(o zo&mm{jkP1EXLYa?Mrk~yT`#vYfC1)iG9J^^XLtSckZBhQkv+wgz(#EQ52ouRHOIn{ zqAC<(y2A@~Akr#45L*R`sF}PIEjy<*X5WTfDzX!k@c>64TI?bAxzSKmQ;|<5?*T?O zxFdGcH4Tc_V{z=}*oVns!luir(r~!SB>zyza`AMKu)18%Po54LhbQs4yhqpEP8Q#{ zbPxEKlLAzZ%|pAnNB-Y!sYthIzIEbk{j8~5J(13>6J+3P2mX27)|PyM93(+L)Azi!hfeP zqoMpIwaSPLsV8}-6v9XD0s-zj{%sZeKqI;*V|MM^YN{;2(eQ9eBp21uaMd!zpOJW1 zBXpV0%X9D6ABEl)WS}U|i5FFP4t;md2Q$#=F>Dx4BVG3tZ zdU>QgKpP+71s8?l(+CGY*Ij8cU}dg%ltcSVAe!T|Z=3^`n*66d4B$1R%9jpLP}vz-lpE8nXjx9iMBwtQ4GGhWh* zH+i=;uzL+(!BTY^1t)_AUS%a+Ukdd2(1z-nm&J&EW^l-Abr2=`E^p=U#r@jnU0~vS z#l@h;inrYLBz2Z*=Uqck;j-+`j4pm_M-4;VjE`U3_89ek^Gselsw{S{)tNjgey=jx z(3=vf3h!>kUH%FzFZ?hBIVp&gVj*XtKkf4std3zt)ORL0j^yLc%?c+)q zAVo)lIS?H}kq1;VQNns0sHTol2<0s56?BwPuYiRJ32@{6jA5qoZjnVX@w^UWZq8pX zLFER(^t?o~gl#TdU$WkYDfA~IGyqZa1jWXw}~gZjk}gv4q=WJh??CI!p7D4LKqc)NHu#{nz(4Ss7eZ@n#S+dQJ}t zGsrDLYte$fkq25HNy^3;j}e9(snw;hjOWGR>-lll0k zdRo;<#X?<=TGn>$a-|eN@yEehqrq^xLhfBqV508OR8k~b(s{WTJ@F9LpArz#*+x8X zYcdzK>UF{SGBjY#`l_CAC+w>U#O5S@h$TXqr*|-Ssb!~}4D`ZO&4?jOT09Hx)|-4 zZ=Dv?Z1Ks2-y}zD(sxIG{Kx|o=xczs!&t~onG9GG8tG;>uY(UKHm@Y~Jn=`zA-2vC z_$egC)sYi+9^#`y7U_I+*jj&vNjl1Srw&9u^5W*xUJepBBQ^B1T6^@s6lC+lq# zo_}0risj-!#*J-sXf#YfG;B_qEbc5Q3kyny1O$9{UYDf+k+(3>l#pm*u6~s)0^;^m{+7i&%WPne+$u=Hv18@y$5%549=X-j@zuIrdGP1riuUs$Eb$mc3O75orO4|AW9fQ020J&i38si32BZr%9$^JY?<8kJ=~@H~}jCnvYIR z5GyWXO^A>hJ@cKCgpn;Q$Gj!H-Vu5wB)*a~rKsgdZ_8ODF(3{G2aJY4f*<}uIm5(_ zpeJ^6@uIrC8m5u`d4h5WE2ov{k9V%NRT(d(Hw>FR>vD0j*sIIhb{|zJa8ihojI(e= z=DZ_=c)FN4R>c|!9j3&cW5muQ)tAVFgoLdS|9t;yhYP)+6jHc?5-r=DM~t-GwO;#9 zLd!7G&z(cbkjS*(l;fZBv@a!HpCwCiY_yOp|N49R8{Wc%_j7ZRf#>$m2_0h~f2ePm zZF|BvB$?xo2`Mh_*Q{6~JpTLjqw)Scy>Bq4Z|?L;m=6c3S_sKtR58C4zN~gk0vawk zzTq59WX--1lYGv{5I?$5Dc2!$+n3Ax;Wh*JiZt%>_6S*aeZ$h^%OR?@t7ka`FjNa@ z^;+j+p)j`NH;bWZGjEruXk(eqp(5saghbpum|%rx0BkgquoG^U?=WeJ(IG|lnvB;7 z1n(h~E@8MW`NvK2E!F+0yzt>P_d`!NPR7b1*GM*w(?Y5H{^84ZAM5d$aed+#qXOue zj(MfV_)jkB-MU+;x(%`bf~qRMiwr^jg-7O%9W3*L8Se~0xTvQyF<+hN@|>FE*kK^yqIYmbl~YP~)IY!97mpbe>Bm~-p#j~& zYK^aXT~ePjULvi5^WC;G}l(+57ID5A36- zyFnU98On>|V)wfi&n#*F-;-( z2K?qo7Gr@QZ0lx%^b>`zvR_sXuUi#6Qyb~dHwIZd(bnOP z>_Epir)B14C4beFWKt`?c;+B$>On@vdgVk8K~|co;kRAK0)qddUc8BP0tmn|g2@1M}U0gRE0&UcAMJA*j75 z!glPr*6UW~_*K{Lx2!dDa?$1bSY?n}YAsO*cI;q8eL#Z%eSCl+9?gCdM)gl=Z$U$dbh%Z+=YVTV8Dg;y|OF&AHIKA0Yq zT17~=k+3zOu-!PQ^_@dDv?;AzY5c3L@BJZibs6PrkzHAydB0)IJk9%t_P4Lc2zR^5 zVDIavLU731HH)oDX=mwTD>a|Ud3kFiO-6#E?eUuT{L3u|937t2Q9?K_)>vCx3?4~R zfmhpyR7MS9~kRp72QlRQ}ps*`$j69|JiRfv^LEp+mkkECN z`i#G=H`=WD+)G}YEsUpC_<^RY?yBEw{u5&7=Zjj9dojJ05vGP1fNW@5 z)Cwy`Oe756fl=@r@aD*ttmlxZCXKR1&<%(=I|F_iY&=hnbvyUPS?+p#Rbu}mFQyR7!oh49y1wHLnY(>CfaQxY?ll#IyP-&8RFIp8eT7cxGmI=>pm7NI<4cDWT32gpo|%C2pSu!r-!%m zHztC8qWX5ie$W%9b!KGq?{MdEd=!_CN)*}(+Q`|_5{!A1!+p+d`hi3I?Q5gZ?do@Y z07FkC6Z7CWaD2fC*TTO+A;tsO0I{jRyz&?eh8*a{pQ2o^9WVqHB|bMsrfn4qfB`hz zyZ_7+><#s_6pf39M|;BcaF;&p8ibQN9V$d*_*uS^?6^>Q1QS~C=PtfGt)PaWD)rhE zfUw#}g+nkRofxs*Nv*EsVW)j6Lw01eAWB?*eg?z{1dSO?;?;Qoim`)&Nj$9NQ-R_` z*i9CXy^I>Ad)*1Soj^1FR2le)B|`^P_gtLWhc8< zI^5qv*rGhq|5-IkX}q{#vu)OGMh8$>?6y{sRNV|KPyRS;yKZ;dPzeP^{mHQPH&uZ+^q+~Ysxf69KKM*hyl$FVHx6H_GO&P6*38=|!wrg6Q za}d5rHbDGSoi-J(+k&^$kjw0o37Xs2C$sYN0JZY2RKXpo`oi!?=_RniGb`dRi?p?o z8|D=&!c1e2OL=`vig`5ZN`HIBfk39R?VtE;dc}i4w3tpsGv7+*QAY7&qP!oZ3 zvj1xDBD2`%V99<-=`cDB3?iq_QgKK2c&`4=tbMk1rqOYa`;>0A*@Nx)etd4G+~Q=u ztRp1STa--Y`Q<3z3K2hI{#$sGn9fnxem>k<8ugproVqXC-`n}?s|qMP#Q<`KOQObG z0-Xl+{t!%Kw3F~jUWHejV1^oz)qYaCXf1r4qob|!r@40A?jX1-0$vO(3SgyHR`L+c zYcH=q{d6^@{XX7|cI$^8o60DT3W^R5pGy`(6)Dey_ta;IQG*=KsmMXyvL1O3iJ=%%&#tIk?iU-+y)vA7z z;is&l52ob;RE#hdH{eJ4%V%$4kUP>iE8rku(8E<5_MyWPE$*5g5EhDQQm60yG7i8B&^ zRkau=Pb$ry{~aS&i~h4D4PuEin_@=lFMM7yPaYo@7j!53-GD1FYe!58BFyd}Wc0t~ zta{ZU*2A_M{m;~N9j9fU&o0$Gzki3jhng5d)EI|AGM5zDHnFF3C400k&~$7m!=vj%a8MT6BLn^^r+y9K#VKUE zKRD0{PI3CqFFJv@v^je_7iJ1r&9{Dn?I0N*K#%7hAJ`cnfs-0=aJk_7mCfS=ko(@5 ze}lwx|7!E|Dt&S+LGV=h*ykk=R z1Oduu^0WIYxr9EA%xZQW#5*lwpo$k`n-Hb&pYJ zE*u>X=Pim-YV=hXwb_JB<&dm^y;8|_z+gZBey@4HmGVeHe;LRSBS|pF^r=wHdWE<{UmcUx{k-G6+zZEJ2IH&x^I{;NNmt`Ci@`yzF9B{wuQ72P~LqE7^9K? z*%J%IH4_+vhI&OeMi9fxVx0@0mua@+nrHphx|!ngIbrR1yTQjh1Uf`@@9~qa+Jbsd za&gM5Oj)UIQZQhLlX{J`aU2PbVdLSNBSGMOt%!0ozHM_v7lG~vEBRnHofVPqIopHC zi^NmP3XNU#`3hBK?+J)91r{!VB9Y7sHb;Bguyb<;<( zid)R!%T=}{Pr4Jz=?f6GQW4&=A7wHt%-m5{r_Qd+axqPOUnH>!C+Ey$waVc1ir&qO zUK`2($dZ-1z3Y@nBma|w*p<}~l zZ%+gc_6?2A@}*SOBj)*uh0p}Hb>eh1nkN#)KXODxO&;XyIbsRZPBbj*1ywbW5Ydc; zwKJTY3E)LgL(}4%k=sr@c&*JVZ<+OZ&;622>6n8^EBvd1@@yx>b=Jcl<<0^XORgSp zuU{s(Kz;aw$#2&RS`fm~2V8D&5#uJVZ)*5nW$Ff!O6xM&%FQQq!Ij3Pjk zd&mh;c#MpCkTCBJ)XScWC95=`@4&~M^)mf!2#yjFfQCVVwpW7&D2(dGXM>aXjmq-$ z5En;|NxY=j7qv(t;#YHhX$3no;yW)YUX40`PK!r~swq{mDqAWts2)dymujnE-gC#V z3F&8v;!?51^B~Wf0hyhj49xFOw?6F*_kz;g?C&@fnXiw4*`bI9sKP0m?D<5jMW;QM z>IEdk#ORjo09WQj5VY&G9T$MTI#hGl+4Y~A0IbAoPeMHn?;k>nMy)w`-m3~!&iTPljc^EI z1U{MpA~hhgP;8?-(pjQ1XcfE&!Z>LExo|}k%Zm|psr|EqBiJX0;|o@pMz+cU_pazZ zi+`pJRp^Amv%dxbFv$Gm5n}$(>Nz|GM5#;gIIHKvjHoHIsaX|I-yZ}vt;bvC~f z6zAwq42ZCDKY@!3Pd_Nxj*mN=X9ch$)Fh}Xo%l&#O_NHni9gT+PpLuNrlYsj5dK3Ozs(w9Y}57D4y5F}N_{Z*)uK%d3N@$Y(SKi68|iJzoQR3uvh8J$ z!^Rj%wJ~jSSMoV##k9bJl=?|Lq>{1#{p+I`qyuBJO&ew(!}F09C6zy+T8Mi_(e_S| zRB+JVoH!wjo$8%L!&}a+eefauHd7&j&%Rhs|!a6*tcVXwos#5s2Gbz2l zV`Cp0!T!Z}v5KqFW4+YP6$OWfbTW7dQGbVJUG+7@gD*T!StzVMTBgs=+n%{iM3R4X zw5$4>&u?SkyZ!y$nQ|?EEMVjt>=S)75?WtE5h!+X(z`|Vyeum#t{G7~ID@9(c2KCU zmSQa?+c^cPh(thl1bQ3W2U*3q_ixjOu{n^*4HWoRp~lPPnbg$(k)&ckUe$KZ{0Qwp zw7-;N?4U_0nm;Aes)<{qZ9mbFttQP=MqObmC?FwU) zAe}9#p8H8nX1MTk(vTy2)?mYI$rhPA)#v?ds$22K{ck&eAE?YY<2YhdAWvdnAqtEMg(?zv6B)c5bPBlDo<0#`hXo1^QPjAp07r z6d%)flr`M~hW;Rr1O8_;^_T_IpQlk3qkSvBoAb4t)#=7fcy(jwlY=A5MpJ=^u1(Vr zx^U9%HNza++QFVf;pIa59E+ECA~|hrbT&pjYW6ZFSLJP8npaK0UaI(<#jY_+VBjoA zZJwF>Y{Fu~dU;hjv~(I;q+>8=+c;=R=-mILAnikBmE(-a%tcwpA zBy!prTL!i~gwn_g9OmX%zjsnnt@3X_v}GJ2GNy0w&{*!nt&l+#^mDLd^mf%F!THu! z;0Dv>U+Tu0-<->_g3bZ1WS2h9W0QU8I}#vRTj5B_7y_cAnk?5Gf#jMoIh5K~NIUYX zhU_oW(>H+`H9eg9F&ib(cmX>?UJI?xMa4S%=V)|xT;9$*2@932F5kvSU@GCr!|-Q! zk+?;7FOma+rA%hn^(4h(LFSe7$R+eLnF!;MB3O30`9;8*7aa&$f^}ONN4+KYHtpcB z^84!z=N(ss0<$cp>V10hxl6=r_p13B9qnzpr|kjIVtg&=De5 zHhbnz{j%-Mx3ToGK@XOayq_a)B{*9Xkg;asxF0MgzKvS@gv<-KydQ@IXi6?kQBeuJ zA{v`25q$sb%CMpkwaCaBpA|g?v-w54SDVaHtCD*sp1j$2i1{p+h6N0PPx(jn3VH0u z)Rm^J>!DY1K7XEv^qgErWOD|6$Fb|?z<4p7*edQ|wkKn5Y7TG1E&=oD3NYExRXqP) z(lBmNF#c;WdD-2*wh}8Lqg`ky2&sfY?nG5C-H;GhE^J^rfjj%>-5Dg@#kwa8UaAv- zcVH+a<0#>b#Y-um(6Z-@ezGa-2uF(%DxcZRH5mm-L%Q(4qc7}z@j>VIxM(i&!j;wp zeKxk@DmG5(w1-D7GK3H*D+W^26Pc;QCol;?J7df#A)$0&Zld6!yBnLfj@V9nsTYf# zYP(Nun5J?m&WNJl2>S~!o~H&EF^0j5!^~NP3HT*YFZ;(Z>VVI55d+jF-{&a?>*C0F z-NgSU0oUe5AcVEFN98V##8M&o)IE74{mT0aKGE==_gzZDvyH_y*+5^oeE$&m=1OB1 z)2I$uc|SPH2(CHom_4>ME$KJN;-DEj=shr!!=5<{NwozI*9ESNyAAi1Wpn6NE>^CD zH#TY7Wg%7dN;m{WoVR_9U@|bL-QVbk=4s0UJyO*IMVn96w-qyIG+lSGkZ8YXHA)de z41^Xgv66fy&{YpP=6!j&0E_)LY@;#}y{~p6%vR7LZCal|={H=Nvn7{PaBR!7!%t#eo0L zo?yZZ2{qYO`&wMHA}qhC(xe%#+>}y{O^r}fX;}9ktcTxT2q6ZtlcFfe821R)!3(dk z(6l2vMiIaTNt6BaDTrc@;`;&dH!?gaWa-Nxh?Rs9P6eSEvXIPsfEv^EVZ3CCwDuV7 z4H01v&F=5d0yZ~9@C7d9X_XKddW&h7KWW;Ay9y|yW6Y;t$94`k!im*MYfhE?r!l4F z;>>RfzrG&gHyWnnJZdxktu-1IzBO|$9c9AF8xLTMp7Qd+SW$?hd?YfG9I|2;5O<&EpQNp6?F zBYcXmB1G{8*$ykh*cI8AXh@Z$6dJUjx5w6?bCTw5Vqfu%^umN%MnT3ThzLnHG5KV$ zguwjNYcC(F^Xnl0yi0?eql+G#FNyqqZ?8@kc^IOL;6Dtk5vF?+@fbOg6n$s;8(Iw9 z-+*0W&57@OwzqIv(^1`*(()#;Y`AA4Xn}L)l(ET?*K3gfLR0yd|KT?!8bHHm!-0lbN}@8d=|s4?+k4gj)B^LXz9)y@R@ZER%+#YdW*F%pQ7rOq zoMWFSZN;U&>Ya)WU2D`zV_HiTA+9{3r~eB!vOP@PxoJUak$~fJ(ssLqK_^-yOQ@XhSP~m<&=!@uB54w_X)QzZ98=YD+^aF?!W7T zVb_1%?JB+IFnCZkVMQ$0)|iLVpD>2qf$Cu+)?=h3Jn>SA)H9fp1A+N|9#90I`g#cp zy>_H2uGV$YeY-M7WZ!UH1vIq)jRIbY8UiIMS6e=86;@Gk6Q#vc!OP#lWSWF$+S6VN zC9xdqHd;$_cw9ACs~ktqH*zlvLR&)TZ$`M=lR-a5A|@qM?c zuwdX=?&s04+mP7C42MAaK!et_q!x}=JeLXLzFcPBp5nyw|1lkHuxYxL*}KgGW>hwM{Tn zZXS>NiSimwQd_d(QW{suZvexk*S(CSUKdou(>bi{IXEDhe_!D#^{Wp58w0co_k2Ig=LC?hXtNrO14e*K0L8 zw?4?!S&-`A!og1S=bVZtAze#>teM3CpV7?+$qAd`RIlodbqW#U?maAm-G4F6%6bFOqg zTcHOW4?UXefJqqxi|NX;kT`t!Es$;+>2qP_=(t>LBg z8wCTm85Pzarw?DyzZYDKS&0^{kH?O-o5x-MZQ^uZ7&ZMDXj}31MN^gsWz5#tS76?| zLhoo(T3TUgCEe6YpcC`8(SMJg!vBsZ46Ck*EfVnD>N*KJZ?H+B@dOnfC*b3qSUU9z zxP;PFrWH~%iKuy{(X3ALYI|>yQ<^I8eC9lia(v31&efrH%N5c>*3v?W7@sulpL2*Y zJ47P$dpNdhyJa`qSO*PMPa)9u+fl8_;4m zIN7U^NOYVCy$}7~WV=jTYGV=ke(qP7=FId1uch()uGvBv{Tdl>742WeN32iMl4lyo zj4;@(1wi*@2GnU>A!?D`OP0ky+k`b~Zlya8|3J#3tvtSH&Y>gTxB;k%Lo&zf@k#)7 zIt+f99s_c`kiP?Cz>;0a?%^-7;?GkXjc>@LUIb$wv%3ja;EJZvbe zCWM!%|3}KgVqL!NW#qgX7#k6==NpS^AAh@~WL$MD`W?h)=MH$o&Ed`bOp7$c#77+J zdK!7WeH@jmJVZF}Uf-mr&o#O{g!T!)2ncuV7*??fS~H#btgfq(Bzy(xR*0WO!g(3| zR((j|BTm#jHXbh!Iu??)hmp81<4AI$8lg)x4(E6g)lM`THjf+WunXvVL|?m#&k-6I zac}z!Rh^&yR(UlquApJ^Y(nP^VCT|XxmMxb84m$5-k z`YYQWvMjW&UaOXf0fD*`ed`mObm1_b~&^V^Q|yZ}!LQ zEd$SZot{bqk0MLYfzjomptEHTg6`(AvhaD$$f@@2q1N$ejG~2R8w+WNbxhNyUF43s zOW?q|FBHL(Uq;|cv^#W!D@#6-U^1Mx1{`|*xdtKd1jX&}I!>;bI`pFhSc^%7I%mBHaZij$=8J(gx7+UG~F@(BfieI|Rw7`*key0YOj^G#6fjB ze{c|8M<>uORbWL2E~N{gD1IaK^>yi&4iz)uIgLe{0ee-4FXUHJeIEHlhU5D{Jf*wb z#csEh2oPAT-|9@V*@qS7lpp>7JlQ2XEF(uFBSSzp(!7X6J~l#Uwr!utku3c7`4T)PJ`o@qsx z4GfA+DJ4+iD|IYob{;Dl@m>y>pr=EkI*iKOUO&HxF>CblD5!l$nPO|lON5J7ttxjZ zQUsT6RQS)wsfYGHDeccoxfXpRj@+-)z_6CQ*Y77L%*Ay{Nia1f9 zCYlw!qRL*7sV!ikX>M@@_1JqH3i_F`h2$ zRI)Y?t0B+I7#L9O3}_@Ch6gQL$6j!x!EW6NG-k;7=;Mh(ukn|0ha;T+ms)!ShER|w z_gnE-IsV%$JUSktPErLl8HcY1HUnFs9MgAgqRt6w{{bwrq&Wro5CMk?n7Xvv)8pzX zjj6H@H;+D>4T8?_;>hVfCL8wgUP1N0GX7Cb`Ld$0|F{Ku|816vf&CKvmg_b*=5k{r zo#jPhF*Pn750|Z4uc+^(U7z#VCp<&@hF;5N1kTp>=LB5G#of@uUd1aME0b$YVvC-S zujyq}K}4;kMqgT@H68!TdS4d#B9=|wx z!*58Pul$M(#-x16KY2#4nybY$e&UN4*`DvHr+w{T#%NuKdw=v(K64o(PJt z)5&Od7A4I7V-;qMTlvJrth{y$O1kSy`{}>iBbMajUwd-=;y{2=l!%eAiD;h(>np4K z?z@H&sk$Ig+~2!(mJdOyC)I&So}%q{xn(Nw`1hUU8Uw|^1&p?}ZiM9(l48+ZljdQ{ zpSVb;{D#@YCo@L9K=5FoG#?h#Z2bR_bogv1kBf_HL8-T?ocr^u z3(UH^u)6~X{O59{I#66AqJgdWEi!URu$72|#p};^Ex?9E@R?{ZvlHoNq=TIb`pYJU zoeqCLQ(5tl{nT062QwIhQC3l*_)TA{O7I~d?}kxiQheXe;#H=*gQ zJ=)kOqchOgbrY00qCXdJ!&Q{8c|_srw@RsX<)JkGrtpxJcvGb@wl=T)%$qLLwVN9U z7hSaKXt{=xj75n&{e4hTTG{`7R|sZZ0HRZ=_w%ae-Fe~JupbkxcN#jI1`qo_k63SL z(+LK13UY!}YDeUf=FUFbc0$bJ|5WsI9@2MR2R6Ki&fD4PX{8Uy%tjDTgq0`|TZ4zK z=g*K(V4$ALo6nNsH5QchhtDBH7r?lJ1O5d3+JyO@Y}D)E*{p+qLQqA-3Bcx z`WaI~I~yxb)8LvmOC>3mLSHf0`6(ifv*)~yvWFpJ7Qek+dET1e{?fJwV-n+PAsOlK zZM;FhqV52AAXz6;7CT(Iv0Qr2S(zaNy!(7c`Ej|Fo|x8O^kXPbsD`~IZy(3Ngm2I} z%!#tT`S{qN(bP9m_%^_|RwiTY6b-^j&jV?1M0V__8^HXsznOH5m!ylj#(5otz0fPk z(b72i5!}lkiH1a}w=h+mV^=BST%6$>ojSl>?cNTkmW#Yc@i% zyV)0)?gxE#b*y!xDmJ_VkM2H4k{lq#)&k* z_>gSTOO7x1oFJ353rY_8uVA%~p#5&JWd;m{5D&VWW`RLsgph>qHx7E<4V0HgpK^`* zW87gIdq_-c55iw_j1@*IPVryfpUv9zA}dAo;y; zyJZk?GXZS)09h&*0%mW9t2`mOt)qxlL)LhLy!`iJGP59`@o+h?F*z!(s;wsKy0qv& z%CDE|Z!u@y$1>a6*U>G-8w_I}Bar>+PkB|`Yc~pJ-o=GcC?&|EpG*Q%2+tJ%7m*EC z%_2Q6>jSRa7A!aGFlajA(#KE`DkcmJ5E{#1V4}xAMNP#+!(wmOOT*Lg%Rz7ZLe`;R&%X;3aW@hl5) zhhM?>WL)nVsr@|QQWe86IIYZo-&LNZD|;+HdYU(WkrRISd$!e>Z(Eln z5_s`Qdp`vJF;=QxP-s3ld{X}%mC0texBB;3aLl!o{hIs;=vGl0%XcMnvDf8U#K&+V zIw~cG6@K$wF?s5O!)b0KiKwv&DX|Y6HS@ON66f>gtO#DgAy}35b0;i#uKD}zXA>E_ zC1V_PgBCLm-~!*r;!@GkGk?kK+#bY>x4(&hkcZlTr;Ut z`F&IMsj>1=`B{hQ^Hda$w(!0*8+%%)#*MZxz#nnVf3AX_-QU-BCFfm>a^a*Yv^WZ5 zw?|m!pX`GCb8M(?LE!gUM#9P!IIA(5mQtk(K zxx3dw$>gmW@pTD?vi}42KncGtPVfE{;7sR-V_o0aXYes^wyla^wo&C|=ZwdP0l^8r z_j-fV8F;5$Bc3u)PG36j-t?m@`JnxX5P=OJiXJ_-P#7OY3N_{)QIQ)1mn@db!50|F zoKBsYR6$uu>1jyV7}^*M8L0Z?kD@)YR8bncX}D&&_~6d}5*COh0(i zuypDH>**GV@PWsrR(*znSqzlO}ZP~g8v8aKyFEWEc1UySRxA-&RwzN-ZE-{!Towch}L<>}ae zi(0W2s*BURyMH-zImZ0dHT4~K^sKf~^*CK)frbgqe@vg6Ix-!<&-heZdf6o}5jf=Z zJJauPe$urfP?!TLC^iuo3>rm^RB;GNw^*aHq=(^R#>D}AIrASD1 zy*J)wH5@fkF%62J|5Mou>8PDPTp`FV1zY$h~}j`o=lR5C0<8wGJHqwb?Br>f7sl$T`80zPa6) zbo8|15wDc-8P$8Ae0#d|{ueDKDMYd$Rs@&;9K3FZ!J@y~&Z$^a83K;|E*!8UN?!m>Tw%jCg|2D0qXo*8oFFeCfoEg*%Cfpod!~@p=%;mnS_V;xW$TO6 zKnh*QyNYDcWKYyzQAHyvHS&*YavH@mT1}2U%j^Lng5;0#s8q#K^sy zY6xs#D2&BMCLFKTC$YA)DdskcO$NkS}|UN*n*;d0?+nJ`=lkSf-e7>{3o)et6VW-*w0uU!B!F6tCT| z;pQa%#U5*?Pi-=?!uqWFi_$K~-{YH}Dc&u{#-s`4=Cl9Nz{KnjcX~*B(lK%cQ-YJI`PJ1RUATjx^0TMg*W0NrUcC4_Lxv1l@OE@p7M<$i^q=3;oag*-ajI_{wdRp? zS_aoFt!Yekke+hDL|w~NHv05;9*z<7Nd=GN4n!X)FQI9y zqRKOIapf+!`n#N7m^mL$ue>Qe_#$52ga*z?Tas92ura`Fo?5*4UvKKUHR%ZmfiDE& zg3MLiBx=s3zy3}@JpZdI4E0E5LCPPkgC}y?0f2RyYzJ?~JFn?b!OUB`Edmq;7xC(2 z2r_Rh5{4%r{{XIjcChcJ>`03h61yGGIii%SZ`lL#tHpAtC_#@t%YFHTlK!`+4Mz4$ z-`QhB<*r?k@oZ}Y9(oz6Jl-K&xOhqW{<+sJJ?+AChqtY{e)l=|-|;%yWUiEl`bw7< zqqov^{ax!Yr?wWSZezxOVQ%w)n#ShVbRKR;>9{aG{rX(JA*EQDI4^B(+QLL16%8yX zpkk8isG?A3bW=fkbbux1OeXab0}e&Qmk0#%dM*dps15oND&Hv=3kZ({Sc|wqnbCnvQsxvzkV#0Hgoa~T$-=4}ftYJLz zj{kJSDnrsGpW8A`9)VXrl@gDXrv2)s^un9-3n_!eqQL@B(^3(rDXd^ZyXA~I=<5d~ z3$h4D${CzCtI6pmoWIOd7-z{C3?G;XACkBQ@{<&~B+ehaa^B&gAT;fZCHxH^&4%oH z{|FZ?O|fJIMQ(!dNY>0g4fw)IDSdUk+9|?6ax%9v6{!k&*g*>`l(piqCr50JGjLmv zTkaXSEAD+Nee2xY(u{cv(v9ETv*IfM7iP>!pFH`RWv+Xw=MRvy!-U7?%xM|krEB}6 z-|9bYujwyd_7}bokBuw!?@!gmiIe01o;cI^(ZbX+cyza(J$fH;(ou)CP8eFbFx~sy zOuZrH)!BS?lg&T|i7N5~q2s>`F}S#3`4qMQk`C2@b^-%6ueZ7cKIpG`XN2+Mc|7r^kv@s4?MjzbfvFBFTOCRyb^B%;r7L49f7!GGp;0v5vpn2&PyuFc%r9 zoaiiQVe-f|8RXt3w@Xtt+Kp98t^y_J3NseWVb@5)R^TI-L)j@`7~~T)Xf(KKO>&vd z@>n?NQIY5qYS9<)h#7c^MQi;hXQ&b{2qpaQcst@mqwKpj%F!n(7h`NkP2sl|R`3|F zacsZN(Dd!yHdyW%xF_%;gYRB&2VSW8g!~}*YdcR&Q^yRXi#h$^qQ9q`A9-H&^0UP$ zz$;ixk3P$OO-L4={rmXrLkrT)kH407`Q0sP*8IgLz`2kWmE9bMset41`Kbkbj<9~29g#Jv%v*Hc=z5a z=7U5ivR${O$S3CnO)Lb)sV#J&j1+BV7aS6*NoCf#Owl8?8O)t2+s7!NjeR9&IT1~m z;f=uTqXnw#kL;Viv-|pK(yIN-uv~r>M?gQpV|Y)w;eoU`USt3oE=phAZr!ZM!gUWm zn|^WmKV*wukbfy|c0XZynumMlm>3ojXcD+ABF`o& z7AMST&PIh13W{UM?1+F!3o7PZn5b+u!Slr?P zI`~i|g=}&oa>o-R{(?&jf%93T^$$o3RJ6%%2OV0lz8QDemtndmf{ymx9q(j}P4?HE zBQlM`8MtG1otSo>Fhcc?8{SfM{MGkivAHMBp1;tYHLX5fo6`x0Y>^ioO8Ew0{?4>i zHj4f)!TShn^?SEr>oG=)e39e6<+yVo<}?rE?@wPIxcxpiKY!_;ZiF`7Ug>^+sxD48 zPu}(x6KFV;Z^!3tJ$r{Y@A4*I-pm`x0z@I@B1)LNMS+b;T~<-EwcFw$fNxB(Lft2N z9GUP?NH~iE4=}UgG2~nfCJPkV$0m54Qf>yX+Sm|X09FenWQGegauLu>xo^JUP^<)^ z@DcUGy6LHjayeJ28+clzDW(F?{u{NNg%TH*ukNsJ`q;*+FY}uavHU7_<}Ht=|Gwys z^wOL2eEir#BIEkq$G52XO^QcHhvRpqm*1SbO#c@zX~!i7#9OMvkK%Zvmv}`8F?~<9 z4QX!Ekk-~!+Ry4fX|oA4reDYJhTbD8RFoxH5jWMv>7DP@D*$<03)9Vyznb>KO}vXS zVIyhsp~Ao9{BJR41qK}D5kxtb1tteV%?T{{xBwJE8#YMLIhTV;Vg#Pa8*#)x5RCp5y5uPnL$UCTy*h|UkBCa zkW)qfXhRkd|=pr6D&36_1!+{3W>GzWQu4!Ro&(`k4*Z4CQr!B2TM^BZc1^o(F zsxD6NV3%S7aVAziM&K>*;|8sg7p4dhPCWisAm$%f(8{2~WYuCr1(xDyhc*}))X~(m zMaja>xr8x^eke8uNDZUFBGw2h0~P@4$uVr1=r2>Y=N<+GL~U^hS!To0w2bq-A0Q zQeKompfconfpy=gSHY9*h=9$qUko3#fsFtc4S9j1d49;p3%2*4Jo;^%fx9hD$Ay%^ zNicmBlgRPPuF(UrI8A`8NH=0pI_}atrD^KcMw;H0@sx4-rI&(>0W&e8 z1{1#GuRhupXf)=F5(6cQ#R9ghGZye36yS+PM9$K#2Y_c5ZxB`lUtTyMWjR3P%RXE8 z6|0U9J(|geN)Cki=xeAEm+i||VN-qeRY<%>p6QJPPSFmWv2pYMqcPOCAICFrya%rH zN9XDc+=YKn7u@-TFwsqozkKo$17WW&E$KvjovYJ@l=2|Gg0FqNY=FNrsV_D#HvXoP z_?uHX=`r#28AAHG%=n>C^O}ct>C*n#X9n%O_YKco^k@FgN%bIdryyAYQ+08AXFHWY zgum^?i9N7eQ+j5`{L;j$TsVr^#l^=0j0R$|#?b*~obOq#lBC=ysSO;=xrsnkV<}jI z$H*8=Ozw%}<1{^`=9cS~eUuG>g)jS0sq91tP%*Az;V}EwC)zjaVwoa9_w0!)>^m=H z;TLVuSjjc-8t;L_8Mr+*NRviYUZF1xHpLmZ2h(D5cms2dUkI!-YCZno&Go8rSJhd# zPo5mVGX;PI_(Tr2BLjnA>wX16xsTp`>8c+t zi2l)|GG4wyM>z=P7wmHcn1ZjswoXluU(EiI+fPVat~sQF?x&aDrAr42kI>A0VeX-9TpfRN;*pnOsEE=sV`gTS)U~BW zjeS~LhqnJEJ=$lJ^vWwZWtiV{XG$yJwz@dIvz>Y#)Tg~&2oLdJ!wsJC7cu6l&33XB zi-rQuLOLx0=437me(y~AW*xU+y^Ka56^E~yL1U;T{3UT@S>jk;`*`zhv*MQfj4<3P{BMhuJE4? zxj!X5du#gPW{vKF`_lI7qjD#IkwIQBT z=CY%pfPjhE4tO#tWnWyv3?JEd74bnPA892pf|x(n$FO5~5=n&{eYo-QLy0%g9<=f3 zbk}$GNZ-JHaGgCKm-oPJf7~T$-``%D9(~Cl=*O{f-y*ZR?H9>PI=?w+^E9YO@sdpT zjTc(Jhzl-RrcdS|{Rb2NckF-1uXVOKQPDlHLAS9FJo23InCE%8{;94@*He3Kw$pIP zxj<1PJ}9m@srt3IT-IVWk9Wh5p(axZEjp$>7LB1ja@fmd!IYc~l0Dp4BA%)3P9BY0hgEEY8JmvzF2 zuoeDGUWu1TK?uvx4>!pvehSd8z|MGbpyJBLf$!IA4@t+qI>VAPur|LG?bM89Ji)ieg88y{Bnq5;?4i?)j;eg&)VAWk0!%mkIwe7 zZ~PSf6RG;@W8wgg$&$Jj{QlItsmp-&b6@K*a!PvRiFs(|(L703s8l_2dS|~PKYP*jHbu%yry#Eg?}gPKQaB`pbb(R9)!=iq#yx9Aa*>lfp>lK+Jb_D{m)#5f%rmL=nakF=+Ls02mY2m*0% z0gzI*uNs9N@aAWMH*V6C!0bAc?t#1P$nDdapW3R^O9OZY?yG;kHEnq8Il44}xbTa- zTBK#)tk~8at?)mk4My}!U*CECFs*$0V=ts1;;Qk$mAwl)R3ugYgMf0!vvm#rb>A>% zzhqu`>%Y$xa^ka~p=ZO=y0&4%w%+2WsIIVI@7S?w`u8KiIl&LdtA4kVpZ3IfwD^U&##+G4_1Gl!C6rBUL|Il_AZqGC~rZ0RjLq7LlCaouUq>0zEjfo_uL>2?pb`ravGc&U7Y^s zZ}+ByPq;SS`ov2(#qa#!h$Aucn@1$M`%=m_S@TNwmd12G9)r>yzc)pCefIpc`_C^) zGx6-EqW=X(*icrE-nsn)r>>d$cW=wBr{DO1?!%KH9AYl-LXD$iM^H8lw*ME^wly{` zowa(`k!w8q=EL_q+yNBe4|QH$oc{gpXk5JUkO%f>DF55?07?TAxWqrCd;b{RA)2|NLIE@{&?3;A?b?JT5 z2z<*CSNIEy$F=GP8`k*2N zmG<}Bc#Sl$^t;vRH$9ME#KY+Ff(E*b7h%*({<{MGTT^| z`On}PZ^R_9z%r}_wTGgy6BxdEP?=xE5pFqo_^+H)cX{X$Lq6GK8SSmO!v4$C#^DUy zo^O2y?z=bxcgD>)12-2q94smTQDmPwm&+`?J zVjVl6TN=}^tJojG*S@L8U6+o(;qhgxOdu>T1x$XD|?`|Hi{cwi%Vn7+6_kaD@zg>Ue};L8Jvzn$m#KDo%?E z1%@x|d#ijid%XUKXW;&E?C$C7Z+QmpuW6s(UWJ=;-=qU>x#Tls*)|FoJ=v!$eAPHo zTENr9;eFFL@w%sSy5sQ|)AugCjhYe%E(~Zau6+5Pt4pfv0aWK>}-b zfh}B+psj1(fQ)YG< z-PB%Zzd_hk=ag?xZ_HhUSBSq{!S;~}gKXA=g^x`6uXFBBA3gCF=<)*JA`M)nC0+j6 zP0}~EAFFrUSpfLlV$`5W8xc@MDwyN|HVgvk7^h8QX#rJAt|Vq*2{n9~Kqg&@YAPx( z8!_I%(>Io>OT2+tC?{hIKDGv)_KDkm+>kW=o4cl8?6*zU#= zEK!i0Nq8H7l_g>w{b4!mBVYD9CxlmboYZly|}(@=-j5kwJ~|9z`v8G*uKnHyFDii(Ik}) zl!&|N-WSq_KfDq@h`(B4nNLRi#FVe?C^_u5n1!6&1WQ z>OU^fY#P{B)7bjUfm`f87A-5p0##j{Vg^>`oD=+LVd^z`o1rz$E#G>h%i3LPu!yjM zQ^_VYwmvg2zVC&^mnlZ#*SfU^b~8;*EF$s$S^QYO^)IeVKf2;U-835d%r{!KPrB`> ztOM9B>cZIBP|qKv#;~MA*pz$pfwc2A)`MEQVPp-Wb%zcFG{4TrF%6 z)-^KT&o~s!H>ZtCiPr+qtpFQ%S|cwZbWNKu96TOkvQHJq^F46K?L8^o zcJ!`kmricZ#@1i@?oCSQ8QGqWy2%+05`^ENblPUz$7uYZRNjVp_ z@x%J0V|Lq6TCutF$(Pcx=f_)`yko+H!)*p$7_^yKqd{h~MGVS$VRZxbxCoQBGlqEg zlq#}Fcf%YD_%I8#Sdgi|sJ>h6;`%OqdX8W3cvL_1zduzMC+^kCr1=Js7N#LP?YrmF zx~BDJVR6!AL4!>o3t7#+m@LTua@R8zYmPX~ZGf+Ne|zA?3LGn+ z-fXpW{g-f|VZQanRXCfCz|F^`rcc3ay0YoBoG|+k?*u;Kr9w37EHh_fU!zSqHz0|2 zI#4VMRu-^?R~>qUw&4uiS9icYaQLzN&`mpkJnrSV9ANTy&&NG*H{)@)^WDC&L?67H zP7Q8zvV;F-)0?1+5-a?;e;lu_%?;_4Pi%$DJ$SHTX(n#i{`iSkl^zi1<0U*cj;9}a z%m6LQ)OWv#LF)LM6Ccw{eZ!y(9r`2pyq5b*0`H{g!hhw4A$op&pVn00wB4}X4mlWz z^3jufL81>wsxD5I`-wY{AC601&$ZU?RoA8aFW%@nt_xp$$jQS-P9~-ZTM1c5<0b!m zq2hVrtKw_R_`%)L8k3Mn4;oE2Jh=exu-o;wH>R&&aBs!eLTau9?9E@@I_*7iC=HvD zun&9YiE)-ZqConb>|V^u*?*txYR5)@#idOq!ijYY$XKGBelXX<2VuxK04h_533_24 z-(YYCZZ$gtx8u6Yxd-l*H0`)c{S4ep$C~|)zR-{jU$k32JfI)eB1by(#hecRe|g}h zX;^Q5{VvjH|8QM;`c<3-3>J=s#skdC&0b-nY~Q@7^trRIb>;B$F?AQ@Kgngl=8o*L zK|A=@VP5iDy2ItZ?bKSKXT~xf@T!EQpJ;`ej%H z(Kj!Z?W1gX4ZL(Z{vaMtC9wHW_Mc@Rb8@Tf1&)CYfGN>V%s^h_MIZ2jg!v79@q*5N z=eDi2A^$Q~7bn|AAKESVK0jKR25%yP1t^Ep>D zENNn^*@A@vevs4#p72e5FjZ%#t_he$P)nc65X!#ug{FCCXW%YBa$5S+C$~te^{c#% zhq|xM!Wp=;Z%Z2-dmhff{j0EPMB37p^-RWOQ?eL$7%TjbQ89!QIGZmP;0yoP9@0A< zyZdBsx9skxUP;HEH@)yhgJfU1%4-70BJ9c^z-c}DUS8NJWG^}{PJGxL5jG!y2dCjn z^FsCkc0YUo(Tr7 zGrFwV+K$Ima1v-Da58X~MJks$$x!D6{_B0ur+Eu`qqwEvz1q^mVLcpb3t6;9P6uqV z*Q7V`!}`ZgnVya~^A6oF7COxNHSyXbw@RPfbhW~#tY8jBuux2YRHt+Nc%q!0v3{8_ zS_7|8F~@{0XO)#zm8Kt=zBM@39%x148;+tUI(JrmdDO&zw!AxA*)q5FDQ zU7WDjJaNBr2l5t`V-_~{>NdZzPo4z!Hw{~AF>#`x997oU1~DJad)fUjn4WL<;x$hw z7k?XV_6ZNH0@DeUq2bh9o=8)Fa#gzjd3y#?5Hg^Cy4Qqs_9v$5A!rey#$EUolU+8< zcu)Xh6FK~0gY@7@^f7eQ2BH44o09N002EkX`gUzeKmF(i>GorGNxQ5&jIfHtGjM;< z8Mt$B2JWsj8)x9mmiOO1piE{s#vCzLLS}%%e<{+8^@k-pYC(hW5>O~kho_(KzeyU= zr}Aj-^QT>(p2F`;p%ddrPT8jqYJ|T&ej=4Z*}epYeaCk^a^l%KhaxM+Js$zlfA!Bg zX|@AmW3CqHX{6|zAhxe4o8Q#GOHE_bu>-c(}z#Y3=I&67o;O^rYxC_(gaR%-se4UFxi~g$r zvSHT9tB%k&SF-+D5L+Cw&GnU;k5&vkYQl&ADSdpiv1zx7tCoT3#B1+M7u|*1i-NCg zpPDoUnZ^~>!hQg<5(~z|fsO>V#ieWDDCTWC*xYBx$!~TU)5M!9Db!A8 zfi>nP)Fe=Vb1q0`Itn^WlFRX?-?%o1Ekk;?rb#0>H<)Y^35Rk7y6Cy_WL&uQ1+Rtu z{8@LV{eORRn$h9bmq9&R)1{x^A{~XR%W*={d=cJ=@?_7vn#OtZQdJY3lbU673MC7I za*)VB4uD0Gg6m+MfxDac!0kBxEziL1{oAW?58O=IAU=Y}?2y@mF8fJUw`C;?l-zvw zyMUl4#m*qwd_@*IYYpz1zPo4T?@agNDFerxbBnmMqA;}YVkclG5%;?*2e&ly__<9w zi$rn9#mU~j!u>B5v6cOoAn>L_CfM&de<03&f()4OiJEvpLe1iau3g(kuXP4AKQzBT zt?Wlmu@5;8yhY`R#r0i>%xN0vf76PIlXp0GJZBRoLlI!C`t0z@G!OUE{S}uLloM}f z2{rO%43xBp&>PY?M=UH7Lw3R4&!)*gydvHHWaX_d{9*N*JB&-0es0UuA7{hlmw{}O zMwvMbL{PcM6oyIjiT2UQ;8Ay}41D;kGjK=mj5BbPmwyKC_t&MZe)89J>*KiRP@FR5 zn@;*4EurgvlzlP(skl5m91UNcC;acyT%S(<#MY^~f$?c+*8GL~ooT_M;>!N;HTH?& zZNJm1hVw1$(*btb^1$KpB z$Q*TUhp%?K;;S9$2qD=1Id91ho5P0+S1{?Gd3{dW`sY`tpZ)Ct^GC|InYvovbnDlq zrR~>53z_!x^~EmwyKCES!P+&INc1 z<-df@8@F$LD)k=@mit+@ZDs3#!6C8#h5s4v9nA)*TCn|WzfJJKI-XfC(ii@CU3&7B znc04!Z91yE-`xMfM|swjyx>&H-@c-Lxm(K|Ir(o+#!D0wL7I6XDIdvl0g6V-?W5f% zlWG$_7)s-TKezK6`{MVfhNA}V@R5xG_`v@_zA`UP+@aiy9E19{fum2D)iSbSab2r~ znh0LX!OIB&Wlkn7RM_{*A+@6^%8itGUiCd|{vseO4eH*4htlxdXGWQS1zu*r7jorcAM~t; zUCT|zd*DtxY}0flZq8l)8Mu4XI!B+APPqO-yO^}xCw0)rhMlGl^21~?YH%PI!w;y* zC(;=uk(R6d4-Cou`QWq+unn)x73^7==<2~U|TIzDrN zfoGoQH}q+3?Ar6J9_wwz)2N*Lf(Pn@l&`FdQ|u5eOhb0w?;|+m9Y3dKxZk1@OF`b# zw$EZR04)vXnprkcNE(Ul{H^Gcd!G-H^8BO%VTt!<&ML1IGPe^9#mr+vy7r+L)8rpq zp00kd^6MkjKeOfP>FOi5!D)SeoNd@a8;j;QymYzD6R1=u_wCx8j^i0Pe1&_fGjQXN zJ~w^u;(z$>Y%zW$GcJS%$_!IVtlj*e4wH{Hgkjw!)+`O;El@)z`jPR_i#Ezw9y_RK z`u<0zWF8jofA+QX&9iSU_1`>WTY;}EoA++tmCQasr2))%bs%bd<8SG!BOv_EDf?f> z2ZYWrsg4bFVlm@>vkx0#h-B-@cYyLlSZaz z-k6i_e{P0*RQM*n3iB3t6!9LoFK)ADI_2Y=q{*w|`RFCy;J+Ma;P(67wdwR*9>e7m zOLE|#+vc5Be+&6b*p&VF0P>9TVs-Du3Ey67-HcNcBXrkLh;J%Hy%Urur1x#DzE_0|R zv(27#)x$60Vfe*YyZ*Qvf0Na4{ggQduKPj~|+7-#=d6@l>9SjIA3_N5i&cJ_W($L3$$@_nXRbv}S1^ffxv;>N?AN5@@H^A! z{*})iKk|(0)8jAOnZv@q$EPT5VM0-MWrp^_NtsggISZmeUb%fM8_%+5>-cJ@7`u*3 zL-@~r2VVA3iCl8{!JH{_Reby@l$;xvu@|^NlP~D}+~6Je-5f0+$lsqoYQXG0vkdMSKt*O%!4C=dfJG0-$Tk#AEV}hvop!oi#m1%K`L1$jub>Qf> z`c-i$Jh_h%Ho#MTy#mi4dgzxg8dZopAA_{%TU2 zc!Ovbu^3gdDRE-DRFjfVStQP7c~ZmuSqm1W15dg!eg03k;}#lw22#+B#;cXC{qlC{ z(5W7gNA4zd7-29hH0lV_c2dnrO@VRs2B;J@?eTAgb^* zMqe^;6h1m1w3pO1r^WTFv@{PI^_SGAPa{M;f(lv`-`}({ElwP14m@{2efKGweS1M; zudX<})y`+55@#3^CjfVB5oD-V>^7eW$Sio+$dcndC;l3IwJSe2ybpe4-h6Z)=$jpy zD$vARjqNgtjY%z#$!TsNV5BZv977o~ z?!P#tUxXR8nF4Nj2#6su1jO@-^^@@&@Ycroqr>0k7U_DX`Fd@X*^+6&EAgZv5BXRY zHFBj8xbN&`_#F0y4|OLAB;;h` zuExgZXKAu%$E%eZdD5ALdgY0OD)jk}r~WU*1(fBq`-GvTiDxaGhy_>R9pzlC<%!0L zYwjt}^Vt1(;Ds4!lOJ7*``Pd{rbF6s{LpmWSGP|)Bk!2(9=J!-cq~fa#TmHS2yE+% zjZxT-z>&}JBiF4Hptb(9sX`8CsCQQTf%eoOyhJ{JKi+4f zwEDoFlE>!DXWW1*)cqdCj0Zs;zX*Oql&GB_iqQ$r)!%RlTE+gO|5RtADX_fEtD$s~ z!3l`RheTY82Rb=&$*}dT`muQcbfB9P=7_wbkO5t-(9*I}PFw z%mTHtElwQx#+F`qi%Qqkaf?bL_g_xhks;tbusw)&PF&yXXfMh!QHnN_spfiCPWWo~ zOt!yx5B33Y?$1pcQOZp+%o%YVwFBdyK#m0KAuqGO97Nv zCdO*m7CID_eX%;X#J8utCyz<{PFdXqzCHcs2h&-%JwgR)Ii8J0T_v>hU$n$7C$2nJ zRyIB3GX(8tsW76FU1B=k_{%H%g_oL-WkDdD8aK2PuO{;1pDaNTV@e+NFYu@rK6fY+ z4<07wHMRWxsaI=#TenktZ@fMKS}{&I{P98Yy-zE{;uLe7eXSd|$HAY)Eh+=?whHxK z6VP{5u`_M2Xm_wpGlVh?}XJV4C9qmlKU9JvgD>_`$Rrc?0yD?`~-)3mGRssHg4qCqK7s-iW33K}Lr2z4KbDbbYHp~-eRJF3`mEx2rnw94ccyv%wKO{m9y4%^;jAu{Xf8y{@dm)^no^?oumVKU65q|6sVWwVicC3rYU0Ws-0eK8B z+vbtHl+2UxHgyVxq;UYhqc1VNXEH2JAT7!R9N=IVdy{O-?=TcsuekjHqIgxLU`rc01y% zT?5~);_W%h=BpOz!|tNZmlzLqdR#DMf>u{7_3?>kfFv_VNA^`fHE+s6Jw^a_8#Ou#>jZb z3jH$Ph<~)tR9y)kW|)8Fj2qL#FT7zQ`pSReLeH{&HmgHwXD$27PDKJvez`9h?>;lO9x!wq5QSxSw2pUmE+3bJK)xUy!bQsPfhq@_ugXwe*6^ z5xwo!7aN04?4yBWMh|oU-5E-A|5;&@ss$nKNNnWZ9HUF!02O8JAILc_{Mfi8-7~lU zMwP!aoptNO=}*%i7O!p0*9gkOeyFGi-XFJF?f~3TUy)Os@$7~BN`JE3MS)~8vh1Y< zwwSYB)-%Rz$vk9+CPh(lfXmEL+ZUy=k=2ra~jZkhcIIjh?~+5jEdTz zVT+P02*t?|lg!G-1s&&C9R!>Q!+PS8y~E?VPB^^`;YIfpXY%BeJ792|xYTF7vD(x|N7Fjh>ykPpwlnGY0LH%t43wAyq}{}0%4Q!BA^t>IlUD7 zOq^vkbcsikS#8M9wKHO-i@BzHO#A$Ygbihkt8{?)Cr+PI(N z7L~?D4R|bT>^bDIlep{Rua33bj*fOti}?Fh=yljNfM{nsFFC_jo)8e7)8T^59o8O{ znt9p|xUfSPgG&?N2#U#6Xr{Kty7bj)Yo&X>y>mKjlQFuPw1a2qviqJ*8}ba?nbXsY zZ_Y(C0~ATVrgR*Rq^AAsay*Wu^41sr4t4U!H%usWko-a^j(kPl0DJ5slNX_)5&@Gj+i&I+1wc=^?+nBF)9~w4`Nl z$Ftw1arSbhFJRUVFgT5AkzoJaZ)IvWF51pr5uDzIFguWzeV!Air|HThiBV8r9nVEY z1L%6$5Mge{{0W|-fh8?U*v44LIBGs;r%B~x%pu3i5$cxIH4PfR!Pfr+ntFWbY&e7B zyPs%qMN2V&>|5Qy9roU`y`f>-H``XTUxPW>(7=gifzN&6lPnt89;4y~%VL6>g49ur z9WMxkvJf~L(tfuIGX05H(hD<+H~jM6x2?wx$Oau2)-Cr*WSI=$G6dWMxACf2ly*tq z*m0eT01faRPrj12``O?83>@CUrhXc}Y=;x`cIZz1?~BvLcRv;IoPTVi(dpW+?VQFA zup=@HQgfL+n%E^mYli^+E{Q9~NkId}zz3&BFF>(hYycf^jokhSSAegDH>i};QGdQM zJ^0+~CUW~4en|3u0$BFLJK71GWir^MIB=0ug>v)W+PUm0iQ5&3Hv%;+TEYdhNPJ`p z|CM}?yc?hje88= z?PDKB!+ZYqZbe(1IIQuh?Xv2sT^hUfJaI-F-lBp-RVFraCvks9u6@BhZ9?gBT9~fs zNm$;Hga42dYLn1)9cYxMM^W&d_m8K`Olgnxa6zV84Iem>(kD)Qf{~c( z$6ub64*dNMX{(=Jk#2eHCHZK0pu-7rDp^kZd<5|Rc*?+$f1Zx(qbhHGSse@0HAn4; z=OT^KFwv*@M@oK}I3B7ohfsBm(dD7p002M$Nkln>BT84c&kXxkdN`-ysQG-c%y%XNd3ayHUM z>an0)2%)TzdH}INPQO&*59D<{+EP=?ulj7Y^BR!8rwi1Iwm5NEIm#Tw`d;g9aqRr2 z-d%W$igpZl20onhtaFF*qY8J1X_*W)rqEM~Dv`v&V4((*!-GyD$h3; zN=_uY%|9=NB_Y!fx~VnSLy z>=?IAQuqt;NjE&4w*1jwiz}*~fRDH3i!o0-Z1c1VzVZ=i=&)dT)r`I*U|%@?9FqbS zYBqpSfj}}i5Rgk|6r4?y$;+V##Zs68< z6>~$wK{H#|(4#Fm(epwAMePc1xU;px3AG(m?0*K6$x<- zX!fg5+s`RRlk z9?=incGN+uv26NaZ7zp-nYI08@g z#6R*dmHn3u@?}oikQLL_Rvr*(wX7Rs>`K%?8_|te#Cbha*Jj?QYKcdAeBaz`UK#-< z>B>~uH|t|y+#;3jll22n0h4K#_oJjvpgxmBb_Eb$G1sU_G%j@X{<#(|j^&d|tyTNW zZG}zgiCfNj2a5eju#Guo#bRz_#Um`b5@U%RlahYW*CLsJ*%md3kx91Uh_AM_7Qa9B z?Ky7h@u_1yY{QYo?3Ov7tT#5(EJ^s?nwD}r5 zx+|v3D!2?{-$|=!A)L+^7?3MAEU>Rg;g3W;`uw%#%a5ChCAA_ zul(u8bmc$&tFrJm{$hj!F$A$sB@c#Zj5>0Pxc|&M;?2t(w5~%)j32wf(C4G3F0)`p6XaLx;JIALj!2sHl(iDf9%^ zrh`NTXy`EhOKN!BykJ`&d$Eq4DrF2x_9TU0Q} z9L4g)(rnPsV&gl3;Mn0diDRdg_5pV`^{5=s5>HH&SeR`8QNcM%_;ddL6j^fGcFh5L z`Xp6b@#KJ`a0c$)Z||hXo7gLavZ@MPj5hqHdU|Ol&cJEH#$QXDpi*T2%6){8 z{0WIANiBDjgV;O~#6KiqJ?bTs}_FKijU6Z_7a{Xo6g ze;lJcN^U?|!Eyb!-E|zyl)v7I8nXXr zD*hDyM`_vOeS6_{t%ubz-KG*aVFDn+eu^TNJUE6 z+=16Z9dhzb=@(btpI)1@K)#zTvmLHP9=QM4-gf}#RaN=lOvz+YAqfFONkVT4C4dw` ziUJ}cWd%{OP;BeE_O-4ch-K||L^xZ=e&2n zFOvY8#RT3>-n-|Xe$T!4%==!o+WNvEhz{S)efLuh9#$H92z z3x8Eu;a5|R`RSC@nZ29a#ihsOYwdHT5mVY%!!;65(EKZB1{bz#vksK{fesl{Vtm{S zGV|hDZ>^!}J$p?wHtc6ESe(wjWvc6T#)6sypvg$^l>W$$ce=3Y{^t(J7guS?3ynZh zKJ$}o8b9Y(8#~^hiE~hmg9bJq(%sN`X6LL~e8ix1D&t=j;A4*sdX&$}XoBxbQ zR5rp_Cio82GRQu1R&gIOwJ;jSaD=1SB(|x^J;#xoiG!aFF18Q=fqeFttQ>!m#e`S zDq;rATE*;!YeXB#_o!;*UB+a&4@uLn_~A2AlxA{z@7_GDi8!;z4t4#IeYCcfR;(==q{x|0?@ zbkBJ}nT(mA z^5Qi;(!tTNS)hSOqZkmX%Lg;35#~QSOcCigt@+ovg?eS6wy z)KJMptV%B-;XaxFra`d@)2Inb2E3=dbN964cz(&KEMNKaEothr^9o&Lw*DB8$y0+_ zHa`f1q5-fm2wzN|vXtaTwmNxJ<@jK8WHcdb(o!H^wyCQA@s;?frdtCSN(j49;R6)7 z+h8)AV##C}&xnE1z^6V?f-)u-Re$qG>L@HnC}lGAga1gv1H!GTsd=AqM}FWVXz~Du zd4wcMqFyCsO-@b`3>`J$5tVfs1`qk}f?*T!QB0r^Ob!$WK!d80;9!*lXai>fC&4Cy z$2H^`?_36Cn*cFvY>P$mhP43|LF&vG>R};2pMm?_fwkvPjB5LK-0A#3mp_;$eeFVg z2kuck<^p^sZ@wE&{(x3a<_xR_2lP7#8Y&>Fem=!s!P+k(@pC~?N@Pg>zA!m#~VHRssrTIf$ z6e3=}xqNs1O_(?$G#NP{85xtj>vdAtw0@ag?P-SqYU&Cf#Ai;6ElZhD?EHnDQl89E z!@bzdyvIU#h~p~b(#8=(1`KRHb?ia!+!@SVgWSk&A^`r`LWOH`VIbIt#^GB``ceC! z4F`5L4a=g#!9uji#lSki0W45}IFKwcmWpo9D+C7!OjIbP1{*2{GNyn8s8YO!HTiY$ z(aY~2H96h=mA9pL?68sXWPAGdyIUStQaa`0`}Iz68*g@m`lyn+{LmdZn~?P1?PlgF zklUC0#F1Be%tyK4@}Y(}O>#FZwNhZd@ZE6VnQ~ux`}Z$RGqPJFD98Jte*2;Q)A!!} zM*Y+yP632y#IL3wBA5OX+Gy0U^pp4R1E?%d&s~&0@w@933%XLo0W2m>0~t&vr+7DH z-h5$f=#KHk=uzHeIDZ%`!-R}0uzr9Puh=Q#AL61xlZ@DNl&(ot6BfDYHlNi1Zb${| z_u^|utK@_rurQ9?l$gdIeDs@fnE94Pt(&&kKr*aW6mSI}xa9^*;r9WJq4B{IuMN0~ zQ$DnolORKZ=qZC+Vqt6frrd(xC?!+Q_oMA-Z!19YRU)q z-a1|Vg~RaBE1qi%FQ7EK5DS9{!e*~m@i_zU0Ut4#?>)29iJKUY$Nfxv)k`TKRMXA` zQ|KfK9aEcqQ~u!!5=iu;kf)lP?skNIqkueg5nJHN8~e%(TIlYWSi- zDciCQTJ$}{Lf*y8{)>@1W0QIhzT;d->&E$tWJ`AFVB1_W|D+bX?uM2D_(a-gln$xDW8(8-4_ zCb?v&kbo-))kIsN5^qo*|0vJDj>o-m(=_$ux2G?^ZBoA{yF0Q+MRXvP55<#NNv4;KV6Pkv}<;7Bn6#9g=$Y?RJEw!dWFuUsZVgb#^mBV9CzdC*4 zoLdu5R_BFZ+`aaTN2FuJUgWq}ZKw#BOQwgmrT@>1*fRhLiEQrz-jn zY9UWp+#VN)SqaabHtuNOUTEhg%UF>OEaXbTLs+t;nY{Y%X!(PkQD~p@hu!3u6r%WP zUi%U~et}1Y@uU)0EgTd?jMyeg8}a-HU9?dY*s`|yA-weFOu&+fc=`s75p#3ZdBH{o%d@X) z7i=caCL!;>^8iCxCumX^@@33UzwV**mTzB}9>?P(d1=KLQGRsn-sy}J_D{oG?Vc#A zlRjsZe?S~GXplj)b&;NCyTSpRL2%BP~o>`$RA0IyNW|A-%Z2WA(r zRv8bWX(&qg8vITQbCM4pY6cwA!PDK(<13+FwRF=zNV=Va>^!F2-7Evg!z|GGP1jfg+L8i332aJXrOsp%;KUG+<+_bM1^-oe(;x&bH4cN}M5J?6NDH0upV4og4%z`iw7&(3SZd8X?P2PQ2N z`DJ}IdDG)va(OZrVvdvh?1KCz+2ozi8-Y{&ND?BCOUPM#j6#@<&IeF&GS&S&_{-%7 z-dqIoX4Co;7RjQ`byB5lQJgkQlO%Z&Ga1T0QVDqF7hx2eGp0c9yE3sdP+~G2wwD6Q zNf|znu&i-tYx7#8&K4yZnH7N(jQLPGhOqU zH`m<1L^2Dnz`KFk-odEpJ2hvI&x=oBYk*e`0EVI~nBmjrzB zerw@_rpXsL*4EL2FWTaRUfFP8Y_UG3%_cPq1 z4Drz?)$|WI>r2SfqW5VzU6jX087kyX2Gk-3rB+_NZl~}oXwV;{2O_Rc)>oiXJPdofa7P{vC8;rS1oP3Mge5Qt;0nK;^ z)S#imHre`*Xs(iYTqt-Ey4`UCPoA_tkG6pKrUn3%>Be9Oxh zlwv_pT!ZEYjKlV0a0TuM@7g)tjh}G+U4i3UD9_XtxQo)Mmpp(sP2I0mQ$=jVZwdd7wmqUJTUgZ&>?&w_jad z$9D~AZRu-l-eKI4$A1ay?&yv>?W7z=JpOikS4|hDNqY1Mvr<|1oJ^)weI?{*t*|HGd7_aj^8HTb2F7V9$vx$8x#6%UKF4a&t#g=Hz_fpfa#6&T3VSCGODVcZlic6{RC-3n#A`uLM6 zx2r92nW~jnigS7Lu=Lx1(ZEO@?pNM_;Yb24<}`H$Kl^XCtp64V@){FW$^@I_^f(2d z5qSrB)4=A2?p=mW*y6rLf1mnuo`|F8E}zNLjr(T;0C1Qc?IsILvRIdVj7x#j!3-N@fo-Se{fYg z`WN`p?5suZ2Z!G#pKMF5S$u5vqV&MCFNU}t3Ox)z?P5Zb5R)hGmGz?38y${*v27Mi zgU>eEF(0@$c_IsYA>?&584|*@8#U#qLq&|(sRqVt{yh8QIk%)^es)=UadG*P2gne`(`XmXM*lW87Ia^dZNQV)-|%~Qm(U% z6NPhm%^$}xh$dwogo*k)#5s>*oFc9@KC7Q4MmE@g*^zbFaZB!hG_8&WU@@a6V>20P zz>amqHLQt4;y$kA(HL7h{Ej+?zmE^M3_yMwJnWMY$&a2w^~hs@_fMSf7%_6pLWqT0>wmRUsbFR1g-nlN9zZW z6jq-cCL@Y17LyGPArAqUVL-s_^73DHffR{EvWN*#9MKTzv6g*unHu_k#RE^LH=X*I zbnD-rH-hiWc-vzhC;aLvoM+-kNZn58jJDggzLJMdp;iU^>?GU4U&x!m@LTekoT%P5 zS+_4e>NrhEH)U;uS+{E>lc9V-gnR35C1@<}5SQBpH)Uvp($uf<@;daHX0~|V(j^LF z#GfQXs-I|*SN;nkxwIi$po*9M(LeKrk{mx*XL{LSpWHm*fxU~{8+(>M@${AFz8@m| zWyXyr!z{p;S627r6rG}y?Vc(fQbi1Jg^j8NxjxO66sMkbAplv}rlk+H+ z5R`JSDQH~S1E&EZj&z`Q;}y71?T0IHd#&^e+|H+5nl8Y{GfR2%Q3IPz46{X>0i|@o zJx|k(qVS6->yE(5VX?p`%>s~%8A9n&-Wla{*r)UK-*%>&;vpF|DTO8ioOGzI|E42i z8g-3KoqX7*?bzr)RX;y}aeBx1FHYaN_?I3{U~nVXR|*|w2aTL!`q>gn?qr+c5Cmv$OwpN`@=tiyL4mrlQ)-(T|y<^1%S zMSB?G>nvoG82k_W^v5{GUf=_(lm{O1hY$q_*UK_`{LffHt4&tMob2VKmL+2`nsHm~ z9nYhPyPkMKto1wI6ZN}~zBQe3-PH6|JmAZ7GjgdIJ(DOFi+3=_Ki9{8>cSh4G+h++ zjXv9)5zT#*)6Qsdasn#xVNcsEMu8r})&OL&AYFsuRkzCsjU8{$HB^li#a-c-oarK; z=Co{tA&ZJ=K&dZ`l%b`kG)biSE49vNKD&Jx1q`8b{g|9&X;uY*6uYgyo;0Ytt)pxH z>=oqFj5`cP&p)KEaj zZ4H}2wdI2QNki%YkRrls)ED5YT_h*6^QYbE*H0s;qB(*7aq_UA6s1iq7B!4dd+D`y z^hZ;H?&>iwbGe%oCt-+9J+kQTQY&;Jp?TV9C2z#!Cg%WsJnh zS2PJ&+wHr2bWCGG9;jHZ1djS}D?u?&z{o#7G{=&$EeBh_$V4Cc-^O2p*oFU^Z;n(q zicT0mcZRP&V)?tveVU1_nDA6doeG9Eq?cT163tSEkIX0(l1#w1 zz)de1Op-kzWI8G9TOkjx)ED=IYGEwwh9=(6^s5uzl12|J`^9Ac%WEIdQ4;y0Y`fv; zbnTaqO2_Q6l~87%Ae7U*Y8o;)9!sGgq}AJI-xYt6jmP>=PP8RgRFJ0tbz5K5M4S3G z2CT6ei{r>FJ@%LTzNozUIXyR5sVtF3MReYc;C#B@DKytEt}f7WrML$LvT z`DJxYPV50g!b8KRQM>GM>avzGgFBn?@H4PBa12sJjqDb*8jOvT5fM!OBVFT1pco|> zLffvuy$x63w(I{2-1K?40(VV1idW!pbuG-B&2l0Ho?{Ze(zcCJS(SXi(KjHTeQ`;; z8{d$NLfm2apn%Dm|562v*^L_d!5g@c&+X^^H$LvmAele84*GBUbN@qU$?4imL70YS z%A5XN{)>n1lJ?kQY{5t18yDV@zJ%Kb4*J&l>B-p(YiNUUHS{MR*gu_q!T~leuCo{b zasyZIKP_`pFD2u?5awYH1%$FX=9BdX1n`;P1g$Tji+wNor9_rYmZ0gR4i%U0MiS0T zgPbN0$2Qk#HqAd$%^#CBorNtSqyJ_)_=@;@UziH%63v=v(?x#aOi;d&p|o z0(ZtOY2W|3GTn-o`OBW`(xABm?v6PF2Q=B}7dFBsf_*qlL+%@8`PYg(=hdAUKXT{s zk|#U0<$cNR>m;^YAt%i+kcEt3Hy_T@Aem&J4FRfY$4KmpDA-dTEwVW;o9u^tAp_N5 zSKoi)_rZq}K6%hiWGKqD4?UB9blKk|fB!SH(|)I%gPS*bo~^bVxhp>I`PFx&9meBc zDYrqp@&~MDp9B%*bKyUnj4p@&#s^+9K_-qCLN)tkq|ydCDIa}TEv}F^+cHLBZkdn$ z-cA4}N@@J%q-#}RK>*c2*G;A1Ge4b%M#MkjVDVQEiZP=$>Ixp}2;IcFR?$oI(SL9w zF}3tAUXt1uJpSC2vp=_N=CjX1MTc*AnY~~pDJCc$MMZL=DQ+*jtZvDPk&LY&t+iH5 z%c%8#TRftoXrX@`Y%sE?s=XbHLA?h-(ICy`SfS~H1JWDDk4#td3fzfrT6ot;vs%nEtiKdpiSdB+CDB`3b5B=mc=c-DVfEl+~;>7pD_MZ8(ZhZEyzdN~S$r7FjWMW!^ZL!x(Q2iw*pkG;8-I5c%XN+{z zw9%W7{7hH#@G%`N>-wtnK?p9^C={^b^eosUGWbG?%JsSEYHlDbwU@gVX#r%r*8HF5b8qDNH@mRzWtXRn94p4i4$w{)o z2O&Fx1Gg(MuPN>GxDXHuJERoyy8W=1+b1dX7~Yl?u?3&^CypRx21M#>=e4-w9legDy-y0 z2_LP6efSS1(##AQ%~ruDQi*duYNZWYV*+w3Ze7}?A;wfTfNARO?8S-i_5}~z^_|%_ zUwIi&JTi*J#3QCSYGT6TowNP@qyzeumDMRZ(R0R&J~g$jxA7T^2X8hIA1pvq;Z(2* zmq8}pgamMhK_fk%7=s7$3fwkn>emil`4zZtT#oO+*%i1L7}+*6(y0mWlaFrs{3&dP zA2!*&RmhkAX_62h3dowlf4z4)e%`wvNIQ-joyLtCA|Eu3=tD`Su+zN%9w)<7ISTc= z4O<8y1C3NCeWsLq#!*ECm#vYf3^~h~y9B)fsF6r}ZLvZ6?7@?Rr22+Oo=xAM!kaJ+ zElxB-j&<1Y?CMQl_~Q-feLuTYZzsqV@rx$sfBJ~@g?H?xK7_(@{OO(fC>jkyaV$Xd zKRg8u^zel-p|m#oCjZQOk;u?AKTTHbWTBn1k`MHw{>UjlazGoVA97G=(o#`2sMGb< zkDqKws`TFjQpiX@IHzru=J1I^W&3>wY7?aP|G+g_Y?%hg|4@X$T?!cP7(E|O9l zF)=amclbYb#KgX^tS-rk}X0$kffv+%I^|CviK@ zCb%$CKnQ}8#|}95AQ91$^pBO*AvrM|+~`Z= zh+W>eUTPis%>~1@8){v$h{Ysf60pg{LAQYyAA-$7I(gr|{n&K(mk!VsxOIo)5s8Mr6raz2zL)5XEE`2q$2 ztNy3m5lBv=OJ^xxE-k9G{<~S9Y;Ks&P=U*=Vzs#(#mt#0Pt$@Mw@Hz~{8J%h$luX1%<8B9i)Hmy7XvJP>^7cg{=Szx2+UBTMSp z9Y<)_f9*Z^T=|5;Z!KWz@Sm~fDYcQy^ zo%mif@2+PrJo~F%Z42-Xu9rMwVp8G}lU(o`)jwWVhvY=>IRuuRHrehc9jzNTcQ@N# zKn+e7cr73oprDG=sf5qKU4tud=dA1s+@J4BlTWz}SH*Z0l%<>mnhV&ews8}OwQaHr z<`6qyF|gubl&6R7bYG$+b`z-&5- zqFFINC?0gHZBKcJaL0E~YVBFDbm}ep`^(*;P>f)yau5 z;ikm2!6EN@Yib&}&!W}|dPIfMw~1rHvjDQ!Tqvwj6!8IRbG(J}tmAi2*M53GyoGYK z(Xu_Cf%}%Oz+IYtal>P&OP6#xU9805#6Y=Ph1mw*I5;89HhnO8Qj7|~h>|m2SO6{- za^NH>Uh{x>7UBs~H?Lyd^!M85&yLu6g7k?qY=Ph8DP79jA_pe;s2E$ci8g*LCMgJB zFGChVD{YEf56ByZR@8!V{tG+ozc7BHn0_cG z!RP)-j~u5K%A##uMw5tR``H{0s61;Zwdz{idYpGYJNYfc_(dq9EXPquFywtf1= zdv;FSjO9eEE&NM$=B-brV}5;0;$tVBc;cPf3Oxn8O(qgZ6WxIVE_e+n_|)W>029or zi83H2KU=wdiE8KeogS)MaM}E=lQt^YMh_j7&bV=!+NR4)Mr>-$x^3ztXtUW|Bu<+M zfUIbd5j4@Fw#DZ>AewAJ2Epbe?Tmtt=YQ%@eaOHhQkt`Q7xl(1H`IAX)5P~g4N33XbK5i%Z$Wr?2H(W#V?+iw z!G2LsEM~$qrkXH{!PrY2J9g3;2u57R%sDEgu{fJ>wDG=uc1&YN_!$cb{Qic&AvRq1 zR*WC}LeTMQTmN%jdP-7^KO1a|-C^?6C+oe%!pX z23&#L8du=nnm)AGrYn2~ZWmsG`{Ui|g@w46rbdG+(}J}D(*y-te^qFVG|>q=KmBk= zh@yW+#NZda?yJj-9dJyLS~pCdzJ+}FLE9JMr(J#$b(;;=#y8;Bk~3l7VnliK(R_kc zw@<9~$sIR`8wDDdEu1;y+6zCkaqo6T5WIqHbGq07;O-8KiUiV`R&Wn z4Uf(wLs1ULvv#gh_6b{`Z0gA;21`t6p;kYEBg#mn=O%CLQdKfZ&g9Lf&`lQGLfUM9 z>a9Da{kGj$YGU_;%kD`xJn|f(=QiY;$%Qm|3zz_%Bo4-9pQwmG@!HTI0Nhyan?5V; z`}MQZ`FB2Qh-`naDs(iHNn zV}1p${@FUdHYG~*LmhG(MgL5JSdB!4^+5pqjzXDojv60nfdk}%eQ8i{+mgN|^QS&{ z(b-?=A!VH z0;!C>I!V{uGqy`7&5H97ZzS5bV*Q{DDJYT$jG8ny)hE(|g=~d=5>k4@=IbRs)SH)E z|2`vq`=UF=DbYgS^pdy6KAn<%(^A+kyv_Xg4oF@y1JdG-&h+75UYb66=9PH=I!}p| zWz%)nPM3fA-RZ>rca+VLQ$HXdiG*siQFaBu!gsoFVMZG=;X8FiA`=WI>M^?2l+BNV zshF?o4IQRkLI#=%qqBb~v`szXv(<8?@Jpc(w>)sDoS#Gt=1V(_){*<8nkqo$4bKjOde6_wFT@rVisDb@!$ zY!EQ%CyY$jd~#o0f!h<$z{S<7G93I4+^5gIC+&2~<>|uvXNbUwLU|j;a^kR!(FWcO zh|d_s?dXCo{(Ry=R(_JhomEA(CS`g=C&=joN_0%N$!VpqU!vS^(B)TOy7d01^_lXJ z;m^8l$F5!Q(GS_69mh&E`c-4z{2?jG>WA!?PKX{pL$BFj=<4Os12B&q(vp6S*Pz%P z&*tgewvKe-udc+GZW(`%L{0pRFSj2ii@2yy<@m|E^qF01kURn4qf=+!@=$v7sb{5! zpIiRXm(z~fFa6;^4zo{C!LjID$Xi;WLKKFCdZ`!{K_wG*h*-+IQTpa+qiGK;{q>I3 zf0CS6>!QyxAQ@Tn9c?>^TJev&yCGN;r>y_s2&K(0_Y8=Jo@KaIU{Oc=Q;+;|=2aK` z0WA5srTmDA_()9k_b-`aVEDf(uXb``gko!4XYWI|958Uue=ZoZHO>%P@HCKMV!avO zLiwi?_DI*Q^a|X=X}gmyO{d@V1U_0AS0PNELF1%w&1_oHY(Q)kUQQnJfkYq%Sg^^P zkWD-}AM#RAD`OMU?MtuQ2OJZ|03ujo@|q0g#mAZjwidK^rfVOzq-#jLgl^~Y<~|S> zGk8W3mzN&Ea+N|<3l{m$wu;**Yyz3dM_+<9Vre~cSt^Fi&p)_-8oPFRqL;EC{ms?7 znbd62wn>&g)!VmNke~8$rDQIA7m7q~&MTrdu(zX;$7a5m4m|bjbmsNdXX4UeewHu5G(TS%O~V}{`wxTAJT_kQ_+^bWkvq`vS9 z+;8{{+{shYDVIEmXW*FBq1Z<<2C*DK5Pc$x(cyajawUt4Ld7Jrp*4Il@nX7!UDRP) z6Qr0xl;dPF`2Y%T7t%xy1rni*30-i{Qz5KccZ)m5%!7|WE)P1zi7I1}9~T%UWTZ!Z z2qix8Yvf&*+h+9%yaMGd6USwklzw>Gy@{_vVXO-K9)O6S#TyQKDaIqr)9a8&ZJbOz z5Fig7b~=gjU%Ff;XiK}h)93zhb^74XFH8&DdD^5bV{te61)n=AeeTdbOcv@2(`1vO zQkU9faFqG$4WK!m;e)vbnsl%-xc+CIjx^n<^wl^}h|4G=AoI}0TDDw2?XVr62qC-kd8z27u z<9eC~Z{9v=!$uoOzJ;0T+`(lf&dW8T$A*fhL9N3;SlZl&1zWK@NJw1VOmg zzEQ;@THz;Jmp$-ITGoY^$m30&EWCebB5ulj;F)>mC-w0zXciiaifH@`U9>DJC$;x4 ze4~98V=%oT4}$c#|Livv?Xtns=DLxndzGs-C=GN^44dnwz!(S=>eAU3?%~S zC%K6){U=J@bN|U2ZKBFzGA5tWrT0Fr6Xw7C@FDnc=h%Xc$>HXI~T z0e{t?xPHp>f5eaYh?idLtG17)+Yh?>Df!a>sb86b61}K?Vcrz@SK+JVTz-sNK(w+{29k{ORM~jZ|HnQ z<<+{+iLvFT#57`${l>PA9{ZR1!*?9o*Vv-VsVCrF-RGRRM>=@Bv3MoM@&|Nx;VqOO zIP;G5y{o3-=EP+_A*mH=eOA%{3sC_Z#`ZCY8=g~37&GX`Jo*a06t*kV1qjew+uaV3xar>rYbKiR~lN$QZP z_a2GY9QIrL9Zg}qEvwzUnwT#&}k`jNy#m~6<#FQT` z*#J@Bt5}xL0;||K0yq*IZ+c;DX|Hj_MqB=@qjjU^?xvyX?UTl)d-V+5PAj+qw-{I8 zZcT6c?lpJ@jwd(sI04Qkwk8WX99K*cwK+{pmX5I0v~A=KAM%(bLQ9UakT4a(vytl{ z&D0ogBn!%!HPp`IhJ+7)q&)K@CaGD$OX%u;+8wsj1j$pEbhraW|AAx=!bkF%|Lz|v zFJ{&kotVr~;oX1r!4yd9XCK%X58BqAXZm-%Any73{Dpub`?Oga%0T0B68lP;{LPq( zw)3gt5vzPEtD;Sm%hPxIz_xIxdF1kq^KMQ@{opTY_Cl`9vW&z-%4hxOk?EA9_f5?> zqLMtxuFooWq!cS4(kT=6*(m%M7wy-|>zos4vLyrjV@c*RDX|@MK={HjvAJ|P&SAd+ zSLFC83Le@$tDko;`X^pDWgR{B@5BQ!HSg82O+Q zR?%Q`n&2N{4mL)-&_yQ72N6xN*(|Y%qrJQB-ZScz)3{} zK@v@dI>bX73fuf?x3tU7uMszXo#FUW%VB^~3L)N0A7*SW|fM6O4`=W*a>WOU2zD365T}Sk>h*PecPEZCo z$-_pFWtVt&;QEKAr+rU4GhO%KQajI7yCuK{3^C(;P6o z_TpIn**e|lWXUspGfqo3vWH~MS{sZouEpFC1MS!LwXt_uFCIl{U+|#b!88R7Og?db zDNjss;yZ2=h!fwY6hi;1vdVwfA!zSJXRg4u3pT!tDaxp(rlx%l+I`n|>^yO&meY=y zoHoR(XX^`}f#XlNV}Em7y6vfXHU|+V4!q50YZQx23OF$3Uk;5|u9#sak~Ier$|UcT zidZXRa9t)Me2u*M67rgmuCrK-p{~CgHcR_pfL}^a(LtivS94arPz~MZC%q?aI0|=& zuT7K+z803QdjwbV3VG8{dCIUD%oaN0eF^`A$;6}oxeQ>SNmoH*GR`yo5^o#GZyVq@ z=st|M4Tx8kGW+VYkfsB=3j4x%GLv)G*BbxBK9#f37dhnZ=#=-b2n7Je)5<}g1hXge(5VEWxNtsMn~`JRisS)4 z;-Xh?o4i%pYQr(Ya(C%HkEBOu;1@yRG~-q`mmyw|rQh3x@sEXj^$$m81N%CbHFYgs zIQzD1{(JFb4^0CpPfYPME|Su#@+b;@e5GZTCMOKY9&q3iY~@LaI*-;GI;0_ATtVjH z?wPkfg=gSyO_x466Q2m;CvvhuM)WldKJ z+K0>zRVqG44OL}K$bik)O7LbOHF=49RNlrp8CH4=b8BQf0PAFmZrmhz^h?6 z&qVz&po&l-4-ApqioP0`P+82up%Sd(GS<@n2vWi6GQme5;0*0kh<=e|o14-hlebJ8jvAhB!jTsLI8Y+SMEa>yNS}XfFqr_)61;(V>rPvy zZ8lohh}d6t@3iy?lau_U4zoZ8Q)|iyGi3$U+ScNp1vd(&L49}^O-I|aGq1kjzn9IL z!53fkcYcXSQ2G5!JX&H4uOqu&S#ei>yV?cQ{#Uy zhO?~Ap~5F)Df?H-v#+ke<$vrh6jpKK zyPGGzm-H@}clWaw{^6@#3+C|=iAYTO&p0L~9!W7d{mZY&!G{%=RhFC(&~0I5rxjZ2 zhj1sZz`gm0*QKL=c}sc}&%m*0qIvD(F1&V*AWv+>$Alp)dF)hYV`pt&fNmugUQC9N z7cU!_0LA12ayhD4_od73f2u@;eq!qM zMmu6^_Ujij=7BmNunruYr^nlc%;^ zj|M-OLOC^PQqTbP8iU~gi)MBRi;A6z`+{;mT!AV|u#kf?;*>55iv%PEvkhqJt&z*w zO=rlTf7g?)$f|eXs-3y-ez_jgKs(N!`A?i&a6gOw2eW*3Ilw!NACtazYk@& zrIXLURc_gmg#D0XZNV}_>N^*8EJov{Yb04{3`dxHS@3hYi1S8izWllbvfK|C_9Jw{DZZGJcuLA!dSXhPg3FcN zXX_F2yY)D+E8zE566B`lza1+p+;O?=-3*F>?`Vax>UeE~e*2w9FpgxHNRUIYRwaiO z#kXGm?B#hPnh-GRrHIuJE_H6aB5?PYUWguT0&hqiXHE$pF(08|(k*e!a+B^b6GMvb z(_-@K8ywar0%&E;@PbyU2Xx)F( z!Wv&Icim2+AQ^z$x*l<}FT2|5E(wi+`<#B=EmW1uoxJ_n75Qh(;Qp=;_}poBjzTo- zo_i8DmzC?_`X6bEra9SR&r)SE44%Fiq@NQ@#=GZpy8KE2Dd=@8J%SD{53I~3hNpW< zH)BqwPOfjEfd^>12axJ|D8TXRPB0v`Hu@}(Y?E6&@$4+HCiPg>BjY&cDP~CB9^)NA z^Zti9B}n=HZQ7baiykx6O{<~)PYREs~PU|ME%VuDK{hh?81LAcVvX+XHDg#ip`M)tu>k1y+XpPyLXEQr4dZ; zJ?lPFk&|#Omg_aKsFFTe)RdK@h5r8gN*iFJuadOiZAT8dT}gK7eKLwCo)tsZ^$f%J zdpRkx#65>|f2s(TP}f_fj(-t>-q8V3gN^mN4DvO9wA{qCA zXW~jMUG?|*W2H(grbp0m5x^!mw?WEq7G3SNx_5Ee_8F0Rz zHqQX*D2N-L9s|r^Q=)%1P$TBD;)7&<0cR)&skrGHVk$pLfJn@()EMla>Aa?C674US z06~9wivGbSF7VF+=!t&I79M`NX8s~Ww~Q9wjWBIq^uv#h(ws5W@y@}5l1G+M)r0(3 zPa=y}b9u@>x{0R)f(o5a3g3#nAyvLNnauTF9{u9`6A)kgRePa+j!-Fy4}mDRA9)9h zp05USS8%Y@lVZe)9&sfxZ~Fb{YdFMi#yTw?r6&ws=KMKwL=G|DTtJ+h8VDml8l z;cRZRbG)1GP!WsZ!jwwF1do(3%0&fh87H%+{}2oi|HgE?IkBUf7cw3LgH+3l+~q00QQt)0fd zt3uTufonH{eb9@g2)bYVdZ7|uuD)hL&aEL!kjBj7xuz-8P_|OUN2~UNe>CR3Q;6=* z(m1|W1?KUMuV(`3&855S$_`IEC({$}+bGTIR4r3Fnlj>R?vRnzU6EP&c3#o1C*{^Z zBCdx=JPEnKe-uG-YJQIN$Gh1SEAtj#>v7qx_kVbR8f{Sicdn03-*d5jq4RVe#d%j- zr9?V8O`vN86LgaG$#Z|$5$F=K8%<`t=b35PDj&-2y?V7CO@@W{{cY!Vk)8ff?3&YQ zM1sg`(n-N8@cMA>*YQHki^~pQJi%%$fvZe4QUw*#t|aZGxYt$R!CP^}_Y3UK4 zRYjSDN`4&{zZwwA_)48Oa%aIz{W7snTvL_(MB=5f^Ig%e$<7h|wn3qTl;%sW$ASyF zv$5l>+||ZA;`brDO>V0UEBEfJf6X2*|5#tQyjTf5HEx&ueNV+-CiJk=heeD}mgGPH zC=S)6f2^ui^K*uFOlKrwIOn9eMvht zx?ZIA#L3#c9)A(S%XiY zpWUeaW|jK!H2moj@^lNi4jjxo+o@eZB3s+qQel}~^@y|Z$EMSsHcn&VZ#x|09h6`_ zauM<{w{e5EMjJPf^cwe#XkI?lltxC~))IsfqUHnRC4T)A@e?Ww1O`lh;>}7CXA2l} z0VLy;bCp?rI|tC>yAZriG}%zMrA_a2Mv?EgpkdJYzNM5^GUg@e=~o5MX*;iNBwD@r zS6@l>b8F8R=Wa$y*Y%m3#Vh)XTa_&Rk>uptUqg@?1{jIGI?*rEQ(jn7MdP={its)s zfB`=e9FX!T62gAMn|>F>P`Un?*(s{7*r9X<($-Gm|{g55F#x#WC*x(1}i-2 zgbGz49W)QOb68Vun{!H=bF3$GrTA^OO$o7i_EM<&5m}n%zok#8^kbCT9E5UJQuL{F za_zIOKHEV){$iQr^z$*XcISa{aWtPaIkbB`<+!5^p!XN`hx9|7s_50v9ED?3&->B; zWol{fCji@ygU@Uj%6TOI<>MpfmY3Tx5;Z1V=MAs-oQ5cez#1G2>0ca4i(^T#@@eG@nZ1ITQld54TK%HCSgnzT6}m@ z*_A@Jfq4@;DBBHjBW2cBJg+G->%~<@#$GBGI7x=7q;%iW(W6CGm|^8+VwfOt3eY-7bL=G6FVpV*I##zk$L2I6>C6}Iu0-rKsc9jwi zgGda&<*cRw^##kL=LY06{A_>2JNHx8$~)iJ_;~X=H}cxL&d!b_bn%JvuYX#*Vpuin zC*tqYY+}jDZeVj&8h#^1ffaZSWM2CI{j@ zdpo{P0vtX9fM=4?350RQ}49_nX*JaEDs>Zhg58Yy&3{18$c@CrlHF@IXx zXcVlTA;;2KU>S+8)%ciIO-eS?WwZ%--!5k;0o_%T|)VLP* zC+oU;`(^>(+p4jD_f&R1=Lqj&PO29F{0%IUmH1BI$Ht%VUuXmOeU++Sr)Pn&vT9=f z+ZbzlnyWpZU2KV77~DLO6#Ko(7%<vW3n+oYxo=qKc+kCi6&g$?q8H`{Bb{4*FuMy_%?<28JLRv%q^er9uowpn(eML9 zP8m?h+69D##uWOr8|NS&v)BTdaaR9D zkW5l`EbV-Ef^t!D+}_S=YI2e={B6)#0O%O{_cLr;v%q-Hyus#SY4{dUxVIz7@2P+n z{{(FU@lCY{dpmy^FZ8K%3(fAl=fJJd!FgOLTClaNMw`iuAj{z2ALTT2%$_%i4kar@ zWwaNt!HA!fZW!QY6q)%d1x2GDySJXW(?U?uc>XW$7Jto-goUItdHo(-PPeM3?c!9` zC~ww#LHK{sdbGAg+qUWpK931D>gd$chzOC_YCf0YwxqlcS(4gS> zrIHR(dcIw_{R3!gluJWe)sDgzNVDXq5I?I6bE{$t3(B7R&wNn9jVQO7EoKU5btV7J zc-mk&TP`QFLnP46maKE`we`o%JXoL&bM|Kkgl)euRIWlc^2OmTByGijTi;&?Hpjfecku8_ zsEyAAoI+wXaz}rMuH(fxr1A=9P`JZEFO zD#H=7!M)={0cqZ;EjOIB+tk<_KVwkWyw7UL&z9N+yZY&i7t-_SxSa6%$qGR1>iRZ# zt6iBRtoRgYr1}h%*v`4O=N5#gN)PvASU+05+Lnmf_0ErgYV2NH4gwq=TY*X%?sq^A zyfI>E$v_{byZHRna=iy@=Rg!BKt*tW6s(o)d`k*+qxhILyq!QC(VG#$Ba=u=5Mn@Y zx>%+!s)tt~iQ1rxZE7f9-Xn{{p}Ky(7`y1JM5X~(Yl9g2cDcMUjfVb%z_zZSgE}Gp z*MFcTmR56fV5Y+fy4rhd~1x!^U^fHaAMfUzEFErQ25!6l3$~eN4naKi7ZCVf||bv(ry} z^q5}zw|Ex0Ip-dhRx?N4G8fR;w)BK7g*@#kIu8BO$W8Wm21%FeqGhgGO|#k8mIKcG z8%2oR?kbKFR0nR2}DeDDOkXB_J1_ks|}hnRwtAw4|E0>9dKCeabuksWoGi%@a}dVpf*@Hn>g zCi10Ztou8Od&KodXV&?O-wAR**@bA;=+kJL^G+CJp!WUm<{h?^{h~um%S+rMw^9xAar;PuT zHoEa`!kym7UdC}u>Eeba3kWH_;V0HrGvqLAHF>8+mwxCo{qp3s>d&$-7HELkyc$a?E%jFemPINS9!0lG z4G}JXn1!&L76SWJKytKQV1T4OU*m6O<^(g$z=EwnI&)~KXU|IG$bKGT=#Y)xaOJb5 zGdz7O&==ocQc(C#2>UbgK?T3GYx&lFx;i$Uh)3~ty;AiN?w4Nw!YKhMx%$V<2@5ws zS$_Bf?^Ajv19QblxZx-(N&*UA+cza$u1q5bk+>PuuV*rp*cBg8oab(g5BQDy--r~@ zFte$nkv!n1z?eiKE#9GDyHT(xCVy-Z3g~dNZbwG?H(1j(>6H16n_ka0>IwbMtM4bB z@5@HJAL|K`N#9M^1@-Ew7nm7ea7Yy3hyvf5&c%o!>q=6DC^-{`Dj`EhjtY0E?+K&X$@og$i2=O-wM@WSb<&)oYTiv1{&2_HCmPHl6!Ly zuNO|MleaPRrNs@YG4f`zx%!z(x{(n)(PVkgTfaUu#C1yw)i z50Il6Ij6k}n^`*B=*7rZ+SW>YqpR?+R~RRb&mEV2pyo5ACZyO`Iqzn=K9>v#?h$Uf zua$pAaVYpNhRw@~I|mVkNSCT(vYrq3EWt4%IYRWcD(_W@K0Wu8-OjAVb#{X*RKCzM;q1FQQ(j zN3`CH^Y@MM&lS(!)~(a2wD+CY3Tff-OAZ5AN)vZm`PL6d7jPvts!`Th=;m4k2&rKHD>JTqTfDe~n^>8Ws5| zlS}u@phe2jur!DVs+d8+jmbpiXJ4LQ%LA`p^;Pkuq(t*hby*Km)B&sMn}e?ti}6+v zoJCg{YFF>GU5$6M-?~??ET3+ncq4wP?2IZ(iX!=TJ5GF0+%?jur_qx z0#|Nc3YOJdduZavR-6jbEU5qd*i5q#L8DDX88a}n_>B_fGd~OByrK#LPPuQ}2nDD4 zJ~|F<=2sRQT8~ZKV?ws%YVgE$2x29h5|q1E&N1~9Ve#cN^`3a7AH;hdWWSJb^O8xRbWkeXQ>+cxZe7K;L){Fsg*j}3crZ~2-wLQ|Wvl!vj*V7xV%@@}6pZ%S|(OeeW2=-H8J2Ctc?&go`n$({@Snf@JP;Lg0fStqVm z<<1vJ8dVukPU2n#KHGYUR$ahK$|fmlg{pcA(IXn9Qr3Blf^s;^}t60a@+fdx&*YxaFT; zWZptKqqbre&2g10?(4upOZ|MiAAH%2RG1C60q0V>&wPtHYJ}XxZrw|of#5Izjt$A zR$$P8HpTkXp@3C>3-dX-1AcrG_Eu~u+af) zr>>$oXXAi1`nfFKC_Jz7@pi($laearN&L}1<=UGtr>7;Xrbhqt=Y2wHj@w_lJWoa1 zd4-PJHUN0LarLYnf&v;xmlABs>_-&^zE09)$d&)hCR~8CS|+U;iQ`2U_G8U?Gk=qR z#czENk4fOQZMF&X+&ooKAzM2RCYU7m)F7I&dZC@&AID4ocn3VUJ^~ypV(W5b_^k`a zqL*;L!h2@tS(dE%A5~i);*IXMdR*-2tKK~~DG%$EBR<52!WdX$a?xjC0My`=wEMCV zmjpC2?zfxFuOGk<`QJP}NwDfYHW+wdw|f>~V|+h*S6ALE*Gss;o52ETaV^vVC2?YT z6?U~K7QJNY%4waow9@2wRY$V>8<9S?6|BEoEi?Z93My+>Ry(3(V0($kmhg!3TJH|* zm@ZMdBlT;d41v1-kb$%%BvbVyOkFpvdL7#23&2vsa!>J!=8Yp;&b$si|A~CRf;e`{ zPDK>~6qpX=#S^N|h1CqU@K@tG04AGxv>biVzxE>SCJx~{aGOmGZ#-$o&$H2m6+NOA zy$D?0wZ();;x^{@?hCQmd%`>1d()kFD9iC*#D!6$xLm5pbGs0z5jzRu-79MFhF-$dC}`p9-_!iOT;cr}8tdskm5^L$%7W z+{z!fB4zr8-*<-8oDvkxJHUNBJJLikQFi&@NExCL#zYl{JWC`*t+g|x+0rYZn8LcW2gxF^p_T)pkf4wP?&W#EyfN|K2%&+hxH3lOXqoY0 zmFiCr;rznV(zTg`4oR$!@bl+23ZRD!fLn<<(%WoT*_rY}XY`=?<)NygN(^D^(Qf{( z>**B36K+KLMQ0~2$wXtQt_9vW}|T>=l!zH@z1$Bf4B6 z+bZF|KHODWXKub}cr@8SrFNW2#`TbO^%oU-zy5eY0TNuBOtza7>ZE!td^qoWW_{Bd z*Efp#hU?(J?Dt<808nLkmS5_3%l&zfq&G~`nl(=)hRQq#buh%AKzU6iN-%+Pb;*n8 zFsMG;ox787jf_+67s)VMeekPy?AvLA+;JFuG@XiEMR`Br5QzW@a(`LR?;4>3R8|Hi-p3dkhR`^Fl+va{&!@=#`Ejf(NC} zH$b4^#k@~N(^K?iRfq7_O@au8q0Es8#_-5Zx-lRTTwa$mrHqdCSB;up zLYcfxzhsh_d(hw*hdFtg5#QYotH1+~0BJAr-I6hPnHrg=x2)sd=;e01MQ&%0GkAFMS@mIOLXIW34Pn^LVki3wL5L8flC@u#;i=n)XUR`#43pf(# z=^h^8* z%(dEe<_sO%Cu+}>qahyrez1BtSWOyUw&>J*OAONL=i(Y5&|+Yt{4$=-wb^y%(SK)1 zJbR82r<5&b+8gqO5*isE-6@VDD^iNFY^-7s3#y+~5!_S9ise&^75D*1jm`AAES4}3 zQ=2tHqmSnd#llHdb z*2Cl#wEug~rwFmQ7=uwe(WQaAJ ze0?rC^?J3C3ywr>l{)0gi64~KZwC=mG~+(4BCvA#_`Dmu9hOBk&7r2QbeTSG$?F#-1y)-`ynouTCIVc=xvlm=nX0G+!hu^8Zud zXjJ0ws8d>`_ from your fork. Before you send it, summarize your change in the ``[Unreleased]`` section of ``CHANGELOG.md`` and make sure develop is the destination branch. We also appreciate `squash commits `_ before pull requests are merged. -Merlin uses a rough approximation of the Git Flow branching model. The develop branch contains the latest contributions, and master is always tagged and points to the latest stable release. +Merlin uses a rough approximation of the Git Flow branching model. The develop branch contains the latest contributions, and main is always tagged and points to the latest stable release. If you're a contributor, try to test and run on develop. That's where all the magic is happening (and where we hope bugs stop). diff --git a/merlin/__init__.py b/merlin/__init__.py index 75513329f..b99002f88 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.8.3" +__version__ = "1.8.4" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 8acd698b4..2b3366693 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index c2e98f3a3..a2e1de2bd 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index f78f9c91a..da4ad9f74 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index f78f9c91a..da4ad9f74 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index e16c45498..c6357ef4b 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 4c87bb978..beb8a69aa 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -1,14 +1,14 @@ #!/usr/bin/env python ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index bcb298035..793a869b6 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -1,14 +1,14 @@ #!/usr/bin/env python ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 6d4511307..8f6d4f9bf 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index e23fb9081..5161821b2 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index f78f9c91a..da4ad9f74 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 7c12a70d4..c76eb0688 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 1c566a377..b5f9953bc 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 0737bd887..a3b63e0c3 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 2c71cee2d..e11721cae 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 47945b01b..f7c96c35e 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 4084d13c2..5a0430dc4 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 2d5141ae3..e237d741a 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 01c210a7f..ad1504754 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index f78f9c91a..da4ad9f74 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index 2ad038034..9d58acbd0 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index f78f9c91a..da4ad9f74 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index bbbad64d9..059e0ae9c 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index d1ce08b4e..31ab74c5d 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/workflows/feature_demo/scripts/pgen.py b/merlin/examples/workflows/feature_demo/scripts/pgen.py index 99b9b105d..6ffb3ed37 100644 --- a/merlin/examples/workflows/feature_demo/scripts/pgen.py +++ b/merlin/examples/workflows/feature_demo/scripts/pgen.py @@ -5,7 +5,7 @@ def get_custom_generator(env, **kwargs): p_gen = ParameterGenerator() params = { "X2": {"values": [1 / i for i in range(3, 6)], "label": "X2.%%"}, - "N_NEW": {"values": [2 ** i for i in range(1, 4)], "label": "N_NEW.%%"}, + "N_NEW": {"values": [2**i for i in range(1, 4)], "label": "N_NEW.%%"}, } for key, value in params.items(): diff --git a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py index bb5ba8a4d..99c3c3d6a 100644 --- a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py +++ b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py @@ -21,8 +21,8 @@ # launch n_samples * n_conc merlin workflow jobs submit_path: str = os.path.abspath(os.path.dirname(os.path.abspath(__file__))) -concurrencies: List[int] = [2 ** 0, 2 ** 1, 2 ** 2, 2 ** 3, 2 ** 4, 2 ** 5, 2 ** 6] -samples: List[int] = [10 ** 1, 10 ** 2, 10 ** 3, 10 ** 4, 10 ** 5] +concurrencies: List[int] = [2**0, 2**1, 2**2, 2**3, 2**4, 2**5, 2**6] +samples: List[int] = [10**1, 10**2, 10**3, 10**4, 10**5] nodes: List = [] c: int for c in concurrencies: diff --git a/merlin/examples/workflows/openfoam_wf/scripts/learn.py b/merlin/examples/workflows/openfoam_wf/scripts/learn.py index 1a4ba7c00..b7c40f6d3 100644 --- a/merlin/examples/workflows/openfoam_wf/scripts/learn.py +++ b/merlin/examples/workflows/openfoam_wf/scripts/learn.py @@ -26,7 +26,7 @@ U = outputs["arr_0"] enstrophy = outputs["arr_1"] -energy_byhand = np.sum(np.sum(U ** 2, axis=3), axis=2) / U.shape[2] / 2 +energy_byhand = np.sum(np.sum(U**2, axis=3), axis=2) / U.shape[2] / 2 enstrophy_all = np.sum(enstrophy, axis=2) X = np.load(inputs_dir + "/samples.npy") diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py index 1a4ba7c00..b7c40f6d3 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py +++ b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py @@ -26,7 +26,7 @@ U = outputs["arr_0"] enstrophy = outputs["arr_1"] -energy_byhand = np.sum(np.sum(U ** 2, axis=3), axis=2) / U.shape[2] / 2 +energy_byhand = np.sum(np.sum(U**2, axis=3), axis=2) / U.shape[2] / 2 enstrophy_all = np.sum(enstrophy, axis=2) X = np.load(inputs_dir + "/samples.npy") diff --git a/merlin/examples/workflows/optimization/scripts/test_functions.py b/merlin/examples/workflows/optimization/scripts/test_functions.py index d6f391a0a..ab1a3ae4a 100644 --- a/merlin/examples/workflows/optimization/scripts/test_functions.py +++ b/merlin/examples/workflows/optimization/scripts/test_functions.py @@ -17,14 +17,14 @@ def rosenbrock(X): def rastrigin(X, A=10): first_term = A * len(inputs) - return first_term + sum([(x ** 2 - A * np.cos(2 * math.pi * x)) for x in X]) + return first_term + sum([(x**2 - A * np.cos(2 * math.pi * x)) for x in X]) def ackley(X): firstSum = 0.0 secondSum = 0.0 for x in X: - firstSum += x ** 2.0 + firstSum += x**2.0 secondSum += np.cos(2.0 * np.pi * x) n = float(len(X)) @@ -32,7 +32,7 @@ def ackley(X): def griewank(X): - term_1 = (1.0 / 4000.0) * sum(X ** 2) + term_1 = (1.0 / 4000.0) * sum(X**2) term_2 = 1.0 for i, x in enumerate(X): term_2 *= np.cos(x) / np.sqrt(i + 1) diff --git a/merlin/examples/workflows/remote_feature_demo/scripts/pgen.py b/merlin/examples/workflows/remote_feature_demo/scripts/pgen.py index ad13cc50b..5332d5b93 100644 --- a/merlin/examples/workflows/remote_feature_demo/scripts/pgen.py +++ b/merlin/examples/workflows/remote_feature_demo/scripts/pgen.py @@ -10,7 +10,7 @@ def get_custom_generator(env, **kwargs) -> ParameterGenerator: p_gen: ParameterGenerator = ParameterGenerator() params: Dict[str, Union[List[float], str]] = { "X2": {"values": [1 / i for i in range(3, 6)], "label": "X2.%%"}, - "N_NEW": {"values": [2 ** i for i in range(1, 4)], "label": "N_NEW.%%"}, + "N_NEW": {"values": [2**i for i in range(1, 4)], "label": "N_NEW.%%"}, } key: str value: Union[List[float], str] diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 8fac45395..20a32755f 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index cbb285618..4e3969c2c 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -1,14 +1,14 @@ """This module handles setting up the extensive logging system in Merlin.""" ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 8a7b5d302..9f6c90a73 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -1,14 +1,14 @@ """The top level main function for invoking Merlin.""" ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index f9885b242..94c57445e 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 85e298510..c559883fa 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index f78f9c91a..da4ad9f74 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index cd4bf238c..3a344a98e 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index ef3f6da3e..74dc0be46 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 6d4ec0329..30a71b25c 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index eb77c3157..122b8f002 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index f78f9c91a..da4ad9f74 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index dec4cfac4..0be2e8739 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index eef29aa63..b7045bc46 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 31ef0f08b..6821d486e 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 07c4622b9..63a11d0c7 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index af331c3cf..649610ab6 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 1cf1e2adc..d98a31419 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index b35ef2b77..657349fed 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # diff --git a/requirements/dev.txt b/requirements/dev.txt index c66854a61..9321694f8 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,4 +1,5 @@ # Development dependencies. +build black dep-license flake8 diff --git a/setup.py b/setup.py index e9ed37693..68e4b8ed7 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # @@ -100,6 +100,7 @@ def extras_require(): "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], keywords="machine learning workflow", url="https://github.com/LLNL/merlin", diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index f92f20354..c7d3745b1 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -1,12 +1,12 @@ ############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.3. +# This file is part of Merlin, Version: 1.8.4. # # For details, see https://github.com/LLNL/merlin. # From 681d1754ac400c4ecb06375f5ef20378c71e7ebd Mon Sep 17 00:00:00 2001 From: Joe Koning Date: Mon, 7 Mar 2022 13:22:07 -0800 Subject: [PATCH 021/126] Add updates for lgtm CI security site (#357) * Update code to remove LGTM Errors and Warnings and implement Recommendations. * Change BaseException to Exception. * Add lgtm config file. * Changes for flake8. * Add TypeError yo yam read. * Add TypeError to yaml read. * Just return when successful on the yaml read. * Fix typo. * Add merlin/examples to lgtm exclude list as well. * Add ssl comment. * Fix typo. --- CHANGELOG.md | 7 ++++ lgtm.yml | 25 ++++++++++++ merlin/common/util_sampling.py | 2 +- merlin/config/broker.py | 1 - merlin/config/configfile.py | 3 +- .../feature_demo/scripts/hello_world.py | 3 +- .../workflows/flux/scripts/flux_info.py | 40 +++++++++---------- .../workflows/flux/scripts/make_samples.py | 3 +- .../hpc_demo/cumulative_sample_processor.py | 3 +- .../workflows/hpc_demo/faker_sample.py | 3 +- .../workflows/hpc_demo/sample_collector.py | 3 +- .../workflows/hpc_demo/sample_processor.py | 3 +- .../cumulative_sample_processor.py | 3 +- .../workflows/iterative_demo/faker_sample.py | 3 +- .../iterative_demo/sample_collector.py | 3 +- .../iterative_demo/sample_processor.py | 3 +- .../workflows/lsf/scripts/make_samples.py | 3 +- .../null_spec/scripts/read_output.py | 17 ++++---- .../null_spec/scripts/read_output_chain.py | 12 +++--- .../workflows/openfoam_wf/scripts/learn.py | 2 - .../openfoam_wf_no_docker/scripts/learn.py | 2 - .../optimization/scripts/visualizer.py | 1 - .../scripts/hello_world.py | 3 +- .../workflows/restart/scripts/make_samples.py | 3 +- .../restart_delay/scripts/make_samples.py | 3 +- .../workflows/slurm/scripts/make_samples.py | 3 +- merlin/exceptions/__init__.py | 3 +- merlin/main.py | 4 +- merlin/merlin_templates.py | 3 +- merlin/spec/specification.py | 2 +- merlin/study/celeryadapter.py | 4 +- merlin/utils.py | 17 ++++---- tests/integration/run_tests.py | 5 +-- 33 files changed, 102 insertions(+), 93 deletions(-) create mode 100644 lgtm.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index b44333f52..d7b0d980d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Added +- Code updates to satisfy lgtm CI security checker + +### Fixed +- A bug in the ssl config was not returning the proper values + ## [1.8.4] ### Added - Auto-release of pypi packages diff --git a/lgtm.yml b/lgtm.yml new file mode 100644 index 000000000..e3f53c87d --- /dev/null +++ b/lgtm.yml @@ -0,0 +1,25 @@ +########################################################################################## +# Customize file classifications. # +# Results from files under any classifier will be excluded from LGTM # +# statistics. # +########################################################################################## + +########################################################################################## +# Use the `path_classifiers` block to define changes to the default classification of # +# files. # +########################################################################################## + +path_classifiers: + test: + # Classify all files in the top-level directories tests/ as test code. + - exclude: + - tests + - merlin/examples + +######################################################################################### +# Use the `queries` block to change the default display of query results. # +######################################################################################### + +queries: + # Specifically hide the results of clear-text-logging-sensitive-data + - exclude: py/clear-text-logging-sensitive-data diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index e11721cae..638795e3b 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -90,7 +90,7 @@ def scale_samples(samples_norm, limits, limits_norm=(0, 1), do_log=False): if not hasattr(do_log, "__iter__"): do_log = ndims * [do_log] logs = np.asarray(do_log) - lims_norm = np.array([limits_norm for i in logs]) + lims_norm = np.array([limits_norm] * len(logs)) _lims = [] for limit, log in zip(limits, logs): if log: diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 5a0430dc4..8cb5312f0 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -219,7 +219,6 @@ def get_connection_string(include_password=True): raise ValueError(f"Error: {broker} is not a supported broker.") else: return _sort_valid_broker(broker, config_path, include_password) - return None def _sort_valid_broker(broker, config_path, include_password): diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index e237d741a..f7430ac68 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -298,7 +298,8 @@ def merge_sslmap(server_ssl: Dict[str, Union[str, ssl.VerifyMode]], ssl_map: Dic new_server_ssl[ssl_map[k]] = server_ssl[k] else: new_server_ssl[k] = server_ssl[k] - server_ssl = new_server_ssl + + return new_server_ssl app_config: Dict = get_config(None) diff --git a/merlin/examples/workflows/feature_demo/scripts/hello_world.py b/merlin/examples/workflows/feature_demo/scripts/hello_world.py index 9d029bbf0..ab14bedf4 100644 --- a/merlin/examples/workflows/feature_demo/scripts/hello_world.py +++ b/merlin/examples/workflows/feature_demo/scripts/hello_world.py @@ -1,6 +1,5 @@ import argparse import json -import sys def process_args(args): @@ -32,4 +31,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/flux/scripts/flux_info.py b/merlin/examples/workflows/flux/scripts/flux_info.py index cc3114f04..d4599c652 100755 --- a/merlin/examples/workflows/flux/scripts/flux_info.py +++ b/merlin/examples/workflows/flux/scripts/flux_info.py @@ -39,34 +39,32 @@ kvs.get(f, "lwj") for d in kvs.walk("lwj", flux_handle=f): - try: - # print(type(d)) - fdir = "lwj.{0}".format(d[0]) + # print(type(d)) + fdir = "lwj.{0}".format(d[0]) - qcreate = "{0}.create-time".format(fdir) - create_time = kvs.get(f, qcreate) + qcreate = "{0}.create-time".format(fdir) + create_time = kvs.get(f, qcreate) - qstart = "{0}.starting-time".format(fdir) - start_time = kvs.get(f, qstart) + qstart = "{0}.starting-time".format(fdir) + start_time = kvs.get(f, qstart) - qrun = "{0}.running-time".format(fdir) - start_time = kvs.get(f, qrun) + qrun = "{0}.running-time".format(fdir) + start_time = kvs.get(f, qrun) - qcomplete = "{0}.complete-time".format(fdir) - complete_time = kvs.get(f, qcomplete) + qcomplete = "{0}.complete-time".format(fdir) + complete_time = kvs.get(f, qcomplete) - qcompleting = "{0}.completing-time".format(fdir) - completing_time = kvs.get(f, qcompleting) + qcompleting = "{0}.completing-time".format(fdir) + completing_time = kvs.get(f, qcompleting) - qwall = "{0}.walltime".format(fdir) - wall_time = kvs.get(f, qwall) + qwall = "{0}.walltime".format(fdir) + wall_time = kvs.get(f, qwall) - print( - f"Job {d[0]}: create: {create_time} start {start_time} run {start_time} completing {completing_time} complete {complete_time} wall {wall_time}" - ) - except BaseException: - pass -except BaseException: + print( + f"Job {d[0]}: create: {create_time} start {start_time} run {start_time} completing {completing_time} complete {complete_time} wall {wall_time}" + ) + +except KeyError: top_dir = "job" def get_data_dict(key: str) -> Dict: diff --git a/merlin/examples/workflows/flux/scripts/make_samples.py b/merlin/examples/workflows/flux/scripts/make_samples.py index 913a94062..e6c807bc9 100644 --- a/merlin/examples/workflows/flux/scripts/make_samples.py +++ b/merlin/examples/workflows/flux/scripts/make_samples.py @@ -1,6 +1,5 @@ import argparse import ast -import sys import numpy as np @@ -58,4 +57,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py b/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py index ea84f7d6f..7d06ab594 100644 --- a/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py +++ b/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py @@ -1,6 +1,5 @@ import argparse import os -import sys from concurrent.futures import ProcessPoolExecutor import matplotlib.pyplot as plt @@ -98,4 +97,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/hpc_demo/faker_sample.py b/merlin/examples/workflows/hpc_demo/faker_sample.py index d6f7020e8..ee8bf2f5c 100644 --- a/merlin/examples/workflows/hpc_demo/faker_sample.py +++ b/merlin/examples/workflows/hpc_demo/faker_sample.py @@ -1,5 +1,4 @@ import argparse -import sys from faker import Faker @@ -38,4 +37,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/hpc_demo/sample_collector.py b/merlin/examples/workflows/hpc_demo/sample_collector.py index cfaac6e17..f62111e8e 100644 --- a/merlin/examples/workflows/hpc_demo/sample_collector.py +++ b/merlin/examples/workflows/hpc_demo/sample_collector.py @@ -1,6 +1,5 @@ import argparse import os -import sys from concurrent.futures import ProcessPoolExecutor @@ -46,4 +45,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/hpc_demo/sample_processor.py b/merlin/examples/workflows/hpc_demo/sample_processor.py index ed4dd596b..9ec0951e9 100644 --- a/merlin/examples/workflows/hpc_demo/sample_processor.py +++ b/merlin/examples/workflows/hpc_demo/sample_processor.py @@ -1,7 +1,6 @@ import argparse import os import pathlib -import sys import pandas as pd @@ -51,4 +50,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py b/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py index ea84f7d6f..7d06ab594 100644 --- a/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py +++ b/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py @@ -1,6 +1,5 @@ import argparse import os -import sys from concurrent.futures import ProcessPoolExecutor import matplotlib.pyplot as plt @@ -98,4 +97,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/iterative_demo/faker_sample.py b/merlin/examples/workflows/iterative_demo/faker_sample.py index d6f7020e8..ee8bf2f5c 100644 --- a/merlin/examples/workflows/iterative_demo/faker_sample.py +++ b/merlin/examples/workflows/iterative_demo/faker_sample.py @@ -1,5 +1,4 @@ import argparse -import sys from faker import Faker @@ -38,4 +37,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/iterative_demo/sample_collector.py b/merlin/examples/workflows/iterative_demo/sample_collector.py index cfaac6e17..f62111e8e 100644 --- a/merlin/examples/workflows/iterative_demo/sample_collector.py +++ b/merlin/examples/workflows/iterative_demo/sample_collector.py @@ -1,6 +1,5 @@ import argparse import os -import sys from concurrent.futures import ProcessPoolExecutor @@ -46,4 +45,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/iterative_demo/sample_processor.py b/merlin/examples/workflows/iterative_demo/sample_processor.py index ed4dd596b..9ec0951e9 100644 --- a/merlin/examples/workflows/iterative_demo/sample_processor.py +++ b/merlin/examples/workflows/iterative_demo/sample_processor.py @@ -1,7 +1,6 @@ import argparse import os import pathlib -import sys import pandas as pd @@ -51,4 +50,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/lsf/scripts/make_samples.py b/merlin/examples/workflows/lsf/scripts/make_samples.py index 913a94062..e6c807bc9 100644 --- a/merlin/examples/workflows/lsf/scripts/make_samples.py +++ b/merlin/examples/workflows/lsf/scripts/make_samples.py @@ -1,6 +1,5 @@ import argparse import ast -import sys import numpy as np @@ -58,4 +57,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/null_spec/scripts/read_output.py b/merlin/examples/workflows/null_spec/scripts/read_output.py index bc238d9d2..7c0f8017e 100644 --- a/merlin/examples/workflows/null_spec/scripts/read_output.py +++ b/merlin/examples/workflows/null_spec/scripts/read_output.py @@ -41,7 +41,8 @@ def single_task_times(): match = matches.group(0) match = float(match.strip("s:")) task_durations.append(match) - except BaseException: + except Exception as e: + print(f"single_task_times Exception= {e}\n") continue print(str(task_durations)) @@ -58,9 +59,9 @@ def merlin_run_time(): total += result try: print(f"c{args.c}_s{args.s} merlin run : " + str(result)) - except BaseException: + except Exception as e: result = None - print(f"c{args.c}_s{args.s} merlin run : ERROR -- result={result}, args.errfile={args.errfile}") + print(f"c{args.c}_s{args.s} merlin run : ERROR -- result={result}, args.errfile={args.errfile}\n{e}") def start_verify_time(): @@ -74,12 +75,12 @@ def start_verify_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception as e: continue try: print(f"c{args.c}_s{args.s} start verify : " + str(all_timestamps[0])) - except BaseException: - print(f"c{args.c}_s{args.s} start verify : ERROR") + except Exception as e: + print(f"c{args.c}_s{args.s} start verify : ERROR\n{e}") def start_run_workers_time(): @@ -93,7 +94,7 @@ def start_run_workers_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception: continue earliest = min(all_timestamps) print(f"c{args.c}_s{args.s} start run-workers : " + str(earliest)) @@ -110,7 +111,7 @@ def start_sample1_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception: continue earliest = min(all_timestamps) print(f"c{args.c}_s{args.s} start samp1 : " + str(earliest)) diff --git a/merlin/examples/workflows/null_spec/scripts/read_output_chain.py b/merlin/examples/workflows/null_spec/scripts/read_output_chain.py index f9fdfd521..ae9fb11a5 100644 --- a/merlin/examples/workflows/null_spec/scripts/read_output_chain.py +++ b/merlin/examples/workflows/null_spec/scripts/read_output_chain.py @@ -53,7 +53,7 @@ def single_task_times(): match = matches.group(0) match = float(match.strip("s:")) task_durations.append(match) - except BaseException: + except Exception: print(f"c{filled_c}_s{k} task times : ERROR") continue @@ -71,7 +71,7 @@ def merlin_run_time(): total += result try: print(f"c{filled_c} merlin run : " + str(result)) - except BaseException: + except Exception: result = None print(f"c{filled_c} merlin run : ERROR -- result={result}, args.errfile={args.errfile}") @@ -87,12 +87,12 @@ def start_verify_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception: print(f"c{filled_c}_s{k} start verify : ERROR") continue try: print(f"c{filled_c}_s{k} start verify : " + str(all_timestamps[0])) - except BaseException: + except Exception: print(f"c{filled_c}_s{k} start verify : ERROR") @@ -107,7 +107,7 @@ def start_run_workers_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception: continue earliest = min(all_timestamps) print(f"c{filled_c}_s{k} start run-workers : " + str(earliest)) @@ -124,7 +124,7 @@ def start_sample1_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception: print(f"c{filled_c}_s{k} start samp1 : ERROR") continue earliest = min(all_timestamps) diff --git a/merlin/examples/workflows/openfoam_wf/scripts/learn.py b/merlin/examples/workflows/openfoam_wf/scripts/learn.py index b7c40f6d3..96124c46f 100644 --- a/merlin/examples/workflows/openfoam_wf/scripts/learn.py +++ b/merlin/examples/workflows/openfoam_wf/scripts/learn.py @@ -103,8 +103,6 @@ ax[0][1].set_ylim([y_min, y_max]) -y_pred_all = regr.predict(X) -input_enstrophy = ax[1][1].scatter(X[:, 0], 10 ** y[:, 1], s=100, edgecolors="black") ax[1][1].set_xlabel(r"Lidspeed ($\frac{m}{s}$)", fontsize=fontsize) ax[1][1].set_ylabel(r"$Energy$", fontsize=fontsize) ax[1][1].set_title("Average Energy Variation with Lidspeed") diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py index b7c40f6d3..96124c46f 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py +++ b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py @@ -103,8 +103,6 @@ ax[0][1].set_ylim([y_min, y_max]) -y_pred_all = regr.predict(X) -input_enstrophy = ax[1][1].scatter(X[:, 0], 10 ** y[:, 1], s=100, edgecolors="black") ax[1][1].set_xlabel(r"Lidspeed ($\frac{m}{s}$)", fontsize=fontsize) ax[1][1].set_ylabel(r"$Energy$", fontsize=fontsize) ax[1][1].set_title("Average Energy Variation with Lidspeed") diff --git a/merlin/examples/workflows/optimization/scripts/visualizer.py b/merlin/examples/workflows/optimization/scripts/visualizer.py index f41885bf9..c373c9cd1 100644 --- a/merlin/examples/workflows/optimization/scripts/visualizer.py +++ b/merlin/examples/workflows/optimization/scripts/visualizer.py @@ -9,7 +9,6 @@ import matplotlib.pyplot as plt import numpy as np -from mpl_toolkits.mplot3d import Axes3D plt.style.use("seaborn-white") diff --git a/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py b/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py index 0e2b624d6..232c43e86 100644 --- a/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py +++ b/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py @@ -1,6 +1,5 @@ import argparse import json -import sys from typing import Dict @@ -42,4 +41,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/restart/scripts/make_samples.py b/merlin/examples/workflows/restart/scripts/make_samples.py index 913a94062..e6c807bc9 100644 --- a/merlin/examples/workflows/restart/scripts/make_samples.py +++ b/merlin/examples/workflows/restart/scripts/make_samples.py @@ -1,6 +1,5 @@ import argparse import ast -import sys import numpy as np @@ -58,4 +57,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/restart_delay/scripts/make_samples.py b/merlin/examples/workflows/restart_delay/scripts/make_samples.py index 913a94062..e6c807bc9 100644 --- a/merlin/examples/workflows/restart_delay/scripts/make_samples.py +++ b/merlin/examples/workflows/restart_delay/scripts/make_samples.py @@ -1,6 +1,5 @@ import argparse import ast -import sys import numpy as np @@ -58,4 +57,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/slurm/scripts/make_samples.py b/merlin/examples/workflows/slurm/scripts/make_samples.py index 913a94062..e6c807bc9 100644 --- a/merlin/examples/workflows/slurm/scripts/make_samples.py +++ b/merlin/examples/workflows/slurm/scripts/make_samples.py @@ -1,6 +1,5 @@ import argparse import ast -import sys import numpy as np @@ -58,4 +57,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 20a32755f..e90fd6afa 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -36,6 +36,7 @@ "RetryException", "SoftFailException", "HardFailException", + "InvalidChainException", "RestartException", ) @@ -76,7 +77,7 @@ class InvalidChainException(Exception): """ def __init__(self): - super(HardFailException, self).__init__() + super(InvalidChainException, self).__init__() class RestartException(Exception): diff --git a/merlin/main.py b/merlin/main.py index 9f6c90a73..dec90a034 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -140,7 +140,7 @@ def parse_override_vars( val = int(val) result[key] = val - except BaseException as excpt: + except Exception as excpt: raise ValueError( f"{excpt} Bad '--vars' formatting on command line. See 'merlin run --help' for an example." ) from excpt @@ -756,4 +756,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 94c57445e..67a9d455f 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -33,7 +33,6 @@ """ import argparse import logging -import sys from merlin.ascii_art import banner_small from merlin.log_formatter import setup_logging @@ -65,4 +64,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 122b8f002..7dda416b1 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -228,7 +228,7 @@ def dump(self): result = result.replace("\n\n\n", "\n\n") try: yaml.safe_load(result) - except BaseException as e: + except Exception as e: raise ValueError(f"Error parsing provenance spec:\n{e}") return result diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index b7045bc46..67598be87 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -155,8 +155,8 @@ def query_celery_queues(queues): try: name, jobs, consumers = channel.queue_declare(queue=queue, passive=True) found_queues.append((name, jobs, consumers)) - except BaseException: - LOG.warning(f"Cannot find queue {queue} on server.") + except Exception as e: + LOG.warning(f"Cannot find queue {queue} on server.{e}") finally: connection.close() return found_queues diff --git a/merlin/utils.py b/merlin/utils.py index 657349fed..0cd06371e 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -37,7 +37,7 @@ import re import socket import subprocess -from contextlib import contextmanager, suppress +from contextlib import contextmanager from copy import deepcopy from datetime import timedelta from types import SimpleNamespace @@ -205,15 +205,14 @@ def get_yaml_var(entry, var, default): :param `var`: a yaml key :param `default`: default value in the absence of data """ - ret = default - if isinstance(entry, dict): - with suppress(KeyError): - ret = entry[var] - else: - with suppress(AttributeError): - ret = getattr(entry, var) - return ret + try: + return entry[var] + except (TypeError, KeyError): + try: + return getattr(entry, var) + except AttributeError: + return default def load_array_file(filename, ndmin=2): diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index c7d3745b1..3a8038d06 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -34,7 +34,6 @@ """ import argparse import shutil -import sys import time from contextlib import suppress from subprocess import PIPE, Popen @@ -155,7 +154,7 @@ def run_tests(args, tests): continue try: passed, info = run_single_test(test_name, test, test_label) - except BaseException as e: + except Exception as e: print(e) passed = False info = None @@ -215,4 +214,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() From 8159975f47b757d9b68563bf89747956b319d15a Mon Sep 17 00:00:00 2001 From: "Joseph M. Koning" Date: Mon, 7 Mar 2022 13:29:28 -0800 Subject: [PATCH 022/126] Update version to 1.8.5. --- CHANGELOG.md | 2 +- merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- 43 files changed, 44 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7b0d980d..b86f399ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.8.5] ### Added - Code updates to satisfy lgtm CI security checker diff --git a/merlin/__init__.py b/merlin/__init__.py index b99002f88..fa68134d6 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.8.4" +__version__ = "1.8.5" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 2b3366693..de04bfc78 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index a2e1de2bd..3b8769bb5 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index da4ad9f74..13db3dccc 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index da4ad9f74..13db3dccc 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index c6357ef4b..fa4a5f7c1 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index beb8a69aa..601af41c7 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 793a869b6..3c2d0dba0 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 8f6d4f9bf..cb2a221d8 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 5161821b2..c3af9cdb6 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index da4ad9f74..13db3dccc 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index c76eb0688..af29d394f 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index b5f9953bc..8adc75228 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index a3b63e0c3..e3a7ed525 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 638795e3b..bd2795915 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index f7c96c35e..a78a937d4 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 8cb5312f0..b73330f6f 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index f7430ac68..cc3dedd13 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index ad1504754..a5effe13e 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index da4ad9f74..13db3dccc 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index 9d58acbd0..1cd9af01e 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index da4ad9f74..13db3dccc 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 059e0ae9c..686f04013 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 31ab74c5d..d0cf1acb9 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index e90fd6afa..20970fe7a 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 4e3969c2c..1f6befa88 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index dec90a034..6e491ba1a 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 67a9d455f..a1e80fea1 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index c559883fa..90aa9db38 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index da4ad9f74..13db3dccc 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 3a344a98e..9af27d456 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 74dc0be46..34b2113cd 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 30a71b25c..0be5b21b9 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 7dda416b1..6bb612cb4 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index da4ad9f74..13db3dccc 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 0be2e8739..69c68856a 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 67598be87..f78044d41 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 6821d486e..01f7aae91 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 63a11d0c7..928e246e1 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 649610ab6..5b03bf2cb 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index d98a31419..efa43dd9f 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index 0cd06371e..6529e8fbb 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # From 14e5a171fce42fb5fe6058fe336817a0848b5614 Mon Sep 17 00:00:00 2001 From: Joe Koning Date: Mon, 7 Mar 2022 13:40:30 -0800 Subject: [PATCH 023/126] Update for v1.8.5 (#358) * Add updates for lgtm CI security site (#357) * Update version to 1.8.5. Co-authored-by: Joe Koning --- CHANGELOG.md | 7 ++++ lgtm.yml | 25 ++++++++++++ merlin/__init__.py | 4 +- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- .../security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 4 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 3 +- merlin/config/configfile.py | 5 ++- merlin/config/results_backend.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- .../feature_demo/scripts/hello_world.py | 3 +- .../workflows/flux/scripts/flux_info.py | 40 +++++++++---------- .../workflows/flux/scripts/make_samples.py | 3 +- .../hpc_demo/cumulative_sample_processor.py | 3 +- .../workflows/hpc_demo/faker_sample.py | 3 +- .../workflows/hpc_demo/sample_collector.py | 3 +- .../workflows/hpc_demo/sample_processor.py | 3 +- .../cumulative_sample_processor.py | 3 +- .../workflows/iterative_demo/faker_sample.py | 3 +- .../iterative_demo/sample_collector.py | 3 +- .../iterative_demo/sample_processor.py | 3 +- .../workflows/lsf/scripts/make_samples.py | 3 +- .../null_spec/scripts/read_output.py | 17 ++++---- .../null_spec/scripts/read_output_chain.py | 12 +++--- .../workflows/openfoam_wf/scripts/learn.py | 2 - .../openfoam_wf_no_docker/scripts/learn.py | 2 - .../optimization/scripts/visualizer.py | 1 - .../scripts/hello_world.py | 3 +- .../workflows/restart/scripts/make_samples.py | 3 +- .../restart_delay/scripts/make_samples.py | 3 +- .../workflows/slurm/scripts/make_samples.py | 3 +- merlin/exceptions/__init__.py | 5 ++- merlin/log_formatter.py | 2 +- merlin/main.py | 6 +-- merlin/merlin_templates.py | 5 +-- merlin/router.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/specification.py | 4 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 6 +-- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 19 +++++---- tests/integration/run_tests.py | 5 +-- 66 files changed, 145 insertions(+), 136 deletions(-) create mode 100644 lgtm.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index b44333f52..b86f399ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.8.5] +### Added +- Code updates to satisfy lgtm CI security checker + +### Fixed +- A bug in the ssl config was not returning the proper values + ## [1.8.4] ### Added - Auto-release of pypi packages diff --git a/lgtm.yml b/lgtm.yml new file mode 100644 index 000000000..e3f53c87d --- /dev/null +++ b/lgtm.yml @@ -0,0 +1,25 @@ +########################################################################################## +# Customize file classifications. # +# Results from files under any classifier will be excluded from LGTM # +# statistics. # +########################################################################################## + +########################################################################################## +# Use the `path_classifiers` block to define changes to the default classification of # +# files. # +########################################################################################## + +path_classifiers: + test: + # Classify all files in the top-level directories tests/ as test code. + - exclude: + - tests + - merlin/examples + +######################################################################################### +# Use the `queries` block to change the default display of query results. # +######################################################################################### + +queries: + # Specifically hide the results of clear-text-logging-sensitive-data + - exclude: py/clear-text-logging-sensitive-data diff --git a/merlin/__init__.py b/merlin/__init__.py index b99002f88..fa68134d6 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.8.4" +__version__ = "1.8.5" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 2b3366693..de04bfc78 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index a2e1de2bd..3b8769bb5 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index da4ad9f74..13db3dccc 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index da4ad9f74..13db3dccc 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index c6357ef4b..fa4a5f7c1 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index beb8a69aa..601af41c7 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 793a869b6..3c2d0dba0 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 8f6d4f9bf..cb2a221d8 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 5161821b2..c3af9cdb6 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index da4ad9f74..13db3dccc 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index c76eb0688..af29d394f 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index b5f9953bc..8adc75228 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index a3b63e0c3..e3a7ed525 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index e11721cae..bd2795915 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # @@ -90,7 +90,7 @@ def scale_samples(samples_norm, limits, limits_norm=(0, 1), do_log=False): if not hasattr(do_log, "__iter__"): do_log = ndims * [do_log] logs = np.asarray(do_log) - lims_norm = np.array([limits_norm for i in logs]) + lims_norm = np.array([limits_norm] * len(logs)) _lims = [] for limit, log in zip(limits, logs): if log: diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index f7c96c35e..a78a937d4 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 5a0430dc4..b73330f6f 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # @@ -219,7 +219,6 @@ def get_connection_string(include_password=True): raise ValueError(f"Error: {broker} is not a supported broker.") else: return _sort_valid_broker(broker, config_path, include_password) - return None def _sort_valid_broker(broker, config_path, include_password): diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index e237d741a..cc3dedd13 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # @@ -298,7 +298,8 @@ def merge_sslmap(server_ssl: Dict[str, Union[str, ssl.VerifyMode]], ssl_map: Dic new_server_ssl[ssl_map[k]] = server_ssl[k] else: new_server_ssl[k] = server_ssl[k] - server_ssl = new_server_ssl + + return new_server_ssl app_config: Dict = get_config(None) diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index ad1504754..a5effe13e 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index da4ad9f74..13db3dccc 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index 9d58acbd0..1cd9af01e 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index da4ad9f74..13db3dccc 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 059e0ae9c..686f04013 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 31ab74c5d..d0cf1acb9 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/workflows/feature_demo/scripts/hello_world.py b/merlin/examples/workflows/feature_demo/scripts/hello_world.py index 9d029bbf0..ab14bedf4 100644 --- a/merlin/examples/workflows/feature_demo/scripts/hello_world.py +++ b/merlin/examples/workflows/feature_demo/scripts/hello_world.py @@ -1,6 +1,5 @@ import argparse import json -import sys def process_args(args): @@ -32,4 +31,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/flux/scripts/flux_info.py b/merlin/examples/workflows/flux/scripts/flux_info.py index cc3114f04..d4599c652 100755 --- a/merlin/examples/workflows/flux/scripts/flux_info.py +++ b/merlin/examples/workflows/flux/scripts/flux_info.py @@ -39,34 +39,32 @@ kvs.get(f, "lwj") for d in kvs.walk("lwj", flux_handle=f): - try: - # print(type(d)) - fdir = "lwj.{0}".format(d[0]) + # print(type(d)) + fdir = "lwj.{0}".format(d[0]) - qcreate = "{0}.create-time".format(fdir) - create_time = kvs.get(f, qcreate) + qcreate = "{0}.create-time".format(fdir) + create_time = kvs.get(f, qcreate) - qstart = "{0}.starting-time".format(fdir) - start_time = kvs.get(f, qstart) + qstart = "{0}.starting-time".format(fdir) + start_time = kvs.get(f, qstart) - qrun = "{0}.running-time".format(fdir) - start_time = kvs.get(f, qrun) + qrun = "{0}.running-time".format(fdir) + start_time = kvs.get(f, qrun) - qcomplete = "{0}.complete-time".format(fdir) - complete_time = kvs.get(f, qcomplete) + qcomplete = "{0}.complete-time".format(fdir) + complete_time = kvs.get(f, qcomplete) - qcompleting = "{0}.completing-time".format(fdir) - completing_time = kvs.get(f, qcompleting) + qcompleting = "{0}.completing-time".format(fdir) + completing_time = kvs.get(f, qcompleting) - qwall = "{0}.walltime".format(fdir) - wall_time = kvs.get(f, qwall) + qwall = "{0}.walltime".format(fdir) + wall_time = kvs.get(f, qwall) - print( - f"Job {d[0]}: create: {create_time} start {start_time} run {start_time} completing {completing_time} complete {complete_time} wall {wall_time}" - ) - except BaseException: - pass -except BaseException: + print( + f"Job {d[0]}: create: {create_time} start {start_time} run {start_time} completing {completing_time} complete {complete_time} wall {wall_time}" + ) + +except KeyError: top_dir = "job" def get_data_dict(key: str) -> Dict: diff --git a/merlin/examples/workflows/flux/scripts/make_samples.py b/merlin/examples/workflows/flux/scripts/make_samples.py index 913a94062..e6c807bc9 100644 --- a/merlin/examples/workflows/flux/scripts/make_samples.py +++ b/merlin/examples/workflows/flux/scripts/make_samples.py @@ -1,6 +1,5 @@ import argparse import ast -import sys import numpy as np @@ -58,4 +57,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py b/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py index ea84f7d6f..7d06ab594 100644 --- a/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py +++ b/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py @@ -1,6 +1,5 @@ import argparse import os -import sys from concurrent.futures import ProcessPoolExecutor import matplotlib.pyplot as plt @@ -98,4 +97,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/hpc_demo/faker_sample.py b/merlin/examples/workflows/hpc_demo/faker_sample.py index d6f7020e8..ee8bf2f5c 100644 --- a/merlin/examples/workflows/hpc_demo/faker_sample.py +++ b/merlin/examples/workflows/hpc_demo/faker_sample.py @@ -1,5 +1,4 @@ import argparse -import sys from faker import Faker @@ -38,4 +37,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/hpc_demo/sample_collector.py b/merlin/examples/workflows/hpc_demo/sample_collector.py index cfaac6e17..f62111e8e 100644 --- a/merlin/examples/workflows/hpc_demo/sample_collector.py +++ b/merlin/examples/workflows/hpc_demo/sample_collector.py @@ -1,6 +1,5 @@ import argparse import os -import sys from concurrent.futures import ProcessPoolExecutor @@ -46,4 +45,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/hpc_demo/sample_processor.py b/merlin/examples/workflows/hpc_demo/sample_processor.py index ed4dd596b..9ec0951e9 100644 --- a/merlin/examples/workflows/hpc_demo/sample_processor.py +++ b/merlin/examples/workflows/hpc_demo/sample_processor.py @@ -1,7 +1,6 @@ import argparse import os import pathlib -import sys import pandas as pd @@ -51,4 +50,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py b/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py index ea84f7d6f..7d06ab594 100644 --- a/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py +++ b/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py @@ -1,6 +1,5 @@ import argparse import os -import sys from concurrent.futures import ProcessPoolExecutor import matplotlib.pyplot as plt @@ -98,4 +97,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/iterative_demo/faker_sample.py b/merlin/examples/workflows/iterative_demo/faker_sample.py index d6f7020e8..ee8bf2f5c 100644 --- a/merlin/examples/workflows/iterative_demo/faker_sample.py +++ b/merlin/examples/workflows/iterative_demo/faker_sample.py @@ -1,5 +1,4 @@ import argparse -import sys from faker import Faker @@ -38,4 +37,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/iterative_demo/sample_collector.py b/merlin/examples/workflows/iterative_demo/sample_collector.py index cfaac6e17..f62111e8e 100644 --- a/merlin/examples/workflows/iterative_demo/sample_collector.py +++ b/merlin/examples/workflows/iterative_demo/sample_collector.py @@ -1,6 +1,5 @@ import argparse import os -import sys from concurrent.futures import ProcessPoolExecutor @@ -46,4 +45,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/iterative_demo/sample_processor.py b/merlin/examples/workflows/iterative_demo/sample_processor.py index ed4dd596b..9ec0951e9 100644 --- a/merlin/examples/workflows/iterative_demo/sample_processor.py +++ b/merlin/examples/workflows/iterative_demo/sample_processor.py @@ -1,7 +1,6 @@ import argparse import os import pathlib -import sys import pandas as pd @@ -51,4 +50,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/lsf/scripts/make_samples.py b/merlin/examples/workflows/lsf/scripts/make_samples.py index 913a94062..e6c807bc9 100644 --- a/merlin/examples/workflows/lsf/scripts/make_samples.py +++ b/merlin/examples/workflows/lsf/scripts/make_samples.py @@ -1,6 +1,5 @@ import argparse import ast -import sys import numpy as np @@ -58,4 +57,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/null_spec/scripts/read_output.py b/merlin/examples/workflows/null_spec/scripts/read_output.py index bc238d9d2..7c0f8017e 100644 --- a/merlin/examples/workflows/null_spec/scripts/read_output.py +++ b/merlin/examples/workflows/null_spec/scripts/read_output.py @@ -41,7 +41,8 @@ def single_task_times(): match = matches.group(0) match = float(match.strip("s:")) task_durations.append(match) - except BaseException: + except Exception as e: + print(f"single_task_times Exception= {e}\n") continue print(str(task_durations)) @@ -58,9 +59,9 @@ def merlin_run_time(): total += result try: print(f"c{args.c}_s{args.s} merlin run : " + str(result)) - except BaseException: + except Exception as e: result = None - print(f"c{args.c}_s{args.s} merlin run : ERROR -- result={result}, args.errfile={args.errfile}") + print(f"c{args.c}_s{args.s} merlin run : ERROR -- result={result}, args.errfile={args.errfile}\n{e}") def start_verify_time(): @@ -74,12 +75,12 @@ def start_verify_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception as e: continue try: print(f"c{args.c}_s{args.s} start verify : " + str(all_timestamps[0])) - except BaseException: - print(f"c{args.c}_s{args.s} start verify : ERROR") + except Exception as e: + print(f"c{args.c}_s{args.s} start verify : ERROR\n{e}") def start_run_workers_time(): @@ -93,7 +94,7 @@ def start_run_workers_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception: continue earliest = min(all_timestamps) print(f"c{args.c}_s{args.s} start run-workers : " + str(earliest)) @@ -110,7 +111,7 @@ def start_sample1_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception: continue earliest = min(all_timestamps) print(f"c{args.c}_s{args.s} start samp1 : " + str(earliest)) diff --git a/merlin/examples/workflows/null_spec/scripts/read_output_chain.py b/merlin/examples/workflows/null_spec/scripts/read_output_chain.py index f9fdfd521..ae9fb11a5 100644 --- a/merlin/examples/workflows/null_spec/scripts/read_output_chain.py +++ b/merlin/examples/workflows/null_spec/scripts/read_output_chain.py @@ -53,7 +53,7 @@ def single_task_times(): match = matches.group(0) match = float(match.strip("s:")) task_durations.append(match) - except BaseException: + except Exception: print(f"c{filled_c}_s{k} task times : ERROR") continue @@ -71,7 +71,7 @@ def merlin_run_time(): total += result try: print(f"c{filled_c} merlin run : " + str(result)) - except BaseException: + except Exception: result = None print(f"c{filled_c} merlin run : ERROR -- result={result}, args.errfile={args.errfile}") @@ -87,12 +87,12 @@ def start_verify_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception: print(f"c{filled_c}_s{k} start verify : ERROR") continue try: print(f"c{filled_c}_s{k} start verify : " + str(all_timestamps[0])) - except BaseException: + except Exception: print(f"c{filled_c}_s{k} start verify : ERROR") @@ -107,7 +107,7 @@ def start_run_workers_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception: continue earliest = min(all_timestamps) print(f"c{filled_c}_s{k} start run-workers : " + str(earliest)) @@ -124,7 +124,7 @@ def start_sample1_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception: print(f"c{filled_c}_s{k} start samp1 : ERROR") continue earliest = min(all_timestamps) diff --git a/merlin/examples/workflows/openfoam_wf/scripts/learn.py b/merlin/examples/workflows/openfoam_wf/scripts/learn.py index b7c40f6d3..96124c46f 100644 --- a/merlin/examples/workflows/openfoam_wf/scripts/learn.py +++ b/merlin/examples/workflows/openfoam_wf/scripts/learn.py @@ -103,8 +103,6 @@ ax[0][1].set_ylim([y_min, y_max]) -y_pred_all = regr.predict(X) -input_enstrophy = ax[1][1].scatter(X[:, 0], 10 ** y[:, 1], s=100, edgecolors="black") ax[1][1].set_xlabel(r"Lidspeed ($\frac{m}{s}$)", fontsize=fontsize) ax[1][1].set_ylabel(r"$Energy$", fontsize=fontsize) ax[1][1].set_title("Average Energy Variation with Lidspeed") diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py index b7c40f6d3..96124c46f 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py +++ b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py @@ -103,8 +103,6 @@ ax[0][1].set_ylim([y_min, y_max]) -y_pred_all = regr.predict(X) -input_enstrophy = ax[1][1].scatter(X[:, 0], 10 ** y[:, 1], s=100, edgecolors="black") ax[1][1].set_xlabel(r"Lidspeed ($\frac{m}{s}$)", fontsize=fontsize) ax[1][1].set_ylabel(r"$Energy$", fontsize=fontsize) ax[1][1].set_title("Average Energy Variation with Lidspeed") diff --git a/merlin/examples/workflows/optimization/scripts/visualizer.py b/merlin/examples/workflows/optimization/scripts/visualizer.py index f41885bf9..c373c9cd1 100644 --- a/merlin/examples/workflows/optimization/scripts/visualizer.py +++ b/merlin/examples/workflows/optimization/scripts/visualizer.py @@ -9,7 +9,6 @@ import matplotlib.pyplot as plt import numpy as np -from mpl_toolkits.mplot3d import Axes3D plt.style.use("seaborn-white") diff --git a/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py b/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py index 0e2b624d6..232c43e86 100644 --- a/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py +++ b/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py @@ -1,6 +1,5 @@ import argparse import json -import sys from typing import Dict @@ -42,4 +41,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/restart/scripts/make_samples.py b/merlin/examples/workflows/restart/scripts/make_samples.py index 913a94062..e6c807bc9 100644 --- a/merlin/examples/workflows/restart/scripts/make_samples.py +++ b/merlin/examples/workflows/restart/scripts/make_samples.py @@ -1,6 +1,5 @@ import argparse import ast -import sys import numpy as np @@ -58,4 +57,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/restart_delay/scripts/make_samples.py b/merlin/examples/workflows/restart_delay/scripts/make_samples.py index 913a94062..e6c807bc9 100644 --- a/merlin/examples/workflows/restart_delay/scripts/make_samples.py +++ b/merlin/examples/workflows/restart_delay/scripts/make_samples.py @@ -1,6 +1,5 @@ import argparse import ast -import sys import numpy as np @@ -58,4 +57,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/slurm/scripts/make_samples.py b/merlin/examples/workflows/slurm/scripts/make_samples.py index 913a94062..e6c807bc9 100644 --- a/merlin/examples/workflows/slurm/scripts/make_samples.py +++ b/merlin/examples/workflows/slurm/scripts/make_samples.py @@ -1,6 +1,5 @@ import argparse import ast -import sys import numpy as np @@ -58,4 +57,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 20a32755f..20970fe7a 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # @@ -36,6 +36,7 @@ "RetryException", "SoftFailException", "HardFailException", + "InvalidChainException", "RestartException", ) @@ -76,7 +77,7 @@ class InvalidChainException(Exception): """ def __init__(self): - super(HardFailException, self).__init__() + super(InvalidChainException, self).__init__() class RestartException(Exception): diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 4e3969c2c..1f6befa88 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 9f6c90a73..6e491ba1a 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # @@ -140,7 +140,7 @@ def parse_override_vars( val = int(val) result[key] = val - except BaseException as excpt: + except Exception as excpt: raise ValueError( f"{excpt} Bad '--vars' formatting on command line. See 'merlin run --help' for an example." ) from excpt @@ -756,4 +756,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 94c57445e..a1e80fea1 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # @@ -33,7 +33,6 @@ """ import argparse import logging -import sys from merlin.ascii_art import banner_small from merlin.log_formatter import setup_logging @@ -65,4 +64,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/router.py b/merlin/router.py index c559883fa..90aa9db38 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index da4ad9f74..13db3dccc 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 3a344a98e..9af27d456 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 74dc0be46..34b2113cd 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 30a71b25c..0be5b21b9 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 122b8f002..6bb612cb4 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # @@ -228,7 +228,7 @@ def dump(self): result = result.replace("\n\n\n", "\n\n") try: yaml.safe_load(result) - except BaseException as e: + except Exception as e: raise ValueError(f"Error parsing provenance spec:\n{e}") return result diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index da4ad9f74..13db3dccc 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 0be2e8739..69c68856a 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index b7045bc46..f78044d41 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # @@ -155,8 +155,8 @@ def query_celery_queues(queues): try: name, jobs, consumers = channel.queue_declare(queue=queue, passive=True) found_queues.append((name, jobs, consumers)) - except BaseException: - LOG.warning(f"Cannot find queue {queue} on server.") + except Exception as e: + LOG.warning(f"Cannot find queue {queue} on server.{e}") finally: connection.close() return found_queues diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 6821d486e..01f7aae91 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 63a11d0c7..928e246e1 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 649610ab6..5b03bf2cb 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index d98a31419..efa43dd9f 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index 657349fed..6529e8fbb 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.8.5. # # For details, see https://github.com/LLNL/merlin. # @@ -37,7 +37,7 @@ import re import socket import subprocess -from contextlib import contextmanager, suppress +from contextlib import contextmanager from copy import deepcopy from datetime import timedelta from types import SimpleNamespace @@ -205,15 +205,14 @@ def get_yaml_var(entry, var, default): :param `var`: a yaml key :param `default`: default value in the absence of data """ - ret = default - if isinstance(entry, dict): - with suppress(KeyError): - ret = entry[var] - else: - with suppress(AttributeError): - ret = getattr(entry, var) - return ret + try: + return entry[var] + except (TypeError, KeyError): + try: + return getattr(entry, var) + except AttributeError: + return default def load_array_file(filename, ndmin=2): diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index c7d3745b1..3a8038d06 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -34,7 +34,6 @@ """ import argparse import shutil -import sys import time from contextlib import suppress from subprocess import PIPE, Popen @@ -155,7 +154,7 @@ def run_tests(args, tests): continue try: passed, info = run_single_test(test_name, test, test_label) - except BaseException as e: + except Exception as e: print(e) passed = False info = None @@ -215,4 +214,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() From 3bce97e53b05bdc489008040d6c060977cc7c483 Mon Sep 17 00:00:00 2001 From: Joe Koning Date: Mon, 21 Mar 2022 12:17:26 -0700 Subject: [PATCH 024/126] Update lgtm.yml I think this is yaml format issue, add a newline at end of file. --- lgtm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/lgtm.yml b/lgtm.yml index e3f53c87d..9c1433974 100644 --- a/lgtm.yml +++ b/lgtm.yml @@ -23,3 +23,4 @@ path_classifiers: queries: # Specifically hide the results of clear-text-logging-sensitive-data - exclude: py/clear-text-logging-sensitive-data + From 5d7508fbfcbdbcf476a8203e7137072548bf876b Mon Sep 17 00:00:00 2001 From: Joe Koning Date: Mon, 28 Mar 2022 10:26:02 -0700 Subject: [PATCH 025/126] Update lgtm.yml --- lgtm.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lgtm.yml b/lgtm.yml index 9c1433974..7f2dfb613 100644 --- a/lgtm.yml +++ b/lgtm.yml @@ -21,6 +21,10 @@ path_classifiers: ######################################################################################### queries: - # Specifically hide the results of clear-text-logging-sensitive-data - - exclude: py/clear-text-logging-sensitive-data + - include: "*" + - exclude: "py/clear-text-logging-sensitive-data" +extraction: + python: + python_setup: + version: "3" From 64b7c284213c970864dc007db07903c50c9b3f33 Mon Sep 17 00:00:00 2001 From: "Joseph M. Koning" Date: Mon, 28 Mar 2022 10:28:26 -0700 Subject: [PATCH 026/126] Update docker config docs. --- .../docker-compose_rabbit_redis_tls.yml | 25 +++---------------- .../modules/installation/installation.rst | 15 +++++++++++ 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/docs/source/modules/installation/docker-compose_rabbit_redis_tls.yml b/docs/source/modules/installation/docker-compose_rabbit_redis_tls.yml index b0a82a7b0..b80b71dc7 100644 --- a/docs/source/modules/installation/docker-compose_rabbit_redis_tls.yml +++ b/docs/source/modules/installation/docker-compose_rabbit_redis_tls.yml @@ -6,7 +6,7 @@ networks: services: redis: - image: 'redis:6.0-rc2' + image: 'redis' container_name: my-redis command: - --port 0 @@ -18,7 +18,7 @@ services: ports: - "6379:6379" volumes: - - ~/merlinu/cert_redis:/cert_redis + - "~/merlinu/cert_redis:/cert_redis" networks: - mernet @@ -31,25 +31,8 @@ services: - "15671:15671" - "5672:5672" - "5671:5671" - environment: - - RABBITMQ_SSL_CACERTFILE=/cert_rabbitmq/ca_certificate.pem - - RABBITMQ_SSL_KEYFILE=/cert_rabbitmq/server_key.pem - - RABBITMQ_SSL_CERTFILE=/cert_rabbitmq/server_certificate.pem - - RABBITMQ_SSL_VERIFY=verify_none - - RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=false - - RABBITMQ_DEFAULT_USER=merlinu - - RABBITMQ_DEFAULT_VHOST=/merlinu - - RABBITMQ_DEFAULT_PASS=guest volumes: - - ~/merlinu/cert_rabbitmq:/cert_rabbitmq - networks: - - mernet - - merlin: - image: 'llnl/merlin' - container_name: my-merlin - tty: true - volumes: - - ~/merlinu/:/home/merlinu + - "~/merlinu/rabbbitmq.conf:/etc/rabbitmq/rabbitmq.conf" + - "~/merlinu/cert_rabbitmq:/cert_rambbitmq" networks: - mernet diff --git a/docs/source/modules/installation/installation.rst b/docs/source/modules/installation/installation.rst index dabc4f807..2eb1ac95d 100644 --- a/docs/source/modules/installation/installation.rst +++ b/docs/source/modules/installation/installation.rst @@ -369,6 +369,21 @@ be defined as shown in the :ref:`broker_redis_ssl` section. .. literalinclude:: ./docker-compose_rabbit_redis_tls.yml :language: yaml +The ``rabbitmq.conf`` file contains the configuration, including ssl, for +the rabbitmq server. + +.. code-block:: bash + + default_vhost = /merlinu + default_user = merlinu + default_pass = guest + listeners.ssl.default = 5671 + ssl.options.ccertfile = /cert_rabbitmq/ca_certificate.pem + ssl.options.certfile = /cert_rabbitmq/server_certificate.pem + ssl.options.keyfile = /cert_rabbitmq/server_key.pem + ssl.options.verify = verify_none + ssl.options.fail_if_no_peer_cert = false + Once this docker-compose file is run, the merlin ``app.yaml`` file is changed to use the redis TLS server ``rediss`` instead of ``redis``. From e6a1279e4b151a62e99db4c3a6421762569a73e0 Mon Sep 17 00:00:00 2001 From: Joe Koning Date: Tue, 29 Mar 2022 09:03:29 -0700 Subject: [PATCH 027/126] Docs update and rename lgtm.yml (#359) * Create python-publish.yml (#353) Adding GitHub action for publishing to pypi * Workflows Community Initiative Metadata (#355) * Added Workflows Community Initiative metadata info; fixed some old links * Run black * Add updates for lgtm CI security site (#357) * Update code to remove LGTM Errors and Warnings and implement Recommendations. * Change BaseException to Exception. * Add lgtm config file. * Changes for flake8. * Add TypeError yo yam read. * Add TypeError to yaml read. * Just return when successful on the yaml read. * Fix typo. * Add merlin/examples to lgtm exclude list as well. * Add ssl comment. * Fix typo. * Update version to 1.8.5. * Update docker config docs. * Rename lgtm.yml to .lgtm.yml * Add changelog entries. * Add newline at end. Co-authored-by: Luc Peterson --- lgtm.yml => .lgtm.yml | 1 + CHANGELOG.md | 6 +++++ .../docker-compose_rabbit_redis_tls.yml | 25 +++---------------- .../modules/installation/installation.rst | 15 +++++++++++ 4 files changed, 26 insertions(+), 21 deletions(-) rename lgtm.yml => .lgtm.yml (99%) diff --git a/lgtm.yml b/.lgtm.yml similarity index 99% rename from lgtm.yml rename to .lgtm.yml index 7f2dfb613..19667c722 100644 --- a/lgtm.yml +++ b/.lgtm.yml @@ -28,3 +28,4 @@ extraction: python: python_setup: version: "3" + diff --git a/CHANGELOG.md b/CHANGELOG.md index b86f399ae..79b89fc87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] +### Added +- Update docker docs for new rabbitmq and redis server versions +### Changed +- Rename lgtm.yml to .lgtm.yml + ## [1.8.5] ### Added - Code updates to satisfy lgtm CI security checker diff --git a/docs/source/modules/installation/docker-compose_rabbit_redis_tls.yml b/docs/source/modules/installation/docker-compose_rabbit_redis_tls.yml index b0a82a7b0..b80b71dc7 100644 --- a/docs/source/modules/installation/docker-compose_rabbit_redis_tls.yml +++ b/docs/source/modules/installation/docker-compose_rabbit_redis_tls.yml @@ -6,7 +6,7 @@ networks: services: redis: - image: 'redis:6.0-rc2' + image: 'redis' container_name: my-redis command: - --port 0 @@ -18,7 +18,7 @@ services: ports: - "6379:6379" volumes: - - ~/merlinu/cert_redis:/cert_redis + - "~/merlinu/cert_redis:/cert_redis" networks: - mernet @@ -31,25 +31,8 @@ services: - "15671:15671" - "5672:5672" - "5671:5671" - environment: - - RABBITMQ_SSL_CACERTFILE=/cert_rabbitmq/ca_certificate.pem - - RABBITMQ_SSL_KEYFILE=/cert_rabbitmq/server_key.pem - - RABBITMQ_SSL_CERTFILE=/cert_rabbitmq/server_certificate.pem - - RABBITMQ_SSL_VERIFY=verify_none - - RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=false - - RABBITMQ_DEFAULT_USER=merlinu - - RABBITMQ_DEFAULT_VHOST=/merlinu - - RABBITMQ_DEFAULT_PASS=guest volumes: - - ~/merlinu/cert_rabbitmq:/cert_rabbitmq - networks: - - mernet - - merlin: - image: 'llnl/merlin' - container_name: my-merlin - tty: true - volumes: - - ~/merlinu/:/home/merlinu + - "~/merlinu/rabbbitmq.conf:/etc/rabbitmq/rabbitmq.conf" + - "~/merlinu/cert_rabbitmq:/cert_rambbitmq" networks: - mernet diff --git a/docs/source/modules/installation/installation.rst b/docs/source/modules/installation/installation.rst index dabc4f807..2eb1ac95d 100644 --- a/docs/source/modules/installation/installation.rst +++ b/docs/source/modules/installation/installation.rst @@ -369,6 +369,21 @@ be defined as shown in the :ref:`broker_redis_ssl` section. .. literalinclude:: ./docker-compose_rabbit_redis_tls.yml :language: yaml +The ``rabbitmq.conf`` file contains the configuration, including ssl, for +the rabbitmq server. + +.. code-block:: bash + + default_vhost = /merlinu + default_user = merlinu + default_pass = guest + listeners.ssl.default = 5671 + ssl.options.ccertfile = /cert_rabbitmq/ca_certificate.pem + ssl.options.certfile = /cert_rabbitmq/server_certificate.pem + ssl.options.keyfile = /cert_rabbitmq/server_key.pem + ssl.options.verify = verify_none + ssl.options.fail_if_no_peer_cert = false + Once this docker-compose file is run, the merlin ``app.yaml`` file is changed to use the redis TLS server ``rediss`` instead of ``redis``. From f52afb1b02ee3326f2ead87f942dd21cffb970c1 Mon Sep 17 00:00:00 2001 From: Joe Koning Date: Fri, 1 Apr 2022 08:01:28 -0700 Subject: [PATCH 028/126] Update .lgtm.yml Simple edit to trigger lgtm build. --- .lgtm.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.lgtm.yml b/.lgtm.yml index 19667c722..98729306e 100644 --- a/.lgtm.yml +++ b/.lgtm.yml @@ -18,6 +18,8 @@ path_classifiers: ######################################################################################### # Use the `queries` block to change the default display of query results. # +# The py/clear-text-logging-sensitive-data exclusion is due to cert and password # +# keywords being flagged as security leaks. # ######################################################################################### queries: From 0f57a733f8a96a1ecb6093c84eabade1e6a1bb41 Mon Sep 17 00:00:00 2001 From: "Joseph M. Koning" Date: Tue, 5 Apr 2022 09:06:40 -0700 Subject: [PATCH 029/126] Fix formatting in .lgtm.yml. --- .lgtm.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.lgtm.yml b/.lgtm.yml index 98729306e..4ecd0f36b 100644 --- a/.lgtm.yml +++ b/.lgtm.yml @@ -13,8 +13,8 @@ path_classifiers: test: # Classify all files in the top-level directories tests/ as test code. - exclude: - - tests - - merlin/examples + - tests + - merlin/examples ######################################################################################### # Use the `queries` block to change the default display of query results. # From f9e8efa5cd4460805f95def80869cf5154e9cd00 Mon Sep 17 00:00:00 2001 From: "Joseph M. Koning" Date: Tue, 5 Apr 2022 09:32:59 -0700 Subject: [PATCH 030/126] Update conf.py for new sphinx versions. --- docs/source/conf.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 5b8c881d0..d5cef3d64 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -185,6 +185,11 @@ def setup(app): - app.add_stylesheet('custom.css') - app.add_javascript("custom.js") - app.add_javascript("https://cdn.jsdelivr.net/npm/clipboard@1/dist/clipboard.min.js") + try: + app.add_javascript("custom.js") + app.add_javascript("https://cdn.jsdelivr.net/npm/clipboard@1/dist/clipboard.min.js") + app.add_stylesheet('custom.css') + except AttributeError: + app.add_css_file('custom.css') + app.add_js_file("custom.js") + app.add_js_file("https://cdn.jsdelivr.net/npm/clipboard@1/dist/clipboard.min.js") From cbb3e6d9b937bd3244fba4d08997a7b5328d6b52 Mon Sep 17 00:00:00 2001 From: "Joseph M. Koning" Date: Thu, 7 Apr 2022 10:21:53 -0700 Subject: [PATCH 031/126] Fix lgtm test clause. --- .lgtm.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.lgtm.yml b/.lgtm.yml index 4ecd0f36b..b1b073e2b 100644 --- a/.lgtm.yml +++ b/.lgtm.yml @@ -11,10 +11,12 @@ path_classifiers: test: - # Classify all files in the top-level directories tests/ as test code. - - exclude: - - tests - - merlin/examples + # Exclude all files to ovveride lgtm defaults + - exclude: / + # Classify all files in the top-level directories tests/ + # and merlin/examples as test code. + - tests + - merlin/examples ######################################################################################### # Use the `queries` block to change the default display of query results. # From 0497a98f5d29ef0047b1db39ff362696df7d79e5 Mon Sep 17 00:00:00 2001 From: Joe Koning Date: Wed, 13 Apr 2022 12:31:47 -0700 Subject: [PATCH 032/126] Lgtm fixes 2 (#361) * Create python-publish.yml (#353) Adding GitHub action for publishing to pypi * Workflows Community Initiative Metadata (#355) * Added Workflows Community Initiative metadata info; fixed some old links * Run black * Add updates for lgtm CI security site (#357) * Update code to remove LGTM Errors and Warnings and implement Recommendations. * Change BaseException to Exception. * Add lgtm config file. * Changes for flake8. * Add TypeError yo yam read. * Add TypeError to yaml read. * Just return when successful on the yaml read. * Fix typo. * Add merlin/examples to lgtm exclude list as well. * Add ssl comment. * Fix typo. * Update version to 1.8.5. * Update conf.py for new sphinx versions. * Fix an error and some Warnings and Recommendations from lgtm.com * Add some comments for exceptions with pass. * More lgtm fixes. Add lgtm badge. * Update CHANGELOG.md. * Remove extraneous lgtm.yml. Co-authored-by: Luc Peterson --- CHANGELOG.md | 2 ++ README.md | 1 + docs/source/conf.py | 11 ++++++++--- merlin/common/openfilelist.py | 4 ++-- merlin/common/opennpylib.py | 6 +++--- merlin/common/tasks.py | 3 ++- merlin/config/__init__.py | 1 + merlin/config/broker.py | 3 +++ merlin/config/results_backend.py | 2 ++ merlin/spec/override.py | 3 ++- merlin/study/celeryadapter.py | 2 ++ merlin/study/script_adapter.py | 4 ++-- merlin/utils.py | 1 + setup.py | 2 +- 14 files changed, 32 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79b89fc87..b472cfcf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Added - Update docker docs for new rabbitmq and redis server versions +- Added lgtm.com Badge for README.md +- More fixes for lgtm checks. ### Changed - Rename lgtm.yml to .lgtm.yml diff --git a/README.md b/README.md index fad1de93f..909ba869a 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ ![Activity](https://img.shields.io/github/commit-activity/m/LLNL/merlin) [![Issues](https://img.shields.io/github/issues/LLNL/merlin)](https://github.com/LLNL/merlin/issues) [![Pull requests](https://img.shields.io/github/issues-pr/LLNL/merlin)](https://github.com/LLNL/merlin/pulls) +[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/LLNL/merlin.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/LLNL/merlin/context:python) ![Merlin](https://raw.githubusercontent.com/LLNL/merlin/main/docs/images/merlin.png) diff --git a/docs/source/conf.py b/docs/source/conf.py index 5b8c881d0..d5cef3d64 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -185,6 +185,11 @@ def setup(app): - app.add_stylesheet('custom.css') - app.add_javascript("custom.js") - app.add_javascript("https://cdn.jsdelivr.net/npm/clipboard@1/dist/clipboard.min.js") + try: + app.add_javascript("custom.js") + app.add_javascript("https://cdn.jsdelivr.net/npm/clipboard@1/dist/clipboard.min.js") + app.add_stylesheet('custom.css') + except AttributeError: + app.add_css_file('custom.css') + app.add_js_file("custom.js") + app.add_js_file("https://cdn.jsdelivr.net/npm/clipboard@1/dist/clipboard.min.js") diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 601af41c7..814fb1881 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -73,7 +73,7 @@ def __init__(self, files, *v, **kw): self.argv, self.argkw = (v, kw) if self.files: self.fnnow = self.files.pop(0) - self.fnow = open(self.fnnow, *v, **kw) if files else None + self.fnow = open(self.fnnow, *v, **kw) if files else None # noqa self.atend = False else: self.fnnow = self.fnow = None @@ -89,7 +89,7 @@ def _tonext(self): self.fnow.close() if self.files: self.fnnow = self.files.pop(0) - self.fnow = open(self.fnnow, *self.argv, **self.argkw) + self.fnow = open(self.fnnow, *self.argv, **self.argkw) # noqa else: self.fnnow = self.fnow = None self.atend = True diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 3c2d0dba0..a8be89486 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -96,7 +96,7 @@ def _get_npy_info2(f): if isinstance(f, unistr): - f = open(f, "rb") + f = open(f, "rb") # noqa magic = f.read(6) # must be .npy file assert magic == npy_magic # must be .npy file or ELSE major, _ = list(map(ord, f.read(2))) @@ -127,7 +127,7 @@ def _get_npy_info2(f): def _get_npy_info3(f): if isinstance(f, unistr): - f = open(f, "rb") + f = open(f, "rb") # noqa magic = f.read(6) # must be .npy file assert magic == npy_magic # must be .npy file or ELSE major, _ = list(f.read(2)) @@ -373,7 +373,7 @@ def test_seeknpylist(self): en3 = np.asarray(a[1:14]) en4 = a.to_array() # test __len__ method - self.assertTrue(len(a) == 3 * len(e)) + self.assertEqual(len(a), 3 * len(e)) os.unlink(fn) # test read slice of whole file self.assertTrue((en == e).all()) diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index e3a7ed525..f36134fec 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -110,7 +110,8 @@ def merlin_step(self, *args: Any, **kwargs: Any) -> Optional[ReturnCode]: # noq if result == ReturnCode.OK: LOG.info(f"Step '{step_name}' in '{step_dir}' finished successfully.") # touch a file indicating we're done with this step - open(finished_filename, "a").close() + with open(finished_filename, "a"): + pass elif result == ReturnCode.DRY_OK: LOG.info(f"Dry-ran step '{step_name}' in '{step_dir}'.") elif result == ReturnCode.RESTART: diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index a78a937d4..ebbdc662e 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -62,4 +62,5 @@ def load_app_into_namespaces(self, app_dict: Dict) -> None: try: setattr(self, field, nested_dict_to_namespaces(app_dict[field])) except KeyError: + # The keywords are optional pass diff --git a/merlin/config/broker.py b/merlin/config/broker.py index b73330f6f..72cd6ec27 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -202,6 +202,7 @@ def get_connection_string(include_password=True): try: return CONFIG.broker.url except AttributeError: + # The broker may not have a url pass try: @@ -250,11 +251,13 @@ def get_ssl_config() -> Union[bool, Dict[str, Union[str, ssl.VerifyMode]]]: try: broker = CONFIG.broker.url.split(":")[0] except AttributeError: + # The broker may not have a url pass try: broker = CONFIG.broker.name.lower() except AttributeError: + # The broker may not have a name pass if broker not in BROKERS: diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index a5effe13e..843930cac 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -298,11 +298,13 @@ def get_ssl_config(celery_check=False): try: results_backend = CONFIG.results_backend.url.split(":")[0] except AttributeError: + # The results_backend may not have a url pass try: results_backend = CONFIG.results_backend.name.lower() except AttributeError: + # The results_backend may not have a name pass if results_backend not in BACKENDS: diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 538f895e8..a3fbf281b 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -11,7 +11,8 @@ def error_override_vars(override_vars, spec_filepath): """ if override_vars is None: return - original_text = open(spec_filepath, "r").read() + with open(spec_filepath, "r") as ospec: + original_text = ospec.read() for variable in override_vars.keys(): if variable not in original_text: raise ValueError(f"Command line override variable '{variable}' not found in spec file '{spec_filepath}'.") diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index f78044d41..67bae9d74 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -320,6 +320,8 @@ def examine_and_log_machines(worker_val, yenv) -> bool: "The env:variables section does not have an OUTPUT_PATH specified, multi-machine checks cannot be performed." ) return False + else: + return False def verify_args(spec, worker_args, worker_name, overlap): diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 928e246e1..23398e117 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -299,7 +299,7 @@ def __init__(self, **kwargs): "gpus per task": "-g", "walltime": "-t", "flux": "", - } + } # noqa if "wreck" in flux_command: self._cmd_flags["walltime"] = "-T" @@ -321,7 +321,7 @@ def __init__(self, **kwargs): "lsf", "slurm", ] - self._unsupported = set(new_unsupported) + self._unsupported = set(new_unsupported) # noqa def time_format(self, val): """ diff --git a/merlin/utils.py b/merlin/utils.py index 6529e8fbb..b9f8742bb 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -297,6 +297,7 @@ def get_source_root(filepath): parent = os.path.dirname(filepath) # Walk backwards testing for integers. + break_point = parent.split(sep)[-1] # Initial value for lgtm.com for _, _dir in enumerate(parent.split(sep)[::-1]): try: int(_dir) diff --git a/setup.py b/setup.py index 68e4b8ed7..f60536d30 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ def _pip_requirement(req): def _reqs(*f): return [ _pip_requirement(r) - for r in (_strip_comments(line) for line in open(os.path.join(os.getcwd(), "requirements", *f)).readlines()) + for r in (_strip_comments(line) for line in open(os.path.join(os.getcwd(), "requirements", *f)).readlines()) # noqa if r ] From 271021961c86415c106ce530a87b91efe028642a Mon Sep 17 00:00:00 2001 From: Ryan Lee <44886374+ryannova@users.noreply.github.com> Date: Thu, 28 Apr 2022 13:29:46 -0700 Subject: [PATCH 033/126] Added Server Command Feature to Merlin (#360) * Added merlin server capability to initalize, monitor and stop redis containers * Added configuration for singularity, docker, and podman in merlin/server/ * Created documentation for "merlin server" command * Added tests to initialize, start and stop a singularity+redis server to the local test suite. (Future: add to the "distributed tests" connecting to that server and running merlin) Co-authored-by: Ryan Lee & Joe Koning --- .github/workflows/push-pr_workflow.yml | 26 + CHANGELOG.md | 5 + CONTRIBUTORS | 1 + MANIFEST.in | 1 + docs/source/index.rst | 1 + docs/source/merlin_server.rst | 72 + docs/source/server/commands.rst | 40 + docs/source/server/configuration.rst | 75 + merlin/main.py | 77 +- merlin/server/docker.yaml | 6 + merlin/server/merlin_server.yaml | 19 + merlin/server/podman.yaml | 6 + merlin/server/redis.conf | 2051 ++++++++++++++++++++++++ merlin/server/server_config.py | 105 ++ merlin/server/server_setup.py | 291 ++++ merlin/server/singularity.yaml | 6 + tests/integration/run_tests.py | 3 +- tests/integration/test_definitions.py | 14 + 18 files changed, 2797 insertions(+), 2 deletions(-) create mode 100644 docs/source/merlin_server.rst create mode 100644 docs/source/server/commands.rst create mode 100644 docs/source/server/configuration.rst create mode 100644 merlin/server/docker.yaml create mode 100644 merlin/server/merlin_server.yaml create mode 100644 merlin/server/podman.yaml create mode 100644 merlin/server/redis.conf create mode 100644 merlin/server/server_config.py create mode 100644 merlin/server/server_setup.py create mode 100644 merlin/server/singularity.yaml diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 3b2f809eb..44a2255b9 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -67,6 +67,11 @@ jobs: Local-test-suite: runs-on: ubuntu-latest + env: + GO_VERSION: 1.18.1 + SINGULARITY_VERSION: 3.9.9 + OS: linux + ARCH: amd64 strategy: matrix: @@ -90,6 +95,27 @@ jobs: python3 -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt + + - name: Install singularity + run: | + sudo apt-get update && sudo apt-get install -y \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config + wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz + sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz + rm go$GO_VERSION.$OS-$ARCH.tar.gz + export PATH=$PATH:/usr/local/go/bin + wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz + tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz + cd singularity-ce-$SINGULARITY_VERSION + ./mconfig && \ + make -C ./builddir && \ + sudo make -C ./builddir install - name: Install merlin to run unit tests run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index b472cfcf7..0142d6d08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update docker docs for new rabbitmq and redis server versions - Added lgtm.com Badge for README.md - More fixes for lgtm checks. +- Added merlin server capabilities under merlin/server/ +- Added merlin server commands init, start, status, stop to main.py +- Added redis.conf for default redis configuration for merlin in server/redis.conf +- Added default configurations for merlin server command in merlin/server/*.yaml +- Added documentation page docs/merlin_server.rst, docs/modules/server/configuration.rst, and docs/modules/server/commands.rst ### Changed - Rename lgtm.yml to .lgtm.yml diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 9f96bc6e7..a920e45b7 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -5,3 +5,4 @@ Benjamin Bay Joe Koning Jeremy White Aidan Keogh +Ryan Lee diff --git a/MANIFEST.in b/MANIFEST.in index f5b32237d..93fa2f74e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ recursive-include merlin/data *.yaml *.py +recursive-include merlin/server *.yaml *.py recursive-include merlin/examples *.py *.yaml *.c *.json *.sbatch *.bsub *.txt include requirements.txt include requirements/* diff --git a/docs/source/index.rst b/docs/source/index.rst index 3747adca4..72469f968 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -75,6 +75,7 @@ Need help? `merlin@llnl.gov `_ merlin_specification merlin_config merlin_variables + merlin_server celery_overview virtualenv spack diff --git a/docs/source/merlin_server.rst b/docs/source/merlin_server.rst new file mode 100644 index 000000000..66e773ba9 --- /dev/null +++ b/docs/source/merlin_server.rst @@ -0,0 +1,72 @@ +Merlin Server +============= +The merlin server command allows users easy access to containerized broker +and results servers for merlin workflows. This allowsusers to run merlin without +a dedicated external server. + +The main configuration will be stored in the subdirectory called "server/" by +default in the main merlin configuration "~/.merlin". However different server +images can be created for different use cases or studies just by simplying creating +a new directory to store local configuration files for merlin server instances. + +Below is an example of how merlin server can be utilized. + +First create and navigate into a directory to store your local merlin +configuration for a specific use case or study. + +.. code-block:: bash + + mkdir study1/ + cd study1/ + +Afterwards you can instantiate merlin server in this directory by running + +.. code-block:: bash + + merlin server init + +A main server configuration will be created in the ~/.merlin/server and a local +configuration will be created in a subdirectory called "merlin_server/" + +We should expect the following files in each directory + +.. code-block:: bash + + ~/study1$ ls ~/.merlin/server/ + docker.yaml merlin_server.yaml podman.yaml singularity.yaml + + ~/study1$ ls + merlin_server + + ~/study1$ ls merlin_server/ + redis.conf redis_latest.sif + +The main configuration in "~/.merlin/server" deals with defaults and +technical commands that might be used for setting up the merlin server +local configuration and its containers. Each container has their own +configuration file to allow users to be able to switch between different +containerized services freely. + +The local configuration "merlin_server" folder contains configuration files +specific to a certain use case or run. In the case above you can see that we have a +redis singularity container called "redis_latest.sif" with the redis configuration +file called "redis.conf". This redis configuration will allow the user to +configurate redis to their specified needs without have to manage or edit +the redis container. When the server is run this configuration will be dynamically +read, so settings can be changed between runs if needed. + +Once the merlin server has been initialized in the local directory the user will be allowed +to run other merlin server commands such as "run, status, stop" to interact with the +merlin server. A detailed list of commands can be found in the `Merlin Server Commands <./server/commands.html>`_ page. + +Note: Running "merlin server init" again will NOT override any exisiting configuration +that the users might have set or edited. By running this command again any missing files +will be created for the users with exisiting defaults. HOWEVER it is highly advised that +users back up their configuration in case an error occurs where configuration files are override. + +.. toctree:: + :maxdepth: 1 + :caption: Merlin Server Settings: + + server/configuration + server/commands \ No newline at end of file diff --git a/docs/source/server/commands.rst b/docs/source/server/commands.rst new file mode 100644 index 000000000..f3bbec276 --- /dev/null +++ b/docs/source/server/commands.rst @@ -0,0 +1,40 @@ +Merlin Server Commands +====================== + +Merlin server has a list of commands for interacting with the broker and results server. +These commands allow the user to manage and monitor the exisiting server and create +instances of servers if needed. + +Initializing Merlin Server (``merlin server init``) +--------------------------------------------------- +The merlin server init command creates configurations for merlin server commands. + +A main merlin sever configuration subdirectory is created in "~/.merlin/server" which contains +configuration for local merlin configuration, and configurations for different containerized +services that merlin server supports, which includes singularity (docker and podman implemented +in the future). + +A local merlin server configuration subdirectory called "merlin_server/" will also +be created when this command is run. This will contain a container for merlin server and associated +configuration files that might be used to start the server. For example, for a redis server a "redis.conf" +will contain settings which will be dynamically loaded when the redis server is run. This local configuration +will also contain information about currently running containers as well. + +Note: If there is an exisiting subdirectory containing a merlin server configuration then only +missing files will be replaced. However it is recommended that users backup their local configurations. + + +Checking Merlin Server Status (``merlin server status``) +-------------------------------------------------------- + +Displays the current status of the merlin server. + +Starting up a Merlin Server (``merlin server start``) +----------------------------------------------------- + +Starts the container located in the local merlin server configuration. + +Stopping an exisiting Merlin Server (``merlin server stop``) +------------------------------------------------------------ + +Stop any exisiting container being managaed and monitored by merlin server. diff --git a/docs/source/server/configuration.rst b/docs/source/server/configuration.rst new file mode 100644 index 000000000..84429c079 --- /dev/null +++ b/docs/source/server/configuration.rst @@ -0,0 +1,75 @@ +Merlin Server Configuration +=========================== + +Below are a sample list of configurations for the merlin server command + +Main Configuration ``~/.merlin/server/`` +---------------------------------------- + +merlin_server.yaml + +.. code-block:: yaml + + container: + # Select the format for the recipe e.g. singularity, docker, podman (currently singularity is the only working option.) + format: singularity + # The image name + image: redis_latest.sif + # The url to pull the image from + url: docker://redis + # The config file + config: redis.conf + # Subdirectory name to store configurations Default: merlin_server/ + config_dir: merlin_server/ + # Process file containing information regarding the redis process + pfile: merlin_server.pf + + process: + # Command for determining the process of the command + status: pgrep -P {pid} #ps -e | grep {pid} + # Command for killing process + kill: kill {pid} + + +singularity.yaml + +.. code-block:: yaml + + singularity: + command: singularity + # init_command: \{command} .. (optional or default) + run_command: \{command} run {image} {config} + stop_command: kill # \{command} (optional or kill default) + pull_command: \{command} pull {image} {url} + + +Local Configuration ``merlin_server/`` +-------------------------------------- + +redis.conf + +.. code-block:: yaml + + bind 127.0.0.1 -::1 + protected-mode yes + port 6379 + logfile "" + dir ./ + ... + +see documentation on redis configuration `here `_ for more detail + +merlin_server.pf + +.. code-block:: yaml + + bits: '64' + commit: '00000000' + hostname: ubuntu + image_pid: '1111' + mode: standalone + modified: '0' + parent_pid: 1112 + port: '6379' + version: 6.2.6 + diff --git a/merlin/main.py b/merlin/main.py index 6e491ba1a..113a15534 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -52,6 +52,14 @@ from merlin.ascii_art import banner_small from merlin.examples.generator import list_examples, setup_example from merlin.log_formatter import setup_logging +from merlin.server.server_setup import ( + ServerStatus, + create_server_config, + get_server_status, + pull_server_image, + start_server, + stop_server, +) from merlin.spec.expansion import RESERVED, get_spec_with_expansion from merlin.spec.specification import MerlinSpec from merlin.study.study import MerlinStudy @@ -342,6 +350,32 @@ def process_monitor(args): LOG.info("Monitor: ... stop condition met") +def process_server(args): + if args.commands == "init": + if not create_server_config(): + LOG.info("Merlin server initialization failed.") + return + if pull_server_image(): + LOG.info("New merlin server image fetched") + LOG.info("Merlin server initialization successful.") + elif args.commands == "start": + start_server() + elif args.commands == "stop": + stop_server() + elif args.commands == "status": + current_status = get_server_status() + if current_status == ServerStatus.NOT_INITALIZED: + LOG.info("Merlin server has not been initialized.") + LOG.info("Please initalize server by running 'merlin server init'") + elif current_status == ServerStatus.MISSING_CONTAINER: + LOG.info("Unable to find server image.") + LOG.info("Ensure there is a .sif file in merlin server directory.") + elif current_status == ServerStatus.NOT_RUNNING: + LOG.info("Merlin server is not running.") + elif current_status == ServerStatus.RUNNING: + LOG.info("Merlin server is running.") + + def setup_argparse() -> None: """ Setup argparse and any CLI options we want available via the package. @@ -551,6 +585,47 @@ def setup_argparse() -> None: generate_diagnostic_parsers(subparsers) + # merlin server + server: ArgumentParser = subparsers.add_parser( + "server", + help="Manage broker and results server for merlin workflow.", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + + server_commands: ArgumentParser = server.add_subparsers(dest="commands") + + server_init: ArgumentParser = server_commands.add_parser( + "init", + help="Initialize merlin server resources.", + description="Initialize merlin server", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + server_init.set_defaults(func=process_server) + + server_status: ArgumentParser = server_commands.add_parser( + "status", + help="View status of the current server containers.", + description="View status", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + server_status.set_defaults(func=process_server) + + server_start: ArgumentParser = server_commands.add_parser( + "start", + help="Start a containerized server to be used as an broker and results server.", + description="Start server", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + server_start.set_defaults(func=process_server) + + server_stop: ArgumentParser = server_commands.add_parser( + "stop", + help="Stop an instance of redis containers currently running.", + description="Stop server.", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + server_stop.set_defaults(func=process_server) + return parser @@ -756,4 +831,4 @@ def main(): if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/merlin/server/docker.yaml b/merlin/server/docker.yaml new file mode 100644 index 000000000..d7d5bc00a --- /dev/null +++ b/merlin/server/docker.yaml @@ -0,0 +1,6 @@ +docker: + command: docker + # init_command: ? + run_command: \{command} run --name {name} -d {image} + stop_command: \{command} stop {name} + pull_command: \{command} pull {url} diff --git a/merlin/server/merlin_server.yaml b/merlin/server/merlin_server.yaml new file mode 100644 index 000000000..6963a2d9f --- /dev/null +++ b/merlin/server/merlin_server.yaml @@ -0,0 +1,19 @@ +container: + # Select the format for the recipe e.g. singularity, docker, podman (currently singularity is the only working option.) + format: singularity + # The image name + image: redis_latest.sif + # The url to pull the image from + url: docker://redis + # The config file + config: redis.conf + # Subdirectory name to store configurations Default: merlin_server/ + config_dir: merlin_server/ + # Process file containing information regarding the redis process + pfile: merlin_server.pf + +process: + # Command for determining the process of the command + status: pgrep -P {pid} #ps -e | grep {pid} + # Command for killing process + kill: kill {pid} diff --git a/merlin/server/podman.yaml b/merlin/server/podman.yaml new file mode 100644 index 000000000..1632840bb --- /dev/null +++ b/merlin/server/podman.yaml @@ -0,0 +1,6 @@ +podman: + command: podman + # init_command: \{command} .. (optional or default) + run_command: \{command} run --name {name} -d {image} + stop_command: \{command} stop {name} + pull_command: \{command} pull {url} diff --git a/merlin/server/redis.conf b/merlin/server/redis.conf new file mode 100644 index 000000000..b50ae9e76 --- /dev/null +++ b/merlin/server/redis.conf @@ -0,0 +1,2051 @@ +# Redis configuration file example. +# +# Note that in order to read the configuration file, Redis must be +# started with the file path as first argument: +# +# ./redis-server /path/to/redis.conf + +# Note on units: when memory size is needed, it is possible to specify +# it in the usual form of 1k 5GB 4M and so forth: +# +# 1k => 1000 bytes +# 1kb => 1024 bytes +# 1m => 1000000 bytes +# 1mb => 1024*1024 bytes +# 1g => 1000000000 bytes +# 1gb => 1024*1024*1024 bytes +# +# units are case insensitive so 1GB 1Gb 1gB are all the same. + +################################## INCLUDES ################################### + +# Include one or more other config files here. This is useful if you +# have a standard template that goes to all Redis servers but also need +# to customize a few per-server settings. Include files can include +# other files, so use this wisely. +# +# Note that option "include" won't be rewritten by command "CONFIG REWRITE" +# from admin or Redis Sentinel. Since Redis always uses the last processed +# line as value of a configuration directive, you'd better put includes +# at the beginning of this file to avoid overwriting config change at runtime. +# +# If instead you are interested in using includes to override configuration +# options, it is better to use include as the last line. +# +# include /path/to/local.conf +# include /path/to/other.conf + +################################## MODULES ##################################### + +# Load modules at startup. If the server is not able to load modules +# it will abort. It is possible to use multiple loadmodule directives. +# +# loadmodule /path/to/my_module.so +# loadmodule /path/to/other_module.so + +################################## NETWORK ##################################### + +# By default, if no "bind" configuration directive is specified, Redis listens +# for connections from all available network interfaces on the host machine. +# It is possible to listen to just one or multiple selected interfaces using +# the "bind" configuration directive, followed by one or more IP addresses. +# Each address can be prefixed by "-", which means that redis will not fail to +# start if the address is not available. Being not available only refers to +# addresses that does not correspond to any network interfece. Addresses that +# are already in use will always fail, and unsupported protocols will always BE +# silently skipped. +# +# Examples: +# +# bind 192.168.1.100 10.0.0.1 # listens on two specific IPv4 addresses +# bind 127.0.0.1 ::1 # listens on loopback IPv4 and IPv6 +# bind * -::* # like the default, all available interfaces +# +# ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the +# internet, binding to all the interfaces is dangerous and will expose the +# instance to everybody on the internet. So by default we uncomment the +# following bind directive, that will force Redis to listen only on the +# IPv4 and IPv6 (if available) loopback interface addresses (this means Redis +# will only be able to accept client connections from the same host that it is +# running on). +# +# IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES +# JUST COMMENT OUT THE FOLLOWING LINE. +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +bind 127.0.0.1 -::1 + +# Protected mode is a layer of security protection, in order to avoid that +# Redis instances left open on the internet are accessed and exploited. +# +# When protected mode is on and if: +# +# 1) The server is not binding explicitly to a set of addresses using the +# "bind" directive. +# 2) No password is configured. +# +# The server only accepts connections from clients connecting from the +# IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain +# sockets. +# +# By default protected mode is enabled. You should disable it only if +# you are sure you want clients from other hosts to connect to Redis +# even if no authentication is configured, nor a specific set of interfaces +# are explicitly listed using the "bind" directive. +protected-mode yes + +# Accept connections on the specified port, default is 6379 (IANA #815344). +# If port 0 is specified Redis will not listen on a TCP socket. +port 6379 + +# TCP listen() backlog. +# +# In high requests-per-second environments you need a high backlog in order +# to avoid slow clients connection issues. Note that the Linux kernel +# will silently truncate it to the value of /proc/sys/net/core/somaxconn so +# make sure to raise both the value of somaxconn and tcp_max_syn_backlog +# in order to get the desired effect. +tcp-backlog 511 + +# Unix socket. +# +# Specify the path for the Unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# +# unixsocket /run/redis.sock +# unixsocketperm 700 + +# Close the connection after a client is idle for N seconds (0 to disable) +timeout 0 + +# TCP keepalive. +# +# If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence +# of communication. This is useful for two reasons: +# +# 1) Detect dead peers. +# 2) Force network equipment in the middle to consider the connection to be +# alive. +# +# On Linux, the specified value (in seconds) is the period used to send ACKs. +# Note that to close the connection the double of the time is needed. +# On other kernels the period depends on the kernel configuration. +# +# A reasonable value for this option is 300 seconds, which is the new +# Redis default starting with Redis 3.2.1. +tcp-keepalive 300 + +################################# TLS/SSL ##################################### + +# By default, TLS/SSL is disabled. To enable it, the "tls-port" configuration +# directive can be used to define TLS-listening ports. To enable TLS on the +# default port, use: +# +# port 0 +# tls-port 6379 + +# Configure a X.509 certificate and private key to use for authenticating the +# server to connected clients, masters or cluster peers. These files should be +# PEM formatted. +# +# tls-cert-file redis.crt +# tls-key-file redis.key +# +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-key-file-pass secret + +# Normally Redis uses the same certificate for both server functions (accepting +# connections) and client functions (replicating from a master, establishing +# cluster bus connections, etc.). +# +# Sometimes certificates are issued with attributes that designate them as +# client-only or server-only certificates. In that case it may be desired to use +# different certificates for incoming (server) and outgoing (client) +# connections. To do that, use the following directives: +# +# tls-client-cert-file client.crt +# tls-client-key-file client.key +# +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-client-key-file-pass secret + +# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: +# +# tls-dh-params-file redis.dh + +# Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL +# clients and peers. Redis requires an explicit configuration of at least one +# of these, and will not implicitly use the system wide configuration. +# +# tls-ca-cert-file ca.crt +# tls-ca-cert-dir /etc/ssl/certs + +# By default, clients (including replica servers) on a TLS port are required +# to authenticate using valid client side certificates. +# +# If "no" is specified, client certificates are not required and not accepted. +# If "optional" is specified, client certificates are accepted and must be +# valid if provided, but are not required. +# +# tls-auth-clients no +# tls-auth-clients optional + +# By default, a Redis replica does not attempt to establish a TLS connection +# with its master. +# +# Use the following directive to enable TLS on replication links. +# +# tls-replication yes + +# By default, the Redis Cluster bus uses a plain TCP connection. To enable +# TLS for the bus protocol, use the following directive: +# +# tls-cluster yes + +# By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended +# that older formally deprecated versions are kept disabled to reduce the attack surface. +# You can explicitly specify TLS versions to support. +# Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2", +# "TLSv1.3" (OpenSSL >= 1.1.1) or any combination. +# To enable only TLSv1.2 and TLSv1.3, use: +# +# tls-protocols "TLSv1.2 TLSv1.3" + +# Configure allowed ciphers. See the ciphers(1ssl) manpage for more information +# about the syntax of this string. +# +# Note: this configuration applies only to <= TLSv1.2. +# +# tls-ciphers DEFAULT:!MEDIUM + +# Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more +# information about the syntax of this string, and specifically for TLSv1.3 +# ciphersuites. +# +# tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 + +# When choosing a cipher, use the server's preference instead of the client +# preference. By default, the server follows the client's preference. +# +# tls-prefer-server-ciphers yes + +# By default, TLS session caching is enabled to allow faster and less expensive +# reconnections by clients that support it. Use the following directive to disable +# caching. +# +# tls-session-caching no + +# Change the default number of TLS sessions cached. A zero value sets the cache +# to unlimited size. The default size is 20480. +# +# tls-session-cache-size 5000 + +# Change the default timeout of cached TLS sessions. The default timeout is 300 +# seconds. +# +# tls-session-cache-timeout 60 + +################################# GENERAL ##################################### + +# By default Redis does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +# When Redis is supervised by upstart or systemd, this parameter has no impact. +daemonize no + +# If you run Redis from upstart or systemd, Redis can interact with your +# supervision tree. Options: +# supervised no - no supervision interaction +# supervised upstart - signal upstart by putting Redis into SIGSTOP mode +# requires "expect stop" in your upstart job config +# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET +# on startup, and updating Redis status on a regular +# basis. +# supervised auto - detect upstart or systemd method based on +# UPSTART_JOB or NOTIFY_SOCKET environment variables +# Note: these supervision methods only signal "process is ready." +# They do not enable continuous pings back to your supervisor. +# +# The default is "no". To run under upstart/systemd, you can simply uncomment +# the line below: +# +# supervised auto + +# If a pid file is specified, Redis writes it where specified at startup +# and removes it at exit. +# +# When the server runs non daemonized, no pid file is created if none is +# specified in the configuration. When the server is daemonized, the pid file +# is used even if not specified, defaulting to "/var/run/redis.pid". +# +# Creating a pid file is best effort: if Redis is not able to create it +# nothing bad happens, the server will start and run normally. +# +# Note that on modern Linux systems "/run/redis.pid" is more conforming +# and should be used instead. +pidfile /var/run/redis_6379.pid + +# Specify the server verbosity level. +# This can be one of: +# debug (a lot of information, useful for development/testing) +# verbose (many rarely useful info, but not a mess like the debug level) +# notice (moderately verbose, what you want in production probably) +# warning (only very important / critical messages are logged) +loglevel notice + +# Specify the log file name. Also the empty string can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile "" + +# To enable logging to the system logger, just set 'syslog-enabled' to yes, +# and optionally update the other syslog parameters to suit your needs. +# syslog-enabled no + +# Specify the syslog identity. +# syslog-ident redis + +# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. +# syslog-facility local0 + +# To disable the built in crash log, which will possibly produce cleaner core +# dumps when they are needed, uncomment the following: +# +# crash-log-enabled no + +# To disable the fast memory check that's run as part of the crash log, which +# will possibly let redis terminate sooner, uncomment the following: +# +# crash-memcheck-enabled no + +# Set the number of databases. The default database is DB 0, you can select +# a different one on a per-connection basis using SELECT where +# dbid is a number between 0 and 'databases'-1 +databases 16 + +# By default Redis shows an ASCII art logo only when started to log to the +# standard output and if the standard output is a TTY and syslog logging is +# disabled. Basically this means that normally a logo is displayed only in +# interactive sessions. +# +# However it is possible to force the pre-4.0 behavior and always show a +# ASCII art logo in startup logs by setting the following option to yes. +always-show-logo no + +# By default, Redis modifies the process title (as seen in 'top' and 'ps') to +# provide some runtime information. It is possible to disable this and leave +# the process name as executed by setting the following to no. +set-proc-title yes + +# When changing the process title, Redis uses the following template to construct +# the modified title. +# +# Template variables are specified in curly brackets. The following variables are +# supported: +# +# {title} Name of process as executed if parent, or type of child process. +# {listen-addr} Bind address or '*' followed by TCP or TLS port listening on, or +# Unix socket if only that's available. +# {server-mode} Special mode, i.e. "[sentinel]" or "[cluster]". +# {port} TCP port listening on, or 0. +# {tls-port} TLS port listening on, or 0. +# {unixsocket} Unix domain socket listening on, or "". +# {config-file} Name of configuration file used. +# +proc-title-template "{title} {listen-addr} {server-mode}" + +################################ SNAPSHOTTING ################################ + +# Save the DB to disk. +# +# save +# +# Redis will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. +# +# Snapshotting can be completely disabled with a single empty string argument +# as in following example: +# +# save "" +# +# Unless specified otherwise, by default Redis will save the DB: +# * After 3600 seconds (an hour) if at least 1 key changed +# * After 300 seconds (5 minutes) if at least 100 keys changed +# * After 60 seconds if at least 10000 keys changed +# +# You can set these explicitly by uncommenting the three following lines. +# +# save 3600 1 +save 300 100 +# save 60 10000 + +# By default Redis will stop accepting writes if RDB snapshots are enabled +# (at least one save point) and the latest background save failed. +# This will make the user aware (in a hard way) that data is not persisting +# on disk properly, otherwise chances are that no one will notice and some +# disaster will happen. +# +# If the background saving process will start working again Redis will +# automatically allow writes again. +# +# However if you have setup your proper monitoring of the Redis server +# and persistence, you may want to disable this feature so that Redis will +# continue to work as usual even if there are problems with disk, +# permissions, and so forth. +stop-writes-on-bgsave-error no + +# Compress string objects using LZF when dump .rdb databases? +# By default compression is enabled as it's almost always a win. +# If you want to save some CPU in the saving child set it to 'no' but +# the dataset will likely be bigger if you have compressible values or keys. +rdbcompression yes + +# Since version 5 of RDB a CRC64 checksum is placed at the end of the file. +# This makes the format more resistant to corruption but there is a performance +# hit to pay (around 10%) when saving and loading RDB files, so you can disable it +# for maximum performances. +# +# RDB files created with checksum disabled have a checksum of zero that will +# tell the loading code to skip the check. +rdbchecksum yes + +# Enables or disables full sanitation checks for ziplist and listpack etc when +# loading an RDB or RESTORE payload. This reduces the chances of a assertion or +# crash later on while processing commands. +# Options: +# no - Never perform full sanitation +# yes - Always perform full sanitation +# clients - Perform full sanitation only for user connections. +# Excludes: RDB files, RESTORE commands received from the master +# connection, and client connections which have the +# skip-sanitize-payload ACL flag. +# The default should be 'clients' but since it currently affects cluster +# resharding via MIGRATE, it is temporarily set to 'no' by default. +# +# sanitize-dump-payload no + +# The filename where to dump the DB +dbfilename dump.rdb + +# Remove RDB files used by replication in instances without persistence +# enabled. By default this option is disabled, however there are environments +# where for regulations or other security concerns, RDB files persisted on +# disk by masters in order to feed replicas, or stored on disk by replicas +# in order to load them for the initial synchronization, should be deleted +# ASAP. Note that this option ONLY WORKS in instances that have both AOF +# and RDB persistence disabled, otherwise is completely ignored. +# +# An alternative (and sometimes better) way to obtain the same effect is +# to use diskless replication on both master and replicas instances. However +# in the case of replicas, diskless is not always an option. +rdb-del-sync-files no + +# The working directory. +# +# The DB will be written inside this directory, with the filename specified +# above using the 'dbfilename' configuration directive. +# +# The Append Only File will also be created inside this directory. +# +# Note that you must specify a directory here, not a file name. +dir ./ + +################################# REPLICATION ################################# + +# Master-Replica replication. Use replicaof to make a Redis instance a copy of +# another Redis server. A few things to understand ASAP about Redis replication. +# +# +------------------+ +---------------+ +# | Master | ---> | Replica | +# | (receive writes) | | (exact copy) | +# +------------------+ +---------------+ +# +# 1) Redis replication is asynchronous, but you can configure a master to +# stop accepting writes if it appears to be not connected with at least +# a given number of replicas. +# 2) Redis replicas are able to perform a partial resynchronization with the +# master if the replication link is lost for a relatively small amount of +# time. You may want to configure the replication backlog size (see the next +# sections of this file) with a sensible value depending on your needs. +# 3) Replication is automatic and does not need user intervention. After a +# network partition replicas automatically try to reconnect to masters +# and resynchronize with them. +# +# replicaof + +# If the master is password protected (using the "requirepass" configuration +# directive below) it is possible to tell the replica to authenticate before +# starting the replication synchronization process, otherwise the master will +# refuse the replica request. +# +# masterauth +# +# However this is not enough if you are using Redis ACLs (for Redis version +# 6 or greater), and the default user is not capable of running the PSYNC +# command and/or other commands needed for replication. In this case it's +# better to configure a special user to use with replication, and specify the +# masteruser configuration as such: +# +# masteruser +# +# When masteruser is specified, the replica will authenticate against its +# master using the new AUTH form: AUTH . + +# When a replica loses its connection with the master, or when the replication +# is still in progress, the replica can act in two different ways: +# +# 1) if replica-serve-stale-data is set to 'yes' (the default) the replica will +# still reply to client requests, possibly with out of date data, or the +# data set may just be empty if this is the first synchronization. +# +# 2) If replica-serve-stale-data is set to 'no' the replica will reply with +# an error "SYNC with master in progress" to all commands except: +# INFO, REPLICAOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, SUBSCRIBE, +# UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, COMMAND, POST, +# HOST and LATENCY. +# +replica-serve-stale-data yes + +# You can configure a replica instance to accept writes or not. Writing against +# a replica instance may be useful to store some ephemeral data (because data +# written on a replica will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it because of a +# misconfiguration. +# +# Since Redis 2.6 by default replicas are read-only. +# +# Note: read only replicas are not designed to be exposed to untrusted clients +# on the internet. It's just a protection layer against misuse of the instance. +# Still a read only replica exports by default all the administrative commands +# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve +# security of read only replicas using 'rename-command' to shadow all the +# administrative / dangerous commands. +replica-read-only yes + +# Replication SYNC strategy: disk or socket. +# +# New replicas and reconnecting replicas that are not able to continue the +# replication process just receiving differences, need to do what is called a +# "full synchronization". An RDB file is transmitted from the master to the +# replicas. +# +# The transmission can happen in two different ways: +# +# 1) Disk-backed: The Redis master creates a new process that writes the RDB +# file on disk. Later the file is transferred by the parent +# process to the replicas incrementally. +# 2) Diskless: The Redis master creates a new process that directly writes the +# RDB file to replica sockets, without touching the disk at all. +# +# With disk-backed replication, while the RDB file is generated, more replicas +# can be queued and served with the RDB file as soon as the current child +# producing the RDB file finishes its work. With diskless replication instead +# once the transfer starts, new replicas arriving will be queued and a new +# transfer will start when the current one terminates. +# +# When diskless replication is used, the master waits a configurable amount of +# time (in seconds) before starting the transfer in the hope that multiple +# replicas will arrive and the transfer can be parallelized. +# +# With slow disks and fast (large bandwidth) networks, diskless replication +# works better. +repl-diskless-sync no + +# When diskless replication is enabled, it is possible to configure the delay +# the server waits in order to spawn the child that transfers the RDB via socket +# to the replicas. +# +# This is important since once the transfer starts, it is not possible to serve +# new replicas arriving, that will be queued for the next RDB transfer, so the +# server waits a delay in order to let more replicas arrive. +# +# The delay is specified in seconds, and by default is 5 seconds. To disable +# it entirely just set it to 0 seconds and the transfer will start ASAP. +repl-diskless-sync-delay 5 + +# ----------------------------------------------------------------------------- +# WARNING: RDB diskless load is experimental. Since in this setup the replica +# does not immediately store an RDB on disk, it may cause data loss during +# failovers. RDB diskless load + Redis modules not handling I/O reads may also +# cause Redis to abort in case of I/O errors during the initial synchronization +# stage with the master. Use only if you know what you are doing. +# ----------------------------------------------------------------------------- +# +# Replica can load the RDB it reads from the replication link directly from the +# socket, or store the RDB to a file and read that file after it was completely +# received from the master. +# +# In many cases the disk is slower than the network, and storing and loading +# the RDB file may increase replication time (and even increase the master's +# Copy on Write memory and salve buffers). +# However, parsing the RDB file directly from the socket may mean that we have +# to flush the contents of the current database before the full rdb was +# received. For this reason we have the following options: +# +# "disabled" - Don't use diskless load (store the rdb file to the disk first) +# "on-empty-db" - Use diskless load only when it is completely safe. +# "swapdb" - Keep a copy of the current db contents in RAM while parsing +# the data directly from the socket. note that this requires +# sufficient memory, if you don't have it, you risk an OOM kill. +repl-diskless-load disabled + +# Replicas send PINGs to server in a predefined interval. It's possible to +# change this interval with the repl_ping_replica_period option. The default +# value is 10 seconds. +# +# repl-ping-replica-period 10 + +# The following option sets the replication timeout for: +# +# 1) Bulk transfer I/O during SYNC, from the point of view of replica. +# 2) Master timeout from the point of view of replicas (data, pings). +# 3) Replica timeout from the point of view of masters (REPLCONF ACK pings). +# +# It is important to make sure that this value is greater than the value +# specified for repl-ping-replica-period otherwise a timeout will be detected +# every time there is low traffic between the master and the replica. The default +# value is 60 seconds. +# +# repl-timeout 60 + +# Disable TCP_NODELAY on the replica socket after SYNC? +# +# If you select "yes" Redis will use a smaller number of TCP packets and +# less bandwidth to send data to replicas. But this can add a delay for +# the data to appear on the replica side, up to 40 milliseconds with +# Linux kernels using a default configuration. +# +# If you select "no" the delay for data to appear on the replica side will +# be reduced but more bandwidth will be used for replication. +# +# By default we optimize for low latency, but in very high traffic conditions +# or when the master and replicas are many hops away, turning this to "yes" may +# be a good idea. +repl-disable-tcp-nodelay no + +# Set the replication backlog size. The backlog is a buffer that accumulates +# replica data when replicas are disconnected for some time, so that when a +# replica wants to reconnect again, often a full resync is not needed, but a +# partial resync is enough, just passing the portion of data the replica +# missed while disconnected. +# +# The bigger the replication backlog, the longer the replica can endure the +# disconnect and later be able to perform a partial resynchronization. +# +# The backlog is only allocated if there is at least one replica connected. +# +# repl-backlog-size 1mb + +# After a master has no connected replicas for some time, the backlog will be +# freed. The following option configures the amount of seconds that need to +# elapse, starting from the time the last replica disconnected, for the backlog +# buffer to be freed. +# +# Note that replicas never free the backlog for timeout, since they may be +# promoted to masters later, and should be able to correctly "partially +# resynchronize" with other replicas: hence they should always accumulate backlog. +# +# A value of 0 means to never release the backlog. +# +# repl-backlog-ttl 3600 + +# The replica priority is an integer number published by Redis in the INFO +# output. It is used by Redis Sentinel in order to select a replica to promote +# into a master if the master is no longer working correctly. +# +# A replica with a low priority number is considered better for promotion, so +# for instance if there are three replicas with priority 10, 100, 25 Sentinel +# will pick the one with priority 10, that is the lowest. +# +# However a special priority of 0 marks the replica as not able to perform the +# role of master, so a replica with priority of 0 will never be selected by +# Redis Sentinel for promotion. +# +# By default the priority is 100. +replica-priority 100 + +# ----------------------------------------------------------------------------- +# By default, Redis Sentinel includes all replicas in its reports. A replica +# can be excluded from Redis Sentinel's announcements. An unannounced replica +# will be ignored by the 'sentinel replicas ' command and won't be +# exposed to Redis Sentinel's clients. +# +# This option does not change the behavior of replica-priority. Even with +# replica-announced set to 'no', the replica can be promoted to master. To +# prevent this behavior, set replica-priority to 0. +# +# replica-announced yes + +# It is possible for a master to stop accepting writes if there are less than +# N replicas connected, having a lag less or equal than M seconds. +# +# The N replicas need to be in "online" state. +# +# The lag in seconds, that must be <= the specified value, is calculated from +# the last ping received from the replica, that is usually sent every second. +# +# This option does not GUARANTEE that N replicas will accept the write, but +# will limit the window of exposure for lost writes in case not enough replicas +# are available, to the specified number of seconds. +# +# For example to require at least 3 replicas with a lag <= 10 seconds use: +# +# min-replicas-to-write 3 +# min-replicas-max-lag 10 +# +# Setting one or the other to 0 disables the feature. +# +# By default min-replicas-to-write is set to 0 (feature disabled) and +# min-replicas-max-lag is set to 10. + +# A Redis master is able to list the address and port of the attached +# replicas in different ways. For example the "INFO replication" section +# offers this information, which is used, among other tools, by +# Redis Sentinel in order to discover replica instances. +# Another place where this info is available is in the output of the +# "ROLE" command of a master. +# +# The listed IP address and port normally reported by a replica is +# obtained in the following way: +# +# IP: The address is auto detected by checking the peer address +# of the socket used by the replica to connect with the master. +# +# Port: The port is communicated by the replica during the replication +# handshake, and is normally the port that the replica is using to +# listen for connections. +# +# However when port forwarding or Network Address Translation (NAT) is +# used, the replica may actually be reachable via different IP and port +# pairs. The following two options can be used by a replica in order to +# report to its master a specific set of IP and port, so that both INFO +# and ROLE will report those values. +# +# There is no need to use both the options if you need to override just +# the port or the IP address. +# +# replica-announce-ip 5.5.5.5 +# replica-announce-port 1234 + +############################### KEYS TRACKING ################################# + +# Redis implements server assisted support for client side caching of values. +# This is implemented using an invalidation table that remembers, using +# a radix key indexed by key name, what clients have which keys. In turn +# this is used in order to send invalidation messages to clients. Please +# check this page to understand more about the feature: +# +# https://redis.io/topics/client-side-caching +# +# When tracking is enabled for a client, all the read only queries are assumed +# to be cached: this will force Redis to store information in the invalidation +# table. When keys are modified, such information is flushed away, and +# invalidation messages are sent to the clients. However if the workload is +# heavily dominated by reads, Redis could use more and more memory in order +# to track the keys fetched by many clients. +# +# For this reason it is possible to configure a maximum fill value for the +# invalidation table. By default it is set to 1M of keys, and once this limit +# is reached, Redis will start to evict keys in the invalidation table +# even if they were not modified, just to reclaim memory: this will in turn +# force the clients to invalidate the cached values. Basically the table +# maximum size is a trade off between the memory you want to spend server +# side to track information about who cached what, and the ability of clients +# to retain cached objects in memory. +# +# If you set the value to 0, it means there are no limits, and Redis will +# retain as many keys as needed in the invalidation table. +# In the "stats" INFO section, you can find information about the number of +# keys in the invalidation table at every given moment. +# +# Note: when key tracking is used in broadcasting mode, no memory is used +# in the server side so this setting is useless. +# +# tracking-table-max-keys 1000000 + +################################## SECURITY ################################### + +# Warning: since Redis is pretty fast, an outside user can try up to +# 1 million passwords per second against a modern box. This means that you +# should use very strong passwords, otherwise they will be very easy to break. +# Note that because the password is really a shared secret between the client +# and the server, and should not be memorized by any human, the password +# can be easily a long string from /dev/urandom or whatever, so by using a +# long and unguessable password no brute force attack will be possible. + +# Redis ACL users are defined in the following format: +# +# user ... acl rules ... +# +# For example: +# +# user worker +@list +@connection ~jobs:* on >ffa9203c493aa99 +# +# The special username "default" is used for new connections. If this user +# has the "nopass" rule, then new connections will be immediately authenticated +# as the "default" user without the need of any password provided via the +# AUTH command. Otherwise if the "default" user is not flagged with "nopass" +# the connections will start in not authenticated state, and will require +# AUTH (or the HELLO command AUTH option) in order to be authenticated and +# start to work. +# +# The ACL rules that describe what a user can do are the following: +# +# on Enable the user: it is possible to authenticate as this user. +# off Disable the user: it's no longer possible to authenticate +# with this user, however the already authenticated connections +# will still work. +# skip-sanitize-payload RESTORE dump-payload sanitation is skipped. +# sanitize-payload RESTORE dump-payload is sanitized (default). +# + Allow the execution of that command +# - Disallow the execution of that command +# +@ Allow the execution of all the commands in such category +# with valid categories are like @admin, @set, @sortedset, ... +# and so forth, see the full list in the server.c file where +# the Redis command table is described and defined. +# The special category @all means all the commands, but currently +# present in the server, and that will be loaded in the future +# via modules. +# +|subcommand Allow a specific subcommand of an otherwise +# disabled command. Note that this form is not +# allowed as negative like -DEBUG|SEGFAULT, but +# only additive starting with "+". +# allcommands Alias for +@all. Note that it implies the ability to execute +# all the future commands loaded via the modules system. +# nocommands Alias for -@all. +# ~ Add a pattern of keys that can be mentioned as part of +# commands. For instance ~* allows all the keys. The pattern +# is a glob-style pattern like the one of KEYS. +# It is possible to specify multiple patterns. +# allkeys Alias for ~* +# resetkeys Flush the list of allowed keys patterns. +# & Add a glob-style pattern of Pub/Sub channels that can be +# accessed by the user. It is possible to specify multiple channel +# patterns. +# allchannels Alias for &* +# resetchannels Flush the list of allowed channel patterns. +# > Add this password to the list of valid password for the user. +# For example >mypass will add "mypass" to the list. +# This directive clears the "nopass" flag (see later). +# < Remove this password from the list of valid passwords. +# nopass All the set passwords of the user are removed, and the user +# is flagged as requiring no password: it means that every +# password will work against this user. If this directive is +# used for the default user, every new connection will be +# immediately authenticated with the default user without +# any explicit AUTH command required. Note that the "resetpass" +# directive will clear this condition. +# resetpass Flush the list of allowed passwords. Moreover removes the +# "nopass" status. After "resetpass" the user has no associated +# passwords and there is no way to authenticate without adding +# some password (or setting it as "nopass" later). +# reset Performs the following actions: resetpass, resetkeys, off, +# -@all. The user returns to the same state it has immediately +# after its creation. +# +# ACL rules can be specified in any order: for instance you can start with +# passwords, then flags, or key patterns. However note that the additive +# and subtractive rules will CHANGE MEANING depending on the ordering. +# For instance see the following example: +# +# user alice on +@all -DEBUG ~* >somepassword +# +# This will allow "alice" to use all the commands with the exception of the +# DEBUG command, since +@all added all the commands to the set of the commands +# alice can use, and later DEBUG was removed. However if we invert the order +# of two ACL rules the result will be different: +# +# user alice on -DEBUG +@all ~* >somepassword +# +# Now DEBUG was removed when alice had yet no commands in the set of allowed +# commands, later all the commands are added, so the user will be able to +# execute everything. +# +# Basically ACL rules are processed left-to-right. +# +# For more information about ACL configuration please refer to +# the Redis web site at https://redis.io/topics/acl + +# ACL LOG +# +# The ACL Log tracks failed commands and authentication events associated +# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked +# by ACLs. The ACL Log is stored in memory. You can reclaim memory with +# ACL LOG RESET. Define the maximum entry length of the ACL Log below. +acllog-max-len 128 + +# Using an external ACL file +# +# Instead of configuring users here in this file, it is possible to use +# a stand-alone file just listing users. The two methods cannot be mixed: +# if you configure users here and at the same time you activate the external +# ACL file, the server will refuse to start. +# +# The format of the external ACL user file is exactly the same as the +# format that is used inside redis.conf to describe users. +# +# aclfile /etc/redis/users.acl + +# IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatibility +# layer on top of the new ACL system. The option effect will be just setting +# the password for the default user. Clients will still authenticate using +# AUTH as usually, or more explicitly with AUTH default +# if they follow the new protocol: both will work. +# +# The requirepass is not compatable with aclfile option and the ACL LOAD +# command, these will cause requirepass to be ignored. +# +# requirepass foobared + +# New users are initialized with restrictive permissions by default, via the +# equivalent of this ACL rule 'off resetkeys -@all'. Starting with Redis 6.2, it +# is possible to manage access to Pub/Sub channels with ACL rules as well. The +# default Pub/Sub channels permission if new users is controlled by the +# acl-pubsub-default configuration directive, which accepts one of these values: +# +# allchannels: grants access to all Pub/Sub channels +# resetchannels: revokes access to all Pub/Sub channels +# +# To ensure backward compatibility while upgrading Redis 6.0, acl-pubsub-default +# defaults to the 'allchannels' permission. +# +# Future compatibility note: it is very likely that in a future version of Redis +# the directive's default of 'allchannels' will be changed to 'resetchannels' in +# order to provide better out-of-the-box Pub/Sub security. Therefore, it is +# recommended that you explicitly define Pub/Sub permissions for all users +# rather then rely on implicit default values. Once you've set explicit +# Pub/Sub for all existing users, you should uncomment the following line. +# +# acl-pubsub-default resetchannels + +# Command renaming (DEPRECATED). +# +# ------------------------------------------------------------------------ +# WARNING: avoid using this option if possible. Instead use ACLs to remove +# commands from the default user, and put them only in some admin user you +# create for administrative purposes. +# ------------------------------------------------------------------------ +# +# It is possible to change the name of dangerous commands in a shared +# environment. For instance the CONFIG command may be renamed into something +# hard to guess so that it will still be available for internal-use tools +# but not available for general clients. +# +# Example: +# +# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 +# +# It is also possible to completely kill a command by renaming it into +# an empty string: +# +# rename-command CONFIG "" +# +# Please note that changing the name of commands that are logged into the +# AOF file or transmitted to replicas may cause problems. + +################################### CLIENTS #################################### + +# Set the max number of connected clients at the same time. By default +# this limit is set to 10000 clients, however if the Redis server is not +# able to configure the process file limit to allow for the specified limit +# the max number of allowed clients is set to the current file limit +# minus 32 (as Redis reserves a few file descriptors for internal uses). +# +# Once the limit is reached Redis will close all the new connections sending +# an error 'max number of clients reached'. +# +# IMPORTANT: When Redis Cluster is used, the max number of connections is also +# shared with the cluster bus: every node in the cluster will use two +# connections, one incoming and another outgoing. It is important to size the +# limit accordingly in case of very large clusters. +# +# maxclients 10000 + +############################## MEMORY MANAGEMENT ################################ + +# Set a memory usage limit to the specified amount of bytes. +# When the memory limit is reached Redis will try to remove keys +# according to the eviction policy selected (see maxmemory-policy). +# +# If Redis can't remove keys according to the policy, or if the policy is +# set to 'noeviction', Redis will start to reply with errors to commands +# that would use more memory, like SET, LPUSH, and so on, and will continue +# to reply to read-only commands like GET. +# +# This option is usually useful when using Redis as an LRU or LFU cache, or to +# set a hard memory limit for an instance (using the 'noeviction' policy). +# +# WARNING: If you have replicas attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the replicas are subtracted +# from the used memory count, so that network problems / resyncs will +# not trigger a loop where keys are evicted, and in turn the output +# buffer of replicas is full with DELs of keys evicted triggering the deletion +# of more keys, and so forth until the database is completely emptied. +# +# In short... if you have replicas attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for replica +# output buffers (but this is not needed if the policy is 'noeviction'). +# +# maxmemory + +# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory +# is reached. You can select one from the following behaviors: +# +# volatile-lru -> Evict using approximated LRU, only keys with an expire set. +# allkeys-lru -> Evict any key using approximated LRU. +# volatile-lfu -> Evict using approximated LFU, only keys with an expire set. +# allkeys-lfu -> Evict any key using approximated LFU. +# volatile-random -> Remove a random key having an expire set. +# allkeys-random -> Remove a random key, any key. +# volatile-ttl -> Remove the key with the nearest expire time (minor TTL) +# noeviction -> Don't evict anything, just return an error on write operations. +# +# LRU means Least Recently Used +# LFU means Least Frequently Used +# +# Both LRU, LFU and volatile-ttl are implemented using approximated +# randomized algorithms. +# +# Note: with any of the above policies, when there are no suitable keys for +# eviction, Redis will return an error on write operations that require +# more memory. These are usually commands that create new keys, add data or +# modify existing keys. A few examples are: SET, INCR, HSET, LPUSH, SUNIONSTORE, +# SORT (due to the STORE argument), and EXEC (if the transaction includes any +# command that requires memory). +# +# The default is: +# +# maxmemory-policy noeviction + +# LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated +# algorithms (in order to save memory), so you can tune it for speed or +# accuracy. By default Redis will check five keys and pick the one that was +# used least recently, you can change the sample size using the following +# configuration directive. +# +# The default of 5 produces good enough results. 10 Approximates very closely +# true LRU but costs more CPU. 3 is faster but not very accurate. +# +# maxmemory-samples 5 + +# Eviction processing is designed to function well with the default setting. +# If there is an unusually large amount of write traffic, this value may need to +# be increased. Decreasing this value may reduce latency at the risk of +# eviction processing effectiveness +# 0 = minimum latency, 10 = default, 100 = process without regard to latency +# +# maxmemory-eviction-tenacity 10 + +# Starting from Redis 5, by default a replica will ignore its maxmemory setting +# (unless it is promoted to master after a failover or manually). It means +# that the eviction of keys will be just handled by the master, sending the +# DEL commands to the replica as keys evict in the master side. +# +# This behavior ensures that masters and replicas stay consistent, and is usually +# what you want, however if your replica is writable, or you want the replica +# to have a different memory setting, and you are sure all the writes performed +# to the replica are idempotent, then you may change this default (but be sure +# to understand what you are doing). +# +# Note that since the replica by default does not evict, it may end using more +# memory than the one set via maxmemory (there are certain buffers that may +# be larger on the replica, or data structures may sometimes take more memory +# and so forth). So make sure you monitor your replicas and make sure they +# have enough memory to never hit a real out-of-memory condition before the +# master hits the configured maxmemory setting. +# +# replica-ignore-maxmemory yes + +# Redis reclaims expired keys in two ways: upon access when those keys are +# found to be expired, and also in background, in what is called the +# "active expire key". The key space is slowly and interactively scanned +# looking for expired keys to reclaim, so that it is possible to free memory +# of keys that are expired and will never be accessed again in a short time. +# +# The default effort of the expire cycle will try to avoid having more than +# ten percent of expired keys still in memory, and will try to avoid consuming +# more than 25% of total memory and to add latency to the system. However +# it is possible to increase the expire "effort" that is normally set to +# "1", to a greater value, up to the value "10". At its maximum value the +# system will use more CPU, longer cycles (and technically may introduce +# more latency), and will tolerate less already expired keys still present +# in the system. It's a tradeoff between memory, CPU and latency. +# +# active-expire-effort 1 + +############################# LAZY FREEING #################################### + +# Redis has two primitives to delete keys. One is called DEL and is a blocking +# deletion of the object. It means that the server stops processing new commands +# in order to reclaim all the memory associated with an object in a synchronous +# way. If the key deleted is associated with a small object, the time needed +# in order to execute the DEL command is very small and comparable to most other +# O(1) or O(log_N) commands in Redis. However if the key is associated with an +# aggregated value containing millions of elements, the server can block for +# a long time (even seconds) in order to complete the operation. +# +# For the above reasons Redis also offers non blocking deletion primitives +# such as UNLINK (non blocking DEL) and the ASYNC option of FLUSHALL and +# FLUSHDB commands, in order to reclaim memory in background. Those commands +# are executed in constant time. Another thread will incrementally free the +# object in the background as fast as possible. +# +# DEL, UNLINK and ASYNC option of FLUSHALL and FLUSHDB are user-controlled. +# It's up to the design of the application to understand when it is a good +# idea to use one or the other. However the Redis server sometimes has to +# delete keys or flush the whole database as a side effect of other operations. +# Specifically Redis deletes objects independently of a user call in the +# following scenarios: +# +# 1) On eviction, because of the maxmemory and maxmemory policy configurations, +# in order to make room for new data, without going over the specified +# memory limit. +# 2) Because of expire: when a key with an associated time to live (see the +# EXPIRE command) must be deleted from memory. +# 3) Because of a side effect of a command that stores data on a key that may +# already exist. For example the RENAME command may delete the old key +# content when it is replaced with another one. Similarly SUNIONSTORE +# or SORT with STORE option may delete existing keys. The SET command +# itself removes any old content of the specified key in order to replace +# it with the specified string. +# 4) During replication, when a replica performs a full resynchronization with +# its master, the content of the whole database is removed in order to +# load the RDB file just transferred. +# +# In all the above cases the default is to delete objects in a blocking way, +# like if DEL was called. However you can configure each case specifically +# in order to instead release memory in a non-blocking way like if UNLINK +# was called, using the following configuration directives. + +lazyfree-lazy-eviction no +lazyfree-lazy-expire no +lazyfree-lazy-server-del no +replica-lazy-flush no + +# It is also possible, for the case when to replace the user code DEL calls +# with UNLINK calls is not easy, to modify the default behavior of the DEL +# command to act exactly like UNLINK, using the following configuration +# directive: + +lazyfree-lazy-user-del no + +# FLUSHDB, FLUSHALL, and SCRIPT FLUSH support both asynchronous and synchronous +# deletion, which can be controlled by passing the [SYNC|ASYNC] flags into the +# commands. When neither flag is passed, this directive will be used to determine +# if the data should be deleted asynchronously. + +lazyfree-lazy-user-flush no + +################################ THREADED I/O ################################# + +# Redis is mostly single threaded, however there are certain threaded +# operations such as UNLINK, slow I/O accesses and other things that are +# performed on side threads. +# +# Now it is also possible to handle Redis clients socket reads and writes +# in different I/O threads. Since especially writing is so slow, normally +# Redis users use pipelining in order to speed up the Redis performances per +# core, and spawn multiple instances in order to scale more. Using I/O +# threads it is possible to easily speedup two times Redis without resorting +# to pipelining nor sharding of the instance. +# +# By default threading is disabled, we suggest enabling it only in machines +# that have at least 4 or more cores, leaving at least one spare core. +# Using more than 8 threads is unlikely to help much. We also recommend using +# threaded I/O only if you actually have performance problems, with Redis +# instances being able to use a quite big percentage of CPU time, otherwise +# there is no point in using this feature. +# +# So for instance if you have a four cores boxes, try to use 2 or 3 I/O +# threads, if you have a 8 cores, try to use 6 threads. In order to +# enable I/O threads use the following configuration directive: +# +# io-threads 4 +# +# Setting io-threads to 1 will just use the main thread as usual. +# When I/O threads are enabled, we only use threads for writes, that is +# to thread the write(2) syscall and transfer the client buffers to the +# socket. However it is also possible to enable threading of reads and +# protocol parsing using the following configuration directive, by setting +# it to yes: +# +# io-threads-do-reads no +# +# Usually threading reads doesn't help much. +# +# NOTE 1: This configuration directive cannot be changed at runtime via +# CONFIG SET. Aso this feature currently does not work when SSL is +# enabled. +# +# NOTE 2: If you want to test the Redis speedup using redis-benchmark, make +# sure you also run the benchmark itself in threaded mode, using the +# --threads option to match the number of Redis threads, otherwise you'll not +# be able to notice the improvements. + +############################ KERNEL OOM CONTROL ############################## + +# On Linux, it is possible to hint the kernel OOM killer on what processes +# should be killed first when out of memory. +# +# Enabling this feature makes Redis actively control the oom_score_adj value +# for all its processes, depending on their role. The default scores will +# attempt to have background child processes killed before all others, and +# replicas killed before masters. +# +# Redis supports three options: +# +# no: Don't make changes to oom-score-adj (default). +# yes: Alias to "relative" see below. +# absolute: Values in oom-score-adj-values are written as is to the kernel. +# relative: Values are used relative to the initial value of oom_score_adj when +# the server starts and are then clamped to a range of -1000 to 1000. +# Because typically the initial value is 0, they will often match the +# absolute values. +oom-score-adj no + +# When oom-score-adj is used, this directive controls the specific values used +# for master, replica and background child processes. Values range -2000 to +# 2000 (higher means more likely to be killed). +# +# Unprivileged processes (not root, and without CAP_SYS_RESOURCE capabilities) +# can freely increase their value, but not decrease it below its initial +# settings. This means that setting oom-score-adj to "relative" and setting the +# oom-score-adj-values to positive values will always succeed. +oom-score-adj-values 0 200 800 + + +#################### KERNEL transparent hugepage CONTROL ###################### + +# Usually the kernel Transparent Huge Pages control is set to "madvise" or +# or "never" by default (/sys/kernel/mm/transparent_hugepage/enabled), in which +# case this config has no effect. On systems in which it is set to "always", +# redis will attempt to disable it specifically for the redis process in order +# to avoid latency problems specifically with fork(2) and CoW. +# If for some reason you prefer to keep it enabled, you can set this config to +# "no" and the kernel global to "always". + +disable-thp yes + +############################## APPEND ONLY MODE ############################### + +# By default Redis asynchronously dumps the dataset on disk. This mode is +# good enough in many applications, but an issue with the Redis process or +# a power outage may result into a few minutes of writes lost (depending on +# the configured save points). +# +# The Append Only File is an alternative persistence mode that provides +# much better durability. For instance using the default data fsync policy +# (see later in the config file) Redis can lose just one second of writes in a +# dramatic event like a server power outage, or a single write if something +# wrong with the Redis process itself happens, but the operating system is +# still running correctly. +# +# AOF and RDB persistence can be enabled at the same time without problems. +# If the AOF is enabled on startup Redis will load the AOF, that is the file +# with the better durability guarantees. +# +# Please check https://redis.io/topics/persistence for more information. + +appendonly yes + +# The name of the append only file (default: "appendonly.aof") + +appendfilename "appendonly.aof" + +# The fsync() call tells the Operating System to actually write data on disk +# instead of waiting for more data in the output buffer. Some OS will really flush +# data on disk, some other OS will just try to do it ASAP. +# +# Redis supports three different modes: +# +# no: don't fsync, just let the OS flush the data when it wants. Faster. +# always: fsync after every write to the append only log. Slow, Safest. +# everysec: fsync only one time every second. Compromise. +# +# The default is "everysec", as that's usually the right compromise between +# speed and data safety. It's up to you to understand if you can relax this to +# "no" that will let the operating system flush the output buffer when +# it wants, for better performances (but if you can live with the idea of +# some data loss consider the default persistence mode that's snapshotting), +# or on the contrary, use "always" that's very slow but a bit safer than +# everysec. +# +# More details please check the following article: +# http://antirez.com/post/redis-persistence-demystified.html +# +# If unsure, use "everysec". + +# appendfsync always +appendfsync everysec +# appendfsync no + +# When the AOF fsync policy is set to always or everysec, and a background +# saving process (a background save or AOF log background rewriting) is +# performing a lot of I/O against the disk, in some Linux configurations +# Redis may block too long on the fsync() call. Note that there is no fix for +# this currently, as even performing fsync in a different thread will block +# our synchronous write(2) call. +# +# In order to mitigate this problem it's possible to use the following option +# that will prevent fsync() from being called in the main process while a +# BGSAVE or BGREWRITEAOF is in progress. +# +# This means that while another child is saving, the durability of Redis is +# the same as "appendfsync none". In practical terms, this means that it is +# possible to lose up to 30 seconds of log in the worst scenario (with the +# default Linux settings). +# +# If you have latency problems turn this to "yes". Otherwise leave it as +# "no" that is the safest pick from the point of view of durability. + +no-appendfsync-on-rewrite no + +# Automatic rewrite of the append only file. +# Redis is able to automatically rewrite the log file implicitly calling +# BGREWRITEAOF when the AOF log size grows by the specified percentage. +# +# This is how it works: Redis remembers the size of the AOF file after the +# latest rewrite (if no rewrite has happened since the restart, the size of +# the AOF at startup is used). +# +# This base size is compared to the current size. If the current size is +# bigger than the specified percentage, the rewrite is triggered. Also +# you need to specify a minimal size for the AOF file to be rewritten, this +# is useful to avoid rewriting the AOF file even if the percentage increase +# is reached but it is still pretty small. +# +# Specify a percentage of zero in order to disable the automatic AOF +# rewrite feature. + +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb + +# An AOF file may be found to be truncated at the end during the Redis +# startup process, when the AOF data gets loaded back into memory. +# This may happen when the system where Redis is running +# crashes, especially when an ext4 filesystem is mounted without the +# data=ordered option (however this can't happen when Redis itself +# crashes or aborts but the operating system still works correctly). +# +# Redis can either exit with an error when this happens, or load as much +# data as possible (the default now) and start if the AOF file is found +# to be truncated at the end. The following option controls this behavior. +# +# If aof-load-truncated is set to yes, a truncated AOF file is loaded and +# the Redis server starts emitting a log to inform the user of the event. +# Otherwise if the option is set to no, the server aborts with an error +# and refuses to start. When the option is set to no, the user requires +# to fix the AOF file using the "redis-check-aof" utility before to restart +# the server. +# +# Note that if the AOF file will be found to be corrupted in the middle +# the server will still exit with an error. This option only applies when +# Redis will try to read more data from the AOF file but not enough bytes +# will be found. +aof-load-truncated yes + +# When rewriting the AOF file, Redis is able to use an RDB preamble in the +# AOF file for faster rewrites and recoveries. When this option is turned +# on the rewritten AOF file is composed of two different stanzas: +# +# [RDB file][AOF tail] +# +# When loading, Redis recognizes that the AOF file starts with the "REDIS" +# string and loads the prefixed RDB file, then continues loading the AOF +# tail. +aof-use-rdb-preamble yes + +################################ LUA SCRIPTING ############################### + +# Max execution time of a Lua script in milliseconds. +# +# If the maximum execution time is reached Redis will log that a script is +# still in execution after the maximum allowed time and will start to +# reply to queries with an error. +# +# When a long running script exceeds the maximum execution time only the +# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be +# used to stop a script that did not yet call any write commands. The second +# is the only way to shut down the server in the case a write command was +# already issued by the script but the user doesn't want to wait for the natural +# termination of the script. +# +# Set it to 0 or a negative value for unlimited execution without warnings. +lua-time-limit 5000 + +################################ REDIS CLUSTER ############################### + +# Normal Redis instances can't be part of a Redis Cluster; only nodes that are +# started as cluster nodes can. In order to start a Redis instance as a +# cluster node enable the cluster support uncommenting the following: +# +# cluster-enabled yes + +# Every cluster node has a cluster configuration file. This file is not +# intended to be edited by hand. It is created and updated by Redis nodes. +# Every Redis Cluster node requires a different cluster configuration file. +# Make sure that instances running in the same system do not have +# overlapping cluster configuration file names. +# +# cluster-config-file nodes-6379.conf + +# Cluster node timeout is the amount of milliseconds a node must be unreachable +# for it to be considered in failure state. +# Most other internal time limits are a multiple of the node timeout. +# +# cluster-node-timeout 15000 + +# A replica of a failing master will avoid to start a failover if its data +# looks too old. +# +# There is no simple way for a replica to actually have an exact measure of +# its "data age", so the following two checks are performed: +# +# 1) If there are multiple replicas able to failover, they exchange messages +# in order to try to give an advantage to the replica with the best +# replication offset (more data from the master processed). +# Replicas will try to get their rank by offset, and apply to the start +# of the failover a delay proportional to their rank. +# +# 2) Every single replica computes the time of the last interaction with +# its master. This can be the last ping or command received (if the master +# is still in the "connected" state), or the time that elapsed since the +# disconnection with the master (if the replication link is currently down). +# If the last interaction is too old, the replica will not try to failover +# at all. +# +# The point "2" can be tuned by user. Specifically a replica will not perform +# the failover if, since the last interaction with the master, the time +# elapsed is greater than: +# +# (node-timeout * cluster-replica-validity-factor) + repl-ping-replica-period +# +# So for example if node-timeout is 30 seconds, and the cluster-replica-validity-factor +# is 10, and assuming a default repl-ping-replica-period of 10 seconds, the +# replica will not try to failover if it was not able to talk with the master +# for longer than 310 seconds. +# +# A large cluster-replica-validity-factor may allow replicas with too old data to failover +# a master, while a too small value may prevent the cluster from being able to +# elect a replica at all. +# +# For maximum availability, it is possible to set the cluster-replica-validity-factor +# to a value of 0, which means, that replicas will always try to failover the +# master regardless of the last time they interacted with the master. +# (However they'll always try to apply a delay proportional to their +# offset rank). +# +# Zero is the only value able to guarantee that when all the partitions heal +# the cluster will always be able to continue. +# +# cluster-replica-validity-factor 10 + +# Cluster replicas are able to migrate to orphaned masters, that are masters +# that are left without working replicas. This improves the cluster ability +# to resist to failures as otherwise an orphaned master can't be failed over +# in case of failure if it has no working replicas. +# +# Replicas migrate to orphaned masters only if there are still at least a +# given number of other working replicas for their old master. This number +# is the "migration barrier". A migration barrier of 1 means that a replica +# will migrate only if there is at least 1 other working replica for its master +# and so forth. It usually reflects the number of replicas you want for every +# master in your cluster. +# +# Default is 1 (replicas migrate only if their masters remain with at least +# one replica). To disable migration just set it to a very large value or +# set cluster-allow-replica-migration to 'no'. +# A value of 0 can be set but is useful only for debugging and dangerous +# in production. +# +# cluster-migration-barrier 1 + +# Turning off this option allows to use less automatic cluster configuration. +# It both disables migration to orphaned masters and migration from masters +# that became empty. +# +# Default is 'yes' (allow automatic migrations). +# +# cluster-allow-replica-migration yes + +# By default Redis Cluster nodes stop accepting queries if they detect there +# is at least a hash slot uncovered (no available node is serving it). +# This way if the cluster is partially down (for example a range of hash slots +# are no longer covered) all the cluster becomes, eventually, unavailable. +# It automatically returns available as soon as all the slots are covered again. +# +# However sometimes you want the subset of the cluster which is working, +# to continue to accept queries for the part of the key space that is still +# covered. In order to do so, just set the cluster-require-full-coverage +# option to no. +# +# cluster-require-full-coverage yes + +# This option, when set to yes, prevents replicas from trying to failover its +# master during master failures. However the replica can still perform a +# manual failover, if forced to do so. +# +# This is useful in different scenarios, especially in the case of multiple +# data center operations, where we want one side to never be promoted if not +# in the case of a total DC failure. +# +# cluster-replica-no-failover no + +# This option, when set to yes, allows nodes to serve read traffic while the +# the cluster is in a down state, as long as it believes it owns the slots. +# +# This is useful for two cases. The first case is for when an application +# doesn't require consistency of data during node failures or network partitions. +# One example of this is a cache, where as long as the node has the data it +# should be able to serve it. +# +# The second use case is for configurations that don't meet the recommended +# three shards but want to enable cluster mode and scale later. A +# master outage in a 1 or 2 shard configuration causes a read/write outage to the +# entire cluster without this option set, with it set there is only a write outage. +# Without a quorum of masters, slot ownership will not change automatically. +# +# cluster-allow-reads-when-down no + +# In order to setup your cluster make sure to read the documentation +# available at https://redis.io web site. + +########################## CLUSTER DOCKER/NAT support ######################## + +# In certain deployments, Redis Cluster nodes address discovery fails, because +# addresses are NAT-ted or because ports are forwarded (the typical case is +# Docker and other containers). +# +# In order to make Redis Cluster working in such environments, a static +# configuration where each node knows its public address is needed. The +# following four options are used for this scope, and are: +# +# * cluster-announce-ip +# * cluster-announce-port +# * cluster-announce-tls-port +# * cluster-announce-bus-port +# +# Each instructs the node about its address, client ports (for connections +# without and with TLS) and cluster message bus port. The information is then +# published in the header of the bus packets so that other nodes will be able to +# correctly map the address of the node publishing the information. +# +# If cluster-tls is set to yes and cluster-announce-tls-port is omitted or set +# to zero, then cluster-announce-port refers to the TLS port. Note also that +# cluster-announce-tls-port has no effect if cluster-tls is set to no. +# +# If the above options are not used, the normal Redis Cluster auto-detection +# will be used instead. +# +# Note that when remapped, the bus port may not be at the fixed offset of +# clients port + 10000, so you can specify any port and bus-port depending +# on how they get remapped. If the bus-port is not set, a fixed offset of +# 10000 will be used as usual. +# +# Example: +# +# cluster-announce-ip 10.1.1.5 +# cluster-announce-tls-port 6379 +# cluster-announce-port 0 +# cluster-announce-bus-port 6380 + +################################## SLOW LOG ################################### + +# The Redis Slow Log is a system to log queries that exceeded a specified +# execution time. The execution time does not include the I/O operations +# like talking with the client, sending the reply and so forth, +# but just the time needed to actually execute the command (this is the only +# stage of command execution where the thread is blocked and can not serve +# other requests in the meantime). +# +# You can configure the slow log with two parameters: one tells Redis +# what is the execution time, in microseconds, to exceed in order for the +# command to get logged, and the other parameter is the length of the +# slow log. When a new command is logged the oldest one is removed from the +# queue of logged commands. + +# The following time is expressed in microseconds, so 1000000 is equivalent +# to one second. Note that a negative number disables the slow log, while +# a value of zero forces the logging of every command. +slowlog-log-slower-than 10000 + +# There is no limit to this length. Just be aware that it will consume memory. +# You can reclaim memory used by the slow log with SLOWLOG RESET. +slowlog-max-len 128 + +################################ LATENCY MONITOR ############################## + +# The Redis latency monitoring subsystem samples different operations +# at runtime in order to collect data related to possible sources of +# latency of a Redis instance. +# +# Via the LATENCY command this information is available to the user that can +# print graphs and obtain reports. +# +# The system only logs operations that were performed in a time equal or +# greater than the amount of milliseconds specified via the +# latency-monitor-threshold configuration directive. When its value is set +# to zero, the latency monitor is turned off. +# +# By default latency monitoring is disabled since it is mostly not needed +# if you don't have latency issues, and collecting data has a performance +# impact, that while very small, can be measured under big load. Latency +# monitoring can easily be enabled at runtime using the command +# "CONFIG SET latency-monitor-threshold " if needed. +latency-monitor-threshold 0 + +############################# EVENT NOTIFICATION ############################## + +# Redis can notify Pub/Sub clients about events happening in the key space. +# This feature is documented at https://redis.io/topics/notifications +# +# For instance if keyspace events notification is enabled, and a client +# performs a DEL operation on key "foo" stored in the Database 0, two +# messages will be published via Pub/Sub: +# +# PUBLISH __keyspace@0__:foo del +# PUBLISH __keyevent@0__:del foo +# +# It is possible to select the events that Redis will notify among a set +# of classes. Every class is identified by a single character: +# +# K Keyspace events, published with __keyspace@__ prefix. +# E Keyevent events, published with __keyevent@__ prefix. +# g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... +# $ String commands +# l List commands +# s Set commands +# h Hash commands +# z Sorted set commands +# x Expired events (events generated every time a key expires) +# e Evicted events (events generated when a key is evicted for maxmemory) +# t Stream commands +# d Module key type events +# m Key-miss events (Note: It is not included in the 'A' class) +# A Alias for g$lshzxetd, so that the "AKE" string means all the events +# (Except key-miss events which are excluded from 'A' due to their +# unique nature). +# +# The "notify-keyspace-events" takes as argument a string that is composed +# of zero or multiple characters. The empty string means that notifications +# are disabled. +# +# Example: to enable list and generic events, from the point of view of the +# event name, use: +# +# notify-keyspace-events Elg +# +# Example 2: to get the stream of the expired keys subscribing to channel +# name __keyevent@0__:expired use: +# +# notify-keyspace-events Ex +# +# By default all notifications are disabled because most users don't need +# this feature and the feature has some overhead. Note that if you don't +# specify at least one of K or E, no events will be delivered. +notify-keyspace-events "" + +############################### GOPHER SERVER ################################# + +# Redis contains an implementation of the Gopher protocol, as specified in +# the RFC 1436 (https://www.ietf.org/rfc/rfc1436.txt). +# +# The Gopher protocol was very popular in the late '90s. It is an alternative +# to the web, and the implementation both server and client side is so simple +# that the Redis server has just 100 lines of code in order to implement this +# support. +# +# What do you do with Gopher nowadays? Well Gopher never *really* died, and +# lately there is a movement in order for the Gopher more hierarchical content +# composed of just plain text documents to be resurrected. Some want a simpler +# internet, others believe that the mainstream internet became too much +# controlled, and it's cool to create an alternative space for people that +# want a bit of fresh air. +# +# Anyway for the 10nth birthday of the Redis, we gave it the Gopher protocol +# as a gift. +# +# --- HOW IT WORKS? --- +# +# The Redis Gopher support uses the inline protocol of Redis, and specifically +# two kind of inline requests that were anyway illegal: an empty request +# or any request that starts with "/" (there are no Redis commands starting +# with such a slash). Normal RESP2/RESP3 requests are completely out of the +# path of the Gopher protocol implementation and are served as usual as well. +# +# If you open a connection to Redis when Gopher is enabled and send it +# a string like "/foo", if there is a key named "/foo" it is served via the +# Gopher protocol. +# +# In order to create a real Gopher "hole" (the name of a Gopher site in Gopher +# talking), you likely need a script like the following: +# +# https://github.com/antirez/gopher2redis +# +# --- SECURITY WARNING --- +# +# If you plan to put Redis on the internet in a publicly accessible address +# to server Gopher pages MAKE SURE TO SET A PASSWORD to the instance. +# Once a password is set: +# +# 1. The Gopher server (when enabled, not by default) will still serve +# content via Gopher. +# 2. However other commands cannot be called before the client will +# authenticate. +# +# So use the 'requirepass' option to protect your instance. +# +# Note that Gopher is not currently supported when 'io-threads-do-reads' +# is enabled. +# +# To enable Gopher support, uncomment the following line and set the option +# from no (the default) to yes. +# +# gopher-enabled no + +############################### ADVANCED CONFIG ############################### + +# Hashes are encoded using a memory efficient data structure when they have a +# small number of entries, and the biggest entry does not exceed a given +# threshold. These thresholds can be configured using the following directives. +hash-max-ziplist-entries 512 +hash-max-ziplist-value 64 + +# Lists are also encoded in a special way to save a lot of space. +# The number of entries allowed per internal list node can be specified +# as a fixed maximum size or a maximum number of elements. +# For a fixed maximum size, use -5 through -1, meaning: +# -5: max size: 64 Kb <-- not recommended for normal workloads +# -4: max size: 32 Kb <-- not recommended +# -3: max size: 16 Kb <-- probably not recommended +# -2: max size: 8 Kb <-- good +# -1: max size: 4 Kb <-- good +# Positive numbers mean store up to _exactly_ that number of elements +# per list node. +# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), +# but if your use case is unique, adjust the settings as necessary. +list-max-ziplist-size -2 + +# Lists may also be compressed. +# Compress depth is the number of quicklist ziplist nodes from *each* side of +# the list to *exclude* from compression. The head and tail of the list +# are always uncompressed for fast push/pop operations. Settings are: +# 0: disable all list compression +# 1: depth 1 means "don't start compressing until after 1 node into the list, +# going from either the head or tail" +# So: [head]->node->node->...->node->[tail] +# [head], [tail] will always be uncompressed; inner nodes will compress. +# 2: [head]->[next]->node->node->...->node->[prev]->[tail] +# 2 here means: don't compress head or head->next or tail->prev or tail, +# but compress all nodes between them. +# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] +# etc. +list-compress-depth 0 + +# Sets have a special encoding in just one case: when a set is composed +# of just strings that happen to be integers in radix 10 in the range +# of 64 bit signed integers. +# The following configuration setting sets the limit in the size of the +# set in order to use this special memory saving encoding. +set-max-intset-entries 512 + +# Similarly to hashes and lists, sorted sets are also specially encoded in +# order to save a lot of space. This encoding is only used when the length and +# elements of a sorted set are below the following limits: +zset-max-ziplist-entries 128 +zset-max-ziplist-value 64 + +# HyperLogLog sparse representation bytes limit. The limit includes the +# 16 bytes header. When an HyperLogLog using the sparse representation crosses +# this limit, it is converted into the dense representation. +# +# A value greater than 16000 is totally useless, since at that point the +# dense representation is more memory efficient. +# +# The suggested value is ~ 3000 in order to have the benefits of +# the space efficient encoding without slowing down too much PFADD, +# which is O(N) with the sparse encoding. The value can be raised to +# ~ 10000 when CPU is not a concern, but space is, and the data set is +# composed of many HyperLogLogs with cardinality in the 0 - 15000 range. +hll-sparse-max-bytes 3000 + +# Streams macro node max size / items. The stream data structure is a radix +# tree of big nodes that encode multiple items inside. Using this configuration +# it is possible to configure how big a single node can be in bytes, and the +# maximum number of items it may contain before switching to a new node when +# appending new stream entries. If any of the following settings are set to +# zero, the limit is ignored, so for instance it is possible to set just a +# max entries limit by setting max-bytes to 0 and max-entries to the desired +# value. +stream-node-max-bytes 4096 +stream-node-max-entries 100 + +# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in +# order to help rehashing the main Redis hash table (the one mapping top-level +# keys to values). The hash table implementation Redis uses (see dict.c) +# performs a lazy rehashing: the more operation you run into a hash table +# that is rehashing, the more rehashing "steps" are performed, so if the +# server is idle the rehashing is never complete and some more memory is used +# by the hash table. +# +# The default is to use this millisecond 10 times every second in order to +# actively rehash the main dictionaries, freeing memory when possible. +# +# If unsure: +# use "activerehashing no" if you have hard latency requirements and it is +# not a good thing in your environment that Redis can reply from time to time +# to queries with 2 milliseconds delay. +# +# use "activerehashing yes" if you don't have such hard requirements but +# want to free memory asap when possible. +activerehashing yes + +# The client output buffer limits can be used to force disconnection of clients +# that are not reading data from the server fast enough for some reason (a +# common reason is that a Pub/Sub client can't consume messages as fast as the +# publisher can produce them). +# +# The limit can be set differently for the three different classes of clients: +# +# normal -> normal clients including MONITOR clients +# replica -> replica clients +# pubsub -> clients subscribed to at least one pubsub channel or pattern +# +# The syntax of every client-output-buffer-limit directive is the following: +# +# client-output-buffer-limit +# +# A client is immediately disconnected once the hard limit is reached, or if +# the soft limit is reached and remains reached for the specified number of +# seconds (continuously). +# So for instance if the hard limit is 32 megabytes and the soft limit is +# 16 megabytes / 10 seconds, the client will get disconnected immediately +# if the size of the output buffers reach 32 megabytes, but will also get +# disconnected if the client reaches 16 megabytes and continuously overcomes +# the limit for 10 seconds. +# +# By default normal clients are not limited because they don't receive data +# without asking (in a push way), but just after a request, so only +# asynchronous clients may create a scenario where data is requested faster +# than it can read. +# +# Instead there is a default limit for pubsub and replica clients, since +# subscribers and replicas receive data in a push fashion. +# +# Both the hard or the soft limit can be disabled by setting them to zero. +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit replica 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 + +# Client query buffers accumulate new commands. They are limited to a fixed +# amount by default in order to avoid that a protocol desynchronization (for +# instance due to a bug in the client) will lead to unbound memory usage in +# the query buffer. However you can configure it here if you have very special +# needs, such us huge multi/exec requests or alike. +# +# client-query-buffer-limit 1gb + +# In the Redis protocol, bulk requests, that are, elements representing single +# strings, are normally limited to 512 mb. However you can change this limit +# here, but must be 1mb or greater +# +# proto-max-bulk-len 512mb + +# Redis calls an internal function to perform many background tasks, like +# closing connections of clients in timeout, purging expired keys that are +# never requested, and so forth. +# +# Not all tasks are performed with the same frequency, but Redis checks for +# tasks to perform according to the specified "hz" value. +# +# By default "hz" is set to 10. Raising the value will use more CPU when +# Redis is idle, but at the same time will make Redis more responsive when +# there are many keys expiring at the same time, and timeouts may be +# handled with more precision. +# +# The range is between 1 and 500, however a value over 100 is usually not +# a good idea. Most users should use the default of 10 and raise this up to +# 100 only in environments where very low latency is required. +hz 10 + +# Normally it is useful to have an HZ value which is proportional to the +# number of clients connected. This is useful in order, for instance, to +# avoid too many clients are processed for each background task invocation +# in order to avoid latency spikes. +# +# Since the default HZ value by default is conservatively set to 10, Redis +# offers, and enables by default, the ability to use an adaptive HZ value +# which will temporarily raise when there are many connected clients. +# +# When dynamic HZ is enabled, the actual configured HZ will be used +# as a baseline, but multiples of the configured HZ value will be actually +# used as needed once more clients are connected. In this way an idle +# instance will use very little CPU time while a busy instance will be +# more responsive. +dynamic-hz yes + +# When a child rewrites the AOF file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +aof-rewrite-incremental-fsync yes + +# When redis saves RDB file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +rdb-save-incremental-fsync yes + +# Redis LFU eviction (see maxmemory setting) can be tuned. However it is a good +# idea to start with the default settings and only change them after investigating +# how to improve the performances and how the keys LFU change over time, which +# is possible to inspect via the OBJECT FREQ command. +# +# There are two tunable parameters in the Redis LFU implementation: the +# counter logarithm factor and the counter decay time. It is important to +# understand what the two parameters mean before changing them. +# +# The LFU counter is just 8 bits per key, it's maximum value is 255, so Redis +# uses a probabilistic increment with logarithmic behavior. Given the value +# of the old counter, when a key is accessed, the counter is incremented in +# this way: +# +# 1. A random number R between 0 and 1 is extracted. +# 2. A probability P is calculated as 1/(old_value*lfu_log_factor+1). +# 3. The counter is incremented only if R < P. +# +# The default lfu-log-factor is 10. This is a table of how the frequency +# counter changes with a different number of accesses with different +# logarithmic factors: +# +# +--------+------------+------------+------------+------------+------------+ +# | factor | 100 hits | 1000 hits | 100K hits | 1M hits | 10M hits | +# +--------+------------+------------+------------+------------+------------+ +# | 0 | 104 | 255 | 255 | 255 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# | 1 | 18 | 49 | 255 | 255 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# | 10 | 10 | 18 | 142 | 255 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# | 100 | 8 | 11 | 49 | 143 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# +# NOTE: The above table was obtained by running the following commands: +# +# redis-benchmark -n 1000000 incr foo +# redis-cli object freq foo +# +# NOTE 2: The counter initial value is 5 in order to give new objects a chance +# to accumulate hits. +# +# The counter decay time is the time, in minutes, that must elapse in order +# for the key counter to be divided by two (or decremented if it has a value +# less <= 10). +# +# The default value for the lfu-decay-time is 1. A special value of 0 means to +# decay the counter every time it happens to be scanned. +# +# lfu-log-factor 10 +# lfu-decay-time 1 + +########################### ACTIVE DEFRAGMENTATION ####################### +# +# What is active defragmentation? +# ------------------------------- +# +# Active (online) defragmentation allows a Redis server to compact the +# spaces left between small allocations and deallocations of data in memory, +# thus allowing to reclaim back memory. +# +# Fragmentation is a natural process that happens with every allocator (but +# less so with Jemalloc, fortunately) and certain workloads. Normally a server +# restart is needed in order to lower the fragmentation, or at least to flush +# away all the data and create it again. However thanks to this feature +# implemented by Oran Agra for Redis 4.0 this process can happen at runtime +# in a "hot" way, while the server is running. +# +# Basically when the fragmentation is over a certain level (see the +# configuration options below) Redis will start to create new copies of the +# values in contiguous memory regions by exploiting certain specific Jemalloc +# features (in order to understand if an allocation is causing fragmentation +# and to allocate it in a better place), and at the same time, will release the +# old copies of the data. This process, repeated incrementally for all the keys +# will cause the fragmentation to drop back to normal values. +# +# Important things to understand: +# +# 1. This feature is disabled by default, and only works if you compiled Redis +# to use the copy of Jemalloc we ship with the source code of Redis. +# This is the default with Linux builds. +# +# 2. You never need to enable this feature if you don't have fragmentation +# issues. +# +# 3. Once you experience fragmentation, you can enable this feature when +# needed with the command "CONFIG SET activedefrag yes". +# +# The configuration parameters are able to fine tune the behavior of the +# defragmentation process. If you are not sure about what they mean it is +# a good idea to leave the defaults untouched. + +# Enabled active defragmentation +# activedefrag no + +# Minimum amount of fragmentation waste to start active defrag +# active-defrag-ignore-bytes 100mb + +# Minimum percentage of fragmentation to start active defrag +# active-defrag-threshold-lower 10 + +# Maximum percentage of fragmentation at which we use maximum effort +# active-defrag-threshold-upper 100 + +# Minimal effort for defrag in CPU percentage, to be used when the lower +# threshold is reached +# active-defrag-cycle-min 1 + +# Maximal effort for defrag in CPU percentage, to be used when the upper +# threshold is reached +# active-defrag-cycle-max 25 + +# Maximum number of set/hash/zset/list fields that will be processed from +# the main dictionary scan +# active-defrag-max-scan-fields 1000 + +# Jemalloc background thread for purging will be enabled by default +jemalloc-bg-thread yes + +# It is possible to pin different threads and processes of Redis to specific +# CPUs in your system, in order to maximize the performances of the server. +# This is useful both in order to pin different Redis threads in different +# CPUs, but also in order to make sure that multiple Redis instances running +# in the same host will be pinned to different CPUs. +# +# Normally you can do this using the "taskset" command, however it is also +# possible to this via Redis configuration directly, both in Linux and FreeBSD. +# +# You can pin the server/IO threads, bio threads, aof rewrite child process, and +# the bgsave child process. The syntax to specify the cpu list is the same as +# the taskset command: +# +# Set redis server/io threads to cpu affinity 0,2,4,6: +# server_cpulist 0-7:2 +# +# Set bio threads to cpu affinity 1,3: +# bio_cpulist 1,3 +# +# Set aof rewrite child process to cpu affinity 8,9,10,11: +# aof_rewrite_cpulist 8-11 +# +# Set bgsave child process to cpu affinity 1,10,11 +# bgsave_cpulist 1,10-11 + +# In some cases redis will emit warnings and even refuse to start if it detects +# that the system is in bad state, it is possible to suppress these warnings +# by setting the following config which takes a space delimited list of warnings +# to suppress +# +# ignore-warnings ARM64-COW-BUG \ No newline at end of file diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py new file mode 100644 index 000000000..7b1f9ee9f --- /dev/null +++ b/merlin/server/server_config.py @@ -0,0 +1,105 @@ +import logging +import os + +import yaml + + +LOG = logging.getLogger("merlin") + +MERLIN_CONFIG_DIR = os.path.join(os.path.expanduser("~"), ".merlin") +MERLIN_SERVER_SUBDIR = "server/" +MERLIN_SERVER_CONFIG = "merlin_server.yaml" + + +def parse_redis_output(redis_stdout): + if redis_stdout is None: + return False, "None passed as redis output" + server_init = False + redis_config = {} + for line in redis_stdout: + if not server_init: + values = [ln for ln in line.split() if b"=" in ln] + for val in values: + key, value = val.split(b"=") + redis_config[key.decode("utf-8")] = value.strip(b",").strip(b".").decode("utf-8") + if b"Server initialized" in line: + server_init = True + if b"Ready to accept connections" in line: + return True, redis_config + if b"aborting" in line: + return False, line.decode("utf-8") + + +def pull_server_config() -> dict: + """ + Pull the main configuration file and corresponding format configuration file + as well. Returns the values as a dictionary. + + :return: A dictionary containing the main and corresponding format configuration file + """ + return_data = {} + format_needed_keys = ["command", "run_command", "stop_command", "pull_command"] + process_needed_keys = ["status", "kill"] + + config_dir = os.path.join(MERLIN_CONFIG_DIR, MERLIN_SERVER_SUBDIR) + config_path = os.path.join(config_dir, MERLIN_SERVER_CONFIG) + if not os.path.exists(config_path): + LOG.error(f"Unable to pull merlin server configuration from {config_path}") + return None + + with open(config_path, "r") as cf: + server_config = yaml.load(cf, yaml.Loader) + return_data.update(server_config) + + if "container" in server_config: + if "format" in server_config["container"]: + format_file = os.path.join(config_dir, server_config["container"]["format"] + ".yaml") + with open(format_file, "r") as ff: + format_data = yaml.load(ff, yaml.Loader) + for key in format_needed_keys: + if key not in format_data[server_config["container"]["format"]]: + LOG.error(f'Unable to find necessary "{key}" value in format config file {format_file}') + return None + return_data.update(format_data) + else: + LOG.error(f'Unable to find "format" in {MERLIN_SERVER_CONFIG}') + return None + else: + LOG.error(f'Unable to find "container" object in {MERLIN_SERVER_CONFIG}') + return None + + # Checking for process values that are needed for main functions and defaults + if "process" not in server_config: + LOG.error("Process config not found in " + MERLIN_SERVER_CONFIG) + return None + + for key in process_needed_keys: + if key not in server_config["process"]: + LOG.error(f'Process necessary "{key}" command configuration not found in {MERLIN_SERVER_CONFIG}') + return None + + return return_data + + +def check_process_file_format(data): + required_keys = ["parent_pid", "image_pid", "port", "hostname"] + for key in required_keys: + if key not in data: + return False + return True + + +def pull_process_file(file_path): + with open(file_path, "r") as f: + data = yaml.load(f, yaml.Loader) + if check_process_file_format(data): + return data + return None + + +def dump_process_file(data, file_path): + if not check_process_file_format(data): + return False + with open(file_path, "w+") as f: + yaml.dump(data, f, yaml.Dumper) + return True diff --git a/merlin/server/server_setup.py b/merlin/server/server_setup.py new file mode 100644 index 000000000..75d1f68a3 --- /dev/null +++ b/merlin/server/server_setup.py @@ -0,0 +1,291 @@ +"""Main functions for instantiating and running Merlin server containers.""" + +import enum +import logging +import os +import shutil +import socket +import subprocess +import time + +from merlin.server.server_config import ( + MERLIN_CONFIG_DIR, + MERLIN_SERVER_CONFIG, + MERLIN_SERVER_SUBDIR, + dump_process_file, + parse_redis_output, + pull_process_file, + pull_server_config, +) + + +# Default values for configuration +CONFIG_DIR = "./merlin_server/" +IMAGE_NAME = "redis_latest.sif" +PROCESS_FILE = "merlin_server.pf" +CONFIG_FILE = "redis.conf" +REDIS_URL = "docker://redis" +CONTAINER_TYPES = ["singularity", "docker", "podman"] + +LOG = logging.getLogger("merlin") + + +class ServerStatus(enum.Enum): + """ + Different states in which the server can be in. + """ + + RUNNING = 0 + NOT_INITALIZED = 1 + MISSING_CONTAINER = 2 + NOT_RUNNING = 3 + ERROR = 4 + + +def create_server_config(): + """ + Create main configuration file for merlin server in the + merlin configuration directory. If a configuration already + exists it will not replace the current configuration and exit. + """ + if not os.path.exists(MERLIN_CONFIG_DIR): + LOG.error("Unable to find main merlin configuration directory at " + MERLIN_CONFIG_DIR) + return False + + config_dir = os.path.join(MERLIN_CONFIG_DIR, MERLIN_SERVER_SUBDIR) + if not os.path.exists(config_dir): + LOG.info("Unable to find exisiting server configuration.") + LOG.info(f"Creating default configuration in {config_dir}") + try: + os.mkdir(config_dir) + except OSError as err: + LOG.error(err) + return False + + files = [i + ".yaml" for i in CONTAINER_TYPES] + files.append(MERLIN_SERVER_CONFIG) + for file in files: + file_path = os.path.join(config_dir, file) + if os.path.exists(file_path): + LOG.info(f"{file} already exists.") + continue + LOG.info(f"Copying file {file} to configuration directory.") + try: + shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), file), config_dir) + except OSError: + LOG.error(f"Destination location {config_dir} is not writable.") + return False + + return True + + +def pull_server_image(): + """ + Fetch the server image using singularity. + """ + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + + container_config = server_config["container"] + config_dir = container_config["config_dir"] if "config_dir" in container_config else CONFIG_DIR + image_name = container_config["image"] if "image" in container_config else IMAGE_NAME + config_file = container_config["config"] if "config" in container_config else CONFIG_FILE + image_url = container_config["url"] if "url" in container_config else REDIS_URL + + if not os.path.exists(config_dir): + LOG.info("Creating merlin server directory.") + os.mkdir(config_dir) + + image_path = os.path.join(config_dir, image_name) + + if os.path.exists(image_path): + LOG.info(f"{image_path} already exists.") + return False + + LOG.info(f"Fetching redis image from {image_url}") + format_config = server_config[container_config["format"]] + subprocess.run( + format_config["pull_command"] + .strip("\\") + .format(command=format_config["command"], image=image_path, url=image_url) + .split(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + LOG.info("Copying default redis configuration file.") + try: + file_dir = os.path.dirname(os.path.abspath(__file__)) + shutil.copy(os.path.join(file_dir, config_file), config_dir) + except OSError: + LOG.error(f"Destination location {config_dir} is not writable.") + return False + return True + + +def get_server_status(): + """ + Determine the status of the current server. + This function can be used to check if the servers + have been initalized, started, or stopped. + + :param `server_dir`: location of all server related files. + :param `image_name`: name of the image when fetched. + """ + server_config = pull_server_config() + if not server_config: + return ServerStatus.NOT_INITALIZED + + container_config = server_config["container"] + config_dir = container_config["config_dir"] if "config_dir" in container_config else CONFIG_DIR + image_name = container_config["image"] if "image" in container_config else IMAGE_NAME + pfile = container_config["pfile"] if "pfile" in container_config else PROCESS_FILE + + if not os.path.exists(config_dir): + return ServerStatus.NOT_INITALIZED + + if not os.path.exists(os.path.join(config_dir, image_name)): + return ServerStatus.MISSING_CONTAINER + + if not os.path.exists(os.path.join(config_dir, pfile)): + return ServerStatus.NOT_RUNNING + + pf_data = pull_process_file(os.path.join(config_dir, pfile)) + parent_pid = pf_data["parent_pid"] + + check_process = subprocess.run( + server_config["process"]["status"].strip("\\").format(pid=parent_pid).split(), + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + + if check_process.stdout == b"": + return ServerStatus.NOT_RUNNING + + return ServerStatus.RUNNING + + +def start_server(): + """ + Start a merlin server container using singularity. + + :param `server_dir`: location of all server related files. + :param `image_name`: name of the image when fetched. + """ + current_status = get_server_status() + + if current_status == ServerStatus.NOT_INITALIZED or current_status == ServerStatus.MISSING_CONTAINER: + LOG.info("Merlin server has not been initialized. Please run 'merlin server init' first.") + return False + + if current_status == ServerStatus.RUNNING: + LOG.info("Merlin server already running.") + LOG.info("Stop current server with 'merlin server stop' before attempting to start a new server.") + return False + + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + container_config = server_config["container"] + + config_dir = container_config["config_dir"] if "config_dir" in container_config else CONFIG_DIR + config_file = container_config["config"] if "config_dir" in container_config else CONFIG_FILE + image_name = container_config["image"] if "image" in container_config else IMAGE_NAME + pfile = container_config["pfile"] if "pfile" in container_config else PROCESS_FILE + + image_path = os.path.join(config_dir, image_name) + if not os.path.exists(image_path): + LOG.error("Unable to find image at " + image_path) + return False + + config_path = os.path.join(config_dir, config_file) + if not os.path.exists(config_path): + LOG.error("Unable to find config file at " + config_path) + return False + + format_config = server_config[container_config["format"]] + process = subprocess.Popen( + format_config["run_command"] + .strip("\\") + .format(command=container_config["format"], image=image_path, config=config_path) + .split(), + start_new_session=True, + close_fds=True, + stdout=subprocess.PIPE, + ) + + time.sleep(1) + + redis_start, redis_out = parse_redis_output(process.stdout) + + if not redis_start: + LOG.error("Redis is unable to start") + LOG.error('Check to see if there is an unresponsive instance of redis with "ps -e"') + LOG.error(redis_out.strip("\n")) + return False + + redis_out["image_pid"] = redis_out.pop("pid") + redis_out["parent_pid"] = process.pid + redis_out["hostname"] = socket.gethostname() + if not dump_process_file(redis_out, os.path.join(config_dir, pfile)): + LOG.error("Unable to create process file for container.") + return False + + if get_server_status() != ServerStatus.RUNNING: + LOG.error("Unable to start merlin server.") + return False + + LOG.info(f"Server started with PID {str(process.pid)}.") + LOG.info(f'Merlin server operating on "{redis_out["hostname"]}" and port "{redis_out["port"]}".') + + return True + + +def stop_server(): + """ + Stop running merlin server containers. + + """ + if get_server_status() != ServerStatus.RUNNING: + LOG.info("There is no instance of merlin server running.") + LOG.info("Start a merlin server first with 'merlin server start'") + return False + + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + container_config = server_config["container"] + + config_dir = container_config["config_dir"] if "config_dir" in container_config else CONFIG_DIR + pfile = container_config["pfile"] if "pfile" in container_config else PROCESS_FILE + image_name = container_config["name"] if "name" in container_config else IMAGE_NAME + + pf_data = pull_process_file(os.path.join(config_dir, pfile)) + read_pid = pf_data["parent_pid"] + + process = subprocess.run( + server_config["process"]["status"].strip("\\").format(pid=read_pid).split(), stdout=subprocess.PIPE + ) + if process.stdout == b"": + LOG.error("Unable to get the PID for the current merlin server.") + return False + + format_config = server_config[container_config["format"]] + command = server_config["process"]["kill"].strip("\\").format(pid=read_pid).split() + if format_config["stop_command"] != "kill": + command = format_config["stop_command"].strip("\\").format(name=image_name).split() + + LOG.info(f"Attempting to close merlin server PID {str(read_pid)}") + + subprocess.run(command, stdout=subprocess.PIPE) + time.sleep(1) + if get_server_status() == ServerStatus.RUNNING: + LOG.error("Unable to kill process.") + return False + + LOG.info("Merlin server terminated.") + return True diff --git a/merlin/server/singularity.yaml b/merlin/server/singularity.yaml new file mode 100644 index 000000000..277800f4f --- /dev/null +++ b/merlin/server/singularity.yaml @@ -0,0 +1,6 @@ +singularity: + command: singularity + # init_command: \{command} .. (optional or default) + run_command: \{command} run {image} {config} + stop_command: kill # \{command} (optional or kill default) + pull_command: \{command} pull {image} {url} diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 3a8038d06..769e74a4c 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -34,6 +34,7 @@ """ import argparse import shutil +import sys import time from contextlib import suppress from subprocess import PIPE, Popen @@ -214,4 +215,4 @@ def main(): if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index dbdffd2cd..714e22ff5 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -45,6 +45,19 @@ def define_tests(): "local", ), } + server_tests = { + "merlin server init": ("merlin server init", HasRegex(".*successful"), "local"), + "merlin server start/stop": ( + "merlin server start; merlin server status; merlin server stop", + [ + HasRegex("Server started with PID [0-9]*"), + HasRegex("Merlin server is running"), + HasRegex("Merlin server terminated"), + ], + "local", + ), + "clean merlin server": ("rm -rf appendonly.aof dump.rdb merlin_server/"), + } examples_check = { "example list": ( "merlin example list", @@ -371,6 +384,7 @@ def define_tests(): all_tests = {} for test_dict in [ basic_checks, + server_tests, examples_check, run_workers_echo_tests, wf_format_tests, From df9612676a41b89e3b341656337d11a46c4d8c7e Mon Sep 17 00:00:00 2001 From: Joe Koning Date: Thu, 28 Apr 2022 15:55:17 -0700 Subject: [PATCH 034/126] Fix lgtm returns (#363) * Changed script sys.exit commands to use try/catch, per lgtm recommendation --- CHANGELOG.md | 2 + .../feature_demo/scripts/hello_world.py | 12 +++- .../workflows/flux/scripts/make_samples.py | 12 +++- .../hpc_demo/cumulative_sample_processor.py | 62 ++++++++++--------- .../workflows/hpc_demo/faker_sample.py | 12 +++- .../workflows/hpc_demo/sample_collector.py | 18 ++++-- .../workflows/hpc_demo/sample_processor.py | 44 +++++++------ .../cumulative_sample_processor.py | 62 ++++++++++--------- .../workflows/iterative_demo/faker_sample.py | 12 +++- .../iterative_demo/sample_collector.py | 18 ++++-- .../iterative_demo/sample_processor.py | 44 +++++++------ .../workflows/lsf/scripts/make_samples.py | 12 +++- .../null_spec/scripts/read_output.py | 15 +++-- .../scripts/hello_world.py | 12 +++- .../workflows/restart/scripts/make_samples.py | 12 +++- .../restart_delay/scripts/make_samples.py | 12 +++- .../workflows/slurm/scripts/make_samples.py | 12 +++- merlin/main.py | 6 +- merlin/merlin_templates.py | 14 +++-- tests/integration/run_tests.py | 4 +- 20 files changed, 250 insertions(+), 147 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0142d6d08..587778da5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added documentation page docs/merlin_server.rst, docs/modules/server/configuration.rst, and docs/modules/server/commands.rst ### Changed - Rename lgtm.yml to .lgtm.yml +### Fixed +- Fixed return values from scripts with main() to fix testing errors. ## [1.8.5] ### Added diff --git a/merlin/examples/workflows/feature_demo/scripts/hello_world.py b/merlin/examples/workflows/feature_demo/scripts/hello_world.py index ab14bedf4..634dfe417 100644 --- a/merlin/examples/workflows/feature_demo/scripts/hello_world.py +++ b/merlin/examples/workflows/feature_demo/scripts/hello_world.py @@ -1,5 +1,6 @@ import argparse import json +import sys def process_args(args): @@ -25,9 +26,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/flux/scripts/make_samples.py b/merlin/examples/workflows/flux/scripts/make_samples.py index e6c807bc9..8ec1c7e2f 100644 --- a/merlin/examples/workflows/flux/scripts/make_samples.py +++ b/merlin/examples/workflows/flux/scripts/make_samples.py @@ -1,5 +1,6 @@ import argparse import ast +import sys import numpy as np @@ -51,9 +52,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py b/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py index 7d06ab594..43b732ae2 100644 --- a/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py +++ b/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py @@ -1,5 +1,6 @@ import argparse import os +import sys from concurrent.futures import ProcessPoolExecutor import matplotlib.pyplot as plt @@ -55,45 +56,50 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() + try: + parser = setup_argparse() + args = parser.parse_args() - # Load all iterations' data into single pandas dataframe for further analysis - all_iter_df = load_samples(args.sample_file_paths, args.np) + # Load all iterations' data into single pandas dataframe for further analysis + all_iter_df = load_samples(args.sample_file_paths, args.np) - # PLOTS: - # counts vs index for each iter range (1, [1,2], [1-3], [1-4], ...) - # num names vs iter - # median, min, max counts vs iter -> same plot - fig, ax = plt.subplots(nrows=2, ncols=1, constrained_layout=True, sharex=True) + # PLOTS: + # counts vs index for each iter range (1, [1,2], [1-3], [1-4], ...) + # num names vs iter + # median, min, max counts vs iter -> same plot + fig, ax = plt.subplots(nrows=2, ncols=1, constrained_layout=True, sharex=True) - iterations = sorted(all_iter_df.Iter.unique()) + iterations = sorted(all_iter_df.Iter.unique()) - max_counts = [] - min_counts = [] - med_counts = [] - unique_names = [] + max_counts = [] + min_counts = [] + med_counts = [] + unique_names = [] - for it in iterations: - max_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].max()) - min_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].min()) - med_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].median()) + for it in iterations: + max_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].max()) + min_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].min()) + med_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].median()) - unique_names.append(len(all_iter_df[all_iter_df["Iter"] <= it].index.value_counts())) + unique_names.append(len(all_iter_df[all_iter_df["Iter"] <= it].index.value_counts())) - ax[0].plot(iterations, min_counts, label="Minimum Occurances") - ax[0].plot(iterations, max_counts, label="Maximum Occurances") + ax[0].plot(iterations, min_counts, label="Minimum Occurances") + ax[0].plot(iterations, max_counts, label="Maximum Occurances") - ax[0].plot(iterations, med_counts, label="Median Occurances") + ax[0].plot(iterations, med_counts, label="Median Occurances") - ax[0].set_ylabel("Counts") - ax[0].legend() + ax[0].set_ylabel("Counts") + ax[0].legend() - ax[1].set_xlabel("Iteration") - ax[1].set_ylabel("Unique Names") - ax[1].plot(iterations, unique_names) + ax[1].set_xlabel("Iteration") + ax[1].set_ylabel("Unique Names") + ax[1].plot(iterations, unique_names) - fig.savefig(args.hardcopy, dpi=150) + fig.savefig(args.hardcopy, dpi=150) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/hpc_demo/faker_sample.py b/merlin/examples/workflows/hpc_demo/faker_sample.py index ee8bf2f5c..be16be5de 100644 --- a/merlin/examples/workflows/hpc_demo/faker_sample.py +++ b/merlin/examples/workflows/hpc_demo/faker_sample.py @@ -1,4 +1,5 @@ import argparse +import sys from faker import Faker @@ -31,9 +32,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/hpc_demo/sample_collector.py b/merlin/examples/workflows/hpc_demo/sample_collector.py index f62111e8e..ad06dc6c5 100644 --- a/merlin/examples/workflows/hpc_demo/sample_collector.py +++ b/merlin/examples/workflows/hpc_demo/sample_collector.py @@ -1,5 +1,6 @@ import argparse import os +import sys from concurrent.futures import ProcessPoolExecutor @@ -36,12 +37,17 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - - # Collect sample files into single file - sample_paths = [sample_path for sample_path in args.sample_file_paths] - serialize_samples(sample_paths, args.outfile, args.np) + try: + parser = setup_argparse() + args = parser.parse_args() + + # Collect sample files into single file + sample_paths = [sample_path for sample_path in args.sample_file_paths] + serialize_samples(sample_paths, args.outfile, args.np) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/hpc_demo/sample_processor.py b/merlin/examples/workflows/hpc_demo/sample_processor.py index 9ec0951e9..8523dcc80 100644 --- a/merlin/examples/workflows/hpc_demo/sample_processor.py +++ b/merlin/examples/workflows/hpc_demo/sample_processor.py @@ -1,6 +1,7 @@ import argparse import os import pathlib +import sys import pandas as pd @@ -28,25 +29,30 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - - # Collect the samples - samples = load_samples(args.sample_file_paths) - - # Count up the occurences - namesdf = pd.DataFrame({"Name": samples}) - - names = namesdf["Name"].value_counts() - - # Serialize processed samples - # create directory if it doesn't exist already - abspath = os.path.abspath(args.results) - absdir = os.path.dirname(abspath) - if not os.path.isdir(absdir): - pathlib.Path(absdir).mkdir(parents=True, exist_ok=True) - - names.to_json(args.results) + try: + parser = setup_argparse() + args = parser.parse_args() + + # Collect the samples + samples = load_samples(args.sample_file_paths) + + # Count up the occurences + namesdf = pd.DataFrame({"Name": samples}) + + names = namesdf["Name"].value_counts() + + # Serialize processed samples + # create directory if it doesn't exist already + abspath = os.path.abspath(args.results) + absdir = os.path.dirname(abspath) + if not os.path.isdir(absdir): + pathlib.Path(absdir).mkdir(parents=True, exist_ok=True) + + names.to_json(args.results) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py b/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py index 7d06ab594..43b732ae2 100644 --- a/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py +++ b/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py @@ -1,5 +1,6 @@ import argparse import os +import sys from concurrent.futures import ProcessPoolExecutor import matplotlib.pyplot as plt @@ -55,45 +56,50 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() + try: + parser = setup_argparse() + args = parser.parse_args() - # Load all iterations' data into single pandas dataframe for further analysis - all_iter_df = load_samples(args.sample_file_paths, args.np) + # Load all iterations' data into single pandas dataframe for further analysis + all_iter_df = load_samples(args.sample_file_paths, args.np) - # PLOTS: - # counts vs index for each iter range (1, [1,2], [1-3], [1-4], ...) - # num names vs iter - # median, min, max counts vs iter -> same plot - fig, ax = plt.subplots(nrows=2, ncols=1, constrained_layout=True, sharex=True) + # PLOTS: + # counts vs index for each iter range (1, [1,2], [1-3], [1-4], ...) + # num names vs iter + # median, min, max counts vs iter -> same plot + fig, ax = plt.subplots(nrows=2, ncols=1, constrained_layout=True, sharex=True) - iterations = sorted(all_iter_df.Iter.unique()) + iterations = sorted(all_iter_df.Iter.unique()) - max_counts = [] - min_counts = [] - med_counts = [] - unique_names = [] + max_counts = [] + min_counts = [] + med_counts = [] + unique_names = [] - for it in iterations: - max_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].max()) - min_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].min()) - med_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].median()) + for it in iterations: + max_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].max()) + min_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].min()) + med_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].median()) - unique_names.append(len(all_iter_df[all_iter_df["Iter"] <= it].index.value_counts())) + unique_names.append(len(all_iter_df[all_iter_df["Iter"] <= it].index.value_counts())) - ax[0].plot(iterations, min_counts, label="Minimum Occurances") - ax[0].plot(iterations, max_counts, label="Maximum Occurances") + ax[0].plot(iterations, min_counts, label="Minimum Occurances") + ax[0].plot(iterations, max_counts, label="Maximum Occurances") - ax[0].plot(iterations, med_counts, label="Median Occurances") + ax[0].plot(iterations, med_counts, label="Median Occurances") - ax[0].set_ylabel("Counts") - ax[0].legend() + ax[0].set_ylabel("Counts") + ax[0].legend() - ax[1].set_xlabel("Iteration") - ax[1].set_ylabel("Unique Names") - ax[1].plot(iterations, unique_names) + ax[1].set_xlabel("Iteration") + ax[1].set_ylabel("Unique Names") + ax[1].plot(iterations, unique_names) - fig.savefig(args.hardcopy, dpi=150) + fig.savefig(args.hardcopy, dpi=150) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/iterative_demo/faker_sample.py b/merlin/examples/workflows/iterative_demo/faker_sample.py index ee8bf2f5c..be16be5de 100644 --- a/merlin/examples/workflows/iterative_demo/faker_sample.py +++ b/merlin/examples/workflows/iterative_demo/faker_sample.py @@ -1,4 +1,5 @@ import argparse +import sys from faker import Faker @@ -31,9 +32,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/iterative_demo/sample_collector.py b/merlin/examples/workflows/iterative_demo/sample_collector.py index f62111e8e..ad06dc6c5 100644 --- a/merlin/examples/workflows/iterative_demo/sample_collector.py +++ b/merlin/examples/workflows/iterative_demo/sample_collector.py @@ -1,5 +1,6 @@ import argparse import os +import sys from concurrent.futures import ProcessPoolExecutor @@ -36,12 +37,17 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - - # Collect sample files into single file - sample_paths = [sample_path for sample_path in args.sample_file_paths] - serialize_samples(sample_paths, args.outfile, args.np) + try: + parser = setup_argparse() + args = parser.parse_args() + + # Collect sample files into single file + sample_paths = [sample_path for sample_path in args.sample_file_paths] + serialize_samples(sample_paths, args.outfile, args.np) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/iterative_demo/sample_processor.py b/merlin/examples/workflows/iterative_demo/sample_processor.py index 9ec0951e9..8523dcc80 100644 --- a/merlin/examples/workflows/iterative_demo/sample_processor.py +++ b/merlin/examples/workflows/iterative_demo/sample_processor.py @@ -1,6 +1,7 @@ import argparse import os import pathlib +import sys import pandas as pd @@ -28,25 +29,30 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - - # Collect the samples - samples = load_samples(args.sample_file_paths) - - # Count up the occurences - namesdf = pd.DataFrame({"Name": samples}) - - names = namesdf["Name"].value_counts() - - # Serialize processed samples - # create directory if it doesn't exist already - abspath = os.path.abspath(args.results) - absdir = os.path.dirname(abspath) - if not os.path.isdir(absdir): - pathlib.Path(absdir).mkdir(parents=True, exist_ok=True) - - names.to_json(args.results) + try: + parser = setup_argparse() + args = parser.parse_args() + + # Collect the samples + samples = load_samples(args.sample_file_paths) + + # Count up the occurences + namesdf = pd.DataFrame({"Name": samples}) + + names = namesdf["Name"].value_counts() + + # Serialize processed samples + # create directory if it doesn't exist already + abspath = os.path.abspath(args.results) + absdir = os.path.dirname(abspath) + if not os.path.isdir(absdir): + pathlib.Path(absdir).mkdir(parents=True, exist_ok=True) + + names.to_json(args.results) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/lsf/scripts/make_samples.py b/merlin/examples/workflows/lsf/scripts/make_samples.py index e6c807bc9..8ec1c7e2f 100644 --- a/merlin/examples/workflows/lsf/scripts/make_samples.py +++ b/merlin/examples/workflows/lsf/scripts/make_samples.py @@ -1,5 +1,6 @@ import argparse import ast +import sys import numpy as np @@ -51,9 +52,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/null_spec/scripts/read_output.py b/merlin/examples/workflows/null_spec/scripts/read_output.py index 7c0f8017e..278283bd7 100644 --- a/merlin/examples/workflows/null_spec/scripts/read_output.py +++ b/merlin/examples/workflows/null_spec/scripts/read_output.py @@ -118,11 +118,16 @@ def start_sample1_time(): def main(): - single_task_times() - merlin_run_time() - start_verify_time() - start_run_workers_time() - start_sample1_time() + try: + single_task_times() + merlin_run_time() + start_verify_time() + start_run_workers_time() + start_sample1_time() + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py b/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py index 232c43e86..3b9f62df1 100644 --- a/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py +++ b/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py @@ -1,5 +1,6 @@ import argparse import json +import sys from typing import Dict @@ -35,9 +36,14 @@ def main(): """ Primary coordinating method for collecting args and dumping them to a json file for later examination. """ - parser: argparse.ArgumentParser = setup_argparse() - args: argparse.Namespace = parser.parse_args() - process_args(args) + try: + parser: argparse.ArgumentParser = setup_argparse() + args: argparse.Namespace = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/restart/scripts/make_samples.py b/merlin/examples/workflows/restart/scripts/make_samples.py index e6c807bc9..8ec1c7e2f 100644 --- a/merlin/examples/workflows/restart/scripts/make_samples.py +++ b/merlin/examples/workflows/restart/scripts/make_samples.py @@ -1,5 +1,6 @@ import argparse import ast +import sys import numpy as np @@ -51,9 +52,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/restart_delay/scripts/make_samples.py b/merlin/examples/workflows/restart_delay/scripts/make_samples.py index e6c807bc9..8ec1c7e2f 100644 --- a/merlin/examples/workflows/restart_delay/scripts/make_samples.py +++ b/merlin/examples/workflows/restart_delay/scripts/make_samples.py @@ -1,5 +1,6 @@ import argparse import ast +import sys import numpy as np @@ -51,9 +52,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/slurm/scripts/make_samples.py b/merlin/examples/workflows/slurm/scripts/make_samples.py index e6c807bc9..8ec1c7e2f 100644 --- a/merlin/examples/workflows/slurm/scripts/make_samples.py +++ b/merlin/examples/workflows/slurm/scripts/make_samples.py @@ -1,5 +1,6 @@ import argparse import ast +import sys import numpy as np @@ -51,9 +52,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/main.py b/merlin/main.py index 113a15534..25c06619c 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -823,12 +823,12 @@ def main(): except Exception as excpt: # pylint: disable=broad-except LOG.debug(traceback.format_exc()) LOG.error(str(excpt)) - return 1 + sys.exit(1) # All paths in a function ought to return an exit code, or none of them should. Given the # distributed nature of Merlin, maybe it doesn't make sense for it to exit 0 until the work is completed, but # if the work is dispatched with no errors, that is a 'successful' Merlin run - any other failures are runtime. - return 0 + sys.exit() if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index a1e80fea1..bbc213471 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -33,6 +33,7 @@ """ import argparse import logging +import sys from merlin.ascii_art import banner_small from merlin.log_formatter import setup_logging @@ -57,10 +58,15 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - setup_logging(logger=LOG, log_level=DEFAULT_LOG_LEVEL, colors=True) - args.func(args) + try: + parser = setup_argparse() + args = parser.parse_args() + setup_logging(logger=LOG, log_level=DEFAULT_LOG_LEVEL, colors=True) + args.func(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 769e74a4c..5226356c5 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -211,8 +211,8 @@ def main(): clear_test_studies_dir() result = run_tests(args, tests) - return result + sys.exit(result) if __name__ == "__main__": - sys.exit(main()) + main() From 98469c3085e0041e6427743c5fed6f74fb5f4713 Mon Sep 17 00:00:00 2001 From: Joe Koning Date: Tue, 7 Jun 2022 10:21:53 -0700 Subject: [PATCH 035/126] Allow for flux exec arguments to limit the number of celery workers. (#366) * Added the flux_exec batch argument to allow for flux exec arguments, e.g. flux_exec: flux exec -r "0-1" to run celery workers only on ranks 0 and 1 of a multi-rank allocation. * Remove period. --- CHANGELOG.md | 3 +++ docs/source/merlin_specification.rst | 3 +++ merlin/examples/workflows/flux/flux_par.yaml | 1 + merlin/spec/all_keys.py | 1 + merlin/study/batch.py | 2 +- 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 587778da5..7dfe726a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added redis.conf for default redis configuration for merlin in server/redis.conf - Added default configurations for merlin server command in merlin/server/*.yaml - Added documentation page docs/merlin_server.rst, docs/modules/server/configuration.rst, and docs/modules/server/commands.rst +- Added the flux_exec batch argument to allow for flux exec arguments, + e.g. flux_exec: flux exec -r "0-1" to run celery workers only on + ranks 0 and 1 of a multi-rank allocation ### Changed - Rename lgtm.yml to .lgtm.yml ### Fixed diff --git a/docs/source/merlin_specification.rst b/docs/source/merlin_specification.rst index a84eae1a6..d01f6f83d 100644 --- a/docs/source/merlin_specification.rst +++ b/docs/source/merlin_specification.rst @@ -48,6 +48,9 @@ see :doc:`./merlin_variables`. queue: pbatch flux_path: flux_start_opts: + flux_exec: flux_exec_workers: launch_pre: diff --git a/merlin/examples/workflows/flux/flux_par.yaml b/merlin/examples/workflows/flux/flux_par.yaml index d401a0270..1fee4131e 100644 --- a/merlin/examples/workflows/flux/flux_par.yaml +++ b/merlin/examples/workflows/flux/flux_par.yaml @@ -6,6 +6,7 @@ batch: type: flux nodes: 1 queue: pbatch + flux_exec: flux exec -r "0-1" flux_start_opts: -o,-S,log-filename=flux_par.out env: diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 9af27d456..bc9d771ec 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -38,6 +38,7 @@ "shell", "flux_path", "flux_start_opts", + "flux_exec", "flux_exec_workers", "launch_pre", "launch_args", diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 69c68856a..88ab230a1 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -175,7 +175,7 @@ def batch_worker_launch( flux_exec: str = "" if flux_exec_workers: - flux_exec = "flux exec" + flux_exec = get_yaml_var(batch, "flux_exec", "flux exec") if "/" in flux_path: flux_path += "/" From 472d97b726a243620e75c193fb8c0757e9f466a1 Mon Sep 17 00:00:00 2001 From: Ryan Lee <44886374+ryannova@users.noreply.github.com> Date: Wed, 22 Jun 2022 14:05:02 -0700 Subject: [PATCH 036/126] Merlin server configuration through commands (#365) * Reorganized functions within server_setup and server_config * Rename server_setup file to server_commands * Added password generation for redis container in merlin server * Changed redis configuration to require password authentication * Added merlin config flags ipaddress, port, password, directory, snapshot_seconds, snapshot_changes, snapshot_file, append_mode, append_file * Added server_util.py * Added merlin user file into merlin server config * Added RedisConfig class to interact and change config values within the redis config file * Added merlin server restart * Updated info messages * Added function to add/remove users and store info to user file * Update running container with new users and removed users * Added ServerConfig, ProcessConfig, ContainerConfig, and ContainerFormatConfig classes to interact with configuration files * Adjusted adding user and password to use values in config files * Updated host in redis.conf * Updated pull_server_image step * Moved creation of local merlin config to create_server_config() * Added placeholder for documentation of restart and config commands for merlin server --- CHANGELOG.md | 28 ++ docs/source/server/commands.rst | 46 ++- merlin/main.py | 130 ++++++-- merlin/server/merlin_server.yaml | 6 + merlin/server/redis.conf | 6 +- merlin/server/server_commands.py | 259 ++++++++++++++++ merlin/server/server_config.py | 256 +++++++++++++++- merlin/server/server_setup.py | 291 ------------------ merlin/server/server_util.py | 510 +++++++++++++++++++++++++++++++ 9 files changed, 1201 insertions(+), 331 deletions(-) create mode 100644 merlin/server/server_commands.py delete mode 100644 merlin/server/server_setup.py create mode 100644 merlin/server/server_util.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dfe726a7..195483640 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,11 +14,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added redis.conf for default redis configuration for merlin in server/redis.conf - Added default configurations for merlin server command in merlin/server/*.yaml - Added documentation page docs/merlin_server.rst, docs/modules/server/configuration.rst, and docs/modules/server/commands.rst +- Added merlin server config command for editing configuration files. +- Added server_command.py to store command calls. +- Added following flags to config subcommand + - ipaddress (Set the binded ip address of the container) + - port (Set the binded port of the container) + - user (Set the main user file for container) + - password (Set the main user password file for container) + - add-user (Add a user to the container image [outputs an associated password file for user]) + - remove-user (Remove user from list of added users) + - directory (Set the directory of the merlin server container files) + - snapshot-seconds (Set the number of seconds elapsed before snapshot change condition is checked) + - snapshot-changes (Set snapshot change condition for a snapshot to be made) + - snapshot-file (Set the database file that the snapshot will be written to) + - append-mode (Set the append mode for redis) + - append-file (Set the name of the append only file for redis) +- Added user_file to merlin server config +- Added pass_file to merlin server config +- Added add_user function to add user to exisiting merlin server instance if one is running +- Added remove_user function to remove user from merlin server instance if one is running +- Added masteruser in redis config +- Added requirepass in redis config +- Added server_util.py file to store utility functions. +- Created RedisConfig class to interface with redis.conf file +- Created RedisUsers class to interface with redis.user file +- Added better interface for configuration files(ServerConfig, ContainerConfig, ContainerFormatConfig, and ProcessConfig) with getting configuration values from merlin server config file, with classes. +- Added merlin server to reapply users based on the saved redis.users config file. +- Added redis.pass file containing password for default user in main merlin configuration. - Added the flux_exec batch argument to allow for flux exec arguments, e.g. flux_exec: flux exec -r "0-1" to run celery workers only on ranks 0 and 1 of a multi-rank allocation ### Changed - Rename lgtm.yml to .lgtm.yml +- Changed "default" user password to be "merlin_password" as default. ### Fixed - Fixed return values from scripts with main() to fix testing errors. diff --git a/docs/source/server/commands.rst b/docs/source/server/commands.rst index f3bbec276..af3d2e419 100644 --- a/docs/source/server/commands.rst +++ b/docs/source/server/commands.rst @@ -37,4 +37,48 @@ Starts the container located in the local merlin server configuration. Stopping an exisiting Merlin Server (``merlin server stop``) ------------------------------------------------------------ -Stop any exisiting container being managaed and monitored by merlin server. +Stop any exisiting container being managed and monitored by merlin server. + +Restarting a Merlin Server instance (``merlin server restart``) +------------------------------------------------------------ + +Restarting an existing container that is being managed and monitored by merlin server. + +Configurating Merlin Server instance (``merlin server config``) +------------------------------------------------------------ +Place holder for information regarding merlin server config command + +Possible Flags +.. code:: none + -ip IPADDRESS, --ipaddress IPADDRESS + Set the binded IP address for the merlin server + container. (default: None) + -p PORT, --port PORT Set the binded port for the merlin server container. + (default: None) + -pwd PASSWORD, --password PASSWORD + Set the password file to be used for merlin server + container. (default: None) + --add-user ADD_USER ADD_USER + Create a new user for merlin server instance. (Provide + both username and password) (default: None) + --remove-user REMOVE_USER + Remove an exisiting user. (default: None) + -d DIRECTORY, --directory DIRECTORY + Set the working directory of the merlin server + container. (default: None) + -ss SNAPSHOT_SECONDS, --snapshot-seconds SNAPSHOT_SECONDS + Set the number of seconds merlin server waits before + checking if a snapshot is needed. (default: None) + -sc SNAPSHOT_CHANGES, --snapshot-changes SNAPSHOT_CHANGES + Set the number of changes that are required to be made + to the merlin server before a snapshot is made. + (default: None) + -sf SNAPSHOT_FILE, --snapshot-file SNAPSHOT_FILE + Set the snapshot filename for database dumps. + (default: None) + -am APPEND_MODE, --append-mode APPEND_MODE + The appendonly mode to be set. The avaiable options + are always, everysec, no. (default: None) + -af APPEND_FILE, --append-file APPEND_FILE + Set append only filename for merlin server container. + (default: None) diff --git a/merlin/main.py b/merlin/main.py index 25c06619c..bc2ff9900 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -52,14 +52,7 @@ from merlin.ascii_art import banner_small from merlin.examples.generator import list_examples, setup_example from merlin.log_formatter import setup_logging -from merlin.server.server_setup import ( - ServerStatus, - create_server_config, - get_server_status, - pull_server_image, - start_server, - stop_server, -) +from merlin.server.server_commands import config_server, init_server, restart_server, start_server, status_server, stop_server from merlin.spec.expansion import RESERVED, get_spec_with_expansion from merlin.spec.specification import MerlinSpec from merlin.study.study import MerlinStudy @@ -350,30 +343,19 @@ def process_monitor(args): LOG.info("Monitor: ... stop condition met") -def process_server(args): +def process_server(args: Namespace): if args.commands == "init": - if not create_server_config(): - LOG.info("Merlin server initialization failed.") - return - if pull_server_image(): - LOG.info("New merlin server image fetched") - LOG.info("Merlin server initialization successful.") + init_server() elif args.commands == "start": start_server() elif args.commands == "stop": stop_server() elif args.commands == "status": - current_status = get_server_status() - if current_status == ServerStatus.NOT_INITALIZED: - LOG.info("Merlin server has not been initialized.") - LOG.info("Please initalize server by running 'merlin server init'") - elif current_status == ServerStatus.MISSING_CONTAINER: - LOG.info("Unable to find server image.") - LOG.info("Ensure there is a .sif file in merlin server directory.") - elif current_status == ServerStatus.NOT_RUNNING: - LOG.info("Merlin server is not running.") - elif current_status == ServerStatus.RUNNING: - LOG.info("Merlin server is running.") + status_server() + elif args.commands == "restart": + restart_server() + elif args.commands == "config": + config_server(args) def setup_argparse() -> None: @@ -591,6 +573,7 @@ def setup_argparse() -> None: help="Manage broker and results server for merlin workflow.", formatter_class=ArgumentDefaultsHelpFormatter, ) + server.set_defaults(func=process_server) server_commands: ArgumentParser = server.add_subparsers(dest="commands") @@ -626,6 +609,101 @@ def setup_argparse() -> None: ) server_stop.set_defaults(func=process_server) + server_stop: ArgumentParser = server_commands.add_parser( + "restart", + help="Restart merlin server instance", + description="Restart server.", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + server_stop.set_defaults(func=process_server) + + server_config: ArgumentParser = server_commands.add_parser( + "config", + help="Making configurations for to the merlin server instance.", + description="Config server.", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + server_config.add_argument( + "-ip", + "--ipaddress", + action="store", + type=str, + # default="127.0.0.1", + help="Set the binded IP address for the merlin server container.", + ) + server_config.add_argument( + "-p", + "--port", + action="store", + type=int, + # default=6379, + help="Set the binded port for the merlin server container.", + ) + server_config.add_argument( + "-pwd", + "--password", + action="store", + type=str, + # default="~/.merlin/redis.pass", + help="Set the password file to be used for merlin server container.", + ) + server_config.add_argument( + "--add-user", + action="store", + nargs=2, + type=str, + help="Create a new user for merlin server instance. (Provide both username and password)", + ) + server_config.add_argument("--remove-user", action="store", type=str, help="Remove an exisiting user.") + server_config.add_argument( + "-d", + "--directory", + action="store", + type=str, + # default="./", + help="Set the working directory of the merlin server container.", + ) + server_config.add_argument( + "-ss", + "--snapshot-seconds", + action="store", + type=int, + # default=300, + help="Set the number of seconds merlin server waits before checking if a snapshot is needed.", + ) + server_config.add_argument( + "-sc", + "--snapshot-changes", + action="store", + type=int, + # default=100, + help="Set the number of changes that are required to be made to the merlin server before a snapshot is made.", + ) + server_config.add_argument( + "-sf", + "--snapshot-file", + action="store", + type=str, + # default="dump.db", + help="Set the snapshot filename for database dumps.", + ) + server_config.add_argument( + "-am", + "--append-mode", + action="store", + type=str, + # default="everysec", + help="The appendonly mode to be set. The avaiable options are always, everysec, no.", + ) + server_config.add_argument( + "-af", + "--append-file", + action="store", + type=str, + # default="appendonly.aof", + help="Set append only filename for merlin server container.", + ) + return parser diff --git a/merlin/server/merlin_server.yaml b/merlin/server/merlin_server.yaml index 6963a2d9f..c351a06d4 100644 --- a/merlin/server/merlin_server.yaml +++ b/merlin/server/merlin_server.yaml @@ -11,6 +11,12 @@ container: config_dir: merlin_server/ # Process file containing information regarding the redis process pfile: merlin_server.pf + # Password file to be used for accessing container + pass_file: redis.pass + # Password command for generating password file + # pass_command: date +%s | sha256sum + # Users file to track concurrent users. + user_file: redis.users process: # Command for determining the process of the command diff --git a/merlin/server/redis.conf b/merlin/server/redis.conf index b50ae9e76..893677763 100644 --- a/merlin/server/redis.conf +++ b/merlin/server/redis.conf @@ -72,7 +72,7 @@ # IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES # JUST COMMENT OUT THE FOLLOWING LINE. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -bind 127.0.0.1 -::1 +bind 127.0.0.1 # Protected mode is a layer of security protection, in order to avoid that # Redis instances left open on the internet are accessed and exploited. @@ -489,7 +489,7 @@ dir ./ # better to configure a special user to use with replication, and specify the # masteruser configuration as such: # -# masteruser +# masteruser root # # When masteruser is specified, the replica will authenticate against its # master using the new AUTH form: AUTH . @@ -898,7 +898,7 @@ acllog-max-len 128 # The requirepass is not compatable with aclfile option and the ACL LOAD # command, these will cause requirepass to be ignored. # -# requirepass foobared +requirepass merlin_password # New users are initialized with restrictive permissions by default, via the # equivalent of this ACL rule 'off resetkeys -@all'. Starting with Redis 6.2, it diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py new file mode 100644 index 000000000..5782183a4 --- /dev/null +++ b/merlin/server/server_commands.py @@ -0,0 +1,259 @@ +"""Main functions for instantiating and running Merlin server containers.""" + +import logging +import os +import socket +import subprocess +import time +from argparse import Namespace + +from merlin.server.server_config import ( + ServerStatus, + config_merlin_server, + create_server_config, + dump_process_file, + get_server_status, + parse_redis_output, + pull_process_file, + pull_server_config, + pull_server_image, +) +from merlin.server.server_util import RedisConfig, RedisUsers + + +LOG = logging.getLogger("merlin") + + +def init_server() -> None: + """ + Initialize merlin server by checking and initializing main configuration directory + and local server configuration. + """ + + if not create_server_config(): + LOG.info("Merlin server initialization failed.") + return + pull_server_image() + + config_merlin_server() + + LOG.info("Merlin server initialization successful.") + + +def config_server(args: Namespace) -> None: + """ + Process the merlin server config flags to make changes and edits to appropriate configurations + based on the input passed in by the user. + """ + server_config = pull_server_config() + redis_config = RedisConfig(server_config.container.get_config_path()) + + redis_config.set_ip_address(args.ipaddress) + + redis_config.set_port(args.port) + + redis_config.set_password(args.password) + if args.password is not None: + redis_users = RedisUsers(server_config.container.get_user_file_path()) + redis_users.set_password("default", args.password) + redis_users.write() + + redis_config.set_directory(args.directory) + + redis_config.set_snapshot_seconds(args.snapshot_seconds) + + redis_config.set_snapshot_changes(args.snapshot_changes) + + redis_config.set_snapshot_file(args.snapshot_file) + + redis_config.set_append_mode(args.append_mode) + + redis_config.set_append_file(args.append_file) + + if redis_config.changes_made(): + redis_config.write() + LOG.info("Merlin server config has changed. Restart merlin server to apply new configuration.") + LOG.info("Run 'merlin server restart' to restart running merlin server") + LOG.info("Run 'merlin server start' to start merlin server instance.") + else: + LOG.info("Add changes to config file and exisiting containers.") + + server_config = pull_server_config() + + # Read the user from the list of avaliable users + redis_users = RedisUsers(server_config.container.get_user_file_path()) + redis_config = RedisConfig(server_config.container.get_config_path()) + + if args.add_user is not None: + # Log the user in a file + if redis_users.add_user(user=args.add_user[0], password=args.add_user[1]): + redis_users.write() + LOG.info(f"Added user {args.add_user[0]} to merlin server") + # Create a new user in container + if get_server_status() == ServerStatus.RUNNING: + LOG.info("Adding user to current merlin server instance") + redis_users.apply_to_redis(redis_config.get_ip_address(), redis_config.get_port(), redis_config.get_password()) + else: + LOG.error(f"User '{args.add_user[0]}' already exisits within current users") + + if args.remove_user is not None: + # Remove user from file + if redis_users.remove_user(args.remove_user): + redis_users.write() + LOG.info(f"Removed user {args.remove_user} to merlin server") + # Remove user from container + if get_server_status() == ServerStatus.RUNNING: + LOG.info("Removing user to current merlin server instance") + redis_users.apply_to_redis(redis_config.get_ip_address(), redis_config.get_port(), redis_config.get_password()) + else: + LOG.error(f"User '{args.remove_user}' doesn't exist within current users.") + + +def status_server() -> None: + """ + Get the server status of the any current running containers for merlin server + """ + current_status = get_server_status() + if current_status == ServerStatus.NOT_INITALIZED: + LOG.info("Merlin server has not been initialized.") + LOG.info("Please initalize server by running 'merlin server init'") + elif current_status == ServerStatus.MISSING_CONTAINER: + LOG.info("Unable to find server image.") + LOG.info("Ensure there is a .sif file in merlin server directory.") + elif current_status == ServerStatus.NOT_RUNNING: + LOG.info("Merlin server is not running.") + elif current_status == ServerStatus.RUNNING: + LOG.info("Merlin server is running.") + + +def start_server() -> bool: + """ + Start a merlin server container using singularity. + :return:: True if server was successful started and False if failed. + """ + current_status = get_server_status() + + if current_status == ServerStatus.NOT_INITALIZED or current_status == ServerStatus.MISSING_CONTAINER: + LOG.info("Merlin server has not been initialized. Please run 'merlin server init' first.") + return False + + if current_status == ServerStatus.RUNNING: + LOG.info("Merlin server already running.") + LOG.info("Stop current server with 'merlin server stop' before attempting to start a new server.") + return False + + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + + image_path = server_config.container.get_image_path() + if not os.path.exists(image_path): + LOG.error("Unable to find image at " + image_path) + return False + + config_path = server_config.container.get_config_path() + if not os.path.exists(config_path): + LOG.error("Unable to find config file at " + config_path) + return False + + process = subprocess.Popen( + server_config.container_format.get_run_command() + .strip("\\") + .format(command=server_config.container_format.get_command(), image=image_path, config=config_path) + .split(), + start_new_session=True, + close_fds=True, + stdout=subprocess.PIPE, + ) + + time.sleep(1) + + redis_start, redis_out = parse_redis_output(process.stdout) + + if not redis_start: + LOG.error("Redis is unable to start") + LOG.error('Check to see if there is an unresponsive instance of redis with "ps -e"') + LOG.error(redis_out.strip("\n")) + return False + + redis_out["image_pid"] = redis_out.pop("pid") + redis_out["parent_pid"] = process.pid + redis_out["hostname"] = socket.gethostname() + if not dump_process_file(redis_out, server_config.container.get_pfile_path()): + LOG.error("Unable to create process file for container.") + return False + + if get_server_status() != ServerStatus.RUNNING: + LOG.error("Unable to start merlin server.") + return False + + LOG.info(f"Server started with PID {str(process.pid)}.") + LOG.info(f'Merlin server operating on "{redis_out["hostname"]}" and port "{redis_out["port"]}".') + + redis_users = RedisUsers(server_config.container.get_user_file_path()) + redis_config = RedisConfig(server_config.container.get_config_path()) + redis_users.apply_to_redis(redis_config.get_ip_address(), redis_config.get_port(), redis_config.get_password()) + + return True + + +def stop_server(): + """ + Stop running merlin server containers. + :return:: True if server was stopped successfully and False if failed. + """ + if get_server_status() != ServerStatus.RUNNING: + LOG.info("There is no instance of merlin server running.") + LOG.info("Start a merlin server first with 'merlin server start'") + return False + + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + + pf_data = pull_process_file(server_config.container.get_pfile_path()) + read_pid = pf_data["parent_pid"] + + process = subprocess.run( + server_config.process.get_status_command().strip("\\").format(pid=read_pid).split(), stdout=subprocess.PIPE + ) + if process.stdout == b"": + LOG.error("Unable to get the PID for the current merlin server.") + return False + + command = server_config.process.get_kill_command().strip("\\").format(pid=read_pid).split() + if server_config.container_format.get_stop_command() != "kill": + command = ( + server_config.container_format.get_stop_command() + .strip("\\") + .format(name=server_config.container.get_image_name) + .split() + ) + + LOG.info(f"Attempting to close merlin server PID {str(read_pid)}") + + subprocess.run(command, stdout=subprocess.PIPE) + time.sleep(1) + if get_server_status() == ServerStatus.RUNNING: + LOG.error("Unable to kill process.") + return False + + LOG.info("Merlin server terminated.") + return True + + +def restart_server() -> bool: + """ + Restart a running merlin server instance. + :return:: True if server was restarted successfully and False if failed. + """ + if get_server_status() != ServerStatus.RUNNING: + LOG.info("Merlin server is not currently running.") + LOG.info("Please start a merlin server instance first with 'merlin server start'") + return False + stop_server() + time.sleep(1) + start_server() + return True diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 7b1f9ee9f..2b18b1e27 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -1,17 +1,80 @@ +import enum import logging import os +import random +import shutil +import string +import subprocess +from typing import Tuple import yaml +from merlin.server.server_util import ( + CONTAINER_TYPES, + MERLIN_CONFIG_DIR, + MERLIN_SERVER_CONFIG, + MERLIN_SERVER_SUBDIR, + RedisConfig, + RedisUsers, + ServerConfig, +) + LOG = logging.getLogger("merlin") -MERLIN_CONFIG_DIR = os.path.join(os.path.expanduser("~"), ".merlin") -MERLIN_SERVER_SUBDIR = "server/" -MERLIN_SERVER_CONFIG = "merlin_server.yaml" +# Default values for configuration +CONFIG_DIR = "./merlin_server/" +IMAGE_NAME = "redis_latest.sif" +PROCESS_FILE = "merlin_server.pf" +CONFIG_FILE = "redis.conf" +REDIS_URL = "docker://redis" + +PASSWORD_LENGTH = 256 + + +class ServerStatus(enum.Enum): + """ + Different states in which the server can be in. + """ + + RUNNING = 0 + NOT_INITALIZED = 1 + MISSING_CONTAINER = 2 + NOT_RUNNING = 3 + ERROR = 4 + + +def generate_password(length, pass_command: str = None) -> str: + """ + Function for generating passwords for redis container. If a specified command is given + then a password would be generated with the given command. If not a password will be + created by combining a string a characters based on the given length. + + :return:: string value with given length + """ + if pass_command: + process = subprocess.run(pass_command.split(), shell=True, stdout=subprocess.PIPE) + return process.stdout + + characters = list(string.ascii_letters + string.digits + "!@#$%^&*()") + + random.shuffle(characters) + + password = [] + for i in range(length): + password.append(random.choice(characters)) + random.shuffle(password) + return "".join(password) -def parse_redis_output(redis_stdout): + +def parse_redis_output(redis_stdout) -> Tuple[bool, str]: + """ + Parse the redis output for a the redis container. It will get all the necessary information + from the output and returns a dictionary of those values. + + :return:: two values is_successful, dictionary of values from redis output + """ if redis_stdout is None: return False, "None passed as redis output" server_init = False @@ -30,12 +93,91 @@ def parse_redis_output(redis_stdout): return False, line.decode("utf-8") -def pull_server_config() -> dict: +def create_server_config() -> bool: + """ + Create main configuration file for merlin server in the + merlin configuration directory. If a configuration already + exists it will not replace the current configuration and exit. + + :return:: True if success and False if fail + """ + if not os.path.exists(MERLIN_CONFIG_DIR): + LOG.error("Unable to find main merlin configuration directory at " + MERLIN_CONFIG_DIR) + return False + + config_dir = os.path.join(MERLIN_CONFIG_DIR, MERLIN_SERVER_SUBDIR) + if not os.path.exists(config_dir): + LOG.info("Unable to find exisiting server configuration.") + LOG.info(f"Creating default configuration in {config_dir}") + try: + os.mkdir(config_dir) + except OSError as err: + LOG.error(err) + return False + + files = [i + ".yaml" for i in CONTAINER_TYPES] + files.append(MERLIN_SERVER_CONFIG) + for file in files: + file_path = os.path.join(config_dir, file) + if os.path.exists(file_path): + LOG.info(f"{file} already exists.") + continue + LOG.info(f"Copying file {file} to configuration directory.") + try: + shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), file), config_dir) + except OSError: + LOG.error(f"Destination location {config_dir} is not writable.") + return False + + server_config = pull_server_config() + + if not os.path.exists(server_config.container.get_config_dir()): + LOG.info("Creating merlin server directory.") + os.mkdir(server_config.container.get_config_dir()) + + return True + + +def config_merlin_server(): + """ + Configurate the merlin server with configurations such as username password and etc. + """ + + server_config = pull_server_config() + + pass_file = server_config.container.get_pass_file_path() + if os.path.exists(pass_file): + LOG.info("Password file already exists. Skipping password generation step.") + else: + # if "pass_command" in server_config["container"]: + # password = generate_password(PASSWORD_LENGTH, server_config["container"]["pass_command"]) + # else: + password = generate_password(PASSWORD_LENGTH) + + with open(pass_file, "w+") as f: + f.write(password) + + LOG.info("Creating password file for merlin server container.") + + user_file = server_config.container.get_user_file_path() + if os.path.exists(user_file): + LOG.info("User file already exists.") + else: + redis_users = RedisUsers(user_file) + redis_config = RedisConfig(server_config.container.get_config_path()) + redis_users.add_user(user="default", password=redis_config.get_password()) + redis_users.add_user(user=os.environ.get("USER"), password=server_config.container.get_container_password()) + redis_users.write() + + LOG.info("User {} created in user file for merlin server container".format(os.environ.get("USER"))) + + +def pull_server_config() -> ServerConfig: """ Pull the main configuration file and corresponding format configuration file as well. Returns the values as a dictionary. - :return: A dictionary containing the main and corresponding format configuration file + :return: A instance of ServerConfig containing all the necessary configuration values. """ return_data = {} format_needed_keys = ["command", "run_command", "stop_command", "pull_command"] @@ -78,10 +220,95 @@ def pull_server_config() -> dict: LOG.error(f'Process necessary "{key}" command configuration not found in {MERLIN_SERVER_CONFIG}') return None - return return_data + return ServerConfig(return_data) + + +def pull_server_image() -> bool: + """ + Fetch the server image using singularity. + + :return:: True if success and False if fail + """ + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + + config_dir = server_config.container.get_config_dir() + config_file = server_config.container.get_config_name() + image_url = server_config.container.get_image_url() + image_path = server_config.container.get_image_path() + + if not os.path.exists(image_path): + LOG.info(f"Fetching redis image from {image_url}") + subprocess.run( + server_config.container_format.get_pull_command() + .strip("\\") + .format(command=server_config.container_format.get_command(), image=image_path, url=image_url) + .split(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + else: + LOG.info(f"{image_path} already exists.") + if not os.path.exists(os.path.join(config_dir, config_file)): + LOG.info("Copying default redis configuration file.") + try: + file_dir = os.path.dirname(os.path.abspath(__file__)) + shutil.copy(os.path.join(file_dir, config_file), config_dir) + except OSError: + LOG.error(f"Destination location {config_dir} is not writable.") + return False + else: + LOG.info("Redis configuration file already exist.") + + return True -def check_process_file_format(data): + +def get_server_status(): + """ + Determine the status of the current server. + This function can be used to check if the servers + have been initalized, started, or stopped. + + :param `server_dir`: location of all server related files. + :param `image_name`: name of the image when fetched. + :return:: A enum value of ServerStatus describing its current state. + """ + server_config = pull_server_config() + if not server_config: + return ServerStatus.NOT_INITALIZED + + if not os.path.exists(server_config.container.get_config_dir()): + return ServerStatus.NOT_INITALIZED + + if not os.path.exists(server_config.container.get_image_path()): + return ServerStatus.MISSING_CONTAINER + + if not os.path.exists(server_config.container.get_pfile_path()): + return ServerStatus.NOT_RUNNING + + pf_data = pull_process_file(server_config.container.get_pfile_path()) + parent_pid = pf_data["parent_pid"] + + check_process = subprocess.run( + server_config.process.get_status_command().strip("\\").format(pid=parent_pid).split(), + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + + if check_process.stdout == b"": + return ServerStatus.NOT_RUNNING + + return ServerStatus.RUNNING + + +def check_process_file_format(data: dict) -> bool: + """ + Check to see if the process file has the correct format and contains the expected key values. + :return:: True if success and False if fail + """ required_keys = ["parent_pid", "image_pid", "port", "hostname"] for key in required_keys: if key not in data: @@ -89,7 +316,12 @@ def check_process_file_format(data): return True -def pull_process_file(file_path): +def pull_process_file(file_path: str) -> dict: + """ + Pull the data from the process file. If one is found returns the data in a dictionary + if not returns None + :return:: Data containing in process file. + """ with open(file_path, "r") as f: data = yaml.load(f, yaml.Loader) if check_process_file_format(data): @@ -97,7 +329,11 @@ def pull_process_file(file_path): return None -def dump_process_file(data, file_path): +def dump_process_file(data: dict, file_path: str): + """ + Dump the process data from the dictionary to the specified file path. + :return:: True if success and False if fail + """ if not check_process_file_format(data): return False with open(file_path, "w+") as f: diff --git a/merlin/server/server_setup.py b/merlin/server/server_setup.py deleted file mode 100644 index 75d1f68a3..000000000 --- a/merlin/server/server_setup.py +++ /dev/null @@ -1,291 +0,0 @@ -"""Main functions for instantiating and running Merlin server containers.""" - -import enum -import logging -import os -import shutil -import socket -import subprocess -import time - -from merlin.server.server_config import ( - MERLIN_CONFIG_DIR, - MERLIN_SERVER_CONFIG, - MERLIN_SERVER_SUBDIR, - dump_process_file, - parse_redis_output, - pull_process_file, - pull_server_config, -) - - -# Default values for configuration -CONFIG_DIR = "./merlin_server/" -IMAGE_NAME = "redis_latest.sif" -PROCESS_FILE = "merlin_server.pf" -CONFIG_FILE = "redis.conf" -REDIS_URL = "docker://redis" -CONTAINER_TYPES = ["singularity", "docker", "podman"] - -LOG = logging.getLogger("merlin") - - -class ServerStatus(enum.Enum): - """ - Different states in which the server can be in. - """ - - RUNNING = 0 - NOT_INITALIZED = 1 - MISSING_CONTAINER = 2 - NOT_RUNNING = 3 - ERROR = 4 - - -def create_server_config(): - """ - Create main configuration file for merlin server in the - merlin configuration directory. If a configuration already - exists it will not replace the current configuration and exit. - """ - if not os.path.exists(MERLIN_CONFIG_DIR): - LOG.error("Unable to find main merlin configuration directory at " + MERLIN_CONFIG_DIR) - return False - - config_dir = os.path.join(MERLIN_CONFIG_DIR, MERLIN_SERVER_SUBDIR) - if not os.path.exists(config_dir): - LOG.info("Unable to find exisiting server configuration.") - LOG.info(f"Creating default configuration in {config_dir}") - try: - os.mkdir(config_dir) - except OSError as err: - LOG.error(err) - return False - - files = [i + ".yaml" for i in CONTAINER_TYPES] - files.append(MERLIN_SERVER_CONFIG) - for file in files: - file_path = os.path.join(config_dir, file) - if os.path.exists(file_path): - LOG.info(f"{file} already exists.") - continue - LOG.info(f"Copying file {file} to configuration directory.") - try: - shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), file), config_dir) - except OSError: - LOG.error(f"Destination location {config_dir} is not writable.") - return False - - return True - - -def pull_server_image(): - """ - Fetch the server image using singularity. - """ - server_config = pull_server_config() - if not server_config: - LOG.error('Try to run "merlin server init" again to reinitialize values.') - return False - - container_config = server_config["container"] - config_dir = container_config["config_dir"] if "config_dir" in container_config else CONFIG_DIR - image_name = container_config["image"] if "image" in container_config else IMAGE_NAME - config_file = container_config["config"] if "config" in container_config else CONFIG_FILE - image_url = container_config["url"] if "url" in container_config else REDIS_URL - - if not os.path.exists(config_dir): - LOG.info("Creating merlin server directory.") - os.mkdir(config_dir) - - image_path = os.path.join(config_dir, image_name) - - if os.path.exists(image_path): - LOG.info(f"{image_path} already exists.") - return False - - LOG.info(f"Fetching redis image from {image_url}") - format_config = server_config[container_config["format"]] - subprocess.run( - format_config["pull_command"] - .strip("\\") - .format(command=format_config["command"], image=image_path, url=image_url) - .split(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - LOG.info("Copying default redis configuration file.") - try: - file_dir = os.path.dirname(os.path.abspath(__file__)) - shutil.copy(os.path.join(file_dir, config_file), config_dir) - except OSError: - LOG.error(f"Destination location {config_dir} is not writable.") - return False - return True - - -def get_server_status(): - """ - Determine the status of the current server. - This function can be used to check if the servers - have been initalized, started, or stopped. - - :param `server_dir`: location of all server related files. - :param `image_name`: name of the image when fetched. - """ - server_config = pull_server_config() - if not server_config: - return ServerStatus.NOT_INITALIZED - - container_config = server_config["container"] - config_dir = container_config["config_dir"] if "config_dir" in container_config else CONFIG_DIR - image_name = container_config["image"] if "image" in container_config else IMAGE_NAME - pfile = container_config["pfile"] if "pfile" in container_config else PROCESS_FILE - - if not os.path.exists(config_dir): - return ServerStatus.NOT_INITALIZED - - if not os.path.exists(os.path.join(config_dir, image_name)): - return ServerStatus.MISSING_CONTAINER - - if not os.path.exists(os.path.join(config_dir, pfile)): - return ServerStatus.NOT_RUNNING - - pf_data = pull_process_file(os.path.join(config_dir, pfile)) - parent_pid = pf_data["parent_pid"] - - check_process = subprocess.run( - server_config["process"]["status"].strip("\\").format(pid=parent_pid).split(), - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - ) - - if check_process.stdout == b"": - return ServerStatus.NOT_RUNNING - - return ServerStatus.RUNNING - - -def start_server(): - """ - Start a merlin server container using singularity. - - :param `server_dir`: location of all server related files. - :param `image_name`: name of the image when fetched. - """ - current_status = get_server_status() - - if current_status == ServerStatus.NOT_INITALIZED or current_status == ServerStatus.MISSING_CONTAINER: - LOG.info("Merlin server has not been initialized. Please run 'merlin server init' first.") - return False - - if current_status == ServerStatus.RUNNING: - LOG.info("Merlin server already running.") - LOG.info("Stop current server with 'merlin server stop' before attempting to start a new server.") - return False - - server_config = pull_server_config() - if not server_config: - LOG.error('Try to run "merlin server init" again to reinitialize values.') - return False - container_config = server_config["container"] - - config_dir = container_config["config_dir"] if "config_dir" in container_config else CONFIG_DIR - config_file = container_config["config"] if "config_dir" in container_config else CONFIG_FILE - image_name = container_config["image"] if "image" in container_config else IMAGE_NAME - pfile = container_config["pfile"] if "pfile" in container_config else PROCESS_FILE - - image_path = os.path.join(config_dir, image_name) - if not os.path.exists(image_path): - LOG.error("Unable to find image at " + image_path) - return False - - config_path = os.path.join(config_dir, config_file) - if not os.path.exists(config_path): - LOG.error("Unable to find config file at " + config_path) - return False - - format_config = server_config[container_config["format"]] - process = subprocess.Popen( - format_config["run_command"] - .strip("\\") - .format(command=container_config["format"], image=image_path, config=config_path) - .split(), - start_new_session=True, - close_fds=True, - stdout=subprocess.PIPE, - ) - - time.sleep(1) - - redis_start, redis_out = parse_redis_output(process.stdout) - - if not redis_start: - LOG.error("Redis is unable to start") - LOG.error('Check to see if there is an unresponsive instance of redis with "ps -e"') - LOG.error(redis_out.strip("\n")) - return False - - redis_out["image_pid"] = redis_out.pop("pid") - redis_out["parent_pid"] = process.pid - redis_out["hostname"] = socket.gethostname() - if not dump_process_file(redis_out, os.path.join(config_dir, pfile)): - LOG.error("Unable to create process file for container.") - return False - - if get_server_status() != ServerStatus.RUNNING: - LOG.error("Unable to start merlin server.") - return False - - LOG.info(f"Server started with PID {str(process.pid)}.") - LOG.info(f'Merlin server operating on "{redis_out["hostname"]}" and port "{redis_out["port"]}".') - - return True - - -def stop_server(): - """ - Stop running merlin server containers. - - """ - if get_server_status() != ServerStatus.RUNNING: - LOG.info("There is no instance of merlin server running.") - LOG.info("Start a merlin server first with 'merlin server start'") - return False - - server_config = pull_server_config() - if not server_config: - LOG.error('Try to run "merlin server init" again to reinitialize values.') - return False - container_config = server_config["container"] - - config_dir = container_config["config_dir"] if "config_dir" in container_config else CONFIG_DIR - pfile = container_config["pfile"] if "pfile" in container_config else PROCESS_FILE - image_name = container_config["name"] if "name" in container_config else IMAGE_NAME - - pf_data = pull_process_file(os.path.join(config_dir, pfile)) - read_pid = pf_data["parent_pid"] - - process = subprocess.run( - server_config["process"]["status"].strip("\\").format(pid=read_pid).split(), stdout=subprocess.PIPE - ) - if process.stdout == b"": - LOG.error("Unable to get the PID for the current merlin server.") - return False - - format_config = server_config[container_config["format"]] - command = server_config["process"]["kill"].strip("\\").format(pid=read_pid).split() - if format_config["stop_command"] != "kill": - command = format_config["stop_command"].strip("\\").format(name=image_name).split() - - LOG.info(f"Attempting to close merlin server PID {str(read_pid)}") - - subprocess.run(command, stdout=subprocess.PIPE) - time.sleep(1) - if get_server_status() == ServerStatus.RUNNING: - LOG.error("Unable to kill process.") - return False - - LOG.info("Merlin server terminated.") - return True diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py new file mode 100644 index 000000000..0f8569d17 --- /dev/null +++ b/merlin/server/server_util.py @@ -0,0 +1,510 @@ +import hashlib +import logging +import os + +import redis +import yaml + + +LOG = logging.getLogger("merlin") + +# Constants for main merlin server configuration values. +CONTAINER_TYPES = ["singularity", "docker", "podman"] +MERLIN_CONFIG_DIR = os.path.join(os.path.expanduser("~"), ".merlin") +MERLIN_SERVER_SUBDIR = "server/" +MERLIN_SERVER_CONFIG = "merlin_server.yaml" + + +def valid_ipv4(ip: str) -> bool: + """ + Checks valid ip address + """ + if not ip: + return False + + arr = ip.split(".") + if len(arr) != 4: + return False + + for i in arr: + if int(i) < 0 and int(i) > 255: + return False + + return True + + +def valid_port(port: int) -> bool: + """ + Checks valid network port + """ + if port > 0 and port < 65536: + return True + return False + + +class ContainerConfig: + """ + ContainerConfig provides interface for parsing and interacting with the container value specified within + the merlin_server.yaml configuration file. Dictionary of the config values should be passed when initialized + to parse values. This can be done after parsing yaml to data dictionary. + If there are missing values within the configuration it will be populated with default values for + singularity container. + + Configuration contains values for setting up containers and storing values specific to each container. + Values that are stored consist of things within the local configuration directory as different runs + can have differnt configuration values. + """ + + # Default values for configuration + FORMAT = "singularity" + IMAGE_NAME = "redis_latest.sif" + REDIS_URL = "docker://redis" + CONFIG_FILE = "redis.conf" + CONFIG_DIR = "./merlin_server/" + PROCESS_FILE = "merlin_server.pf" + PASSWORD_FILE = "redis.pass" + USERS_FILE = "redis.users" + + format = FORMAT + image = IMAGE_NAME + url = REDIS_URL + config = CONFIG_FILE + config_dir = CONFIG_DIR + pfile = PROCESS_FILE + pass_file = PASSWORD_FILE + user_file = USERS_FILE + + def __init__(self, data: dict) -> None: + self.format = data["format"] if "format" in data else self.FORMAT + self.image = data["image"] if "image" in data else self.IMAGE_NAME + self.url = data["url"] if "url" in data else self.REDIS_URL + self.config = data["config"] if "config" in data else self.CONFIG_FILE + self.config_dir = data["config_dir"] if "config_dir" in data else self.CONFIG_DIR + self.pfile = data["pfile"] if "pfile" in data else self.PROCESS_FILE + self.pass_file = data["pass_file"] if "pass_file" in data else self.PASSWORD_FILE + self.user_file = data["user_file"] if "user_file" in data else self.USERS_FILE + + def get_format(self) -> str: + return self.format + + def get_image_name(self) -> str: + return self.image + + def get_image_url(self) -> str: + return self.url + + def get_image_path(self) -> str: + return os.path.join(self.config_dir, self.image) + + def get_config_name(self) -> str: + return self.config + + def get_config_path(self) -> str: + return os.path.join(self.config_dir, self.config) + + def get_config_dir(self) -> str: + return self.config_dir + + def get_pfile_name(self) -> str: + return self.pfile + + def get_pfile_path(self) -> str: + return os.path.join(self.config_dir, self.pfile) + + def get_pass_file_name(self) -> str: + return self.pass_file + + def get_pass_file_path(self) -> str: + return os.path.join(self.config_dir, self.pass_file) + + def get_user_file_name(self) -> str: + return self.user_file + + def get_user_file_path(self) -> str: + return os.path.join(self.config_dir, self.user_file) + + def get_container_password(self) -> str: + password = None + with open(self.get_pass_file_path(), "r") as f: + password = f.read() + return password + + +class ContainerFormatConfig: + """ + ContainerFormatConfig provides an interface for parsing and interacting with container specific + configuration files .yaml. These configuration files contain container specific + commands to run containerizers such as singularity, docker, and podman. + """ + + COMMAND = "singularity" + RUN_COMMAND = "{command} run {image} {config}" + STOP_COMMAND = "kill" + PULL_COMMAND = "{command} pull {image} {url}" + + command = COMMAND + run_command = RUN_COMMAND + stop_command = STOP_COMMAND + pull_command = PULL_COMMAND + + def __init__(self, data: dict) -> None: + self.command = data["command"] if "command" in data else self.COMMAND + self.run_command = data["run_command"] if "run_command" in data else self.RUN_COMMAND + self.stop_command = data["stop_command"] if "stop_command" in data else self.STOP_COMMAND + self.pull_command = data["pull_command"] if "pull_command" in data else self.PULL_COMMAND + + def get_command(self) -> str: + return self.command + + def get_run_command(self) -> str: + return self.run_command + + def get_stop_command(self) -> str: + return self.stop_command + + def get_pull_command(self) -> str: + return self.pull_command + + +class ProcessConfig: + """ + ProcessConfig provides an interface for parsing and interacting with process config specified + in merlin_server.yaml configuration. This configuration provide commands for interfacing with + host machine while the containers are running. + """ + + STATUS_COMMAND = "pgrep -P {pid}" + KILL_COMMAND = "kill {pid}" + + status = STATUS_COMMAND + kill = KILL_COMMAND + + def __init__(self, data: dict) -> None: + self.status = data["status"] if "status" in data else self.STATUS_COMMAND + self.kill = data["kill"] if "kill" in data else self.KILL_COMMAND + + def get_status_command(self) -> str: + return self.status + + def get_kill_command(self) -> str: + return self.kill + + +class ServerConfig: + """ + ServerConfig is an interface for storing all the necessary configuration for merlin server. + These configuration container things such as ContainerConfig, ProcessConfig, and ContainerFormatConfig. + """ + + container: ContainerConfig = None + process: ProcessConfig = None + container_format: ContainerFormatConfig = None + + def __init__(self, data: dict) -> None: + if "container" in data: + self.container = ContainerConfig(data["container"]) + if "process" in data: + self.process = ProcessConfig(data["process"]) + if self.container.get_format() in data: + self.container_format = ContainerFormatConfig(data[self.container.get_format()]) + + +class RedisConfig: + """ + RedisConfig is an interface for parsing and interacing with redis.conf file that is provided + by redis. This allows users to parse the given redis configuration and make edits and allow users + to write those changes into a redis readable config file. + """ + + filename = "" + entry_order = [] + entries = {} + comments = {} + trailing_comments = "" + changed = False + + def __init__(self, filename) -> None: + self.filename = filename + self.changed = False + self.parse() + + def parse(self) -> None: + self.entries = {} + self.comments = {} + with open(self.filename, "r+") as f: + file_contents = f.read() + file_lines = file_contents.split("\n") + comments = "" + for line in file_lines: + if len(line) > 0 and line[0] != "#": + line_contents = line.split(maxsplit=1) + if line_contents[0] in self.entries: + sub_split = line_contents[1].split(maxsplit=1) + line_contents[0] += " " + sub_split[0] + line_contents[1] = sub_split[1] + self.entry_order.append(line_contents[0]) + self.entries[line_contents[0]] = line_contents[1] + self.comments[line_contents[0]] = comments + comments = "" + else: + comments += line + "\n" + self.trailing_comments = comments[:-1] + + def write(self) -> None: + with open(self.filename, "w") as f: + for entry in self.entry_order: + f.write(self.comments[entry]) + f.write(f"{entry} {self.entries[entry]}\n") + f.write(self.trailing_comments) + + def set_filename(self, filename: str) -> None: + self.filename = filename + + def set_config_value(self, key: str, value: str) -> bool: + if key not in self.entries: + return False + self.entries[key] = value + self.changed = True + return True + + def get_config_value(self, key: str) -> str: + if key in self.entries: + return self.entries[key] + return None + + def changes_made(self) -> bool: + return self.changed + + def get_ip_address(self) -> str: + return self.get_config_value("bind") + + def set_ip_address(self, ipaddress: str) -> bool: + if ipaddress is None: + return False + # Check if ipaddress is valid + if valid_ipv4(ipaddress): + # Set ip address in redis config + if not self.set_config_value("bind", ipaddress): + LOG.error("Unable to set ip address for redis config") + return False + else: + LOG.error("Invalid IPv4 address given.") + return False + LOG.info(f"Ipaddress is set to {ipaddress}") + return True + + def get_port(self) -> str: + return self.get_config_value("port") + + def set_port(self, port: str) -> bool: + if port is None: + return False + # Check if port is valid + if valid_port(port): + # Set port in redis config + if not self.set_config_value("port", port): + LOG.error("Unable to set port for redis config") + return False + else: + LOG.error("Invalid port given.") + return False + LOG.info(f"Port is set to {port}") + return True + + def set_password(self, password: str) -> bool: + if password is None: + return False + self.set_config_value("requirepass", password) + LOG.info(f"Password file set to {password}") + return True + + def get_password(self) -> str: + return self.get_config_value("requirepass") + + def set_directory(self, directory: str) -> bool: + if directory is None: + return False + # Validate the directory input + if os.path.exists(directory): + # Set the save directory to the redis config + if not self.set_config_value("dir", directory): + LOG.error("Unable to set directory for redis config") + return False + else: + LOG.error("Directory given does not exist.") + return False + LOG.info(f"Directory is set to {directory}") + return True + + def set_snapshot_seconds(self, seconds: int) -> bool: + if seconds is None: + return False + # Set the snapshot second in the redis config + value = self.get_config_value("save") + if value is None: + LOG.error("Unable to get exisiting parameter values for snapshot") + return False + else: + value = value.split() + value[0] = str(seconds) + value = " ".join(value) + if not self.set_config_value("save", value): + LOG.error("Unable to set snapshot value seconds") + return False + LOG.info(f"Snapshot wait time is set to {seconds} seconds") + return True + + def set_snapshot_changes(self, changes: int) -> bool: + if changes is None: + return False + # Set the snapshot changes into the redis config + value = self.get_config_value("save") + if value is None: + LOG.error("Unable to get exisiting parameter values for snapshot") + return False + else: + value = value.split() + value[1] = str(changes) + value = " ".join(value) + if not self.set_config_value("save", value): + LOG.error("Unable to set snapshot value seconds") + return False + LOG.info(f"Snapshot threshold is set to {changes} changes") + return True + + def set_snapshot_file(self, file: str) -> bool: + if file is None: + return False + # Set the snapshot file in the redis config + if not self.set_config_value("dbfilename", file): + LOG.error("Unable to set snapshot_file name") + return False + + LOG.info(f"Snapshot file is set to {file}") + return True + + def set_append_mode(self, mode: str) -> bool: + if mode is None: + return False + valid_modes = ["always", "everysec", "no"] + + # Validate the append mode (always, everysec, no) + if mode in valid_modes: + # Set the append mode in the redis config + if not self.set_config_value("appendfsync", mode): + LOG.error("Unable to set append_mode in redis config") + return False + else: + LOG.error("Not a valid append_mode(Only valid modes are always, everysec, no)") + return False + + LOG.info(f"Append mode is set to {mode}") + return True + + def set_append_file(self, file: str) -> bool: + if file is None: + return False + # Set the append file in the redis config + if not self.set_config_value("appendfilename", file): + LOG.error("Unable to set append filename.") + return False + LOG.info(f"Append file is set to {file}") + return True + + +class RedisUsers: + """ + RedisUsers provides an interface for parsing and interacting with redis.users configuration + file. Allow users and merlin server to create, remove, and edit users within the redis files. + Changes can be sync and push to an exisiting redis server if one is available. + """ + + class User: + status = "on" + hash_password = hashlib.sha256(b"password").hexdigest() + keys = "*" + commands = "@all" + + def __init__(self, status="on", keys="*", commands="@all", password=None) -> None: + self.status = status + self.keys = keys + self.commands = commands + if password is not None: + self.set_password(password) + + def parse_dict(self, dict: dict) -> None: + self.status = dict["status"] + self.keys = dict["keys"] + self.commands = dict["commands"] + self.hash_password = dict["hash_password"] + + def get_user_dict(self) -> dict: + self.status = "on" + return {"status": self.status, "hash_password": self.hash_password, "keys": self.keys, "commands": self.commands} + + def __repr__(self) -> str: + return str(self.get_user_dict()) + + def __str__(self) -> str: + return self.__repr__() + + def set_password(self, password: str) -> None: + self.hash_password = hashlib.sha256(bytes(password, "utf-8")).hexdigest() + + filename = "" + users = {} + + def __init__(self, filename) -> None: + self.filename = filename + if os.path.exists(self.filename): + self.parse() + + def parse(self) -> None: + with open(self.filename, "r") as f: + self.users = yaml.load(f, yaml.Loader) + for user in self.users: + new_user = self.User() + new_user.parse_dict(self.users[user]) + self.users[user] = new_user + + def write(self) -> None: + data = self.users.copy() + for key in data: + data[key] = self.users[key].get_user_dict() + with open(self.filename, "w") as f: + yaml.dump(data, f, yaml.Dumper) + + def add_user(self, user, status="on", keys="*", commands="@all", password=None) -> bool: + if user in self.users: + return False + self.users[user] = self.User(status, keys, commands, password) + return True + + def set_password(self, user: str, password: str): + if user not in self.users: + return False + self.users[user].set_password(password) + + def remove_user(self, user) -> bool: + if user in self.users: + del self.users[user] + return True + return False + + def apply_to_redis(self, host: str, port: int, password: str) -> None: + db = redis.Redis(host=host, port=port, password=password) + current_users = db.acl_users() + for user in self.users: + if user not in current_users: + data = self.users[user] + db.acl_setuser( + username=user, + hashed_passwords=[f"+{data.hash_password}"], + enabled=(data.status == "on"), + keys=data.keys, + commands=[f"+{data.commands}"], + ) + + for user in current_users: + if user not in self.users: + db.acl_deluser(user) From aad56efa56534398af9fb9c4dce805994d3bee1a Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Mon, 25 Jul 2022 15:41:00 -0700 Subject: [PATCH 037/126] Bugfix/changelog ci (#370) * remove deprecated gitlab ci file * Change CHANGELOG test to work for PRs other than to main --- .github/workflows/push-pr_workflow.yml | 2 +- .gitlab-ci.yml | 19 ------------------- CHANGELOG.md | 5 ++++- 3 files changed, 5 insertions(+), 21 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 44a2255b9..cc04e3afe 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -14,7 +14,7 @@ jobs: - name: Check that CHANGELOG has been updated run: | # If this step fails, this means you haven't updated the CHANGELOG.md file with notes on your contribution. - git diff --name-only $(git merge-base origin/main HEAD) | grep '^CHANGELOG.md$' && echo "Thanks for helping keep our CHANGELOG up-to-date!" + git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep '^CHANGELOG.md$' && echo "Thanks for helping keep our CHANGELOG up-to-date!" Lint: runs-on: ubuntu-latest diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 9403886e3..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,19 +0,0 @@ -image: python:3.8-slim-buster - -job1: - script: - - python3 -m venv venv - - source venv/bin/activate - - pip3 install --upgrade pip - - pip3 install -r requirements.txt - - pip3 install -r requirements/dev.txt - - pip3 install -r merlin/examples/workflows/feature_demo/requirements.txt - - pip3 install -e . - - pip3 install --upgrade sphinx - - merlin config - - - merlin stop-workers - - - python3 -m pytest tests/ - - python3 tests/integration/run_tests.py --verbose --local - diff --git a/CHANGELOG.md b/CHANGELOG.md index 195483640..769545acb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added merlin server capabilities under merlin/server/ - Added merlin server commands init, start, status, stop to main.py - Added redis.conf for default redis configuration for merlin in server/redis.conf -- Added default configurations for merlin server command in merlin/server/*.yaml +- Added default configurations for merlin server command in `merlin/server/*.yaml` - Added documentation page docs/merlin_server.rst, docs/modules/server/configuration.rst, and docs/modules/server/commands.rst - Added merlin server config command for editing configuration files. - Added server_command.py to store command calls. @@ -44,11 +44,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added the flux_exec batch argument to allow for flux exec arguments, e.g. flux_exec: flux exec -r "0-1" to run celery workers only on ranks 0 and 1 of a multi-rank allocation + ### Changed - Rename lgtm.yml to .lgtm.yml - Changed "default" user password to be "merlin_password" as default. + ### Fixed - Fixed return values from scripts with main() to fix testing errors. +- CI test for CHANGELOG modifcations ## [1.8.5] ### Added From f551d193ecbcef2d0c5656ab01c0f1c008214971 Mon Sep 17 00:00:00 2001 From: Ryan Lee <44886374+ryannova@users.noreply.github.com> Date: Tue, 26 Jul 2022 14:53:21 -0700 Subject: [PATCH 038/126] App.yaml for merlin server (#369) * Added AppYaml class to pull app.yaml and make changes required for merlin server configuration * Applied AppYaml class and added log message to inform users to use new app.yaml to use merlin server * Update LOG messages to inform users regarding local runs and instruct users of how to use app.yaml for local configuration * Changed type to image type in ContainerConfig * Shorten CHANGELOG.md for merlin server changes * Updated read in AppYaml to utilize merlin.util.load_yaml --- CHANGELOG.md | 33 +--------------------- merlin/server/merlin_server.yaml | 2 ++ merlin/server/server_commands.py | 10 ++++++- merlin/server/server_util.py | 48 ++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 769545acb..3940237f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,38 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update docker docs for new rabbitmq and redis server versions - Added lgtm.com Badge for README.md - More fixes for lgtm checks. -- Added merlin server capabilities under merlin/server/ -- Added merlin server commands init, start, status, stop to main.py -- Added redis.conf for default redis configuration for merlin in server/redis.conf -- Added default configurations for merlin server command in `merlin/server/*.yaml` -- Added documentation page docs/merlin_server.rst, docs/modules/server/configuration.rst, and docs/modules/server/commands.rst -- Added merlin server config command for editing configuration files. -- Added server_command.py to store command calls. -- Added following flags to config subcommand - - ipaddress (Set the binded ip address of the container) - - port (Set the binded port of the container) - - user (Set the main user file for container) - - password (Set the main user password file for container) - - add-user (Add a user to the container image [outputs an associated password file for user]) - - remove-user (Remove user from list of added users) - - directory (Set the directory of the merlin server container files) - - snapshot-seconds (Set the number of seconds elapsed before snapshot change condition is checked) - - snapshot-changes (Set snapshot change condition for a snapshot to be made) - - snapshot-file (Set the database file that the snapshot will be written to) - - append-mode (Set the append mode for redis) - - append-file (Set the name of the append only file for redis) -- Added user_file to merlin server config -- Added pass_file to merlin server config -- Added add_user function to add user to exisiting merlin server instance if one is running -- Added remove_user function to remove user from merlin server instance if one is running -- Added masteruser in redis config -- Added requirepass in redis config -- Added server_util.py file to store utility functions. -- Created RedisConfig class to interface with redis.conf file -- Created RedisUsers class to interface with redis.user file -- Added better interface for configuration files(ServerConfig, ContainerConfig, ContainerFormatConfig, and ProcessConfig) with getting configuration values from merlin server config file, with classes. -- Added merlin server to reapply users based on the saved redis.users config file. -- Added redis.pass file containing password for default user in main merlin configuration. +- Added merlin server command as a container option for broker and results_backend servers. - Added the flux_exec batch argument to allow for flux exec arguments, e.g. flux_exec: flux exec -r "0-1" to run celery workers only on ranks 0 and 1 of a multi-rank allocation diff --git a/merlin/server/merlin_server.yaml b/merlin/server/merlin_server.yaml index c351a06d4..5f9b25367 100644 --- a/merlin/server/merlin_server.yaml +++ b/merlin/server/merlin_server.yaml @@ -1,6 +1,8 @@ container: # Select the format for the recipe e.g. singularity, docker, podman (currently singularity is the only working option.) format: singularity + #Type of container that is used + image_type: redis # The image name image: redis_latest.sif # The url to pull the image from diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 5782183a4..a487f4ca9 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -18,7 +18,7 @@ pull_server_config, pull_server_image, ) -from merlin.server.server_util import RedisConfig, RedisUsers +from merlin.server.server_util import AppYaml, RedisConfig, RedisUsers LOG = logging.getLogger("merlin") @@ -195,6 +195,14 @@ def start_server() -> bool: redis_config = RedisConfig(server_config.container.get_config_path()) redis_users.apply_to_redis(redis_config.get_ip_address(), redis_config.get_port(), redis_config.get_password()) + new_app_yaml = os.path.join(server_config.container.get_config_dir(), "app.yaml") + ay = AppYaml() + ay.apply_server_config(server_config=server_config) + ay.write(new_app_yaml) + LOG.info(f"New app.yaml written to {new_app_yaml}.") + LOG.info("Replace app.yaml in ~/.merlin/app.yaml to use merlin server as main configuration.") + LOG.info("To use for local runs, move app.yaml into the running directory.") + return True diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 0f8569d17..90c612ef0 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -5,6 +5,8 @@ import redis import yaml +import merlin.utils + LOG = logging.getLogger("merlin") @@ -57,6 +59,7 @@ class ContainerConfig: # Default values for configuration FORMAT = "singularity" + IMAGE_TYPE = "redis" IMAGE_NAME = "redis_latest.sif" REDIS_URL = "docker://redis" CONFIG_FILE = "redis.conf" @@ -66,6 +69,7 @@ class ContainerConfig: USERS_FILE = "redis.users" format = FORMAT + image_type = IMAGE_TYPE image = IMAGE_NAME url = REDIS_URL config = CONFIG_FILE @@ -76,6 +80,7 @@ class ContainerConfig: def __init__(self, data: dict) -> None: self.format = data["format"] if "format" in data else self.FORMAT + self.image_type = data["image_type"] if "image_type" in data else self.IMAGE_TYPE self.image = data["image"] if "image" in data else self.IMAGE_NAME self.url = data["url"] if "url" in data else self.REDIS_URL self.config = data["config"] if "config" in data else self.CONFIG_FILE @@ -87,6 +92,9 @@ def __init__(self, data: dict) -> None: def get_format(self) -> str: return self.format + def get_image_type(self) -> str: + return self.image_type + def get_image_name(self) -> str: return self.image @@ -508,3 +516,43 @@ def apply_to_redis(self, host: str, port: int, password: str) -> None: for user in current_users: if user not in self.users: db.acl_deluser(user) + + +class AppYaml: + """ + AppYaml allows for an structured way to interact with any app.yaml main merlin configuration file. + It helps to parse each component of the app.yaml and allow users to edit, configure and write the + file. + """ + + default_filename = os.path.join(MERLIN_CONFIG_DIR, "app.yaml") + data = {} + broker_name = "broker" + results_name = "results_backend" + + def __init__(self, filename: str = default_filename) -> None: + if not os.path.exists(filename): + filename = self.default_filename + self.read(filename) + + def apply_server_config(self, server_config: ServerConfig): + rc = RedisConfig(server_config.container.get_config_path()) + + self.data[self.broker_name]["name"] = server_config.container.get_image_type() + self.data[self.broker_name]["username"] = os.environ.get("USER") + self.data[self.broker_name]["password"] = server_config.container.get_pass_file_path() + self.data[self.broker_name]["server"] = rc.get_ip_address() + self.data[self.broker_name]["port"] = rc.get_port() + + self.data[self.results_name]["name"] = server_config.container.get_image_type() + self.data[self.results_name]["username"] = os.environ.get("USER") + self.data[self.results_name]["password"] = server_config.container.get_pass_file_path() + self.data[self.results_name]["server"] = rc.get_ip_address() + self.data[self.results_name]["port"] = rc.get_port() + + def read(self, filename: str = default_filename): + self.data = merlin.utils.load_yaml(filename) + + def write(self, filename: str = default_filename): + with open(filename, "w+") as f: + yaml.dump(self.data, f, yaml.Dumper) From 7c62cc520c1975132d03c34f80c087c58bbcdf0d Mon Sep 17 00:00:00 2001 From: Ryan Lee <44886374+ryannova@users.noreply.github.com> Date: Wed, 7 Sep 2022 17:36:30 -0700 Subject: [PATCH 039/126] Updated merlin server unit testing (#372) * Added additional tests for merlin server in test definitions * Fixed directory change to create a new directory if one doesn't exist * Updated redis version to provide acl user channel support --- CHANGELOG.md | 3 + merlin/server/server_util.py | 25 ++++++-- requirements/release.txt | 1 + tests/integration/conditions.py | 62 +++++++++++++++++++ tests/integration/run_tests.py | 12 +++- tests/integration/test_definitions.py | 85 +++++++++++++++++++++++++-- 6 files changed, 175 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3940237f3..03fcb3558 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,13 +10,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added lgtm.com Badge for README.md - More fixes for lgtm checks. - Added merlin server command as a container option for broker and results_backend servers. +- Added merlin server unit tests to test exisiting merlin server commands. - Added the flux_exec batch argument to allow for flux exec arguments, e.g. flux_exec: flux exec -r "0-1" to run celery workers only on ranks 0 and 1 of a multi-rank allocation +- Added additional argument in test definitions to allow for a "cleanup" command ### Changed - Rename lgtm.yml to .lgtm.yml - Changed "default" user password to be "merlin_password" as default. +- Update requirements to require redis 4.3.4 for acl user channel support ### Fixed - Fixed return values from scripts with main() to fix testing errors. diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 90c612ef0..d9698efe3 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -332,6 +332,9 @@ def get_password(self) -> str: def set_directory(self, directory: str) -> bool: if directory is None: return False + if not os.path.exists(directory): + os.mkdir(directory) + LOG.info(f"Created directory {directory}") # Validate the directory input if os.path.exists(directory): # Set the save directory to the redis config @@ -339,7 +342,7 @@ def set_directory(self, directory: str) -> bool: LOG.error("Unable to set directory for redis config") return False else: - LOG.error("Directory given does not exist.") + LOG.error(f"Directory {directory} given does not exist and could not be created.") return False LOG.info(f"Directory is set to {directory}") return True @@ -413,7 +416,7 @@ def set_append_file(self, file: str) -> bool: if file is None: return False # Set the append file in the redis config - if not self.set_config_value("appendfilename", file): + if not self.set_config_value("appendfilename", f'"{file}"'): LOG.error("Unable to set append filename.") return False LOG.info(f"Append file is set to {file}") @@ -431,11 +434,13 @@ class User: status = "on" hash_password = hashlib.sha256(b"password").hexdigest() keys = "*" + channels = "*" commands = "@all" - def __init__(self, status="on", keys="*", commands="@all", password=None) -> None: + def __init__(self, status="on", keys="*", channels="*", commands="@all", password=None) -> None: self.status = status self.keys = keys + self.channels = channels self.commands = commands if password is not None: self.set_password(password) @@ -443,12 +448,19 @@ def __init__(self, status="on", keys="*", commands="@all", password=None) -> Non def parse_dict(self, dict: dict) -> None: self.status = dict["status"] self.keys = dict["keys"] + self.channels = dict["channels"] self.commands = dict["commands"] self.hash_password = dict["hash_password"] def get_user_dict(self) -> dict: self.status = "on" - return {"status": self.status, "hash_password": self.hash_password, "keys": self.keys, "commands": self.commands} + return { + "status": self.status, + "hash_password": self.hash_password, + "keys": self.keys, + "channels": self.channels, + "commands": self.commands, + } def __repr__(self) -> str: return str(self.get_user_dict()) @@ -482,10 +494,10 @@ def write(self) -> None: with open(self.filename, "w") as f: yaml.dump(data, f, yaml.Dumper) - def add_user(self, user, status="on", keys="*", commands="@all", password=None) -> bool: + def add_user(self, user, status="on", keys="*", channels="*", commands="@all", password=None) -> bool: if user in self.users: return False - self.users[user] = self.User(status, keys, commands, password) + self.users[user] = self.User(status, keys, channels, commands, password) return True def set_password(self, user: str, password: str): @@ -510,6 +522,7 @@ def apply_to_redis(self, host: str, port: int, password: str) -> None: hashed_passwords=[f"+{data.hash_password}"], enabled=(data.status == "on"), keys=data.keys, + channels=data.channels, commands=[f"+{data.commands}"], ) diff --git a/requirements/release.txt b/requirements/release.txt index 4771b7a4c..055357b89 100644 --- a/requirements/release.txt +++ b/requirements/release.txt @@ -9,3 +9,4 @@ parse psutil>=5.1.0 pyyaml>=5.1.2 tabulate +redis>=4.3.4 \ No newline at end of file diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index afafa0d99..3448ec61a 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -238,3 +238,65 @@ def passes(self): if self.negate: return not self.is_within() return self.is_within() + + +class PathExists(Condition): + """ + A condition for checking if a path to a file or directory exists + """ + + def __init__(self, pathname) -> None: + self.pathname = pathname + + def path_exists(self) -> bool: + return os.path.exists(self.pathname) + + def __str__(self) -> str: + return f"{__class__.__name__} expected to find file or directory at {self.pathname}" + + @property + def passes(self): + return self.path_exists() + + +class FileHasRegex(Condition): + """ + A condition that some body of text within a file + MUST match a given regular expression. + """ + + def __init__(self, filename, regex) -> None: + self.filename = filename + self.regex = regex + + def contains(self) -> bool: + try: + with open(self.filename, "r") as f: + filetext = f.read() + return self.is_within(filetext) + except Exception: + return False + + def is_within(self, text): + return search(self.regex, text) is not None + + def __str__(self) -> str: + return f"{__class__.__name__} expected to find {self.regex} regex match within {self.filename} file but no match was found" + + @property + def passes(self): + return self.contains() + + +class FileHasNoRegex(FileHasRegex): + """ + A condition that some body of text within a file + MUST NOT match a given regular expression. + """ + + def __str__(self) -> str: + return f"{__class__.__name__} expected to find {self.regex} regex to not match within {self.filename} file but a match was found" + + @property + def passes(self): + return not self.contains() diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 5226356c5..538ed786a 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -78,6 +78,12 @@ def run_single_test(name, test, test_label="", buffer_length=50): info["violated_condition"] = (condition, i, len(conditions)) break + if len(test) == 4: + end_process = Popen(test[3], stdout=PIPE, stderr=PIPE, shell=True) + end_stdout, end_stderr = end_process.communicate() + info["end_stdout"] = end_stdout + info["end_stderr"] = end_stderr + return passed, info @@ -139,7 +145,11 @@ def run_tests(args, tests): n_to_run = 0 selective = True for test_id, test in enumerate(tests.values()): - if len(test) == 3 and test[2] == "local": + # Ensures that test definitions are atleast size 3. + # 'local' variable is stored in 3rd element of the test definitions, + # but an optional 4th element can be provided for an ending command + # to be ran after all checks have been made. + if len(test) >= 3 and test[2] == "local": args.ids.append(test_id + 1) n_to_run += 1 diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 714e22ff5..d23cbd57e 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -1,9 +1,19 @@ -from conditions import HasRegex, HasReturnCode, ProvenanceYAMLFileHasRegex, StepFileExists, StepFileHasRegex +from conditions import ( + FileHasNoRegex, + FileHasRegex, + HasRegex, + HasReturnCode, + PathExists, + ProvenanceYAMLFileHasRegex, + StepFileExists, + StepFileHasRegex, +) from merlin.utils import get_flux_cmd OUTPUT_DIR = "cli_test_studies" +CLEAN_MERLIN_SERVER = "rm -rf appendonly.aof dump.rdb merlin_server/" def define_tests(): @@ -45,18 +55,80 @@ def define_tests(): "local", ), } - server_tests = { - "merlin server init": ("merlin server init", HasRegex(".*successful"), "local"), + server_basic_tests = { + "merlin server init": ( + "merlin server init", + HasRegex(".*successful"), + "local", + CLEAN_MERLIN_SERVER, + ), "merlin server start/stop": ( - "merlin server start; merlin server status; merlin server stop", + """merlin server init; + merlin server start; + merlin server status; + merlin server stop;""", + [ + HasRegex("Server started with PID [0-9]*"), + HasRegex("Merlin server is running"), + HasRegex("Merlin server terminated"), + ], + "local", + CLEAN_MERLIN_SERVER, + ), + "merlin server restart": ( + """merlin server init; + merlin server start; + merlin server restart; + merlin server status; + merlin server stop;""", [ HasRegex("Server started with PID [0-9]*"), HasRegex("Merlin server is running"), HasRegex("Merlin server terminated"), ], "local", + CLEAN_MERLIN_SERVER, + ), + } + server_config_tests = { + "merlin server change config": ( + """merlin server init; + merlin server config -p 8888 -pwd new_password -d ./config_dir -ss 80 -sc 8 -sf new_sf -am always -af new_af.aof; + merlin server start; + merlin server stop;""", + [ + FileHasRegex("merlin_server/redis.conf", "port 8888"), + FileHasRegex("merlin_server/redis.conf", "requirepass new_password"), + FileHasRegex("merlin_server/redis.conf", "dir ./config_dir"), + FileHasRegex("merlin_server/redis.conf", "save 80 8"), + FileHasRegex("merlin_server/redis.conf", "dbfilename new_sf"), + FileHasRegex("merlin_server/redis.conf", "appendfsync always"), + FileHasRegex("merlin_server/redis.conf", 'appendfilename "new_af.aof"'), + PathExists("./config_dir/new_sf"), + PathExists("./config_dir/appendonlydir"), + HasRegex("Server started with PID [0-9]*"), + HasRegex("Merlin server terminated"), + ], + "local", + "rm -rf appendonly.aof dump.rdb merlin_server/ config_dir/", + ), + "merlin server config add/remove user": ( + """merlin server init; + merlin server start; + merlin server config --add-user new_user new_password; + merlin server stop; + scp ./merlin_server/redis.users ./merlin_server/redis.users_new + merlin server start; + merlin server config --remove-user new_user; + merlin server stop; + """, + [ + FileHasRegex("./merlin_server/redis.users_new", "new_user"), + FileHasNoRegex("./merlin_server/redis.users", "new_user"), + ], + "local", + CLEAN_MERLIN_SERVER, ), - "clean merlin server": ("rm -rf appendonly.aof dump.rdb merlin_server/"), } examples_check = { "example list": ( @@ -384,7 +456,8 @@ def define_tests(): all_tests = {} for test_dict in [ basic_checks, - server_tests, + server_basic_tests, + server_config_tests, examples_check, run_workers_echo_tests, wf_format_tests, From a1470af1488ddfde80e50d29ea48836211811fb0 Mon Sep 17 00:00:00 2001 From: Ryan Lee <44886374+ryannova@users.noreply.github.com> Date: Wed, 28 Sep 2022 11:52:58 -0700 Subject: [PATCH 040/126] Addition of new shortcuts in specification file (#375) * Added five shortcuts to the specification definition MERLIN_SAMPLE_VECTOR, MERLIN_SAMPLE_NAMES, MERLIN_SPEC_ORIGINAL_TEMPLATE, MERLIN_SPEC_EXECUTED_RUN, MERLIN_SPEC_ARCHIVED_COPY * Added documentation for the above shortcuts. Co-authored-by: Jim Gaffney --- CHANGELOG.md | 1 + docs/source/merlin_variables.rst | 15 +++++++++++++++ merlin/study/study.py | 23 ++++++++++++++++++++--- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03fcb3558..cf50ccce9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Rename lgtm.yml to .lgtm.yml +- New shortcuts in specification file (sample_vector, sample_names, spec_original_template, spec_executed_run, spec_archived_copy) - Changed "default" user password to be "merlin_password" as default. - Update requirements to require redis 4.3.4 for acl user channel support diff --git a/docs/source/merlin_variables.rst b/docs/source/merlin_variables.rst index b8da52cb6..c19056574 100644 --- a/docs/source/merlin_variables.rst +++ b/docs/source/merlin_variables.rst @@ -83,6 +83,21 @@ Reserved variables ls $path done - ``0/0/0 0/0/1 0/0/2 0/0/3`` + * - ``$(MERLIN_SAMPLE_VECTOR)`` + - Vector of merlin sample values + - ``$(SAMPLE_COLUMN_1) $(SAMPLE_COLUMN_2) ...`` + * - ``$(MERLIN_SAMPLE_NAMES)`` + - Names of merlin sample values + - ``SAMPLE_COLUMN_1 SAMPLE_COLUMN_2 ...`` + * - ``$(MERLIN_SPEC_ORIGINAL_TEMPLATE)`` + - Copy of original yaml file passed to ``merlin run``. + - ``$(MERLIN_INFO)/*.orig.yaml`` + * - ``$(MERLIN_SPEC_EXECUTED_RUN)`` + - Parsed and processed yaml file with command-line variable substitutions included. + - ``$(MERLIN_INFO)/*.partial.yaml`` + * - ``$(MERLIN_SPEC_ARCHIVED_COPY)`` + - Archive version of ``MERLIN_SPEC_EXECUTED_RUN`` with all variables and paths fully resolved. + - ``$(MERLIN_INFO)/*.expanded.yaml`` User variables diff --git a/merlin/study/study.py b/merlin/study/study.py index efa43dd9f..f464cf232 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -107,6 +107,20 @@ def __init__( "MERLIN_SOFT_FAIL": str(int(ReturnCode.SOFT_FAIL)), "MERLIN_HARD_FAIL": str(int(ReturnCode.HARD_FAIL)), "MERLIN_RETRY": str(int(ReturnCode.RETRY)), + # below will be substituted for sample values on execution + "MERLIN_SAMPLE_VECTOR": " ".join( + ["$({})".format(k) for k in self.get_sample_labels(from_spec=self.original_spec)] + ), + "MERLIN_SAMPLE_NAMES": " ".join(self.get_sample_labels(from_spec=self.original_spec)), + "MERLIN_SPEC_ORIGINAL_TEMPLATE": os.path.join( + self.info, self.original_spec.description["name"].replace(" ", "_") + ".orig.yaml" + ), + "MERLIN_SPEC_EXECUTED_RUN": os.path.join( + self.info, self.original_spec.description["name"].replace(" ", "_") + ".partial.yaml" + ), + "MERLIN_SPEC_ARCHIVED_COPY": os.path.join( + self.info, self.original_spec.description["name"].replace(" ", "_") + ".expanded.yaml" + ), } self.pgen_file = pgen_file @@ -182,6 +196,11 @@ def samples(self): return self.load_samples() return [] + def get_sample_labels(self, from_spec): + if from_spec.merlin["samples"]: + return from_spec.merlin["samples"]["column_labels"] + return [] + @property def sample_labels(self): """ @@ -197,9 +216,7 @@ def sample_labels(self): :return: list of labels (e.g. ["X0", "X1"] ) """ - if self.expanded_spec.merlin["samples"]: - return self.expanded_spec.merlin["samples"]["column_labels"] - return [] + return self.get_sample_labels(from_spec=self.expanded_spec) def load_samples(self): """ From f8aaa7aecc47c8200c62f33bebc15a6010c2e267 Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Thu, 29 Sep 2022 16:15:14 -0700 Subject: [PATCH 041/126] Remove emoji from issue templates (#377) * Update bug_report.md remove "buggy" emoji * Update feature_request.md * Update question.md * Update CHANGELOG.md * Update CHANGELOG.md typo fix Co-authored-by: Ryan Lee --- .github/ISSUE_TEMPLATE/bug_report.md | 4 ++-- .github/ISSUE_TEMPLATE/feature_request.md | 4 ++-- .github/ISSUE_TEMPLATE/question.md | 4 ++-- CHANGELOG.md | 1 + 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 5480abee7..0cc50170d 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,5 +1,5 @@ --- -name: "\U0001F41B Bug report" +name: "Bug report" about: Create a report to help us improve title: "[BUG] " labels: bug @@ -7,7 +7,7 @@ assignees: '' --- -## 🐛 Bug Report +## Bug Report **Describe the bug** A clear and concise description of what the bug is. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 33cfac3d0..992cb3211 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,5 +1,5 @@ --- -name: "\U0001F680 Feature request" +name: "Feature request" about: Suggest an idea for Merlin title: "[FEAT] " labels: enhancement @@ -9,7 +9,7 @@ assignees: '' -## 🚀 Feature Request +## Feature Request **What problem is this feature looking to solve?** A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index b20c3af1f..58824b272 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -1,9 +1,9 @@ --- -name: 🤓 General question +name: General question labels: 'question' title: '[Q/A] ' about: Ask, discuss, debate with the Merlin team --- -## 🤓 Question +## Question diff --git a/CHANGELOG.md b/CHANGELOG.md index cf50ccce9..9296e544f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed return values from scripts with main() to fix testing errors. - CI test for CHANGELOG modifcations +- Removed emoji from issue templates that were breaking doc builds ## [1.8.5] ### Added From acf9072fe66b8ed8d63623c0aa4e818ff97f23dc Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Mon, 10 Oct 2022 11:24:11 -0700 Subject: [PATCH 042/126] Update contribute.rst Remove more emoji from docs that are breaking pdf builds --- docs/source/modules/contribute.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/modules/contribute.rst b/docs/source/modules/contribute.rst index 8e8cde7af..9b0239a44 100644 --- a/docs/source/modules/contribute.rst +++ b/docs/source/modules/contribute.rst @@ -17,16 +17,16 @@ Issues Found a bug? Have an idea? Or just want to ask a question? `Create a new issue `_ on GitHub. -Bug Reports 🐛 --------------- +Bug Reports +----------- To report a bug, simply navigate to `Issues `_, click "New Issue", then click "Bug report". Then simply fill out a few fields such as "Describe the bug" and "Expected behavior". Try to fill out every field as it will help us figure out your bug sooner. -Feature Requests 🚀 -------------------- +Feature Requests +---------------- We are still adding new features to merlin. To suggest one, simply navigate to `Issues `_, click "New Issue", then click "Feature request". Then fill out a few fields such as "What problem is this feature looking to solve?" -Questions 🤓 ------------- +Questions +--------- .. note:: Who knows? Your question may already be answered in the :doc:`FAQ<../faq>`. From 1d7bbffa2192d6108855d20b6048ef2fb85661a1 Mon Sep 17 00:00:00 2001 From: Joe Koning Date: Thu, 13 Oct 2022 09:49:56 -0700 Subject: [PATCH 043/126] Update cert_req to cert_regs in the docs. (#379) Co-authored-by: Ryan Lee <44886374+ryannova@users.noreply.github.com> --- CHANGELOG.md | 1 + docs/source/merlin_config.rst | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9296e544f..b56d450dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed return values from scripts with main() to fix testing errors. - CI test for CHANGELOG modifcations +- Fix the cert_req typo in the merlin config docs, it should read cert_reqs - Removed emoji from issue templates that were breaking doc builds ## [1.8.5] diff --git a/docs/source/merlin_config.rst b/docs/source/merlin_config.rst index 7bb43d3c6..599a50413 100644 --- a/docs/source/merlin_config.rst +++ b/docs/source/merlin_config.rst @@ -153,7 +153,7 @@ show below. ca_certs: /var/ssl/myca.pem # This is optional and can be required, optional or none # (required is the default) - cert_req: required + cert_reqs: required @@ -197,7 +197,7 @@ url when using a redis server version 6 or greater with ssl_. ca_certs: /var/ssl/myca.pem # This is optional and can be required, optional or none # (required is the default) - cert_req: required + cert_reqs: required The resulting ``broker_use_ssl`` configuration for a ``rediss`` server is given below. From a178ec2a8267ed61e68d06814a191c794e8aefc0 Mon Sep 17 00:00:00 2001 From: Joe Koning Date: Fri, 28 Oct 2022 10:56:53 -0700 Subject: [PATCH 044/126] Ssl server check fixes (#380) * Add ssl to the Connection object for checking broker and results server acess. * Update CHANGELOG --- CHANGELOG.md | 1 + merlin/display.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b56d450dd..c5afb8a2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - New shortcuts in specification file (sample_vector, sample_names, spec_original_template, spec_executed_run, spec_archived_copy) - Changed "default" user password to be "merlin_password" as default. - Update requirements to require redis 4.3.4 for acl user channel support +- Added ssl to the broker and results backend server checks when "merlin info" is called ### Fixed - Fixed return values from scripts with main() to fix testing errors. diff --git a/merlin/display.py b/merlin/display.py index 1cd9af01e..e0172b6eb 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -88,7 +88,12 @@ def check_server_access(sconf): def _examine_connection(s, sconf, excpts): connect_timeout = 60 try: - conn = Connection(sconf[s]) + ssl_conf = None + if "broker" in s: + ssl_conf = broker.get_ssl_config() + if "results" in s: + ssl_conf = results_backend.get_ssl_config() + conn = Connection(sconf[s], ssl=ssl_conf) conn_check = ConnProcess(target=conn.connect) conn_check.start() counter = 0 From e6ccd07d4971b3e766b4e9c93b58e723ed816d87 Mon Sep 17 00:00:00 2001 From: Ryan Lee <44886374+ryannova@users.noreply.github.com> Date: Fri, 28 Oct 2022 11:09:51 -0700 Subject: [PATCH 045/126] Update documentation in tutorial and merlin server (#378) * Updated installation in instroduction and removed redis requirements * Removed pip headers and added commands for merlin server into installation * Removed additional references to old redis way and update description of merlin server * Remove more emoji from docs that are breaking pdf builds * Updated CHANGELOG to reflect changes to documentation Co-authored-by: Luc Peterson --- CHANGELOG.md | 1 + docs/source/_static/custom.css | 4 + docs/source/conf.py | 2 +- docs/source/index.rst | 1 - docs/source/merlin_commands.rst | 32 +++ .../modules/installation/installation.rst | 225 +++++++----------- docs/source/server/commands.rst | 9 +- 7 files changed, 124 insertions(+), 150 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5afb8a2a..dbae21b54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - New shortcuts in specification file (sample_vector, sample_names, spec_original_template, spec_executed_run, spec_archived_copy) - Changed "default" user password to be "merlin_password" as default. - Update requirements to require redis 4.3.4 for acl user channel support +- Updated tutorial documentation to use merlin server over manual redis installation. - Added ssl to the broker and results backend server checks when "merlin info" is called ### Fixed diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index 367d8e1f2..72d82b073 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -18,3 +18,7 @@ div.highlight .copybtn:hover { div.highlight { position: relative; } +div.sphinxsidebar { + max-height: 100%; + overflow-y: auto; +} diff --git a/docs/source/conf.py b/docs/source/conf.py index d5cef3d64..2905856e8 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -149,7 +149,7 @@ # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'Merlin.tex', u'Merlin Documentation', - u'MLSI', 'manual'), + u'The Merlin Development Team', 'manual'), ] diff --git a/docs/source/index.rst b/docs/source/index.rst index 72469f968..3776466d3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -81,4 +81,3 @@ Need help? `merlin@llnl.gov `_ spack merlin_developer docker - diff --git a/docs/source/merlin_commands.rst b/docs/source/merlin_commands.rst index 0a1a9f2bc..c15adb6d6 100644 --- a/docs/source/merlin_commands.rst +++ b/docs/source/merlin_commands.rst @@ -406,4 +406,36 @@ The only currently available option for ``--task_server`` is celery, which is th only one might get the signal. In this case, you can send it again. +Hosting Local Server (``merlin server``) +---------------------------------------- + +To create a local server for merlin to connect to. Merlin server creates and configures a server on the current directory. +This allows multiple instances of merlin server to exist for different studies or uses. + +The ``init`` subcommand initalizes a new instance of merlin server. + +The ``status`` subcommand checks to the status of the merlin server. + +The ``start`` subcommand starts the merlin server. + +The ``stop`` subcommand stops the merlin server. + +The ``restart`` subcommand performs stop command followed by a start command on the merlin server. + +The ``config`` subcommand edits configurations for the merlin server. There are multiple flags to allow for different configurations. + +- The ``-ip IPADDRESS, --ipaddress IPADDRESS`` option set the binded IP address for merlin server. +- The ``-p PORT, --port PORT`` option set the binded port for merlin server. +- The ``-pwd PASSWORD, --password PASSWORD`` option set the password file for merlin server. +- The ``--add-user USER PASSWORD`` option add a new user for merlin server. +- The ``--remove-user REMOVE_USER`` option remove an exisiting user from merlin server. +- The ``-d DIRECTORY, --directory DIRECTORY`` option set the working directory for merlin server. +- The ``-ss SNAPSHOT_SECONDS, --snapshot-seconds SNAPSHOT_SECONDS`` option set the number of seconds before each snapshot. +- The ``-sc SNAPSHOT_CHANGES, --snapshot-changes SNAPSHOT_CHANGES`` option set the number of database changes before each snapshot. +- The ``-sf SNAPSHOT_FILE, --snapshot-file SNAPSHOT_FILE`` option set the name of snapshots. +- The ``-am APPEND_MODE, --append-mode APPEND_MODE`` option set the appendonly mode. Options are always, everysec, no. +- The ``-af APPEND_FILE, --append-file APPEND_FILE`` option set the filename for server append/change file. + +More information can be found on :doc:`Merlin Server <./merlin_server>` + diff --git a/docs/source/modules/installation/installation.rst b/docs/source/modules/installation/installation.rst index 2eb1ac95d..c4d58192b 100644 --- a/docs/source/modules/installation/installation.rst +++ b/docs/source/modules/installation/installation.rst @@ -6,7 +6,7 @@ Installation * python3 >= python3.6 * pip3 * wget - * build tools (make, C/C++ compiler for local-redis) + * build tools (make, C/C++ compiler) * docker (required for :doc:`Module 4: Run a Real Simulation<../run_simulation/run_simulation>`) * file editor for docker config file editing @@ -17,9 +17,7 @@ Installation .. admonition:: You will learn * How to install merlin in a virtual environment using pip. - * How to install a local redis server. - * How to install merlin using docker (optional). - * How to start the docker containers, including redis (optional). + * How to install a container platform eg. singularity, docker, or podman. * How to configure merlin. * How to test/verify the installation. @@ -27,27 +25,21 @@ Installation :local: This section details the steps necessary to install merlin and its dependencies. -Merlin will then be configured and this configuration checked to ensure a proper installation. +Merlin will then be configured for the local machine and the configuration +will be checked to ensure a proper installation. Installing merlin ----------------- -A merlin installation is required for the subsequent modules of this tutorial. You can choose between the pip method or the docker method. Choose one or the other but -do not use both unless you are familiar with redis servers run locally and through docker. -**The pip method is recommended.** +A merlin installation is required for the subsequent modules of this tutorial. -Once merlin is installed, it requires servers to operate. -The pip section will inform you how to setup a -local redis server to use in merlin. An alternative method for setting up a -redis server can be found in the docker section. Only setup one redis server either -local-redis or docker-redis. -Your computer/organization may already have a redis server available, please check +Once merlin is installed, it requires servers to operate. While you are able to host your own servers, +we will use merlin's containerized servers in this tutorial. However, if you prefer to host your own servers +you can host a redis server that is accessible to your current machine. +Your computer/organization may already have a redis server available you can use, please check with your local system administrator. -Pip (recommended) -+++++++++++++++++ - Create a virtualenv using python3 to install merlin. .. code-block:: bash @@ -85,168 +77,111 @@ but leave the virtualenv activated for the subsequent steps. deactivate -redis local server -^^^^^^^^^^^^^^^^^^ +redis server +++++++++++++ A redis server is required for the celery results backend server, this same server -can also be used for the celery broker. This method will be called local-redis. +can also be used for the celery broker. We will be using merlin's containerized server +however we will need to download one of the supported container platforms avaliable. For +the purpose of this tutorial we will be using singularity. .. code-block:: bash + # Update and install singularity dependencies + apt-get update && apt-get install -y \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config + + # Download dependency go + wget https://go.dev/dl/go1.18.1.linux-amd64.tar.gz + + # Extract go into local + tar -C /usr/local -xzf go1.18.1.linux-amd64.tar.gz + + # Remove go tar file + rm go1.18.1.linux-amd64.tar.gz + + # Update PATH to include go + export PATH=$PATH:/usr/local/go/bin + + # Download singularity + wget https://github.com/sylabs/singularity/releases/download/v3.9.9/singularity-ce-3.9.9.tar.gz + + # Extract singularity + tar -xzf singularity-ce-3.9.9.tar.gz + + # Configure and install singularity + cd singularity-ce-3.9.9 + ./mconfig && \ + make -C ./builddir && \ + sudo make -C ./builddir install - # Download redis - wget http://download.redis.io/releases/redis-6.0.5.tar.gz - - # Untar - tar xvf redis*.tar.gz - - # cd into redis dir - cd redis*/ - - # make redis - make - - # make test (~3.5 minutes) - make test - - -The redis server is started by calling the ``redis-server`` command located in -the src directory. -This should be run in a separate terminal in the top-level source -directory so the output can be examined. -The redis server will use the default ``redis.conf`` file in the top-level -redis directory. - -.. code:: bash - - # run redis with default config, server is at localhost port 6379 - ./src/redis-server & - -You can shutdown the local-redis server by using the ``redis-cli shutdown`` command -when you are done with the tutorial. +Configuring merlin +------------------ +Merlin requires a configuration script for the celery interface. +Run this configuration method to create the ``app.yaml`` +configuration file. .. code-block:: bash - #cd to redis directory - cd /redis*/ - ./src/redis-cli shutdown - - -Docker -++++++ - -Merlin and the servers required by merlin are all available as docker containers on dockerhub. Do not use this method if you have already set up a virtualenv through -the pip installation method. - -.. note:: - - When using the docker method the celery workers will run inside the - merlin container. This - means that any workflow tools that are also from docker containers must - be installed in, or - otherwise made available to, the merlin container. - + merlin config --broker redis -To run a merlin docker container with a docker redis server, cut -and paste the commands below into a new file called ``docker-compose.yml``. -This file can be placed anywhere in your filesystem but you may want to put it in -a directory ``merlin_docker_redis``. +The ``merlin config`` command above will create a file called ``app.yaml`` +in the ``~/.merlin`` directory. +If you are running a redis server locally then you are all set, look in the ``~/.merlin/app.yaml`` file +to see the configuration, it should look like the configuration below. -.. literalinclude:: ./docker-compose.yml +.. literalinclude:: ./app_local_redis.yaml :language: yaml -This file can then be run with the ``docker-compose`` command in same directory -as the ``docker-compose.yml`` file. - -.. code-block:: bash - - docker-compose up -d - -The ``volume`` option in the ``docker-compose.yml`` file -will link the local ``$HOME/merlinu`` directory to the ``/home/merlinu`` -directory in the container. - -Some aliases can be defined for convenience. -.. code-block:: bash +.. _Verifying installation: - # define some aliases for the merlin and celery commands (assuming Bourne shell) - alias merlin="docker exec my-merlin merlin" - alias celery="docker exec my-merlin celery" - alias python3="docker exec my-merlin python3" +Checking/Verifying installation +------------------------------- -When you are done with the containers you can stop them using ``docker-compose down``. -We will be using the containers in the subsequent modules so leave them running. +First launch the merlin server containers by using the ``merlin server`` commands .. code-block:: bash - docker-compose down + merlin server init + merlin server start -Any required python modules can be installed in the running ``my-merlin`` container -through ``docker exec``. When using docker-compose, these changes will persist -if you stop the containers with ``docker-compose down`` and restart them with -``docker-compose up -d``. +A subdirectory called ``merlin_server/`` will have been created in the current run directory. +This contains all of the proper configuration for the server containers merlin creates. +Configuration can be done through the ``merlin server config`` command, however users have +the flexibility to edit the files directly in the directory. Additionally an preconfigured ``app.yaml`` +file has been created in the ``merlin_server/`` subdirectory to utilize the merlin server +containers . To use it locally simply copy it to the run directory with a cp command. .. code-block:: bash - docker exec my-merlin pip3 install pandas faker - -Configuring merlin ------------------- - -Merlin configuration is slightly different between the pip and docker methods. -The fundamental differences include the app.yaml file location and the server name. + cp ./merlin_server/app.yaml . -Merlin requires a configuration script for the celery interface and optional -passwords for the redis server and encryption. Run this configuration method -to create the ``app.yaml`` configuration file. +You can also make this server container your main server configuration by replacing the one located in your home +directory. Make sure you make back-ups of your current app.yaml file in case you want to use your previous +configurations. Note: since merlin servers are created locally on your run directory you are allowed to create +multiple instances of merlin server with their unique configurations for different studies. Simply create different +directories for each study and run ``merlin server init`` in each directory to create an instance for each. .. code-block:: bash - merlin config --broker redis - -Pip -+++ - -The ``merlin config`` command above will create a file called ``app.yaml`` -in the ``~/.merlin`` directory. -If you are using local-redis then you are all set, look in the ``~/.merlin/app.yaml`` file -to see the configuration, it should look like the configuration below. - -.. literalinclude:: ./app_local_redis.yaml - :language: yaml - -Docker -++++++ - -If you are using the docker merlin with docker-redis server then the -``~/merlinu/.merlin/app.yaml`` will be created by the ``merlin config`` -command above. -This file must be edited to -add the server from the redis docker container my-redis. Change the ``server: localhost``, in both the -broker and backend config definitions, to ``server: my-redis``, the port will remain the same. - -.. note:: - You can use the docker redis server, instead of the local-redis server, - with the virtualenv installed merlin by using the local-redis - ``app.yaml`` file above. - -.. literalinclude:: ./app_docker_redis.yaml - :language: yaml - -.. _Verifying installation: - -Checking/Verifying installation -------------------------------- + mv ~/.merlin/app.yaml ~/.merlin/app.yaml.bak + cp ./merlin_server/app.yaml ~/.merlin/ The ``merlin info`` command will check that the configuration file is installed correctly, display the server configuration strings, and check server -access. This command works for both the pip and docker installed merlin. +access. .. code-block:: bash merlin info -If everything is set up correctly, you should see (assuming local-redis servers): +If everything is set up correctly, you should see: .. code-block:: bash diff --git a/docs/source/server/commands.rst b/docs/source/server/commands.rst index af3d2e419..dd8ca1b02 100644 --- a/docs/source/server/commands.rst +++ b/docs/source/server/commands.rst @@ -40,16 +40,18 @@ Stopping an exisiting Merlin Server (``merlin server stop``) Stop any exisiting container being managed and monitored by merlin server. Restarting a Merlin Server instance (``merlin server restart``) ------------------------------------------------------------- +--------------------------------------------------------------- Restarting an existing container that is being managed and monitored by merlin server. Configurating Merlin Server instance (``merlin server config``) ------------------------------------------------------------- +--------------------------------------------------------------- Place holder for information regarding merlin server config command Possible Flags -.. code:: none + +.. code-block:: none + -ip IPADDRESS, --ipaddress IPADDRESS Set the binded IP address for the merlin server container. (default: None) @@ -82,3 +84,4 @@ Possible Flags -af APPEND_FILE, --append-file APPEND_FILE Set append only filename for merlin server container. (default: None) + From 9947e57cb767dd807c93c3e55988eead798da9ce Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Fri, 28 Oct 2022 13:45:59 -0700 Subject: [PATCH 046/126] Update MANIFEST.in (#381) * Update MANIFEST.in Add .temp to examples in MANIFEST, so that they get bundled with pypi releases * Update CHANGELOG.md --- CHANGELOG.md | 1 + MANIFEST.in | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbae21b54..9bf86fc12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - CI test for CHANGELOG modifcations - Fix the cert_req typo in the merlin config docs, it should read cert_reqs - Removed emoji from issue templates that were breaking doc builds +- Including .temp template files in MANIFEST ## [1.8.5] ### Added diff --git a/MANIFEST.in b/MANIFEST.in index 93fa2f74e..cefbd23a5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ recursive-include merlin/data *.yaml *.py recursive-include merlin/server *.yaml *.py -recursive-include merlin/examples *.py *.yaml *.c *.json *.sbatch *.bsub *.txt +recursive-include merlin/examples *.py *.yaml *.c *.json *.sbatch *.bsub *.txt *.temp include requirements.txt include requirements/* From 92f5ba84ca210286f90c262e12190c16f3285c56 Mon Sep 17 00:00:00 2001 From: Ryan Lee <44886374+ryannova@users.noreply.github.com> Date: Fri, 28 Oct 2022 13:49:59 -0700 Subject: [PATCH 047/126] Add support for non-merlin blocks in specification file (#376) * Adding support for "user" block in _dict_to_string method * Updated CHANGELOG * Updated Merlin Spec docs * Added user block in feature_demo.yaml example Co-authored-by: Jim Gaffney --- CHANGELOG.md | 1 + docs/source/merlin_specification.rst | 54 ++++++++++++++++++- .../workflows/feature_demo/feature_demo.yaml | 34 ++++++++---- merlin/spec/specification.py | 19 ++++++- 4 files changed, 94 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bf86fc12..1eeb104bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 e.g. flux_exec: flux exec -r "0-1" to run celery workers only on ranks 0 and 1 of a multi-rank allocation - Added additional argument in test definitions to allow for a "cleanup" command +- Capability for non-user block in yaml ### Changed - Rename lgtm.yml to .lgtm.yml diff --git a/docs/source/merlin_specification.rst b/docs/source/merlin_specification.rst index d01f6f83d..29c0dff0c 100644 --- a/docs/source/merlin_specification.rst +++ b/docs/source/merlin_specification.rst @@ -229,7 +229,7 @@ see :doc:`./merlin_variables`. #################################### # Merlin Block (Required) #################################### - # The merlin specific block will add any required configuration to + # The merlin specific block will add any configuration to # the DAG created by the study description. # including task server config, data management and sample definitions. # @@ -277,6 +277,56 @@ see :doc:`./merlin_variables`. batch: type: local machines: [host3] + #################################### + # User Block (Optional) + #################################### + # The user block allows other variables in the workflow file to be propagated + # through to the workflow (including in variables .partial.yaml and .expanded.yaml). + # User block uses yaml anchors, which defines a chunk of configuration and use + # their alias to refer to that specific chunk of configuration elsewhere. + ####################################################################### + user: + study: + run: + hello: &hello_run + cmd: | + python3 $(HELLO) -outfile hello_world_output_$(MERLIN_SAMPLE_ID).json $(X0) $(X1) $(X2) + max_retries: 1 + collect: &collect_run + cmd: | + echo $(MERLIN_GLOB_PATH) + echo $(hello.workspace) + ls $(hello.workspace)/X2.$(X2)/$(MERLIN_GLOB_PATH)/hello_world_output_*.json > files_to_collect.txt + spellbook collect -outfile results.json -instring "$(cat files_to_collect.txt)" + translate: &translate_run + cmd: spellbook translate -input $(collect.workspace)/results.json -output results.npz -schema $(FEATURES) + learn: &learn_run + cmd: spellbook learn -infile $(translate.workspace)/results.npz + make_samples: &make_samples_run + cmd: spellbook make-samples -n $(N_NEW) -sample_type grid -outfile grid_$(N_NEW).npy + predict: &predict_run + cmd: spellbook predict -infile $(make_new_samples.workspace)/grid_$(N_NEW).npy -outfile prediction_$(N_NEW).npy -reg $(learn.workspace)/random_forest_reg.pkl + verify: &verify_run + cmd: | + if [[ -f $(learn.workspace)/random_forest_reg.pkl && -f $(predict.workspace)/prediction_$(N_NEW).npy ]] + then + touch FINISHED + exit $(MERLIN_SUCCESS) + else + exit $(MERLIN_SOFT_FAIL) + fi + python3: + run: &python3_run + cmd: | + print("OMG is this in python?") + print("Variable X2 is $(X2)") + shell: /usr/bin/env python3 + python2: + run: &python2_run + cmd: | + print "OMG is this in python2? Change is bad." + print "Variable X2 is $(X2)" + shell: /usr/bin/env python2 ################################################### # Sample definitions @@ -292,4 +342,4 @@ see :doc:`./merlin_variables`. generate: cmd: | python $(SPECROOT)/make_samples.py -dims 2 -n 10 -outfile=$(INPUT_PATH)/samples.npy "[(1.3, 1.3, 'linear'), (3.3, 3.3, 'linear')]" - level_max_dirs: 25 + level_max_dirs: 25 \ No newline at end of file diff --git a/merlin/examples/workflows/feature_demo/feature_demo.yaml b/merlin/examples/workflows/feature_demo/feature_demo.yaml index b4bc1ca46..b07107e7b 100644 --- a/merlin/examples/workflows/feature_demo/feature_demo.yaml +++ b/merlin/examples/workflows/feature_demo/feature_demo.yaml @@ -17,15 +17,33 @@ env: HELLO: $(SCRIPTS)/hello_world.py FEATURES: $(SCRIPTS)/features.json +user: + study: + run: + hello: &hello_run + cmd: | + python3 $(HELLO) -outfile hello_world_output_$(MERLIN_SAMPLE_ID).json $(X0) $(X1) $(X2) + max_retries: 1 + python3: + run: &python3_run + cmd: | + print("OMG is this in python?") + print("Variable X2 is $(X2)") + shell: /usr/bin/env python3 + python2: + run: &python2_run + cmd: | + print "OMG is this in python2? Change is bad." + print "Variable X2 is $(X2)" + shell: /usr/bin/env python2 + study: - name: hello description: | process a sample with hello world run: - cmd: | - python3 $(HELLO) -outfile hello_world_output_$(MERLIN_SAMPLE_ID).json $(X0) $(X1) $(X2) + <<: *hello_run task_queue: hello_queue - max_retries: 1 - name: collect description: | @@ -89,20 +107,14 @@ study: description: | do something in python run: - cmd: | - print("OMG is this in python?") - print("Variable X2 is $(X2)") - shell: /usr/bin/env python3 + <<: *python3_run task_queue: pyth3_q - name: python2_hello description: | do something in python2, because change is bad run: - cmd: | - print "OMG is this in python2? Change is bad." - print "Variable X2 is $(X2)" - shell: /usr/bin/env python2 + <<: *python2_run task_queue: pyth2_hello global.parameters: diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 6bb612cb4..466e9f50e 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -82,6 +82,7 @@ def yaml_sections(self): "study": self.study, "global.parameters": self.globals, "merlin": self.merlin, + "user": self.user, } @property @@ -97,6 +98,7 @@ def sections(self): "study": self.study, "globals": self.globals, "merlin": self.merlin, + "user": self.user, } @classmethod @@ -104,6 +106,8 @@ def load_specification(cls, filepath, suppress_warning=True): spec = super(MerlinSpec, cls).load_specification(filepath) with open(filepath, "r") as f: spec.merlin = MerlinSpec.load_merlin_block(f) + with open(filepath, "r") as f: + spec.user = MerlinSpec.load_user_block(f) spec.specroot = os.path.dirname(spec.path) spec.process_spec_defaults() if not suppress_warning: @@ -114,6 +118,7 @@ def load_specification(cls, filepath, suppress_warning=True): def load_spec_from_string(cls, string): spec = super(MerlinSpec, cls).load_specification_from_stream(StringIO(string)) spec.merlin = MerlinSpec.load_merlin_block(StringIO(string)) + spec.user = MerlinSpec.load_user_block(StringIO(string)) spec.specroot = None spec.process_spec_defaults() return spec @@ -132,6 +137,14 @@ def load_merlin_block(stream): LOG.warning(warning_msg) return merlin_block + @staticmethod + def load_user_block(stream): + try: + user_block = yaml.safe_load(stream)["user"] + except KeyError: + user_block = {} + return user_block + def process_spec_defaults(self): for name, section in self.sections.items(): if section is None: @@ -161,6 +174,8 @@ def process_spec_defaults(self): if self.merlin["samples"] is not None: MerlinSpec.fill_missing_defaults(self.merlin["samples"], defaults.SAMPLES) + # no defaults for user block + @staticmethod def fill_missing_defaults(object_to_update, default_dict): """ @@ -212,6 +227,8 @@ def warn_unrecognized_keys(self): if self.merlin["samples"]: MerlinSpec.check_section("merlin.samples", self.merlin["samples"], all_keys.SAMPLES) + # user block is not checked + @staticmethod def check_section(section_name, section, all_keys): diff = set(section.keys()).difference(all_keys) @@ -268,7 +285,7 @@ def _process_dict_or_list(self, obj, string, key_stack, lvl, tab): list_offset = 2 * " " if isinstance(obj, list): n = len(obj) - use_hyphens = key_stack[-1] in ["paths", "sources", "git", "study"] + use_hyphens = key_stack[-1] in ["paths", "sources", "git", "study"] or key_stack[0] in ["user"] if not use_hyphens: string += "[" else: From 69bace6fac37b8dbeae1c6cd884ee3c0fb15dc96 Mon Sep 17 00:00:00 2001 From: Ryan Lee <44886374+ryannova@users.noreply.github.com> Date: Fri, 18 Nov 2022 12:09:43 -0800 Subject: [PATCH 048/126] Update Merlin Server (#385) * Added condition for fatal error from redis server * Update default value for config_dir * Updated fix-style target in Makefile to be consistent with other style related targets * Update default password to use generated password * Updated run user to be default rather than created user * Updated singularity command to specify configuration directory as home directory to solve unaccessible directory issue * Update merlin to use app.yaml configuration rather than its own configuration file --- CHANGELOG.md | 3 +- Makefile | 12 ++++---- merlin/server/merlin_server.yaml | 4 +-- merlin/server/server_commands.py | 13 +++++++- merlin/server/server_config.py | 52 +++++++++++++++++++++----------- merlin/server/server_util.py | 16 +++++++--- merlin/server/singularity.yaml | 2 +- 7 files changed, 68 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1eeb104bc..6fb8fe0fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,8 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Rename lgtm.yml to .lgtm.yml - New shortcuts in specification file (sample_vector, sample_names, spec_original_template, spec_executed_run, spec_archived_copy) -- Changed "default" user password to be "merlin_password" as default. +- Changed "default" user password to be randomly generated by default. - Update requirements to require redis 4.3.4 for acl user channel support +- Merlin server uses app.yaml as its main configuration - Updated tutorial documentation to use merlin server over manual redis installation. - Added ssl to the broker and results backend server checks when "merlin info" is called diff --git a/Makefile b/Makefile index d1e441a87..4e3735df3 100644 --- a/Makefile +++ b/Makefile @@ -164,12 +164,12 @@ checks: check-style check-camel-case # automatically make python files pep 8-compliant fix-style: . $(VENV)/bin/activate; \ - isort --line-length $(MAX_LINE_LENGTH) $(MRLN); \ - isort --line-length $(MAX_LINE_LENGTH) $(TEST); \ - isort --line-length $(MAX_LINE_LENGTH) *.py; \ - black --target-version py36 -l $(MAX_LINE_LENGTH) $(MRLN); \ - black --target-version py36 -l $(MAX_LINE_LENGTH) $(TEST); \ - black --target-version py36 -l $(MAX_LINE_LENGTH) *.py; \ + $(PYTHON) -m isort --line-length $(MAX_LINE_LENGTH) $(MRLN); \ + $(PYTHON) -m isort --line-length $(MAX_LINE_LENGTH) $(TEST); \ + $(PYTHON) -m isort --line-length $(MAX_LINE_LENGTH) *.py; \ + $(PYTHON) -m black --target-version py36 -l $(MAX_LINE_LENGTH) $(MRLN); \ + $(PYTHON) -m black --target-version py36 -l $(MAX_LINE_LENGTH) $(TEST); \ + $(PYTHON) -m black --target-version py36 -l $(MAX_LINE_LENGTH) *.py; \ # Increment the Merlin version. USE ONLY ON DEVELOP BEFORE MERGING TO MASTER. diff --git a/merlin/server/merlin_server.yaml b/merlin/server/merlin_server.yaml index 5f9b25367..01b3c7ddb 100644 --- a/merlin/server/merlin_server.yaml +++ b/merlin/server/merlin_server.yaml @@ -9,8 +9,8 @@ container: url: docker://redis # The config file config: redis.conf - # Subdirectory name to store configurations Default: merlin_server/ - config_dir: merlin_server/ + # Directory name to store configurations Default: ./merlin_server/ + config_dir: ./merlin_server/ # Process file containing information regarding the redis process pfile: merlin_server.pf # Password file to be used for accessing container diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index a487f4ca9..4c102c23e 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -46,6 +46,9 @@ def config_server(args: Namespace) -> None: based on the input passed in by the user. """ server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False redis_config = RedisConfig(server_config.container.get_config_path()) redis_config.set_ip_address(args.ipaddress) @@ -79,6 +82,9 @@ def config_server(args: Namespace) -> None: LOG.info("Add changes to config file and exisiting containers.") server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False # Read the user from the list of avaliable users redis_users = RedisUsers(server_config.container.get_user_file_path()) @@ -160,7 +166,12 @@ def start_server() -> bool: process = subprocess.Popen( server_config.container_format.get_run_command() .strip("\\") - .format(command=server_config.container_format.get_command(), image=image_path, config=config_path) + .format( + command=server_config.container_format.get_command(), + home_dir=server_config.container.get_config_dir(), + image=image_path, + config=config_path, + ) .split(), start_new_session=True, close_fds=True, diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 2b18b1e27..03142e711 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -5,6 +5,7 @@ import shutil import string import subprocess +from io import BufferedReader from typing import Tuple import yaml @@ -14,6 +15,7 @@ MERLIN_CONFIG_DIR, MERLIN_SERVER_CONFIG, MERLIN_SERVER_SUBDIR, + AppYaml, RedisConfig, RedisUsers, ServerConfig, @@ -23,11 +25,12 @@ LOG = logging.getLogger("merlin") # Default values for configuration -CONFIG_DIR = "./merlin_server/" +CONFIG_DIR = os.path.abspath("./merlin_server/") IMAGE_NAME = "redis_latest.sif" PROCESS_FILE = "merlin_server.pf" CONFIG_FILE = "redis.conf" REDIS_URL = "docker://redis" +LOCAL_APP_YAML = "./app.yaml" PASSWORD_LENGTH = 256 @@ -68,7 +71,7 @@ def generate_password(length, pass_command: str = None) -> str: return "".join(password) -def parse_redis_output(redis_stdout) -> Tuple[bool, str]: +def parse_redis_output(redis_stdout: BufferedReader) -> Tuple[bool, str]: """ Parse the redis output for a the redis container. It will get all the necessary information from the output and returns a dictionary of those values. @@ -79,7 +82,8 @@ def parse_redis_output(redis_stdout) -> Tuple[bool, str]: return False, "None passed as redis output" server_init = False redis_config = {} - for line in redis_stdout: + line = redis_stdout.readline() + while line != "" or line is not None: if not server_init: values = [ln for ln in line.split() if b"=" in ln] for val in values: @@ -89,8 +93,9 @@ def parse_redis_output(redis_stdout) -> Tuple[bool, str]: server_init = True if b"Ready to accept connections" in line: return True, redis_config - if b"aborting" in line: + if b"aborting" in line or b"Fatal error" in line: return False, line.decode("utf-8") + line = redis_stdout.readline() def create_server_config() -> bool: @@ -116,7 +121,6 @@ def create_server_config() -> bool: return False files = [i + ".yaml" for i in CONTAINER_TYPES] - files.append(MERLIN_SERVER_CONFIG) for file in files: file_path = os.path.join(config_dir, file) if os.path.exists(file_path): @@ -129,7 +133,18 @@ def create_server_config() -> bool: LOG.error(f"Destination location {config_dir} is not writable.") return False + # Load Merlin Server Configuration and apply it to app.yaml + with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), MERLIN_SERVER_CONFIG)) as f: + main_server_config = yaml.load(f, yaml.Loader) + filename = LOCAL_APP_YAML if os.path.exists(LOCAL_APP_YAML) else AppYaml.default_filename + merlin_app_yaml = AppYaml(filename) + merlin_app_yaml.update_data(main_server_config) + merlin_app_yaml.write(filename) + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False if not os.path.exists(server_config.container.get_config_dir()): LOG.info("Creating merlin server directory.") @@ -144,6 +159,9 @@ def config_merlin_server(): """ server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False pass_file = server_config.container.get_pass_file_path() if os.path.exists(pass_file): @@ -165,9 +183,11 @@ def config_merlin_server(): else: redis_users = RedisUsers(user_file) redis_config = RedisConfig(server_config.container.get_config_path()) - redis_users.add_user(user="default", password=redis_config.get_password()) + redis_config.set_password(server_config.container.get_container_password()) + redis_users.add_user(user="default", password=server_config.container.get_container_password()) redis_users.add_user(user=os.environ.get("USER"), password=server_config.container.get_container_password()) redis_users.write() + redis_config.write() LOG.info("User {} created in user file for merlin server container".format(os.environ.get("USER"))) @@ -183,15 +203,11 @@ def pull_server_config() -> ServerConfig: format_needed_keys = ["command", "run_command", "stop_command", "pull_command"] process_needed_keys = ["status", "kill"] - config_dir = os.path.join(MERLIN_CONFIG_DIR, MERLIN_SERVER_SUBDIR) - config_path = os.path.join(config_dir, MERLIN_SERVER_CONFIG) - if not os.path.exists(config_path): - LOG.error(f"Unable to pull merlin server configuration from {config_path}") - return None + merlin_app_yaml = AppYaml(LOCAL_APP_YAML) + server_config = merlin_app_yaml.get_data() + return_data.update(server_config) - with open(config_path, "r") as cf: - server_config = yaml.load(cf, yaml.Loader) - return_data.update(server_config) + config_dir = os.path.join(MERLIN_CONFIG_DIR, MERLIN_SERVER_SUBDIR) if "container" in server_config: if "format" in server_config["container"]: @@ -204,20 +220,20 @@ def pull_server_config() -> ServerConfig: return None return_data.update(format_data) else: - LOG.error(f'Unable to find "format" in {MERLIN_SERVER_CONFIG}') + LOG.error(f'Unable to find "format" in {merlin_app_yaml.default_filename}') return None else: - LOG.error(f'Unable to find "container" object in {MERLIN_SERVER_CONFIG}') + LOG.error(f'Unable to find "container" object in {merlin_app_yaml.default_filename}') return None # Checking for process values that are needed for main functions and defaults if "process" not in server_config: - LOG.error("Process config not found in " + MERLIN_SERVER_CONFIG) + LOG.error(f"Process config not found in {merlin_app_yaml.default_filename}") return None for key in process_needed_keys: if key not in server_config["process"]: - LOG.error(f'Process necessary "{key}" command configuration not found in {MERLIN_SERVER_CONFIG}') + LOG.error(f'Process necessary "{key}" command configuration not found in {merlin_app_yaml.default_filename}') return None return ServerConfig(return_data) diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index d9698efe3..666a388f0 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -63,7 +63,7 @@ class ContainerConfig: IMAGE_NAME = "redis_latest.sif" REDIS_URL = "docker://redis" CONFIG_FILE = "redis.conf" - CONFIG_DIR = "./merlin_server/" + CONFIG_DIR = os.path.abspath("./merlin_server/") PROCESS_FILE = "merlin_server.pf" PASSWORD_FILE = "redis.pass" USERS_FILE = "redis.users" @@ -84,7 +84,7 @@ def __init__(self, data: dict) -> None: self.image = data["image"] if "image" in data else self.IMAGE_NAME self.url = data["url"] if "url" in data else self.REDIS_URL self.config = data["config"] if "config" in data else self.CONFIG_FILE - self.config_dir = data["config_dir"] if "config_dir" in data else self.CONFIG_DIR + self.config_dir = os.path.abspath(data["config_dir"]) if "config_dir" in data else self.CONFIG_DIR self.pfile = data["pfile"] if "pfile" in data else self.PROCESS_FILE self.pass_file = data["pass_file"] if "pass_file" in data else self.PASSWORD_FILE self.user_file = data["user_file"] if "user_file" in data else self.USERS_FILE @@ -323,7 +323,7 @@ def set_password(self, password: str) -> bool: if password is None: return False self.set_config_value("requirepass", password) - LOG.info(f"Password file set to {password}") + LOG.info("New password set") return True def get_password(self) -> str: @@ -552,17 +552,23 @@ def apply_server_config(self, server_config: ServerConfig): rc = RedisConfig(server_config.container.get_config_path()) self.data[self.broker_name]["name"] = server_config.container.get_image_type() - self.data[self.broker_name]["username"] = os.environ.get("USER") + self.data[self.broker_name]["username"] = "default" self.data[self.broker_name]["password"] = server_config.container.get_pass_file_path() self.data[self.broker_name]["server"] = rc.get_ip_address() self.data[self.broker_name]["port"] = rc.get_port() self.data[self.results_name]["name"] = server_config.container.get_image_type() - self.data[self.results_name]["username"] = os.environ.get("USER") + self.data[self.results_name]["username"] = "default" self.data[self.results_name]["password"] = server_config.container.get_pass_file_path() self.data[self.results_name]["server"] = rc.get_ip_address() self.data[self.results_name]["port"] = rc.get_port() + def update_data(self, new_data: dict): + self.data.update(new_data) + + def get_data(self): + return self.data + def read(self, filename: str = default_filename): self.data = merlin.utils.load_yaml(filename) diff --git a/merlin/server/singularity.yaml b/merlin/server/singularity.yaml index 277800f4f..d2b34874e 100644 --- a/merlin/server/singularity.yaml +++ b/merlin/server/singularity.yaml @@ -1,6 +1,6 @@ singularity: command: singularity # init_command: \{command} .. (optional or default) - run_command: \{command} run {image} {config} + run_command: \{command} run -H {home_dir} {image} {config} stop_command: kill # \{command} (optional or kill default) pull_command: \{command} pull {image} {url} From c2d2c2b3d8a5a74c51150f71cc6ccc74ba77dca0 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Mon, 21 Nov 2022 14:17:02 -0800 Subject: [PATCH 049/126] Docs/install changes (#383) Many modifications to documentation, including installation instructions and formatting fixes. --- .readthedocs.yaml | 13 ++ CHANGELOG.md | 6 + docs/Makefile | 3 + docs/requirements.txt | 56 +++++ docs/source/_static/custom.css | 15 ++ docs/source/_static/theme_overrides.css | 14 -- docs/source/conf.py | 10 +- docs/source/faq.rst | 47 +++-- docs/source/getting_started.rst | 62 +++++- docs/source/merlin_commands.rst | 2 +- docs/source/merlin_developer.rst | 6 +- docs/source/merlin_server.rst | 4 +- docs/source/merlin_variables.rst | 194 ++++++++++++------ docs/source/merlin_workflows.rst | 2 +- docs/source/modules/before.rst | 31 ++- docs/source/modules/contribute.rst | 2 + .../modules/hello_world/hello_world.rst | 30 ++- .../modules/installation/installation.rst | 34 ++- docs/source/modules/port_your_application.rst | 3 +- .../modules/run_simulation/run_simulation.rst | 23 ++- docs/source/tutorial.rst | 2 +- 21 files changed, 415 insertions(+), 144 deletions(-) create mode 100644 .readthedocs.yaml create mode 100644 docs/requirements.txt delete mode 100644 docs/source/_static/theme_overrides.css diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..c1c252e30 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,13 @@ +version: 2 + +build: + os: "ubuntu-20.04" + tools: + python: "3.8" + +sphinx: + configuration: docs/source/conf.py + +python: + install: + - requirements: docs/requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fb8fe0fe..f487a66eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ranks 0 and 1 of a multi-rank allocation - Added additional argument in test definitions to allow for a "cleanup" command - Capability for non-user block in yaml +- .readthedocs.yaml and requirements.txt files for docs +- Small modifications to the Tutorial, Getting Started, Command Line, and Contributing pages in the docs ### Changed - Rename lgtm.yml to .lgtm.yml @@ -25,6 +27,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Merlin server uses app.yaml as its main configuration - Updated tutorial documentation to use merlin server over manual redis installation. - Added ssl to the broker and results backend server checks when "merlin info" is called +- Removed theme_override.css from docs/_static/ since it is no longer needed with the updated version of sphinx +- Updated docs/Makefile to include a pip install for requirements and a clean command ### Fixed - Fixed return values from scripts with main() to fix testing errors. @@ -32,6 +36,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix the cert_req typo in the merlin config docs, it should read cert_reqs - Removed emoji from issue templates that were breaking doc builds - Including .temp template files in MANIFEST +- Styling in the footer for docs +- Horizontal scroll overlap in the variables page of the docs ## [1.8.5] ### Added diff --git a/docs/Makefile b/docs/Makefile index 97642ceda..662696c6f 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -23,6 +23,9 @@ view: code-docs html # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile + pip install -r requirements.txt echo $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) +clean: + rm -rf build/ diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..064c8002b --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,56 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# pip-compile requirements.in +# +alabaster==0.7.12 + # via sphinx +babel==2.10.3 + # via sphinx +certifi==2022.9.24 + # via requests +charset-normalizer==2.1.1 + # via requests +docutils==0.17.1 + # via sphinx +idna==3.4 + # via requests +imagesize==1.4.1 + # via sphinx +importlib-metadata==5.0.0 + # via sphinx +jinja2==3.0.3 + # via sphinx +markupsafe==2.1.1 + # via jinja2 +packaging==21.3 + # via sphinx +pygments==2.13.0 + # via sphinx +pyparsing==3.0.9 + # via packaging +pytz==2022.5 + # via babel +requests==2.28.1 + # via sphinx +snowballstemmer==2.2.0 + # via sphinx +sphinx==5.3.0 + # via -r requirements.in +sphinxcontrib-applehelp==1.0.2 + # via sphinx +sphinxcontrib-devhelp==1.0.2 + # via sphinx +sphinxcontrib-htmlhelp==2.0.0 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via sphinx +sphinxcontrib-serializinghtml==1.1.5 + # via sphinx +urllib3==1.26.12 + # via requests +zipp==3.10.0 + # via importlib-metadata diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index 72d82b073..b89e9d889 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -22,3 +22,18 @@ div.sphinxsidebar { max-height: 100%; overflow-y: auto; } +td { + max-width: 300px; +} +@media screen and (min-width: 875px) { + .sphinxsidebar { + background-color: #fff; + margin-left: 0; + z-index: 1; + height: 100vh; + top: 0px; + } +} +.underline { + text-decoration: underline; +} diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css deleted file mode 100644 index 0d042ea81..000000000 --- a/docs/source/_static/theme_overrides.css +++ /dev/null @@ -1,14 +0,0 @@ -/* override table width restrictions */ -@media screen and (min-width: 767px) { - - .wy-table-responsive table td { - /* !important prevents the common CSS stylesheets from overriding - this as on RTD they are loaded after this stylesheet */ - white-space: normal !important; - } - - .wy-table-responsive { - overflow: visible !important; - } -} - diff --git a/docs/source/conf.py b/docs/source/conf.py index 2905856e8..4f0004dc2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -47,7 +47,7 @@ # 'sphinx.ext.autodoc', # 'sphinx.ext.intersphinx', # ] -extensions = [] +extensions = ['sphinx.ext.autodoc'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -101,11 +101,7 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] -html_context = { - 'css_files': [ - '_static/theme_overrides.css', # override wide tables in RTD theme - ], -} +html_css_files = ['custom.css'] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -188,8 +184,6 @@ def setup(app): try: app.add_javascript("custom.js") app.add_javascript("https://cdn.jsdelivr.net/npm/clipboard@1/dist/clipboard.min.js") - app.add_stylesheet('custom.css') except AttributeError: - app.add_css_file('custom.css') app.add_js_file("custom.js") app.add_js_file("https://cdn.jsdelivr.net/npm/clipboard@1/dist/clipboard.min.js") diff --git a/docs/source/faq.rst b/docs/source/faq.rst index e08edd88c..28d46460c 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -1,5 +1,8 @@ .. _faq: +.. role:: underline + :class: underline + FAQ === .. contents:: Frequently Asked Questions @@ -100,7 +103,7 @@ Where are some example workflows? .. code:: bash - $ merlin example --help + $ merlin example list How do I launch a workflow? ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -185,7 +188,7 @@ Each step is ultimately designated as: Normally this happens behinds the scenes, so you don't need to worry about it. To hard-code this into your step logic, use a shell command such as ``exit $(MERLIN_HARD_FAIL)``. -.. note:: ``$(MERLIN_HARD_FAIL)`` +.. note:: The ``$(MERLIN_HARD_FAIL)`` exit code will shutdown all workers connected to the queue associated with the failed step. To shutdown *all* workers use the ``$(MERLIN_STOP_WORKERS)`` exit code @@ -403,25 +406,35 @@ Do something like this: nodes: 1 procs: 3 -The arguments the LAUNCHER syntax will use: +:underline:`The arguments the LAUNCHER syntax will use`: + +``procs``: The total number of MPI tasks + +``nodes``: The total number of MPI nodes + +``walltime``: The total walltime of the run (hh:mm:ss or mm:ss or ss) (not available in lsf) + +``cores per task``: The number of hardware threads per MPI task + +``gpus per task``: The number of GPUs per MPI task + +:underline:`SLURM specific run flags`: + +``slurm``: Verbatim flags only for the srun parallel launch (srun -n -n ) + +:underline:`FLUX specific run flags`: + +``flux``: Verbatim flags for the flux parallel launch (flux mini run ) + +:underline:`LSF specific run flags`: -procs: The total number of MPI tasks -nodes: The total number of MPI nodes -walltime: The total walltime of the run (hh:mm:ss or mm:ss or ss) (not available in lsf) -cores per task: The number of hardware threads per MPI task -gpus per task: The number of GPUs per MPI task +``bind``: Flag for MPI binding of tasks on a node (default: -b rs) -SLURM specific run flags: -slurm: Verbatim flags only for the srun parallel launch (srun -n -n ) +``num resource set``: Number of resource sets -FLUX specific run flags: -flux: Verbatim flags for the flux parallel launch (flux mini run ) +``launch_distribution``: The distribution of resources (default: plane:{procs/nodes}) -LSF specific run flags: -bind: Flag for MPI binding of tasks on a node (default: -b rs) -num resource set: Number of resource sets -launch_distribution : The distribution of resources (default: plane:{procs/nodes}) -lsf: Verbatim flags only for the lsf parallel launch (jsrun ... ) +``lsf``: Verbatim flags only for the lsf parallel launch (jsrun ... ) What is level_max_dirs? ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 3d4429b4f..4b1a4c1a3 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -13,6 +13,13 @@ Check out the :doc:`Tutorial<./tutorial>`! Developer Setup ++++++++++++++++++ +The developer setup can be done via pip or via make. This section will cover how to do both. + +Additionally, there is an alternative method to setup merlin on supercomputers. See the :doc:`Spack <./spack>` section for more details. + +Pip Setup +****************** + To install with the additional developer dependencies, use:: pip3 install "merlin[dev]" @@ -21,8 +28,35 @@ or:: pip3 install -e "git+https://github.com/LLNL/merlin.git@develop#egg=merlin[dev]" -See the :doc:`Spack <./spack>` section for an alternative method to setup merlin on supercomputers. +Make Setup +******************* + +Visit the `Merlin repository `_ on github. `Create a fork of the repo `_ and `clone it `_ onto your system. + +Change directories into the merlin repo: + +.. code-block:: bash + + $ cd merlin/ + +Install Merlin with the developer dependencies: + +.. code-block:: bash + + $ make install-dev + +This will create a virtualenv, start it, and install Merlin and it's dependencies for you. + +More documentation about using Virtualenvs with Merlin can be found at +:doc:`Using Virtualenvs with Merlin <./virtualenv>`. + +We can make sure it's installed by running: + +.. code-block:: bash + $ merlin --version + +If you don't see a version number, you may need to restart your virtualenv and try again. Configuring Merlin ******************* @@ -32,6 +66,32 @@ Documentation for merlin configuration is in the :doc:`Configuring Merlin <./mer That's it. To start running Merlin see the :doc:`Merlin Workflows. <./merlin_workflows>` +(Optional) Testing Merlin +************************* + +.. warning:: + + With python 3.6 you may see some tests fail and a unicode error presented. To fix this, you need to reset the LC_ALL environment variable to en_US.utf8. + +If you have ``make`` installed and the `Merlin repository `_ cloned, you can run the test suite provided in the Makefile by running: + +.. code-block:: bash + + $ make tests + +This will run both the unit tests suite and the end-to-end tests suite. + +If you'd just like to run the unit tests you can run: + +.. code-block:: bash + + $ make unit-tests + +Similarly, if you'd just like to run the end-to-end tests you can run: + +.. code-block:: bash + + $ make e2e-tests Custom Setup +++++++++++++ diff --git a/docs/source/merlin_commands.rst b/docs/source/merlin_commands.rst index c15adb6d6..2a767797f 100644 --- a/docs/source/merlin_commands.rst +++ b/docs/source/merlin_commands.rst @@ -71,7 +71,7 @@ If you want to run an example workflow, use Merlin's ``merlin example``: .. code:: bash - $ merlin example --help + $ merlin example list This will list the available example workflows and a description for each one. To select one: diff --git a/docs/source/merlin_developer.rst b/docs/source/merlin_developer.rst index d61cc9794..34ce73b6c 100644 --- a/docs/source/merlin_developer.rst +++ b/docs/source/merlin_developer.rst @@ -43,11 +43,11 @@ To expedite review, please ensure that pull requests - Are from a meaningful branch name (e.g. ``feature/my_name/cool_thing``) -- Into the `appropriate branch `_ +- Are being merged into the `appropriate branch `_ - Include testing for any new features - - unit tests in ``tests/*`` + - unit tests in ``tests/unit`` - integration tests in ``tests/integration`` - Include descriptions of the changes @@ -64,6 +64,8 @@ To expedite review, please ensure that pull requests - in ``CHANGELOG.md`` - in ``merlin.__init__.py`` +- Have `squashed `_ commits + Testing +++++++ diff --git a/docs/source/merlin_server.rst b/docs/source/merlin_server.rst index 66e773ba9..24b37c776 100644 --- a/docs/source/merlin_server.rst +++ b/docs/source/merlin_server.rst @@ -62,11 +62,11 @@ merlin server. A detailed list of commands can be found in the `Merlin Server Co Note: Running "merlin server init" again will NOT override any exisiting configuration that the users might have set or edited. By running this command again any missing files will be created for the users with exisiting defaults. HOWEVER it is highly advised that -users back up their configuration in case an error occurs where configuration files are override. +users back up their configuration in case an error occurs where configuration files are overriden. .. toctree:: :maxdepth: 1 :caption: Merlin Server Settings: server/configuration - server/commands \ No newline at end of file + server/commands diff --git a/docs/source/merlin_variables.rst b/docs/source/merlin_variables.rst index c19056574..5dee0fd7c 100644 --- a/docs/source/merlin_variables.rst +++ b/docs/source/merlin_variables.rst @@ -31,73 +31,133 @@ The directory structure of merlin output looks like this: Reserved variables ------------------ .. list-table:: Study variables that Merlin uses. May be referenced within a specification file, but not reassigned or overridden. - - * - Variable - - Description - - Example Expansion - * - ``$(SPECROOT)`` - - Directory path of the specification file. - - ``/globalfs/user/merlin_workflows`` - * - ``$(OUTPUT_PATH)`` - - Directory path the study output will be written to. If not defined - will default to the current working directory. May be reassigned or - overridden. - - ``./studies`` - * - ``$(MERLIN_TIMESTAMP)`` - - The time a study began. May be used as a unique identifier. - - ``"YYYYMMDD-HHMMSS"`` - * - ``$(MERLIN_WORKSPACE)`` - - Output directory generated by a study at ``OUTPUT_PATH``. Ends with - ``MERLIN_TIMESTAMP``. - - ``$(OUTPUT_PATH)/ensemble_name_$(MERLIN_TIMESTAMP)`` - * - ``$(WORKSPACE)`` - - The workspace directory for a single step. - - ``$(OUTPUT_PATH)/ensemble_name_$(MERLIN_TIMESTAMP)/step_name/`` - * - ``$(MERLIN_INFO)`` - - Directory within ``MERLIN_WORKSPACE`` that holds the provenance specs and sample generation results. - Commonly used to hold ``samples.npy``. - - ``$(MERLIN_WORKSPACE)/merlin_info/`` - * - ``$(MERLIN_SAMPLE_ID)`` - - Sample index in an ensemble - - ``0`` ``1`` ``2`` ``3`` - * - ``$(MERLIN_SAMPLE_PATH)`` - - Path in the sample directory tree to a sample's directory, i.e. where the - task is actually run. - - ``/0/0/0/`` ``/0/0/1/`` ``/0/0/2/`` ``/0/0/3/`` - * - ``$(MERLIN_GLOB_PATH)`` - - All of the directories in a simulation tree as a glob (*) string - - ``/\*/\*/\*/\*`` - * - ``$(MERLIN_PATHS_ALL)`` - - A space delimited string of all of the paths; - can be used as is in bash for loop for instance with: - - .. code-block:: bash - - for path in $(MERLIN_PATHS_ALL) - do - ls $path - done - - for path in $(MERLIN_PATHS_ALL) - do - ls $path - done - - ``0/0/0 0/0/1 0/0/2 0/0/3`` - * - ``$(MERLIN_SAMPLE_VECTOR)`` - - Vector of merlin sample values - - ``$(SAMPLE_COLUMN_1) $(SAMPLE_COLUMN_2) ...`` - * - ``$(MERLIN_SAMPLE_NAMES)`` - - Names of merlin sample values - - ``SAMPLE_COLUMN_1 SAMPLE_COLUMN_2 ...`` - * - ``$(MERLIN_SPEC_ORIGINAL_TEMPLATE)`` - - Copy of original yaml file passed to ``merlin run``. - - ``$(MERLIN_INFO)/*.orig.yaml`` - * - ``$(MERLIN_SPEC_EXECUTED_RUN)`` - - Parsed and processed yaml file with command-line variable substitutions included. - - ``$(MERLIN_INFO)/*.partial.yaml`` - * - ``$(MERLIN_SPEC_ARCHIVED_COPY)`` - - Archive version of ``MERLIN_SPEC_EXECUTED_RUN`` with all variables and paths fully resolved. - - ``$(MERLIN_INFO)/*.expanded.yaml`` + :widths: 25 50 25 + :header-rows: 1 + + * - Variable + - Description + - Example Expansion + + * - ``$(SPECROOT)`` + - Directory path of the specification file. + - + :: + + /globalfs/user/merlin_workflows + + * - ``$(OUTPUT_PATH)`` + - Directory path the study output will be written to. If not defined + will default to the current working directory. May be reassigned or + overridden. + - + :: + + ./studies + + * - ``$(MERLIN_TIMESTAMP)`` + - The time a study began. May be used as a unique identifier. + - + :: + + "YYYYMMDD-HHMMSS" + + * - ``$(MERLIN_WORKSPACE)`` + - Output directory generated by a study at ``OUTPUT_PATH``. Ends with + ``MERLIN_TIMESTAMP``. + - + :: + + $(OUTPUT_PATH)/ensemble_name_$(MERLIN_TIMESTAMP) + + * - ``$(WORKSPACE)`` + - The workspace directory for a single step. + - + :: + + $(OUTPUT_PATH)/ensemble_name_$(MERLIN_TIMESTAMP)/step_name/`` + + * - ``$(MERLIN_INFO)`` + - Directory within ``MERLIN_WORKSPACE`` that holds the provenance specs and sample generation results. + Commonly used to hold ``samples.npy``. + - + :: + + $(MERLIN_WORKSPACE)/merlin_info/ + + * - ``$(MERLIN_SAMPLE_ID)`` + - Sample index in an ensemble + - + :: + + 0 1 2 3 + + * - ``$(MERLIN_SAMPLE_PATH)`` + - Path in the sample directory tree to a sample's directory, i.e. where the + task is actually run. + - + :: + + /0/0/0/ /0/0/1/ /0/0/2/ /0/0/3/ + + * - ``$(MERLIN_GLOB_PATH)`` + - All of the directories in a simulation tree as a glob (*) string + - + :: + + /*/*/*/* + + * - ``$(MERLIN_PATHS_ALL)`` + - A space delimited string of all of the paths; + can be used as is in bash for loop for instance with: + + .. code-block:: bash + + for path in $(MERLIN_PATHS_ALL) + do + ls $path + done + - + :: + + 0/0/0 + 0/0/1 + 0/0/2 + 0/0/3 + + * - ``$(MERLIN_SAMPLE_VECTOR)`` + - Vector of merlin sample values + - + :: + + $(SAMPLE_COLUMN_1) $(SAMPLE_COLUMN_2) ... + + * - ``$(MERLIN_SAMPLE_NAMES)`` + - Names of merlin sample values + - + :: + + SAMPLE_COLUMN_1 SAMPLE_COLUMN_2 ... + + * - ``$(MERLIN_SPEC_ORIGINAL_TEMPLATE)`` + - Copy of original yaml file passed to ``merlin run``. + - + :: + + $(MERLIN_INFO)/*.orig.yaml + + * - ``$(MERLIN_SPEC_EXECUTED_RUN)`` + - Parsed and processed yaml file with command-line variable substitutions included. + - + :: + + $(MERLIN_INFO)/*.partial.yaml + + * - ``$(MERLIN_SPEC_ARCHIVED_COPY)`` + - Archive version of ``MERLIN_SPEC_EXECUTED_RUN`` with all variables and paths fully resolved. + - + :: + + $(MERLIN_INFO)/*.expanded.yaml User variables diff --git a/docs/source/merlin_workflows.rst b/docs/source/merlin_workflows.rst index c06976438..42cb7a39a 100644 --- a/docs/source/merlin_workflows.rst +++ b/docs/source/merlin_workflows.rst @@ -8,7 +8,7 @@ provides documentation on running these Merlin workflow examples. Overview -------- -List the built-in Merlin workflow examples with ``merlin example --help``. +List the built-in Merlin workflow examples with ``merlin example list``. The Merlin team is working on adding a more diverse array of example workflows like these. diff --git a/docs/source/modules/before.rst b/docs/source/modules/before.rst index e9a886548..dab1c8e2c 100644 --- a/docs/source/modules/before.rst +++ b/docs/source/modules/before.rst @@ -8,23 +8,40 @@ start the tutorial modules: __ https://www.python.org/downloads/release/python-360/ +* Make sure you have `pip`__ version 22.3 or newer. + +__ https://www.pypi.org/project/pip/ + + * You can upgrade pip to the latest version with: + + .. code-block:: bash + + pip install --upgrade pip + + * OR you can upgrade to a specific version with: + + .. code-block:: bash + + pip install --upgrade pip==x.y.z + + * Make sure you have `GNU make tools`__ and `compilers`__. __ https://www.gnu.org/software/make/ __ https://gcc.gnu.org/ -* Install `docker`__. +* (OPTIONAL) Install `docker`__. __ https://docs.docker.com/install/ -* Download OpenFOAM image with: + * Download OpenFOAM image with: -.. code-block:: bash + .. code-block:: bash - docker pull cfdengine/openfoam + docker pull cfdengine/openfoam -* Download redis image with: + * Download redis image with: -.. code-block:: bash + .. code-block:: bash - docker pull redis + docker pull redis diff --git a/docs/source/modules/contribute.rst b/docs/source/modules/contribute.rst index 9b0239a44..acf35d323 100644 --- a/docs/source/modules/contribute.rst +++ b/docs/source/modules/contribute.rst @@ -44,3 +44,5 @@ Contributing to Merlin is easy! Just `send us a pull request `. diff --git a/docs/source/modules/hello_world/hello_world.rst b/docs/source/modules/hello_world/hello_world.rst index 31ea31976..2cec6f05c 100644 --- a/docs/source/modules/hello_world/hello_world.rst +++ b/docs/source/modules/hello_world/hello_world.rst @@ -20,9 +20,15 @@ This hands-on module walks through the steps of building and running a simple me .. contents:: Table of Contents: :local: -Get example files +Get Example Files +++++++++++++++++ -``merlin example`` is a command line tool that makes it easy to get a basic workflow up and running. Run the following commands: +``merlin example`` is a command line tool that makes it easy to get a basic workflow up and running. To see a list of all the examples provided with merlin you can run: + +.. code-block:: bash + + $ merlin example list + +For this tutorial we will be using the ``hello`` example. Run the following commands: .. code-block:: bash @@ -44,7 +50,7 @@ This will create and move into directory called ``hello``, which contains these * ``requirements.txt`` -- this is a text file listing this workflow's python dependencies. -Specification file +Specification File ++++++++++++++++++ Central to Merlin is something called a specification file, or a "spec" for short. @@ -97,7 +103,7 @@ So this will give us 1) an English result, and 2) a Spanish one (you could add a Section: ``study`` ~~~~~~~~~~~~~~~~~~ This is where you define workflow steps. -While the convention is to list steps as sequentially as possible, the only factor in determining step order is the dependency DAG created by the ``depends`` field. +While the convention is to list steps as sequentially as possible, the only factor in determining step order is the dependency directed acyclic graph (DAG) created by the ``depends`` field. .. code-block:: yaml @@ -163,7 +169,7 @@ The order of the spec sections doesn't matter. At this point, ``my_hello.yaml`` is still maestro-compatible. The primary difference is that maestro won't understand anything in the ``merlin`` block, which we will still add later. If you want to try it, run: ``$ maestro run my_hello.yaml`` -Try it! +Try It! +++++++ First, we'll run merlin locally. On the command line, run: @@ -200,7 +206,7 @@ A lot of stuff, right? Here's what it means: .. Assuming config is ready -Run distributed! +Run Distributed! ++++++++++++++++ .. important:: @@ -234,6 +240,12 @@ Immediately after that, this will pop up: .. literalinclude :: celery.txt :language: text +You may not see all of the info logs listed after the Celery C is displayed. If you'd like to see them you can change the merlin workers' log levels with the ``--worker-args`` tag: + +.. code-block:: bash + + $ merlin run-workers --worker-args "-l INFO" my_hello.yaml + The terminal you ran workers in is now being taken over by Celery, the powerful task queue library that merlin uses internally. The workers will continue to report their task status here until their tasks are complete. Workers are persistent, even after work is done. Send a stop signal to all your workers with this command: @@ -249,7 +261,7 @@ Workers are persistent, even after work is done. Send a stop signal to all your .. _Using Samples: -Using samples +Using Samples +++++++++++++ It's a little boring to say "hello world" in just two different ways. Let's instead say hello to many people! @@ -283,10 +295,12 @@ This makes ``N_SAMPLES`` into a user-defined variable that you can use elsewhere file: $(MERLIN_INFO)/samples.csv column_labels: [WORLD] -This is the merlin block, an exclusively merlin feature. It provides a way to generate samples for your workflow. In this case, a sample is the name of a person. +This is the merlin block, an exclusively merlin feature. It provides a way to generate samples for your workflow. In this case, a sample is the name of a person. For simplicity we give ``column_labels`` the name ``WORLD``, just like before. +It's also important to note that ``$(SPECROOT)`` and ``$(MERLIN_INFO)`` are reserved variables. The ``$(SPECROOT)`` variable is a shorthand for the directory path of the spec file and the ``$(MERLIN_INFO)`` variable is a shorthand for the directory holding the provenance specs and sample generation results. More information on Merlin variables can be found on the :doc:`variables page<../../merlin_variables>`. + It's good practice to shift larger chunks of code to external scripts. At the same location of your spec, make a new file called ``make_samples.py``: .. literalinclude :: ../../../../merlin/examples/workflows/hello/make_samples.py diff --git a/docs/source/modules/installation/installation.rst b/docs/source/modules/installation/installation.rst index c4d58192b..d18261af5 100644 --- a/docs/source/modules/installation/installation.rst +++ b/docs/source/modules/installation/installation.rst @@ -7,8 +7,8 @@ Installation * pip3 * wget * build tools (make, C/C++ compiler) - * docker (required for :doc:`Module 4: Run a Real Simulation<../run_simulation/run_simulation>`) - * file editor for docker config file editing + * (OPTIONAL) docker (required for :doc:`Module 4: Run a Real Simulation<../run_simulation/run_simulation>`) + * (OPTIONAL) file editor for docker config file editing .. admonition:: Estimated time @@ -29,7 +29,7 @@ Merlin will then be configured for the local machine and the configuration will be checked to ensure a proper installation. -Installing merlin +Installing Merlin ----------------- A merlin installation is required for the subsequent modules of this tutorial. @@ -69,6 +69,20 @@ Install merlin through pip. pip3 install merlin +Check to make sure merlin installed correctly. + +.. code-block:: bash + + which merlin + +You should see that it was installed in your virtualenv, like so: + +.. code-block:: bash + + ~//merlin_venv/bin/merlin + +If this is not the output you see, you may need to restart your virtualenv and try again. + When you are done with the virtualenv you can deactivate it using ``deactivate``, but leave the virtualenv activated for the subsequent steps. @@ -77,7 +91,7 @@ but leave the virtualenv activated for the subsequent steps. deactivate -redis server +Redis Server ++++++++++++ A redis server is required for the celery results backend server, this same server @@ -86,6 +100,7 @@ however we will need to download one of the supported container platforms avalia the purpose of this tutorial we will be using singularity. .. code-block:: bash + # Update and install singularity dependencies apt-get update && apt-get install -y \ build-essential \ @@ -120,7 +135,7 @@ the purpose of this tutorial we will be using singularity. make -C ./builddir && \ sudo make -C ./builddir install -Configuring merlin +Configuring Merlin ------------------ Merlin requires a configuration script for the celery interface. Run this configuration method to create the ``app.yaml`` @@ -138,10 +153,11 @@ to see the configuration, it should look like the configuration below. .. literalinclude:: ./app_local_redis.yaml :language: yaml +More detailed information on configuring Merlin can be found in the :doc:`configuration section<../../merlin_config>`. .. _Verifying installation: -Checking/Verifying installation +Checking/Verifying Installation ------------------------------- First launch the merlin server containers by using the ``merlin server`` commands @@ -212,10 +228,10 @@ If everything is set up correctly, you should see: . -Docker Advanced Installation +(OPTIONAL) Docker Advanced Installation ---------------------------- -RabbitMQ server +RabbitMQ Server +++++++++++++++ This optional section details the setup of a rabbitmq server for merlin. @@ -276,7 +292,7 @@ and add the password ``guest``. The aliases defined previously can be used with this set of docker containers. -Redis TLS server +Redis TLS Server ++++++++++++++++ This optional section details the setup of a redis server with TLS for merlin. diff --git a/docs/source/modules/port_your_application.rst b/docs/source/modules/port_your_application.rst index 26f2cc1d1..c9d89b06d 100644 --- a/docs/source/modules/port_your_application.rst +++ b/docs/source/modules/port_your_application.rst @@ -26,6 +26,7 @@ Tips for porting your app, building workflows The first step of building a new workflow, or porting an existing app to a workflow, is to describe it as a set of discrete, and ideally focused steps. Decoupling the steps and making them generic when possible will facilitate more rapid composition of future workflows. This will also require mapping out the dependencies and parameters that get passed between/shared across these steps. Setting up a template using tools such as `cookiecutter `_ can be useful for more production style workflows that will be frequently reused. Additionally, make use of the built-in examples accessible from the merlin command line with ``merlin example``. + .. (machine learning applications on different data sets?) Use dry runs ``merlin run --dry --local`` to prototype without actually populating task broker's queues. Similarly, once the dry run prototype looks good, try it on a small number of parameters before throwing millions at it. @@ -39,7 +40,7 @@ Make use of exit keys such as ``MERLIN_RESTART`` or ``MERLIN_RETRY`` in your ste Tips for debugging your workflows +++++++++++++++++++++++++++++++++ -The scripts defined in the workflow steps are also written to the output directories; this is a useful debugging tool as it can both catch parameter and variable replacement errors, as well as providing a quick way to reproduce, edit, and retry the step offline before fixing the step in the workflow specification. The ``.out`` and ``.err`` files log all of the output to catch any runtime errors. Additionally, you may need to grep for ``'WARNING'`` and ``'ERROR'`` in the worker logs. +The scripts defined in the workflow steps are also written to the output directories; this is a useful debugging tool as it can both catch parameter and variable replacement errors, as well as provide a quick way to reproduce, edit, and retry the step offline before fixing the step in the workflow specification. The ``.out`` and ``.err`` files log all of the output to catch any runtime errors. Additionally, you may need to grep for ``'WARNING'`` and ``'ERROR'`` in the worker logs. .. where are the worker logs, and what might show up there that .out and .err won't see? -> these more developer focused output? diff --git a/docs/source/modules/run_simulation/run_simulation.rst b/docs/source/modules/run_simulation/run_simulation.rst index a30568946..f48d7dc97 100644 --- a/docs/source/modules/run_simulation/run_simulation.rst +++ b/docs/source/modules/run_simulation/run_simulation.rst @@ -8,7 +8,7 @@ Run a Real Simulation .. admonition:: Prerequisites - * :doc:`Module 0: Before you come<../before>` + * :doc:`Module 0: Before you start<../before>` * :doc:`Module 2: Installation<../installation/installation>` * :doc:`Module 3: Hello World<../hello_world/hello_world>` @@ -53,7 +53,9 @@ This module will be going over: * Combining the outputs of these simulations into a an array * Predictive modeling and visualization -Before moving on, +.. _Before Moving On: + +Before Moving On ~~~~~~~~~~~~~~~~~ check that the virtual environment with merlin installed is activated @@ -65,14 +67,25 @@ and that redis server is set up using this command: This is covered more in depth here: :ref:`Verifying installation` - -Then use the ``merlin example`` to get the necessary files for this module. - +There are two ways to do this example: with docker and without docker. To go through the version with docker, get the necessary files for this module by running: + .. code-block:: bash $ merlin example openfoam_wf $ cd openfoam_wf/ + +For the version without docker you should run: + +.. code-block:: bash + + $ merlin example openfoam_wf_no_docker + + $ cd openfoam_wf_no_docker/ + +.. note:: + + From here on, this tutorial will focus solely on the docker version of running openfoam. However, the docker version of this tutorial is almost identical to the no docker version. If you're using the no docker version of this tutorial you can still follow along but check the openfoam_no_docker_template.yaml file in each step to see what differs. In the ``openfoam_wf`` directory you should see the following: diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 9f90a0b0d..0b69f9553 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -20,7 +20,7 @@ Finally we offer some tips and tricks for porting and scaling up your applicatio .. toctree:: :maxdepth: 1 - :caption: Before you come: + :caption: Before you begin: modules/before From 27e51e71fc5b9bad917c01da1af30a8bab17078d Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Thu, 15 Dec 2022 12:40:17 -0800 Subject: [PATCH 050/126] Maestro v 1.1.9dev1 Compatibility (#388) Maestro up to date compatibility Also unpacked maestro DAG to just use what we need, which should help reduce task message size and perhaps allow us to use other serializers in the future. --- .github/workflows/push-pr_workflow.yml | 4 +- CHANGELOG.md | 13 ++ docs/source/merlin_developer.rst | 84 ++++++++ docs/source/merlin_specification.rst | 33 +-- docs/source/merlin_variables.rst | 33 ++- merlin/spec/defaults.py | 2 +- merlin/spec/merlinspec.json | 285 +++++++++++++++++++++++++ merlin/spec/specification.py | 211 +++++++++++++++++- merlin/study/dag.py | 21 +- merlin/study/step.py | 4 +- merlin/study/study.py | 3 +- requirements/release.txt | 2 +- setup.py | 2 +- tests/unit/spec/test_specification.py | 99 +++++++++ tox.ini | 2 +- 15 files changed, 752 insertions(+), 46 deletions(-) create mode 100644 merlin/spec/merlinspec.json diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index cc04e3afe..4d7a51ccb 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -75,7 +75,7 @@ jobs: strategy: matrix: - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v2 @@ -158,7 +158,7 @@ jobs: strategy: matrix: - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index f487a66eb..aa04a9c79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Added +- Added support for Python 3.11 - Update docker docs for new rabbitmq and redis server versions - Added lgtm.com Badge for README.md - More fixes for lgtm checks. @@ -18,8 +19,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Capability for non-user block in yaml - .readthedocs.yaml and requirements.txt files for docs - Small modifications to the Tutorial, Getting Started, Command Line, and Contributing pages in the docs +- Compatibility with the newest version of Maestro (v. 1.1.9dev1) +- JSON schema validation for Merlin spec files +- New tests related to JSON schema validation +- Instructions in the "Contributing" page of the docs on how to add new blocks/fields to the spec file +- Brief explanation of the $(LAUNCHER) variable in the "Variables" page of the docs ### Changed +- Removed support for Python 3.6 - Rename lgtm.yml to .lgtm.yml - New shortcuts in specification file (sample_vector, sample_names, spec_original_template, spec_executed_run, spec_archived_copy) - Changed "default" user password to be randomly generated by default. @@ -29,6 +36,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added ssl to the broker and results backend server checks when "merlin info" is called - Removed theme_override.css from docs/_static/ since it is no longer needed with the updated version of sphinx - Updated docs/Makefile to include a pip install for requirements and a clean command +- Changed what is stored in a Merlin DAG + - We no longer store the entire Maestro ExecutionGraph object + - We now only store the adjacency table and values obtained from the ExecutionGraph object +- Modified how spec files are verified +- Updated requirements to require maestrowf 1.9.1dev1 or later ### Fixed - Fixed return values from scripts with main() to fix testing errors. @@ -38,6 +50,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Including .temp template files in MANIFEST - Styling in the footer for docs - Horizontal scroll overlap in the variables page of the docs +- Reordered small part of Workflow Specification page in the docs in order to put "samples" back in the merlin block ## [1.8.5] ### Added diff --git a/docs/source/merlin_developer.rst b/docs/source/merlin_developer.rst index 34ce73b6c..08947ca89 100644 --- a/docs/source/merlin_developer.rst +++ b/docs/source/merlin_developer.rst @@ -90,3 +90,87 @@ Merlin has style checkers configured. They can be run from the Makefile: .. code-block:: bash $ make check-style + +Adding New Features to YAML Spec File ++++++++++++++++++++++++++++++++++++++ + +In order to conform to Maestro's verification format introduced in Maestro v1.1.7, +we now use `json schema `_ validation to verify our spec +file. + +If you are adding a new feature to Merlin that requires a new block within the yaml spec +file or a new property within a block, then you are going to need to update the +merlinspec.json file located in the merlin/spec/ directory. You also may want to add +additional verifications within the specification.py file located in the same directory. + +.. note:: + If you add custom verifications beyond the pattern checking that the json schema + checks for, then you should also add tests for this verification in the test_specification.py + file located in the merlin/tests/unit/spec/ directory. Follow the steps for adding new + tests in the docstring of the TestCustomVerification class. + +Adding a New Property +********************* + +To add a new property to a block in the yaml file, you need to create a +template for that property and place it in the correct block in merlinspec.json. For +example, say I wanted to add a new property called ``example`` that's an integer within +the ``description`` block. I would modify the ``description`` block in the merlinspec.json file to look +like this: + +.. code-block:: json + + "DESCRIPTION": { + "type": "object", + "properties": { + "name": {"type": "string", "minLength": 1}, + "description": {"type": "string", "minLength": 1}, + "example": {"type": "integer", "minimum": 1} + }, + "required": ["name", "description"] + } + +If you need help with json schema formatting, check out the `step-by-step getting +started guide `_. + +That's all that's required of adding a new property. If you want to add your own custom +verifications make sure to create unit tests for them (see the note above for more info). + +Adding a New Block +****************** + +Adding a new block is slightly more complicated than adding a new property. You will not +only have to update the merlinspec.json schema file but also add calls to verify that +block within specification.py. + +To add a block to the json schema, you will need to define the template for that entire +block. For example, if I wanted to create a block called ``country`` with two +properties labeled ``name`` and ``population`` that are both required, it would look like so: + +.. code-block:: json + + "COUNTRY": { + "type": "object", + "properties": { + "name": {"type": "string", "minLength": 1}, + "population": { + "anyOf": [ + {"type": "string", "minLength": 1}, + {"type": "integer", "minimum": 1} + ] + } + }, + "required": ["name", "capital"] + } + +Here, ``name`` can only be a string but ``population`` can be both a string and an integer. +For help with json schema formatting, check out the `step-by-step getting started guide +`_. + +The next step is to enable this block in the schema validation process. To do this we need to: + +#. Create a new method called verify_() within the MerlinSpec class +#. Call the YAMLSpecification.validate_schema() method provided to us via Maestro in your new method +#. Add a call to verify_() inside the verify() method + +If you add your own custom verifications on top of this, please add unit tests for them. diff --git a/docs/source/merlin_specification.rst b/docs/source/merlin_specification.rst index 29c0dff0c..f857fabf3 100644 --- a/docs/source/merlin_specification.rst +++ b/docs/source/merlin_specification.rst @@ -277,6 +277,23 @@ see :doc:`./merlin_variables`. batch: type: local machines: [host3] + + ################################################### + # Sample definitions + # + # samples file can be one of + # .npy (numpy binary) + # .csv (comma delimited: '#' = comment line) + # .tab (tab/space delimited: '#' = comment line) + ################################################### + samples: + column_labels: [VAR1, VAR2] + file: $(SPECROOT)/samples.npy + generate: + cmd: | + python $(SPECROOT)/make_samples.py -dims 2 -n 10 -outfile=$(INPUT_PATH)/samples.npy "[(1.3, 1.3, 'linear'), (3.3, 3.3, 'linear')]" + level_max_dirs: 25 + #################################### # User Block (Optional) #################################### @@ -327,19 +344,3 @@ see :doc:`./merlin_variables`. print "OMG is this in python2? Change is bad." print "Variable X2 is $(X2)" shell: /usr/bin/env python2 - - ################################################### - # Sample definitions - # - # samples file can be one of - # .npy (numpy binary) - # .csv (comma delimited: '#' = comment line) - # .tab (tab/space delimited: '#' = comment line) - ################################################### - samples: - column_labels: [VAR1, VAR2] - file: $(SPECROOT)/samples.npy - generate: - cmd: | - python $(SPECROOT)/make_samples.py -dims 2 -n 10 -outfile=$(INPUT_PATH)/samples.npy "[(1.3, 1.3, 'linear'), (3.3, 3.3, 'linear')]" - level_max_dirs: 25 \ No newline at end of file diff --git a/docs/source/merlin_variables.rst b/docs/source/merlin_variables.rst index 5dee0fd7c..7f545a4d2 100644 --- a/docs/source/merlin_variables.rst +++ b/docs/source/merlin_variables.rst @@ -113,9 +113,10 @@ Reserved variables .. code-block:: bash for path in $(MERLIN_PATHS_ALL) - do - ls $path - done + do + ls $path + done + - :: @@ -159,6 +160,32 @@ Reserved variables $(MERLIN_INFO)/*.expanded.yaml +The ``LAUNCHER`` Variable ++++++++++++++++++++++ + +``$(LAUNCHER)`` is a special case of a reserved variable since it's value *can* be changed. +It serves as an abstraction to launch a job with parallel schedulers like :ref:`slurm`, +:ref:`lsf`, and :ref:`flux` and it can be used within a step command. For example, +say we start with this run cmd inside our step: + +.. code:: yaml + + run: + cmd: srun -N 1 -n 3 python script.py + +We can modify this to use the ``$(LAUNCHER)`` variable like so: + +.. code:: yaml + + batch: + type: slurm + + run: + cmd: $(LAUNCHER) python script.py + nodes: 1 + procs: 3 + +In other words, the ``$(LAUNCHER)`` variable would become ``srun -N 1 -n 3``. User variables ------------------- diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 34b2113cd..4912cbf00 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -32,7 +32,7 @@ BATCH = {"batch": {"type": "local", "dry_run": False, "shell": "/bin/bash"}} -ENV = {"env": {"variables": {}, "sources": {}, "labels": {}, "dependencies": {}}} +ENV = {"env": {"variables": {}, "sources": [], "labels": {}, "dependencies": {}}} STUDY_STEP_RUN = {"task_queue": "merlin", "shell": "/bin/bash", "max_retries": 30} diff --git a/merlin/spec/merlinspec.json b/merlin/spec/merlinspec.json new file mode 100644 index 000000000..47e738ee6 --- /dev/null +++ b/merlin/spec/merlinspec.json @@ -0,0 +1,285 @@ +{ + "DESCRIPTION": { + "type": "object", + "properties": { + "name": {"type": "string", "minLength": 1}, + "description": {"type": "string", "minLength": 1} + }, + "required": [ + "name", + "description" + ] + }, + "PARAM": { + "type": "object", + "properties": { + "values": { + "type": "array" + }, + "label": {"type": "string", "minLength": 1} + }, + "required": [ + "values", + "label" + ] + }, + "STUDY_STEP": { + "type": "object", + "properties": { + "name": {"type": "string", "minLength": 1}, + "description": {"type": "string", "minLength": 1}, + "run": { + "type": "object", + "properties": { + "cmd": {"type": "string", "minLength": 1}, + "depends": {"type": "array", "uniqueItems": true}, + "pre": {"type": "string", "minLength": 1}, + "post": {"type": "string", "minLength": 1}, + "restart": {"type": "string", "minLength": 1}, + "slurm": {"type": "string", "minLength": 1}, + "lsf": {"type": "string", "minLength": 1}, + "num resource set": {"type": "integer", "minimum": 1}, + "launch distribution": {"type": "string", "minLength": 1}, + "exit_on_error": {"type": "integer", "minimum": 0, "maximum": 1}, + "shell": {"type": "string", "minLength": 1}, + "flux": {"type": "string", "minLength": 1}, + "batch": { + "type": "object", + "properties": { + "type": {"type": "string", "minLength": 1} + } + }, + "gpus per task": {"type": "integer", "minimum": 1}, + "max_retries": {"type": "integer", "minimum": 1}, + "task_queue": {"type": "string", "minLength": 1}, + "nodes": { + "anyOf": [ + {"type": "integer", "minimum": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ] + }, + "procs": { + "anyOf": [ + {"type": "integer", "minimum": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "gpus": { + "anyOf": [ + {"type": "integer", "minimum": 0}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "cores per task": { + "anyOf": [ + {"type": "integer", "minimum": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "tasks per rs": { + "anyOf": [ + {"type": "integer", "minimum": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "rs per node": { + "anyOf": [ + {"type": "integer", "minimum": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "cpus per rs": { + "anyOf": [ + {"type": "integer", "minimum": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "bind": { + "anyOf": [ + {"type": "string", "minLength": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "bind gpus": { + "anyOf": [ + {"type": "string", "minLength": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "walltime": { + "anyOf": [ + {"type": "string", "minLength": 1}, + {"type": "integer", "minimum": 0}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "reservation": {"type": "string", "minLength": 1}, + "exclusive": { + "anyOf": [ + {"type": "boolean"}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "nested": {"type": "boolean"}, + "priority": { + "anyOf": [ + { + "type": "string", + "enum": [ + "HELD", "MINIMAL", "LOW", "MEDIUM", "HIGH", "EXPEDITED", + "held", "minimal", "low", "medium", "high", "expedited", + "Held", "Minimal", "Low", "Medium", "High", "Expedited" + ] + }, + {"type": "number", "minimum": 0.0, "maximum": 1.0} + ] + }, + "qos": {"type": "string", "minLength": 1} + }, + "required": [ + "cmd" + ] + } + }, + "required": [ + "name", + "description", + "run" + ] + }, + "ENV": { + "type": "object", + "properties": { + "variables": { + "type": "object", + "patternProperties": { + "^.*": { + "anyOf": [ + {"type": "string", "minLength": 1}, + {"type": "number"} + ] + } + } + }, + "labels": {"type": "object"}, + "sources": {"type": "array"}, + "dependencies": { + "type": "object", + "properties": { + "paths": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string", "minLength": 1}, + "path": {"type": "string", "minLength": 1} + }, + "required": [ + "name", + "path" + ] + } + }, + "git": { + "type": "array", + "items": { + "properties": { + "name": {"type": "string", "minLength": 1}, + "path": {"type": "string", "minLength": 1}, + "url": {"type": "string", "minLength": 1}, + "tag": {"type": "string", "minLength": 1} + }, + "required": [ + "name", + "path", + "url" + ] + } + }, + "spack": { + "type": "object", + "properties": { + "name": {"type": "string", "minLength": 1}, + "package_name": {"type": "string", "minLength": 1} + }, + "required": [ + "type", + "package_name" + ] + } + } + } + } + }, + "MERLIN": { + "type": "object", + "properties": { + "resources": { + "type": "object", + "properties": { + "task_server": {"type": "string", "minLength": 1}, + "overlap": {"type": "boolean"}, + "workers": { + "type": "object", + "patternProperties": { + "^.+": { + "type": "object", + "properties": { + "args": {"type": "string", "minLength": 1}, + "steps": {"type": "array", "uniqueItems": true}, + "nodes": { + "anyOf": [ + {"type": "null"}, + {"type": "integer", "minimum": 1} + ] + }, + "batch": { + "anyOf": [ + {"type": "null"}, + { + "type": "object", + "properties": { + "type": {"type": "string", "minLength": 1} + } + } + ] + }, + "machines": {"type": "array", "uniqueItems": true} + } + } + }, + "minProperties": 1 + } + } + }, + "samples": { + "anyOf": [ + {"type": "null"}, + { + "type": "object", + "properties": { + "generate": { + "type": "object", + "properties": { + "cmd": {"type": "string", "minLength": 1} + }, + "required": ["cmd"] + }, + "file": {"type": "string", "minLength": 1}, + "column_labels": {"type": "array", "uniqueItems": true}, + "level_max_dirs": {"type": "integer", "minimum": 1} + } + } + ] + } + } + }, + "BATCH": { + "type": "object", + "properties": { + "type": {"type": "string", "minLength": 1}, + "bank": {"type": "string", "minLength": 1}, + "queue": {"type": "string", "minLength": 1}, + "dry_run": {"type": "boolean"}, + "shell": {"type": "string", "minLength": 1}, + "flux_path": {"type": "string", "minLength": 1}, + "flux_start_opts": {"type": "string", "minLength": 1}, + "flux_exec_workers": {"type": "boolean"}, + "launch_pre": {"type": "string", "minLength": 1}, + "launch_args": {"type": "string", "minLength": 1}, + "worker_launch": {"type": "string", "minLength": 1}, + "nodes": {"type": "integer", "minimum": 1}, + "walltime": {"type": "string", "pattern": "^(?:(?:([0-9][0-9]|2[0-3]):)?([0-5][0-9]):)?([0-5][0-9])$"} + } + } +} diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 466e9f50e..6e83b7c5b 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -33,13 +33,14 @@ data from the Merlin specification file. To see examples of yaml specifications, run `merlin example`. """ +import json import logging import os import shlex from io import StringIO import yaml -from maestrowf.datastructures import YAMLSpecification +from maestrowf.specification import YAMLSpecification from merlin.spec import all_keys, defaults @@ -101,28 +102,212 @@ def sections(self): "user": self.user, } + def __str__(self): + """Magic method to print an instance of our MerlinSpec class.""" + env = "" + globs = "" + merlin = "" + user = "" + if self.environment: + env = f"\n\tenvironment: \n\t\t{self.environment}" + if self.globals: + globs = f"\n\tglobals:\n\t\t{self.globals}" + if self.merlin: + merlin = f"\n\tmerlin:\n\t\t{self.merlin}" + if self.user is not None: + user = f"\n\tuser:\n\t\t{self.user}" + result = f"""MERLIN SPEC OBJECT:\n\tdescription:\n\t\t{self.description} + \n\tbatch:\n\t\t{self.batch}\n\tstudy:\n\t\t{self.study} + {env}{globs}{merlin}{user}""" + + return result + @classmethod def load_specification(cls, filepath, suppress_warning=True): - spec = super(MerlinSpec, cls).load_specification(filepath) - with open(filepath, "r") as f: - spec.merlin = MerlinSpec.load_merlin_block(f) - with open(filepath, "r") as f: - spec.user = MerlinSpec.load_user_block(f) + LOG.info("Loading specification from path: %s", filepath) + try: + # Load the YAML spec from the filepath + with open(filepath, "r") as data: + spec = cls.load_spec_from_string(data, needs_IO=False, needs_verification=True) + except Exception as e: + LOG.exception(e.args) + raise e + + # Path not set in _populate_spec because loading spec with string + # does not have a path so we set it here + spec.path = filepath spec.specroot = os.path.dirname(spec.path) - spec.process_spec_defaults() + if not suppress_warning: spec.warn_unrecognized_keys() return spec @classmethod - def load_spec_from_string(cls, string): - spec = super(MerlinSpec, cls).load_specification_from_stream(StringIO(string)) - spec.merlin = MerlinSpec.load_merlin_block(StringIO(string)) - spec.user = MerlinSpec.load_user_block(StringIO(string)) + def load_spec_from_string(cls, string, needs_IO=True, needs_verification=False): + LOG.debug("Creating Merlin spec object...") + # Create and populate the MerlinSpec object + data = StringIO(string) if needs_IO else string + spec = cls._populate_spec(data) spec.specroot = None spec.process_spec_defaults() + LOG.debug("Merlin spec object created.") + + # Verify the spec object + if needs_verification: + LOG.debug("Verifying Merlin spec...") + spec.verify() + LOG.debug("Merlin spec verified.") + return spec + @classmethod + def _populate_spec(cls, data): + """ + Helper method to load a study spec and populate it's fields. + + NOTE: This is basically a direct copy of YAMLSpecification's + load_specification method from Maestro just without the call to verify. + The verify method was breaking our code since we have no way of modifying + Maestro's schema that they use to verify yaml files. The work around + is to load the yaml file ourselves and create our own schema to verify + against. + + :param data: Raw text stream to study YAML spec data + :returns: A MerlinSpec object containing information from the path + """ + # Read in the spec file + try: + spec = yaml.load(data, yaml.FullLoader) + except AttributeError: + LOG.warn( + "PyYAML is using an unsafe version with a known " + "load vulnerability. Please upgrade your installation " + "to a more recent version!" + ) + spec = yaml.load(data) + LOG.debug("Successfully loaded specification: \n%s", spec["description"]) + + # Load in the parts of the yaml that are the same as Maestro's + merlin_spec = cls() + merlin_spec.path = None + merlin_spec.description = spec.pop("description", {}) + merlin_spec.environment = spec.pop("env", {"variables": {}, "sources": [], "labels": {}, "dependencies": {}}) + merlin_spec.batch = spec.pop("batch", {}) + merlin_spec.study = spec.pop("study", []) + merlin_spec.globals = spec.pop("global.parameters", {}) + + # Reset the file pointer and load the merlin block + data.seek(0) + merlin_spec.merlin = MerlinSpec.load_merlin_block(data) + + # Reset the file pointer and load the user block + data.seek(0) + merlin_spec.user = MerlinSpec.load_user_block(data) + + return merlin_spec + + def verify(self): + """ + Verify the spec against a valid schema. Similar to YAMLSpecification's verify + method from Maestro but specific for Merlin yaml specs. + + NOTE: Maestro v2.0 may add the ability to customize the schema files it + compares against. If that's the case then we can convert this file back to + using Maestro's verification. + """ + # Load the MerlinSpec schema file + dir_path = os.path.dirname(os.path.abspath(__file__)) + schema_path = os.path.join(dir_path, "merlinspec.json") + with open(schema_path, "r") as json_file: + schema = json.load(json_file) + + # Use Maestro's verification methods for shared sections + self.verify_description(schema["DESCRIPTION"]) + self.verify_environment(schema["ENV"]) + self.verify_study(schema["STUDY_STEP"]) + self.verify_parameters(schema["PARAM"]) + + # Merlin specific verification + self.verify_merlin_block(schema["MERLIN"]) + self.verify_batch_block(schema["BATCH"]) + + def get_study_step_names(self): + """ + Get a list of the names of steps in our study. + + :returns: an unsorted list of study step names + """ + names = [] + for step in self.study: + names.append(step["name"]) + return names + + def _verify_workers(self): + """ + Helper method to verify the workers section located within the Merlin block + of our spec file. + """ + # Retrieve the names of the steps in our study + actual_steps = self.get_study_step_names() + + try: + # Verify that the steps in merlin block's worker section actually exist + for worker, worker_vals in self.merlin["resources"]["workers"].items(): + error_prefix = f"Problem in Merlin block with worker {worker} --" + for step in worker_vals["steps"]: + if step != "all" and step not in actual_steps: + error_msg = ( + f"{error_prefix} Step with the name {step}" + " is not defined in the study block of the yaml specification file" + ) + raise ValueError(error_msg) + + except Exception: + raise + + def verify_merlin_block(self, schema): + """ + Method to verify the merlin section of our spec file. + + :param schema: The section of the predefined schema (merlinspec.json) to check + our spec file against. + """ + # Validate merlin block against the json schema + YAMLSpecification.validate_schema("merlin", self.merlin, schema) + # Verify the workers section within merlin block + self._verify_workers() + + def verify_batch_block(self, schema): + """ + Method to verify the batch section of our spec file. + + :param schema: The section of the predefined schema (merlinspec.json) to check + our spec file against. + """ + # Validate batch block against the json schema + YAMLSpecification.validate_schema("batch", self.batch, schema) + + # Additional Walltime checks in case the regex from the schema bypasses an error + if "walltime" in self.batch: + if self.batch["type"] == "lsf": + LOG.warning("The walltime argument is not available in lsf.") + else: + try: + err_msg = "Walltime must be of the form SS, MM:SS, or HH:MM:SS." + walltime = self.batch["walltime"] + if len(walltime) > 2: + # Walltime must have : if it's not of the form SS + if ":" not in walltime: + raise ValueError(err_msg) + else: + # Walltime must have exactly 2 chars between : + time = walltime.split(":") + for section in time: + if len(section) != 2: + raise ValueError(err_msg) + except Exception: + raise + @staticmethod def load_merlin_block(stream): try: @@ -200,6 +385,7 @@ def recurse(result, defaults): recurse(object_to_update, default_dict) + # ***Unsure if this method is still needed after adding json schema verification*** def warn_unrecognized_keys(self): # check description MerlinSpec.check_section("description", self.description, all_keys.DESCRIPTION) @@ -232,6 +418,9 @@ def warn_unrecognized_keys(self): @staticmethod def check_section(section_name, section, all_keys): diff = set(section.keys()).difference(all_keys) + + # TODO: Maybe add a check here for required keys + for extra in diff: LOG.warn(f"Unrecognized key '{extra}' found in spec section '{section_name}'.") diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 01f7aae91..becb9d885 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -44,11 +44,18 @@ class DAG: independent chains of tasks. """ - def __init__(self, maestro_dag, labels): + def __init__(self, maestro_adjacency_table, maestro_values, labels): """ - :param `maestro_dag`: A maestrowf ExecutionGraph. + :param `maestro_adjacency_table`: An ordered dict showing adjacency of nodes. Comes from a maestrowf ExecutionGraph. + :param `maestro_values`: An ordered dict of the values at each node. Comes from a maestrowf ExecutionGraph. + :param `labels`: A list of labels provided in the spec file. """ - self.dag = maestro_dag + # We used to store the entire maestro ExecutionGraph here but now it's + # unpacked so we're only storing the 2 attributes from it that we use: + # the adjacency table and the values. This had to happen to get pickle + # to work for Celery. + self.maestro_adjacency_table = maestro_adjacency_table + self.maestro_values = maestro_values self.backwards_adjacency = {} self.calc_backwards_adjacency() self.labels = labels @@ -59,7 +66,7 @@ def step(self, task_name): :param `task_name`: The task name. :return: A Merlin Step object. """ - return Step(self.dag.values[task_name]) + return Step(self.maestro_values[task_name]) def calc_depth(self, node, depths, current_depth=0): """Calculate the depth of the given node and its children. @@ -116,7 +123,7 @@ def children(self, task_name): :return: list of children of this task. """ - return self.dag.adjacency_table[task_name] + return self.maestro_adjacency_table[task_name] def num_children(self, task_name): """Find the number of children for the given task in the dag. @@ -156,8 +163,8 @@ def find_chain(task_name, list_of_groups_of_chains): def calc_backwards_adjacency(self): """initializes our backwards adjacency table""" - for parent in self.dag.adjacency_table: - for task_name in self.dag.adjacency_table[parent]: + for parent in self.maestro_adjacency_table: + for task_name in self.maestro_adjacency_table[parent]: if task_name in self.backwards_adjacency: self.backwards_adjacency[task_name].append(parent) else: diff --git a/merlin/study/step.py b/merlin/study/step.py index 5b03bf2cb..b533c3a6a 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -121,7 +121,7 @@ def clone_changing_workspace_and_cmd(self, new_cmd=None, cmd_replacement_pairs=N new_workspace = self.get_workspace() LOG.debug(f"cloned step with workspace {new_workspace}") study_step = StudyStep() - study_step.name = step_dict["name"] + study_step.name = step_dict["_name"] study_step.description = step_dict["description"] study_step.run = step_dict["run"] return Step(MerlinStepRecord(new_workspace, study_step)) @@ -218,7 +218,7 @@ def name(self): """ :return : The step name. """ - return self.mstep.step.__dict__["name"] + return self.mstep.step.__dict__["_name"] def execute(self, adapter_config): """ diff --git a/merlin/study/study.py b/merlin/study/study.py index f464cf232..ecad49542 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -519,7 +519,8 @@ def load_dag(self): labels = [] if self.expanded_spec.merlin["samples"]: labels = self.expanded_spec.merlin["samples"]["column_labels"] - self.dag = DAG(maestro_dag, labels) + # To avoid pickling issues with _pass_detect_cycle from maestro, we unpack the dag here + self.dag = DAG(maestro_dag.adjacency_table, maestro_dag.values, labels) def get_adapter_config(self, override_type=None): adapter_config = dict(self.expanded_spec.batch) diff --git a/requirements/release.txt b/requirements/release.txt index 055357b89..821589c41 100644 --- a/requirements/release.txt +++ b/requirements/release.txt @@ -3,7 +3,7 @@ celery[redis,sqlalchemy]>=5.0.3 coloredlogs cryptography importlib_resources; python_version < '3.7' -maestrowf==1.1.7dev0 +maestrowf>=1.1.9dev1 numpy parse psutil>=5.1.0 diff --git a/setup.py b/setup.py index f60536d30..e92c48324 100644 --- a/setup.py +++ b/setup.py @@ -96,11 +96,11 @@ def extras_require(): long_description=readme(), long_description_content_type="text/markdown", classifiers=[ - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], keywords="machine learning workflow", url="https://github.com/LLNL/merlin", diff --git a/tests/unit/spec/test_specification.py b/tests/unit/spec/test_specification.py index 6b3503fb5..e1fb09a6b 100644 --- a/tests/unit/spec/test_specification.py +++ b/tests/unit/spec/test_specification.py @@ -3,6 +3,8 @@ import tempfile import unittest +import yaml + from merlin.spec.specification import MerlinSpec @@ -80,6 +82,31 @@ label : N_NEW.%% """ +INVALID_MERLIN = """ +description: + name: basic_ensemble_invalid_merlin + description: Template yaml to ensure our custom merlin block verification works as intended + +batch: + type: local + +study: + - name: step1 + description: | + this won't actually run + run: + cmd: | + echo "if this is printed something is bad" + +merlin: + resources: + task_server: celery + overlap: false + workers: + worker1: + steps: [] +""" + class TestMerlinSpec(unittest.TestCase): """Test the logic for parsing the Merlin spec into a MerlinSpec.""" @@ -170,3 +197,75 @@ def test_default_merlin_block(self): self.assertEqual(self.spec.merlin["resources"]["workers"]["default_worker"]["batch"], None) self.assertEqual(self.spec.merlin["resources"]["workers"]["default_worker"]["nodes"], None) self.assertEqual(self.spec.merlin["samples"], None) + + +class TestCustomVerification(unittest.TestCase): + """ + Tests to make sure our custom verification on merlin specific parts of our + spec files is working as intended. Verification happens in + merlin/spec/specification.py + + NOTE: reset_spec() should be called at the end of each test to make sure the + test file is reset. + + CREATING A NEW VERIFICATION TEST: + 1. Read in the spec with self.read_spec() + 2. Modify the spec with an invalid value to test for (e.g. a bad step, a bad walltime, etc.) + 3. Update the spec file with self.update_spec(spec) + 4. Assert that the correct error is thrown + 5. Reset the spec file with self.reset_spec() + """ + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.merlin_spec_filepath = os.path.join(self.tmpdir, "merlin_verification.yaml") + self.write_spec(INVALID_MERLIN) + + def tearDown(self): + shutil.rmtree(self.tmpdir, ignore_errors=True) + + def reset_spec(self): + self.write_spec(INVALID_MERLIN) + + def write_spec(self, spec): + with open(self.merlin_spec_filepath, "w+") as _file: + _file.write(spec) + + def read_spec(self): + with open(self.merlin_spec_filepath, "r") as yamfile: + spec = yaml.load(yamfile, yaml.Loader) + return spec + + def update_spec(self, spec): + with open(self.merlin_spec_filepath, "w") as yamfile: + yaml.dump(spec, yamfile, yaml.Dumper) + + def test_invalid_step(self): + # Read in the existing spec and update it with our bad step + spec = self.read_spec() + spec["merlin"]["resources"]["workers"]["worker1"]["steps"].append("bad_step") + self.update_spec(spec) + + # Assert that the invalid format was caught + with self.assertRaises(ValueError): + MerlinSpec.load_specification(self.merlin_spec_filepath) + + # Reset the spec to the default value + self.reset_spec() + + def test_invalid_walltime(self): + # Read in INVALID_MERLIN spec + spec = self.read_spec() + + invalid_walltimes = ["2", "0:1", "111", "1:1:1", "65", "65:12", "66:77", ":02:12", "123:45:33", ""] + + # Loop through the invalid walltimes and make sure they're all caught + for time in invalid_walltimes: + spec["batch"]["walltime"] = time + self.update_spec(spec) + + with self.assertRaises(ValueError): + MerlinSpec.load_specification(self.merlin_spec_filepath) + + # Reset the spec + self.reset_spec() diff --git a/tox.ini b/tox.ini index 457a827ca..4dfe01e03 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py35, py36, py37 +envlist = py37, py38, py39, py310, py311 [testenv] deps = From cd465eabaa1591712b8de80c246086a83456c264 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Dec 2022 13:13:00 -0800 Subject: [PATCH 051/126] Bump certifi from 2022.9.24 to 2022.12.7 in /docs (#387) Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.9.24 to 2022.12.7. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2022.09.24...2022.12.07) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 064c8002b..87333eb50 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.12 # via sphinx babel==2.10.3 # via sphinx -certifi==2022.9.24 +certifi==2022.12.7 # via requests charset-normalizer==2.1.1 # via requests From 1b110473f67ff8cf5c630e97e83f596f856da6f9 Mon Sep 17 00:00:00 2001 From: Ryan Lee <44886374+ryannova@users.noreply.github.com> Date: Thu, 15 Dec 2022 13:55:20 -0800 Subject: [PATCH 052/126] Release 1.9.0 (#390) * Make CHANGELOG more concise * Updated Merlin version and added License to files missing it * Incremented python version for workflow test --- CHANGELOG.md | 16 +++++----- Makefile | 2 +- merlin/__init__.py | 4 +-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- .../security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 30 +++++++++++++++++++ merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 30 +++++++++++++++++++ merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/server_commands.py | 30 +++++++++++++++++++ merlin/server/server_config.py | 30 +++++++++++++++++++ merlin/server/server_util.py | 30 +++++++++++++++++++ merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 30 +++++++++++++++++++ merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/run_tests.py | 2 +- 52 files changed, 233 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa04a9c79..0a9cde753 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,18 +4,18 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] +## [1.9.0] ### Added - Added support for Python 3.11 - Update docker docs for new rabbitmq and redis server versions - Added lgtm.com Badge for README.md - More fixes for lgtm checks. - Added merlin server command as a container option for broker and results_backend servers. -- Added merlin server unit tests to test exisiting merlin server commands. +- Added new documentation for merlin server in docs and tutorial - Added the flux_exec batch argument to allow for flux exec arguments, e.g. flux_exec: flux exec -r "0-1" to run celery workers only on ranks 0 and 1 of a multi-rank allocation -- Added additional argument in test definitions to allow for a "cleanup" command +- Additional argument in test definitions to allow for a post "cleanup" command - Capability for non-user block in yaml - .readthedocs.yaml and requirements.txt files for docs - Small modifications to the Tutorial, Getting Started, Command Line, and Contributing pages in the docs @@ -29,23 +29,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Removed support for Python 3.6 - Rename lgtm.yml to .lgtm.yml - New shortcuts in specification file (sample_vector, sample_names, spec_original_template, spec_executed_run, spec_archived_copy) -- Changed "default" user password to be randomly generated by default. - Update requirements to require redis 4.3.4 for acl user channel support -- Merlin server uses app.yaml as its main configuration -- Updated tutorial documentation to use merlin server over manual redis installation. - Added ssl to the broker and results backend server checks when "merlin info" is called - Removed theme_override.css from docs/_static/ since it is no longer needed with the updated version of sphinx - Updated docs/Makefile to include a pip install for requirements and a clean command +- Update to the Tutorial and Contributing pages in the docs - Changed what is stored in a Merlin DAG - We no longer store the entire Maestro ExecutionGraph object - We now only store the adjacency table and values obtained from the ExecutionGraph object -- Modified how spec files are verified -- Updated requirements to require maestrowf 1.9.1dev1 or later +- Modified spec verification +- Update to require maestrowf 1.9.1dev1 or later ### Fixed - Fixed return values from scripts with main() to fix testing errors. - CI test for CHANGELOG modifcations -- Fix the cert_req typo in the merlin config docs, it should read cert_reqs +- Typo "cert_req" to "cert_reqs" in the merlin config docs - Removed emoji from issue templates that were breaking doc builds - Including .temp template files in MANIFEST - Styling in the footer for docs diff --git a/Makefile b/Makefile index 4e3735df3..b0b1fb0a9 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index fa68134d6..aa33bc4d2 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.8.5" +__version__ = "1.9.0" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index de04bfc78..0b5971627 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index 3b8769bb5..ebac67eec 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 13db3dccc..7155d0c5f 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 13db3dccc..7155d0c5f 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index fa4a5f7c1..b02f9e909 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 814fb1881..26c64e9e2 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index a8be89486..8d9a89285 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index cb2a221d8..1859a55df 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index c3af9cdb6..6f3e58e9a 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 13db3dccc..7155d0c5f 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index af29d394f..e378573c4 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 8adc75228..16365e32c 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index f36134fec..9820e1041 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index bd2795915..027bcd291 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index ebbdc662e..7ade90e05 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 72cd6ec27..a546591f1 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index bf58602d5..a8c0a1ef2 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -2,6 +2,36 @@ Default celery configuration for merlin """ +############################################################################### +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### + from merlin.log_formatter import FORMATS diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index cc3dedd13..5dd08ddfb 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 843930cac..335bd05f5 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 12f7283bb..cd47be750 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -1,3 +1,33 @@ +############################################################################### +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### + import enum from typing import List diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 13db3dccc..7155d0c5f 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index e0172b6eb..3b1469f70 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 13db3dccc..7155d0c5f 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 686f04013..39c5cf2e0 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index d0cf1acb9..d59fc8511 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 20970fe7a..269bf6097 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 1f6befa88..33c6776e4 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index bc2ff9900..9e207b749 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index bbc213471..4195ceacc 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 90aa9db38..8858dcfad 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 4c102c23e..8a570b70d 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -1,5 +1,35 @@ """Main functions for instantiating and running Merlin server containers.""" +############################################################################### +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### + import logging import os import socket diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 03142e711..57cb5af22 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -1,3 +1,33 @@ +############################################################################### +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### + import enum import logging import os diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 666a388f0..e6eb6379d 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -1,3 +1,33 @@ +############################################################################### +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### + import hashlib import logging import os diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 13db3dccc..7155d0c5f 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index bc9d771ec..950ceb253 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 4912cbf00..1c0e9fa42 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 0be5b21b9..254ba80be 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index a3fbf281b..c4fcfee97 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -1,3 +1,33 @@ +############################################################################### +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### + import logging from copy import deepcopy diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 6e83b7c5b..a6ecdae2a 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 13db3dccc..7155d0c5f 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 88ab230a1..f395c5d80 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 67bae9d74..34393f967 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index becb9d885..838c14762 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 23398e117..826c1d2e3 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index b533c3a6a..5e7d89e43 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index ecad49542..b7f990a2a 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index b9f8742bb..2959b79e2 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index e92c48324..9bf170126 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 538ed786a..bfd80eb83 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # From efa425e8ea6a11e98cdc0ad40a0aa2a5bbd1c8ee Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Thu, 15 Dec 2022 14:08:10 -0800 Subject: [PATCH 053/126] Version 1.9.0 (#391) ## [1.9.0] ### Added - Added support for Python 3.11 - Update docker docs for new rabbitmq and redis server versions - Added lgtm.com Badge for README.md - More fixes for lgtm checks. - Added merlin server command as a container option for broker and results_backend servers. - Added new documentation for merlin server in docs and tutorial - Added the flux_exec batch argument to allow for flux exec arguments, e.g. flux_exec: flux exec -r "0-1" to run celery workers only on ranks 0 and 1 of a multi-rank allocation - Additional argument in test definitions to allow for a post "cleanup" command - Capability for non-user block in yaml - .readthedocs.yaml and requirements.txt files for docs - Small modifications to the Tutorial, Getting Started, Command Line, and Contributing pages in the docs - Compatibility with the newest version of Maestro (v. 1.1.9dev1) - JSON schema validation for Merlin spec files - New tests related to JSON schema validation - Instructions in the "Contributing" page of the docs on how to add new blocks/fields to the spec file - Brief explanation of the $(LAUNCHER) variable in the "Variables" page of the docs ### Changed - Removed support for Python 3.6 - Rename lgtm.yml to .lgtm.yml - New shortcuts in specification file (sample_vector, sample_names, spec_original_template, spec_executed_run, spec_archived_copy) - Update requirements to require redis 4.3.4 for acl user channel support - Added ssl to the broker and results backend server checks when "merlin info" is called - Removed theme_override.css from docs/_static/ since it is no longer needed with the updated version of sphinx - Updated docs/Makefile to include a pip install for requirements and a clean command - Update to the Tutorial and Contributing pages in the docs - Changed what is stored in a Merlin DAG - We no longer store the entire Maestro ExecutionGraph object - We now only store the adjacency table and values obtained from the ExecutionGraph object - Modified spec verification - Update to require maestrowf 1.9.1dev1 or later ### Fixed - Fixed return values from scripts with main() to fix testing errors. - CI test for CHANGELOG modifcations - Typo "cert_req" to "cert_reqs" in the merlin config docs - Removed emoji from issue templates that were breaking doc builds - Including .temp template files in MANIFEST - Styling in the footer for docs - Horizontal scroll overlap in the variables page of the docs - Reordered small part of Workflow Specification page in the docs in order to put "samples" back in the merlin block Signed-off-by: dependabot[bot] Co-authored-by: Joseph M. Koning Co-authored-by: Joe Koning Co-authored-by: Ryan Lee Co-authored-by: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/bug_report.md | 4 +- .github/ISSUE_TEMPLATE/feature_request.md | 4 +- .github/ISSUE_TEMPLATE/question.md | 4 +- .github/workflows/push-pr_workflow.yml | 32 +- .gitlab-ci.yml | 19 - .readthedocs.yaml | 13 + CHANGELOG.md | 40 +- CONTRIBUTORS | 1 + MANIFEST.in | 3 +- Makefile | 14 +- docs/Makefile | 3 + docs/requirements.txt | 56 + docs/source/_static/custom.css | 19 + docs/source/_static/theme_overrides.css | 14 - docs/source/conf.py | 12 +- docs/source/faq.rst | 47 +- docs/source/getting_started.rst | 62 +- docs/source/index.rst | 2 +- docs/source/merlin_commands.rst | 34 +- docs/source/merlin_config.rst | 4 +- docs/source/merlin_developer.rst | 90 +- docs/source/merlin_server.rst | 72 + docs/source/merlin_specification.rst | 58 +- docs/source/merlin_variables.rst | 208 +- docs/source/merlin_workflows.rst | 2 +- docs/source/modules/before.rst | 31 +- docs/source/modules/contribute.rst | 14 +- .../modules/hello_world/hello_world.rst | 30 +- .../modules/installation/installation.rst | 247 +- docs/source/modules/port_your_application.rst | 3 +- .../modules/run_simulation/run_simulation.rst | 23 +- docs/source/server/commands.rst | 87 + docs/source/server/configuration.rst | 75 + docs/source/tutorial.rst | 2 +- lgtm.yml | 25 + merlin/__init__.py | 4 +- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- .../security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 30 + merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 30 + merlin/data/celery/__init__.py | 2 +- merlin/display.py | 9 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- .../workflows/feature_demo/feature_demo.yaml | 34 +- .../feature_demo/scripts/hello_world.py | 12 +- merlin/examples/workflows/flux/flux_par.yaml | 1 + .../workflows/flux/scripts/make_samples.py | 12 +- .../hpc_demo/cumulative_sample_processor.py | 62 +- .../workflows/hpc_demo/faker_sample.py | 12 +- .../workflows/hpc_demo/sample_collector.py | 18 +- .../workflows/hpc_demo/sample_processor.py | 44 +- .../cumulative_sample_processor.py | 62 +- .../workflows/iterative_demo/faker_sample.py | 12 +- .../iterative_demo/sample_collector.py | 18 +- .../iterative_demo/sample_processor.py | 44 +- .../workflows/lsf/scripts/make_samples.py | 12 +- .../null_spec/scripts/read_output.py | 15 +- .../scripts/hello_world.py | 12 +- .../workflows/restart/scripts/make_samples.py | 12 +- .../restart_delay/scripts/make_samples.py | 12 +- .../workflows/slurm/scripts/make_samples.py | 12 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 159 +- merlin/merlin_templates.py | 16 +- merlin/router.py | 2 +- merlin/server/docker.yaml | 6 + merlin/server/merlin_server.yaml | 27 + merlin/server/podman.yaml | 6 + merlin/server/redis.conf | 2051 +++++++++++++++++ merlin/server/server_commands.py | 308 +++ merlin/server/server_config.py | 387 ++++ merlin/server/server_util.py | 607 +++++ merlin/server/singularity.yaml | 6 + merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 3 +- merlin/spec/defaults.py | 4 +- merlin/spec/expansion.py | 2 +- merlin/spec/merlinspec.json | 285 +++ merlin/spec/override.py | 30 + merlin/spec/specification.py | 226 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 4 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 23 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 6 +- merlin/study/study.py | 28 +- merlin/utils.py | 2 +- requirements/release.txt | 3 +- setup.py | 4 +- tests/integration/conditions.py | 62 + tests/integration/run_tests.py | 17 +- tests/integration/test_definitions.py | 89 +- tests/unit/spec/test_specification.py | 99 + tox.ini | 2 +- 114 files changed, 5806 insertions(+), 547 deletions(-) delete mode 100644 .gitlab-ci.yml create mode 100644 .readthedocs.yaml create mode 100644 docs/requirements.txt delete mode 100644 docs/source/_static/theme_overrides.css create mode 100644 docs/source/merlin_server.rst create mode 100644 docs/source/server/commands.rst create mode 100644 docs/source/server/configuration.rst create mode 100644 lgtm.yml create mode 100644 merlin/server/docker.yaml create mode 100644 merlin/server/merlin_server.yaml create mode 100644 merlin/server/podman.yaml create mode 100644 merlin/server/redis.conf create mode 100644 merlin/server/server_commands.py create mode 100644 merlin/server/server_config.py create mode 100644 merlin/server/server_util.py create mode 100644 merlin/server/singularity.yaml create mode 100644 merlin/spec/merlinspec.json diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 5480abee7..0cc50170d 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,5 +1,5 @@ --- -name: "\U0001F41B Bug report" +name: "Bug report" about: Create a report to help us improve title: "[BUG] " labels: bug @@ -7,7 +7,7 @@ assignees: '' --- -## 🐛 Bug Report +## Bug Report **Describe the bug** A clear and concise description of what the bug is. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 33cfac3d0..992cb3211 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,5 +1,5 @@ --- -name: "\U0001F680 Feature request" +name: "Feature request" about: Suggest an idea for Merlin title: "[FEAT] " labels: enhancement @@ -9,7 +9,7 @@ assignees: '' -## 🚀 Feature Request +## Feature Request **What problem is this feature looking to solve?** A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index b20c3af1f..58824b272 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -1,9 +1,9 @@ --- -name: 🤓 General question +name: General question labels: 'question' title: '[Q/A] ' about: Ask, discuss, debate with the Merlin team --- -## 🤓 Question +## Question diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 3b2f809eb..4d7a51ccb 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -14,7 +14,7 @@ jobs: - name: Check that CHANGELOG has been updated run: | # If this step fails, this means you haven't updated the CHANGELOG.md file with notes on your contribution. - git diff --name-only $(git merge-base origin/main HEAD) | grep '^CHANGELOG.md$' && echo "Thanks for helping keep our CHANGELOG up-to-date!" + git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep '^CHANGELOG.md$' && echo "Thanks for helping keep our CHANGELOG up-to-date!" Lint: runs-on: ubuntu-latest @@ -67,10 +67,15 @@ jobs: Local-test-suite: runs-on: ubuntu-latest + env: + GO_VERSION: 1.18.1 + SINGULARITY_VERSION: 3.9.9 + OS: linux + ARCH: amd64 strategy: matrix: - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v2 @@ -90,6 +95,27 @@ jobs: python3 -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt + + - name: Install singularity + run: | + sudo apt-get update && sudo apt-get install -y \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config + wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz + sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz + rm go$GO_VERSION.$OS-$ARCH.tar.gz + export PATH=$PATH:/usr/local/go/bin + wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz + tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz + cd singularity-ce-$SINGULARITY_VERSION + ./mconfig && \ + make -C ./builddir && \ + sudo make -C ./builddir install - name: Install merlin to run unit tests run: | @@ -132,7 +158,7 @@ jobs: strategy: matrix: - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v2 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 9403886e3..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,19 +0,0 @@ -image: python:3.8-slim-buster - -job1: - script: - - python3 -m venv venv - - source venv/bin/activate - - pip3 install --upgrade pip - - pip3 install -r requirements.txt - - pip3 install -r requirements/dev.txt - - pip3 install -r merlin/examples/workflows/feature_demo/requirements.txt - - pip3 install -e . - - pip3 install --upgrade sphinx - - merlin config - - - merlin stop-workers - - - python3 -m pytest tests/ - - python3 tests/integration/run_tests.py --verbose --local - diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..c1c252e30 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,13 @@ +version: 2 + +build: + os: "ubuntu-20.04" + tools: + python: "3.8" + +sphinx: + configuration: docs/source/conf.py + +python: + install: + - requirements: docs/requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index b472cfcf7..0a9cde753 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,51 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] +## [1.9.0] ### Added +- Added support for Python 3.11 - Update docker docs for new rabbitmq and redis server versions - Added lgtm.com Badge for README.md - More fixes for lgtm checks. +- Added merlin server command as a container option for broker and results_backend servers. +- Added new documentation for merlin server in docs and tutorial +- Added the flux_exec batch argument to allow for flux exec arguments, + e.g. flux_exec: flux exec -r "0-1" to run celery workers only on + ranks 0 and 1 of a multi-rank allocation +- Additional argument in test definitions to allow for a post "cleanup" command +- Capability for non-user block in yaml +- .readthedocs.yaml and requirements.txt files for docs +- Small modifications to the Tutorial, Getting Started, Command Line, and Contributing pages in the docs +- Compatibility with the newest version of Maestro (v. 1.1.9dev1) +- JSON schema validation for Merlin spec files +- New tests related to JSON schema validation +- Instructions in the "Contributing" page of the docs on how to add new blocks/fields to the spec file +- Brief explanation of the $(LAUNCHER) variable in the "Variables" page of the docs + ### Changed +- Removed support for Python 3.6 - Rename lgtm.yml to .lgtm.yml +- New shortcuts in specification file (sample_vector, sample_names, spec_original_template, spec_executed_run, spec_archived_copy) +- Update requirements to require redis 4.3.4 for acl user channel support +- Added ssl to the broker and results backend server checks when "merlin info" is called +- Removed theme_override.css from docs/_static/ since it is no longer needed with the updated version of sphinx +- Updated docs/Makefile to include a pip install for requirements and a clean command +- Update to the Tutorial and Contributing pages in the docs +- Changed what is stored in a Merlin DAG + - We no longer store the entire Maestro ExecutionGraph object + - We now only store the adjacency table and values obtained from the ExecutionGraph object +- Modified spec verification +- Update to require maestrowf 1.9.1dev1 or later + +### Fixed +- Fixed return values from scripts with main() to fix testing errors. +- CI test for CHANGELOG modifcations +- Typo "cert_req" to "cert_reqs" in the merlin config docs +- Removed emoji from issue templates that were breaking doc builds +- Including .temp template files in MANIFEST +- Styling in the footer for docs +- Horizontal scroll overlap in the variables page of the docs +- Reordered small part of Workflow Specification page in the docs in order to put "samples" back in the merlin block ## [1.8.5] ### Added diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 9f96bc6e7..a920e45b7 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -5,3 +5,4 @@ Benjamin Bay Joe Koning Jeremy White Aidan Keogh +Ryan Lee diff --git a/MANIFEST.in b/MANIFEST.in index f5b32237d..cefbd23a5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ recursive-include merlin/data *.yaml *.py -recursive-include merlin/examples *.py *.yaml *.c *.json *.sbatch *.bsub *.txt +recursive-include merlin/server *.yaml *.py +recursive-include merlin/examples *.py *.yaml *.c *.json *.sbatch *.bsub *.txt *.temp include requirements.txt include requirements/* diff --git a/Makefile b/Makefile index d1e441a87..b0b1fb0a9 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # @@ -164,12 +164,12 @@ checks: check-style check-camel-case # automatically make python files pep 8-compliant fix-style: . $(VENV)/bin/activate; \ - isort --line-length $(MAX_LINE_LENGTH) $(MRLN); \ - isort --line-length $(MAX_LINE_LENGTH) $(TEST); \ - isort --line-length $(MAX_LINE_LENGTH) *.py; \ - black --target-version py36 -l $(MAX_LINE_LENGTH) $(MRLN); \ - black --target-version py36 -l $(MAX_LINE_LENGTH) $(TEST); \ - black --target-version py36 -l $(MAX_LINE_LENGTH) *.py; \ + $(PYTHON) -m isort --line-length $(MAX_LINE_LENGTH) $(MRLN); \ + $(PYTHON) -m isort --line-length $(MAX_LINE_LENGTH) $(TEST); \ + $(PYTHON) -m isort --line-length $(MAX_LINE_LENGTH) *.py; \ + $(PYTHON) -m black --target-version py36 -l $(MAX_LINE_LENGTH) $(MRLN); \ + $(PYTHON) -m black --target-version py36 -l $(MAX_LINE_LENGTH) $(TEST); \ + $(PYTHON) -m black --target-version py36 -l $(MAX_LINE_LENGTH) *.py; \ # Increment the Merlin version. USE ONLY ON DEVELOP BEFORE MERGING TO MASTER. diff --git a/docs/Makefile b/docs/Makefile index 97642ceda..662696c6f 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -23,6 +23,9 @@ view: code-docs html # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile + pip install -r requirements.txt echo $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) +clean: + rm -rf build/ diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..87333eb50 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,56 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# pip-compile requirements.in +# +alabaster==0.7.12 + # via sphinx +babel==2.10.3 + # via sphinx +certifi==2022.12.7 + # via requests +charset-normalizer==2.1.1 + # via requests +docutils==0.17.1 + # via sphinx +idna==3.4 + # via requests +imagesize==1.4.1 + # via sphinx +importlib-metadata==5.0.0 + # via sphinx +jinja2==3.0.3 + # via sphinx +markupsafe==2.1.1 + # via jinja2 +packaging==21.3 + # via sphinx +pygments==2.13.0 + # via sphinx +pyparsing==3.0.9 + # via packaging +pytz==2022.5 + # via babel +requests==2.28.1 + # via sphinx +snowballstemmer==2.2.0 + # via sphinx +sphinx==5.3.0 + # via -r requirements.in +sphinxcontrib-applehelp==1.0.2 + # via sphinx +sphinxcontrib-devhelp==1.0.2 + # via sphinx +sphinxcontrib-htmlhelp==2.0.0 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via sphinx +sphinxcontrib-serializinghtml==1.1.5 + # via sphinx +urllib3==1.26.12 + # via requests +zipp==3.10.0 + # via importlib-metadata diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index 367d8e1f2..b89e9d889 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -18,3 +18,22 @@ div.highlight .copybtn:hover { div.highlight { position: relative; } +div.sphinxsidebar { + max-height: 100%; + overflow-y: auto; +} +td { + max-width: 300px; +} +@media screen and (min-width: 875px) { + .sphinxsidebar { + background-color: #fff; + margin-left: 0; + z-index: 1; + height: 100vh; + top: 0px; + } +} +.underline { + text-decoration: underline; +} diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css deleted file mode 100644 index 0d042ea81..000000000 --- a/docs/source/_static/theme_overrides.css +++ /dev/null @@ -1,14 +0,0 @@ -/* override table width restrictions */ -@media screen and (min-width: 767px) { - - .wy-table-responsive table td { - /* !important prevents the common CSS stylesheets from overriding - this as on RTD they are loaded after this stylesheet */ - white-space: normal !important; - } - - .wy-table-responsive { - overflow: visible !important; - } -} - diff --git a/docs/source/conf.py b/docs/source/conf.py index d5cef3d64..4f0004dc2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -47,7 +47,7 @@ # 'sphinx.ext.autodoc', # 'sphinx.ext.intersphinx', # ] -extensions = [] +extensions = ['sphinx.ext.autodoc'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -101,11 +101,7 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] -html_context = { - 'css_files': [ - '_static/theme_overrides.css', # override wide tables in RTD theme - ], -} +html_css_files = ['custom.css'] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -149,7 +145,7 @@ # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'Merlin.tex', u'Merlin Documentation', - u'MLSI', 'manual'), + u'The Merlin Development Team', 'manual'), ] @@ -188,8 +184,6 @@ def setup(app): try: app.add_javascript("custom.js") app.add_javascript("https://cdn.jsdelivr.net/npm/clipboard@1/dist/clipboard.min.js") - app.add_stylesheet('custom.css') except AttributeError: - app.add_css_file('custom.css') app.add_js_file("custom.js") app.add_js_file("https://cdn.jsdelivr.net/npm/clipboard@1/dist/clipboard.min.js") diff --git a/docs/source/faq.rst b/docs/source/faq.rst index e08edd88c..28d46460c 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -1,5 +1,8 @@ .. _faq: +.. role:: underline + :class: underline + FAQ === .. contents:: Frequently Asked Questions @@ -100,7 +103,7 @@ Where are some example workflows? .. code:: bash - $ merlin example --help + $ merlin example list How do I launch a workflow? ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -185,7 +188,7 @@ Each step is ultimately designated as: Normally this happens behinds the scenes, so you don't need to worry about it. To hard-code this into your step logic, use a shell command such as ``exit $(MERLIN_HARD_FAIL)``. -.. note:: ``$(MERLIN_HARD_FAIL)`` +.. note:: The ``$(MERLIN_HARD_FAIL)`` exit code will shutdown all workers connected to the queue associated with the failed step. To shutdown *all* workers use the ``$(MERLIN_STOP_WORKERS)`` exit code @@ -403,25 +406,35 @@ Do something like this: nodes: 1 procs: 3 -The arguments the LAUNCHER syntax will use: +:underline:`The arguments the LAUNCHER syntax will use`: + +``procs``: The total number of MPI tasks + +``nodes``: The total number of MPI nodes + +``walltime``: The total walltime of the run (hh:mm:ss or mm:ss or ss) (not available in lsf) + +``cores per task``: The number of hardware threads per MPI task + +``gpus per task``: The number of GPUs per MPI task + +:underline:`SLURM specific run flags`: + +``slurm``: Verbatim flags only for the srun parallel launch (srun -n -n ) + +:underline:`FLUX specific run flags`: + +``flux``: Verbatim flags for the flux parallel launch (flux mini run ) + +:underline:`LSF specific run flags`: -procs: The total number of MPI tasks -nodes: The total number of MPI nodes -walltime: The total walltime of the run (hh:mm:ss or mm:ss or ss) (not available in lsf) -cores per task: The number of hardware threads per MPI task -gpus per task: The number of GPUs per MPI task +``bind``: Flag for MPI binding of tasks on a node (default: -b rs) -SLURM specific run flags: -slurm: Verbatim flags only for the srun parallel launch (srun -n -n ) +``num resource set``: Number of resource sets -FLUX specific run flags: -flux: Verbatim flags for the flux parallel launch (flux mini run ) +``launch_distribution``: The distribution of resources (default: plane:{procs/nodes}) -LSF specific run flags: -bind: Flag for MPI binding of tasks on a node (default: -b rs) -num resource set: Number of resource sets -launch_distribution : The distribution of resources (default: plane:{procs/nodes}) -lsf: Verbatim flags only for the lsf parallel launch (jsrun ... ) +``lsf``: Verbatim flags only for the lsf parallel launch (jsrun ... ) What is level_max_dirs? ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 3d4429b4f..4b1a4c1a3 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -13,6 +13,13 @@ Check out the :doc:`Tutorial<./tutorial>`! Developer Setup ++++++++++++++++++ +The developer setup can be done via pip or via make. This section will cover how to do both. + +Additionally, there is an alternative method to setup merlin on supercomputers. See the :doc:`Spack <./spack>` section for more details. + +Pip Setup +****************** + To install with the additional developer dependencies, use:: pip3 install "merlin[dev]" @@ -21,8 +28,35 @@ or:: pip3 install -e "git+https://github.com/LLNL/merlin.git@develop#egg=merlin[dev]" -See the :doc:`Spack <./spack>` section for an alternative method to setup merlin on supercomputers. +Make Setup +******************* + +Visit the `Merlin repository `_ on github. `Create a fork of the repo `_ and `clone it `_ onto your system. + +Change directories into the merlin repo: + +.. code-block:: bash + + $ cd merlin/ + +Install Merlin with the developer dependencies: + +.. code-block:: bash + + $ make install-dev + +This will create a virtualenv, start it, and install Merlin and it's dependencies for you. + +More documentation about using Virtualenvs with Merlin can be found at +:doc:`Using Virtualenvs with Merlin <./virtualenv>`. + +We can make sure it's installed by running: + +.. code-block:: bash + $ merlin --version + +If you don't see a version number, you may need to restart your virtualenv and try again. Configuring Merlin ******************* @@ -32,6 +66,32 @@ Documentation for merlin configuration is in the :doc:`Configuring Merlin <./mer That's it. To start running Merlin see the :doc:`Merlin Workflows. <./merlin_workflows>` +(Optional) Testing Merlin +************************* + +.. warning:: + + With python 3.6 you may see some tests fail and a unicode error presented. To fix this, you need to reset the LC_ALL environment variable to en_US.utf8. + +If you have ``make`` installed and the `Merlin repository `_ cloned, you can run the test suite provided in the Makefile by running: + +.. code-block:: bash + + $ make tests + +This will run both the unit tests suite and the end-to-end tests suite. + +If you'd just like to run the unit tests you can run: + +.. code-block:: bash + + $ make unit-tests + +Similarly, if you'd just like to run the end-to-end tests you can run: + +.. code-block:: bash + + $ make e2e-tests Custom Setup +++++++++++++ diff --git a/docs/source/index.rst b/docs/source/index.rst index 3747adca4..3776466d3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -75,9 +75,9 @@ Need help? `merlin@llnl.gov `_ merlin_specification merlin_config merlin_variables + merlin_server celery_overview virtualenv spack merlin_developer docker - diff --git a/docs/source/merlin_commands.rst b/docs/source/merlin_commands.rst index 0a1a9f2bc..2a767797f 100644 --- a/docs/source/merlin_commands.rst +++ b/docs/source/merlin_commands.rst @@ -71,7 +71,7 @@ If you want to run an example workflow, use Merlin's ``merlin example``: .. code:: bash - $ merlin example --help + $ merlin example list This will list the available example workflows and a description for each one. To select one: @@ -406,4 +406,36 @@ The only currently available option for ``--task_server`` is celery, which is th only one might get the signal. In this case, you can send it again. +Hosting Local Server (``merlin server``) +---------------------------------------- + +To create a local server for merlin to connect to. Merlin server creates and configures a server on the current directory. +This allows multiple instances of merlin server to exist for different studies or uses. + +The ``init`` subcommand initalizes a new instance of merlin server. + +The ``status`` subcommand checks to the status of the merlin server. + +The ``start`` subcommand starts the merlin server. + +The ``stop`` subcommand stops the merlin server. + +The ``restart`` subcommand performs stop command followed by a start command on the merlin server. + +The ``config`` subcommand edits configurations for the merlin server. There are multiple flags to allow for different configurations. + +- The ``-ip IPADDRESS, --ipaddress IPADDRESS`` option set the binded IP address for merlin server. +- The ``-p PORT, --port PORT`` option set the binded port for merlin server. +- The ``-pwd PASSWORD, --password PASSWORD`` option set the password file for merlin server. +- The ``--add-user USER PASSWORD`` option add a new user for merlin server. +- The ``--remove-user REMOVE_USER`` option remove an exisiting user from merlin server. +- The ``-d DIRECTORY, --directory DIRECTORY`` option set the working directory for merlin server. +- The ``-ss SNAPSHOT_SECONDS, --snapshot-seconds SNAPSHOT_SECONDS`` option set the number of seconds before each snapshot. +- The ``-sc SNAPSHOT_CHANGES, --snapshot-changes SNAPSHOT_CHANGES`` option set the number of database changes before each snapshot. +- The ``-sf SNAPSHOT_FILE, --snapshot-file SNAPSHOT_FILE`` option set the name of snapshots. +- The ``-am APPEND_MODE, --append-mode APPEND_MODE`` option set the appendonly mode. Options are always, everysec, no. +- The ``-af APPEND_FILE, --append-file APPEND_FILE`` option set the filename for server append/change file. + +More information can be found on :doc:`Merlin Server <./merlin_server>` + diff --git a/docs/source/merlin_config.rst b/docs/source/merlin_config.rst index 7bb43d3c6..599a50413 100644 --- a/docs/source/merlin_config.rst +++ b/docs/source/merlin_config.rst @@ -153,7 +153,7 @@ show below. ca_certs: /var/ssl/myca.pem # This is optional and can be required, optional or none # (required is the default) - cert_req: required + cert_reqs: required @@ -197,7 +197,7 @@ url when using a redis server version 6 or greater with ssl_. ca_certs: /var/ssl/myca.pem # This is optional and can be required, optional or none # (required is the default) - cert_req: required + cert_reqs: required The resulting ``broker_use_ssl`` configuration for a ``rediss`` server is given below. diff --git a/docs/source/merlin_developer.rst b/docs/source/merlin_developer.rst index d61cc9794..08947ca89 100644 --- a/docs/source/merlin_developer.rst +++ b/docs/source/merlin_developer.rst @@ -43,11 +43,11 @@ To expedite review, please ensure that pull requests - Are from a meaningful branch name (e.g. ``feature/my_name/cool_thing``) -- Into the `appropriate branch `_ +- Are being merged into the `appropriate branch `_ - Include testing for any new features - - unit tests in ``tests/*`` + - unit tests in ``tests/unit`` - integration tests in ``tests/integration`` - Include descriptions of the changes @@ -64,6 +64,8 @@ To expedite review, please ensure that pull requests - in ``CHANGELOG.md`` - in ``merlin.__init__.py`` +- Have `squashed `_ commits + Testing +++++++ @@ -88,3 +90,87 @@ Merlin has style checkers configured. They can be run from the Makefile: .. code-block:: bash $ make check-style + +Adding New Features to YAML Spec File ++++++++++++++++++++++++++++++++++++++ + +In order to conform to Maestro's verification format introduced in Maestro v1.1.7, +we now use `json schema `_ validation to verify our spec +file. + +If you are adding a new feature to Merlin that requires a new block within the yaml spec +file or a new property within a block, then you are going to need to update the +merlinspec.json file located in the merlin/spec/ directory. You also may want to add +additional verifications within the specification.py file located in the same directory. + +.. note:: + If you add custom verifications beyond the pattern checking that the json schema + checks for, then you should also add tests for this verification in the test_specification.py + file located in the merlin/tests/unit/spec/ directory. Follow the steps for adding new + tests in the docstring of the TestCustomVerification class. + +Adding a New Property +********************* + +To add a new property to a block in the yaml file, you need to create a +template for that property and place it in the correct block in merlinspec.json. For +example, say I wanted to add a new property called ``example`` that's an integer within +the ``description`` block. I would modify the ``description`` block in the merlinspec.json file to look +like this: + +.. code-block:: json + + "DESCRIPTION": { + "type": "object", + "properties": { + "name": {"type": "string", "minLength": 1}, + "description": {"type": "string", "minLength": 1}, + "example": {"type": "integer", "minimum": 1} + }, + "required": ["name", "description"] + } + +If you need help with json schema formatting, check out the `step-by-step getting +started guide `_. + +That's all that's required of adding a new property. If you want to add your own custom +verifications make sure to create unit tests for them (see the note above for more info). + +Adding a New Block +****************** + +Adding a new block is slightly more complicated than adding a new property. You will not +only have to update the merlinspec.json schema file but also add calls to verify that +block within specification.py. + +To add a block to the json schema, you will need to define the template for that entire +block. For example, if I wanted to create a block called ``country`` with two +properties labeled ``name`` and ``population`` that are both required, it would look like so: + +.. code-block:: json + + "COUNTRY": { + "type": "object", + "properties": { + "name": {"type": "string", "minLength": 1}, + "population": { + "anyOf": [ + {"type": "string", "minLength": 1}, + {"type": "integer", "minimum": 1} + ] + } + }, + "required": ["name", "capital"] + } + +Here, ``name`` can only be a string but ``population`` can be both a string and an integer. +For help with json schema formatting, check out the `step-by-step getting started guide +`_. + +The next step is to enable this block in the schema validation process. To do this we need to: + +#. Create a new method called verify_() within the MerlinSpec class +#. Call the YAMLSpecification.validate_schema() method provided to us via Maestro in your new method +#. Add a call to verify_() inside the verify() method + +If you add your own custom verifications on top of this, please add unit tests for them. diff --git a/docs/source/merlin_server.rst b/docs/source/merlin_server.rst new file mode 100644 index 000000000..24b37c776 --- /dev/null +++ b/docs/source/merlin_server.rst @@ -0,0 +1,72 @@ +Merlin Server +============= +The merlin server command allows users easy access to containerized broker +and results servers for merlin workflows. This allowsusers to run merlin without +a dedicated external server. + +The main configuration will be stored in the subdirectory called "server/" by +default in the main merlin configuration "~/.merlin". However different server +images can be created for different use cases or studies just by simplying creating +a new directory to store local configuration files for merlin server instances. + +Below is an example of how merlin server can be utilized. + +First create and navigate into a directory to store your local merlin +configuration for a specific use case or study. + +.. code-block:: bash + + mkdir study1/ + cd study1/ + +Afterwards you can instantiate merlin server in this directory by running + +.. code-block:: bash + + merlin server init + +A main server configuration will be created in the ~/.merlin/server and a local +configuration will be created in a subdirectory called "merlin_server/" + +We should expect the following files in each directory + +.. code-block:: bash + + ~/study1$ ls ~/.merlin/server/ + docker.yaml merlin_server.yaml podman.yaml singularity.yaml + + ~/study1$ ls + merlin_server + + ~/study1$ ls merlin_server/ + redis.conf redis_latest.sif + +The main configuration in "~/.merlin/server" deals with defaults and +technical commands that might be used for setting up the merlin server +local configuration and its containers. Each container has their own +configuration file to allow users to be able to switch between different +containerized services freely. + +The local configuration "merlin_server" folder contains configuration files +specific to a certain use case or run. In the case above you can see that we have a +redis singularity container called "redis_latest.sif" with the redis configuration +file called "redis.conf". This redis configuration will allow the user to +configurate redis to their specified needs without have to manage or edit +the redis container. When the server is run this configuration will be dynamically +read, so settings can be changed between runs if needed. + +Once the merlin server has been initialized in the local directory the user will be allowed +to run other merlin server commands such as "run, status, stop" to interact with the +merlin server. A detailed list of commands can be found in the `Merlin Server Commands <./server/commands.html>`_ page. + +Note: Running "merlin server init" again will NOT override any exisiting configuration +that the users might have set or edited. By running this command again any missing files +will be created for the users with exisiting defaults. HOWEVER it is highly advised that +users back up their configuration in case an error occurs where configuration files are overriden. + +.. toctree:: + :maxdepth: 1 + :caption: Merlin Server Settings: + + server/configuration + server/commands diff --git a/docs/source/merlin_specification.rst b/docs/source/merlin_specification.rst index a84eae1a6..f857fabf3 100644 --- a/docs/source/merlin_specification.rst +++ b/docs/source/merlin_specification.rst @@ -48,6 +48,9 @@ see :doc:`./merlin_variables`. queue: pbatch flux_path: flux_start_opts: + flux_exec: flux_exec_workers: launch_pre: @@ -226,7 +229,7 @@ see :doc:`./merlin_variables`. #################################### # Merlin Block (Required) #################################### - # The merlin specific block will add any required configuration to + # The merlin specific block will add any configuration to # the DAG created by the study description. # including task server config, data management and sample definitions. # @@ -274,7 +277,7 @@ see :doc:`./merlin_variables`. batch: type: local machines: [host3] - + ################################################### # Sample definitions # @@ -290,3 +293,54 @@ see :doc:`./merlin_variables`. cmd: | python $(SPECROOT)/make_samples.py -dims 2 -n 10 -outfile=$(INPUT_PATH)/samples.npy "[(1.3, 1.3, 'linear'), (3.3, 3.3, 'linear')]" level_max_dirs: 25 + + #################################### + # User Block (Optional) + #################################### + # The user block allows other variables in the workflow file to be propagated + # through to the workflow (including in variables .partial.yaml and .expanded.yaml). + # User block uses yaml anchors, which defines a chunk of configuration and use + # their alias to refer to that specific chunk of configuration elsewhere. + ####################################################################### + user: + study: + run: + hello: &hello_run + cmd: | + python3 $(HELLO) -outfile hello_world_output_$(MERLIN_SAMPLE_ID).json $(X0) $(X1) $(X2) + max_retries: 1 + collect: &collect_run + cmd: | + echo $(MERLIN_GLOB_PATH) + echo $(hello.workspace) + ls $(hello.workspace)/X2.$(X2)/$(MERLIN_GLOB_PATH)/hello_world_output_*.json > files_to_collect.txt + spellbook collect -outfile results.json -instring "$(cat files_to_collect.txt)" + translate: &translate_run + cmd: spellbook translate -input $(collect.workspace)/results.json -output results.npz -schema $(FEATURES) + learn: &learn_run + cmd: spellbook learn -infile $(translate.workspace)/results.npz + make_samples: &make_samples_run + cmd: spellbook make-samples -n $(N_NEW) -sample_type grid -outfile grid_$(N_NEW).npy + predict: &predict_run + cmd: spellbook predict -infile $(make_new_samples.workspace)/grid_$(N_NEW).npy -outfile prediction_$(N_NEW).npy -reg $(learn.workspace)/random_forest_reg.pkl + verify: &verify_run + cmd: | + if [[ -f $(learn.workspace)/random_forest_reg.pkl && -f $(predict.workspace)/prediction_$(N_NEW).npy ]] + then + touch FINISHED + exit $(MERLIN_SUCCESS) + else + exit $(MERLIN_SOFT_FAIL) + fi + python3: + run: &python3_run + cmd: | + print("OMG is this in python?") + print("Variable X2 is $(X2)") + shell: /usr/bin/env python3 + python2: + run: &python2_run + cmd: | + print "OMG is this in python2? Change is bad." + print "Variable X2 is $(X2)" + shell: /usr/bin/env python2 diff --git a/docs/source/merlin_variables.rst b/docs/source/merlin_variables.rst index b8da52cb6..7f545a4d2 100644 --- a/docs/source/merlin_variables.rst +++ b/docs/source/merlin_variables.rst @@ -31,59 +31,161 @@ The directory structure of merlin output looks like this: Reserved variables ------------------ .. list-table:: Study variables that Merlin uses. May be referenced within a specification file, but not reassigned or overridden. - - * - Variable - - Description - - Example Expansion - * - ``$(SPECROOT)`` - - Directory path of the specification file. - - ``/globalfs/user/merlin_workflows`` - * - ``$(OUTPUT_PATH)`` - - Directory path the study output will be written to. If not defined - will default to the current working directory. May be reassigned or - overridden. - - ``./studies`` - * - ``$(MERLIN_TIMESTAMP)`` - - The time a study began. May be used as a unique identifier. - - ``"YYYYMMDD-HHMMSS"`` - * - ``$(MERLIN_WORKSPACE)`` - - Output directory generated by a study at ``OUTPUT_PATH``. Ends with - ``MERLIN_TIMESTAMP``. - - ``$(OUTPUT_PATH)/ensemble_name_$(MERLIN_TIMESTAMP)`` - * - ``$(WORKSPACE)`` - - The workspace directory for a single step. - - ``$(OUTPUT_PATH)/ensemble_name_$(MERLIN_TIMESTAMP)/step_name/`` - * - ``$(MERLIN_INFO)`` - - Directory within ``MERLIN_WORKSPACE`` that holds the provenance specs and sample generation results. - Commonly used to hold ``samples.npy``. - - ``$(MERLIN_WORKSPACE)/merlin_info/`` - * - ``$(MERLIN_SAMPLE_ID)`` - - Sample index in an ensemble - - ``0`` ``1`` ``2`` ``3`` - * - ``$(MERLIN_SAMPLE_PATH)`` - - Path in the sample directory tree to a sample's directory, i.e. where the - task is actually run. - - ``/0/0/0/`` ``/0/0/1/`` ``/0/0/2/`` ``/0/0/3/`` - * - ``$(MERLIN_GLOB_PATH)`` - - All of the directories in a simulation tree as a glob (*) string - - ``/\*/\*/\*/\*`` - * - ``$(MERLIN_PATHS_ALL)`` - - A space delimited string of all of the paths; - can be used as is in bash for loop for instance with: - - .. code-block:: bash - - for path in $(MERLIN_PATHS_ALL) - do - ls $path - done - - for path in $(MERLIN_PATHS_ALL) - do - ls $path - done - - ``0/0/0 0/0/1 0/0/2 0/0/3`` - + :widths: 25 50 25 + :header-rows: 1 + + * - Variable + - Description + - Example Expansion + + * - ``$(SPECROOT)`` + - Directory path of the specification file. + - + :: + + /globalfs/user/merlin_workflows + + * - ``$(OUTPUT_PATH)`` + - Directory path the study output will be written to. If not defined + will default to the current working directory. May be reassigned or + overridden. + - + :: + + ./studies + + * - ``$(MERLIN_TIMESTAMP)`` + - The time a study began. May be used as a unique identifier. + - + :: + + "YYYYMMDD-HHMMSS" + + * - ``$(MERLIN_WORKSPACE)`` + - Output directory generated by a study at ``OUTPUT_PATH``. Ends with + ``MERLIN_TIMESTAMP``. + - + :: + + $(OUTPUT_PATH)/ensemble_name_$(MERLIN_TIMESTAMP) + + * - ``$(WORKSPACE)`` + - The workspace directory for a single step. + - + :: + + $(OUTPUT_PATH)/ensemble_name_$(MERLIN_TIMESTAMP)/step_name/`` + + * - ``$(MERLIN_INFO)`` + - Directory within ``MERLIN_WORKSPACE`` that holds the provenance specs and sample generation results. + Commonly used to hold ``samples.npy``. + - + :: + + $(MERLIN_WORKSPACE)/merlin_info/ + + * - ``$(MERLIN_SAMPLE_ID)`` + - Sample index in an ensemble + - + :: + + 0 1 2 3 + + * - ``$(MERLIN_SAMPLE_PATH)`` + - Path in the sample directory tree to a sample's directory, i.e. where the + task is actually run. + - + :: + + /0/0/0/ /0/0/1/ /0/0/2/ /0/0/3/ + + * - ``$(MERLIN_GLOB_PATH)`` + - All of the directories in a simulation tree as a glob (*) string + - + :: + + /*/*/*/* + + * - ``$(MERLIN_PATHS_ALL)`` + - A space delimited string of all of the paths; + can be used as is in bash for loop for instance with: + + .. code-block:: bash + + for path in $(MERLIN_PATHS_ALL) + do + ls $path + done + + - + :: + + 0/0/0 + 0/0/1 + 0/0/2 + 0/0/3 + + * - ``$(MERLIN_SAMPLE_VECTOR)`` + - Vector of merlin sample values + - + :: + + $(SAMPLE_COLUMN_1) $(SAMPLE_COLUMN_2) ... + + * - ``$(MERLIN_SAMPLE_NAMES)`` + - Names of merlin sample values + - + :: + + SAMPLE_COLUMN_1 SAMPLE_COLUMN_2 ... + + * - ``$(MERLIN_SPEC_ORIGINAL_TEMPLATE)`` + - Copy of original yaml file passed to ``merlin run``. + - + :: + + $(MERLIN_INFO)/*.orig.yaml + + * - ``$(MERLIN_SPEC_EXECUTED_RUN)`` + - Parsed and processed yaml file with command-line variable substitutions included. + - + :: + + $(MERLIN_INFO)/*.partial.yaml + + * - ``$(MERLIN_SPEC_ARCHIVED_COPY)`` + - Archive version of ``MERLIN_SPEC_EXECUTED_RUN`` with all variables and paths fully resolved. + - + :: + + $(MERLIN_INFO)/*.expanded.yaml + +The ``LAUNCHER`` Variable ++++++++++++++++++++++ + +``$(LAUNCHER)`` is a special case of a reserved variable since it's value *can* be changed. +It serves as an abstraction to launch a job with parallel schedulers like :ref:`slurm`, +:ref:`lsf`, and :ref:`flux` and it can be used within a step command. For example, +say we start with this run cmd inside our step: + +.. code:: yaml + + run: + cmd: srun -N 1 -n 3 python script.py + +We can modify this to use the ``$(LAUNCHER)`` variable like so: + +.. code:: yaml + + batch: + type: slurm + + run: + cmd: $(LAUNCHER) python script.py + nodes: 1 + procs: 3 + +In other words, the ``$(LAUNCHER)`` variable would become ``srun -N 1 -n 3``. User variables ------------------- diff --git a/docs/source/merlin_workflows.rst b/docs/source/merlin_workflows.rst index c06976438..42cb7a39a 100644 --- a/docs/source/merlin_workflows.rst +++ b/docs/source/merlin_workflows.rst @@ -8,7 +8,7 @@ provides documentation on running these Merlin workflow examples. Overview -------- -List the built-in Merlin workflow examples with ``merlin example --help``. +List the built-in Merlin workflow examples with ``merlin example list``. The Merlin team is working on adding a more diverse array of example workflows like these. diff --git a/docs/source/modules/before.rst b/docs/source/modules/before.rst index e9a886548..dab1c8e2c 100644 --- a/docs/source/modules/before.rst +++ b/docs/source/modules/before.rst @@ -8,23 +8,40 @@ start the tutorial modules: __ https://www.python.org/downloads/release/python-360/ +* Make sure you have `pip`__ version 22.3 or newer. + +__ https://www.pypi.org/project/pip/ + + * You can upgrade pip to the latest version with: + + .. code-block:: bash + + pip install --upgrade pip + + * OR you can upgrade to a specific version with: + + .. code-block:: bash + + pip install --upgrade pip==x.y.z + + * Make sure you have `GNU make tools`__ and `compilers`__. __ https://www.gnu.org/software/make/ __ https://gcc.gnu.org/ -* Install `docker`__. +* (OPTIONAL) Install `docker`__. __ https://docs.docker.com/install/ -* Download OpenFOAM image with: + * Download OpenFOAM image with: -.. code-block:: bash + .. code-block:: bash - docker pull cfdengine/openfoam + docker pull cfdengine/openfoam -* Download redis image with: + * Download redis image with: -.. code-block:: bash + .. code-block:: bash - docker pull redis + docker pull redis diff --git a/docs/source/modules/contribute.rst b/docs/source/modules/contribute.rst index 8e8cde7af..acf35d323 100644 --- a/docs/source/modules/contribute.rst +++ b/docs/source/modules/contribute.rst @@ -17,16 +17,16 @@ Issues Found a bug? Have an idea? Or just want to ask a question? `Create a new issue `_ on GitHub. -Bug Reports 🐛 --------------- +Bug Reports +----------- To report a bug, simply navigate to `Issues `_, click "New Issue", then click "Bug report". Then simply fill out a few fields such as "Describe the bug" and "Expected behavior". Try to fill out every field as it will help us figure out your bug sooner. -Feature Requests 🚀 -------------------- +Feature Requests +---------------- We are still adding new features to merlin. To suggest one, simply navigate to `Issues `_, click "New Issue", then click "Feature request". Then fill out a few fields such as "What problem is this feature looking to solve?" -Questions 🤓 ------------- +Questions +--------- .. note:: Who knows? Your question may already be answered in the :doc:`FAQ<../faq>`. @@ -44,3 +44,5 @@ Contributing to Merlin is easy! Just `send us a pull request `. diff --git a/docs/source/modules/hello_world/hello_world.rst b/docs/source/modules/hello_world/hello_world.rst index 31ea31976..2cec6f05c 100644 --- a/docs/source/modules/hello_world/hello_world.rst +++ b/docs/source/modules/hello_world/hello_world.rst @@ -20,9 +20,15 @@ This hands-on module walks through the steps of building and running a simple me .. contents:: Table of Contents: :local: -Get example files +Get Example Files +++++++++++++++++ -``merlin example`` is a command line tool that makes it easy to get a basic workflow up and running. Run the following commands: +``merlin example`` is a command line tool that makes it easy to get a basic workflow up and running. To see a list of all the examples provided with merlin you can run: + +.. code-block:: bash + + $ merlin example list + +For this tutorial we will be using the ``hello`` example. Run the following commands: .. code-block:: bash @@ -44,7 +50,7 @@ This will create and move into directory called ``hello``, which contains these * ``requirements.txt`` -- this is a text file listing this workflow's python dependencies. -Specification file +Specification File ++++++++++++++++++ Central to Merlin is something called a specification file, or a "spec" for short. @@ -97,7 +103,7 @@ So this will give us 1) an English result, and 2) a Spanish one (you could add a Section: ``study`` ~~~~~~~~~~~~~~~~~~ This is where you define workflow steps. -While the convention is to list steps as sequentially as possible, the only factor in determining step order is the dependency DAG created by the ``depends`` field. +While the convention is to list steps as sequentially as possible, the only factor in determining step order is the dependency directed acyclic graph (DAG) created by the ``depends`` field. .. code-block:: yaml @@ -163,7 +169,7 @@ The order of the spec sections doesn't matter. At this point, ``my_hello.yaml`` is still maestro-compatible. The primary difference is that maestro won't understand anything in the ``merlin`` block, which we will still add later. If you want to try it, run: ``$ maestro run my_hello.yaml`` -Try it! +Try It! +++++++ First, we'll run merlin locally. On the command line, run: @@ -200,7 +206,7 @@ A lot of stuff, right? Here's what it means: .. Assuming config is ready -Run distributed! +Run Distributed! ++++++++++++++++ .. important:: @@ -234,6 +240,12 @@ Immediately after that, this will pop up: .. literalinclude :: celery.txt :language: text +You may not see all of the info logs listed after the Celery C is displayed. If you'd like to see them you can change the merlin workers' log levels with the ``--worker-args`` tag: + +.. code-block:: bash + + $ merlin run-workers --worker-args "-l INFO" my_hello.yaml + The terminal you ran workers in is now being taken over by Celery, the powerful task queue library that merlin uses internally. The workers will continue to report their task status here until their tasks are complete. Workers are persistent, even after work is done. Send a stop signal to all your workers with this command: @@ -249,7 +261,7 @@ Workers are persistent, even after work is done. Send a stop signal to all your .. _Using Samples: -Using samples +Using Samples +++++++++++++ It's a little boring to say "hello world" in just two different ways. Let's instead say hello to many people! @@ -283,10 +295,12 @@ This makes ``N_SAMPLES`` into a user-defined variable that you can use elsewhere file: $(MERLIN_INFO)/samples.csv column_labels: [WORLD] -This is the merlin block, an exclusively merlin feature. It provides a way to generate samples for your workflow. In this case, a sample is the name of a person. +This is the merlin block, an exclusively merlin feature. It provides a way to generate samples for your workflow. In this case, a sample is the name of a person. For simplicity we give ``column_labels`` the name ``WORLD``, just like before. +It's also important to note that ``$(SPECROOT)`` and ``$(MERLIN_INFO)`` are reserved variables. The ``$(SPECROOT)`` variable is a shorthand for the directory path of the spec file and the ``$(MERLIN_INFO)`` variable is a shorthand for the directory holding the provenance specs and sample generation results. More information on Merlin variables can be found on the :doc:`variables page<../../merlin_variables>`. + It's good practice to shift larger chunks of code to external scripts. At the same location of your spec, make a new file called ``make_samples.py``: .. literalinclude :: ../../../../merlin/examples/workflows/hello/make_samples.py diff --git a/docs/source/modules/installation/installation.rst b/docs/source/modules/installation/installation.rst index 2eb1ac95d..d18261af5 100644 --- a/docs/source/modules/installation/installation.rst +++ b/docs/source/modules/installation/installation.rst @@ -6,9 +6,9 @@ Installation * python3 >= python3.6 * pip3 * wget - * build tools (make, C/C++ compiler for local-redis) - * docker (required for :doc:`Module 4: Run a Real Simulation<../run_simulation/run_simulation>`) - * file editor for docker config file editing + * build tools (make, C/C++ compiler) + * (OPTIONAL) docker (required for :doc:`Module 4: Run a Real Simulation<../run_simulation/run_simulation>`) + * (OPTIONAL) file editor for docker config file editing .. admonition:: Estimated time @@ -17,9 +17,7 @@ Installation .. admonition:: You will learn * How to install merlin in a virtual environment using pip. - * How to install a local redis server. - * How to install merlin using docker (optional). - * How to start the docker containers, including redis (optional). + * How to install a container platform eg. singularity, docker, or podman. * How to configure merlin. * How to test/verify the installation. @@ -27,27 +25,21 @@ Installation :local: This section details the steps necessary to install merlin and its dependencies. -Merlin will then be configured and this configuration checked to ensure a proper installation. +Merlin will then be configured for the local machine and the configuration +will be checked to ensure a proper installation. -Installing merlin +Installing Merlin ----------------- -A merlin installation is required for the subsequent modules of this tutorial. You can choose between the pip method or the docker method. Choose one or the other but -do not use both unless you are familiar with redis servers run locally and through docker. -**The pip method is recommended.** +A merlin installation is required for the subsequent modules of this tutorial. -Once merlin is installed, it requires servers to operate. -The pip section will inform you how to setup a -local redis server to use in merlin. An alternative method for setting up a -redis server can be found in the docker section. Only setup one redis server either -local-redis or docker-redis. -Your computer/organization may already have a redis server available, please check +Once merlin is installed, it requires servers to operate. While you are able to host your own servers, +we will use merlin's containerized servers in this tutorial. However, if you prefer to host your own servers +you can host a redis server that is accessible to your current machine. +Your computer/organization may already have a redis server available you can use, please check with your local system administrator. -Pip (recommended) -+++++++++++++++++ - Create a virtualenv using python3 to install merlin. .. code-block:: bash @@ -77,176 +69,135 @@ Install merlin through pip. pip3 install merlin -When you are done with the virtualenv you can deactivate it using ``deactivate``, -but leave the virtualenv activated for the subsequent steps. +Check to make sure merlin installed correctly. .. code-block:: bash - deactivate + which merlin - -redis local server -^^^^^^^^^^^^^^^^^^ - -A redis server is required for the celery results backend server, this same server -can also be used for the celery broker. This method will be called local-redis. +You should see that it was installed in your virtualenv, like so: .. code-block:: bash - # Download redis - wget http://download.redis.io/releases/redis-6.0.5.tar.gz + ~//merlin_venv/bin/merlin - # Untar - tar xvf redis*.tar.gz +If this is not the output you see, you may need to restart your virtualenv and try again. - # cd into redis dir - cd redis*/ - - # make redis - make - - # make test (~3.5 minutes) - make test +When you are done with the virtualenv you can deactivate it using ``deactivate``, +but leave the virtualenv activated for the subsequent steps. +.. code-block:: bash -The redis server is started by calling the ``redis-server`` command located in -the src directory. -This should be run in a separate terminal in the top-level source -directory so the output can be examined. -The redis server will use the default ``redis.conf`` file in the top-level -redis directory. + deactivate -.. code:: bash - # run redis with default config, server is at localhost port 6379 - ./src/redis-server & +Redis Server +++++++++++++ -You can shutdown the local-redis server by using the ``redis-cli shutdown`` command -when you are done with the tutorial. +A redis server is required for the celery results backend server, this same server +can also be used for the celery broker. We will be using merlin's containerized server +however we will need to download one of the supported container platforms avaliable. For +the purpose of this tutorial we will be using singularity. .. code-block:: bash - #cd to redis directory - cd /redis*/ - ./src/redis-cli shutdown - - -Docker -++++++ - -Merlin and the servers required by merlin are all available as docker containers on dockerhub. Do not use this method if you have already set up a virtualenv through -the pip installation method. - -.. note:: + # Update and install singularity dependencies + apt-get update && apt-get install -y \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config + + # Download dependency go + wget https://go.dev/dl/go1.18.1.linux-amd64.tar.gz + + # Extract go into local + tar -C /usr/local -xzf go1.18.1.linux-amd64.tar.gz + + # Remove go tar file + rm go1.18.1.linux-amd64.tar.gz + + # Update PATH to include go + export PATH=$PATH:/usr/local/go/bin + + # Download singularity + wget https://github.com/sylabs/singularity/releases/download/v3.9.9/singularity-ce-3.9.9.tar.gz + + # Extract singularity + tar -xzf singularity-ce-3.9.9.tar.gz + + # Configure and install singularity + cd singularity-ce-3.9.9 + ./mconfig && \ + make -C ./builddir && \ + sudo make -C ./builddir install + +Configuring Merlin +------------------ +Merlin requires a configuration script for the celery interface. +Run this configuration method to create the ``app.yaml`` +configuration file. - When using the docker method the celery workers will run inside the - merlin container. This - means that any workflow tools that are also from docker containers must - be installed in, or - otherwise made available to, the merlin container. +.. code-block:: bash + merlin config --broker redis -To run a merlin docker container with a docker redis server, cut -and paste the commands below into a new file called ``docker-compose.yml``. -This file can be placed anywhere in your filesystem but you may want to put it in -a directory ``merlin_docker_redis``. +The ``merlin config`` command above will create a file called ``app.yaml`` +in the ``~/.merlin`` directory. +If you are running a redis server locally then you are all set, look in the ``~/.merlin/app.yaml`` file +to see the configuration, it should look like the configuration below. -.. literalinclude:: ./docker-compose.yml +.. literalinclude:: ./app_local_redis.yaml :language: yaml -This file can then be run with the ``docker-compose`` command in same directory -as the ``docker-compose.yml`` file. - -.. code-block:: bash - - docker-compose up -d - -The ``volume`` option in the ``docker-compose.yml`` file -will link the local ``$HOME/merlinu`` directory to the ``/home/merlinu`` -directory in the container. +More detailed information on configuring Merlin can be found in the :doc:`configuration section<../../merlin_config>`. -Some aliases can be defined for convenience. - -.. code-block:: bash +.. _Verifying installation: - # define some aliases for the merlin and celery commands (assuming Bourne shell) - alias merlin="docker exec my-merlin merlin" - alias celery="docker exec my-merlin celery" - alias python3="docker exec my-merlin python3" +Checking/Verifying Installation +------------------------------- -When you are done with the containers you can stop them using ``docker-compose down``. -We will be using the containers in the subsequent modules so leave them running. +First launch the merlin server containers by using the ``merlin server`` commands .. code-block:: bash - docker-compose down + merlin server init + merlin server start -Any required python modules can be installed in the running ``my-merlin`` container -through ``docker exec``. When using docker-compose, these changes will persist -if you stop the containers with ``docker-compose down`` and restart them with -``docker-compose up -d``. +A subdirectory called ``merlin_server/`` will have been created in the current run directory. +This contains all of the proper configuration for the server containers merlin creates. +Configuration can be done through the ``merlin server config`` command, however users have +the flexibility to edit the files directly in the directory. Additionally an preconfigured ``app.yaml`` +file has been created in the ``merlin_server/`` subdirectory to utilize the merlin server +containers . To use it locally simply copy it to the run directory with a cp command. .. code-block:: bash - docker exec my-merlin pip3 install pandas faker - -Configuring merlin ------------------- - -Merlin configuration is slightly different between the pip and docker methods. -The fundamental differences include the app.yaml file location and the server name. + cp ./merlin_server/app.yaml . -Merlin requires a configuration script for the celery interface and optional -passwords for the redis server and encryption. Run this configuration method -to create the ``app.yaml`` configuration file. +You can also make this server container your main server configuration by replacing the one located in your home +directory. Make sure you make back-ups of your current app.yaml file in case you want to use your previous +configurations. Note: since merlin servers are created locally on your run directory you are allowed to create +multiple instances of merlin server with their unique configurations for different studies. Simply create different +directories for each study and run ``merlin server init`` in each directory to create an instance for each. .. code-block:: bash - merlin config --broker redis - -Pip -+++ - -The ``merlin config`` command above will create a file called ``app.yaml`` -in the ``~/.merlin`` directory. -If you are using local-redis then you are all set, look in the ``~/.merlin/app.yaml`` file -to see the configuration, it should look like the configuration below. - -.. literalinclude:: ./app_local_redis.yaml - :language: yaml - -Docker -++++++ - -If you are using the docker merlin with docker-redis server then the -``~/merlinu/.merlin/app.yaml`` will be created by the ``merlin config`` -command above. -This file must be edited to -add the server from the redis docker container my-redis. Change the ``server: localhost``, in both the -broker and backend config definitions, to ``server: my-redis``, the port will remain the same. - -.. note:: - You can use the docker redis server, instead of the local-redis server, - with the virtualenv installed merlin by using the local-redis - ``app.yaml`` file above. - -.. literalinclude:: ./app_docker_redis.yaml - :language: yaml - -.. _Verifying installation: - -Checking/Verifying installation -------------------------------- + mv ~/.merlin/app.yaml ~/.merlin/app.yaml.bak + cp ./merlin_server/app.yaml ~/.merlin/ The ``merlin info`` command will check that the configuration file is installed correctly, display the server configuration strings, and check server -access. This command works for both the pip and docker installed merlin. +access. .. code-block:: bash merlin info -If everything is set up correctly, you should see (assuming local-redis servers): +If everything is set up correctly, you should see: .. code-block:: bash @@ -277,10 +228,10 @@ If everything is set up correctly, you should see (assuming local-redis servers) . -Docker Advanced Installation +(OPTIONAL) Docker Advanced Installation ---------------------------- -RabbitMQ server +RabbitMQ Server +++++++++++++++ This optional section details the setup of a rabbitmq server for merlin. @@ -341,7 +292,7 @@ and add the password ``guest``. The aliases defined previously can be used with this set of docker containers. -Redis TLS server +Redis TLS Server ++++++++++++++++ This optional section details the setup of a redis server with TLS for merlin. diff --git a/docs/source/modules/port_your_application.rst b/docs/source/modules/port_your_application.rst index 26f2cc1d1..c9d89b06d 100644 --- a/docs/source/modules/port_your_application.rst +++ b/docs/source/modules/port_your_application.rst @@ -26,6 +26,7 @@ Tips for porting your app, building workflows The first step of building a new workflow, or porting an existing app to a workflow, is to describe it as a set of discrete, and ideally focused steps. Decoupling the steps and making them generic when possible will facilitate more rapid composition of future workflows. This will also require mapping out the dependencies and parameters that get passed between/shared across these steps. Setting up a template using tools such as `cookiecutter `_ can be useful for more production style workflows that will be frequently reused. Additionally, make use of the built-in examples accessible from the merlin command line with ``merlin example``. + .. (machine learning applications on different data sets?) Use dry runs ``merlin run --dry --local`` to prototype without actually populating task broker's queues. Similarly, once the dry run prototype looks good, try it on a small number of parameters before throwing millions at it. @@ -39,7 +40,7 @@ Make use of exit keys such as ``MERLIN_RESTART`` or ``MERLIN_RETRY`` in your ste Tips for debugging your workflows +++++++++++++++++++++++++++++++++ -The scripts defined in the workflow steps are also written to the output directories; this is a useful debugging tool as it can both catch parameter and variable replacement errors, as well as providing a quick way to reproduce, edit, and retry the step offline before fixing the step in the workflow specification. The ``.out`` and ``.err`` files log all of the output to catch any runtime errors. Additionally, you may need to grep for ``'WARNING'`` and ``'ERROR'`` in the worker logs. +The scripts defined in the workflow steps are also written to the output directories; this is a useful debugging tool as it can both catch parameter and variable replacement errors, as well as provide a quick way to reproduce, edit, and retry the step offline before fixing the step in the workflow specification. The ``.out`` and ``.err`` files log all of the output to catch any runtime errors. Additionally, you may need to grep for ``'WARNING'`` and ``'ERROR'`` in the worker logs. .. where are the worker logs, and what might show up there that .out and .err won't see? -> these more developer focused output? diff --git a/docs/source/modules/run_simulation/run_simulation.rst b/docs/source/modules/run_simulation/run_simulation.rst index a30568946..f48d7dc97 100644 --- a/docs/source/modules/run_simulation/run_simulation.rst +++ b/docs/source/modules/run_simulation/run_simulation.rst @@ -8,7 +8,7 @@ Run a Real Simulation .. admonition:: Prerequisites - * :doc:`Module 0: Before you come<../before>` + * :doc:`Module 0: Before you start<../before>` * :doc:`Module 2: Installation<../installation/installation>` * :doc:`Module 3: Hello World<../hello_world/hello_world>` @@ -53,7 +53,9 @@ This module will be going over: * Combining the outputs of these simulations into a an array * Predictive modeling and visualization -Before moving on, +.. _Before Moving On: + +Before Moving On ~~~~~~~~~~~~~~~~~ check that the virtual environment with merlin installed is activated @@ -65,14 +67,25 @@ and that redis server is set up using this command: This is covered more in depth here: :ref:`Verifying installation` - -Then use the ``merlin example`` to get the necessary files for this module. - +There are two ways to do this example: with docker and without docker. To go through the version with docker, get the necessary files for this module by running: + .. code-block:: bash $ merlin example openfoam_wf $ cd openfoam_wf/ + +For the version without docker you should run: + +.. code-block:: bash + + $ merlin example openfoam_wf_no_docker + + $ cd openfoam_wf_no_docker/ + +.. note:: + + From here on, this tutorial will focus solely on the docker version of running openfoam. However, the docker version of this tutorial is almost identical to the no docker version. If you're using the no docker version of this tutorial you can still follow along but check the openfoam_no_docker_template.yaml file in each step to see what differs. In the ``openfoam_wf`` directory you should see the following: diff --git a/docs/source/server/commands.rst b/docs/source/server/commands.rst new file mode 100644 index 000000000..dd8ca1b02 --- /dev/null +++ b/docs/source/server/commands.rst @@ -0,0 +1,87 @@ +Merlin Server Commands +====================== + +Merlin server has a list of commands for interacting with the broker and results server. +These commands allow the user to manage and monitor the exisiting server and create +instances of servers if needed. + +Initializing Merlin Server (``merlin server init``) +--------------------------------------------------- +The merlin server init command creates configurations for merlin server commands. + +A main merlin sever configuration subdirectory is created in "~/.merlin/server" which contains +configuration for local merlin configuration, and configurations for different containerized +services that merlin server supports, which includes singularity (docker and podman implemented +in the future). + +A local merlin server configuration subdirectory called "merlin_server/" will also +be created when this command is run. This will contain a container for merlin server and associated +configuration files that might be used to start the server. For example, for a redis server a "redis.conf" +will contain settings which will be dynamically loaded when the redis server is run. This local configuration +will also contain information about currently running containers as well. + +Note: If there is an exisiting subdirectory containing a merlin server configuration then only +missing files will be replaced. However it is recommended that users backup their local configurations. + + +Checking Merlin Server Status (``merlin server status``) +-------------------------------------------------------- + +Displays the current status of the merlin server. + +Starting up a Merlin Server (``merlin server start``) +----------------------------------------------------- + +Starts the container located in the local merlin server configuration. + +Stopping an exisiting Merlin Server (``merlin server stop``) +------------------------------------------------------------ + +Stop any exisiting container being managed and monitored by merlin server. + +Restarting a Merlin Server instance (``merlin server restart``) +--------------------------------------------------------------- + +Restarting an existing container that is being managed and monitored by merlin server. + +Configurating Merlin Server instance (``merlin server config``) +--------------------------------------------------------------- +Place holder for information regarding merlin server config command + +Possible Flags + +.. code-block:: none + + -ip IPADDRESS, --ipaddress IPADDRESS + Set the binded IP address for the merlin server + container. (default: None) + -p PORT, --port PORT Set the binded port for the merlin server container. + (default: None) + -pwd PASSWORD, --password PASSWORD + Set the password file to be used for merlin server + container. (default: None) + --add-user ADD_USER ADD_USER + Create a new user for merlin server instance. (Provide + both username and password) (default: None) + --remove-user REMOVE_USER + Remove an exisiting user. (default: None) + -d DIRECTORY, --directory DIRECTORY + Set the working directory of the merlin server + container. (default: None) + -ss SNAPSHOT_SECONDS, --snapshot-seconds SNAPSHOT_SECONDS + Set the number of seconds merlin server waits before + checking if a snapshot is needed. (default: None) + -sc SNAPSHOT_CHANGES, --snapshot-changes SNAPSHOT_CHANGES + Set the number of changes that are required to be made + to the merlin server before a snapshot is made. + (default: None) + -sf SNAPSHOT_FILE, --snapshot-file SNAPSHOT_FILE + Set the snapshot filename for database dumps. + (default: None) + -am APPEND_MODE, --append-mode APPEND_MODE + The appendonly mode to be set. The avaiable options + are always, everysec, no. (default: None) + -af APPEND_FILE, --append-file APPEND_FILE + Set append only filename for merlin server container. + (default: None) + diff --git a/docs/source/server/configuration.rst b/docs/source/server/configuration.rst new file mode 100644 index 000000000..84429c079 --- /dev/null +++ b/docs/source/server/configuration.rst @@ -0,0 +1,75 @@ +Merlin Server Configuration +=========================== + +Below are a sample list of configurations for the merlin server command + +Main Configuration ``~/.merlin/server/`` +---------------------------------------- + +merlin_server.yaml + +.. code-block:: yaml + + container: + # Select the format for the recipe e.g. singularity, docker, podman (currently singularity is the only working option.) + format: singularity + # The image name + image: redis_latest.sif + # The url to pull the image from + url: docker://redis + # The config file + config: redis.conf + # Subdirectory name to store configurations Default: merlin_server/ + config_dir: merlin_server/ + # Process file containing information regarding the redis process + pfile: merlin_server.pf + + process: + # Command for determining the process of the command + status: pgrep -P {pid} #ps -e | grep {pid} + # Command for killing process + kill: kill {pid} + + +singularity.yaml + +.. code-block:: yaml + + singularity: + command: singularity + # init_command: \{command} .. (optional or default) + run_command: \{command} run {image} {config} + stop_command: kill # \{command} (optional or kill default) + pull_command: \{command} pull {image} {url} + + +Local Configuration ``merlin_server/`` +-------------------------------------- + +redis.conf + +.. code-block:: yaml + + bind 127.0.0.1 -::1 + protected-mode yes + port 6379 + logfile "" + dir ./ + ... + +see documentation on redis configuration `here `_ for more detail + +merlin_server.pf + +.. code-block:: yaml + + bits: '64' + commit: '00000000' + hostname: ubuntu + image_pid: '1111' + mode: standalone + modified: '0' + parent_pid: 1112 + port: '6379' + version: 6.2.6 + diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 9f90a0b0d..0b69f9553 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -20,7 +20,7 @@ Finally we offer some tips and tricks for porting and scaling up your applicatio .. toctree:: :maxdepth: 1 - :caption: Before you come: + :caption: Before you begin: modules/before diff --git a/lgtm.yml b/lgtm.yml new file mode 100644 index 000000000..e3f53c87d --- /dev/null +++ b/lgtm.yml @@ -0,0 +1,25 @@ +########################################################################################## +# Customize file classifications. # +# Results from files under any classifier will be excluded from LGTM # +# statistics. # +########################################################################################## + +########################################################################################## +# Use the `path_classifiers` block to define changes to the default classification of # +# files. # +########################################################################################## + +path_classifiers: + test: + # Classify all files in the top-level directories tests/ as test code. + - exclude: + - tests + - merlin/examples + +######################################################################################### +# Use the `queries` block to change the default display of query results. # +######################################################################################### + +queries: + # Specifically hide the results of clear-text-logging-sensitive-data + - exclude: py/clear-text-logging-sensitive-data diff --git a/merlin/__init__.py b/merlin/__init__.py index fa68134d6..aa33bc4d2 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.8.5" +__version__ = "1.9.0" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index de04bfc78..0b5971627 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index 3b8769bb5..ebac67eec 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 13db3dccc..7155d0c5f 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 13db3dccc..7155d0c5f 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index fa4a5f7c1..b02f9e909 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 814fb1881..26c64e9e2 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index a8be89486..8d9a89285 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index cb2a221d8..1859a55df 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index c3af9cdb6..6f3e58e9a 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 13db3dccc..7155d0c5f 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index af29d394f..e378573c4 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 8adc75228..16365e32c 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index f36134fec..9820e1041 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index bd2795915..027bcd291 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index ebbdc662e..7ade90e05 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 72cd6ec27..a546591f1 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index bf58602d5..a8c0a1ef2 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -2,6 +2,36 @@ Default celery configuration for merlin """ +############################################################################### +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### + from merlin.log_formatter import FORMATS diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index cc3dedd13..5dd08ddfb 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 843930cac..335bd05f5 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 12f7283bb..cd47be750 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -1,3 +1,33 @@ +############################################################################### +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### + import enum from typing import List diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 13db3dccc..7155d0c5f 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index 1cd9af01e..3b1469f70 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # @@ -88,7 +88,12 @@ def check_server_access(sconf): def _examine_connection(s, sconf, excpts): connect_timeout = 60 try: - conn = Connection(sconf[s]) + ssl_conf = None + if "broker" in s: + ssl_conf = broker.get_ssl_config() + if "results" in s: + ssl_conf = results_backend.get_ssl_config() + conn = Connection(sconf[s], ssl=ssl_conf) conn_check = ConnProcess(target=conn.connect) conn_check.start() counter = 0 diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 13db3dccc..7155d0c5f 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 686f04013..39c5cf2e0 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index d0cf1acb9..d59fc8511 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/workflows/feature_demo/feature_demo.yaml b/merlin/examples/workflows/feature_demo/feature_demo.yaml index b4bc1ca46..b07107e7b 100644 --- a/merlin/examples/workflows/feature_demo/feature_demo.yaml +++ b/merlin/examples/workflows/feature_demo/feature_demo.yaml @@ -17,15 +17,33 @@ env: HELLO: $(SCRIPTS)/hello_world.py FEATURES: $(SCRIPTS)/features.json +user: + study: + run: + hello: &hello_run + cmd: | + python3 $(HELLO) -outfile hello_world_output_$(MERLIN_SAMPLE_ID).json $(X0) $(X1) $(X2) + max_retries: 1 + python3: + run: &python3_run + cmd: | + print("OMG is this in python?") + print("Variable X2 is $(X2)") + shell: /usr/bin/env python3 + python2: + run: &python2_run + cmd: | + print "OMG is this in python2? Change is bad." + print "Variable X2 is $(X2)" + shell: /usr/bin/env python2 + study: - name: hello description: | process a sample with hello world run: - cmd: | - python3 $(HELLO) -outfile hello_world_output_$(MERLIN_SAMPLE_ID).json $(X0) $(X1) $(X2) + <<: *hello_run task_queue: hello_queue - max_retries: 1 - name: collect description: | @@ -89,20 +107,14 @@ study: description: | do something in python run: - cmd: | - print("OMG is this in python?") - print("Variable X2 is $(X2)") - shell: /usr/bin/env python3 + <<: *python3_run task_queue: pyth3_q - name: python2_hello description: | do something in python2, because change is bad run: - cmd: | - print "OMG is this in python2? Change is bad." - print "Variable X2 is $(X2)" - shell: /usr/bin/env python2 + <<: *python2_run task_queue: pyth2_hello global.parameters: diff --git a/merlin/examples/workflows/feature_demo/scripts/hello_world.py b/merlin/examples/workflows/feature_demo/scripts/hello_world.py index ab14bedf4..634dfe417 100644 --- a/merlin/examples/workflows/feature_demo/scripts/hello_world.py +++ b/merlin/examples/workflows/feature_demo/scripts/hello_world.py @@ -1,5 +1,6 @@ import argparse import json +import sys def process_args(args): @@ -25,9 +26,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/flux/flux_par.yaml b/merlin/examples/workflows/flux/flux_par.yaml index d401a0270..1fee4131e 100644 --- a/merlin/examples/workflows/flux/flux_par.yaml +++ b/merlin/examples/workflows/flux/flux_par.yaml @@ -6,6 +6,7 @@ batch: type: flux nodes: 1 queue: pbatch + flux_exec: flux exec -r "0-1" flux_start_opts: -o,-S,log-filename=flux_par.out env: diff --git a/merlin/examples/workflows/flux/scripts/make_samples.py b/merlin/examples/workflows/flux/scripts/make_samples.py index e6c807bc9..8ec1c7e2f 100644 --- a/merlin/examples/workflows/flux/scripts/make_samples.py +++ b/merlin/examples/workflows/flux/scripts/make_samples.py @@ -1,5 +1,6 @@ import argparse import ast +import sys import numpy as np @@ -51,9 +52,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py b/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py index 7d06ab594..43b732ae2 100644 --- a/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py +++ b/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py @@ -1,5 +1,6 @@ import argparse import os +import sys from concurrent.futures import ProcessPoolExecutor import matplotlib.pyplot as plt @@ -55,45 +56,50 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() + try: + parser = setup_argparse() + args = parser.parse_args() - # Load all iterations' data into single pandas dataframe for further analysis - all_iter_df = load_samples(args.sample_file_paths, args.np) + # Load all iterations' data into single pandas dataframe for further analysis + all_iter_df = load_samples(args.sample_file_paths, args.np) - # PLOTS: - # counts vs index for each iter range (1, [1,2], [1-3], [1-4], ...) - # num names vs iter - # median, min, max counts vs iter -> same plot - fig, ax = plt.subplots(nrows=2, ncols=1, constrained_layout=True, sharex=True) + # PLOTS: + # counts vs index for each iter range (1, [1,2], [1-3], [1-4], ...) + # num names vs iter + # median, min, max counts vs iter -> same plot + fig, ax = plt.subplots(nrows=2, ncols=1, constrained_layout=True, sharex=True) - iterations = sorted(all_iter_df.Iter.unique()) + iterations = sorted(all_iter_df.Iter.unique()) - max_counts = [] - min_counts = [] - med_counts = [] - unique_names = [] + max_counts = [] + min_counts = [] + med_counts = [] + unique_names = [] - for it in iterations: - max_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].max()) - min_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].min()) - med_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].median()) + for it in iterations: + max_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].max()) + min_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].min()) + med_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].median()) - unique_names.append(len(all_iter_df[all_iter_df["Iter"] <= it].index.value_counts())) + unique_names.append(len(all_iter_df[all_iter_df["Iter"] <= it].index.value_counts())) - ax[0].plot(iterations, min_counts, label="Minimum Occurances") - ax[0].plot(iterations, max_counts, label="Maximum Occurances") + ax[0].plot(iterations, min_counts, label="Minimum Occurances") + ax[0].plot(iterations, max_counts, label="Maximum Occurances") - ax[0].plot(iterations, med_counts, label="Median Occurances") + ax[0].plot(iterations, med_counts, label="Median Occurances") - ax[0].set_ylabel("Counts") - ax[0].legend() + ax[0].set_ylabel("Counts") + ax[0].legend() - ax[1].set_xlabel("Iteration") - ax[1].set_ylabel("Unique Names") - ax[1].plot(iterations, unique_names) + ax[1].set_xlabel("Iteration") + ax[1].set_ylabel("Unique Names") + ax[1].plot(iterations, unique_names) - fig.savefig(args.hardcopy, dpi=150) + fig.savefig(args.hardcopy, dpi=150) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/hpc_demo/faker_sample.py b/merlin/examples/workflows/hpc_demo/faker_sample.py index ee8bf2f5c..be16be5de 100644 --- a/merlin/examples/workflows/hpc_demo/faker_sample.py +++ b/merlin/examples/workflows/hpc_demo/faker_sample.py @@ -1,4 +1,5 @@ import argparse +import sys from faker import Faker @@ -31,9 +32,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/hpc_demo/sample_collector.py b/merlin/examples/workflows/hpc_demo/sample_collector.py index f62111e8e..ad06dc6c5 100644 --- a/merlin/examples/workflows/hpc_demo/sample_collector.py +++ b/merlin/examples/workflows/hpc_demo/sample_collector.py @@ -1,5 +1,6 @@ import argparse import os +import sys from concurrent.futures import ProcessPoolExecutor @@ -36,12 +37,17 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - - # Collect sample files into single file - sample_paths = [sample_path for sample_path in args.sample_file_paths] - serialize_samples(sample_paths, args.outfile, args.np) + try: + parser = setup_argparse() + args = parser.parse_args() + + # Collect sample files into single file + sample_paths = [sample_path for sample_path in args.sample_file_paths] + serialize_samples(sample_paths, args.outfile, args.np) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/hpc_demo/sample_processor.py b/merlin/examples/workflows/hpc_demo/sample_processor.py index 9ec0951e9..8523dcc80 100644 --- a/merlin/examples/workflows/hpc_demo/sample_processor.py +++ b/merlin/examples/workflows/hpc_demo/sample_processor.py @@ -1,6 +1,7 @@ import argparse import os import pathlib +import sys import pandas as pd @@ -28,25 +29,30 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - - # Collect the samples - samples = load_samples(args.sample_file_paths) - - # Count up the occurences - namesdf = pd.DataFrame({"Name": samples}) - - names = namesdf["Name"].value_counts() - - # Serialize processed samples - # create directory if it doesn't exist already - abspath = os.path.abspath(args.results) - absdir = os.path.dirname(abspath) - if not os.path.isdir(absdir): - pathlib.Path(absdir).mkdir(parents=True, exist_ok=True) - - names.to_json(args.results) + try: + parser = setup_argparse() + args = parser.parse_args() + + # Collect the samples + samples = load_samples(args.sample_file_paths) + + # Count up the occurences + namesdf = pd.DataFrame({"Name": samples}) + + names = namesdf["Name"].value_counts() + + # Serialize processed samples + # create directory if it doesn't exist already + abspath = os.path.abspath(args.results) + absdir = os.path.dirname(abspath) + if not os.path.isdir(absdir): + pathlib.Path(absdir).mkdir(parents=True, exist_ok=True) + + names.to_json(args.results) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py b/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py index 7d06ab594..43b732ae2 100644 --- a/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py +++ b/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py @@ -1,5 +1,6 @@ import argparse import os +import sys from concurrent.futures import ProcessPoolExecutor import matplotlib.pyplot as plt @@ -55,45 +56,50 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() + try: + parser = setup_argparse() + args = parser.parse_args() - # Load all iterations' data into single pandas dataframe for further analysis - all_iter_df = load_samples(args.sample_file_paths, args.np) + # Load all iterations' data into single pandas dataframe for further analysis + all_iter_df = load_samples(args.sample_file_paths, args.np) - # PLOTS: - # counts vs index for each iter range (1, [1,2], [1-3], [1-4], ...) - # num names vs iter - # median, min, max counts vs iter -> same plot - fig, ax = plt.subplots(nrows=2, ncols=1, constrained_layout=True, sharex=True) + # PLOTS: + # counts vs index for each iter range (1, [1,2], [1-3], [1-4], ...) + # num names vs iter + # median, min, max counts vs iter -> same plot + fig, ax = plt.subplots(nrows=2, ncols=1, constrained_layout=True, sharex=True) - iterations = sorted(all_iter_df.Iter.unique()) + iterations = sorted(all_iter_df.Iter.unique()) - max_counts = [] - min_counts = [] - med_counts = [] - unique_names = [] + max_counts = [] + min_counts = [] + med_counts = [] + unique_names = [] - for it in iterations: - max_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].max()) - min_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].min()) - med_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].median()) + for it in iterations: + max_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].max()) + min_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].min()) + med_counts.append(all_iter_df[all_iter_df["Iter"] <= it]["Count"].median()) - unique_names.append(len(all_iter_df[all_iter_df["Iter"] <= it].index.value_counts())) + unique_names.append(len(all_iter_df[all_iter_df["Iter"] <= it].index.value_counts())) - ax[0].plot(iterations, min_counts, label="Minimum Occurances") - ax[0].plot(iterations, max_counts, label="Maximum Occurances") + ax[0].plot(iterations, min_counts, label="Minimum Occurances") + ax[0].plot(iterations, max_counts, label="Maximum Occurances") - ax[0].plot(iterations, med_counts, label="Median Occurances") + ax[0].plot(iterations, med_counts, label="Median Occurances") - ax[0].set_ylabel("Counts") - ax[0].legend() + ax[0].set_ylabel("Counts") + ax[0].legend() - ax[1].set_xlabel("Iteration") - ax[1].set_ylabel("Unique Names") - ax[1].plot(iterations, unique_names) + ax[1].set_xlabel("Iteration") + ax[1].set_ylabel("Unique Names") + ax[1].plot(iterations, unique_names) - fig.savefig(args.hardcopy, dpi=150) + fig.savefig(args.hardcopy, dpi=150) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/iterative_demo/faker_sample.py b/merlin/examples/workflows/iterative_demo/faker_sample.py index ee8bf2f5c..be16be5de 100644 --- a/merlin/examples/workflows/iterative_demo/faker_sample.py +++ b/merlin/examples/workflows/iterative_demo/faker_sample.py @@ -1,4 +1,5 @@ import argparse +import sys from faker import Faker @@ -31,9 +32,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/iterative_demo/sample_collector.py b/merlin/examples/workflows/iterative_demo/sample_collector.py index f62111e8e..ad06dc6c5 100644 --- a/merlin/examples/workflows/iterative_demo/sample_collector.py +++ b/merlin/examples/workflows/iterative_demo/sample_collector.py @@ -1,5 +1,6 @@ import argparse import os +import sys from concurrent.futures import ProcessPoolExecutor @@ -36,12 +37,17 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - - # Collect sample files into single file - sample_paths = [sample_path for sample_path in args.sample_file_paths] - serialize_samples(sample_paths, args.outfile, args.np) + try: + parser = setup_argparse() + args = parser.parse_args() + + # Collect sample files into single file + sample_paths = [sample_path for sample_path in args.sample_file_paths] + serialize_samples(sample_paths, args.outfile, args.np) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/iterative_demo/sample_processor.py b/merlin/examples/workflows/iterative_demo/sample_processor.py index 9ec0951e9..8523dcc80 100644 --- a/merlin/examples/workflows/iterative_demo/sample_processor.py +++ b/merlin/examples/workflows/iterative_demo/sample_processor.py @@ -1,6 +1,7 @@ import argparse import os import pathlib +import sys import pandas as pd @@ -28,25 +29,30 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - - # Collect the samples - samples = load_samples(args.sample_file_paths) - - # Count up the occurences - namesdf = pd.DataFrame({"Name": samples}) - - names = namesdf["Name"].value_counts() - - # Serialize processed samples - # create directory if it doesn't exist already - abspath = os.path.abspath(args.results) - absdir = os.path.dirname(abspath) - if not os.path.isdir(absdir): - pathlib.Path(absdir).mkdir(parents=True, exist_ok=True) - - names.to_json(args.results) + try: + parser = setup_argparse() + args = parser.parse_args() + + # Collect the samples + samples = load_samples(args.sample_file_paths) + + # Count up the occurences + namesdf = pd.DataFrame({"Name": samples}) + + names = namesdf["Name"].value_counts() + + # Serialize processed samples + # create directory if it doesn't exist already + abspath = os.path.abspath(args.results) + absdir = os.path.dirname(abspath) + if not os.path.isdir(absdir): + pathlib.Path(absdir).mkdir(parents=True, exist_ok=True) + + names.to_json(args.results) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/lsf/scripts/make_samples.py b/merlin/examples/workflows/lsf/scripts/make_samples.py index e6c807bc9..8ec1c7e2f 100644 --- a/merlin/examples/workflows/lsf/scripts/make_samples.py +++ b/merlin/examples/workflows/lsf/scripts/make_samples.py @@ -1,5 +1,6 @@ import argparse import ast +import sys import numpy as np @@ -51,9 +52,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/null_spec/scripts/read_output.py b/merlin/examples/workflows/null_spec/scripts/read_output.py index 7c0f8017e..278283bd7 100644 --- a/merlin/examples/workflows/null_spec/scripts/read_output.py +++ b/merlin/examples/workflows/null_spec/scripts/read_output.py @@ -118,11 +118,16 @@ def start_sample1_time(): def main(): - single_task_times() - merlin_run_time() - start_verify_time() - start_run_workers_time() - start_sample1_time() + try: + single_task_times() + merlin_run_time() + start_verify_time() + start_run_workers_time() + start_sample1_time() + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py b/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py index 232c43e86..3b9f62df1 100644 --- a/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py +++ b/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py @@ -1,5 +1,6 @@ import argparse import json +import sys from typing import Dict @@ -35,9 +36,14 @@ def main(): """ Primary coordinating method for collecting args and dumping them to a json file for later examination. """ - parser: argparse.ArgumentParser = setup_argparse() - args: argparse.Namespace = parser.parse_args() - process_args(args) + try: + parser: argparse.ArgumentParser = setup_argparse() + args: argparse.Namespace = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/restart/scripts/make_samples.py b/merlin/examples/workflows/restart/scripts/make_samples.py index e6c807bc9..8ec1c7e2f 100644 --- a/merlin/examples/workflows/restart/scripts/make_samples.py +++ b/merlin/examples/workflows/restart/scripts/make_samples.py @@ -1,5 +1,6 @@ import argparse import ast +import sys import numpy as np @@ -51,9 +52,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/restart_delay/scripts/make_samples.py b/merlin/examples/workflows/restart_delay/scripts/make_samples.py index e6c807bc9..8ec1c7e2f 100644 --- a/merlin/examples/workflows/restart_delay/scripts/make_samples.py +++ b/merlin/examples/workflows/restart_delay/scripts/make_samples.py @@ -1,5 +1,6 @@ import argparse import ast +import sys import numpy as np @@ -51,9 +52,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/examples/workflows/slurm/scripts/make_samples.py b/merlin/examples/workflows/slurm/scripts/make_samples.py index e6c807bc9..8ec1c7e2f 100644 --- a/merlin/examples/workflows/slurm/scripts/make_samples.py +++ b/merlin/examples/workflows/slurm/scripts/make_samples.py @@ -1,5 +1,6 @@ import argparse import ast +import sys import numpy as np @@ -51,9 +52,14 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - process_args(args) + try: + parser = setup_argparse() + args = parser.parse_args() + process_args(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 20970fe7a..269bf6097 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 1f6befa88..33c6776e4 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 6e491ba1a..9e207b749 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # @@ -52,6 +52,7 @@ from merlin.ascii_art import banner_small from merlin.examples.generator import list_examples, setup_example from merlin.log_formatter import setup_logging +from merlin.server.server_commands import config_server, init_server, restart_server, start_server, status_server, stop_server from merlin.spec.expansion import RESERVED, get_spec_with_expansion from merlin.spec.specification import MerlinSpec from merlin.study.study import MerlinStudy @@ -342,6 +343,21 @@ def process_monitor(args): LOG.info("Monitor: ... stop condition met") +def process_server(args: Namespace): + if args.commands == "init": + init_server() + elif args.commands == "start": + start_server() + elif args.commands == "stop": + stop_server() + elif args.commands == "status": + status_server() + elif args.commands == "restart": + restart_server() + elif args.commands == "config": + config_server(args) + + def setup_argparse() -> None: """ Setup argparse and any CLI options we want available via the package. @@ -551,6 +567,143 @@ def setup_argparse() -> None: generate_diagnostic_parsers(subparsers) + # merlin server + server: ArgumentParser = subparsers.add_parser( + "server", + help="Manage broker and results server for merlin workflow.", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + server.set_defaults(func=process_server) + + server_commands: ArgumentParser = server.add_subparsers(dest="commands") + + server_init: ArgumentParser = server_commands.add_parser( + "init", + help="Initialize merlin server resources.", + description="Initialize merlin server", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + server_init.set_defaults(func=process_server) + + server_status: ArgumentParser = server_commands.add_parser( + "status", + help="View status of the current server containers.", + description="View status", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + server_status.set_defaults(func=process_server) + + server_start: ArgumentParser = server_commands.add_parser( + "start", + help="Start a containerized server to be used as an broker and results server.", + description="Start server", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + server_start.set_defaults(func=process_server) + + server_stop: ArgumentParser = server_commands.add_parser( + "stop", + help="Stop an instance of redis containers currently running.", + description="Stop server.", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + server_stop.set_defaults(func=process_server) + + server_stop: ArgumentParser = server_commands.add_parser( + "restart", + help="Restart merlin server instance", + description="Restart server.", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + server_stop.set_defaults(func=process_server) + + server_config: ArgumentParser = server_commands.add_parser( + "config", + help="Making configurations for to the merlin server instance.", + description="Config server.", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + server_config.add_argument( + "-ip", + "--ipaddress", + action="store", + type=str, + # default="127.0.0.1", + help="Set the binded IP address for the merlin server container.", + ) + server_config.add_argument( + "-p", + "--port", + action="store", + type=int, + # default=6379, + help="Set the binded port for the merlin server container.", + ) + server_config.add_argument( + "-pwd", + "--password", + action="store", + type=str, + # default="~/.merlin/redis.pass", + help="Set the password file to be used for merlin server container.", + ) + server_config.add_argument( + "--add-user", + action="store", + nargs=2, + type=str, + help="Create a new user for merlin server instance. (Provide both username and password)", + ) + server_config.add_argument("--remove-user", action="store", type=str, help="Remove an exisiting user.") + server_config.add_argument( + "-d", + "--directory", + action="store", + type=str, + # default="./", + help="Set the working directory of the merlin server container.", + ) + server_config.add_argument( + "-ss", + "--snapshot-seconds", + action="store", + type=int, + # default=300, + help="Set the number of seconds merlin server waits before checking if a snapshot is needed.", + ) + server_config.add_argument( + "-sc", + "--snapshot-changes", + action="store", + type=int, + # default=100, + help="Set the number of changes that are required to be made to the merlin server before a snapshot is made.", + ) + server_config.add_argument( + "-sf", + "--snapshot-file", + action="store", + type=str, + # default="dump.db", + help="Set the snapshot filename for database dumps.", + ) + server_config.add_argument( + "-am", + "--append-mode", + action="store", + type=str, + # default="everysec", + help="The appendonly mode to be set. The avaiable options are always, everysec, no.", + ) + server_config.add_argument( + "-af", + "--append-file", + action="store", + type=str, + # default="appendonly.aof", + help="Set append only filename for merlin server container.", + ) + return parser @@ -748,11 +901,11 @@ def main(): except Exception as excpt: # pylint: disable=broad-except LOG.debug(traceback.format_exc()) LOG.error(str(excpt)) - return 1 + sys.exit(1) # All paths in a function ought to return an exit code, or none of them should. Given the # distributed nature of Merlin, maybe it doesn't make sense for it to exit 0 until the work is completed, but # if the work is dispatched with no errors, that is a 'successful' Merlin run - any other failures are runtime. - return 0 + sys.exit() if __name__ == "__main__": diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index a1e80fea1..4195ceacc 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # @@ -33,6 +33,7 @@ """ import argparse import logging +import sys from merlin.ascii_art import banner_small from merlin.log_formatter import setup_logging @@ -57,10 +58,15 @@ def setup_argparse(): def main(): - parser = setup_argparse() - args = parser.parse_args() - setup_logging(logger=LOG, log_level=DEFAULT_LOG_LEVEL, colors=True) - args.func(args) + try: + parser = setup_argparse() + args = parser.parse_args() + setup_logging(logger=LOG, log_level=DEFAULT_LOG_LEVEL, colors=True) + args.func(args) + sys.exit() + except Exception as ex: + print(ex) + sys.exit(1) if __name__ == "__main__": diff --git a/merlin/router.py b/merlin/router.py index 90aa9db38..8858dcfad 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/docker.yaml b/merlin/server/docker.yaml new file mode 100644 index 000000000..d7d5bc00a --- /dev/null +++ b/merlin/server/docker.yaml @@ -0,0 +1,6 @@ +docker: + command: docker + # init_command: ? + run_command: \{command} run --name {name} -d {image} + stop_command: \{command} stop {name} + pull_command: \{command} pull {url} diff --git a/merlin/server/merlin_server.yaml b/merlin/server/merlin_server.yaml new file mode 100644 index 000000000..01b3c7ddb --- /dev/null +++ b/merlin/server/merlin_server.yaml @@ -0,0 +1,27 @@ +container: + # Select the format for the recipe e.g. singularity, docker, podman (currently singularity is the only working option.) + format: singularity + #Type of container that is used + image_type: redis + # The image name + image: redis_latest.sif + # The url to pull the image from + url: docker://redis + # The config file + config: redis.conf + # Directory name to store configurations Default: ./merlin_server/ + config_dir: ./merlin_server/ + # Process file containing information regarding the redis process + pfile: merlin_server.pf + # Password file to be used for accessing container + pass_file: redis.pass + # Password command for generating password file + # pass_command: date +%s | sha256sum + # Users file to track concurrent users. + user_file: redis.users + +process: + # Command for determining the process of the command + status: pgrep -P {pid} #ps -e | grep {pid} + # Command for killing process + kill: kill {pid} diff --git a/merlin/server/podman.yaml b/merlin/server/podman.yaml new file mode 100644 index 000000000..1632840bb --- /dev/null +++ b/merlin/server/podman.yaml @@ -0,0 +1,6 @@ +podman: + command: podman + # init_command: \{command} .. (optional or default) + run_command: \{command} run --name {name} -d {image} + stop_command: \{command} stop {name} + pull_command: \{command} pull {url} diff --git a/merlin/server/redis.conf b/merlin/server/redis.conf new file mode 100644 index 000000000..893677763 --- /dev/null +++ b/merlin/server/redis.conf @@ -0,0 +1,2051 @@ +# Redis configuration file example. +# +# Note that in order to read the configuration file, Redis must be +# started with the file path as first argument: +# +# ./redis-server /path/to/redis.conf + +# Note on units: when memory size is needed, it is possible to specify +# it in the usual form of 1k 5GB 4M and so forth: +# +# 1k => 1000 bytes +# 1kb => 1024 bytes +# 1m => 1000000 bytes +# 1mb => 1024*1024 bytes +# 1g => 1000000000 bytes +# 1gb => 1024*1024*1024 bytes +# +# units are case insensitive so 1GB 1Gb 1gB are all the same. + +################################## INCLUDES ################################### + +# Include one or more other config files here. This is useful if you +# have a standard template that goes to all Redis servers but also need +# to customize a few per-server settings. Include files can include +# other files, so use this wisely. +# +# Note that option "include" won't be rewritten by command "CONFIG REWRITE" +# from admin or Redis Sentinel. Since Redis always uses the last processed +# line as value of a configuration directive, you'd better put includes +# at the beginning of this file to avoid overwriting config change at runtime. +# +# If instead you are interested in using includes to override configuration +# options, it is better to use include as the last line. +# +# include /path/to/local.conf +# include /path/to/other.conf + +################################## MODULES ##################################### + +# Load modules at startup. If the server is not able to load modules +# it will abort. It is possible to use multiple loadmodule directives. +# +# loadmodule /path/to/my_module.so +# loadmodule /path/to/other_module.so + +################################## NETWORK ##################################### + +# By default, if no "bind" configuration directive is specified, Redis listens +# for connections from all available network interfaces on the host machine. +# It is possible to listen to just one or multiple selected interfaces using +# the "bind" configuration directive, followed by one or more IP addresses. +# Each address can be prefixed by "-", which means that redis will not fail to +# start if the address is not available. Being not available only refers to +# addresses that does not correspond to any network interfece. Addresses that +# are already in use will always fail, and unsupported protocols will always BE +# silently skipped. +# +# Examples: +# +# bind 192.168.1.100 10.0.0.1 # listens on two specific IPv4 addresses +# bind 127.0.0.1 ::1 # listens on loopback IPv4 and IPv6 +# bind * -::* # like the default, all available interfaces +# +# ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the +# internet, binding to all the interfaces is dangerous and will expose the +# instance to everybody on the internet. So by default we uncomment the +# following bind directive, that will force Redis to listen only on the +# IPv4 and IPv6 (if available) loopback interface addresses (this means Redis +# will only be able to accept client connections from the same host that it is +# running on). +# +# IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES +# JUST COMMENT OUT THE FOLLOWING LINE. +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +bind 127.0.0.1 + +# Protected mode is a layer of security protection, in order to avoid that +# Redis instances left open on the internet are accessed and exploited. +# +# When protected mode is on and if: +# +# 1) The server is not binding explicitly to a set of addresses using the +# "bind" directive. +# 2) No password is configured. +# +# The server only accepts connections from clients connecting from the +# IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain +# sockets. +# +# By default protected mode is enabled. You should disable it only if +# you are sure you want clients from other hosts to connect to Redis +# even if no authentication is configured, nor a specific set of interfaces +# are explicitly listed using the "bind" directive. +protected-mode yes + +# Accept connections on the specified port, default is 6379 (IANA #815344). +# If port 0 is specified Redis will not listen on a TCP socket. +port 6379 + +# TCP listen() backlog. +# +# In high requests-per-second environments you need a high backlog in order +# to avoid slow clients connection issues. Note that the Linux kernel +# will silently truncate it to the value of /proc/sys/net/core/somaxconn so +# make sure to raise both the value of somaxconn and tcp_max_syn_backlog +# in order to get the desired effect. +tcp-backlog 511 + +# Unix socket. +# +# Specify the path for the Unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# +# unixsocket /run/redis.sock +# unixsocketperm 700 + +# Close the connection after a client is idle for N seconds (0 to disable) +timeout 0 + +# TCP keepalive. +# +# If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence +# of communication. This is useful for two reasons: +# +# 1) Detect dead peers. +# 2) Force network equipment in the middle to consider the connection to be +# alive. +# +# On Linux, the specified value (in seconds) is the period used to send ACKs. +# Note that to close the connection the double of the time is needed. +# On other kernels the period depends on the kernel configuration. +# +# A reasonable value for this option is 300 seconds, which is the new +# Redis default starting with Redis 3.2.1. +tcp-keepalive 300 + +################################# TLS/SSL ##################################### + +# By default, TLS/SSL is disabled. To enable it, the "tls-port" configuration +# directive can be used to define TLS-listening ports. To enable TLS on the +# default port, use: +# +# port 0 +# tls-port 6379 + +# Configure a X.509 certificate and private key to use for authenticating the +# server to connected clients, masters or cluster peers. These files should be +# PEM formatted. +# +# tls-cert-file redis.crt +# tls-key-file redis.key +# +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-key-file-pass secret + +# Normally Redis uses the same certificate for both server functions (accepting +# connections) and client functions (replicating from a master, establishing +# cluster bus connections, etc.). +# +# Sometimes certificates are issued with attributes that designate them as +# client-only or server-only certificates. In that case it may be desired to use +# different certificates for incoming (server) and outgoing (client) +# connections. To do that, use the following directives: +# +# tls-client-cert-file client.crt +# tls-client-key-file client.key +# +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-client-key-file-pass secret + +# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: +# +# tls-dh-params-file redis.dh + +# Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL +# clients and peers. Redis requires an explicit configuration of at least one +# of these, and will not implicitly use the system wide configuration. +# +# tls-ca-cert-file ca.crt +# tls-ca-cert-dir /etc/ssl/certs + +# By default, clients (including replica servers) on a TLS port are required +# to authenticate using valid client side certificates. +# +# If "no" is specified, client certificates are not required and not accepted. +# If "optional" is specified, client certificates are accepted and must be +# valid if provided, but are not required. +# +# tls-auth-clients no +# tls-auth-clients optional + +# By default, a Redis replica does not attempt to establish a TLS connection +# with its master. +# +# Use the following directive to enable TLS on replication links. +# +# tls-replication yes + +# By default, the Redis Cluster bus uses a plain TCP connection. To enable +# TLS for the bus protocol, use the following directive: +# +# tls-cluster yes + +# By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended +# that older formally deprecated versions are kept disabled to reduce the attack surface. +# You can explicitly specify TLS versions to support. +# Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2", +# "TLSv1.3" (OpenSSL >= 1.1.1) or any combination. +# To enable only TLSv1.2 and TLSv1.3, use: +# +# tls-protocols "TLSv1.2 TLSv1.3" + +# Configure allowed ciphers. See the ciphers(1ssl) manpage for more information +# about the syntax of this string. +# +# Note: this configuration applies only to <= TLSv1.2. +# +# tls-ciphers DEFAULT:!MEDIUM + +# Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more +# information about the syntax of this string, and specifically for TLSv1.3 +# ciphersuites. +# +# tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 + +# When choosing a cipher, use the server's preference instead of the client +# preference. By default, the server follows the client's preference. +# +# tls-prefer-server-ciphers yes + +# By default, TLS session caching is enabled to allow faster and less expensive +# reconnections by clients that support it. Use the following directive to disable +# caching. +# +# tls-session-caching no + +# Change the default number of TLS sessions cached. A zero value sets the cache +# to unlimited size. The default size is 20480. +# +# tls-session-cache-size 5000 + +# Change the default timeout of cached TLS sessions. The default timeout is 300 +# seconds. +# +# tls-session-cache-timeout 60 + +################################# GENERAL ##################################### + +# By default Redis does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +# When Redis is supervised by upstart or systemd, this parameter has no impact. +daemonize no + +# If you run Redis from upstart or systemd, Redis can interact with your +# supervision tree. Options: +# supervised no - no supervision interaction +# supervised upstart - signal upstart by putting Redis into SIGSTOP mode +# requires "expect stop" in your upstart job config +# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET +# on startup, and updating Redis status on a regular +# basis. +# supervised auto - detect upstart or systemd method based on +# UPSTART_JOB or NOTIFY_SOCKET environment variables +# Note: these supervision methods only signal "process is ready." +# They do not enable continuous pings back to your supervisor. +# +# The default is "no". To run under upstart/systemd, you can simply uncomment +# the line below: +# +# supervised auto + +# If a pid file is specified, Redis writes it where specified at startup +# and removes it at exit. +# +# When the server runs non daemonized, no pid file is created if none is +# specified in the configuration. When the server is daemonized, the pid file +# is used even if not specified, defaulting to "/var/run/redis.pid". +# +# Creating a pid file is best effort: if Redis is not able to create it +# nothing bad happens, the server will start and run normally. +# +# Note that on modern Linux systems "/run/redis.pid" is more conforming +# and should be used instead. +pidfile /var/run/redis_6379.pid + +# Specify the server verbosity level. +# This can be one of: +# debug (a lot of information, useful for development/testing) +# verbose (many rarely useful info, but not a mess like the debug level) +# notice (moderately verbose, what you want in production probably) +# warning (only very important / critical messages are logged) +loglevel notice + +# Specify the log file name. Also the empty string can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile "" + +# To enable logging to the system logger, just set 'syslog-enabled' to yes, +# and optionally update the other syslog parameters to suit your needs. +# syslog-enabled no + +# Specify the syslog identity. +# syslog-ident redis + +# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. +# syslog-facility local0 + +# To disable the built in crash log, which will possibly produce cleaner core +# dumps when they are needed, uncomment the following: +# +# crash-log-enabled no + +# To disable the fast memory check that's run as part of the crash log, which +# will possibly let redis terminate sooner, uncomment the following: +# +# crash-memcheck-enabled no + +# Set the number of databases. The default database is DB 0, you can select +# a different one on a per-connection basis using SELECT where +# dbid is a number between 0 and 'databases'-1 +databases 16 + +# By default Redis shows an ASCII art logo only when started to log to the +# standard output and if the standard output is a TTY and syslog logging is +# disabled. Basically this means that normally a logo is displayed only in +# interactive sessions. +# +# However it is possible to force the pre-4.0 behavior and always show a +# ASCII art logo in startup logs by setting the following option to yes. +always-show-logo no + +# By default, Redis modifies the process title (as seen in 'top' and 'ps') to +# provide some runtime information. It is possible to disable this and leave +# the process name as executed by setting the following to no. +set-proc-title yes + +# When changing the process title, Redis uses the following template to construct +# the modified title. +# +# Template variables are specified in curly brackets. The following variables are +# supported: +# +# {title} Name of process as executed if parent, or type of child process. +# {listen-addr} Bind address or '*' followed by TCP or TLS port listening on, or +# Unix socket if only that's available. +# {server-mode} Special mode, i.e. "[sentinel]" or "[cluster]". +# {port} TCP port listening on, or 0. +# {tls-port} TLS port listening on, or 0. +# {unixsocket} Unix domain socket listening on, or "". +# {config-file} Name of configuration file used. +# +proc-title-template "{title} {listen-addr} {server-mode}" + +################################ SNAPSHOTTING ################################ + +# Save the DB to disk. +# +# save +# +# Redis will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. +# +# Snapshotting can be completely disabled with a single empty string argument +# as in following example: +# +# save "" +# +# Unless specified otherwise, by default Redis will save the DB: +# * After 3600 seconds (an hour) if at least 1 key changed +# * After 300 seconds (5 minutes) if at least 100 keys changed +# * After 60 seconds if at least 10000 keys changed +# +# You can set these explicitly by uncommenting the three following lines. +# +# save 3600 1 +save 300 100 +# save 60 10000 + +# By default Redis will stop accepting writes if RDB snapshots are enabled +# (at least one save point) and the latest background save failed. +# This will make the user aware (in a hard way) that data is not persisting +# on disk properly, otherwise chances are that no one will notice and some +# disaster will happen. +# +# If the background saving process will start working again Redis will +# automatically allow writes again. +# +# However if you have setup your proper monitoring of the Redis server +# and persistence, you may want to disable this feature so that Redis will +# continue to work as usual even if there are problems with disk, +# permissions, and so forth. +stop-writes-on-bgsave-error no + +# Compress string objects using LZF when dump .rdb databases? +# By default compression is enabled as it's almost always a win. +# If you want to save some CPU in the saving child set it to 'no' but +# the dataset will likely be bigger if you have compressible values or keys. +rdbcompression yes + +# Since version 5 of RDB a CRC64 checksum is placed at the end of the file. +# This makes the format more resistant to corruption but there is a performance +# hit to pay (around 10%) when saving and loading RDB files, so you can disable it +# for maximum performances. +# +# RDB files created with checksum disabled have a checksum of zero that will +# tell the loading code to skip the check. +rdbchecksum yes + +# Enables or disables full sanitation checks for ziplist and listpack etc when +# loading an RDB or RESTORE payload. This reduces the chances of a assertion or +# crash later on while processing commands. +# Options: +# no - Never perform full sanitation +# yes - Always perform full sanitation +# clients - Perform full sanitation only for user connections. +# Excludes: RDB files, RESTORE commands received from the master +# connection, and client connections which have the +# skip-sanitize-payload ACL flag. +# The default should be 'clients' but since it currently affects cluster +# resharding via MIGRATE, it is temporarily set to 'no' by default. +# +# sanitize-dump-payload no + +# The filename where to dump the DB +dbfilename dump.rdb + +# Remove RDB files used by replication in instances without persistence +# enabled. By default this option is disabled, however there are environments +# where for regulations or other security concerns, RDB files persisted on +# disk by masters in order to feed replicas, or stored on disk by replicas +# in order to load them for the initial synchronization, should be deleted +# ASAP. Note that this option ONLY WORKS in instances that have both AOF +# and RDB persistence disabled, otherwise is completely ignored. +# +# An alternative (and sometimes better) way to obtain the same effect is +# to use diskless replication on both master and replicas instances. However +# in the case of replicas, diskless is not always an option. +rdb-del-sync-files no + +# The working directory. +# +# The DB will be written inside this directory, with the filename specified +# above using the 'dbfilename' configuration directive. +# +# The Append Only File will also be created inside this directory. +# +# Note that you must specify a directory here, not a file name. +dir ./ + +################################# REPLICATION ################################# + +# Master-Replica replication. Use replicaof to make a Redis instance a copy of +# another Redis server. A few things to understand ASAP about Redis replication. +# +# +------------------+ +---------------+ +# | Master | ---> | Replica | +# | (receive writes) | | (exact copy) | +# +------------------+ +---------------+ +# +# 1) Redis replication is asynchronous, but you can configure a master to +# stop accepting writes if it appears to be not connected with at least +# a given number of replicas. +# 2) Redis replicas are able to perform a partial resynchronization with the +# master if the replication link is lost for a relatively small amount of +# time. You may want to configure the replication backlog size (see the next +# sections of this file) with a sensible value depending on your needs. +# 3) Replication is automatic and does not need user intervention. After a +# network partition replicas automatically try to reconnect to masters +# and resynchronize with them. +# +# replicaof + +# If the master is password protected (using the "requirepass" configuration +# directive below) it is possible to tell the replica to authenticate before +# starting the replication synchronization process, otherwise the master will +# refuse the replica request. +# +# masterauth +# +# However this is not enough if you are using Redis ACLs (for Redis version +# 6 or greater), and the default user is not capable of running the PSYNC +# command and/or other commands needed for replication. In this case it's +# better to configure a special user to use with replication, and specify the +# masteruser configuration as such: +# +# masteruser root +# +# When masteruser is specified, the replica will authenticate against its +# master using the new AUTH form: AUTH . + +# When a replica loses its connection with the master, or when the replication +# is still in progress, the replica can act in two different ways: +# +# 1) if replica-serve-stale-data is set to 'yes' (the default) the replica will +# still reply to client requests, possibly with out of date data, or the +# data set may just be empty if this is the first synchronization. +# +# 2) If replica-serve-stale-data is set to 'no' the replica will reply with +# an error "SYNC with master in progress" to all commands except: +# INFO, REPLICAOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, SUBSCRIBE, +# UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, COMMAND, POST, +# HOST and LATENCY. +# +replica-serve-stale-data yes + +# You can configure a replica instance to accept writes or not. Writing against +# a replica instance may be useful to store some ephemeral data (because data +# written on a replica will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it because of a +# misconfiguration. +# +# Since Redis 2.6 by default replicas are read-only. +# +# Note: read only replicas are not designed to be exposed to untrusted clients +# on the internet. It's just a protection layer against misuse of the instance. +# Still a read only replica exports by default all the administrative commands +# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve +# security of read only replicas using 'rename-command' to shadow all the +# administrative / dangerous commands. +replica-read-only yes + +# Replication SYNC strategy: disk or socket. +# +# New replicas and reconnecting replicas that are not able to continue the +# replication process just receiving differences, need to do what is called a +# "full synchronization". An RDB file is transmitted from the master to the +# replicas. +# +# The transmission can happen in two different ways: +# +# 1) Disk-backed: The Redis master creates a new process that writes the RDB +# file on disk. Later the file is transferred by the parent +# process to the replicas incrementally. +# 2) Diskless: The Redis master creates a new process that directly writes the +# RDB file to replica sockets, without touching the disk at all. +# +# With disk-backed replication, while the RDB file is generated, more replicas +# can be queued and served with the RDB file as soon as the current child +# producing the RDB file finishes its work. With diskless replication instead +# once the transfer starts, new replicas arriving will be queued and a new +# transfer will start when the current one terminates. +# +# When diskless replication is used, the master waits a configurable amount of +# time (in seconds) before starting the transfer in the hope that multiple +# replicas will arrive and the transfer can be parallelized. +# +# With slow disks and fast (large bandwidth) networks, diskless replication +# works better. +repl-diskless-sync no + +# When diskless replication is enabled, it is possible to configure the delay +# the server waits in order to spawn the child that transfers the RDB via socket +# to the replicas. +# +# This is important since once the transfer starts, it is not possible to serve +# new replicas arriving, that will be queued for the next RDB transfer, so the +# server waits a delay in order to let more replicas arrive. +# +# The delay is specified in seconds, and by default is 5 seconds. To disable +# it entirely just set it to 0 seconds and the transfer will start ASAP. +repl-diskless-sync-delay 5 + +# ----------------------------------------------------------------------------- +# WARNING: RDB diskless load is experimental. Since in this setup the replica +# does not immediately store an RDB on disk, it may cause data loss during +# failovers. RDB diskless load + Redis modules not handling I/O reads may also +# cause Redis to abort in case of I/O errors during the initial synchronization +# stage with the master. Use only if you know what you are doing. +# ----------------------------------------------------------------------------- +# +# Replica can load the RDB it reads from the replication link directly from the +# socket, or store the RDB to a file and read that file after it was completely +# received from the master. +# +# In many cases the disk is slower than the network, and storing and loading +# the RDB file may increase replication time (and even increase the master's +# Copy on Write memory and salve buffers). +# However, parsing the RDB file directly from the socket may mean that we have +# to flush the contents of the current database before the full rdb was +# received. For this reason we have the following options: +# +# "disabled" - Don't use diskless load (store the rdb file to the disk first) +# "on-empty-db" - Use diskless load only when it is completely safe. +# "swapdb" - Keep a copy of the current db contents in RAM while parsing +# the data directly from the socket. note that this requires +# sufficient memory, if you don't have it, you risk an OOM kill. +repl-diskless-load disabled + +# Replicas send PINGs to server in a predefined interval. It's possible to +# change this interval with the repl_ping_replica_period option. The default +# value is 10 seconds. +# +# repl-ping-replica-period 10 + +# The following option sets the replication timeout for: +# +# 1) Bulk transfer I/O during SYNC, from the point of view of replica. +# 2) Master timeout from the point of view of replicas (data, pings). +# 3) Replica timeout from the point of view of masters (REPLCONF ACK pings). +# +# It is important to make sure that this value is greater than the value +# specified for repl-ping-replica-period otherwise a timeout will be detected +# every time there is low traffic between the master and the replica. The default +# value is 60 seconds. +# +# repl-timeout 60 + +# Disable TCP_NODELAY on the replica socket after SYNC? +# +# If you select "yes" Redis will use a smaller number of TCP packets and +# less bandwidth to send data to replicas. But this can add a delay for +# the data to appear on the replica side, up to 40 milliseconds with +# Linux kernels using a default configuration. +# +# If you select "no" the delay for data to appear on the replica side will +# be reduced but more bandwidth will be used for replication. +# +# By default we optimize for low latency, but in very high traffic conditions +# or when the master and replicas are many hops away, turning this to "yes" may +# be a good idea. +repl-disable-tcp-nodelay no + +# Set the replication backlog size. The backlog is a buffer that accumulates +# replica data when replicas are disconnected for some time, so that when a +# replica wants to reconnect again, often a full resync is not needed, but a +# partial resync is enough, just passing the portion of data the replica +# missed while disconnected. +# +# The bigger the replication backlog, the longer the replica can endure the +# disconnect and later be able to perform a partial resynchronization. +# +# The backlog is only allocated if there is at least one replica connected. +# +# repl-backlog-size 1mb + +# After a master has no connected replicas for some time, the backlog will be +# freed. The following option configures the amount of seconds that need to +# elapse, starting from the time the last replica disconnected, for the backlog +# buffer to be freed. +# +# Note that replicas never free the backlog for timeout, since they may be +# promoted to masters later, and should be able to correctly "partially +# resynchronize" with other replicas: hence they should always accumulate backlog. +# +# A value of 0 means to never release the backlog. +# +# repl-backlog-ttl 3600 + +# The replica priority is an integer number published by Redis in the INFO +# output. It is used by Redis Sentinel in order to select a replica to promote +# into a master if the master is no longer working correctly. +# +# A replica with a low priority number is considered better for promotion, so +# for instance if there are three replicas with priority 10, 100, 25 Sentinel +# will pick the one with priority 10, that is the lowest. +# +# However a special priority of 0 marks the replica as not able to perform the +# role of master, so a replica with priority of 0 will never be selected by +# Redis Sentinel for promotion. +# +# By default the priority is 100. +replica-priority 100 + +# ----------------------------------------------------------------------------- +# By default, Redis Sentinel includes all replicas in its reports. A replica +# can be excluded from Redis Sentinel's announcements. An unannounced replica +# will be ignored by the 'sentinel replicas ' command and won't be +# exposed to Redis Sentinel's clients. +# +# This option does not change the behavior of replica-priority. Even with +# replica-announced set to 'no', the replica can be promoted to master. To +# prevent this behavior, set replica-priority to 0. +# +# replica-announced yes + +# It is possible for a master to stop accepting writes if there are less than +# N replicas connected, having a lag less or equal than M seconds. +# +# The N replicas need to be in "online" state. +# +# The lag in seconds, that must be <= the specified value, is calculated from +# the last ping received from the replica, that is usually sent every second. +# +# This option does not GUARANTEE that N replicas will accept the write, but +# will limit the window of exposure for lost writes in case not enough replicas +# are available, to the specified number of seconds. +# +# For example to require at least 3 replicas with a lag <= 10 seconds use: +# +# min-replicas-to-write 3 +# min-replicas-max-lag 10 +# +# Setting one or the other to 0 disables the feature. +# +# By default min-replicas-to-write is set to 0 (feature disabled) and +# min-replicas-max-lag is set to 10. + +# A Redis master is able to list the address and port of the attached +# replicas in different ways. For example the "INFO replication" section +# offers this information, which is used, among other tools, by +# Redis Sentinel in order to discover replica instances. +# Another place where this info is available is in the output of the +# "ROLE" command of a master. +# +# The listed IP address and port normally reported by a replica is +# obtained in the following way: +# +# IP: The address is auto detected by checking the peer address +# of the socket used by the replica to connect with the master. +# +# Port: The port is communicated by the replica during the replication +# handshake, and is normally the port that the replica is using to +# listen for connections. +# +# However when port forwarding or Network Address Translation (NAT) is +# used, the replica may actually be reachable via different IP and port +# pairs. The following two options can be used by a replica in order to +# report to its master a specific set of IP and port, so that both INFO +# and ROLE will report those values. +# +# There is no need to use both the options if you need to override just +# the port or the IP address. +# +# replica-announce-ip 5.5.5.5 +# replica-announce-port 1234 + +############################### KEYS TRACKING ################################# + +# Redis implements server assisted support for client side caching of values. +# This is implemented using an invalidation table that remembers, using +# a radix key indexed by key name, what clients have which keys. In turn +# this is used in order to send invalidation messages to clients. Please +# check this page to understand more about the feature: +# +# https://redis.io/topics/client-side-caching +# +# When tracking is enabled for a client, all the read only queries are assumed +# to be cached: this will force Redis to store information in the invalidation +# table. When keys are modified, such information is flushed away, and +# invalidation messages are sent to the clients. However if the workload is +# heavily dominated by reads, Redis could use more and more memory in order +# to track the keys fetched by many clients. +# +# For this reason it is possible to configure a maximum fill value for the +# invalidation table. By default it is set to 1M of keys, and once this limit +# is reached, Redis will start to evict keys in the invalidation table +# even if they were not modified, just to reclaim memory: this will in turn +# force the clients to invalidate the cached values. Basically the table +# maximum size is a trade off between the memory you want to spend server +# side to track information about who cached what, and the ability of clients +# to retain cached objects in memory. +# +# If you set the value to 0, it means there are no limits, and Redis will +# retain as many keys as needed in the invalidation table. +# In the "stats" INFO section, you can find information about the number of +# keys in the invalidation table at every given moment. +# +# Note: when key tracking is used in broadcasting mode, no memory is used +# in the server side so this setting is useless. +# +# tracking-table-max-keys 1000000 + +################################## SECURITY ################################### + +# Warning: since Redis is pretty fast, an outside user can try up to +# 1 million passwords per second against a modern box. This means that you +# should use very strong passwords, otherwise they will be very easy to break. +# Note that because the password is really a shared secret between the client +# and the server, and should not be memorized by any human, the password +# can be easily a long string from /dev/urandom or whatever, so by using a +# long and unguessable password no brute force attack will be possible. + +# Redis ACL users are defined in the following format: +# +# user ... acl rules ... +# +# For example: +# +# user worker +@list +@connection ~jobs:* on >ffa9203c493aa99 +# +# The special username "default" is used for new connections. If this user +# has the "nopass" rule, then new connections will be immediately authenticated +# as the "default" user without the need of any password provided via the +# AUTH command. Otherwise if the "default" user is not flagged with "nopass" +# the connections will start in not authenticated state, and will require +# AUTH (or the HELLO command AUTH option) in order to be authenticated and +# start to work. +# +# The ACL rules that describe what a user can do are the following: +# +# on Enable the user: it is possible to authenticate as this user. +# off Disable the user: it's no longer possible to authenticate +# with this user, however the already authenticated connections +# will still work. +# skip-sanitize-payload RESTORE dump-payload sanitation is skipped. +# sanitize-payload RESTORE dump-payload is sanitized (default). +# + Allow the execution of that command +# - Disallow the execution of that command +# +@ Allow the execution of all the commands in such category +# with valid categories are like @admin, @set, @sortedset, ... +# and so forth, see the full list in the server.c file where +# the Redis command table is described and defined. +# The special category @all means all the commands, but currently +# present in the server, and that will be loaded in the future +# via modules. +# +|subcommand Allow a specific subcommand of an otherwise +# disabled command. Note that this form is not +# allowed as negative like -DEBUG|SEGFAULT, but +# only additive starting with "+". +# allcommands Alias for +@all. Note that it implies the ability to execute +# all the future commands loaded via the modules system. +# nocommands Alias for -@all. +# ~ Add a pattern of keys that can be mentioned as part of +# commands. For instance ~* allows all the keys. The pattern +# is a glob-style pattern like the one of KEYS. +# It is possible to specify multiple patterns. +# allkeys Alias for ~* +# resetkeys Flush the list of allowed keys patterns. +# & Add a glob-style pattern of Pub/Sub channels that can be +# accessed by the user. It is possible to specify multiple channel +# patterns. +# allchannels Alias for &* +# resetchannels Flush the list of allowed channel patterns. +# > Add this password to the list of valid password for the user. +# For example >mypass will add "mypass" to the list. +# This directive clears the "nopass" flag (see later). +# < Remove this password from the list of valid passwords. +# nopass All the set passwords of the user are removed, and the user +# is flagged as requiring no password: it means that every +# password will work against this user. If this directive is +# used for the default user, every new connection will be +# immediately authenticated with the default user without +# any explicit AUTH command required. Note that the "resetpass" +# directive will clear this condition. +# resetpass Flush the list of allowed passwords. Moreover removes the +# "nopass" status. After "resetpass" the user has no associated +# passwords and there is no way to authenticate without adding +# some password (or setting it as "nopass" later). +# reset Performs the following actions: resetpass, resetkeys, off, +# -@all. The user returns to the same state it has immediately +# after its creation. +# +# ACL rules can be specified in any order: for instance you can start with +# passwords, then flags, or key patterns. However note that the additive +# and subtractive rules will CHANGE MEANING depending on the ordering. +# For instance see the following example: +# +# user alice on +@all -DEBUG ~* >somepassword +# +# This will allow "alice" to use all the commands with the exception of the +# DEBUG command, since +@all added all the commands to the set of the commands +# alice can use, and later DEBUG was removed. However if we invert the order +# of two ACL rules the result will be different: +# +# user alice on -DEBUG +@all ~* >somepassword +# +# Now DEBUG was removed when alice had yet no commands in the set of allowed +# commands, later all the commands are added, so the user will be able to +# execute everything. +# +# Basically ACL rules are processed left-to-right. +# +# For more information about ACL configuration please refer to +# the Redis web site at https://redis.io/topics/acl + +# ACL LOG +# +# The ACL Log tracks failed commands and authentication events associated +# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked +# by ACLs. The ACL Log is stored in memory. You can reclaim memory with +# ACL LOG RESET. Define the maximum entry length of the ACL Log below. +acllog-max-len 128 + +# Using an external ACL file +# +# Instead of configuring users here in this file, it is possible to use +# a stand-alone file just listing users. The two methods cannot be mixed: +# if you configure users here and at the same time you activate the external +# ACL file, the server will refuse to start. +# +# The format of the external ACL user file is exactly the same as the +# format that is used inside redis.conf to describe users. +# +# aclfile /etc/redis/users.acl + +# IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatibility +# layer on top of the new ACL system. The option effect will be just setting +# the password for the default user. Clients will still authenticate using +# AUTH as usually, or more explicitly with AUTH default +# if they follow the new protocol: both will work. +# +# The requirepass is not compatable with aclfile option and the ACL LOAD +# command, these will cause requirepass to be ignored. +# +requirepass merlin_password + +# New users are initialized with restrictive permissions by default, via the +# equivalent of this ACL rule 'off resetkeys -@all'. Starting with Redis 6.2, it +# is possible to manage access to Pub/Sub channels with ACL rules as well. The +# default Pub/Sub channels permission if new users is controlled by the +# acl-pubsub-default configuration directive, which accepts one of these values: +# +# allchannels: grants access to all Pub/Sub channels +# resetchannels: revokes access to all Pub/Sub channels +# +# To ensure backward compatibility while upgrading Redis 6.0, acl-pubsub-default +# defaults to the 'allchannels' permission. +# +# Future compatibility note: it is very likely that in a future version of Redis +# the directive's default of 'allchannels' will be changed to 'resetchannels' in +# order to provide better out-of-the-box Pub/Sub security. Therefore, it is +# recommended that you explicitly define Pub/Sub permissions for all users +# rather then rely on implicit default values. Once you've set explicit +# Pub/Sub for all existing users, you should uncomment the following line. +# +# acl-pubsub-default resetchannels + +# Command renaming (DEPRECATED). +# +# ------------------------------------------------------------------------ +# WARNING: avoid using this option if possible. Instead use ACLs to remove +# commands from the default user, and put them only in some admin user you +# create for administrative purposes. +# ------------------------------------------------------------------------ +# +# It is possible to change the name of dangerous commands in a shared +# environment. For instance the CONFIG command may be renamed into something +# hard to guess so that it will still be available for internal-use tools +# but not available for general clients. +# +# Example: +# +# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 +# +# It is also possible to completely kill a command by renaming it into +# an empty string: +# +# rename-command CONFIG "" +# +# Please note that changing the name of commands that are logged into the +# AOF file or transmitted to replicas may cause problems. + +################################### CLIENTS #################################### + +# Set the max number of connected clients at the same time. By default +# this limit is set to 10000 clients, however if the Redis server is not +# able to configure the process file limit to allow for the specified limit +# the max number of allowed clients is set to the current file limit +# minus 32 (as Redis reserves a few file descriptors for internal uses). +# +# Once the limit is reached Redis will close all the new connections sending +# an error 'max number of clients reached'. +# +# IMPORTANT: When Redis Cluster is used, the max number of connections is also +# shared with the cluster bus: every node in the cluster will use two +# connections, one incoming and another outgoing. It is important to size the +# limit accordingly in case of very large clusters. +# +# maxclients 10000 + +############################## MEMORY MANAGEMENT ################################ + +# Set a memory usage limit to the specified amount of bytes. +# When the memory limit is reached Redis will try to remove keys +# according to the eviction policy selected (see maxmemory-policy). +# +# If Redis can't remove keys according to the policy, or if the policy is +# set to 'noeviction', Redis will start to reply with errors to commands +# that would use more memory, like SET, LPUSH, and so on, and will continue +# to reply to read-only commands like GET. +# +# This option is usually useful when using Redis as an LRU or LFU cache, or to +# set a hard memory limit for an instance (using the 'noeviction' policy). +# +# WARNING: If you have replicas attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the replicas are subtracted +# from the used memory count, so that network problems / resyncs will +# not trigger a loop where keys are evicted, and in turn the output +# buffer of replicas is full with DELs of keys evicted triggering the deletion +# of more keys, and so forth until the database is completely emptied. +# +# In short... if you have replicas attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for replica +# output buffers (but this is not needed if the policy is 'noeviction'). +# +# maxmemory + +# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory +# is reached. You can select one from the following behaviors: +# +# volatile-lru -> Evict using approximated LRU, only keys with an expire set. +# allkeys-lru -> Evict any key using approximated LRU. +# volatile-lfu -> Evict using approximated LFU, only keys with an expire set. +# allkeys-lfu -> Evict any key using approximated LFU. +# volatile-random -> Remove a random key having an expire set. +# allkeys-random -> Remove a random key, any key. +# volatile-ttl -> Remove the key with the nearest expire time (minor TTL) +# noeviction -> Don't evict anything, just return an error on write operations. +# +# LRU means Least Recently Used +# LFU means Least Frequently Used +# +# Both LRU, LFU and volatile-ttl are implemented using approximated +# randomized algorithms. +# +# Note: with any of the above policies, when there are no suitable keys for +# eviction, Redis will return an error on write operations that require +# more memory. These are usually commands that create new keys, add data or +# modify existing keys. A few examples are: SET, INCR, HSET, LPUSH, SUNIONSTORE, +# SORT (due to the STORE argument), and EXEC (if the transaction includes any +# command that requires memory). +# +# The default is: +# +# maxmemory-policy noeviction + +# LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated +# algorithms (in order to save memory), so you can tune it for speed or +# accuracy. By default Redis will check five keys and pick the one that was +# used least recently, you can change the sample size using the following +# configuration directive. +# +# The default of 5 produces good enough results. 10 Approximates very closely +# true LRU but costs more CPU. 3 is faster but not very accurate. +# +# maxmemory-samples 5 + +# Eviction processing is designed to function well with the default setting. +# If there is an unusually large amount of write traffic, this value may need to +# be increased. Decreasing this value may reduce latency at the risk of +# eviction processing effectiveness +# 0 = minimum latency, 10 = default, 100 = process without regard to latency +# +# maxmemory-eviction-tenacity 10 + +# Starting from Redis 5, by default a replica will ignore its maxmemory setting +# (unless it is promoted to master after a failover or manually). It means +# that the eviction of keys will be just handled by the master, sending the +# DEL commands to the replica as keys evict in the master side. +# +# This behavior ensures that masters and replicas stay consistent, and is usually +# what you want, however if your replica is writable, or you want the replica +# to have a different memory setting, and you are sure all the writes performed +# to the replica are idempotent, then you may change this default (but be sure +# to understand what you are doing). +# +# Note that since the replica by default does not evict, it may end using more +# memory than the one set via maxmemory (there are certain buffers that may +# be larger on the replica, or data structures may sometimes take more memory +# and so forth). So make sure you monitor your replicas and make sure they +# have enough memory to never hit a real out-of-memory condition before the +# master hits the configured maxmemory setting. +# +# replica-ignore-maxmemory yes + +# Redis reclaims expired keys in two ways: upon access when those keys are +# found to be expired, and also in background, in what is called the +# "active expire key". The key space is slowly and interactively scanned +# looking for expired keys to reclaim, so that it is possible to free memory +# of keys that are expired and will never be accessed again in a short time. +# +# The default effort of the expire cycle will try to avoid having more than +# ten percent of expired keys still in memory, and will try to avoid consuming +# more than 25% of total memory and to add latency to the system. However +# it is possible to increase the expire "effort" that is normally set to +# "1", to a greater value, up to the value "10". At its maximum value the +# system will use more CPU, longer cycles (and technically may introduce +# more latency), and will tolerate less already expired keys still present +# in the system. It's a tradeoff between memory, CPU and latency. +# +# active-expire-effort 1 + +############################# LAZY FREEING #################################### + +# Redis has two primitives to delete keys. One is called DEL and is a blocking +# deletion of the object. It means that the server stops processing new commands +# in order to reclaim all the memory associated with an object in a synchronous +# way. If the key deleted is associated with a small object, the time needed +# in order to execute the DEL command is very small and comparable to most other +# O(1) or O(log_N) commands in Redis. However if the key is associated with an +# aggregated value containing millions of elements, the server can block for +# a long time (even seconds) in order to complete the operation. +# +# For the above reasons Redis also offers non blocking deletion primitives +# such as UNLINK (non blocking DEL) and the ASYNC option of FLUSHALL and +# FLUSHDB commands, in order to reclaim memory in background. Those commands +# are executed in constant time. Another thread will incrementally free the +# object in the background as fast as possible. +# +# DEL, UNLINK and ASYNC option of FLUSHALL and FLUSHDB are user-controlled. +# It's up to the design of the application to understand when it is a good +# idea to use one or the other. However the Redis server sometimes has to +# delete keys or flush the whole database as a side effect of other operations. +# Specifically Redis deletes objects independently of a user call in the +# following scenarios: +# +# 1) On eviction, because of the maxmemory and maxmemory policy configurations, +# in order to make room for new data, without going over the specified +# memory limit. +# 2) Because of expire: when a key with an associated time to live (see the +# EXPIRE command) must be deleted from memory. +# 3) Because of a side effect of a command that stores data on a key that may +# already exist. For example the RENAME command may delete the old key +# content when it is replaced with another one. Similarly SUNIONSTORE +# or SORT with STORE option may delete existing keys. The SET command +# itself removes any old content of the specified key in order to replace +# it with the specified string. +# 4) During replication, when a replica performs a full resynchronization with +# its master, the content of the whole database is removed in order to +# load the RDB file just transferred. +# +# In all the above cases the default is to delete objects in a blocking way, +# like if DEL was called. However you can configure each case specifically +# in order to instead release memory in a non-blocking way like if UNLINK +# was called, using the following configuration directives. + +lazyfree-lazy-eviction no +lazyfree-lazy-expire no +lazyfree-lazy-server-del no +replica-lazy-flush no + +# It is also possible, for the case when to replace the user code DEL calls +# with UNLINK calls is not easy, to modify the default behavior of the DEL +# command to act exactly like UNLINK, using the following configuration +# directive: + +lazyfree-lazy-user-del no + +# FLUSHDB, FLUSHALL, and SCRIPT FLUSH support both asynchronous and synchronous +# deletion, which can be controlled by passing the [SYNC|ASYNC] flags into the +# commands. When neither flag is passed, this directive will be used to determine +# if the data should be deleted asynchronously. + +lazyfree-lazy-user-flush no + +################################ THREADED I/O ################################# + +# Redis is mostly single threaded, however there are certain threaded +# operations such as UNLINK, slow I/O accesses and other things that are +# performed on side threads. +# +# Now it is also possible to handle Redis clients socket reads and writes +# in different I/O threads. Since especially writing is so slow, normally +# Redis users use pipelining in order to speed up the Redis performances per +# core, and spawn multiple instances in order to scale more. Using I/O +# threads it is possible to easily speedup two times Redis without resorting +# to pipelining nor sharding of the instance. +# +# By default threading is disabled, we suggest enabling it only in machines +# that have at least 4 or more cores, leaving at least one spare core. +# Using more than 8 threads is unlikely to help much. We also recommend using +# threaded I/O only if you actually have performance problems, with Redis +# instances being able to use a quite big percentage of CPU time, otherwise +# there is no point in using this feature. +# +# So for instance if you have a four cores boxes, try to use 2 or 3 I/O +# threads, if you have a 8 cores, try to use 6 threads. In order to +# enable I/O threads use the following configuration directive: +# +# io-threads 4 +# +# Setting io-threads to 1 will just use the main thread as usual. +# When I/O threads are enabled, we only use threads for writes, that is +# to thread the write(2) syscall and transfer the client buffers to the +# socket. However it is also possible to enable threading of reads and +# protocol parsing using the following configuration directive, by setting +# it to yes: +# +# io-threads-do-reads no +# +# Usually threading reads doesn't help much. +# +# NOTE 1: This configuration directive cannot be changed at runtime via +# CONFIG SET. Aso this feature currently does not work when SSL is +# enabled. +# +# NOTE 2: If you want to test the Redis speedup using redis-benchmark, make +# sure you also run the benchmark itself in threaded mode, using the +# --threads option to match the number of Redis threads, otherwise you'll not +# be able to notice the improvements. + +############################ KERNEL OOM CONTROL ############################## + +# On Linux, it is possible to hint the kernel OOM killer on what processes +# should be killed first when out of memory. +# +# Enabling this feature makes Redis actively control the oom_score_adj value +# for all its processes, depending on their role. The default scores will +# attempt to have background child processes killed before all others, and +# replicas killed before masters. +# +# Redis supports three options: +# +# no: Don't make changes to oom-score-adj (default). +# yes: Alias to "relative" see below. +# absolute: Values in oom-score-adj-values are written as is to the kernel. +# relative: Values are used relative to the initial value of oom_score_adj when +# the server starts and are then clamped to a range of -1000 to 1000. +# Because typically the initial value is 0, they will often match the +# absolute values. +oom-score-adj no + +# When oom-score-adj is used, this directive controls the specific values used +# for master, replica and background child processes. Values range -2000 to +# 2000 (higher means more likely to be killed). +# +# Unprivileged processes (not root, and without CAP_SYS_RESOURCE capabilities) +# can freely increase their value, but not decrease it below its initial +# settings. This means that setting oom-score-adj to "relative" and setting the +# oom-score-adj-values to positive values will always succeed. +oom-score-adj-values 0 200 800 + + +#################### KERNEL transparent hugepage CONTROL ###################### + +# Usually the kernel Transparent Huge Pages control is set to "madvise" or +# or "never" by default (/sys/kernel/mm/transparent_hugepage/enabled), in which +# case this config has no effect. On systems in which it is set to "always", +# redis will attempt to disable it specifically for the redis process in order +# to avoid latency problems specifically with fork(2) and CoW. +# If for some reason you prefer to keep it enabled, you can set this config to +# "no" and the kernel global to "always". + +disable-thp yes + +############################## APPEND ONLY MODE ############################### + +# By default Redis asynchronously dumps the dataset on disk. This mode is +# good enough in many applications, but an issue with the Redis process or +# a power outage may result into a few minutes of writes lost (depending on +# the configured save points). +# +# The Append Only File is an alternative persistence mode that provides +# much better durability. For instance using the default data fsync policy +# (see later in the config file) Redis can lose just one second of writes in a +# dramatic event like a server power outage, or a single write if something +# wrong with the Redis process itself happens, but the operating system is +# still running correctly. +# +# AOF and RDB persistence can be enabled at the same time without problems. +# If the AOF is enabled on startup Redis will load the AOF, that is the file +# with the better durability guarantees. +# +# Please check https://redis.io/topics/persistence for more information. + +appendonly yes + +# The name of the append only file (default: "appendonly.aof") + +appendfilename "appendonly.aof" + +# The fsync() call tells the Operating System to actually write data on disk +# instead of waiting for more data in the output buffer. Some OS will really flush +# data on disk, some other OS will just try to do it ASAP. +# +# Redis supports three different modes: +# +# no: don't fsync, just let the OS flush the data when it wants. Faster. +# always: fsync after every write to the append only log. Slow, Safest. +# everysec: fsync only one time every second. Compromise. +# +# The default is "everysec", as that's usually the right compromise between +# speed and data safety. It's up to you to understand if you can relax this to +# "no" that will let the operating system flush the output buffer when +# it wants, for better performances (but if you can live with the idea of +# some data loss consider the default persistence mode that's snapshotting), +# or on the contrary, use "always" that's very slow but a bit safer than +# everysec. +# +# More details please check the following article: +# http://antirez.com/post/redis-persistence-demystified.html +# +# If unsure, use "everysec". + +# appendfsync always +appendfsync everysec +# appendfsync no + +# When the AOF fsync policy is set to always or everysec, and a background +# saving process (a background save or AOF log background rewriting) is +# performing a lot of I/O against the disk, in some Linux configurations +# Redis may block too long on the fsync() call. Note that there is no fix for +# this currently, as even performing fsync in a different thread will block +# our synchronous write(2) call. +# +# In order to mitigate this problem it's possible to use the following option +# that will prevent fsync() from being called in the main process while a +# BGSAVE or BGREWRITEAOF is in progress. +# +# This means that while another child is saving, the durability of Redis is +# the same as "appendfsync none". In practical terms, this means that it is +# possible to lose up to 30 seconds of log in the worst scenario (with the +# default Linux settings). +# +# If you have latency problems turn this to "yes". Otherwise leave it as +# "no" that is the safest pick from the point of view of durability. + +no-appendfsync-on-rewrite no + +# Automatic rewrite of the append only file. +# Redis is able to automatically rewrite the log file implicitly calling +# BGREWRITEAOF when the AOF log size grows by the specified percentage. +# +# This is how it works: Redis remembers the size of the AOF file after the +# latest rewrite (if no rewrite has happened since the restart, the size of +# the AOF at startup is used). +# +# This base size is compared to the current size. If the current size is +# bigger than the specified percentage, the rewrite is triggered. Also +# you need to specify a minimal size for the AOF file to be rewritten, this +# is useful to avoid rewriting the AOF file even if the percentage increase +# is reached but it is still pretty small. +# +# Specify a percentage of zero in order to disable the automatic AOF +# rewrite feature. + +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb + +# An AOF file may be found to be truncated at the end during the Redis +# startup process, when the AOF data gets loaded back into memory. +# This may happen when the system where Redis is running +# crashes, especially when an ext4 filesystem is mounted without the +# data=ordered option (however this can't happen when Redis itself +# crashes or aborts but the operating system still works correctly). +# +# Redis can either exit with an error when this happens, or load as much +# data as possible (the default now) and start if the AOF file is found +# to be truncated at the end. The following option controls this behavior. +# +# If aof-load-truncated is set to yes, a truncated AOF file is loaded and +# the Redis server starts emitting a log to inform the user of the event. +# Otherwise if the option is set to no, the server aborts with an error +# and refuses to start. When the option is set to no, the user requires +# to fix the AOF file using the "redis-check-aof" utility before to restart +# the server. +# +# Note that if the AOF file will be found to be corrupted in the middle +# the server will still exit with an error. This option only applies when +# Redis will try to read more data from the AOF file but not enough bytes +# will be found. +aof-load-truncated yes + +# When rewriting the AOF file, Redis is able to use an RDB preamble in the +# AOF file for faster rewrites and recoveries. When this option is turned +# on the rewritten AOF file is composed of two different stanzas: +# +# [RDB file][AOF tail] +# +# When loading, Redis recognizes that the AOF file starts with the "REDIS" +# string and loads the prefixed RDB file, then continues loading the AOF +# tail. +aof-use-rdb-preamble yes + +################################ LUA SCRIPTING ############################### + +# Max execution time of a Lua script in milliseconds. +# +# If the maximum execution time is reached Redis will log that a script is +# still in execution after the maximum allowed time and will start to +# reply to queries with an error. +# +# When a long running script exceeds the maximum execution time only the +# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be +# used to stop a script that did not yet call any write commands. The second +# is the only way to shut down the server in the case a write command was +# already issued by the script but the user doesn't want to wait for the natural +# termination of the script. +# +# Set it to 0 or a negative value for unlimited execution without warnings. +lua-time-limit 5000 + +################################ REDIS CLUSTER ############################### + +# Normal Redis instances can't be part of a Redis Cluster; only nodes that are +# started as cluster nodes can. In order to start a Redis instance as a +# cluster node enable the cluster support uncommenting the following: +# +# cluster-enabled yes + +# Every cluster node has a cluster configuration file. This file is not +# intended to be edited by hand. It is created and updated by Redis nodes. +# Every Redis Cluster node requires a different cluster configuration file. +# Make sure that instances running in the same system do not have +# overlapping cluster configuration file names. +# +# cluster-config-file nodes-6379.conf + +# Cluster node timeout is the amount of milliseconds a node must be unreachable +# for it to be considered in failure state. +# Most other internal time limits are a multiple of the node timeout. +# +# cluster-node-timeout 15000 + +# A replica of a failing master will avoid to start a failover if its data +# looks too old. +# +# There is no simple way for a replica to actually have an exact measure of +# its "data age", so the following two checks are performed: +# +# 1) If there are multiple replicas able to failover, they exchange messages +# in order to try to give an advantage to the replica with the best +# replication offset (more data from the master processed). +# Replicas will try to get their rank by offset, and apply to the start +# of the failover a delay proportional to their rank. +# +# 2) Every single replica computes the time of the last interaction with +# its master. This can be the last ping or command received (if the master +# is still in the "connected" state), or the time that elapsed since the +# disconnection with the master (if the replication link is currently down). +# If the last interaction is too old, the replica will not try to failover +# at all. +# +# The point "2" can be tuned by user. Specifically a replica will not perform +# the failover if, since the last interaction with the master, the time +# elapsed is greater than: +# +# (node-timeout * cluster-replica-validity-factor) + repl-ping-replica-period +# +# So for example if node-timeout is 30 seconds, and the cluster-replica-validity-factor +# is 10, and assuming a default repl-ping-replica-period of 10 seconds, the +# replica will not try to failover if it was not able to talk with the master +# for longer than 310 seconds. +# +# A large cluster-replica-validity-factor may allow replicas with too old data to failover +# a master, while a too small value may prevent the cluster from being able to +# elect a replica at all. +# +# For maximum availability, it is possible to set the cluster-replica-validity-factor +# to a value of 0, which means, that replicas will always try to failover the +# master regardless of the last time they interacted with the master. +# (However they'll always try to apply a delay proportional to their +# offset rank). +# +# Zero is the only value able to guarantee that when all the partitions heal +# the cluster will always be able to continue. +# +# cluster-replica-validity-factor 10 + +# Cluster replicas are able to migrate to orphaned masters, that are masters +# that are left without working replicas. This improves the cluster ability +# to resist to failures as otherwise an orphaned master can't be failed over +# in case of failure if it has no working replicas. +# +# Replicas migrate to orphaned masters only if there are still at least a +# given number of other working replicas for their old master. This number +# is the "migration barrier". A migration barrier of 1 means that a replica +# will migrate only if there is at least 1 other working replica for its master +# and so forth. It usually reflects the number of replicas you want for every +# master in your cluster. +# +# Default is 1 (replicas migrate only if their masters remain with at least +# one replica). To disable migration just set it to a very large value or +# set cluster-allow-replica-migration to 'no'. +# A value of 0 can be set but is useful only for debugging and dangerous +# in production. +# +# cluster-migration-barrier 1 + +# Turning off this option allows to use less automatic cluster configuration. +# It both disables migration to orphaned masters and migration from masters +# that became empty. +# +# Default is 'yes' (allow automatic migrations). +# +# cluster-allow-replica-migration yes + +# By default Redis Cluster nodes stop accepting queries if they detect there +# is at least a hash slot uncovered (no available node is serving it). +# This way if the cluster is partially down (for example a range of hash slots +# are no longer covered) all the cluster becomes, eventually, unavailable. +# It automatically returns available as soon as all the slots are covered again. +# +# However sometimes you want the subset of the cluster which is working, +# to continue to accept queries for the part of the key space that is still +# covered. In order to do so, just set the cluster-require-full-coverage +# option to no. +# +# cluster-require-full-coverage yes + +# This option, when set to yes, prevents replicas from trying to failover its +# master during master failures. However the replica can still perform a +# manual failover, if forced to do so. +# +# This is useful in different scenarios, especially in the case of multiple +# data center operations, where we want one side to never be promoted if not +# in the case of a total DC failure. +# +# cluster-replica-no-failover no + +# This option, when set to yes, allows nodes to serve read traffic while the +# the cluster is in a down state, as long as it believes it owns the slots. +# +# This is useful for two cases. The first case is for when an application +# doesn't require consistency of data during node failures or network partitions. +# One example of this is a cache, where as long as the node has the data it +# should be able to serve it. +# +# The second use case is for configurations that don't meet the recommended +# three shards but want to enable cluster mode and scale later. A +# master outage in a 1 or 2 shard configuration causes a read/write outage to the +# entire cluster without this option set, with it set there is only a write outage. +# Without a quorum of masters, slot ownership will not change automatically. +# +# cluster-allow-reads-when-down no + +# In order to setup your cluster make sure to read the documentation +# available at https://redis.io web site. + +########################## CLUSTER DOCKER/NAT support ######################## + +# In certain deployments, Redis Cluster nodes address discovery fails, because +# addresses are NAT-ted or because ports are forwarded (the typical case is +# Docker and other containers). +# +# In order to make Redis Cluster working in such environments, a static +# configuration where each node knows its public address is needed. The +# following four options are used for this scope, and are: +# +# * cluster-announce-ip +# * cluster-announce-port +# * cluster-announce-tls-port +# * cluster-announce-bus-port +# +# Each instructs the node about its address, client ports (for connections +# without and with TLS) and cluster message bus port. The information is then +# published in the header of the bus packets so that other nodes will be able to +# correctly map the address of the node publishing the information. +# +# If cluster-tls is set to yes and cluster-announce-tls-port is omitted or set +# to zero, then cluster-announce-port refers to the TLS port. Note also that +# cluster-announce-tls-port has no effect if cluster-tls is set to no. +# +# If the above options are not used, the normal Redis Cluster auto-detection +# will be used instead. +# +# Note that when remapped, the bus port may not be at the fixed offset of +# clients port + 10000, so you can specify any port and bus-port depending +# on how they get remapped. If the bus-port is not set, a fixed offset of +# 10000 will be used as usual. +# +# Example: +# +# cluster-announce-ip 10.1.1.5 +# cluster-announce-tls-port 6379 +# cluster-announce-port 0 +# cluster-announce-bus-port 6380 + +################################## SLOW LOG ################################### + +# The Redis Slow Log is a system to log queries that exceeded a specified +# execution time. The execution time does not include the I/O operations +# like talking with the client, sending the reply and so forth, +# but just the time needed to actually execute the command (this is the only +# stage of command execution where the thread is blocked and can not serve +# other requests in the meantime). +# +# You can configure the slow log with two parameters: one tells Redis +# what is the execution time, in microseconds, to exceed in order for the +# command to get logged, and the other parameter is the length of the +# slow log. When a new command is logged the oldest one is removed from the +# queue of logged commands. + +# The following time is expressed in microseconds, so 1000000 is equivalent +# to one second. Note that a negative number disables the slow log, while +# a value of zero forces the logging of every command. +slowlog-log-slower-than 10000 + +# There is no limit to this length. Just be aware that it will consume memory. +# You can reclaim memory used by the slow log with SLOWLOG RESET. +slowlog-max-len 128 + +################################ LATENCY MONITOR ############################## + +# The Redis latency monitoring subsystem samples different operations +# at runtime in order to collect data related to possible sources of +# latency of a Redis instance. +# +# Via the LATENCY command this information is available to the user that can +# print graphs and obtain reports. +# +# The system only logs operations that were performed in a time equal or +# greater than the amount of milliseconds specified via the +# latency-monitor-threshold configuration directive. When its value is set +# to zero, the latency monitor is turned off. +# +# By default latency monitoring is disabled since it is mostly not needed +# if you don't have latency issues, and collecting data has a performance +# impact, that while very small, can be measured under big load. Latency +# monitoring can easily be enabled at runtime using the command +# "CONFIG SET latency-monitor-threshold " if needed. +latency-monitor-threshold 0 + +############################# EVENT NOTIFICATION ############################## + +# Redis can notify Pub/Sub clients about events happening in the key space. +# This feature is documented at https://redis.io/topics/notifications +# +# For instance if keyspace events notification is enabled, and a client +# performs a DEL operation on key "foo" stored in the Database 0, two +# messages will be published via Pub/Sub: +# +# PUBLISH __keyspace@0__:foo del +# PUBLISH __keyevent@0__:del foo +# +# It is possible to select the events that Redis will notify among a set +# of classes. Every class is identified by a single character: +# +# K Keyspace events, published with __keyspace@__ prefix. +# E Keyevent events, published with __keyevent@__ prefix. +# g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... +# $ String commands +# l List commands +# s Set commands +# h Hash commands +# z Sorted set commands +# x Expired events (events generated every time a key expires) +# e Evicted events (events generated when a key is evicted for maxmemory) +# t Stream commands +# d Module key type events +# m Key-miss events (Note: It is not included in the 'A' class) +# A Alias for g$lshzxetd, so that the "AKE" string means all the events +# (Except key-miss events which are excluded from 'A' due to their +# unique nature). +# +# The "notify-keyspace-events" takes as argument a string that is composed +# of zero or multiple characters. The empty string means that notifications +# are disabled. +# +# Example: to enable list and generic events, from the point of view of the +# event name, use: +# +# notify-keyspace-events Elg +# +# Example 2: to get the stream of the expired keys subscribing to channel +# name __keyevent@0__:expired use: +# +# notify-keyspace-events Ex +# +# By default all notifications are disabled because most users don't need +# this feature and the feature has some overhead. Note that if you don't +# specify at least one of K or E, no events will be delivered. +notify-keyspace-events "" + +############################### GOPHER SERVER ################################# + +# Redis contains an implementation of the Gopher protocol, as specified in +# the RFC 1436 (https://www.ietf.org/rfc/rfc1436.txt). +# +# The Gopher protocol was very popular in the late '90s. It is an alternative +# to the web, and the implementation both server and client side is so simple +# that the Redis server has just 100 lines of code in order to implement this +# support. +# +# What do you do with Gopher nowadays? Well Gopher never *really* died, and +# lately there is a movement in order for the Gopher more hierarchical content +# composed of just plain text documents to be resurrected. Some want a simpler +# internet, others believe that the mainstream internet became too much +# controlled, and it's cool to create an alternative space for people that +# want a bit of fresh air. +# +# Anyway for the 10nth birthday of the Redis, we gave it the Gopher protocol +# as a gift. +# +# --- HOW IT WORKS? --- +# +# The Redis Gopher support uses the inline protocol of Redis, and specifically +# two kind of inline requests that were anyway illegal: an empty request +# or any request that starts with "/" (there are no Redis commands starting +# with such a slash). Normal RESP2/RESP3 requests are completely out of the +# path of the Gopher protocol implementation and are served as usual as well. +# +# If you open a connection to Redis when Gopher is enabled and send it +# a string like "/foo", if there is a key named "/foo" it is served via the +# Gopher protocol. +# +# In order to create a real Gopher "hole" (the name of a Gopher site in Gopher +# talking), you likely need a script like the following: +# +# https://github.com/antirez/gopher2redis +# +# --- SECURITY WARNING --- +# +# If you plan to put Redis on the internet in a publicly accessible address +# to server Gopher pages MAKE SURE TO SET A PASSWORD to the instance. +# Once a password is set: +# +# 1. The Gopher server (when enabled, not by default) will still serve +# content via Gopher. +# 2. However other commands cannot be called before the client will +# authenticate. +# +# So use the 'requirepass' option to protect your instance. +# +# Note that Gopher is not currently supported when 'io-threads-do-reads' +# is enabled. +# +# To enable Gopher support, uncomment the following line and set the option +# from no (the default) to yes. +# +# gopher-enabled no + +############################### ADVANCED CONFIG ############################### + +# Hashes are encoded using a memory efficient data structure when they have a +# small number of entries, and the biggest entry does not exceed a given +# threshold. These thresholds can be configured using the following directives. +hash-max-ziplist-entries 512 +hash-max-ziplist-value 64 + +# Lists are also encoded in a special way to save a lot of space. +# The number of entries allowed per internal list node can be specified +# as a fixed maximum size or a maximum number of elements. +# For a fixed maximum size, use -5 through -1, meaning: +# -5: max size: 64 Kb <-- not recommended for normal workloads +# -4: max size: 32 Kb <-- not recommended +# -3: max size: 16 Kb <-- probably not recommended +# -2: max size: 8 Kb <-- good +# -1: max size: 4 Kb <-- good +# Positive numbers mean store up to _exactly_ that number of elements +# per list node. +# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), +# but if your use case is unique, adjust the settings as necessary. +list-max-ziplist-size -2 + +# Lists may also be compressed. +# Compress depth is the number of quicklist ziplist nodes from *each* side of +# the list to *exclude* from compression. The head and tail of the list +# are always uncompressed for fast push/pop operations. Settings are: +# 0: disable all list compression +# 1: depth 1 means "don't start compressing until after 1 node into the list, +# going from either the head or tail" +# So: [head]->node->node->...->node->[tail] +# [head], [tail] will always be uncompressed; inner nodes will compress. +# 2: [head]->[next]->node->node->...->node->[prev]->[tail] +# 2 here means: don't compress head or head->next or tail->prev or tail, +# but compress all nodes between them. +# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] +# etc. +list-compress-depth 0 + +# Sets have a special encoding in just one case: when a set is composed +# of just strings that happen to be integers in radix 10 in the range +# of 64 bit signed integers. +# The following configuration setting sets the limit in the size of the +# set in order to use this special memory saving encoding. +set-max-intset-entries 512 + +# Similarly to hashes and lists, sorted sets are also specially encoded in +# order to save a lot of space. This encoding is only used when the length and +# elements of a sorted set are below the following limits: +zset-max-ziplist-entries 128 +zset-max-ziplist-value 64 + +# HyperLogLog sparse representation bytes limit. The limit includes the +# 16 bytes header. When an HyperLogLog using the sparse representation crosses +# this limit, it is converted into the dense representation. +# +# A value greater than 16000 is totally useless, since at that point the +# dense representation is more memory efficient. +# +# The suggested value is ~ 3000 in order to have the benefits of +# the space efficient encoding without slowing down too much PFADD, +# which is O(N) with the sparse encoding. The value can be raised to +# ~ 10000 when CPU is not a concern, but space is, and the data set is +# composed of many HyperLogLogs with cardinality in the 0 - 15000 range. +hll-sparse-max-bytes 3000 + +# Streams macro node max size / items. The stream data structure is a radix +# tree of big nodes that encode multiple items inside. Using this configuration +# it is possible to configure how big a single node can be in bytes, and the +# maximum number of items it may contain before switching to a new node when +# appending new stream entries. If any of the following settings are set to +# zero, the limit is ignored, so for instance it is possible to set just a +# max entries limit by setting max-bytes to 0 and max-entries to the desired +# value. +stream-node-max-bytes 4096 +stream-node-max-entries 100 + +# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in +# order to help rehashing the main Redis hash table (the one mapping top-level +# keys to values). The hash table implementation Redis uses (see dict.c) +# performs a lazy rehashing: the more operation you run into a hash table +# that is rehashing, the more rehashing "steps" are performed, so if the +# server is idle the rehashing is never complete and some more memory is used +# by the hash table. +# +# The default is to use this millisecond 10 times every second in order to +# actively rehash the main dictionaries, freeing memory when possible. +# +# If unsure: +# use "activerehashing no" if you have hard latency requirements and it is +# not a good thing in your environment that Redis can reply from time to time +# to queries with 2 milliseconds delay. +# +# use "activerehashing yes" if you don't have such hard requirements but +# want to free memory asap when possible. +activerehashing yes + +# The client output buffer limits can be used to force disconnection of clients +# that are not reading data from the server fast enough for some reason (a +# common reason is that a Pub/Sub client can't consume messages as fast as the +# publisher can produce them). +# +# The limit can be set differently for the three different classes of clients: +# +# normal -> normal clients including MONITOR clients +# replica -> replica clients +# pubsub -> clients subscribed to at least one pubsub channel or pattern +# +# The syntax of every client-output-buffer-limit directive is the following: +# +# client-output-buffer-limit +# +# A client is immediately disconnected once the hard limit is reached, or if +# the soft limit is reached and remains reached for the specified number of +# seconds (continuously). +# So for instance if the hard limit is 32 megabytes and the soft limit is +# 16 megabytes / 10 seconds, the client will get disconnected immediately +# if the size of the output buffers reach 32 megabytes, but will also get +# disconnected if the client reaches 16 megabytes and continuously overcomes +# the limit for 10 seconds. +# +# By default normal clients are not limited because they don't receive data +# without asking (in a push way), but just after a request, so only +# asynchronous clients may create a scenario where data is requested faster +# than it can read. +# +# Instead there is a default limit for pubsub and replica clients, since +# subscribers and replicas receive data in a push fashion. +# +# Both the hard or the soft limit can be disabled by setting them to zero. +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit replica 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 + +# Client query buffers accumulate new commands. They are limited to a fixed +# amount by default in order to avoid that a protocol desynchronization (for +# instance due to a bug in the client) will lead to unbound memory usage in +# the query buffer. However you can configure it here if you have very special +# needs, such us huge multi/exec requests or alike. +# +# client-query-buffer-limit 1gb + +# In the Redis protocol, bulk requests, that are, elements representing single +# strings, are normally limited to 512 mb. However you can change this limit +# here, but must be 1mb or greater +# +# proto-max-bulk-len 512mb + +# Redis calls an internal function to perform many background tasks, like +# closing connections of clients in timeout, purging expired keys that are +# never requested, and so forth. +# +# Not all tasks are performed with the same frequency, but Redis checks for +# tasks to perform according to the specified "hz" value. +# +# By default "hz" is set to 10. Raising the value will use more CPU when +# Redis is idle, but at the same time will make Redis more responsive when +# there are many keys expiring at the same time, and timeouts may be +# handled with more precision. +# +# The range is between 1 and 500, however a value over 100 is usually not +# a good idea. Most users should use the default of 10 and raise this up to +# 100 only in environments where very low latency is required. +hz 10 + +# Normally it is useful to have an HZ value which is proportional to the +# number of clients connected. This is useful in order, for instance, to +# avoid too many clients are processed for each background task invocation +# in order to avoid latency spikes. +# +# Since the default HZ value by default is conservatively set to 10, Redis +# offers, and enables by default, the ability to use an adaptive HZ value +# which will temporarily raise when there are many connected clients. +# +# When dynamic HZ is enabled, the actual configured HZ will be used +# as a baseline, but multiples of the configured HZ value will be actually +# used as needed once more clients are connected. In this way an idle +# instance will use very little CPU time while a busy instance will be +# more responsive. +dynamic-hz yes + +# When a child rewrites the AOF file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +aof-rewrite-incremental-fsync yes + +# When redis saves RDB file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +rdb-save-incremental-fsync yes + +# Redis LFU eviction (see maxmemory setting) can be tuned. However it is a good +# idea to start with the default settings and only change them after investigating +# how to improve the performances and how the keys LFU change over time, which +# is possible to inspect via the OBJECT FREQ command. +# +# There are two tunable parameters in the Redis LFU implementation: the +# counter logarithm factor and the counter decay time. It is important to +# understand what the two parameters mean before changing them. +# +# The LFU counter is just 8 bits per key, it's maximum value is 255, so Redis +# uses a probabilistic increment with logarithmic behavior. Given the value +# of the old counter, when a key is accessed, the counter is incremented in +# this way: +# +# 1. A random number R between 0 and 1 is extracted. +# 2. A probability P is calculated as 1/(old_value*lfu_log_factor+1). +# 3. The counter is incremented only if R < P. +# +# The default lfu-log-factor is 10. This is a table of how the frequency +# counter changes with a different number of accesses with different +# logarithmic factors: +# +# +--------+------------+------------+------------+------------+------------+ +# | factor | 100 hits | 1000 hits | 100K hits | 1M hits | 10M hits | +# +--------+------------+------------+------------+------------+------------+ +# | 0 | 104 | 255 | 255 | 255 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# | 1 | 18 | 49 | 255 | 255 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# | 10 | 10 | 18 | 142 | 255 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# | 100 | 8 | 11 | 49 | 143 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# +# NOTE: The above table was obtained by running the following commands: +# +# redis-benchmark -n 1000000 incr foo +# redis-cli object freq foo +# +# NOTE 2: The counter initial value is 5 in order to give new objects a chance +# to accumulate hits. +# +# The counter decay time is the time, in minutes, that must elapse in order +# for the key counter to be divided by two (or decremented if it has a value +# less <= 10). +# +# The default value for the lfu-decay-time is 1. A special value of 0 means to +# decay the counter every time it happens to be scanned. +# +# lfu-log-factor 10 +# lfu-decay-time 1 + +########################### ACTIVE DEFRAGMENTATION ####################### +# +# What is active defragmentation? +# ------------------------------- +# +# Active (online) defragmentation allows a Redis server to compact the +# spaces left between small allocations and deallocations of data in memory, +# thus allowing to reclaim back memory. +# +# Fragmentation is a natural process that happens with every allocator (but +# less so with Jemalloc, fortunately) and certain workloads. Normally a server +# restart is needed in order to lower the fragmentation, or at least to flush +# away all the data and create it again. However thanks to this feature +# implemented by Oran Agra for Redis 4.0 this process can happen at runtime +# in a "hot" way, while the server is running. +# +# Basically when the fragmentation is over a certain level (see the +# configuration options below) Redis will start to create new copies of the +# values in contiguous memory regions by exploiting certain specific Jemalloc +# features (in order to understand if an allocation is causing fragmentation +# and to allocate it in a better place), and at the same time, will release the +# old copies of the data. This process, repeated incrementally for all the keys +# will cause the fragmentation to drop back to normal values. +# +# Important things to understand: +# +# 1. This feature is disabled by default, and only works if you compiled Redis +# to use the copy of Jemalloc we ship with the source code of Redis. +# This is the default with Linux builds. +# +# 2. You never need to enable this feature if you don't have fragmentation +# issues. +# +# 3. Once you experience fragmentation, you can enable this feature when +# needed with the command "CONFIG SET activedefrag yes". +# +# The configuration parameters are able to fine tune the behavior of the +# defragmentation process. If you are not sure about what they mean it is +# a good idea to leave the defaults untouched. + +# Enabled active defragmentation +# activedefrag no + +# Minimum amount of fragmentation waste to start active defrag +# active-defrag-ignore-bytes 100mb + +# Minimum percentage of fragmentation to start active defrag +# active-defrag-threshold-lower 10 + +# Maximum percentage of fragmentation at which we use maximum effort +# active-defrag-threshold-upper 100 + +# Minimal effort for defrag in CPU percentage, to be used when the lower +# threshold is reached +# active-defrag-cycle-min 1 + +# Maximal effort for defrag in CPU percentage, to be used when the upper +# threshold is reached +# active-defrag-cycle-max 25 + +# Maximum number of set/hash/zset/list fields that will be processed from +# the main dictionary scan +# active-defrag-max-scan-fields 1000 + +# Jemalloc background thread for purging will be enabled by default +jemalloc-bg-thread yes + +# It is possible to pin different threads and processes of Redis to specific +# CPUs in your system, in order to maximize the performances of the server. +# This is useful both in order to pin different Redis threads in different +# CPUs, but also in order to make sure that multiple Redis instances running +# in the same host will be pinned to different CPUs. +# +# Normally you can do this using the "taskset" command, however it is also +# possible to this via Redis configuration directly, both in Linux and FreeBSD. +# +# You can pin the server/IO threads, bio threads, aof rewrite child process, and +# the bgsave child process. The syntax to specify the cpu list is the same as +# the taskset command: +# +# Set redis server/io threads to cpu affinity 0,2,4,6: +# server_cpulist 0-7:2 +# +# Set bio threads to cpu affinity 1,3: +# bio_cpulist 1,3 +# +# Set aof rewrite child process to cpu affinity 8,9,10,11: +# aof_rewrite_cpulist 8-11 +# +# Set bgsave child process to cpu affinity 1,10,11 +# bgsave_cpulist 1,10-11 + +# In some cases redis will emit warnings and even refuse to start if it detects +# that the system is in bad state, it is possible to suppress these warnings +# by setting the following config which takes a space delimited list of warnings +# to suppress +# +# ignore-warnings ARM64-COW-BUG \ No newline at end of file diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py new file mode 100644 index 000000000..8a570b70d --- /dev/null +++ b/merlin/server/server_commands.py @@ -0,0 +1,308 @@ +"""Main functions for instantiating and running Merlin server containers.""" + +############################################################################### +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### + +import logging +import os +import socket +import subprocess +import time +from argparse import Namespace + +from merlin.server.server_config import ( + ServerStatus, + config_merlin_server, + create_server_config, + dump_process_file, + get_server_status, + parse_redis_output, + pull_process_file, + pull_server_config, + pull_server_image, +) +from merlin.server.server_util import AppYaml, RedisConfig, RedisUsers + + +LOG = logging.getLogger("merlin") + + +def init_server() -> None: + """ + Initialize merlin server by checking and initializing main configuration directory + and local server configuration. + """ + + if not create_server_config(): + LOG.info("Merlin server initialization failed.") + return + pull_server_image() + + config_merlin_server() + + LOG.info("Merlin server initialization successful.") + + +def config_server(args: Namespace) -> None: + """ + Process the merlin server config flags to make changes and edits to appropriate configurations + based on the input passed in by the user. + """ + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + redis_config = RedisConfig(server_config.container.get_config_path()) + + redis_config.set_ip_address(args.ipaddress) + + redis_config.set_port(args.port) + + redis_config.set_password(args.password) + if args.password is not None: + redis_users = RedisUsers(server_config.container.get_user_file_path()) + redis_users.set_password("default", args.password) + redis_users.write() + + redis_config.set_directory(args.directory) + + redis_config.set_snapshot_seconds(args.snapshot_seconds) + + redis_config.set_snapshot_changes(args.snapshot_changes) + + redis_config.set_snapshot_file(args.snapshot_file) + + redis_config.set_append_mode(args.append_mode) + + redis_config.set_append_file(args.append_file) + + if redis_config.changes_made(): + redis_config.write() + LOG.info("Merlin server config has changed. Restart merlin server to apply new configuration.") + LOG.info("Run 'merlin server restart' to restart running merlin server") + LOG.info("Run 'merlin server start' to start merlin server instance.") + else: + LOG.info("Add changes to config file and exisiting containers.") + + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + + # Read the user from the list of avaliable users + redis_users = RedisUsers(server_config.container.get_user_file_path()) + redis_config = RedisConfig(server_config.container.get_config_path()) + + if args.add_user is not None: + # Log the user in a file + if redis_users.add_user(user=args.add_user[0], password=args.add_user[1]): + redis_users.write() + LOG.info(f"Added user {args.add_user[0]} to merlin server") + # Create a new user in container + if get_server_status() == ServerStatus.RUNNING: + LOG.info("Adding user to current merlin server instance") + redis_users.apply_to_redis(redis_config.get_ip_address(), redis_config.get_port(), redis_config.get_password()) + else: + LOG.error(f"User '{args.add_user[0]}' already exisits within current users") + + if args.remove_user is not None: + # Remove user from file + if redis_users.remove_user(args.remove_user): + redis_users.write() + LOG.info(f"Removed user {args.remove_user} to merlin server") + # Remove user from container + if get_server_status() == ServerStatus.RUNNING: + LOG.info("Removing user to current merlin server instance") + redis_users.apply_to_redis(redis_config.get_ip_address(), redis_config.get_port(), redis_config.get_password()) + else: + LOG.error(f"User '{args.remove_user}' doesn't exist within current users.") + + +def status_server() -> None: + """ + Get the server status of the any current running containers for merlin server + """ + current_status = get_server_status() + if current_status == ServerStatus.NOT_INITALIZED: + LOG.info("Merlin server has not been initialized.") + LOG.info("Please initalize server by running 'merlin server init'") + elif current_status == ServerStatus.MISSING_CONTAINER: + LOG.info("Unable to find server image.") + LOG.info("Ensure there is a .sif file in merlin server directory.") + elif current_status == ServerStatus.NOT_RUNNING: + LOG.info("Merlin server is not running.") + elif current_status == ServerStatus.RUNNING: + LOG.info("Merlin server is running.") + + +def start_server() -> bool: + """ + Start a merlin server container using singularity. + :return:: True if server was successful started and False if failed. + """ + current_status = get_server_status() + + if current_status == ServerStatus.NOT_INITALIZED or current_status == ServerStatus.MISSING_CONTAINER: + LOG.info("Merlin server has not been initialized. Please run 'merlin server init' first.") + return False + + if current_status == ServerStatus.RUNNING: + LOG.info("Merlin server already running.") + LOG.info("Stop current server with 'merlin server stop' before attempting to start a new server.") + return False + + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + + image_path = server_config.container.get_image_path() + if not os.path.exists(image_path): + LOG.error("Unable to find image at " + image_path) + return False + + config_path = server_config.container.get_config_path() + if not os.path.exists(config_path): + LOG.error("Unable to find config file at " + config_path) + return False + + process = subprocess.Popen( + server_config.container_format.get_run_command() + .strip("\\") + .format( + command=server_config.container_format.get_command(), + home_dir=server_config.container.get_config_dir(), + image=image_path, + config=config_path, + ) + .split(), + start_new_session=True, + close_fds=True, + stdout=subprocess.PIPE, + ) + + time.sleep(1) + + redis_start, redis_out = parse_redis_output(process.stdout) + + if not redis_start: + LOG.error("Redis is unable to start") + LOG.error('Check to see if there is an unresponsive instance of redis with "ps -e"') + LOG.error(redis_out.strip("\n")) + return False + + redis_out["image_pid"] = redis_out.pop("pid") + redis_out["parent_pid"] = process.pid + redis_out["hostname"] = socket.gethostname() + if not dump_process_file(redis_out, server_config.container.get_pfile_path()): + LOG.error("Unable to create process file for container.") + return False + + if get_server_status() != ServerStatus.RUNNING: + LOG.error("Unable to start merlin server.") + return False + + LOG.info(f"Server started with PID {str(process.pid)}.") + LOG.info(f'Merlin server operating on "{redis_out["hostname"]}" and port "{redis_out["port"]}".') + + redis_users = RedisUsers(server_config.container.get_user_file_path()) + redis_config = RedisConfig(server_config.container.get_config_path()) + redis_users.apply_to_redis(redis_config.get_ip_address(), redis_config.get_port(), redis_config.get_password()) + + new_app_yaml = os.path.join(server_config.container.get_config_dir(), "app.yaml") + ay = AppYaml() + ay.apply_server_config(server_config=server_config) + ay.write(new_app_yaml) + LOG.info(f"New app.yaml written to {new_app_yaml}.") + LOG.info("Replace app.yaml in ~/.merlin/app.yaml to use merlin server as main configuration.") + LOG.info("To use for local runs, move app.yaml into the running directory.") + + return True + + +def stop_server(): + """ + Stop running merlin server containers. + :return:: True if server was stopped successfully and False if failed. + """ + if get_server_status() != ServerStatus.RUNNING: + LOG.info("There is no instance of merlin server running.") + LOG.info("Start a merlin server first with 'merlin server start'") + return False + + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + + pf_data = pull_process_file(server_config.container.get_pfile_path()) + read_pid = pf_data["parent_pid"] + + process = subprocess.run( + server_config.process.get_status_command().strip("\\").format(pid=read_pid).split(), stdout=subprocess.PIPE + ) + if process.stdout == b"": + LOG.error("Unable to get the PID for the current merlin server.") + return False + + command = server_config.process.get_kill_command().strip("\\").format(pid=read_pid).split() + if server_config.container_format.get_stop_command() != "kill": + command = ( + server_config.container_format.get_stop_command() + .strip("\\") + .format(name=server_config.container.get_image_name) + .split() + ) + + LOG.info(f"Attempting to close merlin server PID {str(read_pid)}") + + subprocess.run(command, stdout=subprocess.PIPE) + time.sleep(1) + if get_server_status() == ServerStatus.RUNNING: + LOG.error("Unable to kill process.") + return False + + LOG.info("Merlin server terminated.") + return True + + +def restart_server() -> bool: + """ + Restart a running merlin server instance. + :return:: True if server was restarted successfully and False if failed. + """ + if get_server_status() != ServerStatus.RUNNING: + LOG.info("Merlin server is not currently running.") + LOG.info("Please start a merlin server instance first with 'merlin server start'") + return False + stop_server() + time.sleep(1) + start_server() + return True diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py new file mode 100644 index 000000000..57cb5af22 --- /dev/null +++ b/merlin/server/server_config.py @@ -0,0 +1,387 @@ +############################################################################### +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### + +import enum +import logging +import os +import random +import shutil +import string +import subprocess +from io import BufferedReader +from typing import Tuple + +import yaml + +from merlin.server.server_util import ( + CONTAINER_TYPES, + MERLIN_CONFIG_DIR, + MERLIN_SERVER_CONFIG, + MERLIN_SERVER_SUBDIR, + AppYaml, + RedisConfig, + RedisUsers, + ServerConfig, +) + + +LOG = logging.getLogger("merlin") + +# Default values for configuration +CONFIG_DIR = os.path.abspath("./merlin_server/") +IMAGE_NAME = "redis_latest.sif" +PROCESS_FILE = "merlin_server.pf" +CONFIG_FILE = "redis.conf" +REDIS_URL = "docker://redis" +LOCAL_APP_YAML = "./app.yaml" + +PASSWORD_LENGTH = 256 + + +class ServerStatus(enum.Enum): + """ + Different states in which the server can be in. + """ + + RUNNING = 0 + NOT_INITALIZED = 1 + MISSING_CONTAINER = 2 + NOT_RUNNING = 3 + ERROR = 4 + + +def generate_password(length, pass_command: str = None) -> str: + """ + Function for generating passwords for redis container. If a specified command is given + then a password would be generated with the given command. If not a password will be + created by combining a string a characters based on the given length. + + :return:: string value with given length + """ + if pass_command: + process = subprocess.run(pass_command.split(), shell=True, stdout=subprocess.PIPE) + return process.stdout + + characters = list(string.ascii_letters + string.digits + "!@#$%^&*()") + + random.shuffle(characters) + + password = [] + for i in range(length): + password.append(random.choice(characters)) + + random.shuffle(password) + return "".join(password) + + +def parse_redis_output(redis_stdout: BufferedReader) -> Tuple[bool, str]: + """ + Parse the redis output for a the redis container. It will get all the necessary information + from the output and returns a dictionary of those values. + + :return:: two values is_successful, dictionary of values from redis output + """ + if redis_stdout is None: + return False, "None passed as redis output" + server_init = False + redis_config = {} + line = redis_stdout.readline() + while line != "" or line is not None: + if not server_init: + values = [ln for ln in line.split() if b"=" in ln] + for val in values: + key, value = val.split(b"=") + redis_config[key.decode("utf-8")] = value.strip(b",").strip(b".").decode("utf-8") + if b"Server initialized" in line: + server_init = True + if b"Ready to accept connections" in line: + return True, redis_config + if b"aborting" in line or b"Fatal error" in line: + return False, line.decode("utf-8") + line = redis_stdout.readline() + + +def create_server_config() -> bool: + """ + Create main configuration file for merlin server in the + merlin configuration directory. If a configuration already + exists it will not replace the current configuration and exit. + + :return:: True if success and False if fail + """ + if not os.path.exists(MERLIN_CONFIG_DIR): + LOG.error("Unable to find main merlin configuration directory at " + MERLIN_CONFIG_DIR) + return False + + config_dir = os.path.join(MERLIN_CONFIG_DIR, MERLIN_SERVER_SUBDIR) + if not os.path.exists(config_dir): + LOG.info("Unable to find exisiting server configuration.") + LOG.info(f"Creating default configuration in {config_dir}") + try: + os.mkdir(config_dir) + except OSError as err: + LOG.error(err) + return False + + files = [i + ".yaml" for i in CONTAINER_TYPES] + for file in files: + file_path = os.path.join(config_dir, file) + if os.path.exists(file_path): + LOG.info(f"{file} already exists.") + continue + LOG.info(f"Copying file {file} to configuration directory.") + try: + shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), file), config_dir) + except OSError: + LOG.error(f"Destination location {config_dir} is not writable.") + return False + + # Load Merlin Server Configuration and apply it to app.yaml + with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), MERLIN_SERVER_CONFIG)) as f: + main_server_config = yaml.load(f, yaml.Loader) + filename = LOCAL_APP_YAML if os.path.exists(LOCAL_APP_YAML) else AppYaml.default_filename + merlin_app_yaml = AppYaml(filename) + merlin_app_yaml.update_data(main_server_config) + merlin_app_yaml.write(filename) + + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + + if not os.path.exists(server_config.container.get_config_dir()): + LOG.info("Creating merlin server directory.") + os.mkdir(server_config.container.get_config_dir()) + + return True + + +def config_merlin_server(): + """ + Configurate the merlin server with configurations such as username password and etc. + """ + + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + + pass_file = server_config.container.get_pass_file_path() + if os.path.exists(pass_file): + LOG.info("Password file already exists. Skipping password generation step.") + else: + # if "pass_command" in server_config["container"]: + # password = generate_password(PASSWORD_LENGTH, server_config["container"]["pass_command"]) + # else: + password = generate_password(PASSWORD_LENGTH) + + with open(pass_file, "w+") as f: + f.write(password) + + LOG.info("Creating password file for merlin server container.") + + user_file = server_config.container.get_user_file_path() + if os.path.exists(user_file): + LOG.info("User file already exists.") + else: + redis_users = RedisUsers(user_file) + redis_config = RedisConfig(server_config.container.get_config_path()) + redis_config.set_password(server_config.container.get_container_password()) + redis_users.add_user(user="default", password=server_config.container.get_container_password()) + redis_users.add_user(user=os.environ.get("USER"), password=server_config.container.get_container_password()) + redis_users.write() + redis_config.write() + + LOG.info("User {} created in user file for merlin server container".format(os.environ.get("USER"))) + + +def pull_server_config() -> ServerConfig: + """ + Pull the main configuration file and corresponding format configuration file + as well. Returns the values as a dictionary. + + :return: A instance of ServerConfig containing all the necessary configuration values. + """ + return_data = {} + format_needed_keys = ["command", "run_command", "stop_command", "pull_command"] + process_needed_keys = ["status", "kill"] + + merlin_app_yaml = AppYaml(LOCAL_APP_YAML) + server_config = merlin_app_yaml.get_data() + return_data.update(server_config) + + config_dir = os.path.join(MERLIN_CONFIG_DIR, MERLIN_SERVER_SUBDIR) + + if "container" in server_config: + if "format" in server_config["container"]: + format_file = os.path.join(config_dir, server_config["container"]["format"] + ".yaml") + with open(format_file, "r") as ff: + format_data = yaml.load(ff, yaml.Loader) + for key in format_needed_keys: + if key not in format_data[server_config["container"]["format"]]: + LOG.error(f'Unable to find necessary "{key}" value in format config file {format_file}') + return None + return_data.update(format_data) + else: + LOG.error(f'Unable to find "format" in {merlin_app_yaml.default_filename}') + return None + else: + LOG.error(f'Unable to find "container" object in {merlin_app_yaml.default_filename}') + return None + + # Checking for process values that are needed for main functions and defaults + if "process" not in server_config: + LOG.error(f"Process config not found in {merlin_app_yaml.default_filename}") + return None + + for key in process_needed_keys: + if key not in server_config["process"]: + LOG.error(f'Process necessary "{key}" command configuration not found in {merlin_app_yaml.default_filename}') + return None + + return ServerConfig(return_data) + + +def pull_server_image() -> bool: + """ + Fetch the server image using singularity. + + :return:: True if success and False if fail + """ + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + + config_dir = server_config.container.get_config_dir() + config_file = server_config.container.get_config_name() + image_url = server_config.container.get_image_url() + image_path = server_config.container.get_image_path() + + if not os.path.exists(image_path): + LOG.info(f"Fetching redis image from {image_url}") + subprocess.run( + server_config.container_format.get_pull_command() + .strip("\\") + .format(command=server_config.container_format.get_command(), image=image_path, url=image_url) + .split(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + else: + LOG.info(f"{image_path} already exists.") + + if not os.path.exists(os.path.join(config_dir, config_file)): + LOG.info("Copying default redis configuration file.") + try: + file_dir = os.path.dirname(os.path.abspath(__file__)) + shutil.copy(os.path.join(file_dir, config_file), config_dir) + except OSError: + LOG.error(f"Destination location {config_dir} is not writable.") + return False + else: + LOG.info("Redis configuration file already exist.") + + return True + + +def get_server_status(): + """ + Determine the status of the current server. + This function can be used to check if the servers + have been initalized, started, or stopped. + + :param `server_dir`: location of all server related files. + :param `image_name`: name of the image when fetched. + :return:: A enum value of ServerStatus describing its current state. + """ + server_config = pull_server_config() + if not server_config: + return ServerStatus.NOT_INITALIZED + + if not os.path.exists(server_config.container.get_config_dir()): + return ServerStatus.NOT_INITALIZED + + if not os.path.exists(server_config.container.get_image_path()): + return ServerStatus.MISSING_CONTAINER + + if not os.path.exists(server_config.container.get_pfile_path()): + return ServerStatus.NOT_RUNNING + + pf_data = pull_process_file(server_config.container.get_pfile_path()) + parent_pid = pf_data["parent_pid"] + + check_process = subprocess.run( + server_config.process.get_status_command().strip("\\").format(pid=parent_pid).split(), + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + + if check_process.stdout == b"": + return ServerStatus.NOT_RUNNING + + return ServerStatus.RUNNING + + +def check_process_file_format(data: dict) -> bool: + """ + Check to see if the process file has the correct format and contains the expected key values. + :return:: True if success and False if fail + """ + required_keys = ["parent_pid", "image_pid", "port", "hostname"] + for key in required_keys: + if key not in data: + return False + return True + + +def pull_process_file(file_path: str) -> dict: + """ + Pull the data from the process file. If one is found returns the data in a dictionary + if not returns None + :return:: Data containing in process file. + """ + with open(file_path, "r") as f: + data = yaml.load(f, yaml.Loader) + if check_process_file_format(data): + return data + return None + + +def dump_process_file(data: dict, file_path: str): + """ + Dump the process data from the dictionary to the specified file path. + :return:: True if success and False if fail + """ + if not check_process_file_format(data): + return False + with open(file_path, "w+") as f: + yaml.dump(data, f, yaml.Dumper) + return True diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py new file mode 100644 index 000000000..e6eb6379d --- /dev/null +++ b/merlin/server/server_util.py @@ -0,0 +1,607 @@ +############################################################################### +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### + +import hashlib +import logging +import os + +import redis +import yaml + +import merlin.utils + + +LOG = logging.getLogger("merlin") + +# Constants for main merlin server configuration values. +CONTAINER_TYPES = ["singularity", "docker", "podman"] +MERLIN_CONFIG_DIR = os.path.join(os.path.expanduser("~"), ".merlin") +MERLIN_SERVER_SUBDIR = "server/" +MERLIN_SERVER_CONFIG = "merlin_server.yaml" + + +def valid_ipv4(ip: str) -> bool: + """ + Checks valid ip address + """ + if not ip: + return False + + arr = ip.split(".") + if len(arr) != 4: + return False + + for i in arr: + if int(i) < 0 and int(i) > 255: + return False + + return True + + +def valid_port(port: int) -> bool: + """ + Checks valid network port + """ + if port > 0 and port < 65536: + return True + return False + + +class ContainerConfig: + """ + ContainerConfig provides interface for parsing and interacting with the container value specified within + the merlin_server.yaml configuration file. Dictionary of the config values should be passed when initialized + to parse values. This can be done after parsing yaml to data dictionary. + If there are missing values within the configuration it will be populated with default values for + singularity container. + + Configuration contains values for setting up containers and storing values specific to each container. + Values that are stored consist of things within the local configuration directory as different runs + can have differnt configuration values. + """ + + # Default values for configuration + FORMAT = "singularity" + IMAGE_TYPE = "redis" + IMAGE_NAME = "redis_latest.sif" + REDIS_URL = "docker://redis" + CONFIG_FILE = "redis.conf" + CONFIG_DIR = os.path.abspath("./merlin_server/") + PROCESS_FILE = "merlin_server.pf" + PASSWORD_FILE = "redis.pass" + USERS_FILE = "redis.users" + + format = FORMAT + image_type = IMAGE_TYPE + image = IMAGE_NAME + url = REDIS_URL + config = CONFIG_FILE + config_dir = CONFIG_DIR + pfile = PROCESS_FILE + pass_file = PASSWORD_FILE + user_file = USERS_FILE + + def __init__(self, data: dict) -> None: + self.format = data["format"] if "format" in data else self.FORMAT + self.image_type = data["image_type"] if "image_type" in data else self.IMAGE_TYPE + self.image = data["image"] if "image" in data else self.IMAGE_NAME + self.url = data["url"] if "url" in data else self.REDIS_URL + self.config = data["config"] if "config" in data else self.CONFIG_FILE + self.config_dir = os.path.abspath(data["config_dir"]) if "config_dir" in data else self.CONFIG_DIR + self.pfile = data["pfile"] if "pfile" in data else self.PROCESS_FILE + self.pass_file = data["pass_file"] if "pass_file" in data else self.PASSWORD_FILE + self.user_file = data["user_file"] if "user_file" in data else self.USERS_FILE + + def get_format(self) -> str: + return self.format + + def get_image_type(self) -> str: + return self.image_type + + def get_image_name(self) -> str: + return self.image + + def get_image_url(self) -> str: + return self.url + + def get_image_path(self) -> str: + return os.path.join(self.config_dir, self.image) + + def get_config_name(self) -> str: + return self.config + + def get_config_path(self) -> str: + return os.path.join(self.config_dir, self.config) + + def get_config_dir(self) -> str: + return self.config_dir + + def get_pfile_name(self) -> str: + return self.pfile + + def get_pfile_path(self) -> str: + return os.path.join(self.config_dir, self.pfile) + + def get_pass_file_name(self) -> str: + return self.pass_file + + def get_pass_file_path(self) -> str: + return os.path.join(self.config_dir, self.pass_file) + + def get_user_file_name(self) -> str: + return self.user_file + + def get_user_file_path(self) -> str: + return os.path.join(self.config_dir, self.user_file) + + def get_container_password(self) -> str: + password = None + with open(self.get_pass_file_path(), "r") as f: + password = f.read() + return password + + +class ContainerFormatConfig: + """ + ContainerFormatConfig provides an interface for parsing and interacting with container specific + configuration files .yaml. These configuration files contain container specific + commands to run containerizers such as singularity, docker, and podman. + """ + + COMMAND = "singularity" + RUN_COMMAND = "{command} run {image} {config}" + STOP_COMMAND = "kill" + PULL_COMMAND = "{command} pull {image} {url}" + + command = COMMAND + run_command = RUN_COMMAND + stop_command = STOP_COMMAND + pull_command = PULL_COMMAND + + def __init__(self, data: dict) -> None: + self.command = data["command"] if "command" in data else self.COMMAND + self.run_command = data["run_command"] if "run_command" in data else self.RUN_COMMAND + self.stop_command = data["stop_command"] if "stop_command" in data else self.STOP_COMMAND + self.pull_command = data["pull_command"] if "pull_command" in data else self.PULL_COMMAND + + def get_command(self) -> str: + return self.command + + def get_run_command(self) -> str: + return self.run_command + + def get_stop_command(self) -> str: + return self.stop_command + + def get_pull_command(self) -> str: + return self.pull_command + + +class ProcessConfig: + """ + ProcessConfig provides an interface for parsing and interacting with process config specified + in merlin_server.yaml configuration. This configuration provide commands for interfacing with + host machine while the containers are running. + """ + + STATUS_COMMAND = "pgrep -P {pid}" + KILL_COMMAND = "kill {pid}" + + status = STATUS_COMMAND + kill = KILL_COMMAND + + def __init__(self, data: dict) -> None: + self.status = data["status"] if "status" in data else self.STATUS_COMMAND + self.kill = data["kill"] if "kill" in data else self.KILL_COMMAND + + def get_status_command(self) -> str: + return self.status + + def get_kill_command(self) -> str: + return self.kill + + +class ServerConfig: + """ + ServerConfig is an interface for storing all the necessary configuration for merlin server. + These configuration container things such as ContainerConfig, ProcessConfig, and ContainerFormatConfig. + """ + + container: ContainerConfig = None + process: ProcessConfig = None + container_format: ContainerFormatConfig = None + + def __init__(self, data: dict) -> None: + if "container" in data: + self.container = ContainerConfig(data["container"]) + if "process" in data: + self.process = ProcessConfig(data["process"]) + if self.container.get_format() in data: + self.container_format = ContainerFormatConfig(data[self.container.get_format()]) + + +class RedisConfig: + """ + RedisConfig is an interface for parsing and interacing with redis.conf file that is provided + by redis. This allows users to parse the given redis configuration and make edits and allow users + to write those changes into a redis readable config file. + """ + + filename = "" + entry_order = [] + entries = {} + comments = {} + trailing_comments = "" + changed = False + + def __init__(self, filename) -> None: + self.filename = filename + self.changed = False + self.parse() + + def parse(self) -> None: + self.entries = {} + self.comments = {} + with open(self.filename, "r+") as f: + file_contents = f.read() + file_lines = file_contents.split("\n") + comments = "" + for line in file_lines: + if len(line) > 0 and line[0] != "#": + line_contents = line.split(maxsplit=1) + if line_contents[0] in self.entries: + sub_split = line_contents[1].split(maxsplit=1) + line_contents[0] += " " + sub_split[0] + line_contents[1] = sub_split[1] + self.entry_order.append(line_contents[0]) + self.entries[line_contents[0]] = line_contents[1] + self.comments[line_contents[0]] = comments + comments = "" + else: + comments += line + "\n" + self.trailing_comments = comments[:-1] + + def write(self) -> None: + with open(self.filename, "w") as f: + for entry in self.entry_order: + f.write(self.comments[entry]) + f.write(f"{entry} {self.entries[entry]}\n") + f.write(self.trailing_comments) + + def set_filename(self, filename: str) -> None: + self.filename = filename + + def set_config_value(self, key: str, value: str) -> bool: + if key not in self.entries: + return False + self.entries[key] = value + self.changed = True + return True + + def get_config_value(self, key: str) -> str: + if key in self.entries: + return self.entries[key] + return None + + def changes_made(self) -> bool: + return self.changed + + def get_ip_address(self) -> str: + return self.get_config_value("bind") + + def set_ip_address(self, ipaddress: str) -> bool: + if ipaddress is None: + return False + # Check if ipaddress is valid + if valid_ipv4(ipaddress): + # Set ip address in redis config + if not self.set_config_value("bind", ipaddress): + LOG.error("Unable to set ip address for redis config") + return False + else: + LOG.error("Invalid IPv4 address given.") + return False + LOG.info(f"Ipaddress is set to {ipaddress}") + return True + + def get_port(self) -> str: + return self.get_config_value("port") + + def set_port(self, port: str) -> bool: + if port is None: + return False + # Check if port is valid + if valid_port(port): + # Set port in redis config + if not self.set_config_value("port", port): + LOG.error("Unable to set port for redis config") + return False + else: + LOG.error("Invalid port given.") + return False + LOG.info(f"Port is set to {port}") + return True + + def set_password(self, password: str) -> bool: + if password is None: + return False + self.set_config_value("requirepass", password) + LOG.info("New password set") + return True + + def get_password(self) -> str: + return self.get_config_value("requirepass") + + def set_directory(self, directory: str) -> bool: + if directory is None: + return False + if not os.path.exists(directory): + os.mkdir(directory) + LOG.info(f"Created directory {directory}") + # Validate the directory input + if os.path.exists(directory): + # Set the save directory to the redis config + if not self.set_config_value("dir", directory): + LOG.error("Unable to set directory for redis config") + return False + else: + LOG.error(f"Directory {directory} given does not exist and could not be created.") + return False + LOG.info(f"Directory is set to {directory}") + return True + + def set_snapshot_seconds(self, seconds: int) -> bool: + if seconds is None: + return False + # Set the snapshot second in the redis config + value = self.get_config_value("save") + if value is None: + LOG.error("Unable to get exisiting parameter values for snapshot") + return False + else: + value = value.split() + value[0] = str(seconds) + value = " ".join(value) + if not self.set_config_value("save", value): + LOG.error("Unable to set snapshot value seconds") + return False + LOG.info(f"Snapshot wait time is set to {seconds} seconds") + return True + + def set_snapshot_changes(self, changes: int) -> bool: + if changes is None: + return False + # Set the snapshot changes into the redis config + value = self.get_config_value("save") + if value is None: + LOG.error("Unable to get exisiting parameter values for snapshot") + return False + else: + value = value.split() + value[1] = str(changes) + value = " ".join(value) + if not self.set_config_value("save", value): + LOG.error("Unable to set snapshot value seconds") + return False + LOG.info(f"Snapshot threshold is set to {changes} changes") + return True + + def set_snapshot_file(self, file: str) -> bool: + if file is None: + return False + # Set the snapshot file in the redis config + if not self.set_config_value("dbfilename", file): + LOG.error("Unable to set snapshot_file name") + return False + + LOG.info(f"Snapshot file is set to {file}") + return True + + def set_append_mode(self, mode: str) -> bool: + if mode is None: + return False + valid_modes = ["always", "everysec", "no"] + + # Validate the append mode (always, everysec, no) + if mode in valid_modes: + # Set the append mode in the redis config + if not self.set_config_value("appendfsync", mode): + LOG.error("Unable to set append_mode in redis config") + return False + else: + LOG.error("Not a valid append_mode(Only valid modes are always, everysec, no)") + return False + + LOG.info(f"Append mode is set to {mode}") + return True + + def set_append_file(self, file: str) -> bool: + if file is None: + return False + # Set the append file in the redis config + if not self.set_config_value("appendfilename", f'"{file}"'): + LOG.error("Unable to set append filename.") + return False + LOG.info(f"Append file is set to {file}") + return True + + +class RedisUsers: + """ + RedisUsers provides an interface for parsing and interacting with redis.users configuration + file. Allow users and merlin server to create, remove, and edit users within the redis files. + Changes can be sync and push to an exisiting redis server if one is available. + """ + + class User: + status = "on" + hash_password = hashlib.sha256(b"password").hexdigest() + keys = "*" + channels = "*" + commands = "@all" + + def __init__(self, status="on", keys="*", channels="*", commands="@all", password=None) -> None: + self.status = status + self.keys = keys + self.channels = channels + self.commands = commands + if password is not None: + self.set_password(password) + + def parse_dict(self, dict: dict) -> None: + self.status = dict["status"] + self.keys = dict["keys"] + self.channels = dict["channels"] + self.commands = dict["commands"] + self.hash_password = dict["hash_password"] + + def get_user_dict(self) -> dict: + self.status = "on" + return { + "status": self.status, + "hash_password": self.hash_password, + "keys": self.keys, + "channels": self.channels, + "commands": self.commands, + } + + def __repr__(self) -> str: + return str(self.get_user_dict()) + + def __str__(self) -> str: + return self.__repr__() + + def set_password(self, password: str) -> None: + self.hash_password = hashlib.sha256(bytes(password, "utf-8")).hexdigest() + + filename = "" + users = {} + + def __init__(self, filename) -> None: + self.filename = filename + if os.path.exists(self.filename): + self.parse() + + def parse(self) -> None: + with open(self.filename, "r") as f: + self.users = yaml.load(f, yaml.Loader) + for user in self.users: + new_user = self.User() + new_user.parse_dict(self.users[user]) + self.users[user] = new_user + + def write(self) -> None: + data = self.users.copy() + for key in data: + data[key] = self.users[key].get_user_dict() + with open(self.filename, "w") as f: + yaml.dump(data, f, yaml.Dumper) + + def add_user(self, user, status="on", keys="*", channels="*", commands="@all", password=None) -> bool: + if user in self.users: + return False + self.users[user] = self.User(status, keys, channels, commands, password) + return True + + def set_password(self, user: str, password: str): + if user not in self.users: + return False + self.users[user].set_password(password) + + def remove_user(self, user) -> bool: + if user in self.users: + del self.users[user] + return True + return False + + def apply_to_redis(self, host: str, port: int, password: str) -> None: + db = redis.Redis(host=host, port=port, password=password) + current_users = db.acl_users() + for user in self.users: + if user not in current_users: + data = self.users[user] + db.acl_setuser( + username=user, + hashed_passwords=[f"+{data.hash_password}"], + enabled=(data.status == "on"), + keys=data.keys, + channels=data.channels, + commands=[f"+{data.commands}"], + ) + + for user in current_users: + if user not in self.users: + db.acl_deluser(user) + + +class AppYaml: + """ + AppYaml allows for an structured way to interact with any app.yaml main merlin configuration file. + It helps to parse each component of the app.yaml and allow users to edit, configure and write the + file. + """ + + default_filename = os.path.join(MERLIN_CONFIG_DIR, "app.yaml") + data = {} + broker_name = "broker" + results_name = "results_backend" + + def __init__(self, filename: str = default_filename) -> None: + if not os.path.exists(filename): + filename = self.default_filename + self.read(filename) + + def apply_server_config(self, server_config: ServerConfig): + rc = RedisConfig(server_config.container.get_config_path()) + + self.data[self.broker_name]["name"] = server_config.container.get_image_type() + self.data[self.broker_name]["username"] = "default" + self.data[self.broker_name]["password"] = server_config.container.get_pass_file_path() + self.data[self.broker_name]["server"] = rc.get_ip_address() + self.data[self.broker_name]["port"] = rc.get_port() + + self.data[self.results_name]["name"] = server_config.container.get_image_type() + self.data[self.results_name]["username"] = "default" + self.data[self.results_name]["password"] = server_config.container.get_pass_file_path() + self.data[self.results_name]["server"] = rc.get_ip_address() + self.data[self.results_name]["port"] = rc.get_port() + + def update_data(self, new_data: dict): + self.data.update(new_data) + + def get_data(self): + return self.data + + def read(self, filename: str = default_filename): + self.data = merlin.utils.load_yaml(filename) + + def write(self, filename: str = default_filename): + with open(filename, "w+") as f: + yaml.dump(self.data, f, yaml.Dumper) diff --git a/merlin/server/singularity.yaml b/merlin/server/singularity.yaml new file mode 100644 index 000000000..d2b34874e --- /dev/null +++ b/merlin/server/singularity.yaml @@ -0,0 +1,6 @@ +singularity: + command: singularity + # init_command: \{command} .. (optional or default) + run_command: \{command} run -H {home_dir} {image} {config} + stop_command: kill # \{command} (optional or kill default) + pull_command: \{command} pull {image} {url} diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 13db3dccc..7155d0c5f 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 9af27d456..950ceb253 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # @@ -38,6 +38,7 @@ "shell", "flux_path", "flux_start_opts", + "flux_exec", "flux_exec_workers", "launch_pre", "launch_args", diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 34b2113cd..1c0e9fa42 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # @@ -32,7 +32,7 @@ BATCH = {"batch": {"type": "local", "dry_run": False, "shell": "/bin/bash"}} -ENV = {"env": {"variables": {}, "sources": {}, "labels": {}, "dependencies": {}}} +ENV = {"env": {"variables": {}, "sources": [], "labels": {}, "dependencies": {}}} STUDY_STEP_RUN = {"task_queue": "merlin", "shell": "/bin/bash", "max_retries": 30} diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 0be5b21b9..254ba80be 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/merlinspec.json b/merlin/spec/merlinspec.json new file mode 100644 index 000000000..47e738ee6 --- /dev/null +++ b/merlin/spec/merlinspec.json @@ -0,0 +1,285 @@ +{ + "DESCRIPTION": { + "type": "object", + "properties": { + "name": {"type": "string", "minLength": 1}, + "description": {"type": "string", "minLength": 1} + }, + "required": [ + "name", + "description" + ] + }, + "PARAM": { + "type": "object", + "properties": { + "values": { + "type": "array" + }, + "label": {"type": "string", "minLength": 1} + }, + "required": [ + "values", + "label" + ] + }, + "STUDY_STEP": { + "type": "object", + "properties": { + "name": {"type": "string", "minLength": 1}, + "description": {"type": "string", "minLength": 1}, + "run": { + "type": "object", + "properties": { + "cmd": {"type": "string", "minLength": 1}, + "depends": {"type": "array", "uniqueItems": true}, + "pre": {"type": "string", "minLength": 1}, + "post": {"type": "string", "minLength": 1}, + "restart": {"type": "string", "minLength": 1}, + "slurm": {"type": "string", "minLength": 1}, + "lsf": {"type": "string", "minLength": 1}, + "num resource set": {"type": "integer", "minimum": 1}, + "launch distribution": {"type": "string", "minLength": 1}, + "exit_on_error": {"type": "integer", "minimum": 0, "maximum": 1}, + "shell": {"type": "string", "minLength": 1}, + "flux": {"type": "string", "minLength": 1}, + "batch": { + "type": "object", + "properties": { + "type": {"type": "string", "minLength": 1} + } + }, + "gpus per task": {"type": "integer", "minimum": 1}, + "max_retries": {"type": "integer", "minimum": 1}, + "task_queue": {"type": "string", "minLength": 1}, + "nodes": { + "anyOf": [ + {"type": "integer", "minimum": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ] + }, + "procs": { + "anyOf": [ + {"type": "integer", "minimum": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "gpus": { + "anyOf": [ + {"type": "integer", "minimum": 0}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "cores per task": { + "anyOf": [ + {"type": "integer", "minimum": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "tasks per rs": { + "anyOf": [ + {"type": "integer", "minimum": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "rs per node": { + "anyOf": [ + {"type": "integer", "minimum": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "cpus per rs": { + "anyOf": [ + {"type": "integer", "minimum": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "bind": { + "anyOf": [ + {"type": "string", "minLength": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "bind gpus": { + "anyOf": [ + {"type": "string", "minLength": 1}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "walltime": { + "anyOf": [ + {"type": "string", "minLength": 1}, + {"type": "integer", "minimum": 0}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "reservation": {"type": "string", "minLength": 1}, + "exclusive": { + "anyOf": [ + {"type": "boolean"}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ]}, + "nested": {"type": "boolean"}, + "priority": { + "anyOf": [ + { + "type": "string", + "enum": [ + "HELD", "MINIMAL", "LOW", "MEDIUM", "HIGH", "EXPEDITED", + "held", "minimal", "low", "medium", "high", "expedited", + "Held", "Minimal", "Low", "Medium", "High", "Expedited" + ] + }, + {"type": "number", "minimum": 0.0, "maximum": 1.0} + ] + }, + "qos": {"type": "string", "minLength": 1} + }, + "required": [ + "cmd" + ] + } + }, + "required": [ + "name", + "description", + "run" + ] + }, + "ENV": { + "type": "object", + "properties": { + "variables": { + "type": "object", + "patternProperties": { + "^.*": { + "anyOf": [ + {"type": "string", "minLength": 1}, + {"type": "number"} + ] + } + } + }, + "labels": {"type": "object"}, + "sources": {"type": "array"}, + "dependencies": { + "type": "object", + "properties": { + "paths": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string", "minLength": 1}, + "path": {"type": "string", "minLength": 1} + }, + "required": [ + "name", + "path" + ] + } + }, + "git": { + "type": "array", + "items": { + "properties": { + "name": {"type": "string", "minLength": 1}, + "path": {"type": "string", "minLength": 1}, + "url": {"type": "string", "minLength": 1}, + "tag": {"type": "string", "minLength": 1} + }, + "required": [ + "name", + "path", + "url" + ] + } + }, + "spack": { + "type": "object", + "properties": { + "name": {"type": "string", "minLength": 1}, + "package_name": {"type": "string", "minLength": 1} + }, + "required": [ + "type", + "package_name" + ] + } + } + } + } + }, + "MERLIN": { + "type": "object", + "properties": { + "resources": { + "type": "object", + "properties": { + "task_server": {"type": "string", "minLength": 1}, + "overlap": {"type": "boolean"}, + "workers": { + "type": "object", + "patternProperties": { + "^.+": { + "type": "object", + "properties": { + "args": {"type": "string", "minLength": 1}, + "steps": {"type": "array", "uniqueItems": true}, + "nodes": { + "anyOf": [ + {"type": "null"}, + {"type": "integer", "minimum": 1} + ] + }, + "batch": { + "anyOf": [ + {"type": "null"}, + { + "type": "object", + "properties": { + "type": {"type": "string", "minLength": 1} + } + } + ] + }, + "machines": {"type": "array", "uniqueItems": true} + } + } + }, + "minProperties": 1 + } + } + }, + "samples": { + "anyOf": [ + {"type": "null"}, + { + "type": "object", + "properties": { + "generate": { + "type": "object", + "properties": { + "cmd": {"type": "string", "minLength": 1} + }, + "required": ["cmd"] + }, + "file": {"type": "string", "minLength": 1}, + "column_labels": {"type": "array", "uniqueItems": true}, + "level_max_dirs": {"type": "integer", "minimum": 1} + } + } + ] + } + } + }, + "BATCH": { + "type": "object", + "properties": { + "type": {"type": "string", "minLength": 1}, + "bank": {"type": "string", "minLength": 1}, + "queue": {"type": "string", "minLength": 1}, + "dry_run": {"type": "boolean"}, + "shell": {"type": "string", "minLength": 1}, + "flux_path": {"type": "string", "minLength": 1}, + "flux_start_opts": {"type": "string", "minLength": 1}, + "flux_exec_workers": {"type": "boolean"}, + "launch_pre": {"type": "string", "minLength": 1}, + "launch_args": {"type": "string", "minLength": 1}, + "worker_launch": {"type": "string", "minLength": 1}, + "nodes": {"type": "integer", "minimum": 1}, + "walltime": {"type": "string", "pattern": "^(?:(?:([0-9][0-9]|2[0-3]):)?([0-5][0-9]):)?([0-5][0-9])$"} + } + } +} diff --git a/merlin/spec/override.py b/merlin/spec/override.py index a3fbf281b..c4fcfee97 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -1,3 +1,33 @@ +############################################################################### +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### + import logging from copy import deepcopy diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 6bb612cb4..a6ecdae2a 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # @@ -33,13 +33,14 @@ data from the Merlin specification file. To see examples of yaml specifications, run `merlin example`. """ +import json import logging import os import shlex from io import StringIO import yaml -from maestrowf.datastructures import YAMLSpecification +from maestrowf.specification import YAMLSpecification from merlin.spec import all_keys, defaults @@ -82,6 +83,7 @@ def yaml_sections(self): "study": self.study, "global.parameters": self.globals, "merlin": self.merlin, + "user": self.user, } @property @@ -97,27 +99,215 @@ def sections(self): "study": self.study, "globals": self.globals, "merlin": self.merlin, + "user": self.user, } + def __str__(self): + """Magic method to print an instance of our MerlinSpec class.""" + env = "" + globs = "" + merlin = "" + user = "" + if self.environment: + env = f"\n\tenvironment: \n\t\t{self.environment}" + if self.globals: + globs = f"\n\tglobals:\n\t\t{self.globals}" + if self.merlin: + merlin = f"\n\tmerlin:\n\t\t{self.merlin}" + if self.user is not None: + user = f"\n\tuser:\n\t\t{self.user}" + result = f"""MERLIN SPEC OBJECT:\n\tdescription:\n\t\t{self.description} + \n\tbatch:\n\t\t{self.batch}\n\tstudy:\n\t\t{self.study} + {env}{globs}{merlin}{user}""" + + return result + @classmethod def load_specification(cls, filepath, suppress_warning=True): - spec = super(MerlinSpec, cls).load_specification(filepath) - with open(filepath, "r") as f: - spec.merlin = MerlinSpec.load_merlin_block(f) + LOG.info("Loading specification from path: %s", filepath) + try: + # Load the YAML spec from the filepath + with open(filepath, "r") as data: + spec = cls.load_spec_from_string(data, needs_IO=False, needs_verification=True) + except Exception as e: + LOG.exception(e.args) + raise e + + # Path not set in _populate_spec because loading spec with string + # does not have a path so we set it here + spec.path = filepath spec.specroot = os.path.dirname(spec.path) - spec.process_spec_defaults() + if not suppress_warning: spec.warn_unrecognized_keys() return spec @classmethod - def load_spec_from_string(cls, string): - spec = super(MerlinSpec, cls).load_specification_from_stream(StringIO(string)) - spec.merlin = MerlinSpec.load_merlin_block(StringIO(string)) + def load_spec_from_string(cls, string, needs_IO=True, needs_verification=False): + LOG.debug("Creating Merlin spec object...") + # Create and populate the MerlinSpec object + data = StringIO(string) if needs_IO else string + spec = cls._populate_spec(data) spec.specroot = None spec.process_spec_defaults() + LOG.debug("Merlin spec object created.") + + # Verify the spec object + if needs_verification: + LOG.debug("Verifying Merlin spec...") + spec.verify() + LOG.debug("Merlin spec verified.") + return spec + @classmethod + def _populate_spec(cls, data): + """ + Helper method to load a study spec and populate it's fields. + + NOTE: This is basically a direct copy of YAMLSpecification's + load_specification method from Maestro just without the call to verify. + The verify method was breaking our code since we have no way of modifying + Maestro's schema that they use to verify yaml files. The work around + is to load the yaml file ourselves and create our own schema to verify + against. + + :param data: Raw text stream to study YAML spec data + :returns: A MerlinSpec object containing information from the path + """ + # Read in the spec file + try: + spec = yaml.load(data, yaml.FullLoader) + except AttributeError: + LOG.warn( + "PyYAML is using an unsafe version with a known " + "load vulnerability. Please upgrade your installation " + "to a more recent version!" + ) + spec = yaml.load(data) + LOG.debug("Successfully loaded specification: \n%s", spec["description"]) + + # Load in the parts of the yaml that are the same as Maestro's + merlin_spec = cls() + merlin_spec.path = None + merlin_spec.description = spec.pop("description", {}) + merlin_spec.environment = spec.pop("env", {"variables": {}, "sources": [], "labels": {}, "dependencies": {}}) + merlin_spec.batch = spec.pop("batch", {}) + merlin_spec.study = spec.pop("study", []) + merlin_spec.globals = spec.pop("global.parameters", {}) + + # Reset the file pointer and load the merlin block + data.seek(0) + merlin_spec.merlin = MerlinSpec.load_merlin_block(data) + + # Reset the file pointer and load the user block + data.seek(0) + merlin_spec.user = MerlinSpec.load_user_block(data) + + return merlin_spec + + def verify(self): + """ + Verify the spec against a valid schema. Similar to YAMLSpecification's verify + method from Maestro but specific for Merlin yaml specs. + + NOTE: Maestro v2.0 may add the ability to customize the schema files it + compares against. If that's the case then we can convert this file back to + using Maestro's verification. + """ + # Load the MerlinSpec schema file + dir_path = os.path.dirname(os.path.abspath(__file__)) + schema_path = os.path.join(dir_path, "merlinspec.json") + with open(schema_path, "r") as json_file: + schema = json.load(json_file) + + # Use Maestro's verification methods for shared sections + self.verify_description(schema["DESCRIPTION"]) + self.verify_environment(schema["ENV"]) + self.verify_study(schema["STUDY_STEP"]) + self.verify_parameters(schema["PARAM"]) + + # Merlin specific verification + self.verify_merlin_block(schema["MERLIN"]) + self.verify_batch_block(schema["BATCH"]) + + def get_study_step_names(self): + """ + Get a list of the names of steps in our study. + + :returns: an unsorted list of study step names + """ + names = [] + for step in self.study: + names.append(step["name"]) + return names + + def _verify_workers(self): + """ + Helper method to verify the workers section located within the Merlin block + of our spec file. + """ + # Retrieve the names of the steps in our study + actual_steps = self.get_study_step_names() + + try: + # Verify that the steps in merlin block's worker section actually exist + for worker, worker_vals in self.merlin["resources"]["workers"].items(): + error_prefix = f"Problem in Merlin block with worker {worker} --" + for step in worker_vals["steps"]: + if step != "all" and step not in actual_steps: + error_msg = ( + f"{error_prefix} Step with the name {step}" + " is not defined in the study block of the yaml specification file" + ) + raise ValueError(error_msg) + + except Exception: + raise + + def verify_merlin_block(self, schema): + """ + Method to verify the merlin section of our spec file. + + :param schema: The section of the predefined schema (merlinspec.json) to check + our spec file against. + """ + # Validate merlin block against the json schema + YAMLSpecification.validate_schema("merlin", self.merlin, schema) + # Verify the workers section within merlin block + self._verify_workers() + + def verify_batch_block(self, schema): + """ + Method to verify the batch section of our spec file. + + :param schema: The section of the predefined schema (merlinspec.json) to check + our spec file against. + """ + # Validate batch block against the json schema + YAMLSpecification.validate_schema("batch", self.batch, schema) + + # Additional Walltime checks in case the regex from the schema bypasses an error + if "walltime" in self.batch: + if self.batch["type"] == "lsf": + LOG.warning("The walltime argument is not available in lsf.") + else: + try: + err_msg = "Walltime must be of the form SS, MM:SS, or HH:MM:SS." + walltime = self.batch["walltime"] + if len(walltime) > 2: + # Walltime must have : if it's not of the form SS + if ":" not in walltime: + raise ValueError(err_msg) + else: + # Walltime must have exactly 2 chars between : + time = walltime.split(":") + for section in time: + if len(section) != 2: + raise ValueError(err_msg) + except Exception: + raise + @staticmethod def load_merlin_block(stream): try: @@ -132,6 +322,14 @@ def load_merlin_block(stream): LOG.warning(warning_msg) return merlin_block + @staticmethod + def load_user_block(stream): + try: + user_block = yaml.safe_load(stream)["user"] + except KeyError: + user_block = {} + return user_block + def process_spec_defaults(self): for name, section in self.sections.items(): if section is None: @@ -161,6 +359,8 @@ def process_spec_defaults(self): if self.merlin["samples"] is not None: MerlinSpec.fill_missing_defaults(self.merlin["samples"], defaults.SAMPLES) + # no defaults for user block + @staticmethod def fill_missing_defaults(object_to_update, default_dict): """ @@ -185,6 +385,7 @@ def recurse(result, defaults): recurse(object_to_update, default_dict) + # ***Unsure if this method is still needed after adding json schema verification*** def warn_unrecognized_keys(self): # check description MerlinSpec.check_section("description", self.description, all_keys.DESCRIPTION) @@ -212,9 +413,14 @@ def warn_unrecognized_keys(self): if self.merlin["samples"]: MerlinSpec.check_section("merlin.samples", self.merlin["samples"], all_keys.SAMPLES) + # user block is not checked + @staticmethod def check_section(section_name, section, all_keys): diff = set(section.keys()).difference(all_keys) + + # TODO: Maybe add a check here for required keys + for extra in diff: LOG.warn(f"Unrecognized key '{extra}' found in spec section '{section_name}'.") @@ -268,7 +474,7 @@ def _process_dict_or_list(self, obj, string, key_stack, lvl, tab): list_offset = 2 * " " if isinstance(obj, list): n = len(obj) - use_hyphens = key_stack[-1] in ["paths", "sources", "git", "study"] + use_hyphens = key_stack[-1] in ["paths", "sources", "git", "study"] or key_stack[0] in ["user"] if not use_hyphens: string += "[" else: diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 13db3dccc..7155d0c5f 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 69c68856a..f395c5d80 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # @@ -175,7 +175,7 @@ def batch_worker_launch( flux_exec: str = "" if flux_exec_workers: - flux_exec = "flux exec" + flux_exec = get_yaml_var(batch, "flux_exec", "flux exec") if "/" in flux_path: flux_path += "/" diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 67bae9d74..34393f967 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 01f7aae91..838c14762 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # @@ -44,11 +44,18 @@ class DAG: independent chains of tasks. """ - def __init__(self, maestro_dag, labels): + def __init__(self, maestro_adjacency_table, maestro_values, labels): """ - :param `maestro_dag`: A maestrowf ExecutionGraph. + :param `maestro_adjacency_table`: An ordered dict showing adjacency of nodes. Comes from a maestrowf ExecutionGraph. + :param `maestro_values`: An ordered dict of the values at each node. Comes from a maestrowf ExecutionGraph. + :param `labels`: A list of labels provided in the spec file. """ - self.dag = maestro_dag + # We used to store the entire maestro ExecutionGraph here but now it's + # unpacked so we're only storing the 2 attributes from it that we use: + # the adjacency table and the values. This had to happen to get pickle + # to work for Celery. + self.maestro_adjacency_table = maestro_adjacency_table + self.maestro_values = maestro_values self.backwards_adjacency = {} self.calc_backwards_adjacency() self.labels = labels @@ -59,7 +66,7 @@ def step(self, task_name): :param `task_name`: The task name. :return: A Merlin Step object. """ - return Step(self.dag.values[task_name]) + return Step(self.maestro_values[task_name]) def calc_depth(self, node, depths, current_depth=0): """Calculate the depth of the given node and its children. @@ -116,7 +123,7 @@ def children(self, task_name): :return: list of children of this task. """ - return self.dag.adjacency_table[task_name] + return self.maestro_adjacency_table[task_name] def num_children(self, task_name): """Find the number of children for the given task in the dag. @@ -156,8 +163,8 @@ def find_chain(task_name, list_of_groups_of_chains): def calc_backwards_adjacency(self): """initializes our backwards adjacency table""" - for parent in self.dag.adjacency_table: - for task_name in self.dag.adjacency_table[parent]: + for parent in self.maestro_adjacency_table: + for task_name in self.maestro_adjacency_table[parent]: if task_name in self.backwards_adjacency: self.backwards_adjacency[task_name].append(parent) else: diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 23398e117..826c1d2e3 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 5b03bf2cb..5e7d89e43 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # @@ -121,7 +121,7 @@ def clone_changing_workspace_and_cmd(self, new_cmd=None, cmd_replacement_pairs=N new_workspace = self.get_workspace() LOG.debug(f"cloned step with workspace {new_workspace}") study_step = StudyStep() - study_step.name = step_dict["name"] + study_step.name = step_dict["_name"] study_step.description = step_dict["description"] study_step.run = step_dict["run"] return Step(MerlinStepRecord(new_workspace, study_step)) @@ -218,7 +218,7 @@ def name(self): """ :return : The step name. """ - return self.mstep.step.__dict__["name"] + return self.mstep.step.__dict__["_name"] def execute(self, adapter_config): """ diff --git a/merlin/study/study.py b/merlin/study/study.py index efa43dd9f..b7f990a2a 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # @@ -107,6 +107,20 @@ def __init__( "MERLIN_SOFT_FAIL": str(int(ReturnCode.SOFT_FAIL)), "MERLIN_HARD_FAIL": str(int(ReturnCode.HARD_FAIL)), "MERLIN_RETRY": str(int(ReturnCode.RETRY)), + # below will be substituted for sample values on execution + "MERLIN_SAMPLE_VECTOR": " ".join( + ["$({})".format(k) for k in self.get_sample_labels(from_spec=self.original_spec)] + ), + "MERLIN_SAMPLE_NAMES": " ".join(self.get_sample_labels(from_spec=self.original_spec)), + "MERLIN_SPEC_ORIGINAL_TEMPLATE": os.path.join( + self.info, self.original_spec.description["name"].replace(" ", "_") + ".orig.yaml" + ), + "MERLIN_SPEC_EXECUTED_RUN": os.path.join( + self.info, self.original_spec.description["name"].replace(" ", "_") + ".partial.yaml" + ), + "MERLIN_SPEC_ARCHIVED_COPY": os.path.join( + self.info, self.original_spec.description["name"].replace(" ", "_") + ".expanded.yaml" + ), } self.pgen_file = pgen_file @@ -182,6 +196,11 @@ def samples(self): return self.load_samples() return [] + def get_sample_labels(self, from_spec): + if from_spec.merlin["samples"]: + return from_spec.merlin["samples"]["column_labels"] + return [] + @property def sample_labels(self): """ @@ -197,9 +216,7 @@ def sample_labels(self): :return: list of labels (e.g. ["X0", "X1"] ) """ - if self.expanded_spec.merlin["samples"]: - return self.expanded_spec.merlin["samples"]["column_labels"] - return [] + return self.get_sample_labels(from_spec=self.expanded_spec) def load_samples(self): """ @@ -502,7 +519,8 @@ def load_dag(self): labels = [] if self.expanded_spec.merlin["samples"]: labels = self.expanded_spec.merlin["samples"]["column_labels"] - self.dag = DAG(maestro_dag, labels) + # To avoid pickling issues with _pass_detect_cycle from maestro, we unpack the dag here + self.dag = DAG(maestro_dag.adjacency_table, maestro_dag.values, labels) def get_adapter_config(self, override_type=None): adapter_config = dict(self.expanded_spec.batch) diff --git a/merlin/utils.py b/merlin/utils.py index b9f8742bb..2959b79e2 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.5. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/requirements/release.txt b/requirements/release.txt index 4771b7a4c..821589c41 100644 --- a/requirements/release.txt +++ b/requirements/release.txt @@ -3,9 +3,10 @@ celery[redis,sqlalchemy]>=5.0.3 coloredlogs cryptography importlib_resources; python_version < '3.7' -maestrowf==1.1.7dev0 +maestrowf>=1.1.9dev1 numpy parse psutil>=5.1.0 pyyaml>=5.1.2 tabulate +redis>=4.3.4 \ No newline at end of file diff --git a/setup.py b/setup.py index f60536d30..9bf170126 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # @@ -96,11 +96,11 @@ def extras_require(): long_description=readme(), long_description_content_type="text/markdown", classifiers=[ - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], keywords="machine learning workflow", url="https://github.com/LLNL/merlin", diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index afafa0d99..3448ec61a 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -238,3 +238,65 @@ def passes(self): if self.negate: return not self.is_within() return self.is_within() + + +class PathExists(Condition): + """ + A condition for checking if a path to a file or directory exists + """ + + def __init__(self, pathname) -> None: + self.pathname = pathname + + def path_exists(self) -> bool: + return os.path.exists(self.pathname) + + def __str__(self) -> str: + return f"{__class__.__name__} expected to find file or directory at {self.pathname}" + + @property + def passes(self): + return self.path_exists() + + +class FileHasRegex(Condition): + """ + A condition that some body of text within a file + MUST match a given regular expression. + """ + + def __init__(self, filename, regex) -> None: + self.filename = filename + self.regex = regex + + def contains(self) -> bool: + try: + with open(self.filename, "r") as f: + filetext = f.read() + return self.is_within(filetext) + except Exception: + return False + + def is_within(self, text): + return search(self.regex, text) is not None + + def __str__(self) -> str: + return f"{__class__.__name__} expected to find {self.regex} regex match within {self.filename} file but no match was found" + + @property + def passes(self): + return self.contains() + + +class FileHasNoRegex(FileHasRegex): + """ + A condition that some body of text within a file + MUST NOT match a given regular expression. + """ + + def __str__(self) -> str: + return f"{__class__.__name__} expected to find {self.regex} regex to not match within {self.filename} file but a match was found" + + @property + def passes(self): + return not self.contains() diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 3a8038d06..bfd80eb83 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.8.4. +# This file is part of Merlin, Version: 1.9.0. # # For details, see https://github.com/LLNL/merlin. # @@ -34,6 +34,7 @@ """ import argparse import shutil +import sys import time from contextlib import suppress from subprocess import PIPE, Popen @@ -77,6 +78,12 @@ def run_single_test(name, test, test_label="", buffer_length=50): info["violated_condition"] = (condition, i, len(conditions)) break + if len(test) == 4: + end_process = Popen(test[3], stdout=PIPE, stderr=PIPE, shell=True) + end_stdout, end_stderr = end_process.communicate() + info["end_stdout"] = end_stdout + info["end_stderr"] = end_stderr + return passed, info @@ -138,7 +145,11 @@ def run_tests(args, tests): n_to_run = 0 selective = True for test_id, test in enumerate(tests.values()): - if len(test) == 3 and test[2] == "local": + # Ensures that test definitions are atleast size 3. + # 'local' variable is stored in 3rd element of the test definitions, + # but an optional 4th element can be provided for an ending command + # to be ran after all checks have been made. + if len(test) >= 3 and test[2] == "local": args.ids.append(test_id + 1) n_to_run += 1 @@ -210,7 +221,7 @@ def main(): clear_test_studies_dir() result = run_tests(args, tests) - return result + sys.exit(result) if __name__ == "__main__": diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index dbdffd2cd..d23cbd57e 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -1,9 +1,19 @@ -from conditions import HasRegex, HasReturnCode, ProvenanceYAMLFileHasRegex, StepFileExists, StepFileHasRegex +from conditions import ( + FileHasNoRegex, + FileHasRegex, + HasRegex, + HasReturnCode, + PathExists, + ProvenanceYAMLFileHasRegex, + StepFileExists, + StepFileHasRegex, +) from merlin.utils import get_flux_cmd OUTPUT_DIR = "cli_test_studies" +CLEAN_MERLIN_SERVER = "rm -rf appendonly.aof dump.rdb merlin_server/" def define_tests(): @@ -45,6 +55,81 @@ def define_tests(): "local", ), } + server_basic_tests = { + "merlin server init": ( + "merlin server init", + HasRegex(".*successful"), + "local", + CLEAN_MERLIN_SERVER, + ), + "merlin server start/stop": ( + """merlin server init; + merlin server start; + merlin server status; + merlin server stop;""", + [ + HasRegex("Server started with PID [0-9]*"), + HasRegex("Merlin server is running"), + HasRegex("Merlin server terminated"), + ], + "local", + CLEAN_MERLIN_SERVER, + ), + "merlin server restart": ( + """merlin server init; + merlin server start; + merlin server restart; + merlin server status; + merlin server stop;""", + [ + HasRegex("Server started with PID [0-9]*"), + HasRegex("Merlin server is running"), + HasRegex("Merlin server terminated"), + ], + "local", + CLEAN_MERLIN_SERVER, + ), + } + server_config_tests = { + "merlin server change config": ( + """merlin server init; + merlin server config -p 8888 -pwd new_password -d ./config_dir -ss 80 -sc 8 -sf new_sf -am always -af new_af.aof; + merlin server start; + merlin server stop;""", + [ + FileHasRegex("merlin_server/redis.conf", "port 8888"), + FileHasRegex("merlin_server/redis.conf", "requirepass new_password"), + FileHasRegex("merlin_server/redis.conf", "dir ./config_dir"), + FileHasRegex("merlin_server/redis.conf", "save 80 8"), + FileHasRegex("merlin_server/redis.conf", "dbfilename new_sf"), + FileHasRegex("merlin_server/redis.conf", "appendfsync always"), + FileHasRegex("merlin_server/redis.conf", 'appendfilename "new_af.aof"'), + PathExists("./config_dir/new_sf"), + PathExists("./config_dir/appendonlydir"), + HasRegex("Server started with PID [0-9]*"), + HasRegex("Merlin server terminated"), + ], + "local", + "rm -rf appendonly.aof dump.rdb merlin_server/ config_dir/", + ), + "merlin server config add/remove user": ( + """merlin server init; + merlin server start; + merlin server config --add-user new_user new_password; + merlin server stop; + scp ./merlin_server/redis.users ./merlin_server/redis.users_new + merlin server start; + merlin server config --remove-user new_user; + merlin server stop; + """, + [ + FileHasRegex("./merlin_server/redis.users_new", "new_user"), + FileHasNoRegex("./merlin_server/redis.users", "new_user"), + ], + "local", + CLEAN_MERLIN_SERVER, + ), + } examples_check = { "example list": ( "merlin example list", @@ -371,6 +456,8 @@ def define_tests(): all_tests = {} for test_dict in [ basic_checks, + server_basic_tests, + server_config_tests, examples_check, run_workers_echo_tests, wf_format_tests, diff --git a/tests/unit/spec/test_specification.py b/tests/unit/spec/test_specification.py index 6b3503fb5..e1fb09a6b 100644 --- a/tests/unit/spec/test_specification.py +++ b/tests/unit/spec/test_specification.py @@ -3,6 +3,8 @@ import tempfile import unittest +import yaml + from merlin.spec.specification import MerlinSpec @@ -80,6 +82,31 @@ label : N_NEW.%% """ +INVALID_MERLIN = """ +description: + name: basic_ensemble_invalid_merlin + description: Template yaml to ensure our custom merlin block verification works as intended + +batch: + type: local + +study: + - name: step1 + description: | + this won't actually run + run: + cmd: | + echo "if this is printed something is bad" + +merlin: + resources: + task_server: celery + overlap: false + workers: + worker1: + steps: [] +""" + class TestMerlinSpec(unittest.TestCase): """Test the logic for parsing the Merlin spec into a MerlinSpec.""" @@ -170,3 +197,75 @@ def test_default_merlin_block(self): self.assertEqual(self.spec.merlin["resources"]["workers"]["default_worker"]["batch"], None) self.assertEqual(self.spec.merlin["resources"]["workers"]["default_worker"]["nodes"], None) self.assertEqual(self.spec.merlin["samples"], None) + + +class TestCustomVerification(unittest.TestCase): + """ + Tests to make sure our custom verification on merlin specific parts of our + spec files is working as intended. Verification happens in + merlin/spec/specification.py + + NOTE: reset_spec() should be called at the end of each test to make sure the + test file is reset. + + CREATING A NEW VERIFICATION TEST: + 1. Read in the spec with self.read_spec() + 2. Modify the spec with an invalid value to test for (e.g. a bad step, a bad walltime, etc.) + 3. Update the spec file with self.update_spec(spec) + 4. Assert that the correct error is thrown + 5. Reset the spec file with self.reset_spec() + """ + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.merlin_spec_filepath = os.path.join(self.tmpdir, "merlin_verification.yaml") + self.write_spec(INVALID_MERLIN) + + def tearDown(self): + shutil.rmtree(self.tmpdir, ignore_errors=True) + + def reset_spec(self): + self.write_spec(INVALID_MERLIN) + + def write_spec(self, spec): + with open(self.merlin_spec_filepath, "w+") as _file: + _file.write(spec) + + def read_spec(self): + with open(self.merlin_spec_filepath, "r") as yamfile: + spec = yaml.load(yamfile, yaml.Loader) + return spec + + def update_spec(self, spec): + with open(self.merlin_spec_filepath, "w") as yamfile: + yaml.dump(spec, yamfile, yaml.Dumper) + + def test_invalid_step(self): + # Read in the existing spec and update it with our bad step + spec = self.read_spec() + spec["merlin"]["resources"]["workers"]["worker1"]["steps"].append("bad_step") + self.update_spec(spec) + + # Assert that the invalid format was caught + with self.assertRaises(ValueError): + MerlinSpec.load_specification(self.merlin_spec_filepath) + + # Reset the spec to the default value + self.reset_spec() + + def test_invalid_walltime(self): + # Read in INVALID_MERLIN spec + spec = self.read_spec() + + invalid_walltimes = ["2", "0:1", "111", "1:1:1", "65", "65:12", "66:77", ":02:12", "123:45:33", ""] + + # Loop through the invalid walltimes and make sure they're all caught + for time in invalid_walltimes: + spec["batch"]["walltime"] = time + self.update_spec(spec) + + with self.assertRaises(ValueError): + MerlinSpec.load_specification(self.merlin_spec_filepath) + + # Reset the spec + self.reset_spec() diff --git a/tox.ini b/tox.ini index 457a827ca..4dfe01e03 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py35, py36, py37 +envlist = py37, py38, py39, py310, py311 [testenv] deps = From f7ea82b97604155077772a07b164904a5069fc46 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 19 Dec 2022 16:24:54 -0800 Subject: [PATCH 054/126] fix merlinspec not being installed with pip and python 3.7 issues with celery --- MANIFEST.in | 1 + requirements/release.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index cefbd23a5..11e403335 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ recursive-include merlin/server *.yaml *.py recursive-include merlin/examples *.py *.yaml *.c *.json *.sbatch *.bsub *.txt *.temp include requirements.txt include requirements/* +include merlin/spec/merlinspec.json \ No newline at end of file diff --git a/requirements/release.txt b/requirements/release.txt index 821589c41..11fd85129 100644 --- a/requirements/release.txt +++ b/requirements/release.txt @@ -2,6 +2,7 @@ cached_property celery[redis,sqlalchemy]>=5.0.3 coloredlogs cryptography +importlib_metadata<5.0.0; python_version == '3.7' importlib_resources; python_version < '3.7' maestrowf>=1.1.9dev1 numpy From 28587257e32c2e27feba1639720e5de911f97d00 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 19 Dec 2022 16:33:04 -0800 Subject: [PATCH 055/126] update changelog and version to 1.9.1 --- CHANGELOG.md | 5 +++++ merlin/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a9cde753..add3a695f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.9.1] +### Fixed +- Added merlin/spec/merlinspec.json to MANIFEST.in so pip will actually install it when ran +- Fixed a bug where "from celery import Celery" was failing on python 3.7 + ## [1.9.0] ### Added - Added support for Python 3.11 diff --git a/merlin/__init__.py b/merlin/__init__.py index aa33bc4d2..7ed348cf9 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -38,7 +38,7 @@ import sys -__version__ = "1.9.0" +__version__ = "1.9.1" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") From f8176b87595c7594fc5ce5cde130408b5f5551c7 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 20 Dec 2022 07:54:03 -0800 Subject: [PATCH 056/126] fix a numpy issue on new numpy version release --- merlin/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/merlin/utils.py b/merlin/utils.py index 2959b79e2..55781ad5b 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -240,9 +240,9 @@ def load_array_file(filename, ndmin=2): ) # Make sure text files load as strings with minimum number of dimensions elif protocol == "csv": - array = np.loadtxt(filename, delimiter=",", ndmin=ndmin, dtype=np.str) + array = np.loadtxt(filename, delimiter=",", ndmin=ndmin, dtype=np.str_) elif protocol == "tab": - array = np.loadtxt(filename, ndmin=ndmin, dtype=np.str) + array = np.loadtxt(filename, ndmin=ndmin, dtype=np.str_) else: raise TypeError( f"{protocol} is not a valid array file extension.\ From 8fc0e1ad78a4563b12f8690506bbe32cd7995051 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 20 Dec 2022 07:55:33 -0800 Subject: [PATCH 057/126] modify changelog to show numpy fix --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index add3a695f..dca040e7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Added merlin/spec/merlinspec.json to MANIFEST.in so pip will actually install it when ran - Fixed a bug where "from celery import Celery" was failing on python 3.7 +- Numpy error about numpy.str not existing from a new numpy release ## [1.9.0] ### Added From fa59c3683eb6336a8cde92cef7c39484c3ff2ef6 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 20 Dec 2022 09:18:15 -0800 Subject: [PATCH 058/126] add version change to all files --- Makefile | 2 +- merlin/__init__.py | 2 +- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 6 +++--- setup.py | 2 +- tests/integration/run_tests.py | 2 +- 51 files changed, 53 insertions(+), 53 deletions(-) diff --git a/Makefile b/Makefile index b0b1fb0a9..570ee3e25 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index 7ed348cf9..eaf3ff668 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 0b5971627..a9c6093ca 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index ebac67eec..0041523b2 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index b02f9e909..30fea7f47 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 26c64e9e2..da214275b 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 8d9a89285..274ddb062 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 1859a55df..591f9a49f 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 6f3e58e9a..4861a757b 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index e378573c4..229f00851 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 16365e32c..1ba552626 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 9820e1041..6049bb5b5 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 027bcd291..352da5550 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 7ade90e05..b460be069 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index a546591f1..e30a18ca0 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index a8c0a1ef2..7822e7086 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 5dd08ddfb..bd22ddaa3 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 335bd05f5..e435e1ba0 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index cd47be750..626b776d5 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index 3b1469f70..30140981e 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 39c5cf2e0..43573b416 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index d59fc8511..2796465ab 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 269bf6097..816546f67 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 33c6776e4..f540e6f1f 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 9e207b749..5de8423b0 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 4195ceacc..a1e4de372 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 8858dcfad..90711756c 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 8a570b70d..38359938c 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 57cb5af22..834ad1d3a 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index e6eb6379d..16d2c3686 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 950ceb253..46db0fd4f 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 1c0e9fa42..46f71882f 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 254ba80be..710bc2500 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index c4fcfee97..5ca2eea6b 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index a6ecdae2a..9d93197bf 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index f395c5d80..71e89bf84 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 34393f967..97fc40289 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 838c14762..fbd7dd6d7 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 826c1d2e3..ab998140a 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 5e7d89e43..3a55df606 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index b7f990a2a..9d36e4173 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index 55781ad5b..b3a495846 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # @@ -240,9 +240,9 @@ def load_array_file(filename, ndmin=2): ) # Make sure text files load as strings with minimum number of dimensions elif protocol == "csv": - array = np.loadtxt(filename, delimiter=",", ndmin=ndmin, dtype=np.str_) + array = np.loadtxt(filename, delimiter=",", ndmin=ndmin, dtype=np.str) elif protocol == "tab": - array = np.loadtxt(filename, ndmin=ndmin, dtype=np.str_) + array = np.loadtxt(filename, ndmin=ndmin, dtype=np.str) else: raise TypeError( f"{protocol} is not a valid array file extension.\ diff --git a/setup.py b/setup.py index 9bf170126..06706ca57 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index bfd80eb83..913a2ab08 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # From d0520183797ef907769a27ee25cfbfd7a955f676 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 20 Dec 2022 09:31:30 -0800 Subject: [PATCH 059/126] re-add fix for numpy since it got removed in the last commit by accident --- merlin/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/merlin/utils.py b/merlin/utils.py index b3a495846..2e45ab623 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -240,9 +240,9 @@ def load_array_file(filename, ndmin=2): ) # Make sure text files load as strings with minimum number of dimensions elif protocol == "csv": - array = np.loadtxt(filename, delimiter=",", ndmin=ndmin, dtype=np.str) + array = np.loadtxt(filename, delimiter=",", ndmin=ndmin, dtype=np.str_) elif protocol == "tab": - array = np.loadtxt(filename, ndmin=ndmin, dtype=np.str) + array = np.loadtxt(filename, ndmin=ndmin, dtype=np.str_) else: raise TypeError( f"{protocol} is not a valid array file extension.\ From 1f03afd4dd10d3c6dfc95b73a27f56419501d7c4 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 20 Dec 2022 15:16:06 -0800 Subject: [PATCH 060/126] revert utils.py back to previous implementation --- merlin/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/merlin/utils.py b/merlin/utils.py index 2e45ab623..b3a495846 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -240,9 +240,9 @@ def load_array_file(filename, ndmin=2): ) # Make sure text files load as strings with minimum number of dimensions elif protocol == "csv": - array = np.loadtxt(filename, delimiter=",", ndmin=ndmin, dtype=np.str_) + array = np.loadtxt(filename, delimiter=",", ndmin=ndmin, dtype=np.str) elif protocol == "tab": - array = np.loadtxt(filename, ndmin=ndmin, dtype=np.str_) + array = np.loadtxt(filename, ndmin=ndmin, dtype=np.str) else: raise TypeError( f"{protocol} is not a valid array file extension.\ From 1c2b96728d69451fbd45485f3d9afe6e6beffe86 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 20 Dec 2022 15:30:40 -0800 Subject: [PATCH 061/126] change dtype to python str type --- merlin/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/merlin/utils.py b/merlin/utils.py index b3a495846..93777d95d 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -240,9 +240,9 @@ def load_array_file(filename, ndmin=2): ) # Make sure text files load as strings with minimum number of dimensions elif protocol == "csv": - array = np.loadtxt(filename, delimiter=",", ndmin=ndmin, dtype=np.str) + array = np.loadtxt(filename, delimiter=",", ndmin=ndmin, dtype=str) elif protocol == "tab": - array = np.loadtxt(filename, ndmin=ndmin, dtype=np.str) + array = np.loadtxt(filename, ndmin=ndmin, dtype=str) else: raise TypeError( f"{protocol} is not a valid array file extension.\ From f1e4cda3b4665137a655245bca0b223be8f9c808 Mon Sep 17 00:00:00 2001 From: Ryan Lee <44886374+ryannova@users.noreply.github.com> Date: Tue, 20 Dec 2022 17:16:42 -0800 Subject: [PATCH 062/126] Hotfix for merlin server unable to write config files. (#394) * Hotfix for merlin server unable to write config files. Change files to modules and copy files from new file modules --- CHANGELOG.md | 1 + merlin/server/__init__.py | 29 +++++++++++++++++++++++++++++ merlin/server/server_config.py | 30 ++++++++++++++++++++---------- 3 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 merlin/server/__init__.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dca040e7d..5e480c8d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added merlin/spec/merlinspec.json to MANIFEST.in so pip will actually install it when ran - Fixed a bug where "from celery import Celery" was failing on python 3.7 - Numpy error about numpy.str not existing from a new numpy release +- Made merlin server configurations into modules that can be loaded and written to users ## [1.9.0] ### Added diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py new file mode 100644 index 000000000..7155d0c5f --- /dev/null +++ b/merlin/server/__init__.py @@ -0,0 +1,29 @@ +############################################################################### +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 834ad1d3a..ea5c8f1a8 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -32,7 +32,6 @@ import logging import os import random -import shutil import string import subprocess from io import BufferedReader @@ -40,6 +39,12 @@ import yaml + +try: + import importlib.resources as resources +except ImportError: + import importlib_resources as resources + from merlin.server.server_util import ( CONTAINER_TYPES, MERLIN_CONFIG_DIR, @@ -158,18 +163,22 @@ def create_server_config() -> bool: continue LOG.info(f"Copying file {file} to configuration directory.") try: - shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), file), config_dir) + with resources.path("merlin.server", file) as config_file: + with open(file_path, "w") as outfile, open(config_file, "r") as infile: + outfile.write(infile.read()) except OSError: LOG.error(f"Destination location {config_dir} is not writable.") return False # Load Merlin Server Configuration and apply it to app.yaml - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), MERLIN_SERVER_CONFIG)) as f: - main_server_config = yaml.load(f, yaml.Loader) - filename = LOCAL_APP_YAML if os.path.exists(LOCAL_APP_YAML) else AppYaml.default_filename - merlin_app_yaml = AppYaml(filename) - merlin_app_yaml.update_data(main_server_config) - merlin_app_yaml.write(filename) + with resources.path("merlin.server", MERLIN_SERVER_CONFIG) as merlin_server_config: + with open(merlin_server_config) as f: + main_server_config = yaml.load(f, yaml.Loader) + filename = LOCAL_APP_YAML if os.path.exists(LOCAL_APP_YAML) else AppYaml.default_filename + merlin_app_yaml = AppYaml(filename) + merlin_app_yaml.update_data(main_server_config) + merlin_app_yaml.write(filename) + LOG.info("Applying merlin server configuration to app.yaml") server_config = pull_server_config() if not server_config: @@ -301,8 +310,9 @@ def pull_server_image() -> bool: if not os.path.exists(os.path.join(config_dir, config_file)): LOG.info("Copying default redis configuration file.") try: - file_dir = os.path.dirname(os.path.abspath(__file__)) - shutil.copy(os.path.join(file_dir, config_file), config_dir) + with resources.path("merlin.server", config_file) as file: + with open(os.path.join(config_dir, config_file), "w") as outfile, open(file, "r") as infile: + outfile.write(infile.read()) except OSError: LOG.error(f"Destination location {config_dir} is not writable.") return False From a40ed4625d7eb3ad296e5194c3ebd36992f9ad76 Mon Sep 17 00:00:00 2001 From: Ryan Lee <44886374+ryannova@users.noreply.github.com> Date: Wed, 21 Dec 2022 15:01:41 -0800 Subject: [PATCH 063/126] Version 1.9.1 (#395) * Create python-publish.yml (#353) * Adding GitHub action for publishing to pypi * Workflows Community Initiative Metadata (#355) * Added Workflows Community Initiative metadata info; fixed some old links * Run black * Add updates for lgtm CI security site (#357) * Update code to remove LGTM Errors and Warnings and implement Recommendations. * Change BaseException to Exception. * Add lgtm config file. * Changes for flake8. * Add TypeError yo yam read. * Add TypeError to yaml read. * Just return when successful on the yaml read. * Fix typo. * Add merlin/examples to lgtm exclude list as well. * Add ssl comment. * Fix typo. * Update version to 1.8.5. * Update conf.py for new sphinx versions. * Added Server Command Feature to Merlin (#360) * Added merlin server capability to initalize, monitor and stop redis containers * Added configuration for singularity, docker, and podman in merlin/server/ * Created documentation for "merlin server" command * Added tests to initialize, start and stop a singularity+redis server to the local test suite. (Future: add to the "distributed tests" connecting to that server and running merlin) Co-authored-by: Ryan Lee & Joe Koning * Fix lgtm returns (#363) * Changed script sys.exit commands to use try/catch, per lgtm recommendation * Allow for flux exec arguments to limit the number of celery workers. (#366) * Added the flux_exec batch argument to allow for flux exec arguments, e.g. flux_exec: flux exec -r "0-1" to run celery workers only on ranks 0 and 1 of a multi-rank allocation. * Remove period. * Merlin server configuration through commands (#365) * Reorganized functions within server_setup and server_config * Rename server_setup file to server_commands * Added password generation for redis container in merlin server * Changed redis configuration to require password authentication * Added merlin config flags ipaddress, port, password, directory, snapshot_seconds, snapshot_changes, snapshot_file, append_mode, append_file * Added server_util.py * Added merlin user file into merlin server config * Added RedisConfig class to interact and change config values within the redis config file * Added merlin server restart * Updated info messages * Added function to add/remove users and store info to user file * Update running container with new users and removed users * Added ServerConfig, ProcessConfig, ContainerConfig, and ContainerFormatConfig classes to interact with configuration files * Adjusted adding user and password to use values in config files * Updated host in redis.conf * Updated pull_server_image step * Moved creation of local merlin config to create_server_config() * Added placeholder for documentation of restart and config commands for merlin server * Bugfix/changelog ci (#370) * remove deprecated gitlab ci file * Change CHANGELOG test to work for PRs other than to main * App.yaml for merlin server (#369) * Added AppYaml class to pull app.yaml and make changes required for merlin server configuration * Applied AppYaml class and added log message to inform users to use new app.yaml to use merlin server * Update LOG messages to inform users regarding local runs and instruct users of how to use app.yaml for local configuration * Changed type to image type in ContainerConfig * Shorten CHANGELOG.md for merlin server changes * Updated read in AppYaml to utilize merlin.util.load_yaml * Updated merlin server unit testing (#372) * Added additional tests for merlin server in test definitions * Fixed directory change to create a new directory if one doesn't exist * Updated redis version to provide acl user channel support * Addition of new shortcuts in specification file (#375) * Added five shortcuts to the specification definition MERLIN_SAMPLE_VECTOR, MERLIN_SAMPLE_NAMES, MERLIN_SPEC_ORIGINAL_TEMPLATE, MERLIN_SPEC_EXECUTED_RUN, MERLIN_SPEC_ARCHIVED_COPY * Added documentation for the above shortcuts. Co-authored-by: Jim Gaffney * Remove emoji from issue templates (#377) * Update bug_report.md remove "buggy" emoji * Update feature_request.md * Update question.md * Update CHANGELOG.md * Update CHANGELOG.md typo fix Co-authored-by: Ryan Lee * Update contribute.rst Remove more emoji from docs that are breaking pdf builds * Update cert_req to cert_regs in the docs. (#379) Co-authored-by: Ryan Lee <44886374+ryannova@users.noreply.github.com> * Ssl server check fixes (#380) * Add ssl to the Connection object for checking broker and results server acess. * Update CHANGELOG * Update documentation in tutorial and merlin server (#378) * Updated installation in instroduction and removed redis requirements * Removed pip headers and added commands for merlin server into installation * Removed additional references to old redis way and update description of merlin server * Remove more emoji from docs that are breaking pdf builds * Updated CHANGELOG to reflect changes to documentation Co-authored-by: Luc Peterson * Update MANIFEST.in (#381) * Update MANIFEST.in Add .temp to examples in MANIFEST, so that they get bundled with pypi releases * Update CHANGELOG.md * Add support for non-merlin blocks in specification file (#376) * Adding support for "user" block in _dict_to_string method * Updated CHANGELOG * Updated Merlin Spec docs * Added user block in feature_demo.yaml example Co-authored-by: Jim Gaffney * Update Merlin Server (#385) * Added condition for fatal error from redis server * Update default value for config_dir * Updated fix-style target in Makefile to be consistent with other style related targets * Update default password to use generated password * Updated run user to be default rather than created user * Updated singularity command to specify configuration directory as home directory to solve unaccessible directory issue * Update merlin to use app.yaml configuration rather than its own configuration file * Docs/install changes (#383) Many modifications to documentation, including installation instructions and formatting fixes. * Maestro v 1.1.9dev1 Compatibility (#388) Maestro up to date compatibility Also unpacked maestro DAG to just use what we need, which should help reduce task message size and perhaps allow us to use other serializers in the future. * Bump certifi from 2022.9.24 to 2022.12.7 in /docs (#387) Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.9.24 to 2022.12.7. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2022.09.24...2022.12.07) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Release 1.9.0 (#390) * Make CHANGELOG more concise * Updated Merlin version and added License to files missing it * Incremented python version for workflow test * fix merlinspec not being installed with pip and python 3.7 issues with celery * update changelog and version to 1.9.1 * fix a numpy issue on new numpy version release * modify changelog to show numpy fix * add version change to all files * re-add fix for numpy since it got removed in the last commit by accident * revert utils.py back to previous implementation * change dtype to python str type * Hotfix for merlin server unable to write config files. (#394) * Hotfix for merlin server unable to write config files. Change files to modules and copy files from new file modules Signed-off-by: dependabot[bot] Co-authored-by: Luc Peterson Co-authored-by: Joseph M. Koning Co-authored-by: Joe Koning Co-authored-by: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Brian Gunnarson --- CHANGELOG.md | 7 ++++ MANIFEST.in | 1 + Makefile | 2 +- merlin/__init__.py | 4 +-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- .../security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 29 +++++++++++++++++ merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 32 ++++++++++++------- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 6 ++-- requirements/release.txt | 1 + setup.py | 2 +- tests/integration/run_tests.py | 2 +- 55 files changed, 112 insertions(+), 64 deletions(-) create mode 100644 merlin/server/__init__.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a9cde753..5e480c8d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.9.1] +### Fixed +- Added merlin/spec/merlinspec.json to MANIFEST.in so pip will actually install it when ran +- Fixed a bug where "from celery import Celery" was failing on python 3.7 +- Numpy error about numpy.str not existing from a new numpy release +- Made merlin server configurations into modules that can be loaded and written to users + ## [1.9.0] ### Added - Added support for Python 3.11 diff --git a/MANIFEST.in b/MANIFEST.in index cefbd23a5..11e403335 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ recursive-include merlin/server *.yaml *.py recursive-include merlin/examples *.py *.yaml *.c *.json *.sbatch *.bsub *.txt *.temp include requirements.txt include requirements/* +include merlin/spec/merlinspec.json \ No newline at end of file diff --git a/Makefile b/Makefile index b0b1fb0a9..570ee3e25 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index aa33bc4d2..eaf3ff668 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.9.0" +__version__ = "1.9.1" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 0b5971627..a9c6093ca 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index ebac67eec..0041523b2 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index b02f9e909..30fea7f47 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 26c64e9e2..da214275b 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 8d9a89285..274ddb062 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 1859a55df..591f9a49f 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 6f3e58e9a..4861a757b 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index e378573c4..229f00851 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 16365e32c..1ba552626 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 9820e1041..6049bb5b5 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 027bcd291..352da5550 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 7ade90e05..b460be069 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index a546591f1..e30a18ca0 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index a8c0a1ef2..7822e7086 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 5dd08ddfb..bd22ddaa3 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 335bd05f5..e435e1ba0 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index cd47be750..626b776d5 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index 3b1469f70..30140981e 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 39c5cf2e0..43573b416 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index d59fc8511..2796465ab 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 269bf6097..816546f67 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 33c6776e4..f540e6f1f 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 9e207b749..5de8423b0 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 4195ceacc..a1e4de372 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 8858dcfad..90711756c 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py new file mode 100644 index 000000000..7155d0c5f --- /dev/null +++ b/merlin/server/__init__.py @@ -0,0 +1,29 @@ +############################################################################### +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 8a570b70d..38359938c 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 57cb5af22..ea5c8f1a8 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # @@ -32,7 +32,6 @@ import logging import os import random -import shutil import string import subprocess from io import BufferedReader @@ -40,6 +39,12 @@ import yaml + +try: + import importlib.resources as resources +except ImportError: + import importlib_resources as resources + from merlin.server.server_util import ( CONTAINER_TYPES, MERLIN_CONFIG_DIR, @@ -158,18 +163,22 @@ def create_server_config() -> bool: continue LOG.info(f"Copying file {file} to configuration directory.") try: - shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), file), config_dir) + with resources.path("merlin.server", file) as config_file: + with open(file_path, "w") as outfile, open(config_file, "r") as infile: + outfile.write(infile.read()) except OSError: LOG.error(f"Destination location {config_dir} is not writable.") return False # Load Merlin Server Configuration and apply it to app.yaml - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), MERLIN_SERVER_CONFIG)) as f: - main_server_config = yaml.load(f, yaml.Loader) - filename = LOCAL_APP_YAML if os.path.exists(LOCAL_APP_YAML) else AppYaml.default_filename - merlin_app_yaml = AppYaml(filename) - merlin_app_yaml.update_data(main_server_config) - merlin_app_yaml.write(filename) + with resources.path("merlin.server", MERLIN_SERVER_CONFIG) as merlin_server_config: + with open(merlin_server_config) as f: + main_server_config = yaml.load(f, yaml.Loader) + filename = LOCAL_APP_YAML if os.path.exists(LOCAL_APP_YAML) else AppYaml.default_filename + merlin_app_yaml = AppYaml(filename) + merlin_app_yaml.update_data(main_server_config) + merlin_app_yaml.write(filename) + LOG.info("Applying merlin server configuration to app.yaml") server_config = pull_server_config() if not server_config: @@ -301,8 +310,9 @@ def pull_server_image() -> bool: if not os.path.exists(os.path.join(config_dir, config_file)): LOG.info("Copying default redis configuration file.") try: - file_dir = os.path.dirname(os.path.abspath(__file__)) - shutil.copy(os.path.join(file_dir, config_file), config_dir) + with resources.path("merlin.server", config_file) as file: + with open(os.path.join(config_dir, config_file), "w") as outfile, open(file, "r") as infile: + outfile.write(infile.read()) except OSError: LOG.error(f"Destination location {config_dir} is not writable.") return False diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index e6eb6379d..16d2c3686 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 950ceb253..46db0fd4f 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 1c0e9fa42..46f71882f 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 254ba80be..710bc2500 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index c4fcfee97..5ca2eea6b 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index a6ecdae2a..9d93197bf 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index f395c5d80..71e89bf84 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 34393f967..97fc40289 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 838c14762..fbd7dd6d7 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 826c1d2e3..ab998140a 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 5e7d89e43..3a55df606 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index b7f990a2a..9d36e4173 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index 2959b79e2..93777d95d 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # @@ -240,9 +240,9 @@ def load_array_file(filename, ndmin=2): ) # Make sure text files load as strings with minimum number of dimensions elif protocol == "csv": - array = np.loadtxt(filename, delimiter=",", ndmin=ndmin, dtype=np.str) + array = np.loadtxt(filename, delimiter=",", ndmin=ndmin, dtype=str) elif protocol == "tab": - array = np.loadtxt(filename, ndmin=ndmin, dtype=np.str) + array = np.loadtxt(filename, ndmin=ndmin, dtype=str) else: raise TypeError( f"{protocol} is not a valid array file extension.\ diff --git a/requirements/release.txt b/requirements/release.txt index 821589c41..11fd85129 100644 --- a/requirements/release.txt +++ b/requirements/release.txt @@ -2,6 +2,7 @@ cached_property celery[redis,sqlalchemy]>=5.0.3 coloredlogs cryptography +importlib_metadata<5.0.0; python_version == '3.7' importlib_resources; python_version < '3.7' maestrowf>=1.1.9dev1 numpy diff --git a/setup.py b/setup.py index 9bf170126..06706ca57 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index bfd80eb83..913a2ab08 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # From 207ca5f313b59402cc72681f9fbfde72e723e3ed Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Thu, 22 Dec 2022 15:03:11 -0800 Subject: [PATCH 064/126] PR 389 (#398) * Allows loading of nparrays of dtype=object Co-authored-by: Keilbart --- CHANGELOG.md | 4 ++++ merlin/utils.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e480c8d8..fef53c8e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] +### Added +- Now loads np.arrays of dtype='object', allowing mix-type sample npy + ## [1.9.1] ### Fixed - Added merlin/spec/merlinspec.json to MANIFEST.in so pip will actually install it when ran diff --git a/merlin/utils.py b/merlin/utils.py index 93777d95d..5217e1d7a 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -232,7 +232,7 @@ def load_array_file(filename, ndmin=2): # Don't change binary-stored numpy arrays; just check dimensions if protocol == "npy": - array = np.load(filename) + array = np.load(filename, allow_pickle=True) if array.ndim < ndmin: LOG.error( f"Array in {filename} has fewer than the required \ From 34d53f04f0614a5fa175917149a810f97c8cebb4 Mon Sep 17 00:00:00 2001 From: Ryan Lee <44886374+ryannova@users.noreply.github.com> Date: Thu, 22 Dec 2022 15:15:23 -0800 Subject: [PATCH 065/126] Sync Up main with develop Sync up the branches between main and develop. No changes. From f2eb3d3e831ed5e8ec06f4501659e63d7848da1e Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Thu, 22 Dec 2022 15:21:50 -0800 Subject: [PATCH 066/126] Fix issue with the wheel not including .sh files for merlin examples (#397) * include all files for merlin examples Co-authored-by: bgunnar --- CHANGELOG.md | 2 ++ MANIFEST.in | 2 +- merlin/server/__init__.py | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fef53c8e9..5af7efd1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [unreleased] +### Fixed +- Pip wheel wasn't including .sh files for merlin examples ### Added - Now loads np.arrays of dtype='object', allowing mix-type sample npy diff --git a/MANIFEST.in b/MANIFEST.in index 11e403335..da9d411ad 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ recursive-include merlin/data *.yaml *.py recursive-include merlin/server *.yaml *.py -recursive-include merlin/examples *.py *.yaml *.c *.json *.sbatch *.bsub *.txt *.temp +recursive-include merlin/examples * include requirements.txt include requirements/* include merlin/spec/merlinspec.json \ No newline at end of file diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 7155d0c5f..31115004f 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,8 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. + # # For details, see https://github.com/LLNL/merlin. # From ef18489dae6670bad4de529c6fbd69226dd76520 Mon Sep 17 00:00:00 2001 From: Joe Koning Date: Mon, 6 Mar 2023 13:05:24 -0800 Subject: [PATCH 067/126] Feature/openfoam wf update (#400) Add a singularity based openfoam_wf example. Update the learn.py to create the missing Energy vs LidSpeed plot. --- CHANGELOG.md | 2 + Makefile | 12 +- .../workflows/openfoam_wf/scripts/learn.py | 4 +- .../openfoam_wf_no_docker/scripts/learn.py | 4 +- .../openfoam_wf_singularity/openfoam_wf.yaml | 89 +++++++++ .../openfoam_wf_singularity/requirements.txt | 3 + .../scripts/blockMesh_template.txt | 75 ++++++++ .../scripts/cavity_setup.sh | 33 ++++ .../scripts/combine_outputs.py | 40 ++++ .../openfoam_wf_singularity/scripts/learn.py | 172 ++++++++++++++++++ .../scripts/make_samples.py | 27 +++ .../scripts/mesh_param_script.py | 49 +++++ .../scripts/run_openfoam | 45 +++++ 13 files changed, 547 insertions(+), 8 deletions(-) create mode 100644 merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml create mode 100644 merlin/examples/workflows/openfoam_wf_singularity/requirements.txt create mode 100644 merlin/examples/workflows/openfoam_wf_singularity/scripts/blockMesh_template.txt create mode 100755 merlin/examples/workflows/openfoam_wf_singularity/scripts/cavity_setup.sh create mode 100644 merlin/examples/workflows/openfoam_wf_singularity/scripts/combine_outputs.py create mode 100644 merlin/examples/workflows/openfoam_wf_singularity/scripts/learn.py create mode 100644 merlin/examples/workflows/openfoam_wf_singularity/scripts/make_samples.py create mode 100644 merlin/examples/workflows/openfoam_wf_singularity/scripts/mesh_param_script.py create mode 100755 merlin/examples/workflows/openfoam_wf_singularity/scripts/run_openfoam diff --git a/CHANGELOG.md b/CHANGELOG.md index 5af7efd1c..5574fa973 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Fixed - Pip wheel wasn't including .sh files for merlin examples +- The learn.py script in the openfoam_wf* examples will now create the missing Energy v Lidspeed plot ### Added - Now loads np.arrays of dtype='object', allowing mix-type sample npy +- Added a singularity container openfoam_wf example ## [1.9.1] ### Fixed diff --git a/Makefile b/Makefile index 570ee3e25..ac1e870f3 100644 --- a/Makefile +++ b/Makefile @@ -130,9 +130,9 @@ check-black: check-isort: . $(VENV)/bin/activate; \ - $(PYTHON) -m isort --check --line-length $(MAX_LINE_LENGTH) merlin; \ - $(PYTHON) -m isort --check --line-length $(MAX_LINE_LENGTH) tests; \ - $(PYTHON) -m isort --check --line-length $(MAX_LINE_LENGTH) *.py; \ + $(PYTHON) -m isort --check -w $(MAX_LINE_LENGTH) merlin; \ + $(PYTHON) -m isort --check -w $(MAX_LINE_LENGTH) tests; \ + $(PYTHON) -m isort --check -w $(MAX_LINE_LENGTH) *.py; \ check-pylint: @@ -164,9 +164,9 @@ checks: check-style check-camel-case # automatically make python files pep 8-compliant fix-style: . $(VENV)/bin/activate; \ - $(PYTHON) -m isort --line-length $(MAX_LINE_LENGTH) $(MRLN); \ - $(PYTHON) -m isort --line-length $(MAX_LINE_LENGTH) $(TEST); \ - $(PYTHON) -m isort --line-length $(MAX_LINE_LENGTH) *.py; \ + $(PYTHON) -m isort -w $(MAX_LINE_LENGTH) $(MRLN); \ + $(PYTHON) -m isort -w $(MAX_LINE_LENGTH) $(TEST); \ + $(PYTHON) -m isort -w $(MAX_LINE_LENGTH) *.py; \ $(PYTHON) -m black --target-version py36 -l $(MAX_LINE_LENGTH) $(MRLN); \ $(PYTHON) -m black --target-version py36 -l $(MAX_LINE_LENGTH) $(TEST); \ $(PYTHON) -m black --target-version py36 -l $(MAX_LINE_LENGTH) *.py; \ diff --git a/merlin/examples/workflows/openfoam_wf/scripts/learn.py b/merlin/examples/workflows/openfoam_wf/scripts/learn.py index 96124c46f..edf96ebbe 100644 --- a/merlin/examples/workflows/openfoam_wf/scripts/learn.py +++ b/merlin/examples/workflows/openfoam_wf/scripts/learn.py @@ -103,11 +103,13 @@ ax[0][1].set_ylim([y_min, y_max]) +ax[1][1].scatter(X[:, 0], 10 ** y[:, 1]) ax[1][1].set_xlabel(r"Lidspeed ($\frac{m}{s}$)", fontsize=fontsize) ax[1][1].set_ylabel(r"$Energy$", fontsize=fontsize) ax[1][1].set_title("Average Energy Variation with Lidspeed") ax[1][1].grid() - +ax[1][1].set_xlim([np.min(X[:, 0]), np.max(X[:, 0])]) +ax[1][1].set_ylim([np.min(10 ** y[:, 1]), np.max(10 ** y[:, 1])]) input_energy = ax[1][0].scatter( X[:, 0], diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py index 96124c46f..edf96ebbe 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py +++ b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py @@ -103,11 +103,13 @@ ax[0][1].set_ylim([y_min, y_max]) +ax[1][1].scatter(X[:, 0], 10 ** y[:, 1]) ax[1][1].set_xlabel(r"Lidspeed ($\frac{m}{s}$)", fontsize=fontsize) ax[1][1].set_ylabel(r"$Energy$", fontsize=fontsize) ax[1][1].set_title("Average Energy Variation with Lidspeed") ax[1][1].grid() - +ax[1][1].set_xlim([np.min(X[:, 0]), np.max(X[:, 0])]) +ax[1][1].set_ylim([np.min(10 ** y[:, 1]), np.max(10 ** y[:, 1])]) input_energy = ax[1][0].scatter( X[:, 0], diff --git a/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml b/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml new file mode 100644 index 000000000..3f3bbb672 --- /dev/null +++ b/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml @@ -0,0 +1,89 @@ +description: + name: openfoam_wf_singularity + description: | + A parameter study that includes initializing, running, + post-processing, collecting, learning and visualizing OpenFOAM runs + + +env: + variables: + OUTPUT_PATH: ./openfoam_wf_output + + SCRIPTS: $(MERLIN_INFO)/scripts + SIF: $(MERLIN_INFO)/openfoam6.sif + N_SAMPLES: 100 + + +merlin: + samples: + generate: + cmd: | + cp -r $(SPECROOT)/scripts $(MERLIN_INFO)/ + + # Generates the samples + python $(SCRIPTS)/make_samples.py -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples + file: $(MERLIN_INFO)/samples.npy + column_labels: [LID_SPEED, VISCOSITY] + resources: + workers: + nonsimworkers: + args: -l INFO --concurrency 1 + steps: [setup, combine_outputs, learn] + simworkers: + args: -l INFO --concurrency 10 --prefetch-multiplier 1 -Ofair + steps: [sim_runs] + + +study: + - name: setup + description: | + Installs necessary python packages and imports the cavity directory + from the singularity container + run: + cmd: | + pip install -r $(SPECROOT)/requirements.txt + + # Set up the cavity directory in the MERLIN_INFO directory + source $(SCRIPTS)/cavity_setup.sh $(MERLIN_INFO) + + - name: sim_runs + description: | + Edits the Lidspeed and viscosity then runs OpenFOAM simulation + using the icoFoam solver + run: + cmd: | + cp -r $(MERLIN_INFO)/cavity cavity/ + cd cavity + + ## Edits default values for viscosity and lidspeed with + # values specified by samples section of the merlin block + sed -i "18s/.*/nu [0 2 -1 0 0 0 0] $(VISCOSITY);/" constant/transportProperties + sed -i "26s/.*/ value uniform ($(LID_SPEED) 0 0);/" 0/U + + cd .. + cp $(SCRIPTS)/run_openfoam . + + # The local sample directory is bound (--bind) to the /merlin_sample dir in + # the container instance + # The sample/cavity is bound (--bind) to the /cavity dir in the container instance + REPATH=$(WORKSPACE)/$(MERLIN_SAMPLE_PATH) + CBIND=${REPATH}:/merlin_sample,${REPATH}/cavity:/cavity + + # Creating a unique OpenFOAM singularity execution for each sample to run the simulation + singularity exec --bind ${CBIND} $(SIF) /merlin_sample/run_openfoam $(LID_SPEED) + depends: [setup] + task_queue: simqueue + + - name: combine_outputs + description: Combines the outputs of the previous step + run: + cmd: | + python $(SCRIPTS)/combine_outputs.py -data $(sim_runs.workspace) -merlin_paths $(MERLIN_PATHS_ALL) + depends: [sim_runs_*] + + - name: learn + description: Learns the output of the openfoam simulations using input parameters + run: + cmd: | + python $(SCRIPTS)/learn.py -workspace $(MERLIN_WORKSPACE) + depends: [combine_outputs] diff --git a/merlin/examples/workflows/openfoam_wf_singularity/requirements.txt b/merlin/examples/workflows/openfoam_wf_singularity/requirements.txt new file mode 100644 index 000000000..8042c2422 --- /dev/null +++ b/merlin/examples/workflows/openfoam_wf_singularity/requirements.txt @@ -0,0 +1,3 @@ +Ofpp==0.11 +scikit-learn==0.21.3 +matplotlib==3.1.1 diff --git a/merlin/examples/workflows/openfoam_wf_singularity/scripts/blockMesh_template.txt b/merlin/examples/workflows/openfoam_wf_singularity/scripts/blockMesh_template.txt new file mode 100644 index 000000000..fa327d19c --- /dev/null +++ b/merlin/examples/workflows/openfoam_wf_singularity/scripts/blockMesh_template.txt @@ -0,0 +1,75 @@ +/*--------------------------------*- C++ -*----------------------------------*\ +| ========= | | +| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox | +| \\ / O peration | Version: 5 | +| \\ / A nd | Web: www.OpenFOAM.org | +| \\/ M anipulation | | +\*---------------------------------------------------------------------------*/ +FoamFile +{ + version 2.0; + format ascii; + class dictionary; + object blockMeshDict; +} +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +convertToMeters 0.1; + +vertices +( + (0 0 0) + (WIDTH 0 0) + (WIDTH HEIGHT 0) + (0 HEIGHT 0) + (0 0 0.1) + (WIDTH 0 0.1) + (WIDTH HEIGHT 0.1) + (0 HEIGHT 0.1) +); + +blocks +( + hex (0 1 2 3 4 5 6 7) (X_RESOLUTION Y_RESOLUTION 1) simpleGrading (1 1 1) +); + +edges +( +); + +boundary +( + movingWall + { + type wall; + faces + ( + (3 7 6 2) + ); + } + fixedWalls + { + type wall; + faces + ( + (0 4 7 3) + (2 6 5 1) + (1 5 4 0) + ); + } + frontAndBack + { + type empty; + faces + ( + (0 3 2 1) + (4 5 6 7) + ); + } +); + +mergePatchPairs +( +); + +// ************************************************************************* // diff --git a/merlin/examples/workflows/openfoam_wf_singularity/scripts/cavity_setup.sh b/merlin/examples/workflows/openfoam_wf_singularity/scripts/cavity_setup.sh new file mode 100755 index 000000000..66891ca9c --- /dev/null +++ b/merlin/examples/workflows/openfoam_wf_singularity/scripts/cavity_setup.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +MERLIN_INFO=$1 + +DOCKER_IMAGE="docker://cfdengine/openfoam" +#DOCKER_IMAGE="docker://openfoam/openfoam6-paraview56" +SIF=openfoam6.sif +CONTAINER_DST="/merlin_openfoam" +cd $MERLIN_INFO +singularity build ${SIF} ${DOCKER_IMAGE} +singularity exec -B `realpath .`:${CONTAINER_DST} ${SIF} cp -rf /opt/openfoam6/tutorials/incompressible/icoFoam/cavity/cavity ${CONTAINER_DST} + +cd - + +cd $MERLIN_INFO/cavity + +echo "***** Setting Up Mesh *****" +python $MERLIN_INFO/scripts/mesh_param_script.py -scripts_dir $MERLIN_INFO/scripts/ +mv blockMeshDict.txt system/blockMeshDict +if [ -e system/blockMeshDict ]; then + echo "... blockMeshDict.txt complete" +fi + +if [ -e system/controlDict ]; then + CONTROL="system/controlDict" + echo "***** Setting Control Dictionary *****" + sed -i "30s/.*/writeControl runTime;/" ${CONTROL} + sed -i "26s/.*/endTime 1;/" ${CONTROL} + sed -i "32s/.*/writeInterval .1;/" ${CONTROL} + echo "... system/controlDict edits complete" +else + echo "Can't find system/controlDict" +fi diff --git a/merlin/examples/workflows/openfoam_wf_singularity/scripts/combine_outputs.py b/merlin/examples/workflows/openfoam_wf_singularity/scripts/combine_outputs.py new file mode 100644 index 000000000..2c23c840a --- /dev/null +++ b/merlin/examples/workflows/openfoam_wf_singularity/scripts/combine_outputs.py @@ -0,0 +1,40 @@ +import argparse +import glob + +import numpy as np +import Ofpp + + +descript = """Using parameters to edit OpenFOAM parameters""" +parser = argparse.ArgumentParser(description=descript) + +parser.add_argument("-data", "--data_dir", help="The home directory of the data directories") +parser.add_argument("-merlin_paths", nargs="+", help="The path of all merlin runs") + +args = parser.parse_args() + +DATA_DIR = args.data_dir +X = args.merlin_paths + +dir_names = [DATA_DIR + "/" + Xi + "/cavity" for Xi in X] + +num_of_timesteps = 10 +U = [] +enstrophy = [] + +for i, dir_name in enumerate(dir_names): + for name in glob.glob(dir_name + "/[0-9]*/U"): + if name[-4:] == "/0/U": + continue + U.append(Ofpp.parse_internal_field(name)) + + for name in glob.glob(dir_name + "/[0-9]*/enstrophy"): + if name[-12:] == "/0/enstrophy": + continue + enstrophy.append(Ofpp.parse_internal_field(name)) + +resolution = np.array(enstrophy).shape[-1] +U = np.array(U).reshape(len(dir_names), num_of_timesteps, resolution, 3) +enstrophy = np.array(enstrophy).reshape(len(dir_names), num_of_timesteps, resolution) / float(resolution) + +np.savez("data.npz", U, enstrophy) diff --git a/merlin/examples/workflows/openfoam_wf_singularity/scripts/learn.py b/merlin/examples/workflows/openfoam_wf_singularity/scripts/learn.py new file mode 100644 index 000000000..edf96ebbe --- /dev/null +++ b/merlin/examples/workflows/openfoam_wf_singularity/scripts/learn.py @@ -0,0 +1,172 @@ +import argparse + +import matplotlib.pyplot as plt +import numpy as np +from joblib import dump +from sklearn.ensemble import RandomForestRegressor +from sklearn.metrics import mean_squared_error + + +descript = """Using parameters to edit OpenFOAM parameters""" +parser = argparse.ArgumentParser(description=descript) + +parser.add_argument("-workspace", help="The DAG spec root") + +args = parser.parse_args() + +training_percent = 0.6 +timestep = -1 +fontsize = 20 + +WORKSPACE = args.workspace +inputs_dir = WORKSPACE + "/merlin_info" +outputs_dir = WORKSPACE + "/combine_outputs" + +outputs = np.load(outputs_dir + "/data.npz") +U = outputs["arr_0"] +enstrophy = outputs["arr_1"] + +energy_byhand = np.sum(np.sum(U**2, axis=3), axis=2) / U.shape[2] / 2 +enstrophy_all = np.sum(enstrophy, axis=2) + +X = np.load(inputs_dir + "/samples.npy") +y = np.concatenate( + ( + enstrophy_all[:, timestep].reshape(-1, 1), + energy_byhand[:, timestep].reshape(-1, 1), + ), + axis=1, +) +X[:, 1] = np.log10(X[:, 0] / X[:, 1]) # np.log10(X) +y = np.log10(y) + +training_size = int(training_percent * len(X)) + +X_train = X[:training_size] +y_train = y[:training_size] +X_test = X[training_size:] +y_test = y[training_size:] + +regr = RandomForestRegressor(max_depth=10, random_state=0, n_estimators=7) + +regr.fit(X_train, y_train) +print("training score:", regr.score(X_train, y_train)) +print("testing score: ", regr.score(X_test, y_test)) +print(mean_squared_error(y_test, regr.predict(X_test))) + +dump(regr, "trained_model.joblib") + + +fig, ax = plt.subplots(3, 2, figsize=(25, 25), constrained_layout=True) +plt.rcParams.update({"font.size": 25}) +plt.rcParams["lines.linewidth"] = 5 + +x = np.linspace(-5, 8, 100) +y1 = 1 * x +ax[0][0].plot(x, y1, "-r", label="y=x", linewidth=1) + +y_pred = regr.predict(X_train) +ax[0][0].scatter(y_train[:, 0], y_pred[:, 0], label="Log10 Enstrophy") +ax[0][0].scatter(y_train[:, 1], y_pred[:, 1], label="Log10 Energy") +ax[0][0].set_title("Velocity Magnitude %s" % timestep) + +ax[0][0].set_xlabel("Actual", fontsize=fontsize) +ax[0][0].set_ylabel("Predicted", fontsize=fontsize) +ax[0][0].set_title("Training Data, # Points: %s" % len(y_pred)) +ax[0][0].legend() +ax[0][0].grid() + +x_min = np.min([np.min(y_train[:, 0]), np.min(y_train[:, 1])]) +y_min = np.min([np.min(y_pred[:, 0]), np.min(y_pred[:, 1])]) +x_max = np.max([np.max(y_train[:, 0]), np.max(y_train[:, 1])]) +y_max = np.max([np.max(y_pred[:, 0]), np.max(y_pred[:, 1])]) + +y_pred = regr.predict(X_test) +ax[0][1].plot(x, y1, "-r", label="y=x", linewidth=1) +ax[0][1].scatter(y_test[:, 0], y_pred[:, 0], label="Log10 Enstrophy") +ax[0][1].scatter(y_test[:, 1], y_pred[:, 1], label="Log10 Energy") +ax[0][1].set_xlabel("Actual", fontsize=fontsize) +ax[0][1].set_ylabel("Predicted", fontsize=fontsize) +ax[0][1].set_title("Testing Data, # Points: %s" % len(y_pred)) +ax[0][1].legend() +ax[0][1].grid() + +x_min = np.min([np.min(y_test[:, 0]), np.min(y_test[:, 1]), x_min]) - 0.1 +y_min = np.min([np.min(y_pred[:, 0]), np.min(y_pred[:, 1]), y_min]) - 0.1 +x_max = np.max([np.max(y_test[:, 0]), np.max(y_test[:, 1]), x_max]) + 0.1 +y_max = np.max([np.max(y_pred[:, 0]), np.max(y_pred[:, 1]), y_max]) + 0.1 + + +ax[0][0].set_xlim([x_min, x_max]) +ax[0][0].set_ylim([y_min, y_max]) +ax[0][1].set_xlim([x_min, x_max]) +ax[0][1].set_ylim([y_min, y_max]) + + +ax[1][1].scatter(X[:, 0], 10 ** y[:, 1]) +ax[1][1].set_xlabel(r"Lidspeed ($\frac{m}{s}$)", fontsize=fontsize) +ax[1][1].set_ylabel(r"$Energy$", fontsize=fontsize) +ax[1][1].set_title("Average Energy Variation with Lidspeed") +ax[1][1].grid() +ax[1][1].set_xlim([np.min(X[:, 0]), np.max(X[:, 0])]) +ax[1][1].set_ylim([np.min(10 ** y[:, 1]), np.max(10 ** y[:, 1])]) + +input_energy = ax[1][0].scatter( + X[:, 0], + X[:, 1], + s=100, + edgecolors="black", + c=10 ** y[:, 1], + cmap=plt.get_cmap("viridis"), +) +ax[1][0].set_xlabel(r"Lidspeed ($\frac{m}{s}$)", fontsize=fontsize) +ax[1][0].set_ylabel(r"$Log_{10}$(Reynolds Number)", fontsize=fontsize) +ax[1][0].set_title("Inputs vs Average Energy") +ax[1][0].grid() +cbar = plt.colorbar(input_energy, ax=ax[1][0]) +cbar.ax.set_ylabel(r"$Energy$", rotation=270, labelpad=30) + +ax[1][0].tick_params(axis="both", which="major", labelsize=fontsize) +ax[1][1].tick_params(axis="both", which="major", labelsize=fontsize) +ax[1][0].tick_params(axis="both", which="major", labelsize=fontsize) +ax[1][1].tick_params(axis="both", which="major", labelsize=fontsize) + + +y_pred_all = regr.predict(X) +input_enstrophy = ax[2][0].scatter( + X[:, 0], + X[:, 1], + s=100, + edgecolors="black", + c=y[:, 0] - y_pred_all[:, 0], + cmap=plt.get_cmap("Spectral"), +) +ax[2][0].set_xlabel(r"Lidspeed ($\frac{m}{s}$)", fontsize=fontsize) +ax[2][0].set_ylabel(r"$Log_{10}$(Reynolds Number)", fontsize=fontsize) +ax[2][0].set_title("Inputs vs Enstrophy error") +ax[2][0].grid() +cbar = plt.colorbar(input_enstrophy, ax=ax[2][0]) +cbar.ax.set_ylabel(r"$y_{act} - y_{pred}$", rotation=270, labelpad=30) + + +input_energy = ax[2][1].scatter( + X[:, 0], + X[:, 1], + s=100, + edgecolors="black", + c=y[:, 1] - y_pred_all[:, 1], + cmap=plt.get_cmap("Spectral"), +) +ax[2][1].set_xlabel(r"Lidspeed ($\frac{m}{s}$)", fontsize=fontsize) +ax[2][1].set_ylabel(r"$Log_{10}$(Reynolds Number)", fontsize=fontsize) +ax[2][1].set_title("Inputs vs Energy error") +ax[2][1].grid() +cbar = plt.colorbar(input_energy, ax=ax[2][1]) +cbar.ax.set_ylabel(r"$y_{act} - y_{pred}$", rotation=270, labelpad=30) + +ax[0][0].tick_params(axis="both", which="major", labelsize=fontsize) +ax[0][1].tick_params(axis="both", which="major", labelsize=fontsize) +ax[2][0].tick_params(axis="both", which="major", labelsize=fontsize) +ax[2][1].tick_params(axis="both", which="major", labelsize=fontsize) + +plt.savefig("prediction.png") diff --git a/merlin/examples/workflows/openfoam_wf_singularity/scripts/make_samples.py b/merlin/examples/workflows/openfoam_wf_singularity/scripts/make_samples.py new file mode 100644 index 000000000..05b1f213f --- /dev/null +++ b/merlin/examples/workflows/openfoam_wf_singularity/scripts/make_samples.py @@ -0,0 +1,27 @@ +import argparse + +import numpy as np + + +def loguniform(low=-1, high=3, size=None, base=10): + return np.power(base, np.random.uniform(low, high, size)) + + +parser = argparse.ArgumentParser("Generate some samples!") +parser.add_argument("-n", help="number of samples", default=100, type=int) +parser.add_argument("-outfile", help="name of output .npy file", default="samples") + +args = parser.parse_args() + +N_SAMPLES = args.n +REYNOLD_RANGE = [1, 100] +LIDSPEED_RANGE = [0.1, 100] +BASE = 10 + +x = np.empty((N_SAMPLES, 2)) +x[:, 0] = np.random.uniform(LIDSPEED_RANGE[0], LIDSPEED_RANGE[1], size=N_SAMPLES) +vi_low = np.log10(x[:, 0] / REYNOLD_RANGE[1]) +vi_high = np.log10(x[:, 0] / REYNOLD_RANGE[0]) +x[:, 1] = [loguniform(low=vi_low[i], high=vi_high[i], base=BASE) for i in range(N_SAMPLES)] + +np.save(args.outfile, x) diff --git a/merlin/examples/workflows/openfoam_wf_singularity/scripts/mesh_param_script.py b/merlin/examples/workflows/openfoam_wf_singularity/scripts/mesh_param_script.py new file mode 100644 index 000000000..a523e400e --- /dev/null +++ b/merlin/examples/workflows/openfoam_wf_singularity/scripts/mesh_param_script.py @@ -0,0 +1,49 @@ +import argparse + + +def positive_numbers(x): + x = float(x) + if not x > 0: + raise argparse.ArgumentTypeError("Needs to be a positive number") + return x + + +descript = """Using parameters to edit OpenFOAM parameters""" +parser = argparse.ArgumentParser(description=descript) + +parser.add_argument("-x", "--width", type=positive_numbers, default=10, help="length") +parser.add_argument("-y", "--height", type=positive_numbers, default=10, help="height") +parser.add_argument("-r", "--resolution", type=int, default=20, help="Resolution") +parser.add_argument("-p", "--piso", type=int, default=0, help="pisoFoam?") +parser.add_argument("-scripts_dir", help="Name of the scripts directory", default="./") + +args = parser.parse_args() +WIDTH = args.width +HEIGHT = args.height +RESOLUTION = args.resolution +SCRIPTS_DIRECTORY = args.scripts_dir +PISO = args.piso + +template = open(SCRIPTS_DIRECTORY + "blockMesh_template.txt", "r") + +WIDTH = float(WIDTH) +HEIGHT = float(HEIGHT) + +tmp = template.read() +tmp = tmp.replace("WIDTH", str(WIDTH)) +tmp = tmp.replace("HEIGHT", str(HEIGHT)) + +X_RESOLUTION = int(RESOLUTION * WIDTH / HEIGHT) + +tmp = tmp.replace("X_RESOLUTION", str(X_RESOLUTION)) +tmp = tmp.replace("Y_RESOLUTION", str(RESOLUTION)) + +if PISO: + tmp = tmp.replace("lid", "movingWall") + print("Changed lid to movingWall") + +f = open("blockMeshDict.txt", "w+") +f.write(tmp) +f.close() + +template.close() diff --git a/merlin/examples/workflows/openfoam_wf_singularity/scripts/run_openfoam b/merlin/examples/workflows/openfoam_wf_singularity/scripts/run_openfoam new file mode 100755 index 000000000..0e1d3d119 --- /dev/null +++ b/merlin/examples/workflows/openfoam_wf_singularity/scripts/run_openfoam @@ -0,0 +1,45 @@ +#!/bin/bash + +# Source OpenFOAM BASH profile +. /opt/openfoam6/etc/bashrc + +# Export fix for OPENMPI in a container +export OMPI_MCA_btl_vader_single_copy_mechanism=none +cd ${0%/*} || exit 1 # Run from this directory + +# Source tutorial run functions +. $WM_PROJECT_DIR/bin/tools/RunFunctions + +cavityCases="cavity" +LID_SPEED=$1 + +for caseName in $cavityCases +do + cd /$caseName + + blockMesh > blockMesh.out + + echo "***** Setting up control parameters ***** " + checkMesh > out + MIN_AREA=$(grep -oP '(?<=Minimum face area = )[0-9,.,e,-]+' out) + echo MIN_AREA "$MIN_AREA" >> out + MIN_AREA=${MIN_AREA::-1} + echo MIN_AREA $MIN_AREA >> out + DELTA_X=$(awk "BEGIN {printf \"%.30f\n\", sqrt($MIN_AREA)}") + echo DELTA_X $DELTA_X >> out + DELTA_T=$(awk "BEGIN {printf \"%.30f\n\", $DELTA_X / $LID_SPEED}") + echo DELTA_T $DELTA_T >> out + + MIN_DELTA_T=0.05 + DELTA_T=$(echo ${DELTA_T} ${MIN_DELTA_T} | awk '{if ($1 < $2) print $1; else print $2}') + + echo 'DELTA_T after comparison' $DELTA_T>> out + sed -i "28s/.*/deltaT $DELTA_T;/" system/controlDict + + echo "Running $(getApplication)" + $(getApplication) > $(getApplication).out + postProcess -func 'enstrophy' + foamToVTK +done + +#------------------------------------------------------------------------------ From 81a945d4cb4e23eb706a40babb56905583303412 Mon Sep 17 00:00:00 2001 From: "Joseph M. Koning" Date: Tue, 7 Mar 2023 09:36:18 -0800 Subject: [PATCH 068/126] Update docs. --- docs/source/faq.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 28d46460c..b72e7f6ff 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -356,7 +356,9 @@ What is flux? ~~~~~~~~~~~~~ Flux is a hierarchical scheduler and launcher for parallel simulations. It allows the user to specify the same launch command that will work on different HPC clusters with different -default schedulers such as SLURM or LSF. +default schedulers such as SLURM or LSF. Merlin versions earlier than 1.9.2 used the non-flux native +scheduler to launch a flux instance. Subsequent merlin versions can launch the merlin workers +using a native flux scheduler. More information can be found at the `Flux web page `_. @@ -370,6 +372,15 @@ in the ``launch_args`` variable in the batch section. type: flux launch_args: --mpi=none +.. _pbs: + +What is PBS? +~~~~~~~~~~~~ +Another job scheduler. See `Portable Batch System +https://en.wikipedia.org/wiki/Portable_Batch_System`_ +. +This functionality is only available to launch a flux scheduler. + How do I use flux on LC? ~~~~~~~~~~~~~~~~~~~~~~~~ The ``--mpibind=off`` option is currently required when using flux with a slurm launcher From 1523e5077b272a7bf836ff3c4b358a0354c1d69d Mon Sep 17 00:00:00 2001 From: Joe Koning Date: Tue, 7 Mar 2023 17:16:54 -0800 Subject: [PATCH 069/126] Feature/flux cluster (#402) * Add flux native worker launch. * Add flux native to changlog. * Run black. * Run black. * Run fix-style * Add try except for check_for_flux function when flux is not present. * Add function doc. * Add flux native run-workers test. * Added PBS flux launch support Added check_for_flux, check_for_slurm, check_for_lsf, and check_for_pbs utility functions * Return None from get_batch_type instead of slurm, now there are check functions, there should be an error condition returned. * Fix block comment * Fix comment. * Fix isort in Makefile. Run fix-style Add - to pbs for reading from stdin. * Fix SYS_TYPE check * fix-style * Re fix-style with new isort. * New qsub version uses -- for stdin. * Chnage celery_regex ito celery_slurm_regex Remove extra nodes variable in flux_par_native_test.yaml * Change celery_regex ito celery_slurm_regex Remove extra nodes variable in flux_par_native_test.yaml --- CHANGELOG.md | 6 + Makefile | 6 +- docs/source/conf.py | 47 ++--- merlin/ascii_art.py | 1 - merlin/common/tasks.py | 3 - .../workflows/flux/flux_par_native_test.yaml | 76 ++++++++ .../workflows/flux/scripts/flux_test/flux | 2 + .../workflows/flux/scripts/pbs_test/qsub | 2 + .../optimization/scripts/visualizer.py | 9 +- merlin/server/server_config.py | 12 +- merlin/study/batch.py | 165 ++++++++++++++++-- merlin/study/dag.py | 4 - merlin/study/study.py | 9 +- tests/integration/test_definitions.py | 27 ++- 14 files changed, 295 insertions(+), 74 deletions(-) create mode 100644 merlin/examples/workflows/flux/flux_par_native_test.yaml create mode 100755 merlin/examples/workflows/flux/scripts/flux_test/flux create mode 100755 merlin/examples/workflows/flux/scripts/pbs_test/qsub diff --git a/CHANGELOG.md b/CHANGELOG.md index 5574fa973..502d48379 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Now loads np.arrays of dtype='object', allowing mix-type sample npy - Added a singularity container openfoam_wf example +- Added flux native worker launch support +- Added PBS flux launch support +- Added check_for_flux, check_for_slurm, check_for_lsf, and check_for_pbs utility functions + +### Changed +- Changed celery_regex to celery_slurm_regex in test_definitions.py ## [1.9.1] ### Fixed diff --git a/Makefile b/Makefile index ac1e870f3..fd6949bcf 100644 --- a/Makefile +++ b/Makefile @@ -130,9 +130,9 @@ check-black: check-isort: . $(VENV)/bin/activate; \ - $(PYTHON) -m isort --check -w $(MAX_LINE_LENGTH) merlin; \ - $(PYTHON) -m isort --check -w $(MAX_LINE_LENGTH) tests; \ - $(PYTHON) -m isort --check -w $(MAX_LINE_LENGTH) *.py; \ + $(PYTHON) -m isort --check --line-length $(MAX_LINE_LENGTH) $(MRLN); \ + $(PYTHON) -m isort --check --line-length $(MAX_LINE_LENGTH) $(TEST); \ + $(PYTHON) -m isort --check --line-length $(MAX_LINE_LENGTH) *.py; \ check-pylint: diff --git a/docs/source/conf.py b/docs/source/conf.py index 4f0004dc2..315978a6a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,7 +16,7 @@ import os import sys -sys.path.insert(0, os.path.abspath('../..')) +sys.path.insert(0, os.path.abspath("../..")) MERLIN_VERSION = __import__("merlin").VERSION @@ -24,9 +24,9 @@ _year = date.today().year -project = u'Merlin' -copyright = '{}, LLNL: LLNL-CODE-797170'.format(_year) -author = u'Lawrence Livermore National Laboratory' +project = "Merlin" +copyright = "{}, LLNL: LLNL-CODE-797170".format(_year) +author = "Lawrence Livermore National Laboratory" # The short X.Y version version = MERLIN_VERSION @@ -47,19 +47,19 @@ # 'sphinx.ext.autodoc', # 'sphinx.ext.intersphinx', # ] -extensions = ['sphinx.ext.autodoc'] +extensions = ["sphinx.ext.autodoc"] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -74,7 +74,7 @@ exclude_patterns = [] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # -- Options for HTML output ------------------------------------------------- @@ -82,7 +82,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -99,9 +99,9 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] -html_css_files = ['custom.css'] +html_css_files = ["custom.css"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -117,7 +117,7 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'Merlindoc' +htmlhelp_basename = "Merlindoc" # -- Options for LaTeX output ------------------------------------------------ @@ -126,15 +126,12 @@ # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -144,8 +141,7 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'Merlin.tex', u'Merlin Documentation', - u'The Merlin Development Team', 'manual'), + (master_doc, "Merlin.tex", "Merlin Documentation", "The Merlin Development Team", "manual"), ] @@ -153,10 +149,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'merlin', u'Merlin Documentation', - [author], 1) -] +man_pages = [(master_doc, "merlin", "Merlin Documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -165,19 +158,17 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'Merlin', u'Merlin Documentation', - author, 'Merlin', 'One line description of project.', - 'Miscellaneous'), + (master_doc, "Merlin", "Merlin Documentation", author, "Merlin", "One line description of project.", "Miscellaneous"), ] # -- Extension configuration ------------------------------------------------- -primary_domain = 'py' +primary_domain = "py" -highlight_language = 'bash' +highlight_language = "bash" -intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} +intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} def setup(app): diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index a9c6093ca..b961afe50 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -120,7 +120,6 @@ def _make_banner(): - name_lines = merlin_name_small.split("\n") hat_lines = merlin_hat_small.split("\n") diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 6049bb5b5..4ec7c5dbe 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -141,7 +141,6 @@ def merlin_step(self, *args: Any, **kwargs: Any) -> Optional[ReturnCode]: # noq elif result == ReturnCode.SOFT_FAIL: LOG.warning(f"*** Step '{step_name}' in '{step_dir}' soft failed. Continuing with workflow.") elif result == ReturnCode.HARD_FAIL: - # stop all workers attached to this queue step_queue = step.get_task_queue() LOG.error(f"*** Step '{step_name}' in '{step_dir}' hard failed. Quitting workflow.") @@ -260,7 +259,6 @@ def add_merlin_expanded_chain_to_chord( ] LOG.debug(f"recursing grandparent with relative paths {relative_paths}") for step in chain_: - # Make a list of new task objects with modified cmd and workspace # based off of the parameter substitutions and relative_path for # a given sample. @@ -325,7 +323,6 @@ def add_simple_chain_to_chord(self, task_type, chain_, adapter_config): LOG.debug(f"simple chain with {chain_}") all_chains = [] for step in chain_: - # Make a list of new task signatures with modified cmd and workspace # based off of the parameter substitutions and relative_path for # a given sample. diff --git a/merlin/examples/workflows/flux/flux_par_native_test.yaml b/merlin/examples/workflows/flux/flux_par_native_test.yaml new file mode 100644 index 000000000..4edb76905 --- /dev/null +++ b/merlin/examples/workflows/flux/flux_par_native_test.yaml @@ -0,0 +1,76 @@ +description: + description: A simple ensemble of parallel MPI jobs run by flux. + name: flux_par + +batch: + type: flux + flux_exec: flux exec -r "0-1" + flux_start_opts: -o,-S,log-filename=flux_par.out + nodes: 1 + +env: + variables: + OUTPUT_PATH: ./studies + N_SAMPLES: 10 + +study: +- description: Build the code + name: build + run: + cmd: mpicc -o mpi_hello $(SPECROOT)/scripts/hello.c >& build.out + task_queue: flux_par +- description: Echo the params + name: runs + run: + cmd: | + if [ ! -z ${FLUX_PMI_LIBRARY_PATH+x} ]; then + FPMI2LIB=`dirname ${FLUX_PMI_LIBRARY_PATH}`/libpmi2.so + if [ -e ${FPMI2LIB} ]; then + if [ ! -z ${LD_PRELOAD+x} ]; then + export LD_PRELOAD=${LD_PRELOAD}:${FPMI2LIB} + else + export LD_PRELOAD=${FPMI2LIB} + fi + fi + fi + $(LAUNCHER) $(build.workspace)/mpi_hello $(V1) $(V2) > flux_run.out + depends: [build] + task_queue: flux_par + nodes: 1 + procs: 4 + cores per task: 1 + +- description: Dump flux info + name: data + run: + cmd: | + $(SPECROOT)/scripts/flux_info.py > flux_timings.out + depends: [runs_*] + task_queue: flux_par + +- description: Stop workers + name: stop_workers + run: + cmd: | + exit $(MERLIN_STOP_WORKERS) + depends: [data] + task_queue: flux_par + +global.parameters: + STUDY: + label: STUDY.%% + values: + - FLUXTEST + +merlin: + resources: + task_server: celery + workers: + simworkers: + args: -l INFO --concurrency 1 --prefetch-multiplier 1 -Ofair + steps: [runs, data] + samples: + column_labels: [V1, V2] + file: $(MERLIN_INFO)/samples.npy + generate: + cmd: python3 $(SPECROOT)/scripts/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy diff --git a/merlin/examples/workflows/flux/scripts/flux_test/flux b/merlin/examples/workflows/flux/scripts/flux_test/flux new file mode 100755 index 000000000..f907abfc8 --- /dev/null +++ b/merlin/examples/workflows/flux/scripts/flux_test/flux @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +print("Nodes") diff --git a/merlin/examples/workflows/flux/scripts/pbs_test/qsub b/merlin/examples/workflows/flux/scripts/pbs_test/qsub new file mode 100755 index 000000000..b12de9c32 --- /dev/null +++ b/merlin/examples/workflows/flux/scripts/pbs_test/qsub @@ -0,0 +1,2 @@ +#!/bin/sh +echo "pbs_version = 19.0.0" diff --git a/merlin/examples/workflows/optimization/scripts/visualizer.py b/merlin/examples/workflows/optimization/scripts/visualizer.py index c373c9cd1..2d700efda 100644 --- a/merlin/examples/workflows/optimization/scripts/visualizer.py +++ b/merlin/examples/workflows/optimization/scripts/visualizer.py @@ -1,16 +1,15 @@ import argparse - -import matplotlib - - -matplotlib.use("pdf") import ast import pickle +import matplotlib import matplotlib.pyplot as plt import numpy as np +matplotlib.use("pdf") + + plt.style.use("seaborn-white") diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index ea5c8f1a8..298a360b7 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -39,12 +39,6 @@ import yaml - -try: - import importlib.resources as resources -except ImportError: - import importlib_resources as resources - from merlin.server.server_util import ( CONTAINER_TYPES, MERLIN_CONFIG_DIR, @@ -57,6 +51,12 @@ ) +try: + import importlib.resources as resources +except ImportError: + import importlib_resources as resources + + LOG = logging.getLogger("merlin") # Default values for configuration diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 71e89bf84..af6ed22b5 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -37,6 +37,7 @@ """ import logging import os +import subprocess from typing import Dict, Optional, Union from merlin.utils import get_yaml_var @@ -64,24 +65,118 @@ def batch_check_parallel(spec): return parallel +def check_for_flux(): + """ + Check if FLUX is the main scheduler for the cluster + """ + try: + p = subprocess.Popen( + ["flux", "resource", "info"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + result = p.stdout.readlines() + if result and len(result) > 0 and b"Nodes" in result[0]: + return True + else: + return False + except FileNotFoundError: + return False + + +def check_for_slurm(): + """ + Check if SLURM is the main scheduler for the cluster + """ + try: + p = subprocess.Popen( + ["sbatch", "--help"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + result = p.stdout.readlines() + if result and len(result) > 0 and b"sbatch" in result[0]: + return True + else: + return False + except FileNotFoundError: + return False + + +def check_for_lsf(): + """ + Check if LSF is the main scheduler for the cluster + """ + try: + p = subprocess.Popen( + ["jsrun", "--help"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + result = p.stdout.readlines() + if result and len(result) > 0 and b"jsrun" in result[0]: + return True + else: + return False + except FileNotFoundError: + return False + + +def check_for_pbs(): + """ + Check if PBS is the main scheduler for the cluster + """ + try: + p = subprocess.Popen( + ["qsub", "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + result = p.stdout.readlines() + if result and len(result) > 0 and b"pbs_version" in result[0]: + return True + else: + return False + except FileNotFoundError: + return False + + def get_batch_type(default=None): """ Determine which batch scheduler to use. :param default: (str) The default batch scheduler to use if a scheduler - can't be determined. The default is slurm. - :returns: (str) The batch name (available options: slurm, flux, lsf). + can't be determined. The default is None. + :returns: (str) The batch name (available options: slurm, flux, lsf, pbs). """ - if default is None: - default = "slurm" + # Flux should be checked first due to slurm emulation scripts + LOG.debug(f"check for flux = {check_for_flux()}") + if check_for_flux(): + return "flux" + + # PBS should be checked before slurm for testing + LOG.debug(f"check for pbs = {check_for_pbs()}") + if check_for_pbs(): + return "pbs" + + # LSF should be checked before slurm for testing + LOG.debug(f"check for lsf = {check_for_lsf()}") + if check_for_lsf(): + return "lsf" - if "SYS_TYPE" not in os.environ: - return default + LOG.debug(f"check for slurm = {check_for_slurm()}") + if check_for_slurm(): + return "slurm" - if "toss3" in os.environ["SYS_TYPE"]: + SYS_TYPE = os.environ.get("SYS_TYPE", "") + if "toss_3" in SYS_TYPE: return "slurm" - if "blueos" in os.environ["SYS_TYPE"]: + if "blueos" in SYS_TYPE: return "lsf" return default @@ -160,29 +255,37 @@ def batch_worker_launch( if not launch_command: launch_command = construct_worker_launch_command(batch, btype, nodes) - launch_command += f" {launch_args}" + if launch_args: + launch_command += f" {launch_args}" # Allow for any pre launch manipulation, e.g. module load # hwloc/1.11.10-cuda if launch_pre: launch_command = f"{launch_pre} {launch_command}" + LOG.debug(f"launch_command = {launch_command}") + worker_cmd: str = "" if btype == "flux": flux_path: str = get_yaml_var(batch, "flux_path", "") + if "/" in flux_path: + flux_path += "/" + + flux_exe: str = os.path.join(flux_path, "flux") + flux_opts: Union[str, Dict] = get_yaml_var(batch, "flux_start_opts", "") + flux_exec_workers: Union[str, Dict, bool] = get_yaml_var(batch, "flux_exec_workers", True) + default_flux_exec = "flux exec" if launch_command else f"{flux_exe} exec" flux_exec: str = "" if flux_exec_workers: - flux_exec = get_yaml_var(batch, "flux_exec", "flux exec") - - if "/" in flux_path: - flux_path += "/" - - flux_exe: str = os.path.join(flux_path, "flux") + flux_exec = get_yaml_var(batch, "flux_exec", default_flux_exec) - launch: str = f"{launch_command} {flux_exe} start {flux_opts} {flux_exec} `which {shell}` -c" + if launch_command and "flux" not in launch_command: + launch: str = f"{launch_command} {flux_exe} start {flux_opts} {flux_exec} `which {shell}` -c" + else: + launch: str = f"{launch_command} {flux_exec} `which {shell}` -c" worker_cmd = f'{launch} "{com}"' else: worker_cmd = f"{launch_command} {com}" @@ -203,6 +306,10 @@ def construct_worker_launch_command(batch: Optional[Dict], btype: str, nodes: in bank: str = get_yaml_var(batch, "bank", "") queue: str = get_yaml_var(batch, "queue", "") walltime: str = get_yaml_var(batch, "walltime", "") + + if btype == "pbs" and workload_manager == btype: + raise Exception("The PBS scheduler is only enabled for 'batch: flux' type") + if btype == "slurm" or workload_manager == "slurm": launch_command = f"srun -N {nodes} -n {nodes}" if bank: @@ -211,8 +318,34 @@ def construct_worker_launch_command(batch: Optional[Dict], btype: str, nodes: in launch_command += f" -p {queue}" if walltime: launch_command += f" -t {walltime}" + if workload_manager == "lsf": # The jsrun utility does not have a time argument launch_command = f"jsrun -a 1 -c ALL_CPUS -g ALL_GPUS --bind=none -n {nodes}" + if workload_manager == "flux": + flux_path: str = get_yaml_var(batch, "flux_path", "") + if "/" in flux_path: + flux_path += "/" + + flux_exe: str = os.path.join(flux_path, "flux") + launch_command = f"{flux_exe} mini alloc -o pty -N {nodes} --exclusive --job-name=merlin" + if bank: + launch_command += f" --setattr=system.bank={bank}" + if queue: + launch_command += f" --setattr=system.queue={queue}" + if walltime: + launch_command += f" -t {walltime}" + + if workload_manager == "pbs": + launch_command = f"qsub -l nodes={nodes}" + # launch_command = f"qsub -l nodes={nodes} -l procs={nodes}" + if bank: + launch_command += f" -A {bank}" + if queue: + launch_command += f" -q {queue}" + # if walltime: + # launch_command += f" -l walltime={walltime}" + launch_command += " --" # To read from stdin + return launch_command diff --git a/merlin/study/dag.py b/merlin/study/dag.py index fbd7dd6d7..37a429055 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -202,15 +202,11 @@ def find_independent_chains(self, list_of_groups_of_chains): for group in list_of_groups_of_chains: for chain in group: for task_name in chain: - if self.num_children(task_name) == 1 and task_name != "_source": - child = self.children(task_name)[0] if self.num_parents(child) == 1: - if self.compatible_merlin_expansion(child, task_name): - self.find_chain(child, list_of_groups_of_chains).remove(child) chain.append(child) diff --git a/merlin/study/study.py b/merlin/study/study.py index 9d36e4173..dd108005a 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -113,13 +113,16 @@ def __init__( ), "MERLIN_SAMPLE_NAMES": " ".join(self.get_sample_labels(from_spec=self.original_spec)), "MERLIN_SPEC_ORIGINAL_TEMPLATE": os.path.join( - self.info, self.original_spec.description["name"].replace(" ", "_") + ".orig.yaml" + self.info, + self.original_spec.description["name"].replace(" ", "_") + ".orig.yaml", ), "MERLIN_SPEC_EXECUTED_RUN": os.path.join( - self.info, self.original_spec.description["name"].replace(" ", "_") + ".partial.yaml" + self.info, + self.original_spec.description["name"].replace(" ", "_") + ".partial.yaml", ), "MERLIN_SPEC_ARCHIVED_COPY": os.path.join( - self.info, self.original_spec.description["name"].replace(" ", "_") + ".expanded.yaml" + self.info, + self.original_spec.description["name"].replace(" ", "_") + ".expanded.yaml", ), } diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index d23cbd57e..feb5168ba 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -22,7 +22,9 @@ def define_tests(): is the test's name, and the value is a tuple of (shell command, condition(s) to satisfy). """ - celery_regex = r"(srun\s+.*)?celery\s+(-A|--app)\s+merlin\s+worker\s+.*" + celery_slurm_regex = r"(srun\s+.*)?celery\s+(-A|--app)\s+merlin\s+worker\s+.*" + celery_flux_regex = r"(flux mini alloc\s+.*)?celery\s+(-A|--app)\s+merlin\s+worker\s+.*" + celery_pbs_regex = r"(qsub\s+.*)?celery\s+(-A|--app)\s+merlin\s+worker\s+.*" # shortcut string variables err_lvl = "-lvl error" @@ -40,6 +42,11 @@ def define_tests(): slurm_restart = f"{examples}/slurm/slurm_par_restart.yaml" flux = f"{examples}/flux/flux_test.yaml" flux_restart = f"{examples}/flux/flux_par_restart.yaml" + flux_native = f"{examples}/flux/flux_par_native_test.yaml" + flux_native_path = f"{examples}/flux/scripts/flux_test" + workers_flux = f"""PATH="{flux_native_path}:$PATH";merlin {err_lvl} run-workers""" + pbs_path = f"{examples}/flux/scripts/pbs_test" + workers_pbs = f"""PATH="{pbs_path}:$PATH";merlin {err_lvl} run-workers""" lsf = f"{examples}/lsf/lsf_par.yaml" black = "black --check --target-version py36" config_dir = "./CLI_TEST_MERLIN_CONFIG" @@ -140,22 +147,32 @@ def define_tests(): run_workers_echo_tests = { "run-workers echo simple_chain": ( f"{workers} {simple} --echo", - [HasReturnCode(), HasRegex(celery_regex)], + [HasReturnCode(), HasRegex(celery_slurm_regex)], "local", ), "run-workers echo feature_demo": ( f"{workers} {demo} --echo", - [HasReturnCode(), HasRegex(celery_regex)], + [HasReturnCode(), HasRegex(celery_slurm_regex)], "local", ), "run-workers echo slurm_test": ( f"{workers} {slurm} --echo", - [HasReturnCode(), HasRegex(celery_regex)], + [HasReturnCode(), HasRegex(celery_slurm_regex)], "local", ), "run-workers echo flux_test": ( f"{workers} {flux} --echo", - [HasReturnCode(), HasRegex(celery_regex)], + [HasReturnCode(), HasRegex(celery_slurm_regex)], + "local", + ), + "run-workers echo flux_native_test": ( + f"{workers_flux} {flux_native} --echo", + [HasReturnCode(), HasRegex(celery_flux_regex)], + "local", + ), + "run-workers echo pbs_test": ( + f"{workers_pbs} {flux_native} --echo", + [HasReturnCode(), HasRegex(celery_pbs_regex)], "local", ), "run-workers echo override feature_demo": ( From 37661756d084169d3b9c947f72918bd916ff5180 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 15 Mar 2023 14:56:12 -0700 Subject: [PATCH 070/126] Update integration test suite and add stop-workers tests Update integration test suite and add stop-workers tests Update integration test suite and add stop-workers tests modify changelog to show test changes update copyright year and CI explanation --- .github/workflows/push-pr_workflow.yml | 8 +- .gitignore | 3 + CHANGELOG.md | 12 + LICENSE | 2 +- Makefile | 25 +- config.mk | 2 + merlin/__init__.py | 2 +- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- .../security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 98 ++- merlin/examples/__init__.py | 2 +- .../dev_workflows/multiple_workers.yaml | 56 ++ merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/README.md | 163 ++++- tests/integration/conditions.py | 83 ++- tests/integration/run_tests.py | 200 +++++-- tests/integration/test_definitions.py | 563 +++++++++++------- 61 files changed, 942 insertions(+), 371 deletions(-) create mode 100644 merlin/examples/dev_workflows/multiple_workers.yaml diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 4d7a51ccb..e2d9e164e 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -131,7 +131,7 @@ jobs: run: | python3 -m pytest tests/unit/ - - name: Run integration test suite + - name: Run integration test suite for local tests run: | python3 tests/integration/run_tests.py --verbose --local @@ -179,7 +179,7 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt - - name: Install merlin to run unit tests + - name: Install merlin and setup redis as the broker run: | pip3 install -e . merlin config --broker redis @@ -189,12 +189,12 @@ jobs: merlin example feature_demo pip3 install -r feature_demo/requirements.txt - - name: Run integration test suite for Redis + - name: Run integration test suite for distributed tests env: REDIS_HOST: redis REDIS_PORT: 6379 run: | - python3 tests/integration/run_tests.py --verbose --ids 31 32 + python3 tests/integration/run_tests.py --verbose --distributed # - name: Setup rabbitmq config # run: | diff --git a/.gitignore b/.gitignore index e427f8ad4..8b0fb8ad2 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,9 @@ ARCHIVE_DIR *_OUTPUT/ *_OUTPUT_D/ *_ensemble_*/ +studies/ +appendonlydir/ +cli_test_studies/ # Scheduler logs flux.out diff --git a/CHANGELOG.md b/CHANGELOG.md index 502d48379..3aa643c8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,15 +8,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Pip wheel wasn't including .sh files for merlin examples - The learn.py script in the openfoam_wf* examples will now create the missing Energy v Lidspeed plot + ### Added - Now loads np.arrays of dtype='object', allowing mix-type sample npy - Added a singularity container openfoam_wf example - Added flux native worker launch support - Added PBS flux launch support - Added check_for_flux, check_for_slurm, check_for_lsf, and check_for_pbs utility functions +- Tests for the `stop-workers` command +- A function in `run_tests.py` to check that an integration test definition is formatted correctly +- A new dev_workflow example `multiple_workers.yaml` that's used for testing the `stop-workers` command +- Ability to start 2 subprocesses for a single test +- Added the --distributed and --display-table flags to run_tests.py + - --distributed: only run distributed tests + - --display-tests: displays a table of all existing tests and the id associated with each test ### Changed - Changed celery_regex to celery_slurm_regex in test_definitions.py +- Reformatted how integration tests are defined and part of how they run + - Test values are now dictionaries rather than tuples + - Stopped using `subprocess.Popen()` and `subprocess.communicate()` to run tests and now instead use `subprocess.run()` for simplicity and to keep things up-to-date with the latest subprocess release (`run()` will call `Popen()` and `communicate()` under the hood so we don't have to handle that anymore) +- Rewrote the README in the integration tests folder to explain the new integration test format ## [1.9.1] ### Fixed diff --git a/LICENSE b/LICENSE index 746aac58b..3adc85cb9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 Lawrence Livermore National Laboratory +Copyright (c) 2023 Lawrence Livermore National Laboratory Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index fd6949bcf..897b44a31 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # @@ -38,6 +38,8 @@ include config.mk .PHONY : e2e-tests-diagnostic .PHONY : e2e-tests-local .PHONY : e2e-tests-local-diagnostic +.PHONY : e2e-tests-distributed +.PHONY : e2e-tests-distributed-diagnostic .PHONY : tests .PHONY : check-flake8 .PHONY : check-black @@ -109,6 +111,16 @@ e2e-tests-local-diagnostic: $(PYTHON) $(TEST)/integration/run_tests.py --local --verbose +e2e-tests-distributed: + . $(VENV)/bin/activate; \ + $(PYTHON) $(TEST)/integration/run_tests.py --distributed; \ + + +e2e-tests-distributed-diagnostic: + . $(VENV)/bin/activate; \ + $(PYTHON) $(TEST)/integration/run_tests.py --distributed --verbose + + # run unit and CLI tests tests: unit-tests e2e-tests @@ -185,6 +197,17 @@ version: find tests/ -type f -print0 | xargs -0 sed -i 's/Version: $(VSTRING)/Version: $(VER)/g' find Makefile -type f -print0 | xargs -0 sed -i 's/Version: $(VSTRING)/Version: $(VER)/g' +# Increment copyright year +year: +# do LICENSE (no comma after year) + sed -i 's/$(YEAR) Lawrence Livermore/$(NEW_YEAR) Lawrence Livermore/g' LICENSE + +# do all file headers (works on linux) + find merlin/ -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g' + find *.py -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g' + find tests/ -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g' + find Makefile -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g' + # Make a list of all dependencies/requirements reqlist: johnnydep merlin --output-format pinned diff --git a/config.mk b/config.mk index 38a87a9bf..f1cfbcea3 100644 --- a/config.mk +++ b/config.mk @@ -19,6 +19,8 @@ endif VER?=1.0.0 VSTRING=[0-9]\+\.[0-9]\+\.[0-9]\+ +YEAR=20[0-9][0-9] +NEW_YEAR?=2023 CHANGELOG_VSTRING="## \[$(VSTRING)\]" INIT_VSTRING="__version__ = \"$(VSTRING)\"" diff --git a/merlin/__init__.py b/merlin/__init__.py index eaf3ff668..1222a38f6 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index b961afe50..a521d2cc5 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/celery.py b/merlin/celery.py index 0041523b2..52d4e4589 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index 30fea7f47..f242b15fb 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index da214275b..5e82abbc2 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -1,7 +1,7 @@ #!/usr/bin/env python ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 274ddb062..c2391ca6d 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -1,7 +1,7 @@ #!/usr/bin/env python ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 591f9a49f..dd93cf5a7 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 4861a757b..45ddb2ed1 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 229f00851..6a0766427 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 1ba552626..930c67b5f 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 4ec7c5dbe..b0eca1b6c 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 352da5550..bd19ac539 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index b460be069..08a0362ae 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index e30a18ca0..ea26edc8f 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index 7822e7086..38fba1ddc 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -3,7 +3,7 @@ """ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index bd22ddaa3..bb7f79875 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index e435e1ba0..40ea19af7 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 626b776d5..3cce32883 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/display.py b/merlin/display.py index 30140981e..0e0e11e66 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # @@ -45,7 +45,27 @@ from merlin.config.configfile import default_config_info +# TODO: make these color blind compliant +# (see https://mikemol.github.io/technique/colorblind/2018/02/11/color-safe-palette.html) +ANSI_COLORS = { + "RESET": "\033[0m", + "GREY": "\033[90m", + "RED": "\033[91m", + "GREEN": "\033[92m", + "YELLOW": "\033[93m", + "BLUE": "\033[94m", + "MAGENTA": "\033[95m", + "CYAN": "\033[96m", + "WHITE": "\033[97m", +} + + class ConnProcess(Process): + """ + An extension of Multiprocessing's Process class in order + to overwrite the run and exception defintions. + """ + def __init__(self, *args, **kwargs): Process.__init__(self, *args, **kwargs) self._pconn, self._cconn = Pipe() @@ -55,19 +75,24 @@ def run(self): try: Process.run(self) self._cconn.send(None) - except Exception as e: - tb = traceback.format_exc() - self._cconn.send((e, tb)) + except Exception as e: # pylint: disable=W0718,C0103 + trace_back = traceback.format_exc() + self._cconn.send((e, trace_back)) # raise e # You can still rise this exception if you need to @property def exception(self): + """Create custom exception""" if self._pconn.poll(): self._exception = self._pconn.recv() return self._exception def check_server_access(sconf): + """ + Check if there are any issues connecting to the servers. + If there are, output the errors. + """ servers = ["broker server", "results server"] if sconf.keys(): @@ -75,25 +100,25 @@ def check_server_access(sconf): print("-" * 28) excpts = {} - for s in servers: - if s in sconf: - _examine_connection(s, sconf, excpts) + for server in servers: + if server in sconf: + _examine_connection(server, sconf, excpts) if excpts: print("\nExceptions:") - for k, v in excpts.items(): - print(f"{k}: {v}") + for key, val in excpts.items(): + print(f"{key}: {val}") -def _examine_connection(s, sconf, excpts): +def _examine_connection(server, sconf, excpts): connect_timeout = 60 try: ssl_conf = None - if "broker" in s: + if "broker" in server: ssl_conf = broker.get_ssl_config() - if "results" in s: + if "results" in server: ssl_conf = results_backend.get_ssl_config() - conn = Connection(sconf[s], ssl=ssl_conf) + conn = Connection(sconf[server], ssl=ssl_conf) conn_check = ConnProcess(target=conn.connect) conn_check.start() counter = 0 @@ -102,16 +127,16 @@ def _examine_connection(s, sconf, excpts): counter += 1 if counter > connect_timeout: conn_check.kill() - raise Exception(f"Connection was killed due to timeout ({connect_timeout}s)") + raise TimeoutError(f"Connection was killed due to timeout ({connect_timeout}server)") conn.release() if conn_check.exception: - error, traceback = conn_check.exception + error, _ = conn_check.exception raise error - except Exception as e: - print(f"{s} connection: Error") - excpts[s] = e + except Exception as e: # pylint: disable=W0718,C0103 + print(f"{server} connection: Error") + excpts[server] = e else: - print(f"{s} connection: OK") + print(f"{server} connection: OK") def display_config_info(): @@ -129,7 +154,7 @@ def display_config_info(): conf["broker server"] = broker.get_connection_string(include_password=False) sconf["broker server"] = broker.get_connection_string() conf["broker ssl"] = broker.get_ssl_config() - except Exception as e: + except Exception as e: # pylint: disable=W0718,C0103 conf["broker server"] = "Broker server error." excpts["broker server"] = e @@ -137,7 +162,7 @@ def display_config_info(): conf["results server"] = results_backend.get_connection_string(include_password=False) sconf["results server"] = results_backend.get_connection_string() conf["results ssl"] = results_backend.get_ssl_config() - except Exception as e: + except Exception as e: # pylint: disable=W0718,C0103 conf["results server"] = "No results server configured or error." excpts["results server"] = e @@ -145,8 +170,8 @@ def display_config_info(): if excpts: print("\nExceptions:") - for k, v in excpts.items(): - print(f"{k}: {v}") + for key, val in excpts.items(): + print(f"{key}: {val}") check_server_access(sconf) @@ -170,7 +195,7 @@ def display_multiple_configs(files, configs): pprint.pprint(config) -def print_info(args): +def print_info(args): # pylint: disable=W0613 """ Provide version and location information about python and pip to facilitate user troubleshooting. 'merlin info' is a CLI tool only for @@ -187,9 +212,30 @@ def print_info(args): print("") info_calls = ["which python3", "python3 --version", "which pip3", "pip3 --version"] info_str = "" - for x in info_calls: - info_str += 'echo " $ ' + x + '" && ' + x + "\n" + for cmd in info_calls: + info_str += 'echo " $ ' + cmd + '" && ' + cmd + "\n" info_str += "echo \n" info_str += r"echo \"echo \$PYTHONPATH\" && echo $PYTHONPATH" _ = subprocess.run(info_str, shell=True) print("") + + +def tabulate_info(info, headers=None, color=None): + """ + Display info in a table. Colorize the table if you'd like. + Intended for use for functions outside of this file so they don't + need to import tabulate. + :param `info`: The info you want to tabulate. + :param `headers`: A string or list stating what you'd like the headers to be. + Options: "firstrow", "keys", or List[str] + :param `color`: An ANSI color. + """ + # Adds the color at the start of the print + if color: + print(color, end="") + + # \033[0m resets color to white + if headers: + print(tabulate(info, headers=headers), ANSI_COLORS["RESET"]) + else: + print(tabulate(info), ANSI_COLORS["RESET"]) diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/examples/dev_workflows/multiple_workers.yaml b/merlin/examples/dev_workflows/multiple_workers.yaml new file mode 100644 index 000000000..f393f87d3 --- /dev/null +++ b/merlin/examples/dev_workflows/multiple_workers.yaml @@ -0,0 +1,56 @@ +description: + name: multiple_workers + description: a very simple merlin workflow with multiple workers + +global.parameters: + GREET: + values : ["hello","hola"] + label : GREET.%% + WORLD: + values : ["world","mundo"] + label : WORLD.%% + +study: + - name: step_1 + description: say hello + run: + cmd: | + echo "$(GREET), $(WORLD)!" + task_queue: hello_queue + + - name: step_2 + description: step 2 + run: + cmd: | + echo "step_2" + depends: [step_1_*] + task_queue: echo_queue + + - name: step_3 + description: stop workers + run: + cmd: | + echo "stop workers" + depends: [step_2] + task_queue: other_queue + + - name: step_4 + description: another step + run: + cmd: | + echo "another step" + depends: [step_3] + task_queue: other_queue + +merlin: + resources: + workers: + step_1_merlin_test_worker: + args: -l INFO + steps: [step_1] + step_2_merlin_test_worker: + args: -l INFO + steps: [step_2] + other_merlin_test_worker: + args: -l INFO + steps: [step_3, step_4] \ No newline at end of file diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 43573b416..7fbd502b1 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 2796465ab..c97cc2757 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 816546f67..225e69d28 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index f540e6f1f..475982d8c 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -1,7 +1,7 @@ """This module handles setting up the extensive logging system in Merlin.""" ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/main.py b/merlin/main.py index 5de8423b0..47f471dd8 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -1,7 +1,7 @@ """The top level main function for invoking Merlin.""" ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index a1e4de372..a1af0298b 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/router.py b/merlin/router.py index 90711756c..3ec9122ff 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 31115004f..5653cba21 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 38359938c..045ef22e3 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -1,7 +1,7 @@ """Main functions for instantiating and running Merlin server containers.""" ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 298a360b7..e62e12e4f 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 16d2c3686..65f0b2abb 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 46db0fd4f..ac0031823 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 46f71882f..7b54e5200 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 710bc2500..f4ea42e24 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 5ca2eea6b..4c9291693 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 9d93197bf..287fab1f6 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index af6ed22b5..9a42873f1 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 97fc40289..f6d344a25 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 37a429055..f7ba3532f 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index ab998140a..027ebaff9 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 3a55df606..031302480 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/study.py b/merlin/study/study.py index dd108005a..6bf07653f 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/utils.py b/merlin/utils.py index 5217e1d7a..48def3a10 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/setup.py b/setup.py index 06706ca57..1a1ad94ba 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/tests/integration/README.md b/tests/integration/README.md index ef402c9e5..1d575e022 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,21 +1,160 @@ -# Integration test script: run_tests.py +# Integration Tests -To run command line-level tests of Merlin, follow these steps: +This directory contains 3 key files for testing: +1. `run_tests.py` - script to launch tests +2. `test_definitions.py` - test definitions +3. `conditions.py` - test conditions + +## How to Run + +To run command-line-level tests of Merlin, follow these steps: 1. activate the Merlin virtual environment 2. navigate to the top-level `merlin` directory 3. run `python tests/integration/run_tests.py` -This will run all tests found in the `define_tests` function. -A test is a python dict where the key is the test name, and the -value is a tuple holding the test shell command, and a regexp string -to search for, if any. Without a regexp string, the script will -output 'FAIL' on non-zero return codes. With a regexp string, the -script outputs 'FAIL' if the string cannot be found in the -test's stdout. +This will run all tests found in the `define_tests` function located within `test_definitions.py`. + +## How Tests are Defined + +A test is a python dict where the key is the test name and the +value is another dict. The value dict can currently have 5 keys: + +Required: + +1. `cmds` + - Type: Str or List[Str] + - Functionality: Defines the CLI commands to run for a test + - Limitations: The number of strings here should be equal to `num procs` (see `5.num procs` below) +2. `conditions` + - Type: Condition or List[Condition] + - Functionality: Defines the conditions to check against for this test + - Condition classes can be found in `conditions.py` + +Optional: + +3. `run type` + - Type: Str + - Functionality: Defines the type of run (either `local` or `distributed`) + - Default: None +4. `cleanup` + - Type: Str + - Functionality: Defines a CLI command to run that will clean the output of your test + - Default: None +5. `num procs` + - Type: int + - Functionality: Defines the number of subprocesses required for a test + - Default: 1 + - Limitations: + - Currently the value here can only be 1 or 2 + - The number of `cmds` must be equal to `num procs` (i.e. one command will be run per subprocess launched) + +## Examples + +This section will show both valid and invalid test definitions. + +### Valid Test Definitions + +The most basic test you can provide can be written 4 ways since `cmds` and `conditions` can both be 2 different types: + + "cmds as string, conditions as Condition": { + "cmds": "echo hello", + "conditions": HasReturnCode(), + } + + "cmds as list, conditions as Condition": { + "cmds": ["echo hello"], + "conditions": HasReturnCode(), + } + + "cmds as string, conditions as list": { + "cmds": "echo hello", + "conditions": [HasReturnCode()], + } + + "cmds as list, conditions as list": { + "cmds": ["echo hello"], + "conditions": [HasReturnCode()], + } + +Adding slightly more complexity, we provide a run type: + + "basic test with run type": { + "cmds": "echo hello", + "conditions": HasReturnCode(), + "run type": "local" # This could also be "distributed" + } + +Now we'll add a cleanup command: + + "basic test with cleanup": { + "cmds": "mkdir output_dir/", + "conditions": HasReturnCode(), + "run type": "local", + "cleanup": "rm -rf output_dir/" + } + +Finally we'll play with the number of processes to start: + + "test with 1 process": { + "cmds": "mkdir output_dir/", + "conditions": HasReturnCode(), + "run type": "local", + "cleanup": "rm -rf output_dir/", + "num procs": 1 + } + + "test with 2 processes": { + "cmds": ["mkdir output_dir/", "touch output_dir/new_file.txt"], + "conditions": HasReturnCode(), + "run type": "local", + "cleanup": "rm -rf output_dir/", + "num procs": 2 + } + +Similarly, the test with 2 processes can be condensed to a test with 1 process +by placing them in the same string and separating them with a semi-colon: + + "condensing test with 2 processes into 1 process": { + "cmds": ["mkdir output_dir/ ; touch output_dir/new_file.txt"], + "conditions": HasReturnCode(), + "run type": "local", + "cleanup": "rm -rf output_dir/", + "num procs": 1 + } + + +### Invalid Test Definitions + +No `cmds` provided: + + "no cmd": { + "conditions": HasReturnCode(), + } + +No `conditions` provided: + + "no conditions": { + "cmds": "echo hello", + } + +Number of `cmds` does not match `num procs`: + + "num cmds != num procs": { + "cmds": ["echo hello; echo goodbye"], + "conditions": HasReturnCode(), + "num procs": 2 + } + +Note: Technically 2 commands were provided here ("echo hello" and "echo goodbye") +but since they were placed in one string it will be viewed as one command. +Changing the `cmds` section here to be: + + "cmds": ["echo hello", "echo goodbye"] + +would fix this issue and create a valid test definition. -### Continuous integration -Currently run from [Bamboo](https://lc.llnl.gov/bamboo/chain/admin/config/defaultStages.action?buildKey=MLSI-TES). +## Continuous Integration -Our Bamboo agents make the virtual environment, staying at the `merlin/` location, then run: `python tests/integration/run_tests.py`. +Merlin's CI is currently done through [GitHub Actions](https://github.com/features/actions). If you're needing to modify this CI, you'll need to update `/.github/workflows/push-pr_workflow.yml`. diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 3448ec61a..9c44e1f5f 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -1,3 +1,33 @@ +############################################################################### +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.1. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### +"""This module defines the different conditions to test against.""" import os from abc import ABC, abstractmethod from glob import glob @@ -6,6 +36,8 @@ # TODO when moving command line tests to pytest, change Condition boolean returns to assertions class Condition(ABC): + """Abstract Condition class that other conditions will inherit from""" + def ingest_info(self, info): """ This function allows child classes of Condition @@ -14,11 +46,14 @@ def ingest_info(self, info): for key, val in info.items(): setattr(self, key, val) + @property @abstractmethod def passes(self): + """The method that will check if the test passes or not""" pass +# pylint: disable=no-member class HasReturnCode(Condition): """ A condition that some process must return 0 @@ -80,7 +115,7 @@ def is_within(self, text): @property def passes(self): if self.negate: - return not self.is_within(self.stdout) + return not self.is_within(self.stdout) and not self.is_within(self.stderr) return self.is_within(self.stdout) or self.is_within(self.stderr) @@ -99,6 +134,9 @@ def __init__(self, study_name, output_path): self.dirpath_glob = f"{self.output_path}/{self.study_name}" f"_[0-9]*-[0-9]*" def glob(self, glob_string): + """ + Returns a regex string for the glob library to recursively find files with. + """ candidates = glob(glob_string) if isinstance(candidates, list): return sorted(candidates)[-1] @@ -110,7 +148,7 @@ class StepFileExists(StudyOutputAware): A StudyOutputAware that checks for a particular file's existence. """ - def __init__(self, step, filename, study_name, output_path, params=False): + def __init__(self, step, filename, study_name, output_path, params=False): # pylint: disable=R0913 """ :param `step`: the name of a step :param `filename`: name of file to search for in step's workspace directory @@ -127,12 +165,16 @@ def __str__(self): @property def glob_string(self): + """ + Returns a regex string for the glob library to recursively find files with. + """ param_glob = "" if self.params: param_glob = "*/" return f"{self.dirpath_glob}/{self.step}/{param_glob}{self.filename}" def file_exists(self): + """Check if the file path created by glob_string exists""" glob_string = self.glob_string try: filename = self.glob(glob_string) @@ -150,7 +192,7 @@ class StepFileHasRegex(StudyOutputAware): A StudyOutputAware that checks that a particular file contains a regex. """ - def __init__(self, step, filename, study_name, output_path, regex): + def __init__(self, step, filename, study_name, output_path, regex): # pylint: disable=R0913 """ :param `step`: the name of a step :param `filename`: name of file to search for in step's workspace directory @@ -163,20 +205,25 @@ def __init__(self, step, filename, study_name, output_path, regex): self.regex = regex def __str__(self): - return f"{__class__.__name__} expected to find '{self.regex}' regex match in file '{self.glob_string}', but match was not found" + return f"""{__class__.__name__} expected to find '{self.regex}' + regex match in file '{self.glob_string}', but match was not found""" @property def glob_string(self): + """ + Returns a regex string for the glob library to recursively find files with. + """ return f"{self.dirpath_glob}/{self.step}/{self.filename}" def contains(self): + """See if the regex is within the filetext""" glob_string = self.glob_string try: filename = self.glob(glob_string) with open(filename, "r") as textfile: filetext = textfile.read() return self.is_within(filetext) - except Exception: + except Exception: # pylint: disable=W0718 return False def is_within(self, text): @@ -196,7 +243,7 @@ class ProvenanceYAMLFileHasRegex(HasRegex): MUST contain a given regular expression. """ - def __init__(self, regex, name, output_path, provenance_type, negate=False): + def __init__(self, regex, name, output_path, provenance_type, negate=False): # pylint: disable=R0913 """ :param `regex`: a string regex pattern :param `name`: the name of a study @@ -214,14 +261,19 @@ def __init__(self, regex, name, output_path, provenance_type, negate=False): def __str__(self): if self.negate: - return f"{__class__.__name__} expected to find no '{self.regex}' regex match in provenance spec '{self.glob_string}', but match was found" - return f"{__class__.__name__} expected to find '{self.regex}' regex match in provenance spec '{self.glob_string}', but match was not found" + return f"""{__class__.__name__} expected to find no '{self.regex}' + regex match in provenance spec '{self.glob_string}', but match was found""" + return f"""{__class__.__name__} expected to find '{self.regex}' + regex match in provenance spec '{self.glob_string}', but match was not found""" @property def glob_string(self): + """ + Returns a regex string for the glob library to recursively find files with. + """ return f"{self.output_path}/{self.name}" f"_[0-9]*-[0-9]*/merlin_info/{self.name}.{self.prov_type}.yaml" - def is_within(self): + def is_within(self): # pylint: disable=W0221 """ Uses glob to find the correct provenance yaml spec. Returns True if that file contains a match to this @@ -249,6 +301,7 @@ def __init__(self, pathname) -> None: self.pathname = pathname def path_exists(self) -> bool: + """Check if a path exists""" return os.path.exists(self.pathname) def __str__(self) -> str: @@ -270,18 +323,21 @@ def __init__(self, filename, regex) -> None: self.regex = regex def contains(self) -> bool: + """Checks if the regex matches anywhere in the filetext""" try: - with open(self.filename, "r") as f: + with open(self.filename, "r") as f: # pylint: disable=C0103 filetext = f.read() return self.is_within(filetext) - except Exception: + except Exception: # pylint: disable=W0718 return False def is_within(self, text): + """Check if there's a match for the regex in text""" return search(self.regex, text) is not None def __str__(self) -> str: - return f"{__class__.__name__} expected to find {self.regex} regex match within {self.filename} file but no match was found" + return f"""{__class__.__name__} expected to find {self.regex} + regex match within {self.filename} file but no match was found""" @property def passes(self): @@ -295,7 +351,8 @@ class FileHasNoRegex(FileHasRegex): """ def __str__(self) -> str: - return f"{__class__.__name__} expected to find {self.regex} regex to not match within {self.filename} file but a match was found" + return f"""{__class__.__name__} expected to find {self.regex} + regex to not match within {self.filename} file but a match was found""" @property def passes(self): diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 913a2ab08..0282f888b 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # @@ -37,36 +37,89 @@ import sys import time from contextlib import suppress -from subprocess import PIPE, Popen +from subprocess import TimeoutExpired, run -from test_definitions import OUTPUT_DIR, define_tests +from test_definitions import OUTPUT_DIR, define_tests # pylint: disable=E0401 -def run_single_test(name, test, test_label="", buffer_length=50): - dot_length = buffer_length - len(name) - len(str(test_label)) - print(f"TEST {test_label}: {name}{'.'*dot_length}", end="") - command = test[0] - conditions = test[1] +def get_definition_issues(test): + """ + Function to make sure the test definition was written properly. + :param `test`: The test definition we're checking + :returns: A list of errors found with the test definition + """ + errors = [] + # Check that commands were provided + try: + commands = test["cmds"] + if not isinstance(commands, list): + commands = [commands] + except KeyError: + errors.append("'cmds' flag not defined") + commands = None + + # Check that conditions were provided + if "conditions" not in test: + errors.append("'conditions' flag not defined") + + # Check that correct number of cmds were given depending on + # the number of processes we'll need to start + if commands: + if "num procs" not in test: + num_procs = 1 + else: + num_procs = test["num procs"] + + if num_procs == 1 and len(commands) != 1: + errors.append(f"Need 1 'cmds' since 'num procs' is 1 but {len(commands)} 'cmds' were given") + elif num_procs == 2 and len(commands) != 2: + errors.append(f"Need 2 'cmds' since 'num procs' is 2 but {len(commands)} 'cmds' were given") + + return errors + + +def run_single_test(test): + """ + Runs a single test and returns whether it passed or not + and information about the test for logging purposes. + :param `test`: A dictionary that defines the test + :returns: A tuple of type (bool, dict) where the bool + represents if the test passed and the dict + contains info about the test. + """ + # Parse the test definition + commands = test.pop("cmds", None) + if not isinstance(commands, list): + commands = [commands] + conditions = test.pop("conditions", None) if not isinstance(conditions, list): conditions = [conditions] + cleanup = test.pop("cleanup", None) + num_procs = test.pop("num procs", 1) start_time = time.time() - process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) - stdout, stderr = process.communicate() + # As of now the only time we need 2 processes is to test stop-workers + # Therefore we only care about the result of the second process + if num_procs == 2: + # First command should start the workers + try: + run(commands[0], timeout=8, capture_output=True, shell=True) + except TimeoutExpired: + pass + # Second command should stop the workers + result = run(commands[1], capture_output=True, text=True, shell=True) + else: + # Run the commands + result = run(commands[0], capture_output=True, text=True, shell=True) end_time = time.time() total_time = end_time - start_time - if stdout is not None: - stdout = stdout.decode("utf-8") - if stderr is not None: - stderr = stderr.decode("utf-8") - return_code = process.returncode info = { "total_time": total_time, - "command": command, - "stdout": stdout, - "stderr": stderr, - "return_code": return_code, + "command": commands, + "stdout": result.stdout, + "stderr": result.stderr, + "return_code": result.returncode, "violated_condition": None, } @@ -78,11 +131,10 @@ def run_single_test(name, test, test_label="", buffer_length=50): info["violated_condition"] = (condition, i, len(conditions)) break - if len(test) == 4: - end_process = Popen(test[3], stdout=PIPE, stderr=PIPE, shell=True) - end_stdout, end_stderr = end_process.communicate() - info["end_stdout"] = end_stdout - info["end_stderr"] = end_stderr + if cleanup: + end_process = run(cleanup, capture_output=True, text=True, shell=True) + info["end_stdout"] = end_process.stdout + info["end_stderr"] = end_process.stderr return passed, info @@ -96,7 +148,7 @@ def clear_test_studies_dir(): shutil.rmtree(f"./{OUTPUT_DIR}") -def process_test_result(passed, info, is_verbose, exit): +def process_test_result(passed, info, is_verbose, exit_on_failure): """ Process and print test results to the console. """ @@ -104,9 +156,9 @@ def process_test_result(passed, info, is_verbose, exit): if passed is False and "merlin: command not found" in info["stderr"]: print(f"\nMissing from environment:\n\t{info['stderr']}") return None - elif passed is False: + if passed is False: print("FAIL") - if exit is True: + if exit_on_failure is True: return None else: print("pass") @@ -127,11 +179,18 @@ def process_test_result(passed, info, is_verbose, exit): return passed -def run_tests(args, tests): +def filter_tests_to_run(args, tests): """ - Run all inputted tests. - :param `tests`: a dictionary of - {"test_name" : ("test_command", [conditions])} + Filter which tests to run based on args. The tests to + run will be what makes up the args.ids list. This function + will return whether we're being selective with what tests + we run and also the number of tests that match the filter. + :param `args`: CLI args given by user + :param `tests`: a dict of all the tests that exist + :returns: a tuple where the first entry is a bool on whether + we filtered the tests at all and the second entry + is an int representing the number of tests we're + going to run. """ selective = False n_to_run = len(tests) @@ -140,19 +199,27 @@ def run_tests(args, tests): raise ValueError(f"Test ids must be between 1 and {len(tests)}, inclusive.") selective = True n_to_run = len(args.ids) - elif args.local is not None: + elif args.local is not None or args.distributed is not None: args.ids = [] n_to_run = 0 selective = True for test_id, test in enumerate(tests.values()): - # Ensures that test definitions are atleast size 3. - # 'local' variable is stored in 3rd element of the test definitions, - # but an optional 4th element can be provided for an ending command - # to be ran after all checks have been made. - if len(test) >= 3 and test[2] == "local": + run_type = test.pop("run type", None) + if (args.local and run_type == "local") or (args.distributed and run_type == "distributed"): args.ids.append(test_id + 1) n_to_run += 1 + return selective, n_to_run + + +def run_tests(args, tests): # pylint: disable=R0914 + """ + Run all inputted tests. + :param `tests`: a dictionary of + {"test_name" : ("test_command", [conditions])} + """ + selective, n_to_run = filter_tests_to_run(args, tests) + print(f"Running {n_to_run} integration tests...") start_time = time.time() @@ -163,14 +230,32 @@ def run_tests(args, tests): if selective and test_label not in args.ids: total += 1 continue - try: - passed, info = run_single_test(test_name, test, test_label) - except Exception as e: - print(e) + dot_length = 50 - len(test_name) - len(str(test_label)) + print(f"TEST {test_label}: {test_name}{'.'*dot_length}", end="") + # Check the format of the test definition + definition_issues = get_definition_issues(test) + if definition_issues: + print("FAIL") + print(f"\tTest with name '{test_name}' has problems with its' test definition. Skipping...") + if args.verbose: + print(f"\tFound {len(definition_issues)} problems with the definition of '{test_name}':") + for error in definition_issues: + print(f"\t- {error}") + total += 1 passed = False - info = None + if args.exit: + result = None + else: + result = False + else: + try: + passed, info = run_single_test(test) + except Exception as e: # pylint: disable=C0103,W0718 + print(e) + passed = False + info = None + result = process_test_result(passed, info, args.verbose, args.exit) - result = process_test_result(passed, info, args.verbose, args.exit) clear_test_studies_dir() if result is None: print("Exiting early") @@ -190,6 +275,10 @@ def run_tests(args, tests): def setup_argparse(): + """ + Using ArgumentParser, define the arguments allowed for this script. + :returns: An ArgumentParser object + """ parser = argparse.ArgumentParser(description="run_tests cli parser") parser.add_argument( "--exit", @@ -198,6 +287,7 @@ def setup_argparse(): ) parser.add_argument("--verbose", action="store_true", help="Flag for more detailed output messages") parser.add_argument("--local", action="store_true", default=None, help="Run only local tests") + parser.add_argument("--distributed", action="store_true", default=None, help="Run only distributed tests") parser.add_argument( "--ids", action="store", @@ -207,9 +297,29 @@ def setup_argparse(): default=None, help="Provide space-delimited ids of tests you want to run. Example: '--ids 1 5 8 13'", ) + parser.add_argument( + "--display-tests", action="store_true", default=False, help="Display a table format of test names and ids" + ) return parser +def display_tests(tests): + """ + Helper function to display a table of tests and associated ids. + Helps choose which test to run if you're trying to debug and use + the --id flag. + :param `tests`: A dict of tests (Dict) + """ + from merlin.display import tabulate_info # pylint: disable=C0415 + + test_names = list(tests.keys()) + test_table = [(i + 1, test_names[i]) for i in range(len(test_names))] + test_table.insert(0, ("ID", "Test Name")) + print() + tabulate_info(test_table, headers="firstrow") + print() + + def main(): """ High-level CLI test operations. @@ -219,6 +329,10 @@ def main(): tests = define_tests() + if args.display_tests: + display_tests(tests) + return + clear_test_studies_dir() result = run_tests(args, tests) sys.exit(result) diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index feb5168ba..752b3650d 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -1,4 +1,46 @@ -from conditions import ( +############################################################################### +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.1. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### +""" +This module defines all the integration tests to be ran through run_tests.py. + +Each test looks like: +"test name": { + "cmds": the commands to run, + "conditions": the conditions to check for, + "run type": the type of test (local or distributed), + "cleanup": the command to run after your test in order to clean output, + "num procs": the number of processes you need to start for a test (1 or 2) +} +""" + +from conditions import ( # pylint: disable=E0401 FileHasNoRegex, FileHasRegex, HasRegex, @@ -14,9 +56,11 @@ OUTPUT_DIR = "cli_test_studies" CLEAN_MERLIN_SERVER = "rm -rf appendonly.aof dump.rdb merlin_server/" +# KILL_WORKERS = "pkill -9 -f '.*merlin_test_worker'" +KILL_WORKERS = "pkill -9 -f 'celery'" -def define_tests(): +def define_tests(): # pylint: disable=R0914 """ Returns a dictionary of tests, where the key is the test's name, and the value is a tuple @@ -48,62 +92,64 @@ def define_tests(): pbs_path = f"{examples}/flux/scripts/pbs_test" workers_pbs = f"""PATH="{pbs_path}:$PATH";merlin {err_lvl} run-workers""" lsf = f"{examples}/lsf/lsf_par.yaml" + mul_workers_demo = f"{dev_examples}/multiple_workers.yaml" black = "black --check --target-version py36" config_dir = "./CLI_TEST_MERLIN_CONFIG" release_dependencies = "./requirements/release.txt" basic_checks = { - "merlin": ("merlin", HasReturnCode(1), "local"), - "merlin help": ("merlin --help", HasReturnCode(), "local"), - "merlin version": ("merlin --version", HasReturnCode(), "local"), - "merlin config": ( - f"merlin config -o {config_dir}; rm -rf {config_dir}", - HasReturnCode(), - "local", - ), + "merlin": {"cmds": "merlin", "conditions": HasReturnCode(1), "run type": "local"}, + "merlin help": {"cmds": "merlin --help", "conditions": HasReturnCode(), "run type": "local"}, + "merlin version": {"cmds": "merlin --version", "conditions": HasReturnCode(), "run type": "local"}, + "merlin config": { + "cmds": f"merlin config -o {config_dir}", + "conditions": HasReturnCode(), + "run type": "local", + "cleanup": f"rm -rf {config_dir}", + }, } server_basic_tests = { - "merlin server init": ( - "merlin server init", - HasRegex(".*successful"), - "local", - CLEAN_MERLIN_SERVER, - ), - "merlin server start/stop": ( - """merlin server init; - merlin server start; - merlin server status; - merlin server stop;""", - [ + "merlin server init": { + "cmds": "merlin server init", + "conditions": HasRegex(".*successful"), + "run type": "local", + "cleanup": CLEAN_MERLIN_SERVER, + }, + "merlin server start/stop": { + "cmds": """merlin server init ; + merlin server start ; + merlin server status ; + merlin server stop""", + "conditions": [ HasRegex("Server started with PID [0-9]*"), HasRegex("Merlin server is running"), HasRegex("Merlin server terminated"), ], - "local", - CLEAN_MERLIN_SERVER, - ), - "merlin server restart": ( - """merlin server init; + "run type": "local", + "cleanup": CLEAN_MERLIN_SERVER, + }, + "merlin server restart": { + "cmds": """merlin server init; merlin server start; merlin server restart; merlin server status; merlin server stop;""", - [ + "conditions": [ HasRegex("Server started with PID [0-9]*"), HasRegex("Merlin server is running"), HasRegex("Merlin server terminated"), ], - "local", - CLEAN_MERLIN_SERVER, - ), + "run type": "local", + "cleanup": CLEAN_MERLIN_SERVER, + }, } server_config_tests = { - "merlin server change config": ( - """merlin server init; + "merlin server change config": { + "cmds": """merlin server init; merlin server config -p 8888 -pwd new_password -d ./config_dir -ss 80 -sc 8 -sf new_sf -am always -af new_af.aof; merlin server start; merlin server stop;""", - [ + "conditions": [ FileHasRegex("merlin_server/redis.conf", "port 8888"), FileHasRegex("merlin_server/redis.conf", "requirepass new_password"), FileHasRegex("merlin_server/redis.conf", "dir ./config_dir"), @@ -116,11 +162,11 @@ def define_tests(): HasRegex("Server started with PID [0-9]*"), HasRegex("Merlin server terminated"), ], - "local", - "rm -rf appendonly.aof dump.rdb merlin_server/ config_dir/", - ), - "merlin server config add/remove user": ( - """merlin server init; + "run type": "local", + "cleanup": "rm -rf appendonly.aof dump.rdb merlin_server/ config_dir/", + }, + "merlin server config add/remove user": { + "cmds": """merlin server init; merlin server start; merlin server config --add-user new_user new_password; merlin server stop; @@ -129,129 +175,132 @@ def define_tests(): merlin server config --remove-user new_user; merlin server stop; """, - [ + "conditions": [ FileHasRegex("./merlin_server/redis.users_new", "new_user"), FileHasNoRegex("./merlin_server/redis.users", "new_user"), ], - "local", - CLEAN_MERLIN_SERVER, - ), + "run type": "local", + "cleanup": CLEAN_MERLIN_SERVER, + }, } examples_check = { - "example list": ( - "merlin example list", - HasReturnCode(), - "local", - ), + "example list": { + "cmds": "merlin example list", + "conditions": HasReturnCode(), + "run type": "local", + }, } run_workers_echo_tests = { - "run-workers echo simple_chain": ( - f"{workers} {simple} --echo", - [HasReturnCode(), HasRegex(celery_slurm_regex)], - "local", - ), - "run-workers echo feature_demo": ( - f"{workers} {demo} --echo", - [HasReturnCode(), HasRegex(celery_slurm_regex)], - "local", - ), - "run-workers echo slurm_test": ( - f"{workers} {slurm} --echo", - [HasReturnCode(), HasRegex(celery_slurm_regex)], - "local", - ), - "run-workers echo flux_test": ( - f"{workers} {flux} --echo", - [HasReturnCode(), HasRegex(celery_slurm_regex)], - "local", - ), - "run-workers echo flux_native_test": ( - f"{workers_flux} {flux_native} --echo", - [HasReturnCode(), HasRegex(celery_flux_regex)], - "local", - ), - "run-workers echo pbs_test": ( - f"{workers_pbs} {flux_native} --echo", - [HasReturnCode(), HasRegex(celery_pbs_regex)], - "local", - ), - "run-workers echo override feature_demo": ( - f"{workers} {demo} --echo --vars VERIFY_QUEUE=custom_verify_queue", - [HasReturnCode(), HasRegex("custom_verify_queue")], - "local", - ), + "run-workers echo simple_chain": { + "cmds": f"{workers} {simple} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_slurm_regex)], + "run type": "local", + }, + "run-workers echo feature_demo": { + "cmds": f"{workers} {demo} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_slurm_regex)], + "run type": "local", + }, + "run-workers echo slurm_test": { + "cmds": f"{workers} {slurm} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_slurm_regex)], + "run type": "local", + }, + "run-workers echo flux_test": { + "cmds": f"{workers} {flux} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_slurm_regex)], + "run type": "local", + }, + "run-workers echo flux_native_test": { + "cmds": f"{workers_flux} {flux_native} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_flux_regex)], + "run type": "local", + }, + "run-workers echo pbs_test": { + "cmds": f"{workers_pbs} {flux_native} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_pbs_regex)], + "run type": "local", + }, + "run-workers echo override feature_demo": { + "cmds": f"{workers} {demo} --echo --vars VERIFY_QUEUE=custom_verify_queue", + "conditions": [HasReturnCode(), HasRegex("custom_verify_queue")], + "run type": "local", + }, } wf_format_tests = { - "local minimum_format": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../{dev_examples}/minimum_format.yaml --local", - StepFileExists( + "local minimum_format": { + "cmds": f"mkdir {OUTPUT_DIR}; cd {OUTPUT_DIR}; merlin run ../{dev_examples}/minimum_format.yaml --local", + "conditions": StepFileExists( "step1", "MERLIN_FINISHED", "minimum_format", OUTPUT_DIR, params=False, ), - "local", - ), - "local no_description": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../merlin/examples/dev_workflows/no_description.yaml --local", - HasReturnCode(1), - "local", - ), - "local no_steps": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../merlin/examples/dev_workflows/no_steps.yaml --local", - HasReturnCode(1), - "local", - ), - "local no_study": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../merlin/examples/dev_workflows/no_study.yaml --local", - HasReturnCode(1), - "local", - ), + "run type": "local", + }, + "local no_description": { + "cmds": f"""mkdir {OUTPUT_DIR}; cd {OUTPUT_DIR}; + merlin run ../merlin/examples/dev_workflows/no_description.yaml --local""", + "conditions": HasReturnCode(1), + "run type": "local", + }, + "local no_steps": { + "cmds": f"mkdir {OUTPUT_DIR}; cd {OUTPUT_DIR}; merlin run ../merlin/examples/dev_workflows/no_steps.yaml --local", + "conditions": HasReturnCode(1), + "run type": "local", + }, + "local no_study": { + "cmds": f"mkdir {OUTPUT_DIR}; cd {OUTPUT_DIR}; merlin run ../merlin/examples/dev_workflows/no_study.yaml --local", + "conditions": HasReturnCode(1), + "run type": "local", + }, } example_tests = { - "example failure": ("merlin example failure", HasRegex("not found"), "local"), - "example simple_chain": ( - f"merlin example simple_chain ; {run} simple_chain.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR} ; rm simple_chain.yaml", - HasReturnCode(), - "local", - ), + "example failure": {"cmds": "merlin example failure", "conditions": HasRegex("not found"), "run type": "local"}, + "example simple_chain": { + "cmds": f"""merlin example simple_chain; + {run} simple_chain.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}; rm simple_chain.yaml""", + "conditions": HasReturnCode(), + "run type": "local", + }, } restart_step_tests = { - "local restart_shell": ( - f"{run} {dev_examples}/restart_shell.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileExists( + "local restart_shell": { + "cmds": f"{run} {dev_examples}/restart_shell.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileExists( "step2", "MERLIN_FINISHED", "restart_shell", OUTPUT_DIR, params=False, ), - "local", - ), - "local restart": ( - f"{run} {dev_examples}/restart.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileExists( + "run type": "local", + }, + "local restart": { + "cmds": f"{run} {dev_examples}/restart.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileExists( "final_check_for_no_hard_fails", "MERLIN_FINISHED", "restart", OUTPUT_DIR, params=False, ), - "local", - ), + "run type": "local", + }, } restart_wf_tests = { - "restart local simple_chain": ( - f"{run} {simple} --local --vars OUTPUT_PATH=./{OUTPUT_DIR} ; {restart} $(find ./{OUTPUT_DIR} -type d -name 'simple_chain_*') --local", - HasReturnCode(), - "local", - ), + "restart local simple_chain": { + "cmds": f"""{run} {simple} --local --vars OUTPUT_PATH=./{OUTPUT_DIR}; + {restart} $(find ./{OUTPUT_DIR} -type d -name 'simple_chain_*') --local""", + "conditions": HasReturnCode(), + "run type": "local", + }, } dry_run_tests = { - "dry feature_demo": ( - f"{run} {demo} --local --dry --vars OUTPUT_PATH=./{OUTPUT_DIR}", - [ + "dry feature_demo": { + "cmds": f"{run} {demo} --local --dry --vars OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": [ StepFileExists( "verify", "verify_*.sh", @@ -261,61 +310,61 @@ def define_tests(): ), HasReturnCode(), ], - "local", - ), - "dry launch slurm": ( - f"{run} {slurm} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex("runs", "*/runs.slurm.sh", "slurm_test", OUTPUT_DIR, "srun "), - "local", - ), - "dry launch flux": ( - f"{run} {flux} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex( + "run type": "local", + }, + "dry launch slurm": { + "cmds": f"{run} {slurm} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileHasRegex("runs", "*/runs.slurm.sh", "slurm_test", OUTPUT_DIR, "srun "), + "run type": "local", + }, + "dry launch flux": { + "cmds": f"{run} {flux} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileHasRegex( "runs", "*/runs.slurm.sh", "flux_test", OUTPUT_DIR, get_flux_cmd("flux", no_errors=True), ), - "local", - ), - "dry launch lsf": ( - f"{run} {lsf} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex("runs", "*/runs.slurm.sh", "lsf_par", OUTPUT_DIR, "jsrun "), - "local", - ), - "dry launch slurm restart": ( - f"{run} {slurm_restart} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex( + "run type": "local", + }, + "dry launch lsf": { + "cmds": f"{run} {lsf} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileHasRegex("runs", "*/runs.slurm.sh", "lsf_par", OUTPUT_DIR, "jsrun "), + "run type": "local", + }, + "dry launch slurm restart": { + "cmds": f"{run} {slurm_restart} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileHasRegex( "runs", "*/runs.restart.slurm.sh", "slurm_par_restart", OUTPUT_DIR, "srun ", ), - "local", - ), - "dry launch flux restart": ( - f"{run} {flux_restart} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex( + "run type": "local", + }, + "dry launch flux restart": { + "cmds": f"{run} {flux_restart} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileHasRegex( "runs_rs", "*/runs_rs.restart.slurm.sh", "flux_par_restart", OUTPUT_DIR, get_flux_cmd("flux", no_errors=True), ), - "local", - ), + "run type": "local", + }, } other_local_tests = { - "local simple_chain": ( - f"{run} {simple} --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", - HasReturnCode(), - "local", - ), - "local override feature_demo": ( - f"{run} {demo} --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR} --local", - [ + "local simple_chain": { + "cmds": f"{run} {simple} --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": HasReturnCode(), + "run type": "local", + }, + "local override feature_demo": { + "cmds": f"{run} {demo} --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR} --local", + "conditions": [ HasReturnCode(), ProvenanceYAMLFileHasRegex( regex=r"HELLO: \$\(SCRIPTS\)/hello_world.py", @@ -356,10 +405,11 @@ def define_tests(): params=True, ), ], - "local", - ), + "run type": "local", + }, # "local restart expand name": ( - # f"{run} {demo} --local --vars OUTPUT_PATH=./{OUTPUT_DIR} NAME=test_demo ; {restart} $(find ./{OUTPUT_DIR} -type d -name 'test_demo_*') --local", + # f"""{run} {demo} --local --vars OUTPUT_PATH=./{OUTPUT_DIR} NAME=test_demo; + # {restart} $(find ./{OUTPUT_DIR} -type d -name 'test_demo_*') --local""", # [ # HasReturnCode(), # ProvenanceYAMLFileHasRegex( @@ -374,19 +424,23 @@ def define_tests(): # ], # "local", # ), - "local csv feature_demo": ( - f"echo 42.0,47.0 > foo_testing_temp.csv; merlin run {demo} --samplesfile foo_testing_temp.csv --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; rm -f foo_testing_temp.csv", - [HasRegex("1 sample loaded."), HasReturnCode()], - "local", - ), - "local tab feature_demo": ( - f"echo '42.0\t47.0\n7.0 5.3' > foo_testing_temp.tab; merlin run {demo} --samplesfile foo_testing_temp.tab --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; rm -f foo_testing_temp.tab", - [HasRegex("2 samples loaded."), HasReturnCode()], - "local", - ), - "local pgen feature_demo": ( - f"{run} {demo} --pgen {demo_pgen} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local", - [ + "local csv feature_demo": { + "cmds": f"""echo 42.0,47.0 > foo_testing_temp.csv; + merlin run {demo} --samplesfile foo_testing_temp.csv --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; + rm -f foo_testing_temp.csv""", + "conditions": [HasRegex("1 sample loaded."), HasReturnCode()], + "run type": "local", + }, + "local tab feature_demo": { + "cmds": f"""echo '42.0\t47.0\n7.0 5.3' > foo_testing_temp.tab; + merlin run {demo} --samplesfile foo_testing_temp.tab --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; + rm -f foo_testing_temp.tab""", + "conditions": [HasRegex("2 samples loaded."), HasReturnCode()], + "run type": "local", + }, + "local pgen feature_demo": { + "cmds": f"{run} {demo} --pgen {demo_pgen} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local", + "conditions": [ ProvenanceYAMLFileHasRegex( regex=r"\[0.3333333", name="feature_demo", @@ -402,35 +456,118 @@ def define_tests(): ), HasReturnCode(), ], - "local", - ), + "run type": "local", + }, } - provenence_equality_checks = { # noqa: F841 - "local provenance spec equality": ( - f"{run} {simple} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local ; cp $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml') ./{OUTPUT_DIR}/FILE1 ; rm -rf ./{OUTPUT_DIR}/simple_chain_* ; {run} ./{OUTPUT_DIR}/FILE1 --vars OUTPUT_PATH=./{OUTPUT_DIR} --local ; cmp ./{OUTPUT_DIR}/FILE1 $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml')", - HasReturnCode(), - "local", - ), + provenence_equality_checks = { # noqa: F841 pylint: disable=W0612 + "local provenance spec equality": { + "cmds": f"""{run} {simple} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; + cp $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml') ./{OUTPUT_DIR}/FILE1; + rm -rf ./{OUTPUT_DIR}/simple_chain_*; + {run} ./{OUTPUT_DIR}/FILE1 --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; + cmp ./{OUTPUT_DIR}/FILE1 $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml')""", # pylint: disable=C0301 + "conditions": HasReturnCode(), + "run type": "local", + }, } - style_checks = { # noqa: F841 - "black check merlin": (f"{black} merlin/", HasReturnCode(), "local"), - "black check tests": (f"{black} tests/", HasReturnCode(), "local"), + style_checks = { # noqa: F841 pylint: disable=W0612 + "black check merlin": {"cmds": f"{black} merlin/", "conditions": HasReturnCode(), "run type": "local"}, + "black check tests": {"cmds": f"{black} tests/", "conditions": HasReturnCode(), "run type": "local"}, } dependency_checks = { - "deplic no GNU": ( - f"deplic {release_dependencies}", - [HasRegex("GNU", negate=True), HasRegex("GPL", negate=True)], - "local", - ), + "deplic no GNU": { + "cmds": f"deplic {release_dependencies}", + "conditions": [HasRegex("GNU", negate=True), HasRegex("GPL", negate=True)], + "run type": "local", + }, + } + stop_workers_tests = { + "stop workers no workers": { + "cmds": "merlin stop-workers", + "conditions": [ + HasReturnCode(), + HasRegex("No workers found to stop"), + HasRegex("step_1_merlin_test_worker", negate=True), + HasRegex("step_2_merlin_test_worker", negate=True), + HasRegex("other_merlin_test_worker", negate=True), + ], + "run type": "distributed", + }, + "stop workers no flags": { + "cmds": [ + f"{workers} {mul_workers_demo}", + "merlin stop-workers", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found to stop", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker"), + HasRegex("other_merlin_test_worker"), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, + "stop workers with spec flag": { + "cmds": [ + f"{workers} {mul_workers_demo}", + f"merlin stop-workers --spec {mul_workers_demo}", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found to stop", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker"), + HasRegex("other_merlin_test_worker"), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, + "stop workers with workers flag": { + "cmds": [ + f"{workers} {mul_workers_demo}", + "merlin stop-workers --workers step_1_merlin_test_worker step_2_merlin_test_worker", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found to stop", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker"), + HasRegex("other_merlin_test_worker", negate=True), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, + "stop workers with queues flag": { + "cmds": [ + f"{workers} {mul_workers_demo}", + "merlin stop-workers --queues hello_queue", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found to stop", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker", negate=True), + HasRegex("other_merlin_test_worker", negate=True), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, } distributed_tests = { # noqa: F841 - "run and purge feature_demo": ( - f"{run} {demo} ; {purge} {demo} -f", - HasReturnCode(), - ), - "remote feature_demo": ( - f"{run} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; {workers} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers", - [ + "run and purge feature_demo": { + "cmds": f"{run} {demo}; {purge} {demo} -f", + "conditions": HasReturnCode(), + "run type": "distributed", + }, + "remote feature_demo": { + "cmds": f"""{run} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers; + {workers} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers""", + "conditions": [ HasReturnCode(), ProvenanceYAMLFileHasRegex( regex="cli_test_demo_workers:", @@ -446,27 +583,8 @@ def define_tests(): params=True, ), ], - ), - # this test is deactivated until the --spec option for stop-workers is active again - # "stop workers for distributed feature_demo": ( - # f"{run} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; {workers} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; sleep 20 ; merlin stop-workers --spec {demo}", - # [ - # HasReturnCode(), - # ProvenanceYAMLFileHasRegex( - # regex="cli_test_demo_workers:", - # name="feature_demo", - # output_path=OUTPUT_DIR, - # provenance_type="expanded", - # ), - # StepFileExists( - # "verify", - # "MERLIN_FINISHED", - # "feature_demo", - # OUTPUT_DIR, - # params=True, - # ), - # ], - # ), + "run type": "distributed", + }, } # combine and return test dictionaries @@ -486,6 +604,7 @@ def define_tests(): # provenence_equality_checks, # omitting provenance equality check because it is broken # style_checks, # omitting style checks due to different results on different machines dependency_checks, + stop_workers_tests, distributed_tests, ]: all_tests.update(test_dict) From cea85761e597210a5301e4cb2a44161f11892eda Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 17 Mar 2023 11:22:34 -0700 Subject: [PATCH 071/126] fix bugs for stop-workers and schema validation fix bugs for stop-workers and schema validation fix all flags for stop-workers fix small bug with schema validation modify CHANGELOG run fix-style add myself to contributors list decouple celery logic from main remove unused import make changes Luc requested in PR change worker assignment to use sets --- CHANGELOG.md | 13 ++ CONTRIBUTORS | 1 + merlin/main.py | 20 +- merlin/router.py | 30 +-- merlin/spec/merlinspec.json | 5 +- merlin/spec/specification.py | 145 ++++++++++---- merlin/study/celeryadapter.py | 278 +++++++++++++++++--------- tests/integration/test_definitions.py | 3 +- 8 files changed, 342 insertions(+), 153 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3aa643c8d..5ac7348d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Pip wheel wasn't including .sh files for merlin examples - The learn.py script in the openfoam_wf* examples will now create the missing Energy v Lidspeed plot +- Fixed the flags associated with the `stop-workers` command (--spec, --queues, --workers) +- Fixed the --step flag for the `run-workers` command ### Added - Now loads np.arrays of dtype='object', allowing mix-type sample npy @@ -22,6 +24,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added the --distributed and --display-table flags to run_tests.py - --distributed: only run distributed tests - --display-tests: displays a table of all existing tests and the id associated with each test +- Added the --disable-logs flag to the `run-workers` command +- Merlin will now assign `default_worker` to any step not associated with a worker +- Added `get_step_worker_map()` as a method in `specification.py` ### Changed - Changed celery_regex to celery_slurm_regex in test_definitions.py @@ -29,6 +34,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Test values are now dictionaries rather than tuples - Stopped using `subprocess.Popen()` and `subprocess.communicate()` to run tests and now instead use `subprocess.run()` for simplicity and to keep things up-to-date with the latest subprocess release (`run()` will call `Popen()` and `communicate()` under the hood so we don't have to handle that anymore) - Rewrote the README in the integration tests folder to explain the new integration test format +- Reformatted `start_celery_workers()` in `celeryadapter.py` file. This involved: + - Modifying `verify_args()` to return the arguments it verifies/updates + - Changing `launch_celery_worker()` to launch the subprocess (no longer builds the celery command) + - Creating `get_celery_cmd()` to do what `launch_celery_worker()` used to do and build the celery command to run + - Creating `_get_steps_to_start()`, `_create_kwargs()`, and `_get_workers_to_start()` as helper functions to simplify logic in `start_celery_workers()` +- Modified the `merlinspec.json` file: + - the minimum `gpus per task` is now 0 instead of 1 + - variables defined in the `env` block of a spec file can now be arrays ## [1.9.1] ### Fixed diff --git a/CONTRIBUTORS b/CONTRIBUTORS index a920e45b7..2d4a7c9f5 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -6,3 +6,4 @@ Joe Koning Jeremy White Aidan Keogh Ryan Lee +Brian Gunnarson \ No newline at end of file diff --git a/merlin/main.py b/merlin/main.py index 47f471dd8..723fadd22 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -68,7 +68,7 @@ class HelpParser(ArgumentParser): print the help message when an error happens.""" def error(self, message): - sys.stderr.write("error: %s\n" % message) + sys.stderr.write(f"error: {message}\n") self.print_help() sys.exit(2) @@ -222,7 +222,7 @@ def launch_workers(args): spec, filepath = get_merlin_spec_with_override(args) if not args.worker_echo_only: LOG.info(f"Launching workers from '{filepath}'") - status = router.launch_workers(spec, args.worker_steps, args.worker_args, args.worker_echo_only) + status = router.launch_workers(spec, args.worker_steps, args.worker_args, args.disable_logs, args.worker_echo_only) if args.worker_echo_only: print(status) else: @@ -280,6 +280,8 @@ def stop_workers(args): """ print(banner_small) worker_names = [] + + # Load in the spec if one was provided via the CLI if args.spec: spec_path = verify_filepath(args.spec) spec = MerlinSpec.load_specification(spec_path) @@ -287,6 +289,8 @@ def stop_workers(args): for worker_name in worker_names: if "$" in worker_name: LOG.warning(f"Worker '{worker_name}' is unexpanded. Target provenance spec instead?") + + # Send stop command to router router.stop_workers(args.task_server, worker_names, args.queues, args.workers) @@ -344,6 +348,10 @@ def process_monitor(args): def process_server(args: Namespace): + """ + Route to the correct function based on the command + given via the CLI + """ if args.commands == "init": init_server() elif args.commands == "start": @@ -755,6 +763,12 @@ def generate_worker_touching_parsers(subparsers: ArgumentParser) -> None: help="Specify desired Merlin variable values to override those found in the specification. Space-delimited. " "Example: '--vars LEARN=path/to/new_learn.py EPOCHS=3'", ) + run_workers.add_argument( + "--disable-logs", + action="store_true", + help="Turn off the logs for the celery workers. Note: having the -l flag " + "in your workers' args section will overwrite this flag for that worker.", + ) # merlin query-workers query: ArgumentParser = subparsers.add_parser("query-workers", help="List connected task server workers.") @@ -787,6 +801,8 @@ def generate_worker_touching_parsers(subparsers: ArgumentParser) -> None: stop.add_argument( "--workers", type=str, + action="store", + nargs="+", default=None, help="regex match for specific workers to stop", ) diff --git a/merlin/router.py b/merlin/router.py index 3ec9122ff..ab4b8e933 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -53,7 +53,7 @@ try: - import importlib.resources as resources + from importlib import resources except ImportError: import importlib_resources as resources @@ -74,7 +74,7 @@ def run_task_server(study, run_mode=None): LOG.error("Celery is not specified as the task server!") -def launch_workers(spec, steps, worker_args="", just_return_command=False): +def launch_workers(spec, steps, worker_args="", disable_logs=False, just_return_command=False): """ Launches workers for the specified study. @@ -83,12 +83,13 @@ def launch_workers(spec, steps, worker_args="", just_return_command=False): :param `worker_args`: Optional arguments for the workers :param `just_return_command`: Don't execute, just return the command """ - if spec.merlin["resources"]["task_server"] == "celery": + if spec.merlin["resources"]["task_server"] == "celery": # pylint: disable=R1705 # Start workers - cproc = start_celery_workers(spec, steps, worker_args, just_return_command) + cproc = start_celery_workers(spec, steps, worker_args, disable_logs, just_return_command) return cproc else: LOG.error("Celery is not specified as the task server!") + return "No workers started" def purge_tasks(task_server, spec, force, steps): @@ -103,12 +104,13 @@ def purge_tasks(task_server, spec, force, steps): """ LOG.info(f"Purging queues for steps = {steps}") - if task_server == "celery": + if task_server == "celery": # pylint: disable=R1705 queues = spec.make_queue_string(steps) # Purge tasks return purge_celery_tasks(queues, force) else: LOG.error("Celery is not specified as the task server!") + return -1 def query_status(task_server, spec, steps, verbose=True): @@ -122,12 +124,13 @@ def query_status(task_server, spec, steps, verbose=True): if verbose: LOG.info(f"Querying queues for steps = {steps}") - if task_server == "celery": + if task_server == "celery": # pylint: disable=R1705 queues = spec.get_queue_list(steps) # Query the queues return query_celery_queues(queues) else: LOG.error("Celery is not specified as the task server!") + return [] def dump_status(query_return, csv_file): @@ -141,7 +144,7 @@ def dump_status(query_return, csv_file): fmode = "a" else: fmode = "w" - with open(csv_file, mode=fmode) as f: + with open(csv_file, mode=fmode) as f: # pylint: disable=W1514,C0103 if f.mode == "w": # add the header f.write("# time") for name, job, consumer in query_return: @@ -162,7 +165,7 @@ def query_workers(task_server): LOG.info("Searching for workers...") if task_server == "celery": - return query_celery_workers() + query_celery_workers() else: LOG.error("Celery is not specified as the task server!") @@ -174,10 +177,11 @@ def get_workers(task_server): :return: A list of all connected workers :rtype: list """ - if task_server == "celery": + if task_server == "celery": # pylint: disable=R1705 return get_workers_from_app() else: LOG.error("Celery is not specified as the task server!") + return [] def stop_workers(task_server, spec_worker_names, queues, workers_regex): @@ -191,14 +195,14 @@ def stop_workers(task_server, spec_worker_names, queues, workers_regex): """ LOG.info("Stopping workers...") - if task_server == "celery": + if task_server == "celery": # pylint: disable=R1705 # Stop workers - return stop_celery_workers(queues, spec_worker_names, workers_regex) + stop_celery_workers(queues, spec_worker_names, workers_regex) else: LOG.error("Celery is not specified as the task server!") -def route_for_task(name, args, kwargs, options, task=None, **kw): +def route_for_task(name, args, kwargs, options, task=None, **kw): # pylint: disable=W0613,R1710 """ Custom task router for queues """ @@ -249,7 +253,7 @@ def check_merlin_status(args, spec): total_jobs = 0 total_consumers = 0 - for name, jobs, consumers in queue_status: + for _, jobs, consumers in queue_status: total_jobs += jobs total_consumers += consumers diff --git a/merlin/spec/merlinspec.json b/merlin/spec/merlinspec.json index 47e738ee6..3044cd506 100644 --- a/merlin/spec/merlinspec.json +++ b/merlin/spec/merlinspec.json @@ -49,7 +49,7 @@ "type": {"type": "string", "minLength": 1} } }, - "gpus per task": {"type": "integer", "minimum": 1}, + "gpus per task": {"type": "integer", "minimum": 0}, "max_retries": {"type": "integer", "minimum": 1}, "task_queue": {"type": "string", "minLength": 1}, "nodes": { @@ -146,7 +146,8 @@ "^.*": { "anyOf": [ {"type": "string", "minLength": 1}, - {"type": "number"} + {"type": "number"}, + {"type": "array"} ] } } diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 287fab1f6..65326d54f 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -48,7 +48,7 @@ LOG = logging.getLogger(__name__) -class MerlinSpec(YAMLSpecification): +class MerlinSpec(YAMLSpecification): # pylint: disable=R0902 """ This class represents the logic for parsing the Merlin yaml specification. @@ -67,8 +67,8 @@ class MerlinSpec(YAMLSpecification): column_labels: [X0, X1] """ - def __init__(self): - super(MerlinSpec, self).__init__() + def __init__(self): # pylint: disable=W0246 + super(MerlinSpec, self).__init__() # pylint: disable=R1725 @property def yaml_sections(self): @@ -123,32 +123,50 @@ def __str__(self): return result @classmethod - def load_specification(cls, filepath, suppress_warning=True): + def load_specification(cls, filepath, suppress_warning=True): # pylint: disable=W0237 + """ + Load in a spec file and create a MerlinSpec object based on its' contents. + + :param `cls`: The class reference (like self) + :param `filepath`: A path to the spec file we're loading in + :param `suppress_warning`: A bool representing whether to warn the user about unrecognized keys + :returns: A MerlinSpec object + """ LOG.info("Loading specification from path: %s", filepath) try: # Load the YAML spec from the filepath with open(filepath, "r") as data: spec = cls.load_spec_from_string(data, needs_IO=False, needs_verification=True) - except Exception as e: + except Exception as e: # pylint: disable=C0103 LOG.exception(e.args) raise e # Path not set in _populate_spec because loading spec with string # does not have a path so we set it here spec.path = filepath - spec.specroot = os.path.dirname(spec.path) + spec.specroot = os.path.dirname(spec.path) # pylint: disable=W0201 if not suppress_warning: spec.warn_unrecognized_keys() return spec @classmethod - def load_spec_from_string(cls, string, needs_IO=True, needs_verification=False): + def load_spec_from_string(cls, string, needs_IO=True, needs_verification=False): # pylint: disable=C0103 + """ + Read in a spec file from a string (or stream) and create a MerlinSpec object from it. + + :param `cls`: The class reference (like self) + :param `string`: A string or stream of the file we're reading in + :param `needs_IO`: A bool representing whether we need to turn the string into a file + object or not + :param `needs_verification`: A bool representing whether we need to verify the spec + :returns: A MerlinSpec object + """ LOG.debug("Creating Merlin spec object...") # Create and populate the MerlinSpec object data = StringIO(string) if needs_IO else string spec = cls._populate_spec(data) - spec.specroot = None + spec.specroot = None # pylint: disable=W0201 spec.process_spec_defaults() LOG.debug("Merlin spec object created.") @@ -179,7 +197,7 @@ def _populate_spec(cls, data): try: spec = yaml.load(data, yaml.FullLoader) except AttributeError: - LOG.warn( + LOG.warning( "PyYAML is using an unsafe version with a known " "load vulnerability. Please upgrade your installation " "to a more recent version!" @@ -198,11 +216,11 @@ def _populate_spec(cls, data): # Reset the file pointer and load the merlin block data.seek(0) - merlin_spec.merlin = MerlinSpec.load_merlin_block(data) + merlin_spec.merlin = MerlinSpec.load_merlin_block(data) # pylint: disable=W0201 # Reset the file pointer and load the user block data.seek(0) - merlin_spec.user = MerlinSpec.load_user_block(data) + merlin_spec.user = MerlinSpec.load_user_block(data) # pylint: disable=W0201 return merlin_spec @@ -262,7 +280,7 @@ def _verify_workers(self): ) raise ValueError(error_msg) - except Exception: + except Exception: # pylint: disable=W0706 raise def verify_merlin_block(self, schema): @@ -288,7 +306,7 @@ def verify_batch_block(self, schema): YAMLSpecification.validate_schema("batch", self.batch, schema) # Additional Walltime checks in case the regex from the schema bypasses an error - if "walltime" in self.batch: + if "walltime" in self.batch: # pylint: disable=R1702 if self.batch["type"] == "lsf": LOG.warning("The walltime argument is not available in lsf.") else: @@ -299,17 +317,17 @@ def verify_batch_block(self, schema): # Walltime must have : if it's not of the form SS if ":" not in walltime: raise ValueError(err_msg) - else: - # Walltime must have exactly 2 chars between : - time = walltime.split(":") - for section in time: - if len(section) != 2: - raise ValueError(err_msg) - except Exception: + # Walltime must have exactly 2 chars between : + time = walltime.split(":") + for section in time: + if len(section) != 2: + raise ValueError(err_msg) + except Exception: # pylint: disable=W0706 raise @staticmethod def load_merlin_block(stream): + """Loads in the merlin block of the spec file""" try: merlin_block = yaml.safe_load(stream)["merlin"] except KeyError: @@ -324,6 +342,7 @@ def load_merlin_block(stream): @staticmethod def load_user_block(stream): + """Loads in the user block of the spec file""" try: user_block = yaml.safe_load(stream)["user"] except KeyError: @@ -331,6 +350,7 @@ def load_user_block(stream): return user_block def process_spec_defaults(self): + """Fills in the default values if they aren't there already""" for name, section in self.sections.items(): if section is None: setattr(self, name, {}) @@ -354,8 +374,25 @@ def process_spec_defaults(self): if self.merlin["resources"]["workers"] is None: self.merlin["resources"]["workers"] = {"default_worker": defaults.WORKER} else: - for worker, vals in self.merlin["resources"]["workers"].items(): - MerlinSpec.fill_missing_defaults(vals, defaults.WORKER) + # Gather a list of step names defined in the study + all_workflow_steps = self.get_study_step_names() + # Create a variable to track the steps assigned to workers + worker_steps = [] + + # Loop through each worker and fill in the defaults + for _, worker_settings in self.merlin["resources"]["workers"].items(): + MerlinSpec.fill_missing_defaults(worker_settings, defaults.WORKER) + worker_steps.extend(worker_settings["steps"]) + + # Figure out which steps still need workers + steps_that_need_workers = list(set(all_workflow_steps) - set(worker_steps)) + + # If there are still steps remaining that haven't been assigned a worker yet, + # assign the remaining steps to the default worker. If all the steps still need workers + # (i.e. no workers were assigned) then default workers' steps should be "all" so we skip this + if steps_that_need_workers and (steps_that_need_workers != all_workflow_steps): + self.merlin["resources"]["workers"]["default_worker"] = defaults.WORKER + self.merlin["resources"]["workers"]["default_worker"]["steps"] = steps_that_need_workers if self.merlin["samples"] is not None: MerlinSpec.fill_missing_defaults(self.merlin["samples"], defaults.SAMPLES) @@ -370,7 +407,7 @@ def fill_missing_defaults(object_to_update, default_dict): existing ones. """ - def recurse(result, defaults): + def recurse(result, defaults): # pylint: disable=W0621 if not isinstance(defaults, dict): return for key, val in defaults.items(): @@ -387,6 +424,7 @@ def recurse(result, defaults): # ***Unsure if this method is still needed after adding json schema verification*** def warn_unrecognized_keys(self): + """Checks if there are any unrecognized keys in the spec file""" # check description MerlinSpec.check_section("description", self.description, all_keys.DESCRIPTION) @@ -397,7 +435,7 @@ def warn_unrecognized_keys(self): MerlinSpec.check_section("env", self.environment, all_keys.ENV) # check parameters - for param, contents in self.globals.items(): + for _, contents in self.globals.items(): MerlinSpec.check_section("global.parameters", contents, all_keys.PARAMETER) # check steps @@ -416,13 +454,14 @@ def warn_unrecognized_keys(self): # user block is not checked @staticmethod - def check_section(section_name, section, all_keys): + def check_section(section_name, section, all_keys): # pylint: disable=W0621 + """Checks a section of the spec file to see if there are any unrecognized keys""" diff = set(section.keys()).difference(all_keys) # TODO: Maybe add a check here for required keys for extra in diff: - LOG.warn(f"Unrecognized key '{extra}' found in spec section '{section_name}'.") + LOG.warning(f"Unrecognized key '{extra}' found in spec section '{section_name}'.") def dump(self): """ @@ -434,11 +473,11 @@ def dump(self): result = result.replace("\n\n\n", "\n\n") try: yaml.safe_load(result) - except Exception as e: - raise ValueError(f"Error parsing provenance spec:\n{e}") + except Exception as e: # pylint: disable=C0103 + raise ValueError(f"Error parsing provenance spec:\n{e}") # pylint: disable=W0707 return result - def _dict_to_yaml(self, obj, string, key_stack, tab, newline=True): + def _dict_to_yaml(self, obj, string, key_stack, tab): """ The if-else ladder for sorting the yaml string prettification of dump(). """ @@ -449,12 +488,11 @@ def _dict_to_yaml(self, obj, string, key_stack, tab, newline=True): if isinstance(obj, str): return self._process_string(obj, lvl, tab) - elif isinstance(obj, bool): + if isinstance(obj, bool): return str(obj).lower() - elif not (isinstance(obj, list) or isinstance(obj, dict)): + if not isinstance(obj, (list, dict)): return obj - else: - return self._process_dict_or_list(obj, string, key_stack, lvl, tab) + return self._process_dict_or_list(obj, string, key_stack, lvl, tab) def _process_string(self, obj, lvl, tab): """ @@ -465,15 +503,15 @@ def _process_string(self, obj, lvl, tab): obj = "|\n" + tab * (lvl + 1) + ("\n" + tab * (lvl + 1)).join(split) return obj - def _process_dict_or_list(self, obj, string, key_stack, lvl, tab): + def _process_dict_or_list(self, obj, string, key_stack, lvl, tab): # pylint: disable=R0912,R0913 """ Processes lists and dicts for _dict_to_yaml() in the dump() method. """ - from copy import deepcopy + from copy import deepcopy # pylint: disable=C0415 list_offset = 2 * " " if isinstance(obj, list): - n = len(obj) + num_entries = len(obj) use_hyphens = key_stack[-1] in ["paths", "sources", "git", "study"] or key_stack[0] in ["user"] if not use_hyphens: string += "[" @@ -485,8 +523,8 @@ def _process_dict_or_list(self, obj, string, key_stack, lvl, tab): if use_hyphens: string += (lvl + 1) * tab + "- " + str(self._dict_to_yaml(elem, "", key_stack, tab)) + "\n" else: - string += str(self._dict_to_yaml(elem, "", key_stack, tab, newline=(i != 0))) - if n > 1 and i != len(obj) - 1: + string += str(self._dict_to_yaml(elem, "", key_stack, tab)) + if num_entries > 1 and i != len(obj) - 1: string += ", " key_stack.pop() if not use_hyphens: @@ -496,9 +534,9 @@ def _process_dict_or_list(self, obj, string, key_stack, lvl, tab): if len(key_stack) > 0 and key_stack[-1] != "elem": string += "\n" i = 0 - for k, v in obj.items(): + for key, val in obj.items(): key_stack = deepcopy(key_stack) - key_stack.append(k) + key_stack.append(key) if len(key_stack) > 1 and key_stack[-2] == "elem" and i == 0: # string += (tab * (lvl - 1)) string += "" @@ -506,14 +544,32 @@ def _process_dict_or_list(self, obj, string, key_stack, lvl, tab): string += list_offset + (tab * lvl) else: string += tab * (lvl + 1) - string += str(k) + ": " + str(self._dict_to_yaml(v, "", key_stack, tab)) + "\n" + string += str(key) + ": " + str(self._dict_to_yaml(val, "", key_stack, tab)) + "\n" key_stack.pop() i += 1 return string + def get_step_worker_map(self): + """ + Creates a dictionary with step names as keys and a list of workers + associated with each step as values. The inverse of get_worker_step_map(). + """ + steps = self.get_study_step_names() + step_worker_map = {step_name: [] for step_name in steps} + for worker_name, worker_val in self.merlin["resources"]["workers"].items(): + # Case 1: worker doesn't have specific steps + if "all" in worker_val["steps"]: + for step_name in step_worker_map: + step_worker_map[step_name].append(worker_name) + # Case 2: worker has specific steps + else: + for step in worker_val["steps"]: + step_worker_map[step].append(worker_name) + return step_worker_map + def get_task_queues(self): """Returns a dictionary of steps and their corresponding task queues.""" - from merlin.config.configfile import CONFIG + from merlin.config.configfile import CONFIG # pylint: disable=C0415 steps = self.get_study_steps() queues = {} @@ -540,8 +596,8 @@ def get_queue_list(self, steps): else: task_queues = [queues[steps]] except KeyError: - nl = "\n" - LOG.error(f"Invalid steps '{steps}'! Try one of these (or 'all'):\n{nl.join(queues.keys())}") + newline = "\n" + LOG.error(f"Invalid steps '{steps}'! Try one of these (or 'all'):\n{newline.join(queues.keys())}") raise return sorted(set(task_queues)) @@ -555,6 +611,7 @@ def make_queue_string(self, steps): return shlex.quote(queues) def get_worker_names(self): + """Builds a list of workers""" result = [] for worker in self.merlin["resources"]["workers"]: result.append(worker) diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index f6d344a25..81f0762f8 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -51,8 +51,8 @@ def run_celery(study, run_mode=None): configure Celery to run locally (without workers). """ # Only import celery stuff if we want celery in charge - from merlin.celery import app - from merlin.common.tasks import queue_merlin_study + from merlin.celery import app # pylint: disable=C0415 + from merlin.common.tasks import queue_merlin_study # pylint: disable=C0415 adapter_config = study.get_adapter_config(override_type="local") @@ -145,7 +145,7 @@ def query_celery_queues(queues): Send results to the log. """ - from merlin.celery import app + from merlin.celery import app # pylint: disable=C0415 connection = app.connection() found_queues = [] @@ -155,7 +155,7 @@ def query_celery_queues(queues): try: name, jobs, consumers = channel.queue_declare(queue=queue, passive=True) found_queues.append((name, jobs, consumers)) - except Exception as e: + except Exception as e: # pylint: disable=C0103,W0718 LOG.warning(f"Cannot find queue {queue} on server.{e}") finally: connection.close() @@ -169,7 +169,7 @@ def get_workers_from_app(): :return: A list of all connected workers :rtype: list """ - from merlin.celery import app + from merlin.celery import app # pylint: disable=C0415 i = app.control.inspect() workers = i.ping() @@ -178,10 +178,90 @@ def get_workers_from_app(): return [*workers] -def start_celery_workers(spec, steps, celery_args, just_return_command): +def _get_workers_to_start(spec, steps): + """ + Helper function to return a set of workers to start based on + the steps provided by the user. + + :param `spec`: A MerlinSpec object + :param `steps`: A list of steps to start workers for + + :returns: A set of workers to start + """ + workers_to_start = [] + step_worker_map = spec.get_step_worker_map() + for step in steps: + try: + workers_to_start.extend(step_worker_map[step]) + except KeyError: + LOG.warning(f"Cannot start workers for step: {step}. This step was not found.") + + workers_to_start = set(workers_to_start) + LOG.debug(f"workers_to_start: {workers_to_start}") + + return workers_to_start + + +def _create_kwargs(spec): + """ + Helper function to handle creating the kwargs dict that + we'll pass to subprocess.Popen when we launch the worker. + + :param `spec`: A MerlinSpec object + :returns: A tuple where the first entry is the kwargs and + the second entry is variables defined in the spec + """ + # Get the environment from the spec and the shell + spec_env = spec.environment + shell_env = os.environ.copy() + yaml_vars = None + + # If the environment from the spec has anything in it, + # read in the variables and save them to the shell environment + if spec_env: + yaml_vars = get_yaml_var(spec_env, "variables", {}) + for var_name, var_val in yaml_vars.items(): + shell_env[str(var_name)] = str(var_val) + # For expandvars + os.environ[str(var_name)] = str(var_val) + + # Create the kwargs dict + kwargs = {"env": shell_env, "shell": True, "universal_newlines": True} + return kwargs, yaml_vars + + +def _get_steps_to_start(wsteps, steps, steps_provided): + """ + Determine which steps to start workers for. + + :param `wsteps`: A list of steps associated with a worker + :param `steps`: A list of steps to start provided by the user + :param `steps`: A bool representing whether the user gave specific + steps to start or not + :returns: A list of steps to start workers for + """ + steps_to_start = [] + if steps_provided: + for wstep in wsteps: + if wstep in steps: + steps_to_start.append(wstep) + else: + steps_to_start.extend(wsteps) + + return steps_to_start + + +def start_celery_workers(spec, steps, celery_args, disable_logs, just_return_command): # pylint: disable=R0914,R0915 """Start the celery workers on the allocation - specs Tuple of (YAMLSpecification, MerlinSpec) + :param MerlinSpec spec: A MerlinSpec object representing our study + :param list steps: A list of steps to start workers for + :param str celery_args: A string of arguments to provide to the celery workers + :param bool disable_logs: A boolean flag to turn off the celery logs for the workers + :param bool just_return_command: When True, workers aren't started and just the launch command(s) + are returned + :side effect: Starts subprocesses for each worker we launch + :returns: A string of all the worker launch commands ... example config: @@ -203,20 +283,22 @@ def start_celery_workers(spec, steps, celery_args, just_return_command): overlap = spec.merlin["resources"]["overlap"] workers = spec.merlin["resources"]["workers"] - senv = spec.environment - spenv = os.environ.copy() - yenv = None - if senv: - yenv = get_yaml_var(senv, "variables", {}) - for k, v in yenv.items(): - spenv[str(k)] = str(v) - # For expandvars - os.environ[str(k)] = str(v) + # Build kwargs dict for subprocess.Popen to use when we launch the worker + kwargs, yenv = _create_kwargs(spec) worker_list = [] local_queues = [] + # Get the workers we need to start if we're only starting certain steps + steps_provided = False if "all" in steps else True # pylint: disable=R1719 + if steps_provided: + workers_to_start = _get_workers_to_start(spec, steps) + for worker_name, worker_val in workers.items(): + # Only triggered if --steps flag provided + if steps_provided and worker_name not in workers_to_start: + continue + skip_loop_step: bool = examine_and_log_machines(worker_val, yenv) if skip_loop_step: continue @@ -227,73 +309,60 @@ def start_celery_workers(spec, steps, celery_args, just_return_command): worker_args = "" worker_nodes = get_yaml_var(worker_val, "nodes", None) - worker_batch = get_yaml_var(worker_val, "batch", None) + # Get the correct steps to start workers for wsteps = get_yaml_var(worker_val, "steps", steps) - queues = spec.make_queue_string(wsteps).split(",") + steps_to_start = _get_steps_to_start(wsteps, steps, steps_provided) + queues = spec.make_queue_string(steps_to_start) # Check for missing arguments - verify_args(spec, worker_args, worker_name, overlap) + worker_args = verify_args(spec, worker_args, worker_name, overlap, disable_logs=disable_logs) # Add a per worker log file (debug) if LOG.isEnabledFor(logging.DEBUG): LOG.debug("Redirecting worker output to individual log files") worker_args += " --logfile %p.%i" - # Get the celery command - celery_com = launch_celery_workers(spec, steps=wsteps, worker_args=worker_args, just_return_command=True) - + # Get the celery command & add it to the batch launch command + celery_com = get_celery_cmd(queues, worker_args=worker_args, just_return_command=True) celery_cmd = os.path.expandvars(celery_com) - worker_cmd = batch_worker_launch(spec, celery_cmd, nodes=worker_nodes, batch=worker_batch) - worker_cmd = os.path.expandvars(worker_cmd) - try: - kwargs = {"env": spenv, "shell": True, "universal_newlines": True} - # These cannot be used with a detached process - # "stdout": subprocess.PIPE, - # "stderr": subprocess.PIPE, - - LOG.debug(f"worker cmd={worker_cmd}") - LOG.debug(f"env={spenv}") - - if just_return_command: - worker_list = "" - print(worker_cmd) - continue - - found = [] - running_queues = [] - - running_queues.extend(local_queues) - if not overlap: - running_queues.extend(get_running_queues()) - # Cache the queues from this worker to use to test - # for existing queues in any subsequent workers. - # If overlap is True, then do not check the local queues. - # This will allow multiple workers to pull from the same - # queue. - local_queues.extend(queues) - - for q in queues: - if q in running_queues: - found.append(q) - - if found: - LOG.warning( - f"A celery worker named '{worker_name}' is already configured/running for queue(s) = {' '.join(found)}" - ) - continue + LOG.debug(f"worker cmd={worker_cmd}") - _ = subprocess.Popen(worker_cmd, **kwargs) + if just_return_command: + worker_list = "" + print(worker_cmd) + continue - worker_list.append(worker_cmd) + # Get the running queues + running_queues = [] + running_queues.extend(local_queues) + queues = queues.split(",") + if not overlap: + running_queues.extend(get_running_queues()) + # Cache the queues from this worker to use to test + # for existing queues in any subsequent workers. + # If overlap is True, then do not check the local queues. + # This will allow multiple workers to pull from the same + # queue. + local_queues.extend(queues) + + # Search for already existing queues and log a warning if we try to start one that already exists + found = [] + for q in queues: # pylint: disable=C0103 + if q in running_queues: + found.append(q) + if found: + LOG.warning( + f"A celery worker named '{worker_name}' is already configured/running for queue(s) = {' '.join(found)}" + ) + continue - except Exception as e: - LOG.error(f"Cannot start celery workers, {e}") - raise + # Start the worker + launch_celery_worker(worker_cmd, worker_list, kwargs) # Return a string with the worker commands for logging return str(worker_list) @@ -306,7 +375,7 @@ def examine_and_log_machines(worker_val, yenv) -> bool: """ worker_machines = get_yaml_var(worker_val, "machines", None) if worker_machines: - LOG.debug("check machines = ", check_machines(worker_machines)) + LOG.debug(f"check machines = {check_machines(worker_machines)}") if not check_machines(worker_machines): return True @@ -320,11 +389,10 @@ def examine_and_log_machines(worker_val, yenv) -> bool: "The env:variables section does not have an OUTPUT_PATH specified, multi-machine checks cannot be performed." ) return False - else: - return False + return False -def verify_args(spec, worker_args, worker_name, overlap): +def verify_args(spec, worker_args, worker_name, overlap, disable_logs=False): """Examines the args passed to a worker for completeness.""" parallel = batch_check_parallel(spec) if parallel: @@ -340,29 +408,46 @@ def verify_args(spec, worker_args, worker_name, overlap): if overlap: nhash = time.strftime("%Y%m%d-%H%M%S") # TODO: Once flux fixes their bug, change this back to %h + # %h in Celery is short for hostname including domain name worker_args += f" -n {worker_name}{nhash}.%%h" - if "-l" not in worker_args: + if not disable_logs and "-l" not in worker_args: worker_args += f" -l {logging.getLevelName(LOG.getEffectiveLevel())}" + return worker_args + + +def launch_celery_worker(worker_cmd, worker_list, kwargs): + """ + Using the worker launch command provided, launch a celery worker. + :param str worker_cmd: The celery command to launch a worker + :param list worker_list: A list of worker launch commands + :param dict kwargs: A dictionary containing additional keyword args to provide + to subprocess.Popen -def launch_celery_workers(spec, steps=None, worker_args="", just_return_command=False): + :side effect: Launches a celery worker via a subprocess """ - Launch celery workers for the specified MerlinStudy. + try: + _ = subprocess.Popen(worker_cmd, **kwargs) # pylint: disable=R1732 + worker_list.append(worker_cmd) + except Exception as e: # pylint: disable=C0103 + LOG.error(f"Cannot start celery workers, {e}") + raise + - spec MerlinSpec object - steps The steps in the spec to tie the workers to +def get_celery_cmd(queue_names, worker_args="", just_return_command=False): + """ + Get the appropriate command to launch celery workers for the specified MerlinStudy. + queue_names The name(s) of the queue(s) to associate a worker with worker_args Optional celery arguments for the workers just_return_command Don't execute, just return the command """ - queues = spec.make_queue_string(steps) - worker_command = " ".join(["celery -A merlin worker", worker_args, "-Q", queues]) + worker_command = " ".join(["celery -A merlin worker", worker_args, "-Q", queue_names]) if just_return_command: return worker_command - else: - # This only runs celery locally the user would need to - # add all of the flux config themselves. - pass + # If we get down here, this only runs celery locally the user would need to + # add all of the flux config themselves. + return "" def purge_celery_tasks(queues, force): @@ -402,7 +487,8 @@ def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): >>> stop_celery_workers() """ - from merlin.celery import app + from merlin.celery import app # pylint: disable=C0415 + from merlin.config.configfile import CONFIG # pylint: disable=C0415 LOG.debug(f"Sending stop to queues: {queues}, worker_regex: {worker_regex}, spec_worker_names: {spec_worker_names}") active_queues, _ = get_queues(app) @@ -410,6 +496,10 @@ def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): # If not specified, get all the queues if queues is None: queues = [*active_queues] + # Celery adds the queue tag in front of each queue so we add that here + else: + for i, queue in enumerate(queues): + queues[i] = f"{CONFIG.celery.queue_tag}{queue}" # Find the set of all workers attached to all of those queues all_workers = set() @@ -424,23 +514,31 @@ def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): LOG.debug(f"Pre-filter worker stop list: {all_workers}") - print(f"all_workers: {all_workers}") - print(f"spec_worker_names: {spec_worker_names}") + # Stop workers with no flags if (spec_worker_names is None or len(spec_worker_names) == 0) and worker_regex is None: workers_to_stop = list(all_workers) + # Flag handling else: workers_to_stop = [] + # --spec flag if (spec_worker_names is not None) and len(spec_worker_names) > 0: for worker_name in spec_worker_names: - print(f"Result of regex_list_filter: {regex_list_filter(worker_name, all_workers)}") + LOG.debug( + f"""Result of regex_list_filter for {worker_name}: + {regex_list_filter(worker_name, all_workers, match=False)}""" + ) workers_to_stop += regex_list_filter(worker_name, all_workers, match=False) + # --workers flag if worker_regex is not None: - workers_to_stop += regex_list_filter(worker_regex, workers_to_stop) + for worker in worker_regex: + LOG.debug(f"Result of regex_list_filter: {regex_list_filter(worker, all_workers, match=False)}") + workers_to_stop += regex_list_filter(worker, all_workers, match=False) - print(f"workers_to_stop: {workers_to_stop}") + # Remove duplicates + workers_to_stop = list(set(workers_to_stop)) if workers_to_stop: LOG.info(f"Sending stop to these workers: {workers_to_stop}") - return app.control.broadcast("shutdown", destination=workers_to_stop) + app.control.broadcast("shutdown", destination=workers_to_stop) else: LOG.warning("No workers found to stop") @@ -454,10 +552,10 @@ def create_celery_config(config_dir, data_file_name, data_file_path): :param `data_file_path`: The full data file path. """ # This will need to come from the server interface - MERLIN_CONFIG = os.path.join(config_dir, data_file_name) + MERLIN_CONFIG = os.path.join(config_dir, data_file_name) # pylint: disable=C0103 if os.path.isfile(MERLIN_CONFIG): - from merlin.common.security import encrypt + from merlin.common.security import encrypt # pylint: disable=C0415 encrypt.init_key() LOG.info(f"The config file already exists, {MERLIN_CONFIG}") @@ -471,6 +569,6 @@ def create_celery_config(config_dir, data_file_name, data_file_path): LOG.info(f"The file {MERLIN_CONFIG} is ready to be edited for your system.") - from merlin.common.security import encrypt + from merlin.common.security import encrypt # pylint: disable=C0415 encrypt.init_key() diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 752b3650d..aafbabe2f 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -56,8 +56,7 @@ OUTPUT_DIR = "cli_test_studies" CLEAN_MERLIN_SERVER = "rm -rf appendonly.aof dump.rdb merlin_server/" -# KILL_WORKERS = "pkill -9 -f '.*merlin_test_worker'" -KILL_WORKERS = "pkill -9 -f 'celery'" +KILL_WORKERS = "pkill -9 -f '.*merlin_test_worker'" def define_tests(): # pylint: disable=R0914 From b17721eaffa78f6698d6e1bb4ad052e4ad89afca Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 3 Mar 2023 09:51:25 -0800 Subject: [PATCH 072/126] Fix pylint errors This is a combination of 4 commits. fix stop-workers, refactor integration tests, and fix pylint errors fix pylint errors in merlin/ directory fixed minor schema validation bugs fix stop-workers flags and refactor integration tests fix stop-workers flags and refactor integration tests added a mapping getter to MerlinSpec object update gitignore to ignore test outputs add ability to see all tests in table format update integration tests to use a different format and add stop-workers tests fix flags for stop-workers update CHANGELOG add Brian to contributors add a cleanup step to the stop-workers tests fix schema validation bugs pylint fixes to top level of merlin/ dir pylint fixes for merlin/common/ dir pylint fixes for merlin/config/ dir pylint fixes for merlin/examples/ dir pylint fixes for merlin/exceptions/ dir pylint fixes for merlin/server/ dir pylint fixes for merlin/spec/ dir pylint fixes for merlin/study/ dir pylint fixes for tests/integration/ dir pylint configuration changes fix-style changes remove a pylint config option that didn't work add negationchecks to two stop-workers tests update changelog to show pylint fix merge changes from develop fix some merge/pylint issues update github workflow to run all distributed tests modify wording in the docs related to stop-workers new pylint fixes after merging develop fixing issues with rebase fix merge issues that somehow got through remove duplicate entries from changelog fix more pylint errors in common/ and config/ fix additional pylint errors in examples/ exceptions/ and merlin/ top-level fix additional pylint errors in server/ and spec/ refactor batch.py for pylint fixes and general cleanup update variable name to make things cleaner move a celery specific function to celery.py add explanation of why we disable similar code warning fix additional pylint errors in study/ directory additional pylint fixes for utils.py and conditions.py final pylint updates refactor construct_worker_launch_command update CHANGELOG run fix-style fix import issues fix conflict between black and isort move import back inside function to fix issues move another import back inside function add some logging messages to debug why tests fail on github add extra log statements for debugging remove batch refactor --- CHANGELOG.md | 30 ++++ CONTRIBUTORS | 2 +- docs/source/merlin_commands.rst | 4 +- docs/source/modules/port_your_application.rst | 2 +- merlin/celery.py | 14 +- merlin/common/abstracts/enums/__init__.py | 12 +- merlin/common/openfilelist.py | 3 + merlin/common/opennpylib.py | 3 + merlin/common/sample_index.py | 6 +- merlin/common/sample_index_factory.py | 5 + merlin/common/security/encrypt.py | 12 +- merlin/common/tasks.py | 53 +++--- merlin/config/__init__.py | 5 +- merlin/config/broker.py | 54 +++--- merlin/config/configfile.py | 29 ++-- merlin/config/results_backend.py | 25 +-- merlin/config/utils.py | 28 +++- merlin/display.py | 3 +- .../dev_workflows/multiple_workers.yaml | 2 +- merlin/examples/examples.py | 1 + merlin/examples/generator.py | 11 +- merlin/exceptions/__init__.py | 14 +- merlin/main.py | 4 +- merlin/merlin_templates.py | 10 +- merlin/router.py | 13 +- merlin/server/server_commands.py | 53 +++--- merlin/server/server_config.py | 23 ++- merlin/server/server_util.py | 157 +++++++++++++----- merlin/spec/all_keys.py | 1 + merlin/spec/defaults.py | 1 + merlin/spec/expansion.py | 17 +- merlin/spec/override.py | 2 + merlin/spec/specification.py | 151 +++++++++-------- merlin/study/celeryadapter.py | 6 +- merlin/study/dag.py | 8 +- merlin/study/script_adapter.py | 111 ++++++------- merlin/study/step.py | 20 ++- merlin/study/study.py | 71 ++++---- merlin/utils.py | 77 +++++---- setup.cfg | 3 +- tests/integration/conditions.py | 8 +- tests/integration/run_tests.py | 10 +- tests/integration/test_definitions.py | 1 + 43 files changed, 631 insertions(+), 434 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ac7348d8..08bac767c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The learn.py script in the openfoam_wf* examples will now create the missing Energy v Lidspeed plot - Fixed the flags associated with the `stop-workers` command (--spec, --queues, --workers) - Fixed the --step flag for the `run-workers` command +- Fixed most of the pylint errors that we're showing up when you ran `make check-style` + - Some errors have been disabled rather than fixed. These include: + - Any pylint errors in merlin_template.py since it's deprecated now + - A "duplicate code" instance between a function in `expansion.py` and a method in `study.py` + - The function is explicitly not creating a MerlinStudy object so the code *must* be duplicate here + - Invalid-name (C0103): These errors typically relate to the names of short variables (i.e. naming files something like f or errors e) + - Unused-argument (W0613): These have been disabled for celery-related functions since celery *does* use these arguments behind the scenes + - Broad-exception (W0718): Pylint wants a more specific exception but sometimes it's ok to have a broad exception + - Import-outside-toplevel (C0415): Sometimes it's necessary for us to import inside a function. Where this is the case, these errors are disabled + - Too-many-statements (R0915): This is disabled for the `setup_argparse` function in `main.py` since it's necessary to be big. It's disabled in `tasks.py` and `celeryadapter.py` too until we can get around to refactoring some code there + - No-else-return (R1705): These are disabled in `router.py` until we refactor the file + - Consider-using-with (R1732): Pylint wants us to consider using with for calls to subprocess.run or subprocess.Popen but it's not necessary + - Too-many-arguments (R0913): These are disabled for functions that I believe *need* to have several arguments + - Note: these could be fixed by using *args and **kwargs but it makes the code harder to follow so I'm opting to not do that + - Too-many-local-variables (R0914): These are disabled for functions that have a lot of variables + - It may be a good idea at some point to go through these and try to find ways to shorten the number of variables used or split the functions up + - Too-many-branches (R0912): These are disabled for certain functions that require a good amount of branching + - Might be able to fix this in the future if we split functions up more + - Too-few-public-methods (R0903): These are disabled for classes we may add to in the future or "wrapper" classes + - Attribute-defined-outside-init (W0201): These errors are only disabled in `specification.py` as they occur in class methods so init() won't be called ### Added - Now loads np.arrays of dtype='object', allowing mix-type sample npy @@ -27,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added the --disable-logs flag to the `run-workers` command - Merlin will now assign `default_worker` to any step not associated with a worker - Added `get_step_worker_map()` as a method in `specification.py` +- Added `tabulate_info()` function in `display.py` to help with table formatting ### Changed - Changed celery_regex to celery_slurm_regex in test_definitions.py @@ -42,6 +63,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Modified the `merlinspec.json` file: - the minimum `gpus per task` is now 0 instead of 1 - variables defined in the `env` block of a spec file can now be arrays +- Refactored `batch.py`: + - Merged 4 functions (`check_for_slurm`, `check_for_lsf`, `check_for_flux`, and `check_for_pbs`) into 1 function named `check_for_scheduler` + - Modified `get_batch_type` to accommodate this change + - Added a function `parse_batch_block` to handle all the logic of reading in the batch block and storing it in one dict + - Added a function `get_flux_launch` to help decrease the amount of logic taking place in `batch_worker_launch` + - Modified `batch_worker_launch` to use the new `parse_batch_block` function + - Added a function `construct_scheduler_legend` to build a dict that keeps as much information as we need about each scheduler stored in one place + - Cleaned up the `construct_worker_launch_command` function to utilize the newly added functions and decrease the amount of repeated code + ## [1.9.1] ### Fixed diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 2d4a7c9f5..145fe0e02 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -6,4 +6,4 @@ Joe Koning Jeremy White Aidan Keogh Ryan Lee -Brian Gunnarson \ No newline at end of file +Brian Gunnarson diff --git a/docs/source/merlin_commands.rst b/docs/source/merlin_commands.rst index 2a767797f..316d675fe 100644 --- a/docs/source/merlin_commands.rst +++ b/docs/source/merlin_commands.rst @@ -378,7 +378,7 @@ To send out a stop signal to some or all connected workers, use: $ merlin stop-workers [--spec ] [--queues ] [--workers ] [--task_server celery] -The default behavior will send a stop to all connected workers, +The default behavior will send a stop to all connected workers across all workflows, having them shutdown softly. The ``--spec`` option targets only workers named in the ``merlin`` block of the spec file. @@ -390,7 +390,7 @@ The ``--queues`` option allows you to pass in the names of specific queues to st # Stop all workers on these queues, no matter their name $ merlin stop-workers --queues queue1 queue2 -The ``--workers`` option allows you to pass in a regular expression of names of queues to stop: +The ``--workers`` option allows you to pass in regular expressions of names of workers to stop: .. code:: bash diff --git a/docs/source/modules/port_your_application.rst b/docs/source/modules/port_your_application.rst index c9d89b06d..0dd501c2e 100644 --- a/docs/source/modules/port_your_application.rst +++ b/docs/source/modules/port_your_application.rst @@ -44,7 +44,7 @@ The scripts defined in the workflow steps are also written to the output directo .. where are the worker logs, and what might show up there that .out and .err won't see? -> these more developer focused output? -When a bug crops up in a running study with many parameters, there are a few other commands to make use of. Rather than trying to spam ``Ctrl-c`` to kill all the workers, you will want to instead use ``merlin stop-workers .yaml`` to stop the workers. This should then be followed up with ``merlin purge .yaml`` to clear out the task queue to prevent the same +When a bug crops up in a running study with many parameters, there are a few other commands to make use of. Rather than trying to spam ``Ctrl-c`` to kill all the workers, you will want to instead use ``merlin stop-workers --spec .yaml`` to stop the workers for that workflow. This should then be followed up with ``merlin purge .yaml`` to clear out the task queue to prevent the same buggy tasks from continuing to run the next time ``run-workers`` is invoked. .. last item from board: use merlin status to see if have workers ... is that 'dangling tasks' in the image? diff --git a/merlin/celery.py b/merlin/celery.py index 52d4e4589..9febf2523 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -44,12 +44,22 @@ from merlin.config import broker, celeryconfig, results_backend from merlin.config.configfile import CONFIG from merlin.config.utils import Priority, get_priority -from merlin.router import route_for_task from merlin.utils import nested_namespace_to_dicts LOG: logging.Logger = logging.getLogger(__name__) + +# This function has to have specific args/return values for celery so ignore pylint +def route_for_task(name, args, kwargs, options, task=None, **kw): # pylint: disable=W0613,R1710 + """ + Custom task router for queues + """ + if ":" in name: + queue, _ = name.split(":") + return {"queue": queue} + + merlin.common.security.encrypt_backend_traffic.set_backend_funcs() @@ -84,7 +94,7 @@ # set task priority defaults to prioritize workflow tasks over task-expansion tasks task_priority_defaults: Dict[str, Union[int, Priority]] = { "task_queue_max_priority": 10, - "task_default_priority": get_priority(Priority.mid), + "task_default_priority": get_priority(Priority.MID), } if CONFIG.broker.name.lower() == "redis": app.conf.broker_transport_options = { diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index f242b15fb..366692ac1 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -32,17 +32,7 @@ from enum import IntEnum -__all__ = ( - "ReturnCode", - "OK_VALUE", - "ERROR_VALUE", - "RESTART_VALUE", - "SOFT_FAIL_VALUE", - "HARD_FAIL_VALUE", - "DRY_OK_VALUE", - "RETRY_VALUE", - "STOP_WORKERS_VALUE", -) +__all__ = ("ReturnCode",) class ReturnCode(IntEnum): diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 5e82abbc2..d7dbf0ff4 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -57,6 +57,9 @@ """ +# This file is not currently used so we don't care what pylint has to say +# pylint: skip-file + import copy diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index c2391ca6d..e91105bf8 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -81,6 +81,9 @@ print a.dtype # dtype of array """ +# This file is not currently used so we don't care what pylint has to say +# pylint: skip-file + from typing import List, Tuple import numpy as np diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index dd93cf5a7..0d786820a 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -69,7 +69,7 @@ class SampleIndex: # Class variable to indicate depth (mostly used for pretty printing). depth = -1 - def __init__(self, minid, maxid, children, name, leafid=-1, num_bundles=0, address=""): + def __init__(self, minid, maxid, children, name, leafid=-1, num_bundles=0, address=""): # pylint: disable=R0913 """The constructor.""" # The direct children of this node, generally also of type SampleIndex. @@ -251,14 +251,14 @@ def get_path_to_sample(self, sample_id): """ path = self.name for child_val in self.children.values(): - if sample_id >= child_val.min and sample_id < child_val.max: + if child_val.min <= sample_id < child_val.max: path = os.path.join(path, child_val.get_path_to_sample(sample_id)) return path def write_single_sample_index_file(self, path): """Writes the index file associated with this node.""" if not self.is_directory: - return + return None fname = os.path.join(path, self.name, "sample_index.txt") with open(fname, "w") as _file: diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 45ddb2ed1..47e600d82 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -37,6 +37,11 @@ from merlin.utils import cd +# These pylint errors I've disabled are for "too many arguments" +# and "too many local variables". I think the functions are still clear +# pylint: disable=R0913,R0914 + + def create_hierarchy( num_samples, bundle_size, diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 6a0766427..1c9bd342b 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -28,9 +28,7 @@ # SOFTWARE. ############################################################################### -""" -TODO -""" +"""This module handles encryption logic""" import logging import os @@ -40,6 +38,10 @@ from merlin.config.configfile import CONFIG +# This disables all the errors about short variable names (like using f to represent a file) +# pylint: disable=invalid-name + + LOG = logging.getLogger(__name__) @@ -52,8 +54,8 @@ def _get_key_path(): try: key_filepath = os.path.abspath(os.path.expanduser(key_filepath)) - except KeyError: - raise ValueError("Error! No password provided for RabbitMQ") + except KeyError as e: + raise ValueError("Error! No password provided for RabbitMQ") from e return key_filepath diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index b0eca1b6c..3a4077ac3 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -35,8 +35,10 @@ import os from typing import Any, Dict, Optional +# Need to disable an overwrite warning here since celery has an exception that we need that directly +# overwrites a python built-in exception from celery import chain, chord, group, shared_task, signature -from celery.exceptions import MaxRetriesExceededError, OperationalError, TimeoutError +from celery.exceptions import MaxRetriesExceededError, OperationalError, TimeoutError # pylint: disable=W0622 from merlin.common.abstracts.enums import ReturnCode from merlin.common.sample_index import uniform_directories @@ -62,14 +64,21 @@ STOP_COUNTDOWN = 60 +# TODO: most of the pylint errors that are disabled in this file are the ones listed below. +# We should refactor this file so that we use more functions to solve all of these errors +# R0912: too many branches +# R0913: too many arguments +# R0914: too many local variables +# R0915: too many statements + @shared_task( # noqa: C901 bind=True, autoretry_for=retry_exceptions, retry_backoff=True, - priority=get_priority(Priority.high), + priority=get_priority(Priority.HIGH), ) -def merlin_step(self, *args: Any, **kwargs: Any) -> Optional[ReturnCode]: # noqa: C901 +def merlin_step(self, *args: Any, **kwargs: Any) -> Optional[ReturnCode]: # noqa: C901 pylint: disable=R0912,R0915 """ Executes a Merlin Step :param args: The arguments, one of which should be an instance of Step @@ -123,7 +132,8 @@ def merlin_step(self, *args: Any, **kwargs: Any) -> Optional[ReturnCode]: # noq self.retry(countdown=step.retry_delay) except MaxRetriesExceededError: LOG.warning( - f"*** Step '{step_name}' in '{step_dir}' exited with a MERLIN_RESTART command, but has already reached its retry limit ({self.max_retries}). Continuing with workflow." + f"""*** Step '{step_name}' in '{step_dir}' exited with a MERLIN_RESTART command, + but has already reached its retry limit ({self.max_retries}). Continuing with workflow.""" ) result = ReturnCode.SOFT_FAIL elif result == ReturnCode.RETRY: @@ -135,7 +145,8 @@ def merlin_step(self, *args: Any, **kwargs: Any) -> Optional[ReturnCode]: # noq self.retry(countdown=step.retry_delay) except MaxRetriesExceededError: LOG.warning( - f"*** Step '{step_name}' in '{step_dir}' exited with a MERLIN_RETRY command, but has already reached its retry limit ({self.max_retries}). Continuing with workflow." + f"""*** Step '{step_name}' in '{step_dir}' exited with a MERLIN_RETRY command, + but has already reached its retry limit ({self.max_retries}). Continuing with workflow.""" ) result = ReturnCode.SOFT_FAIL elif result == ReturnCode.SOFT_FAIL: @@ -226,9 +237,9 @@ def prepare_chain_workspace(sample_index, chain_): bind=True, autoretry_for=retry_exceptions, retry_backoff=True, - priority=get_priority(Priority.low), + priority=get_priority(Priority.LOW), ) -def add_merlin_expanded_chain_to_chord( +def add_merlin_expanded_chain_to_chord( # pylint: disable=R0913,R0914 self, task_type, chain_, @@ -364,14 +375,14 @@ def add_chains_to_chord(self, all_chains): # generates a new task signature, so we need to make # sure we are modifying task signatures before adding them to the # kwargs. - for g in reversed(range(len(all_chains))): - if g < len(all_chains) - 1: + for j in reversed(range(len(all_chains))): + if j < len(all_chains) - 1: # fmt: off - new_kwargs = signature(all_chains[g][i]).kwargs.update( - {"next_in_chain": all_chains[g + 1][i]} + new_kwargs = signature(all_chains[j][i]).kwargs.update( + {"next_in_chain": all_chains[j + 1][i]} ) # fmt: on - all_chains[g][i] = all_chains[g][i].replace(kwargs=new_kwargs) + all_chains[j][i] = all_chains[j][i].replace(kwargs=new_kwargs) chain_steps.append(all_chains[0][i]) for sig in chain_steps: @@ -387,9 +398,9 @@ def add_chains_to_chord(self, all_chains): bind=True, autoretry_for=retry_exceptions, retry_backoff=True, - priority=get_priority(Priority.low), + priority=get_priority(Priority.LOW), ) -def expand_tasks_with_samples( +def expand_tasks_with_samples( # pylint: disable=R0913,R0914 self, dag, chain_, @@ -398,7 +409,6 @@ def expand_tasks_with_samples( task_type, adapter_config, level_max_dirs, - **kwargs, ): """ Generate a group of celery chains of tasks from a chain of task names, using merlin @@ -492,6 +502,7 @@ def expand_tasks_with_samples( LOG.debug("simple chain task queued") +# Pylint complains that "self" is unused but it's needed behind the scenes with celery @shared_task( bind=True, autoretry_for=retry_exceptions, @@ -499,9 +510,9 @@ def expand_tasks_with_samples( acks_late=False, reject_on_worker_lost=False, name="merlin:shutdown_workers", - priority=get_priority(Priority.high), + priority=get_priority(Priority.HIGH), ) -def shutdown_workers(self, shutdown_queues): +def shutdown_workers(self, shutdown_queues): # pylint: disable=W0613 """ This task issues a call to shutdown workers. @@ -518,13 +529,15 @@ def shutdown_workers(self, shutdown_queues): return stop_workers("celery", None, shutdown_queues, None) +# Pylint complains that these args are unused but celery passes args +# here behind the scenes and won't work if these aren't here @shared_task( autoretry_for=retry_exceptions, retry_backoff=True, name="merlin:chordfinisher", - priority=get_priority(Priority.low), + priority=get_priority(Priority.LOW), ) -def chordfinisher(*args, **kwargs): +def chordfinisher(*args, **kwargs): # pylint: disable=W0613 """. It turns out that chain(group,group) in celery does not execute one group after another, but executes the groups as if they were independent from @@ -539,7 +552,7 @@ def chordfinisher(*args, **kwargs): autoretry_for=retry_exceptions, retry_backoff=True, name="merlin:queue_merlin_study", - priority=get_priority(Priority.low), + priority=get_priority(Priority.LOW), ) def queue_merlin_study(study, adapter): """ diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 08a0362ae..24e4ed5c1 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -38,7 +38,10 @@ from merlin.utils import nested_dict_to_namespaces -class Config: +# Pylint complains that there's too few methods here but this class might +# be useful if we ever need to do extra stuff with the configuration so we'll +# ignore it for now +class Config: # pylint: disable=R0903 """ The Config class, meant to store all Merlin config settings in one place. Regardless of the config data loading method, this class is meant to diff --git a/merlin/config/broker.py b/merlin/config/broker.py index ea26edc8f..0bcb3c42f 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -58,17 +58,16 @@ def read_file(filepath): "Safe file read from filepath" - with open(filepath, "r") as f: + with open(filepath, "r") as f: # pylint: disable=C0103 line = f.readline().strip() return quote(line, safe="") -def get_rabbit_connection(config_path, include_password, conn="amqps"): +def get_rabbit_connection(include_password, conn="amqps"): """ Given the path to the directory where the broker configurations are stored setup and return the RabbitMQ connection string. - :param config_path : The path for ssl certificates and passwords :param include_password : Format the connection for ouput by setting this True """ LOG.debug(f"Broker: connection = {conn}") @@ -86,13 +85,13 @@ def get_rabbit_connection(config_path, include_password, conn="amqps"): password_filepath = CONFIG.broker.password LOG.debug(f"Broker: password filepath = {password_filepath}") password_filepath = os.path.abspath(expanduser(password_filepath)) - except KeyError: - raise ValueError("Broker: No password provided for RabbitMQ") + except KeyError as e: # pylint: disable=C0103 + raise ValueError("Broker: No password provided for RabbitMQ") from e try: password = read_file(password_filepath) - except IOError: - raise ValueError(f"Broker: RabbitMQ password file {password_filepath} does not exist") + except IOError as e: # pylint: disable=C0103 + raise ValueError(f"Broker: RabbitMQ password file {password_filepath} does not exist") from e try: port = CONFIG.broker.port @@ -120,13 +119,10 @@ def get_rabbit_connection(config_path, include_password, conn="amqps"): return RABBITMQ_CONNECTION.format(**rabbitmq_config) -def get_redissock_connection(config_path, include_password): +def get_redissock_connection(): """ Given the path to the directory where the broker configurations are stored setup and return the redis+socket connection string. - - :param config_path : The path for ssl certificates and passwords - :param include_password : Format the connection for ouput by setting this True """ try: db_num = CONFIG.broker.db_num @@ -141,18 +137,17 @@ def get_redissock_connection(config_path, include_password): # flake8 complains this function is too complex, we don't gain much nesting any of this as a separate function, # however, cyclomatic complexity examination is off to get around this -def get_redis_connection(config_path, include_password, ssl=False): # noqa C901 +def get_redis_connection(include_password, use_ssl=False): # noqa C901 """ Return the redis or rediss specific connection - :param config_path : The path for ssl certificates and passwords :param include_password : Format the connection for ouput by setting this True - :param ssl : Flag to use rediss output + :param use_ssl : Flag to use rediss output """ server = CONFIG.broker.server LOG.debug(f"Broker: server = {server}") - urlbase = "rediss" if ssl else "redis" + urlbase = "rediss" if use_ssl else "redis" try: port = CONFIG.broker.port @@ -179,9 +174,9 @@ def get_redis_connection(config_path, include_password, ssl=False): # noqa C901 except IOError: password = CONFIG.broker.password if include_password: - spass = "%s:%s@" % (username, password) + spass = f"{username}:{password}@" else: - spass = "%s:%s@" % (username, "******") + spass = f"{username}:******@" except (AttributeError, KeyError): spass = "" LOG.debug(f"Broker: redis using default password = {spass}") @@ -218,25 +213,24 @@ def get_connection_string(include_password=True): if broker not in BROKERS: raise ValueError(f"Error: {broker} is not a supported broker.") - else: - return _sort_valid_broker(broker, config_path, include_password) + return _sort_valid_broker(broker, include_password) -def _sort_valid_broker(broker, config_path, include_password): - if broker == "rabbitmq" or broker == "amqps": - return get_rabbit_connection(config_path, include_password, conn="amqps") +def _sort_valid_broker(broker, include_password): + if broker in ("rabbitmq", "amqps"): + return get_rabbit_connection(include_password, conn="amqps") - elif broker == "amqp": - return get_rabbit_connection(config_path, include_password, conn="amqp") + if broker == "amqp": + return get_rabbit_connection(include_password, conn="amqp") - elif broker == "redis+socket": - return get_redissock_connection(config_path, include_password) + if broker == "redis+socket": + return get_redissock_connection() - elif broker == "redis": - return get_redis_connection(config_path, include_password) + if broker == "redis": + return get_redis_connection(include_password) # broker must be rediss - return get_redis_connection(config_path, include_password, ssl=True) + return get_redis_connection(include_password, use_ssl=True) def get_ssl_config() -> Union[bool, Dict[str, Union[str, ssl.VerifyMode]]]: @@ -276,7 +270,7 @@ def get_ssl_config() -> Union[bool, Dict[str, Union[str, ssl.VerifyMode]]]: if not broker_ssl: broker_ssl = True - if broker == "rabbitmq" or broker == "rediss" or broker == "amqps": + if broker in ("rabbitmq", "rediss", "amqps"): return broker_ssl return False diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index bb7f79875..fc4743b86 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -36,7 +36,7 @@ import logging import os import ssl -from typing import Dict, List, Optional, Union +from typing import Dict, Optional, Union from merlin.config import Config from merlin.utils import load_yaml @@ -60,9 +60,9 @@ def load_config(filepath): """ if not os.path.isfile(filepath): LOG.info(f"No app config file at {filepath}") - else: - LOG.info(f"Reading app config from file {filepath}") - return load_yaml(filepath) + return None + LOG.info(f"Reading app config from file {filepath}") + return load_yaml(filepath) def find_config_file(path=None): @@ -78,10 +78,9 @@ def find_config_file(path=None): if os.path.isfile(local_app): return local_app - elif os.path.isfile(path_app): + if os.path.isfile(path_app): return path_app - else: - return None + return None app_path = os.path.join(path, APP_FILENAME) if os.path.exists(app_path): @@ -133,6 +132,7 @@ def get_config(path: Optional[str]) -> Dict: def load_default_celery(config): + """Creates the celery default configuration""" try: config["celery"] except KeyError: @@ -152,6 +152,7 @@ def load_default_celery(config): def load_defaults(config): + """Loads default configuration values""" load_default_user_names(config) load_default_celery(config) @@ -247,7 +248,7 @@ def get_ssl_entries( except (AttributeError, KeyError): LOG.debug(f"{server_type}: ssl ssl_protocol not present") - if server_ssl and "cert_reqs" not in server_ssl.keys(): + if server_ssl and "cert_reqs" not in server_ssl: server_ssl["cert_reqs"] = ssl.CERT_REQUIRED ssl_map: Dict[str, str] = process_ssl_map(server_name) @@ -290,14 +291,12 @@ def merge_sslmap(server_ssl: Dict[str, Union[str, ssl.VerifyMode]], ssl_map: Dic : param ssl_map : the dict holding special key:value pairs for rediss and mysql """ new_server_ssl: Dict[str, Union[str, ssl.VerifyMode]] = {} - sk: List[str] = server_ssl.keys() - smk: List[str] = ssl_map.keys() - k: str - for k in sk: - if k in smk: - new_server_ssl[ssl_map[k]] = server_ssl[k] + + for key in server_ssl: + if key in ssl_map: + new_server_ssl[ssl_map[key]] = server_ssl[key] else: - new_server_ssl[k] = server_ssl[k] + new_server_ssl[key] = server_ssl[key] return new_server_ssl diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 40ea19af7..3a03b3d79 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -100,7 +100,7 @@ def get_backend_password(password_file, certs_path=None): # The password was given instead of the filepath. password = password_file.strip() else: - with open(password_filepath, "r") as f: + with open(password_filepath, "r") as f: # pylint: disable=C0103 line = f.readline().strip() password = quote(line, safe="") @@ -151,9 +151,9 @@ def get_redis(certs_path=None, include_password=True, ssl=False): # noqa C901 password = CONFIG.results_backend.password if include_password: - spass = "%s:%s@" % (username, password) + spass = f"{username}:{password}@" else: - spass = "%s:%s@" % (username, "******") + spass = f"{username}:******@" except (KeyError, AttributeError): spass = "" LOG.debug(f"Results backend: redis using default password = {spass}") @@ -183,7 +183,7 @@ def get_mysql_config(certs_path, mysql_certs): certs = {} for key, filename in mysql_certs.items(): - for f in files: + for f in files: # pylint: disable=C0103 if not f == filename: continue @@ -215,7 +215,7 @@ def get_mysql(certs_path=None, mysql_certs=None, include_password=True): if not server: msg = f"Results backend: server {server} does not have a configuration" - raise Exception(msg) + raise TypeError(msg) # TypeError since server is None and not str password = get_backend_password(password_file, certs_path=certs_path) @@ -225,8 +225,9 @@ def get_mysql(certs_path=None, mysql_certs=None, include_password=True): mysql_config = get_mysql_config(certs_path, mysql_certs) if not mysql_config: - msg = f"The connection information for MySQL could not be set, cannot find:\n {mysql_certs}\ncheck the celery/certs path or set the ssl information in the app.yaml file." - raise Exception(msg) + msg = f"""The connection information for MySQL could not be set, cannot find:\n + {mysql_certs}\ncheck the celery/certs path or set the ssl information in the app.yaml file.""" + raise TypeError(msg) # TypeError since mysql_config is None when it shouldn't be mysql_config["user"] = CONFIG.results_backend.username if include_password: @@ -275,16 +276,16 @@ def _resolve_backend_string(backend, certs_path, include_password): if "mysql" in backend: return get_mysql(certs_path=certs_path, include_password=include_password) - elif "sqlite" in backend: + if "sqlite" in backend: return SQLITE_CONNECTION_STRING - elif backend == "redis": + if backend == "redis": return get_redis(certs_path=certs_path, include_password=include_password) - elif backend == "rediss": + if backend == "rediss": return get_redis(certs_path=certs_path, include_password=include_password, ssl=True) - else: - return None + + return None def get_ssl_config(celery_check=False): diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 3cce32883..8d85ccf50 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -27,6 +27,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ############################################################################### +"""This module contains priority handling""" import enum from typing import List @@ -35,34 +36,43 @@ class Priority(enum.Enum): - high = 1 - mid = 2 - low = 3 + """Enumerated Priorities""" + + HIGH = 1 + MID = 2 + LOW = 3 def is_rabbit_broker(broker: str) -> bool: + """Check if the broker is a rabbit server""" return broker in ["rabbitmq", "amqps", "amqp"] def is_redis_broker(broker: str) -> bool: + """Check if the broker is a redis server""" return broker in ["redis", "rediss", "redis+socket"] def get_priority(priority: Priority) -> int: + """ + Get the priority based on the broker. For a rabbit broker + a low priority is 1 and high is 10. For redis it's the opposite. + :returns: An int representing the priority level + """ broker: str = CONFIG.broker.name.lower() - priorities: List[Priority] = [Priority.high, Priority.mid, Priority.low] + priorities: List[Priority] = [Priority.HIGH, Priority.MID, Priority.LOW] if not isinstance(priority, Priority): raise TypeError(f"Unrecognized priority '{priority}'! Priority enum options: {[x.name for x in priorities]}") - if priority == Priority.mid: + if priority == Priority.MID: return 5 if is_rabbit_broker(broker): - if priority == Priority.low: + if priority == Priority.LOW: return 1 - if priority == Priority.high: + if priority == Priority.HIGH: return 10 if is_redis_broker(broker): - if priority == Priority.low: + if priority == Priority.LOW: return 10 - if priority == Priority.high: + if priority == Priority.HIGH: return 1 raise ValueError(f"Function get_priority has reached unknown state! Maybe unsupported broker {broker}?") diff --git a/merlin/display.py b/merlin/display.py index 0e0e11e66..cc05fd8ea 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -127,7 +127,7 @@ def _examine_connection(server, sconf, excpts): counter += 1 if counter > connect_timeout: conn_check.kill() - raise TimeoutError(f"Connection was killed due to timeout ({connect_timeout}server)") + raise TimeoutError(f"Connection was killed due to timeout ({connect_timeout}s)") conn.release() if conn_check.exception: error, _ = conn_check.exception @@ -195,6 +195,7 @@ def display_multiple_configs(files, configs): pprint.pprint(config) +# Might use args here in the future so we'll disable the pylint warning for now def print_info(args): # pylint: disable=W0613 """ Provide version and location information about python and pip to diff --git a/merlin/examples/dev_workflows/multiple_workers.yaml b/merlin/examples/dev_workflows/multiple_workers.yaml index f393f87d3..8785d9e9a 100644 --- a/merlin/examples/dev_workflows/multiple_workers.yaml +++ b/merlin/examples/dev_workflows/multiple_workers.yaml @@ -53,4 +53,4 @@ merlin: steps: [step_2] other_merlin_test_worker: args: -l INFO - steps: [step_3, step_4] \ No newline at end of file + steps: [step_3, step_4] diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 7fbd502b1..bfd65a6a8 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -27,6 +27,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ############################################################################### +"""This module contains example spec files with explanations of each block""" # Taken from https://lc.llnl.gov/mlsi/docs/merlin/merlin_config.html TEMPLATE_FILE_CONTENTS = """ diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index c97cc2757..3145b65e3 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -50,13 +50,15 @@ def gather_example_dirs(): + """Get all the example directories""" result = {} - for d in os.listdir(EXAMPLES_DIR): - result[d] = d + for directory in os.listdir(EXAMPLES_DIR): + result[directory] = directory return result def gather_all_examples(): + """Get all the example yaml files""" path = os.path.join(os.path.join(EXAMPLES_DIR, ""), os.path.join("*", "*.yaml")) return glob.glob(path) @@ -81,11 +83,11 @@ def list_examples(): directory = os.path.join(os.path.join(EXAMPLES_DIR, example_dir), "") specs = glob.glob(directory + "*.yaml") for spec in specs: - with open(spec) as f: + with open(spec) as f: # pylint: disable=C0103 try: spec_metadata = yaml.safe_load(f)["description"] except KeyError: - LOG.warn(f"{spec} lacks required section 'description'") + LOG.warning(f"{spec} lacks required section 'description'") continue except TypeError: continue @@ -103,6 +105,7 @@ def setup_example(name, outdir): """Setup the given example.""" example = None spec_paths = gather_all_examples() + spec_path = None for spec_path in spec_paths: spec = os.path.basename(os.path.normpath(spec_path)).replace(".yaml", "") if name == spec: diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 225e69d28..ea376d1b5 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -32,6 +32,10 @@ Module of all Merlin-specific exception types. """ +# Pylint complains that these exceptions are no different from Exception +# but we don't care, we just need new names for exceptions here +# pylint: disable=W0246 + __all__ = ( "RetryException", "SoftFailException", @@ -48,7 +52,7 @@ class RetryException(Exception): """ def __init__(self): - super(RetryException, self).__init__() + super().__init__() class SoftFailException(Exception): @@ -58,7 +62,7 @@ class SoftFailException(Exception): """ def __init__(self): - super(SoftFailException, self).__init__() + super().__init__() class HardFailException(Exception): @@ -68,7 +72,7 @@ class HardFailException(Exception): """ def __init__(self): - super(HardFailException, self).__init__() + super().__init__() class InvalidChainException(Exception): @@ -77,7 +81,7 @@ class InvalidChainException(Exception): """ def __init__(self): - super(InvalidChainException, self).__init__() + super().__init__() class RestartException(Exception): @@ -87,4 +91,4 @@ class RestartException(Exception): """ def __init__(self): - super(RestartException, self).__init__() + super().__init__() diff --git a/merlin/main.py b/merlin/main.py index 723fadd22..3a30df135 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -366,7 +366,9 @@ def process_server(args: Namespace): config_server(args) -def setup_argparse() -> None: +# Pylint complains that there's too many statements here and wants us +# to split the function up but that wouldn't make much sense so we ignore it +def setup_argparse() -> None: # pylint: disable=R0915 """ Setup argparse and any CLI options we want available via the package. """ diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index a1af0298b..0db33d22e 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -42,12 +42,14 @@ LOG = logging.getLogger("merlin-templates") DEFAULT_LOG_LEVEL = "ERROR" +# We disable all pylint errors in this file since this is deprecated anyways -def process_templates(args): + +def process_templates(args): # pylint: disable=W0613,C0116 LOG.error("The command `merlin-templates` has been deprecated in favor of `merlin example`.") -def setup_argparse(): +def setup_argparse(): # pylint: disable=C0116 parser = argparse.ArgumentParser( prog="Merlin Examples", description=banner_small, @@ -57,14 +59,14 @@ def setup_argparse(): return parser -def main(): +def main(): # pylint: disable=C0116 try: parser = setup_argparse() args = parser.parse_args() setup_logging(logger=LOG, log_level=DEFAULT_LOG_LEVEL, colors=True) args.func(args) sys.exit() - except Exception as ex: + except Exception as ex: # pylint: disable=W0718 print(ex) sys.exit(1) diff --git a/merlin/router.py b/merlin/router.py index ab4b8e933..8e8c85bed 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -60,6 +60,10 @@ LOG = logging.getLogger(__name__) +# TODO go through this file and find a way to make a common return format to main.py +# Also, if that doesn't fix them, look into the pylint errors that have been disabled +# and try to resolve them + def run_task_server(study, run_mode=None): """ @@ -202,15 +206,6 @@ def stop_workers(task_server, spec_worker_names, queues, workers_regex): LOG.error("Celery is not specified as the task server!") -def route_for_task(name, args, kwargs, options, task=None, **kw): # pylint: disable=W0613,R1710 - """ - Custom task router for queues - """ - if ":" in name: - queue, _ = name.split(":") - return {"queue": queue} - - def create_config(task_server: str, config_dir: str, broker: str, test: str) -> None: """ Create a config for the given task server. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 045ef22e3..ecaa76b4e 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -70,7 +70,9 @@ def init_server() -> None: LOG.info("Merlin server initialization successful.") -def config_server(args: Namespace) -> None: +# Pylint complains that there's too many branches in this function but +# it looks clean to me so we'll ignore it +def config_server(args: Namespace) -> None: # pylint: disable=R0912 """ Process the merlin server config flags to make changes and edits to appropriate configurations based on the input passed in by the user. @@ -144,6 +146,8 @@ def config_server(args: Namespace) -> None: else: LOG.error(f"User '{args.remove_user}' doesn't exist within current users.") + return None + def status_server() -> None: """ @@ -162,20 +166,22 @@ def status_server() -> None: LOG.info("Merlin server is running.") -def start_server() -> bool: +def start_server() -> bool: # pylint: disable=R0911 """ Start a merlin server container using singularity. :return:: True if server was successful started and False if failed. """ current_status = get_server_status() - - if current_status == ServerStatus.NOT_INITALIZED or current_status == ServerStatus.MISSING_CONTAINER: - LOG.info("Merlin server has not been initialized. Please run 'merlin server init' first.") - return False - - if current_status == ServerStatus.RUNNING: - LOG.info("Merlin server already running.") - LOG.info("Stop current server with 'merlin server stop' before attempting to start a new server.") + uninitialized_err = "Merlin server has not been intitialized. Please run 'merlin server init' first." + status_errors = { + ServerStatus.NOT_INITALIZED: uninitialized_err, + ServerStatus.MISSING_CONTAINER: uninitialized_err, + ServerStatus.RUNNING: """Merlin server already running. + Stop current server with 'merlin server stop' before attempting to start a new server.""", + } + + if current_status in status_errors: + LOG.info(status_errors[current_status]) return False server_config = pull_server_config() @@ -184,16 +190,19 @@ def start_server() -> bool: return False image_path = server_config.container.get_image_path() - if not os.path.exists(image_path): - LOG.error("Unable to find image at " + image_path) - return False - config_path = server_config.container.get_config_path() - if not os.path.exists(config_path): - LOG.error("Unable to find config file at " + config_path) - return False - - process = subprocess.Popen( + path_errors = { + image_path: "image", + config_path: "config file", + } + + for path in (image_path, config_path): + if not os.path.exists(path): + LOG.error(f"Unable to find {path_errors[path]} at {path}") + return False + + # Pylint wants us to use with here but we don't need that + process = subprocess.Popen( # pylint: disable=R1732 server_config.container_format.get_run_command() .strip("\\") .format( @@ -237,9 +246,9 @@ def start_server() -> bool: redis_users.apply_to_redis(redis_config.get_ip_address(), redis_config.get_port(), redis_config.get_password()) new_app_yaml = os.path.join(server_config.container.get_config_dir(), "app.yaml") - ay = AppYaml() - ay.apply_server_config(server_config=server_config) - ay.write(new_app_yaml) + app_yaml = AppYaml() + app_yaml.apply_server_config(server_config=server_config) + app_yaml.write(new_app_yaml) LOG.info(f"New app.yaml written to {new_app_yaml}.") LOG.info("Replace app.yaml in ~/.merlin/app.yaml to use merlin server as main configuration.") LOG.info("To use for local runs, move app.yaml into the running directory.") diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index e62e12e4f..e433a4ebf 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -27,6 +27,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ############################################################################### +"""This module represents everything that goes into server configuration""" import enum import logging @@ -52,7 +53,7 @@ try: - import importlib.resources as resources + from importlib import resources except ImportError: import importlib_resources as resources @@ -99,7 +100,7 @@ def generate_password(length, pass_command: str = None) -> str: random.shuffle(characters) password = [] - for i in range(length): + for _ in range(length): password.append(random.choice(characters)) random.shuffle(password) @@ -132,6 +133,8 @@ def parse_redis_output(redis_stdout: BufferedReader) -> Tuple[bool, str]: return False, line.decode("utf-8") line = redis_stdout.readline() + return False, "Reached end of redis output without seeing 'Ready to accept connections'" + def create_server_config() -> bool: """ @@ -142,7 +145,7 @@ def create_server_config() -> bool: :return:: True if success and False if fail """ if not os.path.exists(MERLIN_CONFIG_DIR): - LOG.error("Unable to find main merlin configuration directory at " + MERLIN_CONFIG_DIR) + LOG.error(f"Unable to find main merlin configuration directory at {MERLIN_CONFIG_DIR}") return False config_dir = os.path.join(MERLIN_CONFIG_DIR, MERLIN_SERVER_SUBDIR) @@ -172,7 +175,7 @@ def create_server_config() -> bool: # Load Merlin Server Configuration and apply it to app.yaml with resources.path("merlin.server", MERLIN_SERVER_CONFIG) as merlin_server_config: - with open(merlin_server_config) as f: + with open(merlin_server_config) as f: # pylint: disable=C0103 main_server_config = yaml.load(f, yaml.Loader) filename = LOCAL_APP_YAML if os.path.exists(LOCAL_APP_YAML) else AppYaml.default_filename merlin_app_yaml = AppYaml(filename) @@ -211,7 +214,7 @@ def config_merlin_server(): # else: password = generate_password(PASSWORD_LENGTH) - with open(pass_file, "w+") as f: + with open(pass_file, "w+") as f: # pylint: disable=C0103 f.write(password) LOG.info("Creating password file for merlin server container.") @@ -228,7 +231,9 @@ def config_merlin_server(): redis_users.write() redis_config.write() - LOG.info("User {} created in user file for merlin server container".format(os.environ.get("USER"))) + LOG.info(f"User {os.environ.get('USER')} created in user file for merlin server container") + + return None def pull_server_config() -> ServerConfig: @@ -251,7 +256,7 @@ def pull_server_config() -> ServerConfig: if "container" in server_config: if "format" in server_config["container"]: format_file = os.path.join(config_dir, server_config["container"]["format"] + ".yaml") - with open(format_file, "r") as ff: + with open(format_file, "r") as ff: # pylint: disable=C0103 format_data = yaml.load(ff, yaml.Loader) for key in format_needed_keys: if key not in format_data[server_config["container"]["format"]]: @@ -378,7 +383,7 @@ def pull_process_file(file_path: str) -> dict: if not returns None :return:: Data containing in process file. """ - with open(file_path, "r") as f: + with open(file_path, "r") as f: # pylint: disable=C0103 data = yaml.load(f, yaml.Loader) if check_process_file_format(data): return data @@ -392,6 +397,6 @@ def dump_process_file(data: dict, file_path: str): """ if not check_process_file_format(data): return False - with open(file_path, "w+") as f: + with open(file_path, "w+") as f: # pylint: disable=C0103 yaml.dump(data, f, yaml.Dumper) return True diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 65f0b2abb..55c475f31 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -27,6 +27,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ############################################################################### +"""Utils relating to merlin server""" import hashlib import logging @@ -47,7 +48,7 @@ MERLIN_SERVER_CONFIG = "merlin_server.yaml" -def valid_ipv4(ip: str) -> bool: +def valid_ipv4(ip: str) -> bool: # pylint: disable=C0103 """ Checks valid ip address """ @@ -69,12 +70,13 @@ def valid_port(port: int) -> bool: """ Checks valid network port """ - if port > 0 and port < 65536: + if 0 < port < 65536: return True return False -class ContainerConfig: +# Pylint complains about too many instance variables but it's necessary here so ignore +class ContainerConfig: # pylint: disable=R0902 """ ContainerConfig provides interface for parsing and interacting with the container value specified within the merlin_server.yaml configuration file. Dictionary of the config values should be passed when initialized @@ -120,50 +122,65 @@ def __init__(self, data: dict) -> None: self.user_file = data["user_file"] if "user_file" in data else self.USERS_FILE def get_format(self) -> str: + """Getter method to get the container format""" return self.format def get_image_type(self) -> str: + """Getter method to get the image type""" return self.image_type def get_image_name(self) -> str: + """Getter method to get the image name""" return self.image def get_image_url(self) -> str: + """Getter method to get the image url""" return self.url def get_image_path(self) -> str: + """Getter method to get the path to the image""" return os.path.join(self.config_dir, self.image) def get_config_name(self) -> str: + """Getter method to get the configuration file name""" return self.config def get_config_path(self) -> str: + """Getter method to get the configuration file path""" return os.path.join(self.config_dir, self.config) def get_config_dir(self) -> str: + """Getter method to get the configuration directory""" return self.config_dir def get_pfile_name(self) -> str: + """Getter method to get the process file name""" return self.pfile def get_pfile_path(self) -> str: + """Getter method to get the process file path""" return os.path.join(self.config_dir, self.pfile) def get_pass_file_name(self) -> str: + """Getter method to get the password file name""" return self.pass_file def get_pass_file_path(self) -> str: + """Getter method to get the password file path""" return os.path.join(self.config_dir, self.pass_file) def get_user_file_name(self) -> str: + """Getter method to get the user file name""" return self.user_file def get_user_file_path(self) -> str: + """Getter method to get the user file path""" return os.path.join(self.config_dir, self.user_file) def get_container_password(self) -> str: + """Getter method to get the container password""" password = None - with open(self.get_pass_file_path(), "r") as f: + with open(self.get_pass_file_path(), "r") as f: # pylint: disable=C0103 password = f.read() return password @@ -192,15 +209,19 @@ def __init__(self, data: dict) -> None: self.pull_command = data["pull_command"] if "pull_command" in data else self.PULL_COMMAND def get_command(self) -> str: + """Getter method to get the container command""" return self.command def get_run_command(self) -> str: + """Getter method to get the run command""" return self.run_command def get_stop_command(self) -> str: + """Getter method to get the stop command""" return self.stop_command def get_pull_command(self) -> str: + """Getter method to get the pull command""" return self.pull_command @@ -222,13 +243,17 @@ def __init__(self, data: dict) -> None: self.kill = data["kill"] if "kill" in data else self.KILL_COMMAND def get_status_command(self) -> str: + """Getter method to get the status command""" return self.status def get_kill_command(self) -> str: + """Getter method to get the kill command""" return self.kill -class ServerConfig: +# Pylint complains there's not enough methods here but this is essentially a wrapper for other +# classes so we can ignore it +class ServerConfig: # pylint: disable=R0903 """ ServerConfig is an interface for storing all the necessary configuration for merlin server. These configuration container things such as ContainerConfig, ProcessConfig, and ContainerFormatConfig. @@ -267,9 +292,10 @@ def __init__(self, filename) -> None: self.parse() def parse(self) -> None: + """Parses the redis configuration file""" self.entries = {} self.comments = {} - with open(self.filename, "r+") as f: + with open(self.filename, "r+") as f: # pylint: disable=C0103 file_contents = f.read() file_lines = file_contents.split("\n") comments = "" @@ -289,16 +315,19 @@ def parse(self) -> None: self.trailing_comments = comments[:-1] def write(self) -> None: - with open(self.filename, "w") as f: + """Writes to the redis configuration file""" + with open(self.filename, "w") as f: # pylint: disable=C0103 for entry in self.entry_order: f.write(self.comments[entry]) f.write(f"{entry} {self.entries[entry]}\n") f.write(self.trailing_comments) def set_filename(self, filename: str) -> None: + """Setter method to set the filename""" self.filename = filename def set_config_value(self, key: str, value: str) -> bool: + """Changes a configuration value""" if key not in self.entries: return False self.entries[key] = value @@ -306,17 +335,21 @@ def set_config_value(self, key: str, value: str) -> bool: return True def get_config_value(self, key: str) -> str: + """Given an entry in the config, get the value""" if key in self.entries: return self.entries[key] return None def changes_made(self) -> bool: + """Getter method to get the changes made""" return self.changed def get_ip_address(self) -> str: + """Getter method to get the ip from the redis config""" return self.get_config_value("bind") def set_ip_address(self, ipaddress: str) -> bool: + """Validates and sets a given ip address""" if ipaddress is None: return False # Check if ipaddress is valid @@ -332,9 +365,11 @@ def set_ip_address(self, ipaddress: str) -> bool: return True def get_port(self) -> str: + """Getter method to get the port from the redis config""" return self.get_config_value("port") def set_port(self, port: str) -> bool: + """Validates and sets a given port""" if port is None: return False # Check if port is valid @@ -350,6 +385,7 @@ def set_port(self, port: str) -> bool: return True def set_password(self, password: str) -> bool: + """Changes the password""" if password is None: return False self.set_config_value("requirepass", password) @@ -357,9 +393,14 @@ def set_password(self, password: str) -> bool: return True def get_password(self) -> str: + """Getter method to get the config password""" return self.get_config_value("requirepass") def set_directory(self, directory: str) -> bool: + """ + Sets the save directory in the redis config file. + Creates the directory if necessary. + """ if directory is None: return False if not os.path.exists(directory): @@ -378,6 +419,7 @@ def set_directory(self, directory: str) -> bool: return True def set_snapshot_seconds(self, seconds: int) -> bool: + """Sets the snapshot wait time""" if seconds is None: return False # Set the snapshot second in the redis config @@ -385,17 +427,19 @@ def set_snapshot_seconds(self, seconds: int) -> bool: if value is None: LOG.error("Unable to get exisiting parameter values for snapshot") return False - else: - value = value.split() - value[0] = str(seconds) - value = " ".join(value) - if not self.set_config_value("save", value): - LOG.error("Unable to set snapshot value seconds") - return False + + value = value.split() + value[0] = str(seconds) + value = " ".join(value) + if not self.set_config_value("save", value): + LOG.error("Unable to set snapshot value seconds") + return False + LOG.info(f"Snapshot wait time is set to {seconds} seconds") return True def set_snapshot_changes(self, changes: int) -> bool: + """Sets the snapshot threshold""" if changes is None: return False # Set the snapshot changes into the redis config @@ -403,17 +447,19 @@ def set_snapshot_changes(self, changes: int) -> bool: if value is None: LOG.error("Unable to get exisiting parameter values for snapshot") return False - else: - value = value.split() - value[1] = str(changes) - value = " ".join(value) - if not self.set_config_value("save", value): - LOG.error("Unable to set snapshot value seconds") - return False + + value = value.split() + value[1] = str(changes) + value = " ".join(value) + if not self.set_config_value("save", value): + LOG.error("Unable to set snapshot value seconds") + return False + LOG.info(f"Snapshot threshold is set to {changes} changes") return True def set_snapshot_file(self, file: str) -> bool: + """Sets the snapshot file""" if file is None: return False # Set the snapshot file in the redis config @@ -425,6 +471,7 @@ def set_snapshot_file(self, file: str) -> bool: return True def set_append_mode(self, mode: str) -> bool: + """Sets the append mode""" if mode is None: return False valid_modes = ["always", "everysec", "no"] @@ -443,6 +490,7 @@ def set_append_mode(self, mode: str) -> bool: return True def set_append_file(self, file: str) -> bool: + """Sets the append file""" if file is None: return False # Set the append file in the redis config @@ -461,13 +509,17 @@ class RedisUsers: """ class User: + """Embedded class to store user specific information""" + status = "on" hash_password = hashlib.sha256(b"password").hexdigest() keys = "*" channels = "*" commands = "@all" - def __init__(self, status="on", keys="*", channels="*", commands="@all", password=None) -> None: + def __init__( # pylint: disable=R0913 + self, status="on", keys="*", channels="*", commands="@all", password=None + ) -> None: self.status = status self.keys = keys self.channels = channels @@ -475,14 +527,20 @@ def __init__(self, status="on", keys="*", channels="*", commands="@all", passwor if password is not None: self.set_password(password) - def parse_dict(self, dict: dict) -> None: - self.status = dict["status"] - self.keys = dict["keys"] - self.channels = dict["channels"] - self.commands = dict["commands"] - self.hash_password = dict["hash_password"] + def parse_dict(self, dictionary: dict) -> None: + """ + Given a dict of user info, parse the dict and store + the values as class attributes. + :param `dictionary`: The dict to parse + """ + self.status = dictionary["status"] + self.keys = dictionary["keys"] + self.channels = dictionary["channels"] + self.commands = dictionary["commands"] + self.hash_password = dictionary["hash_password"] def get_user_dict(self) -> dict: + """Getter method to get the user info""" self.status = "on" return { "status": self.status, @@ -493,12 +551,15 @@ def get_user_dict(self) -> dict: } def __repr__(self) -> str: + """Repr magic method for User class""" return str(self.get_user_dict()) def __str__(self) -> str: + """Str magic method for User class""" return self.__repr__() def set_password(self, password: str) -> None: + """Setter method to set the user's hash password""" self.hash_password = hashlib.sha256(bytes(password, "utf-8")).hexdigest() filename = "" @@ -510,7 +571,8 @@ def __init__(self, filename) -> None: self.parse() def parse(self) -> None: - with open(self.filename, "r") as f: + """Parses the redis user configuration file""" + with open(self.filename, "r") as f: # pylint: disable=C0103 self.users = yaml.load(f, yaml.Loader) for user in self.users: new_user = self.User() @@ -518,36 +580,44 @@ def parse(self) -> None: self.users[user] = new_user def write(self) -> None: + """Writes to the redis user configuration file""" data = self.users.copy() for key in data: data[key] = self.users[key].get_user_dict() - with open(self.filename, "w") as f: + with open(self.filename, "w") as f: # pylint: disable=C0103 yaml.dump(data, f, yaml.Dumper) - def add_user(self, user, status="on", keys="*", channels="*", commands="@all", password=None) -> bool: + def add_user( # pylint: disable=R0913 + self, user, status="on", keys="*", channels="*", commands="@all", password=None + ) -> bool: + """Add a user to the dict of Redis users""" if user in self.users: return False self.users[user] = self.User(status, keys, channels, commands, password) return True def set_password(self, user: str, password: str): + """Set the password for a specific user""" if user not in self.users: return False self.users[user].set_password(password) + return True def remove_user(self, user) -> bool: + """Remove a user from the dict of users""" if user in self.users: del self.users[user] return True return False def apply_to_redis(self, host: str, port: int, password: str) -> None: - db = redis.Redis(host=host, port=port, password=password) - current_users = db.acl_users() + """Apply the changes to users to redis""" + database = redis.Redis(host=host, port=port, password=password) + current_users = database.acl_users() for user in self.users: if user not in current_users: data = self.users[user] - db.acl_setuser( + database.acl_setuser( username=user, hashed_passwords=[f"+{data.hash_password}"], enabled=(data.status == "on"), @@ -558,7 +628,7 @@ def apply_to_redis(self, host: str, port: int, password: str) -> None: for user in current_users: if user not in self.users: - db.acl_deluser(user) + database.acl_deluser(user) class AppYaml: @@ -579,29 +649,34 @@ def __init__(self, filename: str = default_filename) -> None: self.read(filename) def apply_server_config(self, server_config: ServerConfig): - rc = RedisConfig(server_config.container.get_config_path()) + """Store the redis configuration""" + redis_config = RedisConfig(server_config.container.get_config_path()) self.data[self.broker_name]["name"] = server_config.container.get_image_type() self.data[self.broker_name]["username"] = "default" self.data[self.broker_name]["password"] = server_config.container.get_pass_file_path() - self.data[self.broker_name]["server"] = rc.get_ip_address() - self.data[self.broker_name]["port"] = rc.get_port() + self.data[self.broker_name]["server"] = redis_config.get_ip_address() + self.data[self.broker_name]["port"] = redis_config.get_port() self.data[self.results_name]["name"] = server_config.container.get_image_type() self.data[self.results_name]["username"] = "default" self.data[self.results_name]["password"] = server_config.container.get_pass_file_path() - self.data[self.results_name]["server"] = rc.get_ip_address() - self.data[self.results_name]["port"] = rc.get_port() + self.data[self.results_name]["server"] = redis_config.get_ip_address() + self.data[self.results_name]["port"] = redis_config.get_port() def update_data(self, new_data: dict): + """Update the data dict with new entries""" self.data.update(new_data) def get_data(self): + """Getter method to obtain the data""" return self.data def read(self, filename: str = default_filename): + """Load in a yaml file and save it to the data attribute""" self.data = merlin.utils.load_yaml(filename) def write(self, filename: str = default_filename): - with open(filename, "w+") as f: + """Given a filename, dump the data to the file""" + with open(filename, "w+") as f: # pylint: disable=C0103 yaml.dump(self.data, f, yaml.Dumper) diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index ac0031823..5da1fed22 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -27,6 +27,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ############################################################################### +"""This module defines all the keys possible in each block of a merlin spec file""" DESCRIPTION = {"description", "name"} diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 7b54e5200..a1794bed2 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -27,6 +27,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ############################################################################### +"""This module defines the default values of every block in the merlin spec""" DESCRIPTION = {"description": {}} diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index f4ea42e24..a76c56c16 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -27,6 +27,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ############################################################################### +"""This module handles expanding variables in the merlin spec""" import logging from collections import ChainMap @@ -123,10 +124,10 @@ def recurse(section): if isinstance(section, str): return expandvars(expanduser(section)) if isinstance(section, dict): - for k, v in section.items(): - if k in ["cmd", "restart"]: + for key, val in section.items(): + if key in ["cmd", "restart"]: continue - section[k] = recurse(v) + section[key] = recurse(val) elif isinstance(section, list): for i, elem in enumerate(deepcopy(section)): section[i] = recurse(elem) @@ -164,10 +165,10 @@ def determine_user_variables(*user_var_dicts): raise ValueError(f"Cannot reassign value of reserved word '{key}'! Reserved words are: {RESERVED}.") new_val = str(val) if contains_token(new_val): - for determined_key in determined_results.keys(): + for determined_key, determined_val in determined_results.items(): var_determined_key = var_ref(determined_key) if var_determined_key in new_val: - new_val = new_val.replace(var_determined_key, determined_results[determined_key]) + new_val = new_val.replace(var_determined_key, determined_val) new_val = expandvars(expanduser(new_val)) determined_results[key.upper()] = new_val return determined_results @@ -217,6 +218,9 @@ def parameter_substitutions_for_cmd(glob_path, sample_paths): return substitutions +# There's similar code inside study.py but the whole point of this function is to not use +# the MerlinStudy object so we disable this pylint error +# pylint: disable=duplicate-code def expand_spec_no_study(filepath, override_vars=None): """ Get the expanded text of a spec without creating @@ -239,6 +243,9 @@ def expand_spec_no_study(filepath, override_vars=None): return expand_by_line(spec_text, evaluated_uvars) +# pylint: enable=duplicate-code + + def get_spec_with_expansion(filepath, override_vars=None): """ Return a MerlinSpec with overrides and expansion, without diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 4c9291693..2c3194b50 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -27,6 +27,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ############################################################################### +"""This module handles overriding variables in a spec file via the CLI""" import logging from copy import deepcopy @@ -49,6 +50,7 @@ def error_override_vars(override_vars, spec_filepath): def replace_override_vars(env, override_vars): + """Replace override variables in the environment block""" if override_vars is None: return env result = deepcopy(env) diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 65326d54f..45d456067 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -37,6 +37,7 @@ import logging import os import shlex +from copy import deepcopy from io import StringIO import yaml @@ -48,6 +49,7 @@ LOG = logging.getLogger(__name__) +# Pylint complains we have too many instance attributes but it's fine class MerlinSpec(YAMLSpecification): # pylint: disable=R0902 """ This class represents the logic for parsing the Merlin yaml @@ -67,8 +69,9 @@ class MerlinSpec(YAMLSpecification): # pylint: disable=R0902 column_labels: [X0, X1] """ + # Pylint says this call to super is useless but we'll leave it in case we want to add to __init__ in the future def __init__(self): # pylint: disable=W0246 - super(MerlinSpec, self).__init__() # pylint: disable=R1725 + super().__init__() @property def yaml_sections(self): @@ -123,19 +126,19 @@ def __str__(self): return result @classmethod - def load_specification(cls, filepath, suppress_warning=True): # pylint: disable=W0237 + def load_specification(cls, path, suppress_warning=True): """ Load in a spec file and create a MerlinSpec object based on its' contents. :param `cls`: The class reference (like self) - :param `filepath`: A path to the spec file we're loading in + :param `path`: A path to the spec file we're loading in :param `suppress_warning`: A bool representing whether to warn the user about unrecognized keys :returns: A MerlinSpec object """ - LOG.info("Loading specification from path: %s", filepath) + LOG.info("Loading specification from path: %s", path) try: - # Load the YAML spec from the filepath - with open(filepath, "r") as data: + # Load the YAML spec from the path + with open(path, "r") as data: spec = cls.load_spec_from_string(data, needs_IO=False, needs_verification=True) except Exception as e: # pylint: disable=C0103 LOG.exception(e.args) @@ -143,7 +146,7 @@ def load_specification(cls, filepath, suppress_warning=True): # pylint: disable # Path not set in _populate_spec because loading spec with string # does not have a path so we set it here - spec.path = filepath + spec.path = path spec.specroot = os.path.dirname(spec.path) # pylint: disable=W0201 if not suppress_warning: @@ -306,24 +309,23 @@ def verify_batch_block(self, schema): YAMLSpecification.validate_schema("batch", self.batch, schema) # Additional Walltime checks in case the regex from the schema bypasses an error - if "walltime" in self.batch: # pylint: disable=R1702 - if self.batch["type"] == "lsf": - LOG.warning("The walltime argument is not available in lsf.") - else: - try: - err_msg = "Walltime must be of the form SS, MM:SS, or HH:MM:SS." - walltime = self.batch["walltime"] - if len(walltime) > 2: - # Walltime must have : if it's not of the form SS - if ":" not in walltime: + if self.batch["type"] == "lsf" and "walltime" in self.batch: + LOG.warning("The walltime argument is not available in lsf.") + elif "walltime" in self.batch: + try: + err_msg = "Walltime must be of the form SS, MM:SS, or HH:MM:SS." + walltime = self.batch["walltime"] + if len(walltime) > 2: + # Walltime must have : if it's not of the form SS + if ":" not in walltime: + raise ValueError(err_msg) + # Walltime must have exactly 2 chars between : + time = walltime.split(":") + for section in time: + if len(section) != 2: raise ValueError(err_msg) - # Walltime must have exactly 2 chars between : - time = walltime.split(":") - for section in time: - if len(section) != 2: - raise ValueError(err_msg) - except Exception: # pylint: disable=W0706 - raise + except Exception: # pylint: disable=W0706 + raise @staticmethod def load_merlin_block(stream): @@ -407,13 +409,13 @@ def fill_missing_defaults(object_to_update, default_dict): existing ones. """ - def recurse(result, defaults): # pylint: disable=W0621 - if not isinstance(defaults, dict): + def recurse(result, recurse_defaults): + if not isinstance(recurse_defaults, dict): return - for key, val in defaults.items(): + for key, val in recurse_defaults.items(): # fmt: off if (key not in result) or ( - (result[key] is None) and (defaults[key] is not None) + (result[key] is None) and (recurse_defaults[key] is not None) ): result[key] = val else: @@ -454,9 +456,9 @@ def warn_unrecognized_keys(self): # user block is not checked @staticmethod - def check_section(section_name, section, all_keys): # pylint: disable=W0621 + def check_section(section_name, section, known_keys): """Checks a section of the spec file to see if there are any unrecognized keys""" - diff = set(section.keys()).difference(all_keys) + diff = set(section.keys()).difference(known_keys) # TODO: Maybe add a check here for required keys @@ -474,7 +476,7 @@ def dump(self): try: yaml.safe_load(result) except Exception as e: # pylint: disable=C0103 - raise ValueError(f"Error parsing provenance spec:\n{e}") # pylint: disable=W0707 + raise ValueError(f"Error parsing provenance spec:\n{e}") from e return result def _dict_to_yaml(self, obj, string, key_stack, tab): @@ -490,9 +492,11 @@ def _dict_to_yaml(self, obj, string, key_stack, tab): return self._process_string(obj, lvl, tab) if isinstance(obj, bool): return str(obj).lower() - if not isinstance(obj, (list, dict)): - return obj - return self._process_dict_or_list(obj, string, key_stack, lvl, tab) + if isinstance(obj, list): + return self._process_list(obj, string, key_stack, lvl, tab) + if isinstance(obj, dict): + return self._process_dict(obj, string, key_stack, lvl, tab) + return obj def _process_string(self, obj, lvl, tab): """ @@ -503,50 +507,51 @@ def _process_string(self, obj, lvl, tab): obj = "|\n" + tab * (lvl + 1) + ("\n" + tab * (lvl + 1)).join(split) return obj - def _process_dict_or_list(self, obj, string, key_stack, lvl, tab): # pylint: disable=R0912,R0913 + def _process_list(self, obj, string, key_stack, lvl, tab): # pylint: disable=R0913 """ - Processes lists and dicts for _dict_to_yaml() in the dump() method. + Processes lists for _dict_to_yaml() in the dump() method. """ - from copy import deepcopy # pylint: disable=C0415 + num_entries = len(obj) + use_hyphens = key_stack[-1] in ["paths", "sources", "git", "study"] or key_stack[0] in ["user"] + if not use_hyphens: + string += "[" + else: + string += "\n" + for i, elem in enumerate(obj): + key_stack = deepcopy(key_stack) + key_stack.append("elem") + if use_hyphens: + string += (lvl + 1) * tab + "- " + str(self._dict_to_yaml(elem, "", key_stack, tab)) + "\n" + else: + string += str(self._dict_to_yaml(elem, "", key_stack, tab)) + if num_entries > 1 and i != len(obj) - 1: + string += ", " + key_stack.pop() + if not use_hyphens: + string += "]" + return string + def _process_dict(self, obj, string, key_stack, lvl, tab): # pylint: disable=R0913 + """ + Processes dicts for _dict_to_yaml() in the dump() method + """ list_offset = 2 * " " - if isinstance(obj, list): - num_entries = len(obj) - use_hyphens = key_stack[-1] in ["paths", "sources", "git", "study"] or key_stack[0] in ["user"] - if not use_hyphens: - string += "[" + if len(key_stack) > 0 and key_stack[-1] != "elem": + string += "\n" + i = 0 + for key, val in obj.items(): + key_stack = deepcopy(key_stack) + key_stack.append(key) + if len(key_stack) > 1 and key_stack[-2] == "elem" and i == 0: + # string += (tab * (lvl - 1)) + string += "" + elif "elem" in key_stack: + string += list_offset + (tab * lvl) else: - string += "\n" - for i, elem in enumerate(obj): - key_stack = deepcopy(key_stack) - key_stack.append("elem") - if use_hyphens: - string += (lvl + 1) * tab + "- " + str(self._dict_to_yaml(elem, "", key_stack, tab)) + "\n" - else: - string += str(self._dict_to_yaml(elem, "", key_stack, tab)) - if num_entries > 1 and i != len(obj) - 1: - string += ", " - key_stack.pop() - if not use_hyphens: - string += "]" - # must be dict - else: - if len(key_stack) > 0 and key_stack[-1] != "elem": - string += "\n" - i = 0 - for key, val in obj.items(): - key_stack = deepcopy(key_stack) - key_stack.append(key) - if len(key_stack) > 1 and key_stack[-2] == "elem" and i == 0: - # string += (tab * (lvl - 1)) - string += "" - elif "elem" in key_stack: - string += list_offset + (tab * lvl) - else: - string += tab * (lvl + 1) - string += str(key) + ": " + str(self._dict_to_yaml(val, "", key_stack, tab)) + "\n" - key_stack.pop() - i += 1 + string += tab * (lvl + 1) + string += str(key) + ": " + str(self._dict_to_yaml(val, "", key_stack, tab)) + "\n" + key_stack.pop() + i += 1 return string def get_step_worker_map(self): diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 81f0762f8..8daaa1fe0 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -51,8 +51,10 @@ def run_celery(study, run_mode=None): configure Celery to run locally (without workers). """ # Only import celery stuff if we want celery in charge + # Pylint complains about circular import between merlin.common.tasks -> merlin.router -> merlin.study.celeryadapter + # For now I think this is still the best way to do this so we'll ignore it from merlin.celery import app # pylint: disable=C0415 - from merlin.common.tasks import queue_merlin_study # pylint: disable=C0415 + from merlin.common.tasks import queue_merlin_study # pylint: disable=C0415, R0401 adapter_config = study.get_adapter_config(override_type="local") @@ -468,7 +470,7 @@ def purge_celery_tasks(queues, force): return subprocess.run(purge_command, shell=True).returncode -def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): +def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): # pylint: disable=R0912 """Send a stop command to celery workers. Default behavior is to stop all connected workers. diff --git a/merlin/study/dag.py b/merlin/study/dag.py index f7ba3532f..7281875f5 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -205,11 +205,9 @@ def find_independent_chains(self, list_of_groups_of_chains): if self.num_children(task_name) == 1 and task_name != "_source": child = self.children(task_name)[0] - if self.num_parents(child) == 1: - if self.compatible_merlin_expansion(child, task_name): - self.find_chain(child, list_of_groups_of_chains).remove(child) - - chain.append(child) + if self.num_parents(child) == 1 and self.compatible_merlin_expansion(child, task_name): + self.find_chain(child, list_of_groups_of_chains).remove(child) + chain.append(child) new_list = [[chain for chain in group if len(chain) > 0] for group in list_of_groups_of_chains] new_list_2 = [group for group in new_list if len(group) > 0] diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 027ebaff9..ad6843c51 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -65,7 +65,7 @@ def __init__(self, **kwargs): :param **kwargs: A dictionary with default settings for the adapter. """ - super(MerlinLSFScriptAdapter, self).__init__(**kwargs) + super().__init__(**kwargs) self._cmd_flags: Dict[str, str] = { "cmd": "jsrun", @@ -99,6 +99,9 @@ def __init__(self, **kwargs): "walltime", } + def get_priority(self, priority): + """This is implemented to override the abstract method and fix a pylint error""" + def get_header(self, step): """ Generate the header present at the top of LSF execution scripts. @@ -107,7 +110,7 @@ def get_header(self, step): :returns: A string of the header based on internal batch parameters and the parameter step. """ - return "#!{}".format(self._exec) + return f"#!{self._exec}" def get_parallelize_command(self, procs, nodes=None, **kwargs): """ @@ -149,7 +152,7 @@ def get_parallelize_command(self, procs, nodes=None, **kwargs): LOG.warning("'%s' is not supported -- ommitted.", key) continue if value: - args += [self._cmd_flags[key], "{}".format(str(value))] + args += [self._cmd_flags[key], f"{str(value)}"] return " ".join(args) @@ -171,7 +174,7 @@ def __init__(self, **kwargs): :param **kwargs: A dictionary with default settings for the adapter. """ - super(MerlinSlurmScriptAdapter, self).__init__(**kwargs) + super().__init__(**kwargs) self._cmd_flags["slurm"] = "" self._cmd_flags["walltime"] = "-t" @@ -192,6 +195,9 @@ def __init__(self, **kwargs): ] self._unsupported: Set[str] = set(list(self._unsupported) + new_unsupported) + def get_priority(self, priority): + """This is implemented to override the abstract method and fix a pylint error""" + def get_header(self, step): """ Generate the header present at the top of Slurm execution scripts. @@ -200,7 +206,7 @@ def get_header(self, step): :returns: A string of the header based on internal batch parameters and the parameter step. """ - return "#!{}".format(self._exec) + return f"#!{self._exec}" def time_format(self, val): """ @@ -241,35 +247,16 @@ def get_parallelize_command(self, procs, nodes=None, **kwargs): if key == "walltime": args += [ self._cmd_flags[key], - "{}".format(str(self.time_format(value))), + f"{str(self.time_format(value))}", ] elif "=" in self._cmd_flags[key]: - args += ["{0}{1}".format(self._cmd_flags[key], str(value))] + args += [f"{self._cmd_flags[key]}{str(value)}"] else: - args += [self._cmd_flags[key], "{}".format(str(value))] + args += [self._cmd_flags[key], f"{str(value)}"] return " ".join(args) -class MerlinLSFSrunScriptAdapter(MerlinSlurmScriptAdapter): - """ - A SchedulerScriptAdapter class for lsf blocking parallel launches, using the srun wrapper - """ - - key = "merlin-lsf-srun" - - def __init__(self, **kwargs): - """ - Initialize an instance of the MerinLSFSrunScriptAdapter. - The MerlinLSFSrunScriptAdapter is the adapter that is used for workflows that - will execute LSF parallel jobs in a celery worker with an srun wrapper. The only - configurable aspect to this adapter is the shell that scripts are executed in. - - :param **kwargs: A dictionary with default settings for the adapter. - """ - super(MerlinLSFSrunScriptAdapter, self).__init__(**kwargs) - - class MerlinFluxScriptAdapter(MerlinSlurmScriptAdapter): """ A SchedulerScriptAdapter class for flux blocking parallel launches, @@ -288,7 +275,7 @@ def __init__(self, **kwargs): :param **kwargs: A dictionary with default settings for the adapter. """ flux_command = kwargs.pop("flux_command", "flux mini run") - super(MerlinFluxScriptAdapter, self).__init__(**kwargs) + super().__init__(**kwargs) # "cmd": "flux mini run", self._cmd_flags = { @@ -323,6 +310,9 @@ def __init__(self, **kwargs): ] self._unsupported = set(new_unsupported) # noqa + def get_priority(self, priority): + """This is implemented to override the abstract method and fix a pylint error""" + def time_format(self, val): """ Convert a time format to flux standard designation. @@ -346,19 +336,19 @@ def __init__(self, **kwargs): :param **kwargs: A dictionary with default settings for the adapter. """ - super(MerlinScriptAdapter, self).__init__(**kwargs) + super().__init__(**kwargs) self.batch_type = "merlin-" + kwargs.get("batch_type", "local") - if "host" not in kwargs.keys(): + if "host" not in kwargs: kwargs["host"] = "None" - if "bank" not in kwargs.keys(): + if "bank" not in kwargs: kwargs["bank"] = "None" - if "queue" not in kwargs.keys(): + if "queue" not in kwargs: kwargs["queue"] = "None" # Using super prevents recursion. - self.batch_adapter = super(MerlinScriptAdapter, self) + self.batch_adapter = super() if self.batch_type != "merlin-local": self.batch_adapter = MerlinScriptAdapterFactory.get_adapter(self.batch_type)(**kwargs) @@ -369,7 +359,8 @@ def write_script(self, *args, **kwargs): _, script, restart_script = self.batch_adapter.write_script(*args, **kwargs) return True, script, restart_script - def submit(self, step, path, cwd, job_map=None, env=None): + # Pylint complains that there's too many arguments but it's fine in this case + def submit(self, step, path, cwd, job_map=None, env=None): # pylint: disable=R0913 """ Execute the step locally. If cwd is specified, the submit method will operate outside of the path @@ -387,8 +378,8 @@ def submit(self, step, path, cwd, job_map=None, env=None): """ LOG.debug("cwd = %s", cwd) LOG.debug("Script to execute: %s", path) - LOG.debug("starting process %s in cwd %s called %s" % (path, cwd, step.name)) - submission_record = self._execute_subprocess(step.name, path, cwd, env, False) + LOG.debug(f"starting process {path} in cwd {cwd} called {step.name}") + submission_record = self._execute_subprocess(step.name, path, cwd, env=env, join_output=False) retcode = submission_record.return_code if retcode == ReturnCode.OK: LOG.debug("Execution returned status OK.") @@ -406,17 +397,21 @@ def submit(self, step, path, cwd, job_map=None, env=None): LOG.debug("Execution returned status STOP_WORKERS") else: LOG.warning(f"Unrecognized Merlin Return code: {retcode}, returning SOFT_FAIL") - submission_record._info["retcode"] = retcode + submission_record.add_info("retcode", retcode) retcode = ReturnCode.SOFT_FAIL # Currently, we use Maestro's execute method, which is returning the # submission code we want it to return the return code, so we are # setting it in here. - submission_record._subcode = retcode + # TODO: In the refactor/status branch we're overwriting Maestro's execute method (I think) so + # we should be able to change this (i.e. add code in the overridden execute and remove this line) + submission_record._subcode = retcode # pylint: disable=W0212 return submission_record - def _execute_subprocess(self, output_name, script_path, cwd, env=None, join_output=False): + # TODO is there currently ever a scenario where join output is True? We should look into this + # Pylint is complaining there's too many local variables and args but it makes this function cleaner so ignore + def _execute_subprocess(self, output_name, script_path, cwd, env=None, join_output=False): # pylint: disable=R0913,R0914 """ Execute the subprocess script locally. If cwd is specified, the submit method will operate outside of the path @@ -435,16 +430,14 @@ def _execute_subprocess(self, output_name, script_path, cwd, env=None, join_outp script_bn = os.path.basename(script_path) new_output_name = os.path.splitext(script_bn)[0] LOG.debug(f"script_path={script_path}, output_name={output_name}, new_output_name={new_output_name}") - p = start_process(script_path, shell=False, cwd=cwd, env=env) - pid = p.pid - output, err = p.communicate() - retcode = p.wait() + process = start_process(script_path, shell=False, cwd=cwd, env=env) + output, err = process.communicate() + retcode = process.wait() # This allows us to save on iNodes by not writing the output, # or by appending error to output if output_name is not None: - o_path = os.path.join(cwd, "{}.out".format(new_output_name)) - + o_path = os.path.join(cwd, f"{new_output_name}.out") with open(o_path, "a") as out: out.write(output) @@ -453,40 +446,42 @@ def _execute_subprocess(self, output_name, script_path, cwd, env=None, join_outp out.write(err) if not join_output: - e_path = os.path.join(cwd, "{}.err".format(new_output_name)) + e_path = os.path.join(cwd, f"{new_output_name}.err") with open(e_path, "a") as out: out.write(err) if retcode == 0: LOG.info("Execution returned status OK.") - return SubmissionRecord(ReturnCode.OK, retcode, pid) - else: - _record = SubmissionRecord(ReturnCode.ERROR, retcode, pid) - _record.add_info("stderr", str(err)) - return _record + return SubmissionRecord(ReturnCode.OK, retcode, process.pid) + + _record = SubmissionRecord(ReturnCode.ERROR, retcode, process.pid) + _record.add_info("stderr", str(err)) + return _record + +class MerlinScriptAdapterFactory: + """This class routes to the correct ScriptAdapter""" -class MerlinScriptAdapterFactory(object): factories = { "merlin-flux": MerlinFluxScriptAdapter, "merlin-lsf": MerlinLSFScriptAdapter, - "merlin-lsf-srun": MerlinLSFSrunScriptAdapter, + "merlin-lsf-srun": MerlinSlurmScriptAdapter, "merlin-slurm": MerlinSlurmScriptAdapter, "merlin-local": MerlinScriptAdapter, } @classmethod def get_adapter(cls, adapter_id): + """Returns the appropriate ScriptAdapter to use""" if adapter_id.lower() not in cls.factories: - msg = ( - "Adapter '{0}' not found. Specify an adapter that exists " - "or implement a new one mapping to the '{0}'".format(str(adapter_id)) - ) + msg = f"""Adapter '{str(adapter_id)}' not found. Specify an adapter that exists + or implement a new one mapping to the '{str(adapter_id)}'""" LOG.error(msg) - raise Exception(msg) + raise ValueError(msg) return cls.factories[adapter_id] @classmethod def get_valid_adapters(cls): + """Returns the valid ScriptAdapters""" return cls.factories.keys() diff --git a/merlin/study/step.py b/merlin/study/step.py index 031302480..b80dd2766 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -27,6 +27,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ############################################################################### +"""This module represents all of the logic that goes into a step""" import logging import re @@ -78,7 +79,7 @@ def __init__(self, maestro_step_record): :param maestro_step_record: The StepRecord object. """ self.mstep = maestro_step_record - self.restart = False + self.__restart = False def get_cmd(self): """ @@ -133,7 +134,7 @@ def get_task_queue(self): @staticmethod def get_task_queue_from_dict(step_dict): """given a maestro step dict, get the task queue""" - from merlin.config.configfile import CONFIG + from merlin.config.configfile import CONFIG # pylint: disable=C0415 queue_tag = CONFIG.celery.queue_tag omit_tag = CONFIG.celery.omit_queue_tag @@ -153,6 +154,7 @@ def get_task_queue_from_dict(step_dict): @property def retry_delay(self): + """Returns the retry delay (default 1)""" default_retry_delay = 1 return self.mstep.step.__dict__["run"].get("retry_delay", default_retry_delay) @@ -163,20 +165,20 @@ def max_retries(self): """ return self.mstep.step.__dict__["run"]["max_retries"] - def __get_restart(self): + @property + def restart(self): """ - Set the restart property ensuring that restart is false + Get the restart property """ return self.__restart - def __set_restart(self, val): + @restart.setter + def restart(self, val): """ Set the restart property ensuring that restart is false """ self.__restart = val - restart = property(__get_restart, __set_restart) - def needs_merlin_expansion(self, labels): """ :return : True if the cmd has any of the default keywords or spec @@ -273,5 +275,5 @@ def execute(self, adapter_config): # calls to the step execute and restart functions. if self.restart and self.get_restart_cmd(): return ReturnCode(self.mstep.restart(adapter)) - else: - return ReturnCode(self.mstep.execute(adapter)) + + return ReturnCode(self.mstep.execute(adapter)) diff --git a/merlin/study/study.py b/merlin/study/study.py index 6bf07653f..9d016bc0c 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -27,6 +27,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ############################################################################### +"""This module represents all of the logic for a study""" import logging import os @@ -53,7 +54,10 @@ LOG = logging.getLogger(__name__) -class MerlinStudy: +# TODO: see if there's any way to split this class up (pylint doesn't like how many attributes there are) +# - Might be able to create an object to store files and handle file modifications +# - If we don't want to create entirely new classes we could try grouping args into dicts +class MerlinStudy: # pylint: disable=R0902 """ Represents a Merlin study run on a specification. Used for 'merlin run'. @@ -68,7 +72,7 @@ class MerlinStudy: :param `no_errors`: Flag to ignore some errors for testing. """ - def __init__( + def __init__( # pylint: disable=R0913 self, filepath, override_vars=None, @@ -108,9 +112,7 @@ def __init__( "MERLIN_HARD_FAIL": str(int(ReturnCode.HARD_FAIL)), "MERLIN_RETRY": str(int(ReturnCode.RETRY)), # below will be substituted for sample values on execution - "MERLIN_SAMPLE_VECTOR": " ".join( - ["$({})".format(k) for k in self.get_sample_labels(from_spec=self.original_spec)] - ), + "MERLIN_SAMPLE_VECTOR": " ".join([f"$({k})" for k in self.get_sample_labels(from_spec=self.original_spec)]), "MERLIN_SAMPLE_NAMES": " ".join(self.get_sample_labels(from_spec=self.original_spec)), "MERLIN_SPEC_ORIGINAL_TEMPLATE": os.path.join( self.info, @@ -151,6 +153,9 @@ def label_clash_error(self): if label in self.original_spec.globals: raise ValueError(f"column_label {label} cannot also be in global.parameters!") + # There's similar code inside expansion.py but the whole point of the function inside that file is + # to not use the MerlinStudy object so we disable this pylint error + # pylint: disable=duplicate-code @staticmethod def get_user_vars(spec): """ @@ -164,8 +169,11 @@ def get_user_vars(spec): uvars.append(spec.environment["labels"]) return determine_user_variables(*uvars) + # pylint: enable=duplicate-code + @property def user_vars(self): + """Get the user defined variables""" return MerlinStudy.get_user_vars(self.original_spec) def get_expanded_spec(self): @@ -200,6 +208,7 @@ def samples(self): return [] def get_sample_labels(self, from_spec): + """Return the column labels of the samples (if any)""" if from_spec.merlin["samples"]: return from_spec.merlin["samples"]["column_labels"] return [] @@ -289,19 +298,18 @@ def output_path(self): raise ValueError(f"Restart dir '{self.restart_dir}' does not exist!") return os.path.abspath(output_path) - else: - output_path = str(self.original_spec.output_path) + output_path = str(self.original_spec.output_path) - if (self.override_vars is not None) and ("OUTPUT_PATH" in self.override_vars): - output_path = str(self.override_vars["OUTPUT_PATH"]) + if (self.override_vars is not None) and ("OUTPUT_PATH" in self.override_vars): + output_path = str(self.override_vars["OUTPUT_PATH"]) - output_path = expand_line(output_path, self.user_vars, env_vars=True) - output_path = os.path.abspath(output_path) - if not os.path.isdir(output_path): - os.makedirs(output_path) - LOG.info(f"Made dir(s) to output path '{output_path}'.") + output_path = expand_line(output_path, self.user_vars, env_vars=True) + output_path = os.path.abspath(output_path) + if not os.path.isdir(output_path): + os.makedirs(output_path) + LOG.info(f"Made dir(s) to output path '{output_path}'.") - return output_path + return output_path @cached_property def timestamp(self): @@ -313,8 +321,10 @@ def timestamp(self): return self.restart_dir.strip("/")[-15:] return time.strftime("%Y%m%d-%H%M%S") + # TODO look into why pylint complains that this method is hidden + # - might be because we reset self.workspace's value in the expanded_spec method @cached_property - def workspace(self): + def workspace(self): # pylint: disable=E0202 """ Determines, makes, and returns the path to this study's workspace directory. This directory holds workspace directories @@ -334,8 +344,10 @@ def workspace(self): return workspace + # TODO look into why pylint complains that this method is hidden + # - might be because we reset self.info's value in the expanded_spec method @cached_property - def info(self): + def info(self): # pylint: disable=E0202 """ Creates the 'merlin_info' directory inside this study's workspace directory. """ @@ -401,7 +413,7 @@ def expanded_spec(self): ) # write expanded spec for provenance - with open(expanded_filepath, "w") as f: + with open(expanded_filepath, "w") as f: # pylint: disable=C0103 f.write(result.dump()) # write original spec for provenance @@ -417,7 +429,7 @@ def expanded_spec(self): if "labels" in result.environment: partial_spec.environment["labels"] = result.environment["labels"] partial_spec_path = os.path.join(self.info, name + ".partial.yaml") - with open(partial_spec_path, "w") as f: + with open(partial_spec_path, "w") as f: # pylint: disable=C0103 f.write(partial_spec.dump()) LOG.info(f"Study workspace is '{self.workspace}'.") @@ -452,37 +464,39 @@ def generate_samples(self): if not os.path.exists(self.samples_file): sample_generate = self.expanded_spec.merlin["samples"]["generate"]["cmd"] LOG.info("Generating samples...") - sample_process = subprocess.Popen( + sample_process = subprocess.Popen( # pylint: disable=R1732 sample_generate, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, ) stdout, stderr = sample_process.communicate() - with open(os.path.join(self.info, "cmd.sh"), "w") as f: + with open(os.path.join(self.info, "cmd.sh"), "w") as f: # pylint: disable=C0103 f.write(sample_generate) - with open(os.path.join(self.info, "cmd.out"), "wb") as f: + with open(os.path.join(self.info, "cmd.out"), "wb") as f: # pylint: disable=C0103 f.write(stdout) - with open(os.path.join(self.info, "cmd.err"), "wb") as f: + with open(os.path.join(self.info, "cmd.err"), "wb") as f: # pylint: disable=C0103 f.write(stderr) LOG.info("Generating samples complete!") return - except (IndexError, TypeError) as e: + except (IndexError, TypeError) as e: # pylint: disable=C0103 LOG.error(f"Could not generate samples:\n{e}") return def load_pgen(self, filepath, pargs, env): + """Creates a dict of variable names and values defined in a pgen script""" if filepath: if pargs is None: pargs = [] kwargs = create_dictionary(pargs) params = load_parameter_generator(filepath, env, kwargs) result = {} - for k, v in params.labels.items(): - result[k] = {"values": None, "label": v} - for k, v in params.parameters.items(): - result[k]["values"] = v + for key, val in params.labels.items(): + result[key] = {"values": None, "label": val} + for key, val in params.parameters.items(): + result[key]["values"] = val return result + return None def load_dag(self): """ @@ -526,6 +540,7 @@ def load_dag(self): self.dag = DAG(maestro_dag.adjacency_table, maestro_dag.values, labels) def get_adapter_config(self, override_type=None): + """Builds and returns the adapter configuration dictionary""" adapter_config = dict(self.expanded_spec.batch) if "type" not in adapter_config.keys(): diff --git a/merlin/utils.py b/merlin/utils.py index 48def3a10..dd8f69d29 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -78,8 +78,7 @@ def get_user_process_info(user=None, attrs=None): if user == "all_users": return [p.info for p in psutil.process_iter(attrs=attrs)] - else: - return [p.info for p in psutil.process_iter(attrs=attrs) if user in p.info["username"]] + return [p.info for p in psutil.process_iter(attrs=attrs) if user in p.info["username"]] def check_pid(pid, user=None): @@ -91,8 +90,8 @@ def check_pid(pid, user=None): all processes """ user_processes = get_user_process_info(user=user) - for p in user_processes: - if int(p["pid"]) == pid: + for process in user_processes: + if int(process["pid"]) == pid: return True return False @@ -149,12 +148,14 @@ def is_running(name, all_users=False): if all_users: cmd[1] = "aux" + # pylint: disable=consider-using-with try: - ps = subprocess.Popen(cmd, stdout=subprocess.PIPE, encoding="utf8").communicate()[0] + process_status = subprocess.Popen(cmd, stdout=subprocess.PIPE, encoding="utf8").communicate()[0] except TypeError: - ps = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] + process_status = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] + # pylint: enable=consider-using-with - if name in ps: + if name in process_status: return True return False @@ -178,7 +179,7 @@ def regex_list_filter(regex, list_to_filter, match=True): :return `new_list` """ - r = re.compile(regex) + r = re.compile(regex) # pylint: disable=C0103 if match: return list(filter(r.match, list_to_filter)) return list(filter(r.search, list_to_filter)) @@ -268,7 +269,7 @@ def determine_protocol(fname): @contextmanager -def cd(path): +def cd(path): # pylint: disable=C0103 """ TODO """ @@ -282,7 +283,7 @@ def cd(path): def pickle_data(filepath, content): """Dump content to a pickle file""" - with open(filepath, "w") as f: + with open(filepath, "w") as f: # pylint: disable=C0103 pickle.dump(content, f) @@ -343,22 +344,22 @@ def recurse(dic): return recurse(new_dic) -def nested_namespace_to_dicts(ns): +def nested_namespace_to_dicts(namespaces): """Code for recursively converting namespaces of namespaces into dictionaries instead. """ - def recurse(ns): - if not isinstance(ns, SimpleNamespace): - return ns - for key, val in list(ns.__dict__.items()): - setattr(ns, key, recurse(val)) - return ns.__dict__ + def recurse(namespaces): + if not isinstance(namespaces, SimpleNamespace): + return namespaces + for key, val in list(namespaces.__dict__.items()): + setattr(namespaces, key, recurse(val)) + return namespaces.__dict__ - if not isinstance(ns, SimpleNamespace): - raise TypeError(f"{ns} is not a SimpleNamespace") + if not isinstance(namespaces, SimpleNamespace): + raise TypeError(f"{namespaces} is not a SimpleNamespace") - new_ns = deepcopy(ns) + new_ns = deepcopy(namespaces) return recurse(new_ns) @@ -371,26 +372,25 @@ def get_flux_version(flux_path, no_errors=False): """ cmd = [flux_path, "version"] - ps = None + process = None try: - ps = subprocess.Popen(cmd, stdout=subprocess.PIPE, encoding="utf8").communicate() - except FileNotFoundError as e: + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, encoding="utf8").communicate() # pylint: disable=R1732 + except FileNotFoundError as e: # pylint: disable=C0103 if not no_errors: LOG.error(f"The flux path {flux_path} canot be found") LOG.error("Suppress this error with no_errors=True") raise e try: - flux_ver = re.search(r"\s*([\d.]+)", ps[0]).group(1) - except (ValueError, TypeError) as e: + flux_ver = re.search(r"\s*([\d.]+)", process[0]).group(1) + except (ValueError, TypeError) as e: # pylint: disable=C0103 if not no_errors: LOG.error("The flux version cannot be determined") LOG.error("Suppress this error with no_errors=True") raise e - else: - flux_ver = DEFAULT_FLUX_VERSION - LOG.warning(f"Using syntax for default version: {flux_ver}") + flux_ver = DEFAULT_FLUX_VERSION + LOG.warning(f"Using syntax for default version: {flux_ver}") return flux_ver @@ -465,40 +465,39 @@ def convert_to_timedelta(timestr: Union[str, int]) -> timedelta: nfields = len(timestr.split(":")) if nfields > 4: raise ValueError(f"Cannot convert {timestr} to a timedelta. Valid format: days:hours:minutes:seconds.") - _, d, h, m, s = (":0" * 10 + timestr).rsplit(":", 4) + _, d, h, m, s = (":0" * 10 + timestr).rsplit(":", 4) # pylint: disable=C0103 tdelta = timedelta(days=int(d), hours=int(h), minutes=int(m), seconds=int(s)) return tdelta -def _repr_timedelta_HMS(td: timedelta) -> str: +def _repr_timedelta_HMS(time_delta: timedelta) -> str: # pylint: disable=C0103 """Represent a timedelta object as a string in hours:minutes:seconds""" - hours, remainder = divmod(td.total_seconds(), 3600) + hours, remainder = divmod(time_delta.total_seconds(), 3600) minutes, seconds = divmod(remainder, 60) hours, minutes, seconds = int(hours), int(minutes), int(seconds) return f"{hours:02d}:{minutes:02d}:{seconds:02d}" -def _repr_timedelta_FSD(td: timedelta) -> str: +def _repr_timedelta_FSD(time_delta: timedelta) -> str: # pylint: disable=C0103 """Represent a timedelta as a flux standard duration string, using seconds. flux standard duration (FSD) is a floating point number with a single character suffix: s,m,h or d. This uses seconds for simplicity. """ - fsd = f"{td.total_seconds()}s" + fsd = f"{time_delta.total_seconds()}s" return fsd -def repr_timedelta(td: timedelta, method: str = "HMS") -> str: +def repr_timedelta(time_delta: timedelta, method: str = "HMS") -> str: """Represent a timedelta object as a string using a particular method. method - HMS: 'hours:minutes:seconds' method - FSD: flux standard duration: 'seconds.s'""" if method == "HMS": - return _repr_timedelta_HMS(td) - elif method == "FSD": - return _repr_timedelta_FSD(td) - else: - raise ValueError("Invalid method for formatting timedelta! Valid choices: HMS, FSD") + return _repr_timedelta_HMS(time_delta) + if method == "FSD": + return _repr_timedelta_FSD(time_delta) + raise ValueError("Invalid method for formatting timedelta! Valid choices: HMS, FSD") def convert_timestring(timestring: Union[str, int], format_method: str = "HMS") -> str: diff --git a/setup.cfg b/setup.cfg index b81cb1afa..a000df59a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,7 +11,6 @@ lines_after_imports=2 [flake8] ignore = E203, E266, E501, W503 -max-line-length = 127 max-complexity = 15 select = B,C,E,F,W,T4 exclude = .git,__pycache__,ascii_art.py,merlin/examples/*,*venv* @@ -19,6 +18,8 @@ exclude = .git,__pycache__,ascii_art.py,merlin/examples/*,*venv* [pylint.FORMAT] ignore=*venv* +disable=unspecified-encoding,subprocess-run-check +max-line-length = 127 [mypy] diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 9c44e1f5f..b3acdde4e 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -50,7 +50,7 @@ def ingest_info(self, info): @abstractmethod def passes(self): """The method that will check if the test passes or not""" - pass + raise NotImplementedError("The 'passes' property should be defined in all Condition subclasses.") # pylint: disable=no-member @@ -142,6 +142,12 @@ def glob(self, glob_string): return sorted(candidates)[-1] return candidates + @property + @abstractmethod + def passes(self): + """The method that will check if the test passes or not""" + raise NotImplementedError("The 'passes' property should be defined in all StudyOutputAware subclasses.") + class StepFileExists(StudyOutputAware): """ diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 0282f888b..effa9ca6f 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -39,8 +39,11 @@ from contextlib import suppress from subprocess import TimeoutExpired, run +# Pylint complains that we didn't install this module but it's defined locally so ignore from test_definitions import OUTPUT_DIR, define_tests # pylint: disable=E0401 +from merlin.display import tabulate_info + def get_definition_issues(test): """ @@ -84,8 +87,8 @@ def run_single_test(test): and information about the test for logging purposes. :param `test`: A dictionary that defines the test :returns: A tuple of type (bool, dict) where the bool - represents if the test passed and the dict - contains info about the test. + represents if the test passed and the dict + contains info about the test. """ # Parse the test definition commands = test.pop("cmds", None) @@ -212,6 +215,7 @@ def filter_tests_to_run(args, tests): return selective, n_to_run +# TODO split this function up so it's not as large (this will fix the pylint issue here too) def run_tests(args, tests): # pylint: disable=R0914 """ Run all inputted tests. @@ -310,8 +314,6 @@ def display_tests(tests): the --id flag. :param `tests`: A dict of tests (Dict) """ - from merlin.display import tabulate_info # pylint: disable=C0415 - test_names = list(tests.keys()) test_table = [(i + 1, test_names[i]) for i in range(len(test_names))] test_table.insert(0, ("ID", "Test Name")) diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index aafbabe2f..551056f95 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -40,6 +40,7 @@ } """ +# Pylint complains that we didn't install this module but it's defined locally so ignore from conditions import ( # pylint: disable=E0401 FileHasNoRegex, FileHasRegex, From 535a4bf5cf98f092b27262e6fd8396b06ee11d76 Mon Sep 17 00:00:00 2001 From: Joe Koning Date: Tue, 28 Mar 2023 16:16:55 -0700 Subject: [PATCH 073/126] Feature/flux new api (#407) * Update flux run command. * Update get_flux_cmd and add get_flux_alloc. Use these functions in all cases. * Update get_flux_alloc and test_definitions.py to work for testing. * Update actual flux version. * Update CHANGELOG.md with flux version. * Change DEFAULT_FLUX_VERSION . * Fix-style * Return to default flux run in scriptadapter. * fix bugs for stop-workers and schema validation fix bugs for stop-workers and schema validation fix all flags for stop-workers fix small bug with schema validation modify CHANGELOG run fix-style add myself to contributors list decouple celery logic from main remove unused import make changes Luc requested in PR * Fix batch flux exe and have test flux script return version as well. * Make sure test flux decalres itself. Fix flux alloc full path call. * Make sure qsub emits fake info. Check for flux or qsub in env for testing before adding the fake command. * Move fake commands to test directory * Remove unneeded f-string. * Change the scheduler check output strings to be more descriptive. --------- Co-authored-by: Brian Gunnarson --- CHANGELOG.md | 2 ++ .../workflows/flux/scripts/flux_test/flux | 2 -- .../workflows/flux/scripts/pbs_test/qsub | 2 -- merlin/study/batch.py | 14 ++++---- merlin/study/script_adapter.py | 4 +-- merlin/study/study.py | 2 +- merlin/utils.py | 32 ++++++++++++++++--- tests/integration/fake_commands/flux | 8 +++++ tests/integration/fake_commands/qsub | 2 ++ tests/integration/test_definitions.py | 20 ++++++++---- 10 files changed, 65 insertions(+), 23 deletions(-) delete mode 100755 merlin/examples/workflows/flux/scripts/flux_test/flux delete mode 100755 merlin/examples/workflows/flux/scripts/pbs_test/qsub create mode 100755 tests/integration/fake_commands/flux create mode 100755 tests/integration/fake_commands/qsub diff --git a/CHANGELOG.md b/CHANGELOG.md index 08bac767c..6e1c5cbb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Merlin will now assign `default_worker` to any step not associated with a worker - Added `get_step_worker_map()` as a method in `specification.py` - Added `tabulate_info()` function in `display.py` to help with table formatting +- Added get_flux_alloc function for new flux version >= 0.48.x interface change ### Changed - Changed celery_regex to celery_slurm_regex in test_definitions.py @@ -71,6 +72,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Modified `batch_worker_launch` to use the new `parse_batch_block` function - Added a function `construct_scheduler_legend` to build a dict that keeps as much information as we need about each scheduler stored in one place - Cleaned up the `construct_worker_launch_command` function to utilize the newly added functions and decrease the amount of repeated code +- Changed get_flux_cmd for new flux version >=0.48.x interface ## [1.9.1] diff --git a/merlin/examples/workflows/flux/scripts/flux_test/flux b/merlin/examples/workflows/flux/scripts/flux_test/flux deleted file mode 100755 index f907abfc8..000000000 --- a/merlin/examples/workflows/flux/scripts/flux_test/flux +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python3 -print("Nodes") diff --git a/merlin/examples/workflows/flux/scripts/pbs_test/qsub b/merlin/examples/workflows/flux/scripts/pbs_test/qsub deleted file mode 100755 index b12de9c32..000000000 --- a/merlin/examples/workflows/flux/scripts/pbs_test/qsub +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -echo "pbs_version = 19.0.0" diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 9a42873f1..d74da7d7a 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -40,7 +40,7 @@ import subprocess from typing import Dict, Optional, Union -from merlin.utils import get_yaml_var +from merlin.utils import get_flux_alloc, get_yaml_var LOG = logging.getLogger(__name__) @@ -154,21 +154,21 @@ def get_batch_type(default=None): :returns: (str) The batch name (available options: slurm, flux, lsf, pbs). """ # Flux should be checked first due to slurm emulation scripts - LOG.debug(f"check for flux = {check_for_flux()}") + LOG.debug(f"check for flux scheduler = {check_for_flux()}") if check_for_flux(): return "flux" # PBS should be checked before slurm for testing - LOG.debug(f"check for pbs = {check_for_pbs()}") + LOG.debug(f"check for pbs scheduler = {check_for_pbs()}") if check_for_pbs(): return "pbs" # LSF should be checked before slurm for testing - LOG.debug(f"check for lsf = {check_for_lsf()}") + LOG.debug(f"check for lsf scheduler = {check_for_lsf()}") if check_for_lsf(): return "lsf" - LOG.debug(f"check for slurm = {check_for_slurm()}") + LOG.debug(f"check for slurm scheduler = {check_for_slurm()}") if check_for_slurm(): return "slurm" @@ -329,7 +329,9 @@ def construct_worker_launch_command(batch: Optional[Dict], btype: str, nodes: in flux_path += "/" flux_exe: str = os.path.join(flux_path, "flux") - launch_command = f"{flux_exe} mini alloc -o pty -N {nodes} --exclusive --job-name=merlin" + flux_alloc: str = get_flux_alloc(flux_exe) + launch_command = f"{flux_alloc} -o pty -N {nodes} --exclusive --job-name=merlin" + if bank: launch_command += f" --setattr=system.bank={bank}" if queue: diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index ad6843c51..91173ca9c 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -274,10 +274,10 @@ def __init__(self, **kwargs): :param **kwargs: A dictionary with default settings for the adapter. """ - flux_command = kwargs.pop("flux_command", "flux mini run") + # The flux_command should always be overriden by the study object's flux_command property + flux_command = kwargs.pop("flux_command", "flux run") super().__init__(**kwargs) - # "cmd": "flux mini run", self._cmd_flags = { "cmd": flux_command, "ntasks": "-n", diff --git a/merlin/study/study.py b/merlin/study/study.py index 9d016bc0c..f43125dc6 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -438,7 +438,7 @@ def expanded_spec(self): @cached_property def flux_command(self): """ - Returns the flux version. + Returns the flux command, this will include the full path, if flux_path given in the workflow. """ flux_bin = "flux" if "flux_path" in self.expanded_spec.batch.keys(): diff --git a/merlin/utils.py b/merlin/utils.py index dd8f69d29..b3830ccab 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -56,7 +56,7 @@ LOG = logging.getLogger(__name__) ARRAY_FILE_FORMATS = ".npy, .csv, .tab" -DEFAULT_FLUX_VERSION = "0.13" +DEFAULT_FLUX_VERSION = "0.48.0" def get_user_process_info(user=None, attrs=None): @@ -397,24 +397,48 @@ def get_flux_version(flux_path, no_errors=False): def get_flux_cmd(flux_path, no_errors=False): """ - Return the flux command as string + Return the flux run command as string :param `flux_path`: the full path to the flux bin :param `no_errors`: a flag to determine if this a test run to ignore errors """ - # The default is for flux version >= 0.13, + # The default is for flux version >= 0.48.x # this may change in the future. - flux_cmd = "flux mini run" + flux_cmd = "flux run" flux_ver = get_flux_version(flux_path, no_errors=no_errors) vers = [int(n) for n in flux_ver.split(".")] + if vers[0] == 0 and vers[1] < 48: + flux_cmd = "flux mini run" + if vers[0] == 0 and vers[1] < 13: flux_cmd = "flux wreckrun" return flux_cmd +def get_flux_alloc(flux_path, no_errors=False): + """ + Return the flux alloc command as string + + :param `flux_path`: the full path to the flux bin + :param `no_errors`: a flag to determine if this a test run to ignore errors + """ + # The default is for flux version >= 0.48.x + # this may change in the future. + flux_alloc = f"{flux_path} alloc" + + flux_ver = get_flux_version(flux_path, no_errors=no_errors) + + vers = [int(n) for n in flux_ver.split(".")] + + if vers[0] == 0 and vers[1] < 48: + flux_alloc = f"{flux_path} mini alloc" + + return flux_alloc + + def check_machines(machines): """ Return a True if the current machine is in the list of machines. diff --git a/tests/integration/fake_commands/flux b/tests/integration/fake_commands/flux new file mode 100755 index 000000000..1e1e5aa1d --- /dev/null +++ b/tests/integration/fake_commands/flux @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +import sys + + +if len(sys.argv) > 1 and sys.argv[1] == 'version': + print("commands: 0.48.0\n") +else: + print("Nodes\n") diff --git a/tests/integration/fake_commands/qsub b/tests/integration/fake_commands/qsub new file mode 100755 index 000000000..57e42a21f --- /dev/null +++ b/tests/integration/fake_commands/qsub @@ -0,0 +1,2 @@ +#!/bin/sh +echo "pbs_version = 19.0.0\n" diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 551056f95..e6010e6fa 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -40,6 +40,8 @@ } """ +import shutil + # Pylint complains that we didn't install this module but it's defined locally so ignore from conditions import ( # pylint: disable=E0401 FileHasNoRegex, @@ -52,7 +54,7 @@ StepFileHasRegex, ) -from merlin.utils import get_flux_cmd +from merlin.utils import get_flux_alloc, get_flux_cmd OUTPUT_DIR = "cli_test_studies" @@ -66,8 +68,9 @@ def define_tests(): # pylint: disable=R0914 is the test's name, and the value is a tuple of (shell command, condition(s) to satisfy). """ + flux_alloc = (get_flux_alloc("flux", no_errors=True),) celery_slurm_regex = r"(srun\s+.*)?celery\s+(-A|--app)\s+merlin\s+worker\s+.*" - celery_flux_regex = r"(flux mini alloc\s+.*)?celery\s+(-A|--app)\s+merlin\s+worker\s+.*" + celery_flux_regex = rf"({flux_alloc}\s+.*)?celery\s+(-A|--app)\s+merlin\s+worker\s+.*" celery_pbs_regex = r"(qsub\s+.*)?celery\s+(-A|--app)\s+merlin\s+worker\s+.*" # shortcut string variables @@ -87,10 +90,15 @@ def define_tests(): # pylint: disable=R0914 flux = f"{examples}/flux/flux_test.yaml" flux_restart = f"{examples}/flux/flux_par_restart.yaml" flux_native = f"{examples}/flux/flux_par_native_test.yaml" - flux_native_path = f"{examples}/flux/scripts/flux_test" - workers_flux = f"""PATH="{flux_native_path}:$PATH";merlin {err_lvl} run-workers""" - pbs_path = f"{examples}/flux/scripts/pbs_test" - workers_pbs = f"""PATH="{pbs_path}:$PATH";merlin {err_lvl} run-workers""" + workers_flux = f"merlin {err_lvl} run-workers" + fake_cmds_path = "tests/integration/fake_commands" + if not shutil.which("flux"): + # Use bogus flux to test if no flux is present + workers_flux = f"""PATH="{fake_cmds_path}:$PATH";merlin {err_lvl} run-workers""" + workers_pbs = f"merlin {err_lvl} run-workers" "" + if not shutil.which("qsub"): + # Use bogus qsub to test if no pbs scheduler is present + workers_pbs = f"""PATH="{fake_cmds_path}:$PATH";merlin {err_lvl} run-workers""" lsf = f"{examples}/lsf/lsf_par.yaml" mul_workers_demo = f"{dev_examples}/multiple_workers.yaml" black = "black --check --target-version py36" From ae6a9fc57816c160bd7d8c2df9129272ce7f1b81 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 28 Mar 2023 13:01:19 -0700 Subject: [PATCH 074/126] refactor batch.py copy batch refactor changes from pylint-errors branch fix-style and additional logs add more logs attempt to fix the test 14 error remove log statements used for debugging add flux alloc changes to batch refactor playing with flux alloc fix flux alloc issue and add lsf test add lsf run-workers test remove unused import --- merlin/study/batch.py | 360 +++++++++++++------------- tests/integration/fake_commands/lsf | 3 + tests/integration/run_tests.py | 2 + tests/integration/test_definitions.py | 52 +++- 4 files changed, 228 insertions(+), 189 deletions(-) create mode 100644 tests/integration/fake_commands/lsf diff --git a/merlin/study/batch.py b/merlin/study/batch.py index d74da7d7a..676261678 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -65,114 +65,58 @@ def batch_check_parallel(spec): return parallel -def check_for_flux(): +def check_for_scheduler(scheduler, scheduler_legend): """ - Check if FLUX is the main scheduler for the cluster + Check which scheduler (Flux, Slurm, LSF, or PBS) is the main + scheduler for the cluster. + :param `scheduler`: A string representing the scheduler to check for + Options: flux, slurm, lsf, or pbs + :param `scheduler_legend`: A dict of information related to each scheduler + :returns: A bool representing whether `scheduler` is the main scheduler for the cluster """ - try: - p = subprocess.Popen( - ["flux", "resource", "info"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - result = p.stdout.readlines() - if result and len(result) > 0 and b"Nodes" in result[0]: - return True - else: - return False - except FileNotFoundError: + # Check for invalid scheduler + if scheduler not in ("flux", "slurm", "lsf", "pbs"): + LOG.warning(f"Invalid scheduler {scheduler} given to check_for_scheduler.") return False - -def check_for_slurm(): - """ - Check if SLURM is the main scheduler for the cluster - """ + # Try to run the check command provided via the scheduler legend try: - p = subprocess.Popen( - ["sbatch", "--help"], + process = subprocess.Popen( # pylint: disable=R1732 + scheduler_legend[scheduler]["check cmd"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - result = p.stdout.readlines() - if result and len(result) > 0 and b"sbatch" in result[0]: + # If the desired output exists, return True. Otherwise, return False + result = process.stdout.readlines() + if result and len(result) > 0 and scheduler_legend[scheduler]["expected check output"] in result[0]: return True - else: - return False - except FileNotFoundError: return False - - -def check_for_lsf(): - """ - Check if LSF is the main scheduler for the cluster - """ - try: - p = subprocess.Popen( - ["jsrun", "--help"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - result = p.stdout.readlines() - if result and len(result) > 0 and b"jsrun" in result[0]: - return True - else: - return False - except FileNotFoundError: - return False - - -def check_for_pbs(): - """ - Check if PBS is the main scheduler for the cluster - """ - try: - p = subprocess.Popen( - ["qsub", "--version"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - result = p.stdout.readlines() - if result and len(result) > 0 and b"pbs_version" in result[0]: - return True - else: - return False except FileNotFoundError: return False -def get_batch_type(default=None): +def get_batch_type(scheduler_legend, default=None): """ Determine which batch scheduler to use. + :param scheduler_legend: A dict storing info related to each scheduler :param default: (str) The default batch scheduler to use if a scheduler can't be determined. The default is None. :returns: (str) The batch name (available options: slurm, flux, lsf, pbs). """ - # Flux should be checked first due to slurm emulation scripts - LOG.debug(f"check for flux scheduler = {check_for_flux()}") - if check_for_flux(): - return "flux" - - # PBS should be checked before slurm for testing - LOG.debug(f"check for pbs scheduler = {check_for_pbs()}") - if check_for_pbs(): - return "pbs" - - # LSF should be checked before slurm for testing - LOG.debug(f"check for lsf scheduler = {check_for_lsf()}") - if check_for_lsf(): - return "lsf" - - LOG.debug(f"check for slurm scheduler = {check_for_slurm()}") - if check_for_slurm(): - return "slurm" - - SYS_TYPE = os.environ.get("SYS_TYPE", "") + # These schedulers are listed in order of which should be checked for first + # 1. Flux should be checked first due to slurm emulation scripts + # 2. PBS should be checked before slurm for testing + # 3. LSF should be checked before slurm for testing + # 4. Slurm should be checked last + schedulers_to_check = ["flux", "pbs", "lsf", "slurm"] + for scheduler in schedulers_to_check: + LOG.debug(f"check for {scheduler} = {check_for_scheduler(scheduler, scheduler_legend)}") + if check_for_scheduler(scheduler, scheduler_legend): + return scheduler + + SYS_TYPE = os.environ.get("SYS_TYPE", "") # pylint: disable=C0103 if "toss_3" in SYS_TYPE: return "slurm" @@ -198,7 +142,7 @@ def get_node_count(default=1): nodes = set(os.environ["LSB_HOSTS"].split()) n_batch_nodes = len(nodes) - 1 return n_batch_nodes - elif "LSB_MCPU_HOSTS" in os.environ: + if "LSB_MCPU_HOSTS" in os.environ: nodes = os.environ["LSB_MCPU_HOSTS"].split() n_batch_nodes = len(nodes) // 2 - 1 return n_batch_nodes @@ -206,6 +150,66 @@ def get_node_count(default=1): return default +def parse_batch_block(batch: Dict) -> Dict: + """ + A function to parse the batch block of the yaml file. + :param `batch`: The batch block to read in + :returns: A dict with all the info (or defaults) from the batch block + """ + flux_path: str = get_yaml_var(batch, "flux_path", "") + if "/" in flux_path: + flux_path += "/" + + flux_exe: str = os.path.join(flux_path, "flux") + flux_alloc: str + try: + flux_alloc = get_flux_alloc(flux_exe) + except FileNotFoundError as e: # pylint: disable=C0103 + LOG.debug(e) + flux_alloc = "" + + parsed_batch = { + "btype": get_yaml_var(batch, "type", "local"), + "nodes": get_yaml_var(batch, "nodes", None), + "shell": get_yaml_var(batch, "shell", "bash"), + "bank": get_yaml_var(batch, "bank", ""), + "queue": get_yaml_var(batch, "queue", ""), + "walltime": get_yaml_var(batch, "walltime", ""), + "launch pre": get_yaml_var(batch, "launch_pre", ""), + "launch args": get_yaml_var(batch, "launch_args", ""), + "launch command": get_yaml_var(batch, "worker_launch", ""), + "flux path": flux_path, + "flux exe": flux_exe, + "flux exec": get_yaml_var(batch, "flux_exec", None), + "flux alloc": flux_alloc, + "flux opts": get_yaml_var(batch, "flux_start_opts", ""), + "flux exec workers": get_yaml_var(batch, "flux_exec_workers", True), + } + return parsed_batch + + +def get_flux_launch(parsed_batch: Dict) -> str: + """ + Build the flux launch command based on the batch section of the yaml. + :param `parsed_batch`: A dict of batch configurations + :returns: The flux launch command + """ + default_flux_exec = "flux exec" if parsed_batch["launch command"] else f"{parsed_batch['flux exe']} exec" + flux_exec: str = "" + if parsed_batch["flux exec workers"]: + flux_exec = parsed_batch["flux exec"] if parsed_batch["flux exec"] else default_flux_exec + + if parsed_batch["launch command"] and "flux" not in parsed_batch["launch command"]: + launch: str = ( + f"{parsed_batch['launch command']} {parsed_batch['flux exe']}" + f" start {parsed_batch['flux opts']} {flux_exec} `which {parsed_batch['shell']}` -c" + ) + else: + launch: str = f"{parsed_batch['launch command']} {flux_exec} `which {parsed_batch['shell']}` -c" + + return launch + + def batch_worker_launch( spec: Dict, com: str, @@ -229,16 +233,16 @@ def batch_worker_launch( LOG.error("The batch section is required in the specification file.") raise - btype: str = get_yaml_var(batch, "type", "local") + parsed_batch = parse_batch_block(batch) # A jsrun submission cannot be run under a parent jsrun so # all non flux lsf submissions need to be local. - if btype == "local" or "lsf" in btype: + if parsed_batch["btype"] == "local" or "lsf" in parsed_batch["btype"]: return com if nodes is None: # Use the value in the batch section - nodes = get_yaml_var(batch, "nodes", None) + nodes = parsed_batch["nodes"] # Get the number of nodes from the environment if unset if nodes is None or nodes == "all": @@ -246,108 +250,114 @@ def batch_worker_launch( elif not isinstance(nodes, int): raise TypeError("Nodes was passed into batch_worker_launch with an invalid type (likely a string other than 'all').") - shell: str = get_yaml_var(batch, "shell", "bash") - - launch_pre: str = get_yaml_var(batch, "launch_pre", "") - launch_args: str = get_yaml_var(batch, "launch_args", "") - launch_command: str = get_yaml_var(batch, "worker_launch", "") + if not parsed_batch["launch command"]: + parsed_batch["launch command"] = construct_worker_launch_command(parsed_batch, nodes) - if not launch_command: - launch_command = construct_worker_launch_command(batch, btype, nodes) - - if launch_args: - launch_command += f" {launch_args}" + if parsed_batch["launch args"]: + parsed_batch["launch command"] += f" {parsed_batch['launch args']}" # Allow for any pre launch manipulation, e.g. module load # hwloc/1.11.10-cuda - if launch_pre: - launch_command = f"{launch_pre} {launch_command}" + if parsed_batch["launch pre"]: + parsed_batch["launch command"] = f"{parsed_batch['launch pre']} {parsed_batch['launch command']}" - LOG.debug(f"launch_command = {launch_command}") + LOG.debug(f"launch command: {parsed_batch['launch command']}") worker_cmd: str = "" - if btype == "flux": - flux_path: str = get_yaml_var(batch, "flux_path", "") - if "/" in flux_path: - flux_path += "/" - - flux_exe: str = os.path.join(flux_path, "flux") - - flux_opts: Union[str, Dict] = get_yaml_var(batch, "flux_start_opts", "") - - flux_exec_workers: Union[str, Dict, bool] = get_yaml_var(batch, "flux_exec_workers", True) - - default_flux_exec = "flux exec" if launch_command else f"{flux_exe} exec" - flux_exec: str = "" - if flux_exec_workers: - flux_exec = get_yaml_var(batch, "flux_exec", default_flux_exec) - - if launch_command and "flux" not in launch_command: - launch: str = f"{launch_command} {flux_exe} start {flux_opts} {flux_exec} `which {shell}` -c" - else: - launch: str = f"{launch_command} {flux_exec} `which {shell}` -c" + if parsed_batch["btype"] == "flux": + launch = get_flux_launch(parsed_batch) worker_cmd = f'{launch} "{com}"' else: - worker_cmd = f"{launch_command} {com}" + worker_cmd = f"{parsed_batch['launch command']} {com}" return worker_cmd -def construct_worker_launch_command(batch: Optional[Dict], btype: str, nodes: int) -> str: +def construct_scheduler_legend(parsed_batch: Dict, nodes: int) -> Dict: + """ + Constructs a legend of relevant information needed for each scheduler. This includes: + - bank (str): The flag to add a bank to the launch command + - check cmd (list): The command to run to check if this is the main scheduler for the cluster + - expected check output (str): The expected output from running the check cmd + - launch (str): The initial launch command for the scheduler + - queue (str): The flag to add a queue to the launch command + - walltime (str): The flag to add a walltime to the launch command + + :param `parsed_batch`: A dict of batch configurations + :param `nodes`: An int representing the number of nodes to use in a launch command + :returns: A dict of scheduler related information + """ + scheduler_legend = { + "flux": { + "bank": f" --setattr=system.bank={parsed_batch['bank']}", + "check cmd": ["flux", "resource", "info"], + "expected check output": b"Nodes", + "launch": f"{parsed_batch['flux alloc']} -o pty -N {nodes} --exclusive --job-name=merlin", + "queue": f" --setattr=system.queue={parsed_batch['queue']}", + "walltime": f" -t {parsed_batch['walltime']}", + }, + "lsf": { + "check cmd": ["jsrun", "--help"], + "expected check output": b"jsrun", + "launch": f"jsrun -a 1 -c ALL_CPUS -g ALL_SGPUS --bind=none -n {nodes}", + }, + # pbs is mainly a placeholder in case a user wants to try it (we don't have it at the lab so it's mostly untested) + "pbs": { + "bank": f" -A {parsed_batch['bank']}", + "check cmd": ["qsub", "--version"], + "expected check output": b"pbs_version", + "launch": f"qsub -l nodes={nodes}", + "queue": f" -q {parsed_batch['queue']}", + "walltime": f" -l walltime={parsed_batch['walltime']}", + }, + "slurm": { + "bank": f" -A {parsed_batch['bank']}", + "check cmd": ["sbatch", "--help"], + "expected check output": b"sbatch", + "launch": f"srun -N {nodes} -n {nodes}", + "queue": f" -p {parsed_batch['queue']}", + "walltime": f" -t {parsed_batch['walltime']}", + }, + } + return scheduler_legend + + +def construct_worker_launch_command(parsed_batch: Dict, nodes: int) -> str: """ If no 'worker_launch' is found in the batch yaml, this method constructs the needed launch command. - : param batch : (Optional[Dict]): An optional batch override from the worker config - : param btype : (str): The type of batch (flux, local, lsf) - : param nodes : (int): The number of nodes to use in the batch launch + :param `parsed_batch`: A dict of batch configurations + :param `nodes`:: The number of nodes to use in the batch launch + :returns: The launch command """ + # Initialize launch_command and get the scheduler_legend and workload_manager launch_command: str = "" - workload_manager: str = get_batch_type() - bank: str = get_yaml_var(batch, "bank", "") - queue: str = get_yaml_var(batch, "queue", "") - walltime: str = get_yaml_var(batch, "walltime", "") - - if btype == "pbs" and workload_manager == btype: - raise Exception("The PBS scheduler is only enabled for 'batch: flux' type") - - if btype == "slurm" or workload_manager == "slurm": - launch_command = f"srun -N {nodes} -n {nodes}" - if bank: - launch_command += f" -A {bank}" - if queue: - launch_command += f" -p {queue}" - if walltime: - launch_command += f" -t {walltime}" - - if workload_manager == "lsf": - # The jsrun utility does not have a time argument - launch_command = f"jsrun -a 1 -c ALL_CPUS -g ALL_GPUS --bind=none -n {nodes}" - - if workload_manager == "flux": - flux_path: str = get_yaml_var(batch, "flux_path", "") - if "/" in flux_path: - flux_path += "/" - - flux_exe: str = os.path.join(flux_path, "flux") - flux_alloc: str = get_flux_alloc(flux_exe) - launch_command = f"{flux_alloc} -o pty -N {nodes} --exclusive --job-name=merlin" - - if bank: - launch_command += f" --setattr=system.bank={bank}" - if queue: - launch_command += f" --setattr=system.queue={queue}" - if walltime: - launch_command += f" -t {walltime}" - - if workload_manager == "pbs": - launch_command = f"qsub -l nodes={nodes}" - # launch_command = f"qsub -l nodes={nodes} -l procs={nodes}" - if bank: - launch_command += f" -A {bank}" - if queue: - launch_command += f" -q {queue}" - # if walltime: - # launch_command += f" -l walltime={walltime}" - launch_command += " --" # To read from stdin + scheduler_legend: Dict = construct_scheduler_legend(parsed_batch, nodes) + workload_manager: str = get_batch_type(scheduler_legend) + + if parsed_batch["btype"] == "pbs" and workload_manager == parsed_batch["btype"]: + raise TypeError("The PBS scheduler is only enabled for 'batch: flux' type") + + if parsed_batch["btype"] == "slurm" and workload_manager not in ("lsf", "flux", "pbs"): + workload_manager = "slurm" + + try: + launch_command = scheduler_legend[workload_manager]["launch"] + except KeyError as e: # pylint: disable=C0103 + LOG.debug(e) + + # If lsf is the workload manager we stop here (no need to add bank, queue, walltime) + if workload_manager != "lsf" or not launch_command: + # Add bank, queue, and walltime to the launch command as necessary + for key in ("bank", "queue", "walltime"): + if parsed_batch[key]: + try: + launch_command += scheduler_legend[workload_manager][key] + except KeyError as e: # pylint: disable=C0103 + LOG.error(e) + + # To read from stdin we append this to the launch command for pbs + if workload_manager == "pbs": + launch_command += " --" return launch_command diff --git a/tests/integration/fake_commands/lsf b/tests/integration/fake_commands/lsf new file mode 100644 index 000000000..7b6c57f04 --- /dev/null +++ b/tests/integration/fake_commands/lsf @@ -0,0 +1,3 @@ +#!/bin/sh + +echo "jsrun str: + """ + Given a command used by a scheduler (e.g. flux for flux, jsrun for lsf, etc.) + check if that command is found and if it isn't, use a fake scheduler command. + :param `cmd`: A string representing the command used by a scheduler + :param `default`: The default worker launch command to use if a scheduler is found + :returns: The appropriate worker launch command + """ + fake_cmds_path = "tests/integration/fake_commands" + if not shutil.which(cmd): + # Use bogus flux to test if no flux is present + workers_cmd = f"""PATH="{fake_cmds_path}:$PATH";{default}""" + else: + workers_cmd = default + + return workers_cmd + + +def define_tests(): # pylint: disable=R0914,R0915 """ Returns a dictionary of tests, where the key is the test's name, and the value is a tuple of (shell command, condition(s) to satisfy). """ + # Shortcuts for regexs to check against flux_alloc = (get_flux_alloc("flux", no_errors=True),) - celery_slurm_regex = r"(srun\s+.*)?celery\s+(-A|--app)\s+merlin\s+worker\s+.*" - celery_flux_regex = rf"({flux_alloc}\s+.*)?celery\s+(-A|--app)\s+merlin\s+worker\s+.*" - celery_pbs_regex = r"(qsub\s+.*)?celery\s+(-A|--app)\s+merlin\s+worker\s+.*" + celery_regex = r"celery\s+(-A|--app)\s+merlin\s+worker\s+.*" + celery_slurm_regex = rf"(srun\s+.*)?{celery_regex}" + celery_lsf_regex = rf"(jsrun\s+.*)?{celery_regex}" + celery_flux_regex = rf"({flux_alloc}\s+.*)?{celery_regex}" + celery_pbs_regex = rf"(qsub\s+.*)?{celery_regex}" - # shortcut string variables + # Shortcuts for Merlin commands err_lvl = "-lvl error" workers = f"merlin {err_lvl} run-workers" + workers_flux = get_worker_by_cmd("flux", workers) + workers_pbs = get_worker_by_cmd("qsub", workers) + workers_lsf = get_worker_by_cmd("jsrun", workers) run = f"merlin {err_lvl} run" restart = f"merlin {err_lvl} restart" purge = "merlin purge" + + # Shortcuts for example workflow paths examples = "merlin/examples/workflows" dev_examples = "merlin/examples/dev_workflows" demo = f"{examples}/feature_demo/feature_demo.yaml" @@ -90,17 +116,10 @@ def define_tests(): # pylint: disable=R0914 flux = f"{examples}/flux/flux_test.yaml" flux_restart = f"{examples}/flux/flux_par_restart.yaml" flux_native = f"{examples}/flux/flux_par_native_test.yaml" - workers_flux = f"merlin {err_lvl} run-workers" - fake_cmds_path = "tests/integration/fake_commands" - if not shutil.which("flux"): - # Use bogus flux to test if no flux is present - workers_flux = f"""PATH="{fake_cmds_path}:$PATH";merlin {err_lvl} run-workers""" - workers_pbs = f"merlin {err_lvl} run-workers" "" - if not shutil.which("qsub"): - # Use bogus qsub to test if no pbs scheduler is present - workers_pbs = f"""PATH="{fake_cmds_path}:$PATH";merlin {err_lvl} run-workers""" lsf = f"{examples}/lsf/lsf_par.yaml" mul_workers_demo = f"{dev_examples}/multiple_workers.yaml" + + # Other shortcuts black = "black --check --target-version py36" config_dir = "./CLI_TEST_MERLIN_CONFIG" release_dependencies = "./requirements/release.txt" @@ -214,6 +233,11 @@ def define_tests(): # pylint: disable=R0914 "conditions": [HasReturnCode(), HasRegex(celery_slurm_regex)], "run type": "local", }, + "run-workers echo lsf_test": { + "cmds": f"{workers_lsf} {lsf} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_lsf_regex)], + "run type": "local", + }, "run-workers echo flux_test": { "cmds": f"{workers} {flux} --echo", "conditions": [HasReturnCode(), HasRegex(celery_slurm_regex)], From f749e70850416a140ce890844e94b92a6eee0e19 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 17 Mar 2023 17:12:41 -0700 Subject: [PATCH 075/126] Refactor query-workers begin work on refactoring query-workers add --queues, --workers, and --spec flags to query-workers add query-workers tests revert back to pinging workers for querying make regex searches work slightly better clean up code update documentation for query-workers modify changelog run fix-style resolve review requests move an import that was breaking tests --- CHANGELOG.md | 8 +- docs/source/faq.rst | 6 + docs/source/merlin_commands.rst | 41 ++++++- merlin/main.py | 29 ++++- merlin/router.py | 4 +- merlin/study/celeryadapter.py | 154 ++++++++++++++++++++++---- merlin/utils.py | 20 ++++ tests/integration/test_definitions.py | 90 ++++++++++++++- 8 files changed, 320 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e1c5cbb7..8efde6194 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `get_step_worker_map()` as a method in `specification.py` - Added `tabulate_info()` function in `display.py` to help with table formatting - Added get_flux_alloc function for new flux version >= 0.48.x interface change +- New flags to the `query-workers` command + - `--queues`: query workers based on the queues they're associated with + - `--workers`: query workers based on a regex of the names you're looking for + - `--spec`: query workers based on the workers defined in a spec file ### Changed - Changed celery_regex to celery_slurm_regex in test_definitions.py @@ -73,7 +77,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added a function `construct_scheduler_legend` to build a dict that keeps as much information as we need about each scheduler stored in one place - Cleaned up the `construct_worker_launch_command` function to utilize the newly added functions and decrease the amount of repeated code - Changed get_flux_cmd for new flux version >=0.48.x interface - +- The `query-workers` command now prints a table as its' output + - Each row of the `Workers` column has the name of an active worker + - Each row of the `Queues` column has a list of queues associated with the active worker ## [1.9.1] ### Fixed diff --git a/docs/source/faq.rst b/docs/source/faq.rst index b72e7f6ff..d0ef8e109 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -257,6 +257,12 @@ How do I see what workers are connected? $ merlin query-workers +This command gives you fine control over which workers you're looking for via +a regex on their name, the queue names associated with workers, or even by providing +the name of a spec file where workers are defined. + +For more info, see :ref:`query-workers`. + How do I stop workers? ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/merlin_commands.rst b/docs/source/merlin_commands.rst index 316d675fe..cb9b8eefb 100644 --- a/docs/source/merlin_commands.rst +++ b/docs/source/merlin_commands.rst @@ -174,6 +174,8 @@ the input yaml file. ``Example: --vars QUEUE_NAME=new_queue EPOCHS=3`` +.. _query-workers: + Searching for any workers (``merlin query-workers``) ---------------------------------------------------- @@ -185,8 +187,43 @@ the task server you can use: $ merlin query-workers This will broadcast a command to all connected workers and print -the names of any that respond. This is useful for interacting -with workers, such as via ``merlin stop-workers --workers``. +the names of any that respond and the queues they're attached to. +This is useful for interacting with workers, such as via +``merlin stop-workers --workers``. + +The ``--queues`` option will look for workers associated with the +names of the queues you provide here. For example, if you want to +see the names of all workers attached to the queues named ``demo`` +and ``merlin`` you would use: + +.. code-block:: + + merlin query-workers --queues demo merlin + +The ``--spec`` option will query for workers defined in the spec +file you provide. For example, if ``simworker`` and ``nonsimworker`` +are defined in a spec file called ``example_spec.yaml`` then to query +for these workers you would use: + +.. code-block:: + + merlin query-workers --spec example_spec.yaml + +The ``--workers`` option will query for workers based on the worker +names you provide here. For example, if you wanted to query a worker +named ``step_1_worker`` you would use: + +.. code-block:: + + merlin query-workers --workers step_1_worker + +This flag can also take regular expressions as input. For instance, +if you had several workers running but only wanted to find the workers +whose names started with ``step`` you would use: + +.. code-block:: + + merlin query-workers --workers ^step Restart the workflow (``merlin restart``) diff --git a/merlin/main.py b/merlin/main.py index 3a30df135..45e6f0c0c 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -269,7 +269,19 @@ def query_workers(args): :param `args`: parsed CLI arguments """ print(banner_small) - router.query_workers(args.task_server) + + # Get the workers from the spec file if --spec provided + worker_names = [] + if args.spec: + spec_path = verify_filepath(args.spec) + spec = MerlinSpec.load_specification(spec_path) + worker_names = spec.get_worker_names() + for worker_name in worker_names: + if "$" in worker_name: + LOG.warning(f"Worker '{worker_name}' is unexpanded. Target provenance spec instead?") + LOG.debug(f"Searching for the following workers to stop based on the spec {args.spec}: {worker_names}") + + router.query_workers(args.task_server, worker_names, args.queues, args.workers) def stop_workers(args): @@ -782,6 +794,21 @@ def generate_worker_touching_parsers(subparsers: ArgumentParser) -> None: help="Task server type from which to query workers.\ Default: %(default)s", ) + query.add_argument( + "--spec", + type=str, + default=None, + help="Path to a Merlin YAML spec file from which to read worker names to query.", + ) + query.add_argument("--queues", type=str, default=None, nargs="+", help="Specific queues to query workers from.") + query.add_argument( + "--workers", + type=str, + action="store", + nargs="+", + default=None, + help="Regex match for specific workers to query.", + ) # merlin stop-workers stop: ArgumentParser = subparsers.add_parser("stop-workers", help="Attempt to stop all task server workers.") diff --git a/merlin/router.py b/merlin/router.py index 8e8c85bed..d1a356737 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -160,7 +160,7 @@ def dump_status(query_return, csv_file): f.write("\n") -def query_workers(task_server): +def query_workers(task_server, spec_worker_names, queues, workers_regex): """ Gets info from workers. @@ -169,7 +169,7 @@ def query_workers(task_server): LOG.info("Searching for workers...") if task_server == "celery": - query_celery_workers() + query_celery_workers(spec_worker_names, queues, workers_regex) else: LOG.error("Celery is not specified as the task server!") diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 8daaa1fe0..0a47820e6 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -39,7 +39,7 @@ from contextlib import suppress from merlin.study.batch import batch_check_parallel, batch_worker_launch -from merlin.utils import check_machines, get_procs, get_yaml_var, is_running, regex_list_filter +from merlin.utils import apply_list_of_regex, check_machines, get_procs, get_yaml_var, is_running LOG = logging.getLogger(__name__) @@ -128,18 +128,136 @@ def get_queues(app): return queues, [*active_workers] -def query_celery_workers(): - """Look for existing celery workers. +def get_active_workers(app): + """ + This is the inverse of get_queues() defined above. This function + builds a dict where the keys are worker names and the values are lists + of queues attached to the worker. - Send results to the log. + :param `app`: The celery application + :returns: A dict mapping active workers to queues + """ + # Get the information we need from celery + i = app.control.inspect() + active_workers = i.active_queues() + if active_workers is None: + active_workers = {} + + # Build the mapping dictionary + worker_queue_map = {} + for worker, queues in active_workers.items(): + for queue in queues: + if worker in worker_queue_map: + worker_queue_map[worker].append(queue["name"]) + else: + worker_queue_map[worker] = [queue["name"]] + + return worker_queue_map + + +def celerize_queues(queues): """ + Celery requires a queue tag to be prepended to their + queues so this function will 'celerize' every queue in + a list you provide it by prepending the queue tag. + + :param `queues`: A list of queues that need the queue + tag prepended. + """ + from merlin.config.configfile import CONFIG # pylint: disable=C0415 + + for i, queue in enumerate(queues): + queues[i] = f"{CONFIG.celery.queue_tag}{queue}" + + +def _build_output_table(worker_list, output_table): + """ + Helper function for query-status that will build a table + that we'll use as output. + + :param `worker_list`: A list of workers to add to the table + :param `output_table`: A list of tuples where each entry is + of the form (worker name, associated queues) + """ + from merlin.celery import app # pylint: disable=C0415 + + # Get a mapping between workers and the queues they're watching + worker_queue_map = get_active_workers(app) + + # Loop through the list of workers and add an entry in the table + # of the form (worker name, queues attached to this worker) + for worker in worker_list: + if "celery@" not in worker: + worker = f"celery@{worker}" + output_table.append((worker, ", ".join(worker_queue_map[worker]))) + + +def query_celery_workers(spec_worker_names, queues, workers_regex): + """ + Look for existing celery workers. Filter by spec, queues, or + worker names if provided by user. At the end, print a table + of workers and their associated queues. + + :param `spec_worker_names`: The worker names defined in a spec file + :param `queues`: A list of queues to filter by + :param `workers_regex`: A list of regexs to filter by + """ + from merlin.celery import app # pylint: disable=C0415 + from merlin.display import tabulate_info # pylint: disable=C0415 + + # Ping all workers and grab which ones are running workers = get_workers_from_app() - if workers: - LOG.info("Found these connected workers:") - for worker in workers: - LOG.info(worker) - else: + if not workers: LOG.warning("No workers found!") + return + + # Remove prepended celery tag while we filter + workers = [worker.replace("celery@", "") for worker in workers] + workers_to_query = [] + + # --queues flag + if queues: + # Get a mapping between queues and the workers watching them + queue_worker_map, _ = get_queues(app) + # Remove duplicates and prepend the celery queue tag to all queues + queues = list(set(queues)) + celerize_queues(queues) + # Add the workers associated to each queue to the list of workers we're + # going to query + for queue in queues: + try: + workers_to_query.extend(queue_worker_map[queue]) + except KeyError: + LOG.warning(f"No workers connected to {queue}.") + + # --spec flag + if spec_worker_names: + apply_list_of_regex(spec_worker_names, workers, workers_to_query) + + # --workers flag + if workers_regex: + apply_list_of_regex(workers_regex, workers, workers_to_query) + + # Remove any potential duplicates + workers = set(workers) + workers_to_query = set(workers_to_query) + + # If there were filters and nothing was found then we can't display a table + if (queues or spec_worker_names or workers_regex) and not workers_to_query: + LOG.warning("No workers found that match your filters.") + return + + # Build the output table based on our filters + table = [] + if workers_to_query: + _build_output_table(workers_to_query, table) + else: + _build_output_table(workers, table) + + # Display the output table + LOG.info("Found these connected workers:") + tabulate_info(table, headers=["Workers", "Queues"]) + print() def query_celery_queues(queues): @@ -490,7 +608,6 @@ def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): """ from merlin.celery import app # pylint: disable=C0415 - from merlin.config.configfile import CONFIG # pylint: disable=C0415 LOG.debug(f"Sending stop to queues: {queues}, worker_regex: {worker_regex}, spec_worker_names: {spec_worker_names}") active_queues, _ = get_queues(app) @@ -500,8 +617,7 @@ def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): queues = [*active_queues] # Celery adds the queue tag in front of each queue so we add that here else: - for i, queue in enumerate(queues): - queues[i] = f"{CONFIG.celery.queue_tag}{queue}" + celerize_queues(queues) # Find the set of all workers attached to all of those queues all_workers = set() @@ -524,20 +640,16 @@ def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): workers_to_stop = [] # --spec flag if (spec_worker_names is not None) and len(spec_worker_names) > 0: - for worker_name in spec_worker_names: - LOG.debug( - f"""Result of regex_list_filter for {worker_name}: - {regex_list_filter(worker_name, all_workers, match=False)}""" - ) - workers_to_stop += regex_list_filter(worker_name, all_workers, match=False) + apply_list_of_regex(spec_worker_names, all_workers, workers_to_stop) # --workers flag if worker_regex is not None: - for worker in worker_regex: - LOG.debug(f"Result of regex_list_filter: {regex_list_filter(worker, all_workers, match=False)}") - workers_to_stop += regex_list_filter(worker, all_workers, match=False) + LOG.debug(f"Searching for workers to stop based on the following regex's: {worker_regex}") + apply_list_of_regex(worker_regex, all_workers, workers_to_stop) # Remove duplicates workers_to_stop = list(set(workers_to_stop)) + LOG.debug(f"Post-filter worker stop list: {workers_to_stop}") + if workers_to_stop: LOG.info(f"Sending stop to these workers: {workers_to_stop}") app.control.broadcast("shutdown", destination=workers_to_stop) diff --git a/merlin/utils.py b/merlin/utils.py index b3830ccab..1acecd604 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -185,6 +185,26 @@ def regex_list_filter(regex, list_to_filter, match=True): return list(filter(r.search, list_to_filter)) +def apply_list_of_regex(regex_list, list_to_filter, result_list, match=False): + """ + Take a list of regex's, apply each regex to a list we're searching through, + and append each result to a result list. + + :param `regex_list`: A list of regular expressions to apply to the list_to_filter + :param `list_to_filter`: A list that we'll apply regexs to + :param `result_list`: A list that we'll append results of the regex filters to + :param `match`: A bool where when true we use re.match for applying the regex, + when false we use re.search for applying the regex. + """ + for regex in regex_list: + filter_results = set(regex_list_filter(regex, list_to_filter, match)) + + if not filter_results: + LOG.warning(f"No regex match for {regex}.") + else: + result_list += filter_results + + def load_yaml(filepath): """ Safely read a yaml file. diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 91bfac743..8628d8f2b 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -103,6 +103,8 @@ def define_tests(): # pylint: disable=R0914,R0915 run = f"merlin {err_lvl} run" restart = f"merlin {err_lvl} restart" purge = "merlin purge" + stop = "merlin stop-workers" + query = "merlin query-workers" # Shortcuts for example workflow paths examples = "merlin/examples/workflows" @@ -515,7 +517,7 @@ def define_tests(): # pylint: disable=R0914,R0915 } stop_workers_tests = { "stop workers no workers": { - "cmds": "merlin stop-workers", + "cmds": f"{stop}", "conditions": [ HasReturnCode(), HasRegex("No workers found to stop"), @@ -528,7 +530,7 @@ def define_tests(): # pylint: disable=R0914,R0915 "stop workers no flags": { "cmds": [ f"{workers} {mul_workers_demo}", - "merlin stop-workers", + f"{stop}", ], "conditions": [ HasReturnCode(), @@ -544,7 +546,7 @@ def define_tests(): # pylint: disable=R0914,R0915 "stop workers with spec flag": { "cmds": [ f"{workers} {mul_workers_demo}", - f"merlin stop-workers --spec {mul_workers_demo}", + f"{stop} --spec {mul_workers_demo}", ], "conditions": [ HasReturnCode(), @@ -560,7 +562,7 @@ def define_tests(): # pylint: disable=R0914,R0915 "stop workers with workers flag": { "cmds": [ f"{workers} {mul_workers_demo}", - "merlin stop-workers --workers step_1_merlin_test_worker step_2_merlin_test_worker", + f"{stop} --workers step_1_merlin_test_worker step_2_merlin_test_worker", ], "conditions": [ HasReturnCode(), @@ -576,7 +578,7 @@ def define_tests(): # pylint: disable=R0914,R0915 "stop workers with queues flag": { "cmds": [ f"{workers} {mul_workers_demo}", - "merlin stop-workers --queues hello_queue", + f"{stop} --queues hello_queue", ], "conditions": [ HasReturnCode(), @@ -590,6 +592,83 @@ def define_tests(): # pylint: disable=R0914,R0915 "num procs": 2, }, } + query_workers_tests = { + "query workers no workers": { + "cmds": f"{query}", + "conditions": [ + HasReturnCode(), + HasRegex("No workers found!"), + HasRegex("step_1_merlin_test_worker", negate=True), + HasRegex("step_2_merlin_test_worker", negate=True), + HasRegex("other_merlin_test_worker", negate=True), + ], + "run type": "distributed", + }, + "query workers no flags": { + "cmds": [ + f"{workers} {mul_workers_demo}", + f"{query}", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found!", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker"), + HasRegex("other_merlin_test_worker"), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, + "query workers with spec flag": { + "cmds": [ + f"{workers} {mul_workers_demo}", + f"{query} --spec {mul_workers_demo}", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found!", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker"), + HasRegex("other_merlin_test_worker"), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, + "query workers with workers flag": { + "cmds": [ + f"{workers} {mul_workers_demo}", + f"{query} --workers step_1_merlin_test_worker step_2_merlin_test_worker", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found!", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker"), + HasRegex("other_merlin_test_worker", negate=True), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, + "query workers with queues flag": { + "cmds": [ + f"{workers} {mul_workers_demo}", + f"{query} --queues hello_queue", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found!", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker", negate=True), + HasRegex("other_merlin_test_worker", negate=True), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, + } distributed_tests = { # noqa: F841 "run and purge feature_demo": { "cmds": f"{run} {demo}; {purge} {demo} -f", @@ -637,6 +716,7 @@ def define_tests(): # pylint: disable=R0914,R0915 # style_checks, # omitting style checks due to different results on different machines dependency_checks, stop_workers_tests, + query_workers_tests, distributed_tests, ]: all_tests.update(test_dict) From fe98083ff76b8a9708fdc885ea66a2d163f03174 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 3 Apr 2023 11:13:03 -0700 Subject: [PATCH 076/126] fix walltime conversion issue add test for walltime conversion modify changelog run fix-style and fix issue with unit tests modify walltime to work with different formats for all schedulers modify walltime tests --- CHANGELOG.md | 1 + merlin/spec/merlinspec.json | 8 +++- merlin/spec/specification.py | 31 +++++++------ merlin/study/batch.py | 8 ++-- .../{ => flux_fake_command}/flux | 0 tests/integration/fake_commands/lsf | 3 -- .../{ => qsub_fake_command}/qsub | 0 tests/integration/test_definitions.py | 45 ++++++++++++++----- .../test_specs}/flux_par_native_test.yaml | 8 ++-- .../integration/test_specs}/flux_test.yaml | 6 ++- .../integration/test_specs}/slurm_test.yaml | 4 +- tests/unit/spec/test_specification.py | 5 ++- 12 files changed, 76 insertions(+), 43 deletions(-) rename tests/integration/fake_commands/{ => flux_fake_command}/flux (100%) delete mode 100644 tests/integration/fake_commands/lsf rename tests/integration/fake_commands/{ => qsub_fake_command}/qsub (100%) rename {merlin/examples/workflows/flux => tests/integration/test_specs}/flux_par_native_test.yaml (83%) rename {merlin/examples/workflows/flux => tests/integration/test_specs}/flux_test.yaml (78%) rename {merlin/examples/workflows/slurm => tests/integration/test_specs}/slurm_test.yaml (80%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e1c5cbb7..16c1cd091 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Might be able to fix this in the future if we split functions up more - Too-few-public-methods (R0903): These are disabled for classes we may add to in the future or "wrapper" classes - Attribute-defined-outside-init (W0201): These errors are only disabled in `specification.py` as they occur in class methods so init() won't be called +- Fixed an issue where the walltime value in the batch block was being converted to an integer instead of remaining in HH:MM:SS format ### Added - Now loads np.arrays of dtype='object', allowing mix-type sample npy diff --git a/merlin/spec/merlinspec.json b/merlin/spec/merlinspec.json index 3044cd506..4b8ca3633 100644 --- a/merlin/spec/merlinspec.json +++ b/merlin/spec/merlinspec.json @@ -280,7 +280,13 @@ "launch_args": {"type": "string", "minLength": 1}, "worker_launch": {"type": "string", "minLength": 1}, "nodes": {"type": "integer", "minimum": 1}, - "walltime": {"type": "string", "pattern": "^(?:(?:([0-9][0-9]|2[0-3]):)?([0-5][0-9]):)?([0-5][0-9])$"} + "walltime": { + "anyOf": [ + {"type": "string", "minLength": 1}, + {"type": "integer", "minimum": 0}, + {"type": "string", "pattern": "^\\$\\(\\w+\\)$"} + ] + } } } } diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 45d456067..afc70ed84 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -38,12 +38,14 @@ import os import shlex from copy import deepcopy +from datetime import timedelta from io import StringIO import yaml from maestrowf.specification import YAMLSpecification from merlin.spec import all_keys, defaults +from merlin.utils import repr_timedelta LOG = logging.getLogger(__name__) @@ -179,6 +181,18 @@ def load_spec_from_string(cls, string, needs_IO=True, needs_verification=False): spec.verify() LOG.debug("Merlin spec verified.") + # Convert the walltime value back to HMS if PyYAML messed with it + for _, section in spec.yaml_sections.items(): + # Section is a list for the study block + if isinstance(section, list): + for step in section: + if "walltime" in step and isinstance(step["walltime"], int): + step["walltime"] = repr_timedelta(timedelta(seconds=step["walltime"])) + # Section is a dict for all other blocks + if isinstance(section, dict): + if "walltime" in section and isinstance(section["walltime"], int): + section["walltime"] = repr_timedelta(timedelta(seconds=section["walltime"])) + return spec @classmethod @@ -205,7 +219,7 @@ def _populate_spec(cls, data): "load vulnerability. Please upgrade your installation " "to a more recent version!" ) - spec = yaml.load(data) + spec = yaml.load(data, yaml.Loader) LOG.debug("Successfully loaded specification: \n%s", spec["description"]) # Load in the parts of the yaml that are the same as Maestro's @@ -311,21 +325,6 @@ def verify_batch_block(self, schema): # Additional Walltime checks in case the regex from the schema bypasses an error if self.batch["type"] == "lsf" and "walltime" in self.batch: LOG.warning("The walltime argument is not available in lsf.") - elif "walltime" in self.batch: - try: - err_msg = "Walltime must be of the form SS, MM:SS, or HH:MM:SS." - walltime = self.batch["walltime"] - if len(walltime) > 2: - # Walltime must have : if it's not of the form SS - if ":" not in walltime: - raise ValueError(err_msg) - # Walltime must have exactly 2 chars between : - time = walltime.split(":") - for section in time: - if len(section) != 2: - raise ValueError(err_msg) - except Exception: # pylint: disable=W0706 - raise @staticmethod def load_merlin_block(stream): diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 676261678..00e062582 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -40,7 +40,7 @@ import subprocess from typing import Dict, Optional, Union -from merlin.utils import get_flux_alloc, get_yaml_var +from merlin.utils import convert_timestring, get_flux_alloc, get_yaml_var LOG = logging.getLogger(__name__) @@ -294,7 +294,7 @@ def construct_scheduler_legend(parsed_batch: Dict, nodes: int) -> Dict: "expected check output": b"Nodes", "launch": f"{parsed_batch['flux alloc']} -o pty -N {nodes} --exclusive --job-name=merlin", "queue": f" --setattr=system.queue={parsed_batch['queue']}", - "walltime": f" -t {parsed_batch['walltime']}", + "walltime": f" -t {convert_timestring(parsed_batch['walltime'], format_method='FSD')}", }, "lsf": { "check cmd": ["jsrun", "--help"], @@ -308,7 +308,7 @@ def construct_scheduler_legend(parsed_batch: Dict, nodes: int) -> Dict: "expected check output": b"pbs_version", "launch": f"qsub -l nodes={nodes}", "queue": f" -q {parsed_batch['queue']}", - "walltime": f" -l walltime={parsed_batch['walltime']}", + "walltime": f" -l walltime={convert_timestring(parsed_batch['walltime'])}", }, "slurm": { "bank": f" -A {parsed_batch['bank']}", @@ -316,7 +316,7 @@ def construct_scheduler_legend(parsed_batch: Dict, nodes: int) -> Dict: "expected check output": b"sbatch", "launch": f"srun -N {nodes} -n {nodes}", "queue": f" -p {parsed_batch['queue']}", - "walltime": f" -t {parsed_batch['walltime']}", + "walltime": f" -t {convert_timestring(parsed_batch['walltime'])}", }, } return scheduler_legend diff --git a/tests/integration/fake_commands/flux b/tests/integration/fake_commands/flux_fake_command/flux similarity index 100% rename from tests/integration/fake_commands/flux rename to tests/integration/fake_commands/flux_fake_command/flux diff --git a/tests/integration/fake_commands/lsf b/tests/integration/fake_commands/lsf deleted file mode 100644 index 7b6c57f04..000000000 --- a/tests/integration/fake_commands/lsf +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -echo "jsrun str: :param `default`: The default worker launch command to use if a scheduler is found :returns: The appropriate worker launch command """ - fake_cmds_path = "tests/integration/fake_commands" + workers_cmd = default + fake_cmds_path = f"tests/integration/fake_commands/{cmd}_fake_command" + bogus_cmd = f"""PATH="{fake_cmds_path}:$PATH";{default}""" + scheduler_legend = {"flux": {"check cmd": ["flux", "resource", "list"], "expected check output": b"Nodes"}} + + # Use bogus flux/qsub to test if no flux/qsub is present if not shutil.which(cmd): - # Use bogus flux to test if no flux is present - workers_cmd = f"""PATH="{fake_cmds_path}:$PATH";{default}""" - else: - workers_cmd = default + workers_cmd = bogus_cmd + # Use bogus flux if flux is present but slurm is the main scheduler + elif cmd == "flux" and not check_for_scheduler(cmd, scheduler_legend): + workers_cmd = bogus_cmd return workers_cmd @@ -107,15 +113,16 @@ def define_tests(): # pylint: disable=R0914,R0915 # Shortcuts for example workflow paths examples = "merlin/examples/workflows" dev_examples = "merlin/examples/dev_workflows" + test_specs = "tests/integration/test_specs" demo = f"{examples}/feature_demo/feature_demo.yaml" remote_demo = f"{examples}/remote_feature_demo/remote_feature_demo.yaml" demo_pgen = f"{examples}/feature_demo/scripts/pgen.py" simple = f"{examples}/simple_chain/simple_chain.yaml" - slurm = f"{examples}/slurm/slurm_test.yaml" + slurm = f"{test_specs}/slurm_test.yaml" slurm_restart = f"{examples}/slurm/slurm_par_restart.yaml" - flux = f"{examples}/flux/flux_test.yaml" + flux = f"{test_specs}/flux_test.yaml" flux_restart = f"{examples}/flux/flux_par_restart.yaml" - flux_native = f"{examples}/flux/flux_par_native_test.yaml" + flux_native = f"{test_specs}/flux_par_native_test.yaml" lsf = f"{examples}/lsf/lsf_par.yaml" mul_workers_demo = f"{dev_examples}/multiple_workers.yaml" @@ -230,7 +237,13 @@ def define_tests(): # pylint: disable=R0914,R0915 }, "run-workers echo slurm_test": { "cmds": f"{workers} {slurm} --echo", - "conditions": [HasReturnCode(), HasRegex(celery_slurm_regex)], + # Making sure walltime isn't set to an integer w/ last two conditions here + "conditions": [ + HasReturnCode(), + HasRegex(celery_slurm_regex), + HasRegex(r"-t 36000", negate=True), + HasRegex(r"-t 10:00:00"), + ], "run type": "local", }, "run-workers echo lsf_test": { @@ -245,12 +258,22 @@ def define_tests(): # pylint: disable=R0914,R0915 }, "run-workers echo flux_native_test": { "cmds": f"{workers_flux} {flux_native} --echo", - "conditions": [HasReturnCode(), HasRegex(celery_flux_regex)], + "conditions": [ + HasReturnCode(), + HasRegex(celery_flux_regex), + HasRegex(r"-t 36000"), + HasRegex(r"-t 10:00:00", negate=True), + ], "run type": "local", }, "run-workers echo pbs_test": { "cmds": f"{workers_pbs} {flux_native} --echo", - "conditions": [HasReturnCode(), HasRegex(celery_pbs_regex)], + "conditions": [ + HasReturnCode(), + HasRegex(celery_pbs_regex), + HasRegex(r"-l walltime=36000", negate=True), + HasRegex(r"-l walltime=10:00:00"), + ], "run type": "local", }, "run-workers echo override feature_demo": { diff --git a/merlin/examples/workflows/flux/flux_par_native_test.yaml b/tests/integration/test_specs/flux_par_native_test.yaml similarity index 83% rename from merlin/examples/workflows/flux/flux_par_native_test.yaml rename to tests/integration/test_specs/flux_par_native_test.yaml index 4edb76905..8eaf4b024 100644 --- a/merlin/examples/workflows/flux/flux_par_native_test.yaml +++ b/tests/integration/test_specs/flux_par_native_test.yaml @@ -7,17 +7,19 @@ batch: flux_exec: flux exec -r "0-1" flux_start_opts: -o,-S,log-filename=flux_par.out nodes: 1 + walltime: 10:00:00 env: variables: OUTPUT_PATH: ./studies N_SAMPLES: 10 + SCRIPTS: $(SPECROOT)/../../../merlin/examples/workflows/flux/scripts study: - description: Build the code name: build run: - cmd: mpicc -o mpi_hello $(SPECROOT)/scripts/hello.c >& build.out + cmd: mpicc -o mpi_hello $(SCRIPTS)/hello.c >& build.out task_queue: flux_par - description: Echo the params name: runs @@ -44,7 +46,7 @@ study: name: data run: cmd: | - $(SPECROOT)/scripts/flux_info.py > flux_timings.out + $(SCRIPTS)/flux_info.py > flux_timings.out depends: [runs_*] task_queue: flux_par @@ -73,4 +75,4 @@ merlin: column_labels: [V1, V2] file: $(MERLIN_INFO)/samples.npy generate: - cmd: python3 $(SPECROOT)/scripts/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy + cmd: python3 $(SCRIPTS)/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy diff --git a/merlin/examples/workflows/flux/flux_test.yaml b/tests/integration/test_specs/flux_test.yaml similarity index 78% rename from merlin/examples/workflows/flux/flux_test.yaml rename to tests/integration/test_specs/flux_test.yaml index e0f85795f..fe0130526 100644 --- a/merlin/examples/workflows/flux/flux_test.yaml +++ b/tests/integration/test_specs/flux_test.yaml @@ -6,12 +6,14 @@ batch: type: flux nodes: 1 queue: pbatch + walltime: 10:00:00 flux_start_opts: -o,-S,log-filename=flux_test.out env: variables: OUTPUT_PATH: ./studies N_SAMPLES: 10 + SCRIPTS: $(SPECROOT)/../../../merlin/examples/workflows/flux/scripts study: - description: Echo the params @@ -27,7 +29,7 @@ study: name: data run: cmd: | - $(SPECROOT)/scripts/flux_info.py > flux_timings.out + $(SCRIPTS)/flux_info.py > flux_timings.out depends: [runs*] task_queue: flux_test @@ -48,4 +50,4 @@ merlin: column_labels: [V1, V2] file: $(MERLIN_INFO)/samples.npy generate: - cmd: python3 $(SPECROOT)/scripts/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy + cmd: python3 $(SCRIPTS)/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy diff --git a/merlin/examples/workflows/slurm/slurm_test.yaml b/tests/integration/test_specs/slurm_test.yaml similarity index 80% rename from merlin/examples/workflows/slurm/slurm_test.yaml rename to tests/integration/test_specs/slurm_test.yaml index 144a2d871..6398753b5 100644 --- a/merlin/examples/workflows/slurm/slurm_test.yaml +++ b/tests/integration/test_specs/slurm_test.yaml @@ -4,11 +4,13 @@ description: batch: type: slurm + walltime: 10:00:00 env: variables: OUTPUT_PATH: ./studies N_SAMPLES: 10 + SCRIPTS: $(SPECROOT)/../../../merlin/examples/workflows/slurm/scripts study: - description: Echo the params @@ -44,4 +46,4 @@ merlin: column_labels: [V1, V2] file: $(MERLIN_INFO)/samples.npy generate: - cmd: python3 $(SPECROOT)/scripts/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy + cmd: python3 $(SCRIPTS)/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy diff --git a/tests/unit/spec/test_specification.py b/tests/unit/spec/test_specification.py index e1fb09a6b..afc930a6e 100644 --- a/tests/unit/spec/test_specification.py +++ b/tests/unit/spec/test_specification.py @@ -4,6 +4,7 @@ import unittest import yaml +from jsonschema import ValidationError from merlin.spec.specification import MerlinSpec @@ -257,14 +258,14 @@ def test_invalid_walltime(self): # Read in INVALID_MERLIN spec spec = self.read_spec() - invalid_walltimes = ["2", "0:1", "111", "1:1:1", "65", "65:12", "66:77", ":02:12", "123:45:33", ""] + invalid_walltimes = ["", -1] # Loop through the invalid walltimes and make sure they're all caught for time in invalid_walltimes: spec["batch"]["walltime"] = time self.update_spec(spec) - with self.assertRaises(ValueError): + with self.assertRaises((ValidationError, ValueError)): MerlinSpec.load_specification(self.merlin_spec_filepath) # Reset the spec From b6ff025d9ed5766c23da4208da9de28f0f6373a1 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 12 Apr 2023 12:16:55 -0700 Subject: [PATCH 077/126] update version for release of 1.10.0 --- CHANGELOG.md | 4 ++-- Makefile | 2 +- merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/conditions.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 2 +- 55 files changed, 57 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66c4bd313..42811b601 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] +## [1.10.0] ### Fixed - Pip wheel wasn't including .sh files for merlin examples - The learn.py script in the openfoam_wf* examples will now create the missing Energy v Lidspeed plot @@ -42,7 +42,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A function in `run_tests.py` to check that an integration test definition is formatted correctly - A new dev_workflow example `multiple_workers.yaml` that's used for testing the `stop-workers` command - Ability to start 2 subprocesses for a single test -- Added the --distributed and --display-table flags to run_tests.py +- Added the --distributed and --display-tests flags to run_tests.py - --distributed: only run distributed tests - --display-tests: displays a table of all existing tests and the id associated with each test - Added the --disable-logs flag to the `run-workers` command diff --git a/Makefile b/Makefile index 897b44a31..25266ace9 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index 1222a38f6..b4df0fa52 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.9.1" +__version__ = "1.10.0" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index a521d2cc5..02339d429 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index 9febf2523..d09262adc 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 8f5174427..d90050599 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 8f5174427..d90050599 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index 366692ac1..f35a75ae3 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index d7dbf0ff4..dd12cfa93 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index e91105bf8..ceadfa5d2 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 0d786820a..34ccb07b0 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 47e600d82..9f05e43d9 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 8f5174427..d90050599 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 1c9bd342b..819fbc8e7 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 930c67b5f..13304a443 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 3a4077ac3..15c915bf3 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index bd19ac539..6d5f57cff 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 24e4ed5c1..5ee7ed06e 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 0bcb3c42f..466bc56ee 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index 38fba1ddc..c2cc4aab5 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index fc4743b86..38043c115 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 3a03b3d79..de092b7a4 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 8d85ccf50..f4cc093cc 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 8f5174427..d90050599 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index cc05fd8ea..0263e6c7c 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 8f5174427..d90050599 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index bfd65a6a8..7181f055c 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 3145b65e3..84c38d0b2 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index ea376d1b5..5273fb8e7 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 475982d8c..8dc3a83e9 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 45e6f0c0c..41dbb05e3 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 0db33d22e..01a86b2ae 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index d1a356737..a48f86ed4 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 5653cba21..6fae626c9 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index ecaa76b4e..e0a481171 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index e433a4ebf..445495802 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 55c475f31..7a5da16f5 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 8f5174427..d90050599 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 5da1fed22..858c8401e 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index a1794bed2..e2dc3f651 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index a76c56c16..a535c7bd9 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 2c3194b50..7862993fe 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index afc70ed84..6fb524f5b 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 8f5174427..d90050599 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 00e062582..e05342fd1 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 0a47820e6..7c2e42b72 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 7281875f5..bdc508f87 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 91173ca9c..e637c0939 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index b80dd2766..7302344ff 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index f43125dc6..9ee0c131d 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index 1acecd604..ed7f12b9a 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index 1a1ad94ba..efbc5a529 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index b3acdde4e..22d911e7e 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index d809af2c1..c858649c8 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 4465316ad..6fd8e3ce9 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.1. +# This file is part of Merlin, Version: 1.10.0. # # For details, see https://github.com/LLNL/merlin. # From aeaef79d2557c66f9dddb989147a539a2d5b439e Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 3 May 2023 17:20:14 -0700 Subject: [PATCH 078/126] fix default worker bug with all steps --- CHANGELOG.md | 7 +++ merlin/spec/specification.py | 7 ++- tests/integration/test_definitions.py | 10 ++++ .../test_specs/default_worker_test.yaml | 56 +++++++++++++++++++ .../test_specs/no_default_worker_test.yaml | 56 +++++++++++++++++++ 5 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_specs/default_worker_test.yaml create mode 100644 tests/integration/test_specs/no_default_worker_test.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 42811b601..e5ba817ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Fixed +- A bug where assigning a worker all steps also assigned steps to the default worker + +### Added +- Tests to make sure the default worker is being assigned properly + ## [1.10.0] ### Fixed - Pip wheel wasn't including .sh files for merlin examples diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 6fb524f5b..9f3a6c16f 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -385,8 +385,11 @@ def process_spec_defaults(self): MerlinSpec.fill_missing_defaults(worker_settings, defaults.WORKER) worker_steps.extend(worker_settings["steps"]) - # Figure out which steps still need workers - steps_that_need_workers = list(set(all_workflow_steps) - set(worker_steps)) + if "all" in worker_steps: + steps_that_need_workers = [] + else: + # Figure out which steps still need workers + steps_that_need_workers = list(set(all_workflow_steps) - set(worker_steps)) # If there are still steps remaining that haven't been assigned a worker yet, # assign the remaining steps to the default worker. If all the steps still need workers diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 6fd8e3ce9..2b990ea40 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -283,6 +283,16 @@ def define_tests(): # pylint: disable=R0914,R0915 "conditions": [HasReturnCode(), HasRegex("custom_verify_queue")], "run type": "local", }, + "default_worker assigned": { + "cmds": f"{workers} {test_specs}/default_worker_test.yaml --echo", + "conditions": [HasReturnCode(), HasRegex(r"default_worker.*-Q '\[merlin\]_step_4_queue'")], + "run type": "local", + }, + "no default_worker assigned": { + "cmds": f"{workers} {test_specs}/no_default_worker_test.yaml --echo", + "conditions": [HasReturnCode(), HasRegex(r"default_worker", negate=True)], + "run type": "local", + }, } wf_format_tests = { "local minimum_format": { diff --git a/tests/integration/test_specs/default_worker_test.yaml b/tests/integration/test_specs/default_worker_test.yaml new file mode 100644 index 000000000..974921593 --- /dev/null +++ b/tests/integration/test_specs/default_worker_test.yaml @@ -0,0 +1,56 @@ +description: + name: multiple_workers + description: a very simple merlin workflow with multiple workers + +global.parameters: + GREET: + values : ["hello","hola"] + label : GREET.%% + WORLD: + values : ["world","mundo"] + label : WORLD.%% + +study: + - name: step_1 + description: say hello + run: + cmd: | + echo "$(GREET), $(WORLD)!" + task_queue: hello_queue + + - name: step_2 + description: step 2 + run: + cmd: | + echo "step_2" + depends: [step_1_*] + task_queue: echo_queue + + - name: step_3 + description: stop workers + run: + cmd: | + echo "stop workers" + depends: [step_2] + task_queue: other_queue + + - name: step_4 + description: another step + run: + cmd: | + echo "another step" + depends: [step_3] + task_queue: step_4_queue + +merlin: + resources: + workers: + step_1_merlin_test_worker: + args: -l INFO + steps: [step_1] + step_2_merlin_test_worker: + args: -l INFO + steps: [step_2] + other_merlin_test_worker: + args: -l INFO + steps: [step_3] diff --git a/tests/integration/test_specs/no_default_worker_test.yaml b/tests/integration/test_specs/no_default_worker_test.yaml new file mode 100644 index 000000000..b2f15d2ee --- /dev/null +++ b/tests/integration/test_specs/no_default_worker_test.yaml @@ -0,0 +1,56 @@ +description: + name: multiple_workers + description: a very simple merlin workflow with multiple workers + +global.parameters: + GREET: + values : ["hello","hola"] + label : GREET.%% + WORLD: + values : ["world","mundo"] + label : WORLD.%% + +study: + - name: step_1 + description: say hello + run: + cmd: | + echo "$(GREET), $(WORLD)!" + task_queue: hello_queue + + - name: step_2 + description: step 2 + run: + cmd: | + echo "step_2" + depends: [step_1_*] + task_queue: echo_queue + + - name: step_3 + description: stop workers + run: + cmd: | + echo "stop workers" + depends: [step_2] + task_queue: other_queue + + - name: step_4 + description: another step + run: + cmd: | + echo "another step" + depends: [step_3] + task_queue: step_4_queue + +merlin: + resources: + workers: + step_1_merlin_test_worker: + args: -l INFO + steps: [step_1] + step_2_merlin_test_worker: + args: -l INFO + steps: [step_2] + other_merlin_test_worker: + args: -l INFO + steps: [all] From f7508dd9dcf51dc2ada4ba8f64835466975f360f Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 4 May 2023 14:08:56 -0700 Subject: [PATCH 079/126] version bump and requirements fix --- CHANGELOG.md | 5 ++++- Makefile | 2 +- merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/examples/workflows/feature_demo/requirements.txt | 2 +- .../examples/workflows/remote_feature_demo/requirements.txt | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/conditions.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 2 +- 57 files changed, 61 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5ba817ef..15c9e0ffa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,16 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.10.1] ### Fixed - A bug where assigning a worker all steps also assigned steps to the default worker ### Added - Tests to make sure the default worker is being assigned properly +### Changed +- Requirement name in examples/workflows/remote_feature_demo/requirements.txt and examples/workflows/feature_demo/requirements.txt from sklearn to scikit-learn since sklearn is now deprecated + ## [1.10.0] ### Fixed - Pip wheel wasn't including .sh files for merlin examples diff --git a/Makefile b/Makefile index 25266ace9..0153f10d4 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index b4df0fa52..32bf5c875 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.10.0" +__version__ = "1.10.1" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 02339d429..bb804d876 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index d09262adc..072c83b58 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index f35a75ae3..81c8865e5 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index dd12cfa93..f51055ab5 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index ceadfa5d2..65d503564 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 34ccb07b0..635dd617b 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 9f05e43d9..eb24e067e 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 819fbc8e7..22c505df4 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 13304a443..4d7dc176f 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 15c915bf3..3b41cb459 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 6d5f57cff..43dbd133a 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 5ee7ed06e..aee1b62b0 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 466bc56ee..0cf9b7c6d 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index c2cc4aab5..2740aa0a1 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 38043c115..6aa38f9b2 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index de092b7a4..691bf5a6c 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index f4cc093cc..9778cde3c 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index 0263e6c7c..89d14781e 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 7181f055c..a66abf64a 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 84c38d0b2..b859ff605 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/workflows/feature_demo/requirements.txt b/merlin/examples/workflows/feature_demo/requirements.txt index 93909e8aa..e308e1895 100644 --- a/merlin/examples/workflows/feature_demo/requirements.txt +++ b/merlin/examples/workflows/feature_demo/requirements.txt @@ -1,2 +1,2 @@ -sklearn +scikit-learn merlin-spellbook diff --git a/merlin/examples/workflows/remote_feature_demo/requirements.txt b/merlin/examples/workflows/remote_feature_demo/requirements.txt index 93909e8aa..e308e1895 100644 --- a/merlin/examples/workflows/remote_feature_demo/requirements.txt +++ b/merlin/examples/workflows/remote_feature_demo/requirements.txt @@ -1,2 +1,2 @@ -sklearn +scikit-learn merlin-spellbook diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 5273fb8e7..cfbc21e38 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 8dc3a83e9..580952b1d 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 41dbb05e3..f49d3cc94 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 01a86b2ae..4809c7aee 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index a48f86ed4..160909ddf 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 6fae626c9..7c6148fee 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index e0a481171..d53463c2b 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 445495802..14abfb0d4 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 7a5da16f5..d101af2f0 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 858c8401e..f58731336 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index e2dc3f651..b20ce7d09 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index a535c7bd9..38f03d6e9 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 7862993fe..0f1e71f62 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 9f3a6c16f..8d36297ab 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index e05342fd1..eeaead5ee 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 7c2e42b72..4c01cfd73 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index bdc508f87..a85a86e47 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index e637c0939..2df053051 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 7302344ff..14f273dfa 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 9ee0c131d..3a51c926e 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index ed7f12b9a..7c18407e7 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index efbc5a529..9693b06ba 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 22d911e7e..db21e5429 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index c858649c8..9b0270d3b 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 2b990ea40..093644f9f 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # From ab739c667d39c2e7c60d584f667495d893019019 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Thu, 4 May 2023 14:49:23 -0700 Subject: [PATCH 080/126] Version 1.10.1 (#421) * fix default worker bug with all steps * version bump and requirements fix --- CHANGELOG.md | 10 ++++ Makefile | 2 +- merlin/__init__.py | 4 +- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- .../security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- .../workflows/feature_demo/requirements.txt | 2 +- .../remote_feature_demo/requirements.txt | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 9 ++- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/conditions.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 12 +++- .../test_specs/default_worker_test.yaml | 56 +++++++++++++++++++ .../test_specs/no_default_worker_test.yaml | 56 +++++++++++++++++++ 59 files changed, 194 insertions(+), 59 deletions(-) create mode 100644 tests/integration/test_specs/default_worker_test.yaml create mode 100644 tests/integration/test_specs/no_default_worker_test.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 42811b601..15c9e0ffa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.10.1] +### Fixed +- A bug where assigning a worker all steps also assigned steps to the default worker + +### Added +- Tests to make sure the default worker is being assigned properly + +### Changed +- Requirement name in examples/workflows/remote_feature_demo/requirements.txt and examples/workflows/feature_demo/requirements.txt from sklearn to scikit-learn since sklearn is now deprecated + ## [1.10.0] ### Fixed - Pip wheel wasn't including .sh files for merlin examples diff --git a/Makefile b/Makefile index 25266ace9..0153f10d4 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index b4df0fa52..32bf5c875 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.10.0" +__version__ = "1.10.1" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 02339d429..bb804d876 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index d09262adc..072c83b58 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index f35a75ae3..81c8865e5 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index dd12cfa93..f51055ab5 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index ceadfa5d2..65d503564 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 34ccb07b0..635dd617b 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 9f05e43d9..eb24e067e 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 819fbc8e7..22c505df4 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 13304a443..4d7dc176f 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 15c915bf3..3b41cb459 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 6d5f57cff..43dbd133a 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 5ee7ed06e..aee1b62b0 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 466bc56ee..0cf9b7c6d 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index c2cc4aab5..2740aa0a1 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 38043c115..6aa38f9b2 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index de092b7a4..691bf5a6c 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index f4cc093cc..9778cde3c 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index 0263e6c7c..89d14781e 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 7181f055c..a66abf64a 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 84c38d0b2..b859ff605 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/workflows/feature_demo/requirements.txt b/merlin/examples/workflows/feature_demo/requirements.txt index 93909e8aa..e308e1895 100644 --- a/merlin/examples/workflows/feature_demo/requirements.txt +++ b/merlin/examples/workflows/feature_demo/requirements.txt @@ -1,2 +1,2 @@ -sklearn +scikit-learn merlin-spellbook diff --git a/merlin/examples/workflows/remote_feature_demo/requirements.txt b/merlin/examples/workflows/remote_feature_demo/requirements.txt index 93909e8aa..e308e1895 100644 --- a/merlin/examples/workflows/remote_feature_demo/requirements.txt +++ b/merlin/examples/workflows/remote_feature_demo/requirements.txt @@ -1,2 +1,2 @@ -sklearn +scikit-learn merlin-spellbook diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 5273fb8e7..cfbc21e38 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 8dc3a83e9..580952b1d 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 41dbb05e3..f49d3cc94 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 01a86b2ae..4809c7aee 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index a48f86ed4..160909ddf 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 6fae626c9..7c6148fee 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index e0a481171..d53463c2b 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 445495802..14abfb0d4 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 7a5da16f5..d101af2f0 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 858c8401e..f58731336 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index e2dc3f651..b20ce7d09 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index a535c7bd9..38f03d6e9 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 7862993fe..0f1e71f62 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 6fb524f5b..8d36297ab 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # @@ -385,8 +385,11 @@ def process_spec_defaults(self): MerlinSpec.fill_missing_defaults(worker_settings, defaults.WORKER) worker_steps.extend(worker_settings["steps"]) - # Figure out which steps still need workers - steps_that_need_workers = list(set(all_workflow_steps) - set(worker_steps)) + if "all" in worker_steps: + steps_that_need_workers = [] + else: + # Figure out which steps still need workers + steps_that_need_workers = list(set(all_workflow_steps) - set(worker_steps)) # If there are still steps remaining that haven't been assigned a worker yet, # assign the remaining steps to the default worker. If all the steps still need workers diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index e05342fd1..eeaead5ee 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 7c2e42b72..4c01cfd73 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index bdc508f87..a85a86e47 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index e637c0939..2df053051 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 7302344ff..14f273dfa 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 9ee0c131d..3a51c926e 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index ed7f12b9a..7c18407e7 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index efbc5a529..9693b06ba 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 22d911e7e..db21e5429 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index c858649c8..9b0270d3b 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 6fd8e3ce9..093644f9f 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # @@ -283,6 +283,16 @@ def define_tests(): # pylint: disable=R0914,R0915 "conditions": [HasReturnCode(), HasRegex("custom_verify_queue")], "run type": "local", }, + "default_worker assigned": { + "cmds": f"{workers} {test_specs}/default_worker_test.yaml --echo", + "conditions": [HasReturnCode(), HasRegex(r"default_worker.*-Q '\[merlin\]_step_4_queue'")], + "run type": "local", + }, + "no default_worker assigned": { + "cmds": f"{workers} {test_specs}/no_default_worker_test.yaml --echo", + "conditions": [HasReturnCode(), HasRegex(r"default_worker", negate=True)], + "run type": "local", + }, } wf_format_tests = { "local minimum_format": { diff --git a/tests/integration/test_specs/default_worker_test.yaml b/tests/integration/test_specs/default_worker_test.yaml new file mode 100644 index 000000000..974921593 --- /dev/null +++ b/tests/integration/test_specs/default_worker_test.yaml @@ -0,0 +1,56 @@ +description: + name: multiple_workers + description: a very simple merlin workflow with multiple workers + +global.parameters: + GREET: + values : ["hello","hola"] + label : GREET.%% + WORLD: + values : ["world","mundo"] + label : WORLD.%% + +study: + - name: step_1 + description: say hello + run: + cmd: | + echo "$(GREET), $(WORLD)!" + task_queue: hello_queue + + - name: step_2 + description: step 2 + run: + cmd: | + echo "step_2" + depends: [step_1_*] + task_queue: echo_queue + + - name: step_3 + description: stop workers + run: + cmd: | + echo "stop workers" + depends: [step_2] + task_queue: other_queue + + - name: step_4 + description: another step + run: + cmd: | + echo "another step" + depends: [step_3] + task_queue: step_4_queue + +merlin: + resources: + workers: + step_1_merlin_test_worker: + args: -l INFO + steps: [step_1] + step_2_merlin_test_worker: + args: -l INFO + steps: [step_2] + other_merlin_test_worker: + args: -l INFO + steps: [step_3] diff --git a/tests/integration/test_specs/no_default_worker_test.yaml b/tests/integration/test_specs/no_default_worker_test.yaml new file mode 100644 index 000000000..b2f15d2ee --- /dev/null +++ b/tests/integration/test_specs/no_default_worker_test.yaml @@ -0,0 +1,56 @@ +description: + name: multiple_workers + description: a very simple merlin workflow with multiple workers + +global.parameters: + GREET: + values : ["hello","hola"] + label : GREET.%% + WORLD: + values : ["world","mundo"] + label : WORLD.%% + +study: + - name: step_1 + description: say hello + run: + cmd: | + echo "$(GREET), $(WORLD)!" + task_queue: hello_queue + + - name: step_2 + description: step 2 + run: + cmd: | + echo "step_2" + depends: [step_1_*] + task_queue: echo_queue + + - name: step_3 + description: stop workers + run: + cmd: | + echo "stop workers" + depends: [step_2] + task_queue: other_queue + + - name: step_4 + description: another step + run: + cmd: | + echo "another step" + depends: [step_3] + task_queue: step_4_queue + +merlin: + resources: + workers: + step_1_merlin_test_worker: + args: -l INFO + steps: [step_1] + step_2_merlin_test_worker: + args: -l INFO + steps: [step_2] + other_merlin_test_worker: + args: -l INFO + steps: [all] From 98e06f6557774f9cac005d03511151f567c17f56 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Thu, 25 May 2023 13:05:39 -0700 Subject: [PATCH 081/126] Bugfix/filename-special-vars (#425) * fix file naming bug * fix filename bug with variable as study name * add tests for the file name special vars changes * modify changelog * implement Luc's suggestions * remove replace line --- CHANGELOG.md | 11 ++++++ merlin/study/study.py | 57 +++++++++++++-------------- tests/integration/conditions.py | 12 ++++-- tests/integration/test_definitions.py | 24 +++++++---- tests/unit/study/test_study.py | 50 ++++++++++++++++++++++- 5 files changed, 111 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15c9e0ffa..603086aaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] +### Fixed +- A bug where the .orig, .partial, and .expanded file names were using the study name rather than the original file name + +### Added +- Tests for ensuring `$(MERLIN_SPEC_ORIGINAL_TEMPLATE)`, `$(MERLIN_SPEC_ARCHIVED_COPY)`, and `$(MERLIN_SPEC_EXECUTED_RUN)` are stored correctly + +### Changed +- The ProvenanceYAMLFileHasRegex condition for integration tests now saves the study name and spec file name as attributes instead of just the study name + - This lead to minor changes in 3 tests ("local override feature demo", "local pgen feature demo", and "remote feature demo") with what we pass to this specific condition + ## [1.10.1] ### Fixed - A bug where assigning a worker all steps also assigned steps to the default worker diff --git a/merlin/study/study.py b/merlin/study/study.py index 3a51c926e..5446c84a1 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -36,6 +36,7 @@ import time from contextlib import suppress from copy import deepcopy +from pathlib import Path from cached_property import cached_property from maestrowf.datastructures.core import Study @@ -83,6 +84,7 @@ def __init__( # pylint: disable=R0913 pgen_file=None, pargs=None, ): + self.filepath = filepath self.original_spec = MerlinSpec.load_specification(filepath) self.override_vars = override_vars error_override_vars(self.override_vars, self.original_spec.path) @@ -114,19 +116,8 @@ def __init__( # pylint: disable=R0913 # below will be substituted for sample values on execution "MERLIN_SAMPLE_VECTOR": " ".join([f"$({k})" for k in self.get_sample_labels(from_spec=self.original_spec)]), "MERLIN_SAMPLE_NAMES": " ".join(self.get_sample_labels(from_spec=self.original_spec)), - "MERLIN_SPEC_ORIGINAL_TEMPLATE": os.path.join( - self.info, - self.original_spec.description["name"].replace(" ", "_") + ".orig.yaml", - ), - "MERLIN_SPEC_EXECUTED_RUN": os.path.join( - self.info, - self.original_spec.description["name"].replace(" ", "_") + ".partial.yaml", - ), - "MERLIN_SPEC_ARCHIVED_COPY": os.path.join( - self.info, - self.original_spec.description["name"].replace(" ", "_") + ".expanded.yaml", - ), } + self._set_special_file_vars() self.pgen_file = pgen_file self.pargs = pargs @@ -134,12 +125,27 @@ def __init__( # pylint: disable=R0913 self.dag = None self.load_dag() - def write_original_spec(self, filename): + def _set_special_file_vars(self): + """Setter for the orig, partial, and expanded file paths of a study.""" + base_name = Path(self.filepath).stem + self.special_vars["MERLIN_SPEC_ORIGINAL_TEMPLATE"] = os.path.join( + self.info, + base_name + ".orig.yaml", + ) + self.special_vars["MERLIN_SPEC_EXECUTED_RUN"] = os.path.join( + self.info, + base_name + ".partial.yaml", + ) + self.special_vars["MERLIN_SPEC_ARCHIVED_COPY"] = os.path.join( + self.info, + base_name + ".expanded.yaml", + ) + + def write_original_spec(self): """ - Copy the original spec into merlin_info/ as '.orig.yaml'. + Copy the original spec into merlin_info/ as '.orig.yaml'. """ - spec_name = os.path.join(self.info, filename + ".orig.yaml") - shutil.copyfile(self.original_spec.path, spec_name) + shutil.copyfile(self.original_spec.path, self.special_vars["MERLIN_SPEC_ORIGINAL_TEMPLATE"]) def label_clash_error(self): """ @@ -368,10 +374,6 @@ def expanded_spec(self): return self.get_expanded_spec() result = self.get_expanded_spec() - expanded_name = result.description["name"].replace(" ", "_") + ".expanded.yaml" - - # Set expanded filepath - expanded_filepath = os.path.join(self.info, expanded_name) # expand provenance spec filename if contains_token(self.original_spec.name) or contains_shell_ref(self.original_spec.name): @@ -394,8 +396,8 @@ def expanded_spec(self): self.workspace = expanded_workspace self.info = os.path.join(self.workspace, "merlin_info") self.special_vars["MERLIN_INFO"] = self.info + self._set_special_file_vars() - expanded_filepath = os.path.join(self.info, expanded_name) new_spec_text = expand_by_line(result.dump(), MerlinStudy.get_user_vars(result)) result = MerlinSpec.load_spec_from_string(new_spec_text) result = expand_env_vars(result) @@ -412,15 +414,13 @@ def expanded_spec(self): os.path.join(self.info, os.path.basename(self.samples_file)), ) - # write expanded spec for provenance - with open(expanded_filepath, "w") as f: # pylint: disable=C0103 + # write expanded spec for provenance and set the path (necessary for testing) + with open(self.special_vars["MERLIN_SPEC_ARCHIVED_COPY"], "w") as f: # pylint: disable=C0103 f.write(result.dump()) + result.path = self.special_vars["MERLIN_SPEC_ARCHIVED_COPY"] # write original spec for provenance - result = MerlinSpec.load_spec_from_string(result.dump()) - result.path = expanded_filepath - name = result.description["name"].replace(" ", "_") - self.write_original_spec(name) + self.write_original_spec() # write partially-expanded spec for provenance partial_spec = deepcopy(self.original_spec) @@ -428,8 +428,7 @@ def expanded_spec(self): partial_spec.environment["variables"] = result.environment["variables"] if "labels" in result.environment: partial_spec.environment["labels"] = result.environment["labels"] - partial_spec_path = os.path.join(self.info, name + ".partial.yaml") - with open(partial_spec_path, "w") as f: # pylint: disable=C0103 + with open(self.special_vars["MERLIN_SPEC_EXECUTED_RUN"], "w") as f: # pylint: disable=C0103 f.write(partial_spec.dump()) LOG.info(f"Study workspace is '{self.workspace}'.") diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index db21e5429..4da8c36a1 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -249,14 +249,16 @@ class ProvenanceYAMLFileHasRegex(HasRegex): MUST contain a given regular expression. """ - def __init__(self, regex, name, output_path, provenance_type, negate=False): # pylint: disable=R0913 + def __init__(self, regex, spec_file_name, study_name, output_path, provenance_type, negate=False): # pylint: disable=R0913 """ :param `regex`: a string regex pattern - :param `name`: the name of a study + :param `spec_file_name`: the name of the spec file + :param `study_name`: the name of a study :param `output_path`: the $(OUTPUT_PATH) of a study """ super().__init__(regex, negate=negate) - self.name = name + self.spec_file_name = spec_file_name + self.study_name = study_name self.output_path = output_path provenance_types = ["orig", "partial", "expanded"] if provenance_type not in provenance_types: @@ -277,7 +279,9 @@ def glob_string(self): """ Returns a regex string for the glob library to recursively find files with. """ - return f"{self.output_path}/{self.name}" f"_[0-9]*-[0-9]*/merlin_info/{self.name}.{self.prov_type}.yaml" + return ( + f"{self.output_path}/{self.study_name}" f"_[0-9]*-[0-9]*/merlin_info/{self.spec_file_name}.{self.prov_type}.yaml" + ) def is_within(self): # pylint: disable=W0221 """ diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 093644f9f..cf7c008de 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -435,31 +435,36 @@ def define_tests(): # pylint: disable=R0914,R0915 HasReturnCode(), ProvenanceYAMLFileHasRegex( regex=r"HELLO: \$\(SCRIPTS\)/hello_world.py", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="orig", ), ProvenanceYAMLFileHasRegex( regex=r"name: \$\(NAME\)", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="partial", ), ProvenanceYAMLFileHasRegex( regex="studies/feature_demo_", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="partial", ), ProvenanceYAMLFileHasRegex( regex="name: feature_demo", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", ), ProvenanceYAMLFileHasRegex( regex=r"\$\(NAME\)", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", negate=True, @@ -510,13 +515,15 @@ def define_tests(): # pylint: disable=R0914,R0915 "conditions": [ ProvenanceYAMLFileHasRegex( regex=r"\[0.3333333", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", ), ProvenanceYAMLFileHasRegex( regex=r"\[0.5", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", negate=True, @@ -715,7 +722,8 @@ def define_tests(): # pylint: disable=R0914,R0915 HasReturnCode(), ProvenanceYAMLFileHasRegex( regex="cli_test_demo_workers:", - name="feature_demo", + spec_file_name="remote_feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", ), diff --git a/tests/unit/study/test_study.py b/tests/unit/study/test_study.py index a00995d55..cb15805cd 100644 --- a/tests/unit/study/test_study.py +++ b/tests/unit/study/test_study.py @@ -41,6 +41,15 @@ nodes: 1 task_queue: hello_queue + - name: test_special_vars + description: test the special vars + run: + cmd: | + echo $(MERLIN_SPEC_ORIGINAL_TEMPLATE) + echo $(MERLIN_SPEC_EXECUTED_RUN) + echo $(MERLIN_SPEC_ARCHIVED_COPY) + task_queue: special_var_queue + global.parameters: X2: values : [0.5] @@ -234,11 +243,16 @@ class TestMerlinStudy(unittest.TestCase): @staticmethod def file_contains_string(f, string): - return string in open(f, "r").read() + result = False + with open(f, "r") as infile: + if string in infile.read(): + result = True + return result def setUp(self): self.tmpdir = tempfile.mkdtemp() - self.merlin_spec_filepath = os.path.join(self.tmpdir, "basic_ensemble.yaml") + self.base_name = "basic_ensemble" + self.merlin_spec_filepath = os.path.join(self.tmpdir, f"{self.base_name}.yaml") with open(self.merlin_spec_filepath, "w+") as _file: _file.write(MERLIN_SPEC) @@ -263,6 +277,34 @@ def test_expanded_spec(self): assert TestMerlinStudy.file_contains_string(self.study.expanded_spec.path, "$PATH") assert not TestMerlinStudy.file_contains_string(self.study.expanded_spec.path, "PATH_VAR: $PATH") + # Special vars are in the second step of MERLIN_SPEC so grab that step here + original_special_var_step = self.study.original_spec.study[1]["run"]["cmd"] + expanded_special_var_step = self.study.expanded_spec.study[1]["run"]["cmd"] + + # Make sure the special filepath variables aren't expanded in the original spec + assert "$(MERLIN_SPEC_ORIGINAL_TEMPLATE)" in original_special_var_step + assert "$(MERLIN_SPEC_EXECUTED_RUN)" in original_special_var_step + assert "$(MERLIN_SPEC_ARCHIVED_COPY)" in original_special_var_step + + # Make sure the special filepath variables aren't left in their variable form in the expanded spec + assert "$(MERLIN_SPEC_ORIGINAL_TEMPLATE)" not in expanded_special_var_step + assert "$(MERLIN_SPEC_EXECUTED_RUN)" not in expanded_special_var_step + assert "$(MERLIN_SPEC_ARCHIVED_COPY)" not in expanded_special_var_step + + # Make sure the special filepath variables we're expanded appropriately in the expanded spec + assert ( + f"{self.base_name}.orig.yaml" in expanded_special_var_step + and "unit_test1.orig.yaml" not in expanded_special_var_step + ) + assert ( + f"{self.base_name}.partial.yaml" in expanded_special_var_step + and "unit_test1.partial.yaml" not in expanded_special_var_step + ) + assert ( + f"{self.base_name}.expanded.yaml" in expanded_special_var_step + and "unit_test1.expanded.yaml" not in expanded_special_var_step + ) + def test_column_label_conflict(self): """ If there is a common key between Maestro's global.parameters and @@ -291,3 +333,7 @@ def test_no_env(self): assert isinstance(study_no_env, MerlinStudy), bad_type_err except Exception as e: assert False, f"Encountered unexpected exception, {e}, for viable MerlinSpec without optional 'env' section." + + +if __name__ == "__main__": + unittest.main() From b8dd2b2bc69657601b49e07a026f9e1d3c74fc71 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 26 May 2023 08:25:54 -0700 Subject: [PATCH 082/126] Create dependabot-changelog-updater.yml --- .../dependabot-changelog-updater.yml | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/dependabot-changelog-updater.yml diff --git a/.github/workflows/dependabot-changelog-updater.yml b/.github/workflows/dependabot-changelog-updater.yml new file mode 100644 index 000000000..1677cf050 --- /dev/null +++ b/.github/workflows/dependabot-changelog-updater.yml @@ -0,0 +1,34 @@ +# See https://github.com/dangoslen/dependabot-changelog-helper for more info +name: Dependabot Changelog Updater +on: + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + - labeled + - unlabeled + +jobs: + changelog: + runs-on: ubuntu-latest + if: ${{ github.actor == 'dependabot[bot]' }} + steps: + - uses: actions/checkout@v3 + with: + # Depending on your needs, you can use a token that will re-trigger workflows + # See https://github.com/stefanzweifel/git-auto-commit-action#commits-of-this-action-do-not-trigger-new-workflow-runs + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: dangoslen/dependabot-changelog-helper@v3 + with: + version: ${{ needs.setup.outputs.version }} + activationLabel: 'dependabot' + changelogPath: './CHANGELOG.md' + + # This step is required for committing the changes to your branch. + # See https://github.com/stefanzweifel/git-auto-commit-action#commits-of-this-action-do-not-trigger-new-workflow-runs + - uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: "Updated Changelog" From 4c404235c6c5cf775a9db9606413c4a9cfb0fb52 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 26 May 2023 08:40:45 -0700 Subject: [PATCH 083/126] testing outputs of modifying changelog --- .../dependabot-changelog-updater.yml | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/dependabot-changelog-updater.yml b/.github/workflows/dependabot-changelog-updater.yml index 1677cf050..a8afccc92 100644 --- a/.github/workflows/dependabot-changelog-updater.yml +++ b/.github/workflows/dependabot-changelog-updater.yml @@ -11,24 +11,30 @@ on: - unlabeled jobs: - changelog: + changelog-updater: runs-on: ubuntu-latest if: ${{ github.actor == 'dependabot[bot]' }} steps: - - uses: actions/checkout@v3 - with: - # Depending on your needs, you can use a token that will re-trigger workflows - # See https://github.com/stefanzweifel/git-auto-commit-action#commits-of-this-action-do-not-trigger-new-workflow-runs - token: ${{ secrets.GITHUB_TOKEN }} +# - uses: actions/checkout@v3 +# with: +# # Depending on your needs, you can use a token that will re-trigger workflows +# # See https://github.com/stefanzweifel/git-auto-commit-action#commits-of-this-action-do-not-trigger-new-workflow-runs +# token: ${{ secrets.GITHUB_TOKEN }} - - uses: dangoslen/dependabot-changelog-helper@v3 + - name: modify changelog + id: modify-changelog + uses: dangoslen/dependabot-changelog-helper@v3 with: version: ${{ needs.setup.outputs.version }} activationLabel: 'dependabot' changelogPath: './CHANGELOG.md' + + - name: commit changes + run: | + echo ${{ steps.modify-changelog.outputs }} # This step is required for committing the changes to your branch. # See https://github.com/stefanzweifel/git-auto-commit-action#commits-of-this-action-do-not-trigger-new-workflow-runs - - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: "Updated Changelog" +# - uses: stefanzweifel/git-auto-commit-action@v4 +# with: +# commit_message: "Updated Changelog" From 2d231fb937303839934d4ee2048d554252ce9dbd Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 26 May 2023 09:13:57 -0700 Subject: [PATCH 084/126] delete dependabot-changelog-updater --- .../dependabot-changelog-updater.yml | 40 ------------------- 1 file changed, 40 deletions(-) delete mode 100644 .github/workflows/dependabot-changelog-updater.yml diff --git a/.github/workflows/dependabot-changelog-updater.yml b/.github/workflows/dependabot-changelog-updater.yml deleted file mode 100644 index a8afccc92..000000000 --- a/.github/workflows/dependabot-changelog-updater.yml +++ /dev/null @@ -1,40 +0,0 @@ -# See https://github.com/dangoslen/dependabot-changelog-helper for more info -name: Dependabot Changelog Updater -on: - pull_request: - types: - - opened - - synchronize - - reopened - - ready_for_review - - labeled - - unlabeled - -jobs: - changelog-updater: - runs-on: ubuntu-latest - if: ${{ github.actor == 'dependabot[bot]' }} - steps: -# - uses: actions/checkout@v3 -# with: -# # Depending on your needs, you can use a token that will re-trigger workflows -# # See https://github.com/stefanzweifel/git-auto-commit-action#commits-of-this-action-do-not-trigger-new-workflow-runs -# token: ${{ secrets.GITHUB_TOKEN }} - - - name: modify changelog - id: modify-changelog - uses: dangoslen/dependabot-changelog-helper@v3 - with: - version: ${{ needs.setup.outputs.version }} - activationLabel: 'dependabot' - changelogPath: './CHANGELOG.md' - - - name: commit changes - run: | - echo ${{ steps.modify-changelog.outputs }} - - # This step is required for committing the changes to your branch. - # See https://github.com/stefanzweifel/git-auto-commit-action#commits-of-this-action-do-not-trigger-new-workflow-runs -# - uses: stefanzweifel/git-auto-commit-action@v4 -# with: -# commit_message: "Updated Changelog" From 6b142d9b0f67f2cca6856b722b96114ef7a2ca7b Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Wed, 28 Jun 2023 07:56:45 -0700 Subject: [PATCH 085/126] feature/pdf-docs (#427) * first attempt at adding pdf * fixing build error * modify changelog to show docs changes * fix errors Luc found in the build logs * trying out removal of latex * reverting latex changes back * uncommenting the latex_elements settings * adding epub to see if latex will build * adding a latex engine variable to conf * fix naming error with latex_engine * attempting to add a logo to the pdf build * testing an override to the searchtools file * revert back to not using searchtools override * update changelog --- .readthedocs.yaml | 2 ++ CHANGELOG.md | 3 +++ docs/source/conf.py | 13 ++++++++----- docs/source/faq.rst | 2 +- docs/source/merlin_developer.rst | 3 +-- docs/source/merlin_variables.rst | 2 +- docs/source/modules/installation/installation.rst | 2 +- 7 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index c1c252e30..ee62bcb5e 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -11,3 +11,5 @@ sphinx: python: install: - requirements: docs/requirements.txt + +formats: [pdf] \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 603086aaa..25ee73963 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Fixed - A bug where the .orig, .partial, and .expanded file names were using the study name rather than the original file name +- Some build warnings in the docs (unknown targets, duplicate targets, title underlines too short, etc.) ### Added - Tests for ensuring `$(MERLIN_SPEC_ORIGINAL_TEMPLATE)`, `$(MERLIN_SPEC_ARCHIVED_COPY)`, and `$(MERLIN_SPEC_EXECUTED_RUN)` are stored correctly +- A pdf download format for the docs ### Changed - The ProvenanceYAMLFileHasRegex condition for integration tests now saves the study name and spec file name as attributes instead of just the study name - This lead to minor changes in 3 tests ("local override feature demo", "local pgen feature demo", and "remote feature demo") with what we pass to this specific condition +- Uncommented Latex support in the docs configuration to get pdf builds working ## [1.10.1] ### Fixed diff --git a/docs/source/conf.py b/docs/source/conf.py index 315978a6a..b578e8672 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -66,7 +66,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -122,21 +122,24 @@ # -- Options for LaTeX output ------------------------------------------------ +latex_engine = "pdflatex" latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # - # 'papersize': 'letterpaper', + 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # - # 'pointsize': '10pt', + 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # - # 'preamble': '', + 'preamble': '', # Latex figure (float) alignment # - # 'figure_align': 'htbp', + 'figure_align': 'htbp', } +latex_logo = "../images/merlin.png" + # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). diff --git a/docs/source/faq.rst b/docs/source/faq.rst index d0ef8e109..3632aab6c 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -383,7 +383,7 @@ in the ``launch_args`` variable in the batch section. What is PBS? ~~~~~~~~~~~~ Another job scheduler. See `Portable Batch System -https://en.wikipedia.org/wiki/Portable_Batch_System`_ +`_ . This functionality is only available to launch a flux scheduler. diff --git a/docs/source/merlin_developer.rst b/docs/source/merlin_developer.rst index 08947ca89..e88352f3e 100644 --- a/docs/source/merlin_developer.rst +++ b/docs/source/merlin_developer.rst @@ -164,8 +164,7 @@ properties labeled ``name`` and ``population`` that are both required, it would } Here, ``name`` can only be a string but ``population`` can be both a string and an integer. -For help with json schema formatting, check out the `step-by-step getting started guide -`_. +For help with json schema formatting, check out the `step-by-step getting started guide`_. The next step is to enable this block in the schema validation process. To do this we need to: diff --git a/docs/source/merlin_variables.rst b/docs/source/merlin_variables.rst index 7f545a4d2..d67e06412 100644 --- a/docs/source/merlin_variables.rst +++ b/docs/source/merlin_variables.rst @@ -161,7 +161,7 @@ Reserved variables $(MERLIN_INFO)/*.expanded.yaml The ``LAUNCHER`` Variable -+++++++++++++++++++++ ++++++++++++++++++++++++++ ``$(LAUNCHER)`` is a special case of a reserved variable since it's value *can* be changed. It serves as an abstraction to launch a job with parallel schedulers like :ref:`slurm`, diff --git a/docs/source/modules/installation/installation.rst b/docs/source/modules/installation/installation.rst index d18261af5..96195ff3d 100644 --- a/docs/source/modules/installation/installation.rst +++ b/docs/source/modules/installation/installation.rst @@ -229,7 +229,7 @@ If everything is set up correctly, you should see: (OPTIONAL) Docker Advanced Installation ----------------------------- +--------------------------------------- RabbitMQ Server +++++++++++++++ From e3e1a307d6c65fcc834fe4a3aba2271f9109932d Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Wed, 28 Jun 2023 10:18:25 -0700 Subject: [PATCH 086/126] bugfix/openfoam_singularity_issues (#426) * fix openfoam_singularity issues * update requirements and descriptions for openfoam examples --- CHANGELOG.md | 2 ++ merlin/examples/generator.py | 2 ++ merlin/examples/workflows/openfoam_wf/openfoam_wf.yaml | 1 + merlin/examples/workflows/openfoam_wf/openfoam_wf_template.yaml | 1 + merlin/examples/workflows/openfoam_wf/requirements.txt | 2 +- .../workflows/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml | 1 + .../openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml | 1 + .../examples/workflows/openfoam_wf_no_docker/requirements.txt | 2 +- .../{openfoam_wf.yaml => openfoam_wf_singularity.yaml} | 1 + .../examples/workflows/openfoam_wf_singularity/requirements.txt | 2 +- 10 files changed, 12 insertions(+), 3 deletions(-) rename merlin/examples/workflows/openfoam_wf_singularity/{openfoam_wf.yaml => openfoam_wf_singularity.yaml} (99%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25ee73963..ebaea86f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Fixed - A bug where the .orig, .partial, and .expanded file names were using the study name rather than the original file name +- A bug where the openfoam_wf_singularity example was not being found - Some build warnings in the docs (unknown targets, duplicate targets, title underlines too short, etc.) ### Added @@ -16,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - The ProvenanceYAMLFileHasRegex condition for integration tests now saves the study name and spec file name as attributes instead of just the study name - This lead to minor changes in 3 tests ("local override feature demo", "local pgen feature demo", and "remote feature demo") with what we pass to this specific condition +- Updated scikit-learn requirement for the openfoam_wf_singularity example - Uncommented Latex support in the docs configuration to get pdf builds working ## [1.10.1] diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index b859ff605..308785784 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -83,6 +83,8 @@ def list_examples(): directory = os.path.join(os.path.join(EXAMPLES_DIR, example_dir), "") specs = glob.glob(directory + "*.yaml") for spec in specs: + if "template" in spec: + continue with open(spec) as f: # pylint: disable=C0103 try: spec_metadata = yaml.safe_load(f)["description"] diff --git a/merlin/examples/workflows/openfoam_wf/openfoam_wf.yaml b/merlin/examples/workflows/openfoam_wf/openfoam_wf.yaml index e6dabc8c2..0d98445c0 100644 --- a/merlin/examples/workflows/openfoam_wf/openfoam_wf.yaml +++ b/merlin/examples/workflows/openfoam_wf/openfoam_wf.yaml @@ -3,6 +3,7 @@ description: description: | A parameter study that includes initializing, running, post-processing, collecting, learning and visualizing OpenFOAM runs + using docker. env: diff --git a/merlin/examples/workflows/openfoam_wf/openfoam_wf_template.yaml b/merlin/examples/workflows/openfoam_wf/openfoam_wf_template.yaml index 96e13064b..64433d3af 100644 --- a/merlin/examples/workflows/openfoam_wf/openfoam_wf_template.yaml +++ b/merlin/examples/workflows/openfoam_wf/openfoam_wf_template.yaml @@ -3,6 +3,7 @@ description: description: | A parameter study that includes initializing, running, post-processing, collecting, learning and visualizing OpenFOAM runs + using docker. env: diff --git a/merlin/examples/workflows/openfoam_wf/requirements.txt b/merlin/examples/workflows/openfoam_wf/requirements.txt index 8042c2422..ef63ca016 100644 --- a/merlin/examples/workflows/openfoam_wf/requirements.txt +++ b/merlin/examples/workflows/openfoam_wf/requirements.txt @@ -1,3 +1,3 @@ Ofpp==0.11 -scikit-learn==0.21.3 +scikit-learn>=1.0.2 matplotlib==3.1.1 diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml b/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml index ab8224029..688e8ceff 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml +++ b/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml @@ -3,6 +3,7 @@ description: description: | A parameter study that includes initializing, running, post-processing, collecting, learning and vizualizing OpenFOAM runs + without using docker. env: diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml b/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml index 084a1c0bb..3fcbc3588 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml +++ b/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml @@ -3,6 +3,7 @@ description: description: | A parameter study that includes initializing, running, post-processing, collecting, learning and vizualizing OpenFOAM runs + without using docker. env: diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/requirements.txt b/merlin/examples/workflows/openfoam_wf_no_docker/requirements.txt index 8042c2422..ef63ca016 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/requirements.txt +++ b/merlin/examples/workflows/openfoam_wf_no_docker/requirements.txt @@ -1,3 +1,3 @@ Ofpp==0.11 -scikit-learn==0.21.3 +scikit-learn>=1.0.2 matplotlib==3.1.1 diff --git a/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml b/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf_singularity.yaml similarity index 99% rename from merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml rename to merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf_singularity.yaml index 3f3bbb672..03837bcb3 100644 --- a/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml +++ b/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf_singularity.yaml @@ -3,6 +3,7 @@ description: description: | A parameter study that includes initializing, running, post-processing, collecting, learning and visualizing OpenFOAM runs + using singularity. env: diff --git a/merlin/examples/workflows/openfoam_wf_singularity/requirements.txt b/merlin/examples/workflows/openfoam_wf_singularity/requirements.txt index 8042c2422..ef63ca016 100644 --- a/merlin/examples/workflows/openfoam_wf_singularity/requirements.txt +++ b/merlin/examples/workflows/openfoam_wf_singularity/requirements.txt @@ -1,3 +1,3 @@ Ofpp==0.11 -scikit-learn==0.21.3 +scikit-learn>=1.0.2 matplotlib==3.1.1 From 12ff3d782d30579b1387eb314f33924d33d1877f Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Tue, 11 Jul 2023 11:26:15 -0700 Subject: [PATCH 087/126] bugfix/output-path-substitution (#430) * fix bug with output_path and variable substitution * add tests for cli substitutions --- CHANGELOG.md | 2 ++ merlin/study/study.py | 15 +++++++++++-- tests/integration/test_definitions.py | 22 +++++++++++++++++++ .../test_specs/cli_substitution_test.yaml | 14 ++++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_specs/cli_substitution_test.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index ebaea86f2..d1452104b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A bug where the .orig, .partial, and .expanded file names were using the study name rather than the original file name - A bug where the openfoam_wf_singularity example was not being found - Some build warnings in the docs (unknown targets, duplicate targets, title underlines too short, etc.) +- A bug where when the output path contained a variable that was overridden, the overridden variable was not changed in the output_path ### Added - Tests for ensuring `$(MERLIN_SPEC_ORIGINAL_TEMPLATE)`, `$(MERLIN_SPEC_ARCHIVED_COPY)`, and `$(MERLIN_SPEC_EXECUTED_RUN)` are stored correctly - A pdf download format for the docs +- Tests for cli substitutions ### Changed - The ProvenanceYAMLFileHasRegex condition for integration tests now saves the study name and spec file name as attributes instead of just the study name diff --git a/merlin/study/study.py b/merlin/study/study.py index 5446c84a1..74b7c3181 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -306,8 +306,17 @@ def output_path(self): output_path = str(self.original_spec.output_path) - if (self.override_vars is not None) and ("OUTPUT_PATH" in self.override_vars): - output_path = str(self.override_vars["OUTPUT_PATH"]) + # If there are override vars we need to check that the output path doesn't need changed + if self.override_vars is not None: + # Case where output path is directly modified + if "OUTPUT_PATH" in self.override_vars: + output_path = str(self.override_vars["OUTPUT_PATH"]) + else: + for var_name, var_val in self.override_vars.items(): + token = f"$({var_name})" + # Case where output path contains a variable that was overridden + if token in output_path: + output_path = output_path.replace(token, str(var_val)) output_path = expand_line(output_path, self.user_vars, env_vars=True) output_path = os.path.abspath(output_path) @@ -315,6 +324,8 @@ def output_path(self): os.makedirs(output_path) LOG.info(f"Made dir(s) to output path '{output_path}'.") + LOG.info(f"OUTPUT_PATH: {os.path.basename(output_path)}") + return output_path @cached_property diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index cf7c008de..d20efb329 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -127,6 +127,7 @@ def define_tests(): # pylint: disable=R0914,R0915 flux_native = f"{test_specs}/flux_par_native_test.yaml" lsf = f"{examples}/lsf/lsf_par.yaml" mul_workers_demo = f"{dev_examples}/multiple_workers.yaml" + cli_substitution_wf = f"{test_specs}/cli_substitution_test.yaml" # Other shortcuts black = "black --check --target-version py36" @@ -323,6 +324,26 @@ def define_tests(): # pylint: disable=R0914,R0915 "run type": "local", }, } + cli_substitution_tests = { + "no substitutions": { + "cmds": f"merlin run {cli_substitution_wf} --local", + "conditions": [HasReturnCode(), HasRegex(r"OUTPUT_PATH: output_path_no_substitution")], + "run type": "local", + "cleanup": "rm -r output_path_no_substitution", + }, + "output_path substitution": { + "cmds": f"merlin run {cli_substitution_wf} --local --vars OUTPUT_PATH=output_path_substitution", + "conditions": [HasReturnCode(), HasRegex(r"OUTPUT_PATH: output_path_substitution")], + "run type": "local", + "cleanup": "rm -r output_path_substitution", + }, + "output_path w/ variable substitution": { + "cmds": f"merlin run {cli_substitution_wf} --local --vars SUB=variable_sub", + "conditions": [HasReturnCode(), HasRegex(r"OUTPUT_PATH: output_path_variable_sub")], + "run type": "local", + "cleanup": "rm -r output_path_variable_sub", + }, + } example_tests = { "example failure": {"cmds": "merlin example failure", "conditions": HasRegex("not found"), "run type": "local"}, "example simple_chain": { @@ -748,6 +769,7 @@ def define_tests(): # pylint: disable=R0914,R0915 examples_check, run_workers_echo_tests, wf_format_tests, + cli_substitution_tests, example_tests, restart_step_tests, restart_wf_tests, diff --git a/tests/integration/test_specs/cli_substitution_test.yaml b/tests/integration/test_specs/cli_substitution_test.yaml new file mode 100644 index 000000000..5cbbb70d3 --- /dev/null +++ b/tests/integration/test_specs/cli_substitution_test.yaml @@ -0,0 +1,14 @@ +description: + name: cli_substitution_test + description: a spec that helps test cli substitutions + +env: + variables: + SUB: no_substitution + OUTPUT_PATH: output_path_$(SUB) + +study: + - name: step1 + description: step 1 + run: + cmd: echo "test" From 5c69c0beaf74962abd585ea321a6f071d28ad881 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 4 Aug 2023 15:28:55 -0700 Subject: [PATCH 088/126] bugfix/scheduler-permission-error (#436) --- CHANGELOG.md | 1 + merlin/study/batch.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1452104b..981ce7198 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A bug where the openfoam_wf_singularity example was not being found - Some build warnings in the docs (unknown targets, duplicate targets, title underlines too short, etc.) - A bug where when the output path contained a variable that was overridden, the overridden variable was not changed in the output_path +- A bug where permission denied errors happened when checking for system scheduler ### Added - Tests for ensuring `$(MERLIN_SPEC_ORIGINAL_TEMPLATE)`, `$(MERLIN_SPEC_ARCHIVED_COPY)`, and `$(MERLIN_SPEC_EXECUTED_RUN)` are stored correctly diff --git a/merlin/study/batch.py b/merlin/study/batch.py index eeaead5ee..f5b62409f 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -92,7 +92,7 @@ def check_for_scheduler(scheduler, scheduler_legend): if result and len(result) > 0 and scheduler_legend[scheduler]["expected check output"] in result[0]: return True return False - except FileNotFoundError: + except (FileNotFoundError, PermissionError): return False From c01f6358402ee7c20a486560adf2e8f0591150b5 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 4 Aug 2023 16:04:11 -0700 Subject: [PATCH 089/126] Release/1.10.2 (#437) * bump version to 1.10.2 * bump version in CHANGELOG --- .gitignore | 1 + CHANGELOG.md | 2 +- Makefile | 2 +- merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/conditions.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 2 +- 56 files changed, 57 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index 8b0fb8ad2..c22521934 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,4 @@ jupyter/testDistributedSamples.py dist/ build/ .DS_Store +.vscode/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 981ce7198..5e7c0b1cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] +## [1.10.2] ### Fixed - A bug where the .orig, .partial, and .expanded file names were using the study name rather than the original file name - A bug where the openfoam_wf_singularity example was not being found diff --git a/Makefile b/Makefile index 0153f10d4..030ed8d15 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index 32bf5c875..9856d9a7b 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.10.1" +__version__ = "1.10.2" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index bb804d876..5d61d8ed1 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index 072c83b58..dc0cfd9d7 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index 81c8865e5..be663a572 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index f51055ab5..a4c127a12 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 65d503564..f8b881dcf 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 635dd617b..821c1df71 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index eb24e067e..fe9d735d4 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 22c505df4..dfd230a1a 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 4d7dc176f..2c70fadc9 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 3b41cb459..b24e0af3e 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 43dbd133a..1f851abb2 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index aee1b62b0..a53190df7 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 0cf9b7c6d..8c7dc8a2b 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index 2740aa0a1..73605768e 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 6aa38f9b2..6ccf3dea5 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 691bf5a6c..d30d5ab58 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 9778cde3c..2f5434091 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index 89d14781e..bf60bb50d 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index a66abf64a..a7754cec7 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 308785784..5b893ab9e 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index cfbc21e38..b176378a9 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 580952b1d..e90e13324 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index f49d3cc94..d3fe4b80a 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 4809c7aee..b2bcc4949 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 160909ddf..779333dc6 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 7c6148fee..6cd46fa97 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index d53463c2b..93f928f3b 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 14abfb0d4..e088eedcd 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index d101af2f0..a3cc7f021 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index f58731336..145eb00bb 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index b20ce7d09..1560f412d 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 38f03d6e9..abba6ef02 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 0f1e71f62..e9ec4bceb 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 8d36297ab..1cb26d512 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index f5b62409f..b2157ebde 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 4c01cfd73..f9c04b1e3 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index a85a86e47..d2315bf7f 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 2df053051..dc01ad6a9 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 14f273dfa..686b5afa2 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 74b7c3181..8eaf306ca 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index 7c18407e7..a2e4966d0 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index 9693b06ba..0bb413325 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 4da8c36a1..0bc687923 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 9b0270d3b..82a4fdd92 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index d20efb329..e9e9ce590 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # From 9c52ba2799e52f713f622ece8139f67a2d2b04d3 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 4 Aug 2023 17:04:11 -0700 Subject: [PATCH 090/126] resolve develop to main merge issues (#439) * fix default worker bug with all steps * version bump and requirements fix From 261e0350610134ea0c255d9d3f41f2aff689eba2 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:54:22 -0700 Subject: [PATCH 091/126] Version 1.10.2 (#438) * fix default worker bug with all steps * version bump and requirements fix * Bugfix/filename-special-vars (#425) * fix file naming bug * fix filename bug with variable as study name * add tests for the file name special vars changes * modify changelog * implement Luc's suggestions * remove replace line * Create dependabot-changelog-updater.yml * testing outputs of modifying changelog * delete dependabot-changelog-updater * feature/pdf-docs (#427) * first attempt at adding pdf * fixing build error * modify changelog to show docs changes * fix errors Luc found in the build logs * trying out removal of latex * reverting latex changes back * uncommenting the latex_elements settings * adding epub to see if latex will build * adding a latex engine variable to conf * fix naming error with latex_engine * attempting to add a logo to the pdf build * testing an override to the searchtools file * revert back to not using searchtools override * update changelog * bugfix/openfoam_singularity_issues (#426) * fix openfoam_singularity issues * update requirements and descriptions for openfoam examples * bugfix/output-path-substitution (#430) * fix bug with output_path and variable substitution * add tests for cli substitutions * bugfix/scheduler-permission-error (#436) * Release/1.10.2 (#437) * bump version to 1.10.2 * bump version in CHANGELOG * resolve develop to main merge issues (#439) * fix default worker bug with all steps * version bump and requirements fix --- .gitignore | 1 + .readthedocs.yaml | 2 + CHANGELOG.md | 19 +++++ Makefile | 2 +- docs/source/conf.py | 13 ++-- docs/source/faq.rst | 2 +- docs/source/merlin_developer.rst | 3 +- docs/source/merlin_variables.rst | 2 +- .../modules/installation/installation.rst | 2 +- merlin/__init__.py | 4 +- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- .../security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 4 +- .../workflows/openfoam_wf/openfoam_wf.yaml | 1 + .../openfoam_wf/openfoam_wf_template.yaml | 1 + .../workflows/openfoam_wf/requirements.txt | 2 +- .../openfoam_wf_no_docker.yaml | 1 + .../openfoam_wf_no_docker_template.yaml | 1 + .../openfoam_wf_no_docker/requirements.txt | 2 +- ...m_wf.yaml => openfoam_wf_singularity.yaml} | 1 + .../openfoam_wf_singularity/requirements.txt | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 4 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 74 +++++++++++-------- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/conditions.py | 14 ++-- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 48 +++++++++--- .../test_specs/cli_substitution_test.yaml | 14 ++++ tests/unit/study/test_study.py | 50 ++++++++++++- 72 files changed, 249 insertions(+), 114 deletions(-) rename merlin/examples/workflows/openfoam_wf_singularity/{openfoam_wf.yaml => openfoam_wf_singularity.yaml} (99%) create mode 100644 tests/integration/test_specs/cli_substitution_test.yaml diff --git a/.gitignore b/.gitignore index 8b0fb8ad2..c22521934 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,4 @@ jupyter/testDistributedSamples.py dist/ build/ .DS_Store +.vscode/ diff --git a/.readthedocs.yaml b/.readthedocs.yaml index c1c252e30..ee62bcb5e 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -11,3 +11,5 @@ sphinx: python: install: - requirements: docs/requirements.txt + +formats: [pdf] \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 15c9e0ffa..5e7c0b1cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,25 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.10.2] +### Fixed +- A bug where the .orig, .partial, and .expanded file names were using the study name rather than the original file name +- A bug where the openfoam_wf_singularity example was not being found +- Some build warnings in the docs (unknown targets, duplicate targets, title underlines too short, etc.) +- A bug where when the output path contained a variable that was overridden, the overridden variable was not changed in the output_path +- A bug where permission denied errors happened when checking for system scheduler + +### Added +- Tests for ensuring `$(MERLIN_SPEC_ORIGINAL_TEMPLATE)`, `$(MERLIN_SPEC_ARCHIVED_COPY)`, and `$(MERLIN_SPEC_EXECUTED_RUN)` are stored correctly +- A pdf download format for the docs +- Tests for cli substitutions + +### Changed +- The ProvenanceYAMLFileHasRegex condition for integration tests now saves the study name and spec file name as attributes instead of just the study name + - This lead to minor changes in 3 tests ("local override feature demo", "local pgen feature demo", and "remote feature demo") with what we pass to this specific condition +- Updated scikit-learn requirement for the openfoam_wf_singularity example +- Uncommented Latex support in the docs configuration to get pdf builds working + ## [1.10.1] ### Fixed - A bug where assigning a worker all steps also assigned steps to the default worker diff --git a/Makefile b/Makefile index 0153f10d4..030ed8d15 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/docs/source/conf.py b/docs/source/conf.py index 315978a6a..b578e8672 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -66,7 +66,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -122,21 +122,24 @@ # -- Options for LaTeX output ------------------------------------------------ +latex_engine = "pdflatex" latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # - # 'papersize': 'letterpaper', + 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # - # 'pointsize': '10pt', + 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # - # 'preamble': '', + 'preamble': '', # Latex figure (float) alignment # - # 'figure_align': 'htbp', + 'figure_align': 'htbp', } +latex_logo = "../images/merlin.png" + # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). diff --git a/docs/source/faq.rst b/docs/source/faq.rst index d0ef8e109..3632aab6c 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -383,7 +383,7 @@ in the ``launch_args`` variable in the batch section. What is PBS? ~~~~~~~~~~~~ Another job scheduler. See `Portable Batch System -https://en.wikipedia.org/wiki/Portable_Batch_System`_ +`_ . This functionality is only available to launch a flux scheduler. diff --git a/docs/source/merlin_developer.rst b/docs/source/merlin_developer.rst index 08947ca89..e88352f3e 100644 --- a/docs/source/merlin_developer.rst +++ b/docs/source/merlin_developer.rst @@ -164,8 +164,7 @@ properties labeled ``name`` and ``population`` that are both required, it would } Here, ``name`` can only be a string but ``population`` can be both a string and an integer. -For help with json schema formatting, check out the `step-by-step getting started guide -`_. +For help with json schema formatting, check out the `step-by-step getting started guide`_. The next step is to enable this block in the schema validation process. To do this we need to: diff --git a/docs/source/merlin_variables.rst b/docs/source/merlin_variables.rst index 7f545a4d2..d67e06412 100644 --- a/docs/source/merlin_variables.rst +++ b/docs/source/merlin_variables.rst @@ -161,7 +161,7 @@ Reserved variables $(MERLIN_INFO)/*.expanded.yaml The ``LAUNCHER`` Variable -+++++++++++++++++++++ ++++++++++++++++++++++++++ ``$(LAUNCHER)`` is a special case of a reserved variable since it's value *can* be changed. It serves as an abstraction to launch a job with parallel schedulers like :ref:`slurm`, diff --git a/docs/source/modules/installation/installation.rst b/docs/source/modules/installation/installation.rst index d18261af5..96195ff3d 100644 --- a/docs/source/modules/installation/installation.rst +++ b/docs/source/modules/installation/installation.rst @@ -229,7 +229,7 @@ If everything is set up correctly, you should see: (OPTIONAL) Docker Advanced Installation ----------------------------- +--------------------------------------- RabbitMQ Server +++++++++++++++ diff --git a/merlin/__init__.py b/merlin/__init__.py index 32bf5c875..9856d9a7b 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.10.1" +__version__ = "1.10.2" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index bb804d876..5d61d8ed1 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index 072c83b58..dc0cfd9d7 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index 81c8865e5..be663a572 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index f51055ab5..a4c127a12 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 65d503564..f8b881dcf 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 635dd617b..821c1df71 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index eb24e067e..fe9d735d4 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 22c505df4..dfd230a1a 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 4d7dc176f..2c70fadc9 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 3b41cb459..b24e0af3e 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 43dbd133a..1f851abb2 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index aee1b62b0..a53190df7 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 0cf9b7c6d..8c7dc8a2b 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index 2740aa0a1..73605768e 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 6aa38f9b2..6ccf3dea5 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 691bf5a6c..d30d5ab58 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 9778cde3c..2f5434091 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index 89d14781e..bf60bb50d 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index a66abf64a..a7754cec7 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index b859ff605..5b893ab9e 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # @@ -83,6 +83,8 @@ def list_examples(): directory = os.path.join(os.path.join(EXAMPLES_DIR, example_dir), "") specs = glob.glob(directory + "*.yaml") for spec in specs: + if "template" in spec: + continue with open(spec) as f: # pylint: disable=C0103 try: spec_metadata = yaml.safe_load(f)["description"] diff --git a/merlin/examples/workflows/openfoam_wf/openfoam_wf.yaml b/merlin/examples/workflows/openfoam_wf/openfoam_wf.yaml index e6dabc8c2..0d98445c0 100644 --- a/merlin/examples/workflows/openfoam_wf/openfoam_wf.yaml +++ b/merlin/examples/workflows/openfoam_wf/openfoam_wf.yaml @@ -3,6 +3,7 @@ description: description: | A parameter study that includes initializing, running, post-processing, collecting, learning and visualizing OpenFOAM runs + using docker. env: diff --git a/merlin/examples/workflows/openfoam_wf/openfoam_wf_template.yaml b/merlin/examples/workflows/openfoam_wf/openfoam_wf_template.yaml index 96e13064b..64433d3af 100644 --- a/merlin/examples/workflows/openfoam_wf/openfoam_wf_template.yaml +++ b/merlin/examples/workflows/openfoam_wf/openfoam_wf_template.yaml @@ -3,6 +3,7 @@ description: description: | A parameter study that includes initializing, running, post-processing, collecting, learning and visualizing OpenFOAM runs + using docker. env: diff --git a/merlin/examples/workflows/openfoam_wf/requirements.txt b/merlin/examples/workflows/openfoam_wf/requirements.txt index 8042c2422..ef63ca016 100644 --- a/merlin/examples/workflows/openfoam_wf/requirements.txt +++ b/merlin/examples/workflows/openfoam_wf/requirements.txt @@ -1,3 +1,3 @@ Ofpp==0.11 -scikit-learn==0.21.3 +scikit-learn>=1.0.2 matplotlib==3.1.1 diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml b/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml index ab8224029..688e8ceff 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml +++ b/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml @@ -3,6 +3,7 @@ description: description: | A parameter study that includes initializing, running, post-processing, collecting, learning and vizualizing OpenFOAM runs + without using docker. env: diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml b/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml index 084a1c0bb..3fcbc3588 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml +++ b/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml @@ -3,6 +3,7 @@ description: description: | A parameter study that includes initializing, running, post-processing, collecting, learning and vizualizing OpenFOAM runs + without using docker. env: diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/requirements.txt b/merlin/examples/workflows/openfoam_wf_no_docker/requirements.txt index 8042c2422..ef63ca016 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/requirements.txt +++ b/merlin/examples/workflows/openfoam_wf_no_docker/requirements.txt @@ -1,3 +1,3 @@ Ofpp==0.11 -scikit-learn==0.21.3 +scikit-learn>=1.0.2 matplotlib==3.1.1 diff --git a/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml b/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf_singularity.yaml similarity index 99% rename from merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml rename to merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf_singularity.yaml index 3f3bbb672..03837bcb3 100644 --- a/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml +++ b/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf_singularity.yaml @@ -3,6 +3,7 @@ description: description: | A parameter study that includes initializing, running, post-processing, collecting, learning and visualizing OpenFOAM runs + using singularity. env: diff --git a/merlin/examples/workflows/openfoam_wf_singularity/requirements.txt b/merlin/examples/workflows/openfoam_wf_singularity/requirements.txt index 8042c2422..ef63ca016 100644 --- a/merlin/examples/workflows/openfoam_wf_singularity/requirements.txt +++ b/merlin/examples/workflows/openfoam_wf_singularity/requirements.txt @@ -1,3 +1,3 @@ Ofpp==0.11 -scikit-learn==0.21.3 +scikit-learn>=1.0.2 matplotlib==3.1.1 diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index cfbc21e38..b176378a9 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 580952b1d..e90e13324 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index f49d3cc94..d3fe4b80a 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 4809c7aee..b2bcc4949 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 160909ddf..779333dc6 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 7c6148fee..6cd46fa97 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index d53463c2b..93f928f3b 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 14abfb0d4..e088eedcd 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index d101af2f0..a3cc7f021 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index f58731336..145eb00bb 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index b20ce7d09..1560f412d 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 38f03d6e9..abba6ef02 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 0f1e71f62..e9ec4bceb 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 8d36297ab..1cb26d512 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index eeaead5ee..b2157ebde 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # @@ -92,7 +92,7 @@ def check_for_scheduler(scheduler, scheduler_legend): if result and len(result) > 0 and scheduler_legend[scheduler]["expected check output"] in result[0]: return True return False - except FileNotFoundError: + except (FileNotFoundError, PermissionError): return False diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 4c01cfd73..f9c04b1e3 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index a85a86e47..d2315bf7f 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 2df053051..dc01ad6a9 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 14f273dfa..686b5afa2 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 3a51c926e..8eaf306ca 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # @@ -36,6 +36,7 @@ import time from contextlib import suppress from copy import deepcopy +from pathlib import Path from cached_property import cached_property from maestrowf.datastructures.core import Study @@ -83,6 +84,7 @@ def __init__( # pylint: disable=R0913 pgen_file=None, pargs=None, ): + self.filepath = filepath self.original_spec = MerlinSpec.load_specification(filepath) self.override_vars = override_vars error_override_vars(self.override_vars, self.original_spec.path) @@ -114,19 +116,8 @@ def __init__( # pylint: disable=R0913 # below will be substituted for sample values on execution "MERLIN_SAMPLE_VECTOR": " ".join([f"$({k})" for k in self.get_sample_labels(from_spec=self.original_spec)]), "MERLIN_SAMPLE_NAMES": " ".join(self.get_sample_labels(from_spec=self.original_spec)), - "MERLIN_SPEC_ORIGINAL_TEMPLATE": os.path.join( - self.info, - self.original_spec.description["name"].replace(" ", "_") + ".orig.yaml", - ), - "MERLIN_SPEC_EXECUTED_RUN": os.path.join( - self.info, - self.original_spec.description["name"].replace(" ", "_") + ".partial.yaml", - ), - "MERLIN_SPEC_ARCHIVED_COPY": os.path.join( - self.info, - self.original_spec.description["name"].replace(" ", "_") + ".expanded.yaml", - ), } + self._set_special_file_vars() self.pgen_file = pgen_file self.pargs = pargs @@ -134,12 +125,27 @@ def __init__( # pylint: disable=R0913 self.dag = None self.load_dag() - def write_original_spec(self, filename): + def _set_special_file_vars(self): + """Setter for the orig, partial, and expanded file paths of a study.""" + base_name = Path(self.filepath).stem + self.special_vars["MERLIN_SPEC_ORIGINAL_TEMPLATE"] = os.path.join( + self.info, + base_name + ".orig.yaml", + ) + self.special_vars["MERLIN_SPEC_EXECUTED_RUN"] = os.path.join( + self.info, + base_name + ".partial.yaml", + ) + self.special_vars["MERLIN_SPEC_ARCHIVED_COPY"] = os.path.join( + self.info, + base_name + ".expanded.yaml", + ) + + def write_original_spec(self): """ - Copy the original spec into merlin_info/ as '.orig.yaml'. + Copy the original spec into merlin_info/ as '.orig.yaml'. """ - spec_name = os.path.join(self.info, filename + ".orig.yaml") - shutil.copyfile(self.original_spec.path, spec_name) + shutil.copyfile(self.original_spec.path, self.special_vars["MERLIN_SPEC_ORIGINAL_TEMPLATE"]) def label_clash_error(self): """ @@ -300,8 +306,17 @@ def output_path(self): output_path = str(self.original_spec.output_path) - if (self.override_vars is not None) and ("OUTPUT_PATH" in self.override_vars): - output_path = str(self.override_vars["OUTPUT_PATH"]) + # If there are override vars we need to check that the output path doesn't need changed + if self.override_vars is not None: + # Case where output path is directly modified + if "OUTPUT_PATH" in self.override_vars: + output_path = str(self.override_vars["OUTPUT_PATH"]) + else: + for var_name, var_val in self.override_vars.items(): + token = f"$({var_name})" + # Case where output path contains a variable that was overridden + if token in output_path: + output_path = output_path.replace(token, str(var_val)) output_path = expand_line(output_path, self.user_vars, env_vars=True) output_path = os.path.abspath(output_path) @@ -309,6 +324,8 @@ def output_path(self): os.makedirs(output_path) LOG.info(f"Made dir(s) to output path '{output_path}'.") + LOG.info(f"OUTPUT_PATH: {os.path.basename(output_path)}") + return output_path @cached_property @@ -368,10 +385,6 @@ def expanded_spec(self): return self.get_expanded_spec() result = self.get_expanded_spec() - expanded_name = result.description["name"].replace(" ", "_") + ".expanded.yaml" - - # Set expanded filepath - expanded_filepath = os.path.join(self.info, expanded_name) # expand provenance spec filename if contains_token(self.original_spec.name) or contains_shell_ref(self.original_spec.name): @@ -394,8 +407,8 @@ def expanded_spec(self): self.workspace = expanded_workspace self.info = os.path.join(self.workspace, "merlin_info") self.special_vars["MERLIN_INFO"] = self.info + self._set_special_file_vars() - expanded_filepath = os.path.join(self.info, expanded_name) new_spec_text = expand_by_line(result.dump(), MerlinStudy.get_user_vars(result)) result = MerlinSpec.load_spec_from_string(new_spec_text) result = expand_env_vars(result) @@ -412,15 +425,13 @@ def expanded_spec(self): os.path.join(self.info, os.path.basename(self.samples_file)), ) - # write expanded spec for provenance - with open(expanded_filepath, "w") as f: # pylint: disable=C0103 + # write expanded spec for provenance and set the path (necessary for testing) + with open(self.special_vars["MERLIN_SPEC_ARCHIVED_COPY"], "w") as f: # pylint: disable=C0103 f.write(result.dump()) + result.path = self.special_vars["MERLIN_SPEC_ARCHIVED_COPY"] # write original spec for provenance - result = MerlinSpec.load_spec_from_string(result.dump()) - result.path = expanded_filepath - name = result.description["name"].replace(" ", "_") - self.write_original_spec(name) + self.write_original_spec() # write partially-expanded spec for provenance partial_spec = deepcopy(self.original_spec) @@ -428,8 +439,7 @@ def expanded_spec(self): partial_spec.environment["variables"] = result.environment["variables"] if "labels" in result.environment: partial_spec.environment["labels"] = result.environment["labels"] - partial_spec_path = os.path.join(self.info, name + ".partial.yaml") - with open(partial_spec_path, "w") as f: # pylint: disable=C0103 + with open(self.special_vars["MERLIN_SPEC_EXECUTED_RUN"], "w") as f: # pylint: disable=C0103 f.write(partial_spec.dump()) LOG.info(f"Study workspace is '{self.workspace}'.") diff --git a/merlin/utils.py b/merlin/utils.py index 7c18407e7..a2e4966d0 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index 9693b06ba..0bb413325 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index db21e5429..0bc687923 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # @@ -249,14 +249,16 @@ class ProvenanceYAMLFileHasRegex(HasRegex): MUST contain a given regular expression. """ - def __init__(self, regex, name, output_path, provenance_type, negate=False): # pylint: disable=R0913 + def __init__(self, regex, spec_file_name, study_name, output_path, provenance_type, negate=False): # pylint: disable=R0913 """ :param `regex`: a string regex pattern - :param `name`: the name of a study + :param `spec_file_name`: the name of the spec file + :param `study_name`: the name of a study :param `output_path`: the $(OUTPUT_PATH) of a study """ super().__init__(regex, negate=negate) - self.name = name + self.spec_file_name = spec_file_name + self.study_name = study_name self.output_path = output_path provenance_types = ["orig", "partial", "expanded"] if provenance_type not in provenance_types: @@ -277,7 +279,9 @@ def glob_string(self): """ Returns a regex string for the glob library to recursively find files with. """ - return f"{self.output_path}/{self.name}" f"_[0-9]*-[0-9]*/merlin_info/{self.name}.{self.prov_type}.yaml" + return ( + f"{self.output_path}/{self.study_name}" f"_[0-9]*-[0-9]*/merlin_info/{self.spec_file_name}.{self.prov_type}.yaml" + ) def is_within(self): # pylint: disable=W0221 """ diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 9b0270d3b..82a4fdd92 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 093644f9f..e9e9ce590 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # @@ -127,6 +127,7 @@ def define_tests(): # pylint: disable=R0914,R0915 flux_native = f"{test_specs}/flux_par_native_test.yaml" lsf = f"{examples}/lsf/lsf_par.yaml" mul_workers_demo = f"{dev_examples}/multiple_workers.yaml" + cli_substitution_wf = f"{test_specs}/cli_substitution_test.yaml" # Other shortcuts black = "black --check --target-version py36" @@ -323,6 +324,26 @@ def define_tests(): # pylint: disable=R0914,R0915 "run type": "local", }, } + cli_substitution_tests = { + "no substitutions": { + "cmds": f"merlin run {cli_substitution_wf} --local", + "conditions": [HasReturnCode(), HasRegex(r"OUTPUT_PATH: output_path_no_substitution")], + "run type": "local", + "cleanup": "rm -r output_path_no_substitution", + }, + "output_path substitution": { + "cmds": f"merlin run {cli_substitution_wf} --local --vars OUTPUT_PATH=output_path_substitution", + "conditions": [HasReturnCode(), HasRegex(r"OUTPUT_PATH: output_path_substitution")], + "run type": "local", + "cleanup": "rm -r output_path_substitution", + }, + "output_path w/ variable substitution": { + "cmds": f"merlin run {cli_substitution_wf} --local --vars SUB=variable_sub", + "conditions": [HasReturnCode(), HasRegex(r"OUTPUT_PATH: output_path_variable_sub")], + "run type": "local", + "cleanup": "rm -r output_path_variable_sub", + }, + } example_tests = { "example failure": {"cmds": "merlin example failure", "conditions": HasRegex("not found"), "run type": "local"}, "example simple_chain": { @@ -435,31 +456,36 @@ def define_tests(): # pylint: disable=R0914,R0915 HasReturnCode(), ProvenanceYAMLFileHasRegex( regex=r"HELLO: \$\(SCRIPTS\)/hello_world.py", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="orig", ), ProvenanceYAMLFileHasRegex( regex=r"name: \$\(NAME\)", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="partial", ), ProvenanceYAMLFileHasRegex( regex="studies/feature_demo_", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="partial", ), ProvenanceYAMLFileHasRegex( regex="name: feature_demo", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", ), ProvenanceYAMLFileHasRegex( regex=r"\$\(NAME\)", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", negate=True, @@ -510,13 +536,15 @@ def define_tests(): # pylint: disable=R0914,R0915 "conditions": [ ProvenanceYAMLFileHasRegex( regex=r"\[0.3333333", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", ), ProvenanceYAMLFileHasRegex( regex=r"\[0.5", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", negate=True, @@ -715,7 +743,8 @@ def define_tests(): # pylint: disable=R0914,R0915 HasReturnCode(), ProvenanceYAMLFileHasRegex( regex="cli_test_demo_workers:", - name="feature_demo", + spec_file_name="remote_feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", ), @@ -740,6 +769,7 @@ def define_tests(): # pylint: disable=R0914,R0915 examples_check, run_workers_echo_tests, wf_format_tests, + cli_substitution_tests, example_tests, restart_step_tests, restart_wf_tests, diff --git a/tests/integration/test_specs/cli_substitution_test.yaml b/tests/integration/test_specs/cli_substitution_test.yaml new file mode 100644 index 000000000..5cbbb70d3 --- /dev/null +++ b/tests/integration/test_specs/cli_substitution_test.yaml @@ -0,0 +1,14 @@ +description: + name: cli_substitution_test + description: a spec that helps test cli substitutions + +env: + variables: + SUB: no_substitution + OUTPUT_PATH: output_path_$(SUB) + +study: + - name: step1 + description: step 1 + run: + cmd: echo "test" diff --git a/tests/unit/study/test_study.py b/tests/unit/study/test_study.py index a00995d55..cb15805cd 100644 --- a/tests/unit/study/test_study.py +++ b/tests/unit/study/test_study.py @@ -41,6 +41,15 @@ nodes: 1 task_queue: hello_queue + - name: test_special_vars + description: test the special vars + run: + cmd: | + echo $(MERLIN_SPEC_ORIGINAL_TEMPLATE) + echo $(MERLIN_SPEC_EXECUTED_RUN) + echo $(MERLIN_SPEC_ARCHIVED_COPY) + task_queue: special_var_queue + global.parameters: X2: values : [0.5] @@ -234,11 +243,16 @@ class TestMerlinStudy(unittest.TestCase): @staticmethod def file_contains_string(f, string): - return string in open(f, "r").read() + result = False + with open(f, "r") as infile: + if string in infile.read(): + result = True + return result def setUp(self): self.tmpdir = tempfile.mkdtemp() - self.merlin_spec_filepath = os.path.join(self.tmpdir, "basic_ensemble.yaml") + self.base_name = "basic_ensemble" + self.merlin_spec_filepath = os.path.join(self.tmpdir, f"{self.base_name}.yaml") with open(self.merlin_spec_filepath, "w+") as _file: _file.write(MERLIN_SPEC) @@ -263,6 +277,34 @@ def test_expanded_spec(self): assert TestMerlinStudy.file_contains_string(self.study.expanded_spec.path, "$PATH") assert not TestMerlinStudy.file_contains_string(self.study.expanded_spec.path, "PATH_VAR: $PATH") + # Special vars are in the second step of MERLIN_SPEC so grab that step here + original_special_var_step = self.study.original_spec.study[1]["run"]["cmd"] + expanded_special_var_step = self.study.expanded_spec.study[1]["run"]["cmd"] + + # Make sure the special filepath variables aren't expanded in the original spec + assert "$(MERLIN_SPEC_ORIGINAL_TEMPLATE)" in original_special_var_step + assert "$(MERLIN_SPEC_EXECUTED_RUN)" in original_special_var_step + assert "$(MERLIN_SPEC_ARCHIVED_COPY)" in original_special_var_step + + # Make sure the special filepath variables aren't left in their variable form in the expanded spec + assert "$(MERLIN_SPEC_ORIGINAL_TEMPLATE)" not in expanded_special_var_step + assert "$(MERLIN_SPEC_EXECUTED_RUN)" not in expanded_special_var_step + assert "$(MERLIN_SPEC_ARCHIVED_COPY)" not in expanded_special_var_step + + # Make sure the special filepath variables we're expanded appropriately in the expanded spec + assert ( + f"{self.base_name}.orig.yaml" in expanded_special_var_step + and "unit_test1.orig.yaml" not in expanded_special_var_step + ) + assert ( + f"{self.base_name}.partial.yaml" in expanded_special_var_step + and "unit_test1.partial.yaml" not in expanded_special_var_step + ) + assert ( + f"{self.base_name}.expanded.yaml" in expanded_special_var_step + and "unit_test1.expanded.yaml" not in expanded_special_var_step + ) + def test_column_label_conflict(self): """ If there is a common key between Maestro's global.parameters and @@ -291,3 +333,7 @@ def test_no_env(self): assert isinstance(study_no_env, MerlinStudy), bad_type_err except Exception as e: assert False, f"Encountered unexpected exception, {e}, for viable MerlinSpec without optional 'env' section." + + +if __name__ == "__main__": + unittest.main() From b0f4d866e57b2edb324d5773a041122a4f1930d3 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Thu, 17 Aug 2023 12:37:08 -0700 Subject: [PATCH 092/126] dependabot/certifi-requests-pygments (#441) * Bump certifi from 2022.12.7 to 2023.7.22 in /docs Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.12.7 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2022.12.07...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] * add all dependabot changes and update CHANGELOG --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- CHANGELOG.md | 10 ++++++++++ docs/requirements.txt | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e7c0b1cd..b3ac21def 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] +### Fixed + +### Added + +### Changed +- Bump certifi from 2022.12.7 to 2023.7.22 in /docs +- Bump pygments from 2.13.0 to 2.15.0 in /docs +- Bump requests from 2.28.1 to 2.31.0 in /docs + ## [1.10.2] ### Fixed - A bug where the .orig, .partial, and .expanded file names were using the study name rather than the original file name diff --git a/docs/requirements.txt b/docs/requirements.txt index 87333eb50..c771e60dc 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.12 # via sphinx babel==2.10.3 # via sphinx -certifi==2022.12.7 +certifi==2023.7.22 # via requests charset-normalizer==2.1.1 # via requests @@ -26,13 +26,13 @@ markupsafe==2.1.1 # via jinja2 packaging==21.3 # via sphinx -pygments==2.13.0 +pygments==2.15.0 # via sphinx pyparsing==3.0.9 # via packaging pytz==2022.5 # via babel -requests==2.28.1 +requests==2.31.0 # via sphinx snowballstemmer==2.2.0 # via sphinx From c641c5c6495ebc0e789019b8ff80e5d0419ff597 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 18 Aug 2023 13:02:19 -0700 Subject: [PATCH 093/126] bugfix/server-pip-redis-conf (#443) * add *.conf to the MANIFEST file so pip will grab the redis.conf file * add note explaining how to fix a hanging merlin server start * modify CHANGELOG * add second export option to docs and fix typo --- CHANGELOG.md | 2 ++ MANIFEST.in | 2 +- docs/source/merlin_server.rst | 2 +- docs/source/server/commands.rst | 7 +++++++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3ac21def..f5e60ea9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed ### Added +- The *.conf regex for the recursive-include of the merlin server directory so that pip will add it to the wheel +- A note to the docs for how to fix an issue where the `merlin server start` command hangs ### Changed - Bump certifi from 2022.12.7 to 2023.7.22 in /docs diff --git a/MANIFEST.in b/MANIFEST.in index da9d411ad..d5526e37c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ recursive-include merlin/data *.yaml *.py -recursive-include merlin/server *.yaml *.py +recursive-include merlin/server *.yaml *.py *.conf recursive-include merlin/examples * include requirements.txt include requirements/* diff --git a/docs/source/merlin_server.rst b/docs/source/merlin_server.rst index 24b37c776..23f6d4a1d 100644 --- a/docs/source/merlin_server.rst +++ b/docs/source/merlin_server.rst @@ -1,7 +1,7 @@ Merlin Server ============= The merlin server command allows users easy access to containerized broker -and results servers for merlin workflows. This allowsusers to run merlin without +and results servers for merlin workflows. This allows users to run merlin without a dedicated external server. The main configuration will be stored in the subdirectory called "server/" by diff --git a/docs/source/server/commands.rst b/docs/source/server/commands.rst index dd8ca1b02..fc40bf182 100644 --- a/docs/source/server/commands.rst +++ b/docs/source/server/commands.rst @@ -34,6 +34,13 @@ Starting up a Merlin Server (``merlin server start``) Starts the container located in the local merlin server configuration. +.. note:: + If this command seems to hang and never release control back to you, follow these steps: + + 1. Kill the command with ``Ctrl+C`` + 2. Run either ``export LC_ALL="C.UTF-8"`` or ``export LC_ALL="C"`` + 3. Re-run the ``merlin server start`` command + Stopping an exisiting Merlin Server (``merlin server stop``) ------------------------------------------------------------ From 970a06fd6e4ba927b44180ad3bb4819df0fbdb73 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 18 Aug 2023 16:20:26 -0700 Subject: [PATCH 094/126] bump to version 1.10.3 (#444) --- CHANGELOG.md | 4 +--- Makefile | 2 +- merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/conditions.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 2 +- 55 files changed, 56 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5e60ea9d..c0760da46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,7 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] -### Fixed - +## [1.10.3] ### Added - The *.conf regex for the recursive-include of the merlin server directory so that pip will add it to the wheel - A note to the docs for how to fix an issue where the `merlin server start` command hangs diff --git a/Makefile b/Makefile index 030ed8d15..74c407db0 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index 9856d9a7b..12ba225cd 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.10.2" +__version__ = "1.10.3" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 5d61d8ed1..b56da4d7a 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index dc0cfd9d7..9921bbb89 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index be663a572..a90133b73 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index a4c127a12..00aaea917 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index f8b881dcf..0f7607a8e 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 821c1df71..4f7333f6d 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index fe9d735d4..55601073e 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index dfd230a1a..1059383d9 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 2c70fadc9..cee757b91 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index b24e0af3e..56051756b 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 1f851abb2..8137e0543 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index a53190df7..7af320b52 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 8c7dc8a2b..78658333a 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index 73605768e..e688945cc 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 6ccf3dea5..d46a1d038 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index d30d5ab58..a619ecb03 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 2f5434091..1385c4f35 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index bf60bb50d..f59255ddb 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index a7754cec7..78152e6ee 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 5b893ab9e..5dbe2ebf5 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index b176378a9..ed7156aa5 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index e90e13324..6a6da63d8 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index d3fe4b80a..a29546798 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index b2bcc4949..477887794 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 779333dc6..465f0ad3d 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 6cd46fa97..88f37fd2c 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 93f928f3b..45e6ef3d3 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index e088eedcd..5f109f7c8 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index a3cc7f021..a280abac5 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 145eb00bb..7f9f66188 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 1560f412d..c4aad952c 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index abba6ef02..e29f0e7b5 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index e9ec4bceb..50d4f1c35 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 1cb26d512..eb165b617 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index b2157ebde..4298fca32 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index f9c04b1e3..31ef03b7c 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index d2315bf7f..6c977a756 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index dc01ad6a9..be66e1b97 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 686b5afa2..bdba0250d 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 8eaf306ca..8f4ddb19d 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index a2e4966d0..3eb4e5acc 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index 0bb413325..d409c0a74 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 0bc687923..81c063112 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 82a4fdd92..bc19fd9c8 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index e9e9ce590..0fdedf07b 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # From faf71edb866548cf686c2cef0fee559e01db28ce Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 18 Aug 2023 16:53:14 -0700 Subject: [PATCH 095/126] Version/1.10.3 (#445) * fix default worker bug with all steps * version bump and requirements fix * Bugfix/filename-special-vars (#425) * fix file naming bug * fix filename bug with variable as study name * add tests for the file name special vars changes * modify changelog * implement Luc's suggestions * remove replace line * Create dependabot-changelog-updater.yml * testing outputs of modifying changelog * delete dependabot-changelog-updater * feature/pdf-docs (#427) * first attempt at adding pdf * fixing build error * modify changelog to show docs changes * fix errors Luc found in the build logs * trying out removal of latex * reverting latex changes back * uncommenting the latex_elements settings * adding epub to see if latex will build * adding a latex engine variable to conf * fix naming error with latex_engine * attempting to add a logo to the pdf build * testing an override to the searchtools file * revert back to not using searchtools override * update changelog * bugfix/openfoam_singularity_issues (#426) * fix openfoam_singularity issues * update requirements and descriptions for openfoam examples * bugfix/output-path-substitution (#430) * fix bug with output_path and variable substitution * add tests for cli substitutions * bugfix/scheduler-permission-error (#436) * Release/1.10.2 (#437) * bump version to 1.10.2 * bump version in CHANGELOG * resolve develop to main merge issues (#439) * fix default worker bug with all steps * version bump and requirements fix * dependabot/certifi-requests-pygments (#441) * Bump certifi from 2022.12.7 to 2023.7.22 in /docs Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.12.7 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2022.12.07...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] * add all dependabot changes and update CHANGELOG --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * bugfix/server-pip-redis-conf (#443) * add *.conf to the MANIFEST file so pip will grab the redis.conf file * add note explaining how to fix a hanging merlin server start * modify CHANGELOG * add second export option to docs and fix typo * bump to version 1.10.3 (#444) --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- CHANGELOG.md | 10 ++++++++++ MANIFEST.in | 2 +- Makefile | 2 +- docs/requirements.txt | 6 +++--- docs/source/merlin_server.rst | 2 +- docs/source/server/commands.rst | 7 +++++++ merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/conditions.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 2 +- 59 files changed, 77 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e7c0b1cd..c0760da46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.10.3] +### Added +- The *.conf regex for the recursive-include of the merlin server directory so that pip will add it to the wheel +- A note to the docs for how to fix an issue where the `merlin server start` command hangs + +### Changed +- Bump certifi from 2022.12.7 to 2023.7.22 in /docs +- Bump pygments from 2.13.0 to 2.15.0 in /docs +- Bump requests from 2.28.1 to 2.31.0 in /docs + ## [1.10.2] ### Fixed - A bug where the .orig, .partial, and .expanded file names were using the study name rather than the original file name diff --git a/MANIFEST.in b/MANIFEST.in index da9d411ad..d5526e37c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ recursive-include merlin/data *.yaml *.py -recursive-include merlin/server *.yaml *.py +recursive-include merlin/server *.yaml *.py *.conf recursive-include merlin/examples * include requirements.txt include requirements/* diff --git a/Makefile b/Makefile index 030ed8d15..74c407db0 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/docs/requirements.txt b/docs/requirements.txt index 87333eb50..c771e60dc 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.12 # via sphinx babel==2.10.3 # via sphinx -certifi==2022.12.7 +certifi==2023.7.22 # via requests charset-normalizer==2.1.1 # via requests @@ -26,13 +26,13 @@ markupsafe==2.1.1 # via jinja2 packaging==21.3 # via sphinx -pygments==2.13.0 +pygments==2.15.0 # via sphinx pyparsing==3.0.9 # via packaging pytz==2022.5 # via babel -requests==2.28.1 +requests==2.31.0 # via sphinx snowballstemmer==2.2.0 # via sphinx diff --git a/docs/source/merlin_server.rst b/docs/source/merlin_server.rst index 24b37c776..23f6d4a1d 100644 --- a/docs/source/merlin_server.rst +++ b/docs/source/merlin_server.rst @@ -1,7 +1,7 @@ Merlin Server ============= The merlin server command allows users easy access to containerized broker -and results servers for merlin workflows. This allowsusers to run merlin without +and results servers for merlin workflows. This allows users to run merlin without a dedicated external server. The main configuration will be stored in the subdirectory called "server/" by diff --git a/docs/source/server/commands.rst b/docs/source/server/commands.rst index dd8ca1b02..fc40bf182 100644 --- a/docs/source/server/commands.rst +++ b/docs/source/server/commands.rst @@ -34,6 +34,13 @@ Starting up a Merlin Server (``merlin server start``) Starts the container located in the local merlin server configuration. +.. note:: + If this command seems to hang and never release control back to you, follow these steps: + + 1. Kill the command with ``Ctrl+C`` + 2. Run either ``export LC_ALL="C.UTF-8"`` or ``export LC_ALL="C"`` + 3. Re-run the ``merlin server start`` command + Stopping an exisiting Merlin Server (``merlin server stop``) ------------------------------------------------------------ diff --git a/merlin/__init__.py b/merlin/__init__.py index 9856d9a7b..12ba225cd 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.10.2" +__version__ = "1.10.3" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 5d61d8ed1..b56da4d7a 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index dc0cfd9d7..9921bbb89 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index be663a572..a90133b73 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index a4c127a12..00aaea917 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index f8b881dcf..0f7607a8e 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 821c1df71..4f7333f6d 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index fe9d735d4..55601073e 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index dfd230a1a..1059383d9 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 2c70fadc9..cee757b91 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index b24e0af3e..56051756b 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 1f851abb2..8137e0543 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index a53190df7..7af320b52 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 8c7dc8a2b..78658333a 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index 73605768e..e688945cc 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 6ccf3dea5..d46a1d038 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index d30d5ab58..a619ecb03 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 2f5434091..1385c4f35 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index bf60bb50d..f59255ddb 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index a7754cec7..78152e6ee 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 5b893ab9e..5dbe2ebf5 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index b176378a9..ed7156aa5 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index e90e13324..6a6da63d8 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index d3fe4b80a..a29546798 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index b2bcc4949..477887794 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 779333dc6..465f0ad3d 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 6cd46fa97..88f37fd2c 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 93f928f3b..45e6ef3d3 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index e088eedcd..5f109f7c8 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index a3cc7f021..a280abac5 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 145eb00bb..7f9f66188 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 1560f412d..c4aad952c 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index abba6ef02..e29f0e7b5 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index e9ec4bceb..50d4f1c35 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 1cb26d512..eb165b617 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index b2157ebde..4298fca32 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index f9c04b1e3..31ef03b7c 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index d2315bf7f..6c977a756 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index dc01ad6a9..be66e1b97 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 686b5afa2..bdba0250d 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 8eaf306ca..8f4ddb19d 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index a2e4966d0..3eb4e5acc 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index 0bb413325..d409c0a74 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 0bc687923..81c063112 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 82a4fdd92..bc19fd9c8 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index e9e9ce590..0fdedf07b 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # From d8bfbdbd5a07a314064ce24a6201662a34fc8028 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Tue, 22 Aug 2023 12:54:18 -0700 Subject: [PATCH 096/126] bugfix/sphinx-5.3.0-requirement (#446) * Version/1.10.3 (#445) * fix default worker bug with all steps * version bump and requirements fix * Bugfix/filename-special-vars (#425) * fix file naming bug * fix filename bug with variable as study name * add tests for the file name special vars changes * modify changelog * implement Luc's suggestions * remove replace line * Create dependabot-changelog-updater.yml * testing outputs of modifying changelog * delete dependabot-changelog-updater * feature/pdf-docs (#427) * first attempt at adding pdf * fixing build error * modify changelog to show docs changes * fix errors Luc found in the build logs * trying out removal of latex * reverting latex changes back * uncommenting the latex_elements settings * adding epub to see if latex will build * adding a latex engine variable to conf * fix naming error with latex_engine * attempting to add a logo to the pdf build * testing an override to the searchtools file * revert back to not using searchtools override * update changelog * bugfix/openfoam_singularity_issues (#426) * fix openfoam_singularity issues * update requirements and descriptions for openfoam examples * bugfix/output-path-substitution (#430) * fix bug with output_path and variable substitution * add tests for cli substitutions * bugfix/scheduler-permission-error (#436) * Release/1.10.2 (#437) * bump version to 1.10.2 * bump version in CHANGELOG * resolve develop to main merge issues (#439) * fix default worker bug with all steps * version bump and requirements fix * dependabot/certifi-requests-pygments (#441) * Bump certifi from 2022.12.7 to 2023.7.22 in /docs Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.12.7 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2022.12.07...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] * add all dependabot changes and update CHANGELOG --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * bugfix/server-pip-redis-conf (#443) * add *.conf to the MANIFEST file so pip will grab the redis.conf file * add note explaining how to fix a hanging merlin server start * modify CHANGELOG * add second export option to docs and fix typo * bump to version 1.10.3 (#444) --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * change hardcoded sphinx requirement * update CHANGELOG --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- CHANGELOG.md | 4 +++ docs/requirements.in | 4 +++ docs/requirements.txt | 57 +------------------------------------------ 3 files changed, 9 insertions(+), 56 deletions(-) create mode 100644 docs/requirements.in diff --git a/CHANGELOG.md b/CHANGELOG.md index c0760da46..beecf82f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] +### Changed +- Hardcoded Sphinx v5.3.0 requirement is now removed so we can use latest Sphinx + ## [1.10.3] ### Added - The *.conf regex for the recursive-include of the merlin server directory so that pip will add it to the wheel diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 000000000..268785121 --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1,4 @@ +# This file will list all requirements for the docs so we can freeze a version of them for release. +# To freeze the versions run: +# pip-compile requirements.in +sphinx \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index c771e60dc..5d3faecfe 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,56 +1 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# pip-compile requirements.in -# -alabaster==0.7.12 - # via sphinx -babel==2.10.3 - # via sphinx -certifi==2023.7.22 - # via requests -charset-normalizer==2.1.1 - # via requests -docutils==0.17.1 - # via sphinx -idna==3.4 - # via requests -imagesize==1.4.1 - # via sphinx -importlib-metadata==5.0.0 - # via sphinx -jinja2==3.0.3 - # via sphinx -markupsafe==2.1.1 - # via jinja2 -packaging==21.3 - # via sphinx -pygments==2.15.0 - # via sphinx -pyparsing==3.0.9 - # via packaging -pytz==2022.5 - # via babel -requests==2.31.0 - # via sphinx -snowballstemmer==2.2.0 - # via sphinx -sphinx==5.3.0 - # via -r requirements.in -sphinxcontrib-applehelp==1.0.2 - # via sphinx -sphinxcontrib-devhelp==1.0.2 - # via sphinx -sphinxcontrib-htmlhelp==2.0.0 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.3 - # via sphinx -sphinxcontrib-serializinghtml==1.1.5 - # via sphinx -urllib3==1.26.12 - # via requests -zipp==3.10.0 - # via importlib-metadata +sphinx>=5.3.0 From 8241bfe21d5eb58fd2f9f8a3235181c0ea27faaf Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Wed, 27 Sep 2023 17:19:59 -0700 Subject: [PATCH 097/126] feature/vlauncher (#447) * fix file naming error for iterative workflows * fixed small bug with new filepath naming * add VLAUNCHER functionality * add docs for VLAUNCHER and modify changelog * re-word docs and fix table format * add a test for vlauncher * run fix-style and add a test for vlauncher * Add the find_vlaunch_var and setup_vlaunch functions. The numeric value of the shell variables may not be defined until run time, so replace with variable strings instead of values. Consolidate the commands into one function. * Add variable set for (t)csh. * Run fix-style * make step settings the defaults and ignore commented lines * add some additional tests * remove regex library import --------- Co-authored-by: Joseph M. Koning --- CHANGELOG.md | 7 ++ docs/source/merlin_variables.rst | 53 ++++++++++++- merlin/spec/defaults.py | 8 ++ merlin/spec/specification.py | 13 +++- merlin/study/script_adapter.py | 77 ++++++++++++++++++- merlin/study/study.py | 3 +- merlin/utils.py | 20 +++++ tests/integration/test_definitions.py | 82 +++++++++++++++++++-- tests/integration/test_specs/flux_test.yaml | 28 +++++++ 9 files changed, 279 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index beecf82f7..135508a8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [unreleased] +### Added +- New reserved variables: + - `VLAUNCHER`: The same functionality as the `LAUNCHER` variable, but will substitute shell variables `MERLIN_NODES`, `MERLIN_PROCS`, `MERLIN_CORES`, and `MERLIN_GPUS` for nodes, procs, cores per task, and gpus + ### Changed - Hardcoded Sphinx v5.3.0 requirement is now removed so we can use latest Sphinx +### Fixed +- A bug where the filenames in iterative workflows kept appending `.out`, `.partial`, or `.expanded` to the filenames stored in the `merlin_info/` subdirectory + ## [1.10.3] ### Added - The *.conf regex for the recursive-include of the merlin server directory so that pip will add it to the wheel diff --git a/docs/source/merlin_variables.rst b/docs/source/merlin_variables.rst index d67e06412..f8ea7fce7 100644 --- a/docs/source/merlin_variables.rst +++ b/docs/source/merlin_variables.rst @@ -160,8 +160,9 @@ Reserved variables $(MERLIN_INFO)/*.expanded.yaml -The ``LAUNCHER`` Variable -+++++++++++++++++++++++++ + +The ``LAUNCHER`` and ``VLAUNCHER`` Variables ++++++++++++++++++++++++++++++++++++++++++++++++ ``$(LAUNCHER)`` is a special case of a reserved variable since it's value *can* be changed. It serves as an abstraction to launch a job with parallel schedulers like :ref:`slurm`, @@ -187,6 +188,54 @@ We can modify this to use the ``$(LAUNCHER)`` variable like so: In other words, the ``$(LAUNCHER)`` variable would become ``srun -N 1 -n 3``. +Similarly, the ``$(VLAUNCHER)`` variable behaves similarly to the ``$(LAUNCHER)`` variable. +The key distinction lies in its source of information. Instead of drawing certain configuration +options from the ``run`` section of a step, it retrieves specific shell variables. These shell +variables are automatically generated by Merlin when you include the ``$(VLAUNCHER)`` variable +in a step command, but they can also be customized by the user. Currently, the following shell +variables are: + +.. list-table:: VLAUNCHER Variables + :widths: 25 50 25 + :header-rows: 1 + + * - Variable + - Description + - Default + + * - ``${MERLIN_NODES}`` + - The number of nodes + - 1 + + * - ``${MERLIN_PROCS}`` + - The number of tasks/procs + - 1 + + * - ``${MERLIN_CORES}`` + - The number of cores per task/proc + - 1 + + * - ``${MERLIN_GPUS}`` + - The number of gpus per task/proc + - 0 + +Let's say we have the following defined in our yaml file: + +.. code:: yaml + + batch: + type: flux + + run: + cmd: | + MERLIN_NODES=4 + MERLIN_PROCS=2 + MERLIN_CORES=8 + MERLIN_GPUS=2 + $(VLAUNCHER) python script.py + +The ``$(VLAUNCHER)`` variable would be substituted to ``flux run -N 4 -n 2 -c 8 -g 2``. + User variables ------------------- Variables defined by a specification file in the ``env`` section, as in this example: diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index c4aad952c..6845b8b08 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -52,3 +52,11 @@ "generate": {"cmd": "echo 'Insert sample-generating command here'"}, "level_max_dirs": 25, } + +# Values of the form (step key to search for, default value if no step key found) +VLAUNCHER_VARS = { + "MERLIN_NODES": ("nodes", 1), + "MERLIN_PROCS": ("procs", 1), + "MERLIN_CORES": ("cores per task", 1), + "MERLIN_GPUS": ("gpus", 0), +} diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index eb165b617..60a47964d 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -45,7 +45,7 @@ from maestrowf.specification import YAMLSpecification from merlin.spec import all_keys, defaults -from merlin.utils import repr_timedelta +from merlin.utils import find_vlaunch_var, repr_timedelta LOG = logging.getLogger(__name__) @@ -369,6 +369,17 @@ def process_spec_defaults(self): defaults.STUDY_STEP_RUN["shell"] = self.batch["shell"] for step in self.study: MerlinSpec.fill_missing_defaults(step["run"], defaults.STUDY_STEP_RUN) + # Insert VLAUNCHER specific variables if necessary + if "$(VLAUNCHER)" in step["run"]["cmd"]: + SHSET = "" + if "csh" in step["run"]["shell"]: + SHSET = "set " + # We need to set default values for VLAUNCHER variables if they're not defined by the user + for vlaunch_var, vlaunch_val in defaults.VLAUNCHER_VARS.items(): + if not find_vlaunch_var(vlaunch_var.replace("MERLIN_", ""), step["run"]["cmd"], accept_no_matches=True): + # Look for predefined nodes/procs/cores/gpus values in the step and default to those + vlaunch_val = step["run"][vlaunch_val[0]] if vlaunch_val[0] in step["run"] else vlaunch_val[1] + step["run"]["cmd"] = f"{SHSET}{vlaunch_var}={vlaunch_val}\n" + step["run"]["cmd"] # fill in missing merlin section defaults MerlinSpec.fill_missing_defaults(self.merlin, defaults.MERLIN["merlin"]) diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index be66e1b97..80e56f279 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -42,12 +42,36 @@ from maestrowf.utils import start_process from merlin.common.abstracts.enums import ReturnCode -from merlin.utils import convert_timestring +from merlin.utils import convert_timestring, find_vlaunch_var LOG = logging.getLogger(__name__) +def setup_vlaunch(step_run: str, batch_type: str, gpu_config: bool) -> None: + """ + Check for the VLAUNCHER keyword int the step run string, find + the MERLIN variables and configure VLAUNCHER. + + :param `step_run`: the step.run command string + :param `batch_type`: the batch type string + :param `gpu_config`: bool to determin if gpus should be configured + :returns: None + """ + if "$(VLAUNCHER)" in step_run["cmd"]: + step_run["cmd"] = step_run["cmd"].replace("$(VLAUNCHER)", "$(LAUNCHER)") + + step_run["nodes"] = find_vlaunch_var("NODES", step_run["cmd"]) + step_run["procs"] = find_vlaunch_var("PROCS", step_run["cmd"]) + step_run["cores per task"] = find_vlaunch_var("CORES", step_run["cmd"]) + + if find_vlaunch_var("GPUS", step_run["cmd"]): + if gpu_config: + step_run["gpus"] = find_vlaunch_var("GPUS", step_run["cmd"]) + else: + LOG.warning(f"Merlin does not yet have the ability to set GPUs per task with {batch_type}. Coming soon.") + + class MerlinLSFScriptAdapter(SlurmScriptAdapter): """ A SchedulerScriptAdapter class for slurm blocking parallel launches, @@ -156,6 +180,23 @@ def get_parallelize_command(self, procs, nodes=None, **kwargs): return " ".join(args) + def write_script(self, ws_path, step): + """ + This will overwrite the write_script in method from Maestro's base ScriptAdapter + class but will eventually call it. This is necessary for the VLAUNCHER to work. + + :param `ws_path`: the path to the workspace where we'll write the scripts + :param `step`: the Maestro StudyStep object containing info for our step + :returns: a tuple containing: + - a boolean representing whether this step is to be scheduled or not + - Merlin can ignore this + - a path to the script for the cmd + - a path to the script for the restart cmd + """ + setup_vlaunch(step.run, "lsf", False) + + return super().write_script(ws_path, step) + class MerlinSlurmScriptAdapter(SlurmScriptAdapter): """ @@ -256,6 +297,23 @@ def get_parallelize_command(self, procs, nodes=None, **kwargs): return " ".join(args) + def write_script(self, ws_path, step): + """ + This will overwrite the write_script in method from Maestro's base ScriptAdapter + class but will eventually call it. This is necessary for the VLAUNCHER to work. + + :param `ws_path`: the path to the workspace where we'll write the scripts + :param `step`: the Maestro StudyStep object containing info for our step + :returns: a tuple containing: + - a boolean representing whether this step is to be scheduled or not + - Merlin can ignore this + - a path to the script for the cmd + - a path to the script for the restart cmd + """ + setup_vlaunch(step.run, "slurm", False) + + return super().write_script(ws_path, step) + class MerlinFluxScriptAdapter(MerlinSlurmScriptAdapter): """ @@ -319,6 +377,23 @@ def time_format(self, val): """ return convert_timestring(val, format_method="FSD") + def write_script(self, ws_path, step): + """ + This will overwrite the write_script in method from Maestro's base ScriptAdapter + class but will eventually call it. This is necessary for the VLAUNCHER to work. + + :param `ws_path`: the path to the workspace where we'll write the scripts + :param `step`: the Maestro StudyStep object containing info for our step + :returns: a tuple containing: + - a boolean representing whether this step is to be scheduled or not + - Merlin can ignore this + - a path to the script for the cmd + - a path to the script for the restart cmd + """ + setup_vlaunch(step.run, "flux", True) + + return super().write_script(ws_path, step) + class MerlinScriptAdapter(LocalScriptAdapter): """ diff --git a/merlin/study/study.py b/merlin/study/study.py index 8f4ddb19d..831062e17 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -127,7 +127,8 @@ def __init__( # pylint: disable=R0913 def _set_special_file_vars(self): """Setter for the orig, partial, and expanded file paths of a study.""" - base_name = Path(self.filepath).stem + shortened_filepath = self.filepath.replace(".out", "").replace(".partial", "").replace(".expanded", "") + base_name = Path(shortened_filepath).stem self.special_vars["MERLIN_SPEC_ORIGINAL_TEMPLATE"] = os.path.join( self.info, base_name + ".orig.yaml", diff --git a/merlin/utils.py b/merlin/utils.py index 3eb4e5acc..51a1fd8c0 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -497,6 +497,26 @@ def contains_shell_ref(string): return False +def find_vlaunch_var(vlaunch_var: str, step_cmd: str, accept_no_matches=False) -> str: + """ + Given a variable used for VLAUNCHER and the step cmd value, find + the variable. + + :param `vlaunch_var`: The name of the VLAUNCHER variable (without MERLIN_) + :param `step_cmd`: The string for the cmd of a step + :param `accept_no_matches`: If True, return None if we couldn't find the variable. Otherwise, raise an error. + :returns: the `vlaunch_var` variable or None + """ + matches = list(re.findall(rf"^(?!#).*MERLIN_{vlaunch_var}=\d+", step_cmd, re.MULTILINE)) + + if matches: + return f"${{MERLIN_{vlaunch_var}}}" + + if accept_no_matches: + return None + raise ValueError(f"VLAUNCHER used but could not find MERLIN_{vlaunch_var} in the step.") + + # Time utilities def convert_to_timedelta(timestr: Union[str, int]) -> timedelta: """Convert a timestring to a timedelta object. diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 0fdedf07b..2f37dcc95 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -407,13 +407,81 @@ def define_tests(): # pylint: disable=R0914,R0915 }, "dry launch flux": { "cmds": f"{run} {flux} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - "conditions": StepFileHasRegex( - "runs", - "*/runs.slurm.sh", - "flux_test", - OUTPUT_DIR, - get_flux_cmd("flux", no_errors=True), - ), + "conditions": [ + StepFileHasRegex( + "runs", + "*/runs.slurm.sh", + "flux_test", + OUTPUT_DIR, + get_flux_cmd("flux", no_errors=True), + ), + ################## + # VLAUNCHER TESTS + ################## + StepFileHasRegex( + "vlauncher_test", + "vlauncher_test.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"flux run -n \$\{MERLIN_PROCS\} -N \$\{MERLIN_NODES\} -c \$\{MERLIN_CORES\}", + ), + StepFileHasRegex( + "vlauncher_test_step_defaults", + "vlauncher_test_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_GPUS=1", + ), + StepFileHasRegex( + "vlauncher_test_step_defaults", + "vlauncher_test_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_NODES=6", + ), + StepFileHasRegex( + "vlauncher_test_step_defaults", + "vlauncher_test_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_PROCS=3", + ), + StepFileHasRegex( + "vlauncher_test_step_defaults", + "vlauncher_test_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_CORES=2", + ), + StepFileHasRegex( + "vlauncher_test_no_step_defaults", + "vlauncher_test_no_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_GPUS=0", + ), + StepFileHasRegex( + "vlauncher_test_no_step_defaults", + "vlauncher_test_no_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_NODES=1", + ), + StepFileHasRegex( + "vlauncher_test_no_step_defaults", + "vlauncher_test_no_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_PROCS=1", + ), + StepFileHasRegex( + "vlauncher_test_no_step_defaults", + "vlauncher_test_no_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_CORES=1", + ), + ], "run type": "local", }, "dry launch lsf": { diff --git a/tests/integration/test_specs/flux_test.yaml b/tests/integration/test_specs/flux_test.yaml index fe0130526..99f15205c 100644 --- a/tests/integration/test_specs/flux_test.yaml +++ b/tests/integration/test_specs/flux_test.yaml @@ -33,6 +33,34 @@ study: depends: [runs*] task_queue: flux_test +- description: step that uses vlauncher + name: vlauncher_test + run: + cmd: | + MERLIN_NODES=6 + MERLIN_PROCS=3 + MERLIN_CORES=2 + $(VLAUNCHER) echo "step that uses vlauncher" + task_queue: flux_test + +- description: test vlauncher step defaults + name: vlauncher_test_step_defaults + run: + cmd: | + $(VLAUNCHER) echo "test vlauncher step defaults" + task_queue: flux_test + nodes: 6 + procs: 3 + cores per task: 2 + gpus: 1 + +- description: test vlauncher no step defaults + name: vlauncher_test_no_step_defaults + run: + cmd: | + $(VLAUNCHER) echo "test vlauncher no step defaults" + task_queue: flux_test + global.parameters: STUDY: label: STUDY.%% From 50d0fb6a906146cf40fe06fb15edf0f85f87670b Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Thu, 28 Sep 2023 08:14:38 -0700 Subject: [PATCH 098/126] release/1.11.0 (#448) --- CHANGELOG.md | 4 ++-- Makefile | 2 +- merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/conditions.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 2 +- 55 files changed, 57 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 135508a8c..914d8616a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,9 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] +## [1.11.0] ### Added -- New reserved variables: +- New reserved variable: - `VLAUNCHER`: The same functionality as the `LAUNCHER` variable, but will substitute shell variables `MERLIN_NODES`, `MERLIN_PROCS`, `MERLIN_CORES`, and `MERLIN_GPUS` for nodes, procs, cores per task, and gpus ### Changed diff --git a/Makefile b/Makefile index 74c407db0..4a857a217 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index 12ba225cd..20a0e8b3e 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.10.3" +__version__ = "1.11.0" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index b56da4d7a..f823937a6 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index 9921bbb89..95f26530e 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index a90133b73..7b8ab80f5 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 00aaea917..124c7851d 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 0f7607a8e..a8f8dffb2 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 4f7333f6d..149d52e13 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 55601073e..dc13d41d1 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 1059383d9..125ec5bed 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index cee757b91..68e178b77 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 56051756b..f1d06077a 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 8137e0543..c29763485 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 7af320b52..0594ffe45 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 78658333a..fe49ff162 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index e688945cc..0ff305962 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index d46a1d038..1f3418377 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index a619ecb03..d3e7002e7 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 1385c4f35..65fc6f85c 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index f59255ddb..a0470938c 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 78152e6ee..1d756f00e 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 5dbe2ebf5..294787857 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index ed7156aa5..cf272d93b 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 6a6da63d8..3fba8cfc8 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index a29546798..198cf3804 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 477887794..7936db03b 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 465f0ad3d..476ab1c0f 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 88f37fd2c..d04c75d72 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 45e6ef3d3..45411131b 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 5f109f7c8..414f7a407 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index a280abac5..2b8f1216d 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 7f9f66188..556f5924e 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 6845b8b08..8972d5cfe 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index e29f0e7b5..381bc72f4 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 50d4f1c35..f3192a38e 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 60a47964d..32fe0f635 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 4298fca32..e02a65a32 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 31ef03b7c..8b5ff196d 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 6c977a756..ea4d22926 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 80e56f279..6ecc79c5f 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index bdba0250d..5d877ba4f 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 831062e17..b9ada35ea 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index 51a1fd8c0..33735085d 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index d409c0a74..7c91d26c7 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 81c063112..b25010ca2 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index bc19fd9c8..58460e18f 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 2f37dcc95..f59acf237 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # From 99d56598bcd2ed90743f5a0470f604fabfbc6704 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Mon, 9 Oct 2023 13:46:23 -0700 Subject: [PATCH 099/126] bugfix/skewed-sample-hierarchy (#450) * add patch for skewed sample hierarchy/additional samples * update changelog * catch narrower range of exceptions --- CHANGELOG.md | 1 + merlin/common/tasks.py | 48 ++++++++++++++++++++++++------------------ 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 914d8616a..f994a4d05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - A bug where the filenames in iterative workflows kept appending `.out`, `.partial`, or `.expanded` to the filenames stored in the `merlin_info/` subdirectory +- A bug where a skewed sample hierarchy was created when a restart was necessary in the `add_merlin_expanded_chain_to_chord` task ## [1.10.3] ### Added diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index f1d06077a..fbd401826 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -298,27 +298,33 @@ def add_merlin_expanded_chain_to_chord( # pylint: disable=R0913,R0914 LOG.debug("chain added to chord") else: # recurse down the sample_index hierarchy - LOG.debug("recursing down sample_index hierarchy") - for next_index in sample_index.children.values(): - next_index.name = os.path.join(sample_index.name, next_index.name) - LOG.debug("generating next step") - next_step = add_merlin_expanded_chain_to_chord.s( - task_type, - chain_, - samples[next_index.min - min_sample_id : next_index.max - min_sample_id], - labels, - next_index, - adapter_config, - next_index.min, - ) - next_step.set(queue=chain_[0].get_task_queue()) - LOG.debug(f"recursing with range {next_index.min}:{next_index.max}, {next_index.name} {signature(next_step)}") - LOG.debug(f"queuing samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}...") - if self.request.is_eager: - next_step.delay() - else: - self.add_to_chord(next_step, lazy=False) - LOG.debug(f"queued for samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}") + try: + LOG.debug("recursing down sample_index hierarchy") + for next_index in sample_index.children.values(): + next_index_name_before = next_index.name + next_index.name = os.path.join(sample_index.name, next_index.name) + LOG.debug("generating next step") + next_step = add_merlin_expanded_chain_to_chord.s( + task_type, + chain_, + samples[next_index.min - min_sample_id : next_index.max - min_sample_id], + labels, + next_index, + adapter_config, + next_index.min, + ) + next_step.set(queue=chain_[0].get_task_queue()) + LOG.debug(f"recursing with range {next_index.min}:{next_index.max}, {next_index.name} {signature(next_step)}") + LOG.debug(f"queuing samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}...") + if self.request.is_eager: + next_step.delay() + else: + self.add_to_chord(next_step, lazy=False) + LOG.debug(f"queued for samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}") + except retry_exceptions as e: + # Reset the index to what it was before so we don't accidentally create a bunch of extra samples upon restart + next_index.name = next_index_name_before + raise e return ReturnCode.OK From 093c867baaab4f5d7b4b15474f7d24394d20ce24 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Mon, 9 Oct 2023 13:56:14 -0700 Subject: [PATCH 100/126] Version/1.11.0 (#449) * fix default worker bug with all steps * version bump and requirements fix * Bugfix/filename-special-vars (#425) * fix file naming bug * fix filename bug with variable as study name * add tests for the file name special vars changes * modify changelog * implement Luc's suggestions * remove replace line * Create dependabot-changelog-updater.yml * testing outputs of modifying changelog * delete dependabot-changelog-updater * feature/pdf-docs (#427) * first attempt at adding pdf * fixing build error * modify changelog to show docs changes * fix errors Luc found in the build logs * trying out removal of latex * reverting latex changes back * uncommenting the latex_elements settings * adding epub to see if latex will build * adding a latex engine variable to conf * fix naming error with latex_engine * attempting to add a logo to the pdf build * testing an override to the searchtools file * revert back to not using searchtools override * update changelog * bugfix/openfoam_singularity_issues (#426) * fix openfoam_singularity issues * update requirements and descriptions for openfoam examples * bugfix/output-path-substitution (#430) * fix bug with output_path and variable substitution * add tests for cli substitutions * bugfix/scheduler-permission-error (#436) * Release/1.10.2 (#437) * bump version to 1.10.2 * bump version in CHANGELOG * resolve develop to main merge issues (#439) * fix default worker bug with all steps * version bump and requirements fix * dependabot/certifi-requests-pygments (#441) * Bump certifi from 2022.12.7 to 2023.7.22 in /docs Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.12.7 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2022.12.07...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] * add all dependabot changes and update CHANGELOG --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * bugfix/server-pip-redis-conf (#443) * add *.conf to the MANIFEST file so pip will grab the redis.conf file * add note explaining how to fix a hanging merlin server start * modify CHANGELOG * add second export option to docs and fix typo * bump to version 1.10.3 (#444) * bugfix/sphinx-5.3.0-requirement (#446) * Version/1.10.3 (#445) * fix default worker bug with all steps * version bump and requirements fix * Bugfix/filename-special-vars (#425) * fix file naming bug * fix filename bug with variable as study name * add tests for the file name special vars changes * modify changelog * implement Luc's suggestions * remove replace line * Create dependabot-changelog-updater.yml * testing outputs of modifying changelog * delete dependabot-changelog-updater * feature/pdf-docs (#427) * first attempt at adding pdf * fixing build error * modify changelog to show docs changes * fix errors Luc found in the build logs * trying out removal of latex * reverting latex changes back * uncommenting the latex_elements settings * adding epub to see if latex will build * adding a latex engine variable to conf * fix naming error with latex_engine * attempting to add a logo to the pdf build * testing an override to the searchtools file * revert back to not using searchtools override * update changelog * bugfix/openfoam_singularity_issues (#426) * fix openfoam_singularity issues * update requirements and descriptions for openfoam examples * bugfix/output-path-substitution (#430) * fix bug with output_path and variable substitution * add tests for cli substitutions * bugfix/scheduler-permission-error (#436) * Release/1.10.2 (#437) * bump version to 1.10.2 * bump version in CHANGELOG * resolve develop to main merge issues (#439) * fix default worker bug with all steps * version bump and requirements fix * dependabot/certifi-requests-pygments (#441) * Bump certifi from 2022.12.7 to 2023.7.22 in /docs Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.12.7 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2022.12.07...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] * add all dependabot changes and update CHANGELOG --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * bugfix/server-pip-redis-conf (#443) * add *.conf to the MANIFEST file so pip will grab the redis.conf file * add note explaining how to fix a hanging merlin server start * modify CHANGELOG * add second export option to docs and fix typo * bump to version 1.10.3 (#444) --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * change hardcoded sphinx requirement * update CHANGELOG --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feature/vlauncher (#447) * fix file naming error for iterative workflows * fixed small bug with new filepath naming * add VLAUNCHER functionality * add docs for VLAUNCHER and modify changelog * re-word docs and fix table format * add a test for vlauncher * run fix-style and add a test for vlauncher * Add the find_vlaunch_var and setup_vlaunch functions. The numeric value of the shell variables may not be defined until run time, so replace with variable strings instead of values. Consolidate the commands into one function. * Add variable set for (t)csh. * Run fix-style * make step settings the defaults and ignore commented lines * add some additional tests * remove regex library import --------- Co-authored-by: Joseph M. Koning * release/1.11.0 (#448) * bugfix/skewed-sample-hierarchy (#450) * add patch for skewed sample hierarchy/additional samples * update changelog * catch narrower range of exceptions --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joseph M. Koning --- CHANGELOG.md | 12 +++ Makefile | 2 +- docs/requirements.in | 4 + docs/requirements.txt | 57 +------------ docs/source/merlin_variables.rst | 53 +++++++++++- merlin/__init__.py | 4 +- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- .../security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 50 ++++++----- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 10 ++- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 15 +++- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 79 ++++++++++++++++- merlin/study/step.py | 2 +- merlin/study/study.py | 5 +- merlin/utils.py | 22 ++++- setup.py | 2 +- tests/integration/conditions.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 84 +++++++++++++++++-- tests/integration/test_specs/flux_test.yaml | 28 +++++++ 59 files changed, 371 insertions(+), 144 deletions(-) create mode 100644 docs/requirements.in diff --git a/CHANGELOG.md b/CHANGELOG.md index c0760da46..f994a4d05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.11.0] +### Added +- New reserved variable: + - `VLAUNCHER`: The same functionality as the `LAUNCHER` variable, but will substitute shell variables `MERLIN_NODES`, `MERLIN_PROCS`, `MERLIN_CORES`, and `MERLIN_GPUS` for nodes, procs, cores per task, and gpus + +### Changed +- Hardcoded Sphinx v5.3.0 requirement is now removed so we can use latest Sphinx + +### Fixed +- A bug where the filenames in iterative workflows kept appending `.out`, `.partial`, or `.expanded` to the filenames stored in the `merlin_info/` subdirectory +- A bug where a skewed sample hierarchy was created when a restart was necessary in the `add_merlin_expanded_chain_to_chord` task + ## [1.10.3] ### Added - The *.conf regex for the recursive-include of the merlin server directory so that pip will add it to the wheel diff --git a/Makefile b/Makefile index 74c407db0..4a857a217 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 000000000..268785121 --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1,4 @@ +# This file will list all requirements for the docs so we can freeze a version of them for release. +# To freeze the versions run: +# pip-compile requirements.in +sphinx \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index c771e60dc..5d3faecfe 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,56 +1 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# pip-compile requirements.in -# -alabaster==0.7.12 - # via sphinx -babel==2.10.3 - # via sphinx -certifi==2023.7.22 - # via requests -charset-normalizer==2.1.1 - # via requests -docutils==0.17.1 - # via sphinx -idna==3.4 - # via requests -imagesize==1.4.1 - # via sphinx -importlib-metadata==5.0.0 - # via sphinx -jinja2==3.0.3 - # via sphinx -markupsafe==2.1.1 - # via jinja2 -packaging==21.3 - # via sphinx -pygments==2.15.0 - # via sphinx -pyparsing==3.0.9 - # via packaging -pytz==2022.5 - # via babel -requests==2.31.0 - # via sphinx -snowballstemmer==2.2.0 - # via sphinx -sphinx==5.3.0 - # via -r requirements.in -sphinxcontrib-applehelp==1.0.2 - # via sphinx -sphinxcontrib-devhelp==1.0.2 - # via sphinx -sphinxcontrib-htmlhelp==2.0.0 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.3 - # via sphinx -sphinxcontrib-serializinghtml==1.1.5 - # via sphinx -urllib3==1.26.12 - # via requests -zipp==3.10.0 - # via importlib-metadata +sphinx>=5.3.0 diff --git a/docs/source/merlin_variables.rst b/docs/source/merlin_variables.rst index d67e06412..f8ea7fce7 100644 --- a/docs/source/merlin_variables.rst +++ b/docs/source/merlin_variables.rst @@ -160,8 +160,9 @@ Reserved variables $(MERLIN_INFO)/*.expanded.yaml -The ``LAUNCHER`` Variable -+++++++++++++++++++++++++ + +The ``LAUNCHER`` and ``VLAUNCHER`` Variables ++++++++++++++++++++++++++++++++++++++++++++++++ ``$(LAUNCHER)`` is a special case of a reserved variable since it's value *can* be changed. It serves as an abstraction to launch a job with parallel schedulers like :ref:`slurm`, @@ -187,6 +188,54 @@ We can modify this to use the ``$(LAUNCHER)`` variable like so: In other words, the ``$(LAUNCHER)`` variable would become ``srun -N 1 -n 3``. +Similarly, the ``$(VLAUNCHER)`` variable behaves similarly to the ``$(LAUNCHER)`` variable. +The key distinction lies in its source of information. Instead of drawing certain configuration +options from the ``run`` section of a step, it retrieves specific shell variables. These shell +variables are automatically generated by Merlin when you include the ``$(VLAUNCHER)`` variable +in a step command, but they can also be customized by the user. Currently, the following shell +variables are: + +.. list-table:: VLAUNCHER Variables + :widths: 25 50 25 + :header-rows: 1 + + * - Variable + - Description + - Default + + * - ``${MERLIN_NODES}`` + - The number of nodes + - 1 + + * - ``${MERLIN_PROCS}`` + - The number of tasks/procs + - 1 + + * - ``${MERLIN_CORES}`` + - The number of cores per task/proc + - 1 + + * - ``${MERLIN_GPUS}`` + - The number of gpus per task/proc + - 0 + +Let's say we have the following defined in our yaml file: + +.. code:: yaml + + batch: + type: flux + + run: + cmd: | + MERLIN_NODES=4 + MERLIN_PROCS=2 + MERLIN_CORES=8 + MERLIN_GPUS=2 + $(VLAUNCHER) python script.py + +The ``$(VLAUNCHER)`` variable would be substituted to ``flux run -N 4 -n 2 -c 8 -g 2``. + User variables ------------------- Variables defined by a specification file in the ``env`` section, as in this example: diff --git a/merlin/__init__.py b/merlin/__init__.py index 12ba225cd..20a0e8b3e 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.10.3" +__version__ = "1.11.0" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index b56da4d7a..f823937a6 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index 9921bbb89..95f26530e 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index a90133b73..7b8ab80f5 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 00aaea917..124c7851d 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 0f7607a8e..a8f8dffb2 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 4f7333f6d..149d52e13 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 55601073e..dc13d41d1 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 1059383d9..125ec5bed 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index cee757b91..68e178b77 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 56051756b..fbd401826 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # @@ -298,27 +298,33 @@ def add_merlin_expanded_chain_to_chord( # pylint: disable=R0913,R0914 LOG.debug("chain added to chord") else: # recurse down the sample_index hierarchy - LOG.debug("recursing down sample_index hierarchy") - for next_index in sample_index.children.values(): - next_index.name = os.path.join(sample_index.name, next_index.name) - LOG.debug("generating next step") - next_step = add_merlin_expanded_chain_to_chord.s( - task_type, - chain_, - samples[next_index.min - min_sample_id : next_index.max - min_sample_id], - labels, - next_index, - adapter_config, - next_index.min, - ) - next_step.set(queue=chain_[0].get_task_queue()) - LOG.debug(f"recursing with range {next_index.min}:{next_index.max}, {next_index.name} {signature(next_step)}") - LOG.debug(f"queuing samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}...") - if self.request.is_eager: - next_step.delay() - else: - self.add_to_chord(next_step, lazy=False) - LOG.debug(f"queued for samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}") + try: + LOG.debug("recursing down sample_index hierarchy") + for next_index in sample_index.children.values(): + next_index_name_before = next_index.name + next_index.name = os.path.join(sample_index.name, next_index.name) + LOG.debug("generating next step") + next_step = add_merlin_expanded_chain_to_chord.s( + task_type, + chain_, + samples[next_index.min - min_sample_id : next_index.max - min_sample_id], + labels, + next_index, + adapter_config, + next_index.min, + ) + next_step.set(queue=chain_[0].get_task_queue()) + LOG.debug(f"recursing with range {next_index.min}:{next_index.max}, {next_index.name} {signature(next_step)}") + LOG.debug(f"queuing samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}...") + if self.request.is_eager: + next_step.delay() + else: + self.add_to_chord(next_step, lazy=False) + LOG.debug(f"queued for samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}") + except retry_exceptions as e: + # Reset the index to what it was before so we don't accidentally create a bunch of extra samples upon restart + next_index.name = next_index_name_before + raise e return ReturnCode.OK diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 8137e0543..c29763485 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 7af320b52..0594ffe45 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 78658333a..fe49ff162 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index e688945cc..0ff305962 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index d46a1d038..1f3418377 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index a619ecb03..d3e7002e7 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 1385c4f35..65fc6f85c 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index f59255ddb..a0470938c 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 78152e6ee..1d756f00e 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 5dbe2ebf5..294787857 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index ed7156aa5..cf272d93b 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 6a6da63d8..3fba8cfc8 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index a29546798..198cf3804 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 477887794..7936db03b 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 465f0ad3d..476ab1c0f 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 88f37fd2c..d04c75d72 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 45e6ef3d3..45411131b 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 5f109f7c8..414f7a407 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index a280abac5..2b8f1216d 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 7f9f66188..556f5924e 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index c4aad952c..8972d5cfe 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # @@ -52,3 +52,11 @@ "generate": {"cmd": "echo 'Insert sample-generating command here'"}, "level_max_dirs": 25, } + +# Values of the form (step key to search for, default value if no step key found) +VLAUNCHER_VARS = { + "MERLIN_NODES": ("nodes", 1), + "MERLIN_PROCS": ("procs", 1), + "MERLIN_CORES": ("cores per task", 1), + "MERLIN_GPUS": ("gpus", 0), +} diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index e29f0e7b5..381bc72f4 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 50d4f1c35..f3192a38e 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index eb165b617..32fe0f635 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # @@ -45,7 +45,7 @@ from maestrowf.specification import YAMLSpecification from merlin.spec import all_keys, defaults -from merlin.utils import repr_timedelta +from merlin.utils import find_vlaunch_var, repr_timedelta LOG = logging.getLogger(__name__) @@ -369,6 +369,17 @@ def process_spec_defaults(self): defaults.STUDY_STEP_RUN["shell"] = self.batch["shell"] for step in self.study: MerlinSpec.fill_missing_defaults(step["run"], defaults.STUDY_STEP_RUN) + # Insert VLAUNCHER specific variables if necessary + if "$(VLAUNCHER)" in step["run"]["cmd"]: + SHSET = "" + if "csh" in step["run"]["shell"]: + SHSET = "set " + # We need to set default values for VLAUNCHER variables if they're not defined by the user + for vlaunch_var, vlaunch_val in defaults.VLAUNCHER_VARS.items(): + if not find_vlaunch_var(vlaunch_var.replace("MERLIN_", ""), step["run"]["cmd"], accept_no_matches=True): + # Look for predefined nodes/procs/cores/gpus values in the step and default to those + vlaunch_val = step["run"][vlaunch_val[0]] if vlaunch_val[0] in step["run"] else vlaunch_val[1] + step["run"]["cmd"] = f"{SHSET}{vlaunch_var}={vlaunch_val}\n" + step["run"]["cmd"] # fill in missing merlin section defaults MerlinSpec.fill_missing_defaults(self.merlin, defaults.MERLIN["merlin"]) diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 4298fca32..e02a65a32 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 31ef03b7c..8b5ff196d 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 6c977a756..ea4d22926 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index be66e1b97..6ecc79c5f 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # @@ -42,12 +42,36 @@ from maestrowf.utils import start_process from merlin.common.abstracts.enums import ReturnCode -from merlin.utils import convert_timestring +from merlin.utils import convert_timestring, find_vlaunch_var LOG = logging.getLogger(__name__) +def setup_vlaunch(step_run: str, batch_type: str, gpu_config: bool) -> None: + """ + Check for the VLAUNCHER keyword int the step run string, find + the MERLIN variables and configure VLAUNCHER. + + :param `step_run`: the step.run command string + :param `batch_type`: the batch type string + :param `gpu_config`: bool to determin if gpus should be configured + :returns: None + """ + if "$(VLAUNCHER)" in step_run["cmd"]: + step_run["cmd"] = step_run["cmd"].replace("$(VLAUNCHER)", "$(LAUNCHER)") + + step_run["nodes"] = find_vlaunch_var("NODES", step_run["cmd"]) + step_run["procs"] = find_vlaunch_var("PROCS", step_run["cmd"]) + step_run["cores per task"] = find_vlaunch_var("CORES", step_run["cmd"]) + + if find_vlaunch_var("GPUS", step_run["cmd"]): + if gpu_config: + step_run["gpus"] = find_vlaunch_var("GPUS", step_run["cmd"]) + else: + LOG.warning(f"Merlin does not yet have the ability to set GPUs per task with {batch_type}. Coming soon.") + + class MerlinLSFScriptAdapter(SlurmScriptAdapter): """ A SchedulerScriptAdapter class for slurm blocking parallel launches, @@ -156,6 +180,23 @@ def get_parallelize_command(self, procs, nodes=None, **kwargs): return " ".join(args) + def write_script(self, ws_path, step): + """ + This will overwrite the write_script in method from Maestro's base ScriptAdapter + class but will eventually call it. This is necessary for the VLAUNCHER to work. + + :param `ws_path`: the path to the workspace where we'll write the scripts + :param `step`: the Maestro StudyStep object containing info for our step + :returns: a tuple containing: + - a boolean representing whether this step is to be scheduled or not + - Merlin can ignore this + - a path to the script for the cmd + - a path to the script for the restart cmd + """ + setup_vlaunch(step.run, "lsf", False) + + return super().write_script(ws_path, step) + class MerlinSlurmScriptAdapter(SlurmScriptAdapter): """ @@ -256,6 +297,23 @@ def get_parallelize_command(self, procs, nodes=None, **kwargs): return " ".join(args) + def write_script(self, ws_path, step): + """ + This will overwrite the write_script in method from Maestro's base ScriptAdapter + class but will eventually call it. This is necessary for the VLAUNCHER to work. + + :param `ws_path`: the path to the workspace where we'll write the scripts + :param `step`: the Maestro StudyStep object containing info for our step + :returns: a tuple containing: + - a boolean representing whether this step is to be scheduled or not + - Merlin can ignore this + - a path to the script for the cmd + - a path to the script for the restart cmd + """ + setup_vlaunch(step.run, "slurm", False) + + return super().write_script(ws_path, step) + class MerlinFluxScriptAdapter(MerlinSlurmScriptAdapter): """ @@ -319,6 +377,23 @@ def time_format(self, val): """ return convert_timestring(val, format_method="FSD") + def write_script(self, ws_path, step): + """ + This will overwrite the write_script in method from Maestro's base ScriptAdapter + class but will eventually call it. This is necessary for the VLAUNCHER to work. + + :param `ws_path`: the path to the workspace where we'll write the scripts + :param `step`: the Maestro StudyStep object containing info for our step + :returns: a tuple containing: + - a boolean representing whether this step is to be scheduled or not + - Merlin can ignore this + - a path to the script for the cmd + - a path to the script for the restart cmd + """ + setup_vlaunch(step.run, "flux", True) + + return super().write_script(ws_path, step) + class MerlinScriptAdapter(LocalScriptAdapter): """ diff --git a/merlin/study/step.py b/merlin/study/step.py index bdba0250d..5d877ba4f 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 8f4ddb19d..b9ada35ea 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # @@ -127,7 +127,8 @@ def __init__( # pylint: disable=R0913 def _set_special_file_vars(self): """Setter for the orig, partial, and expanded file paths of a study.""" - base_name = Path(self.filepath).stem + shortened_filepath = self.filepath.replace(".out", "").replace(".partial", "").replace(".expanded", "") + base_name = Path(shortened_filepath).stem self.special_vars["MERLIN_SPEC_ORIGINAL_TEMPLATE"] = os.path.join( self.info, base_name + ".orig.yaml", diff --git a/merlin/utils.py b/merlin/utils.py index 3eb4e5acc..33735085d 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # @@ -497,6 +497,26 @@ def contains_shell_ref(string): return False +def find_vlaunch_var(vlaunch_var: str, step_cmd: str, accept_no_matches=False) -> str: + """ + Given a variable used for VLAUNCHER and the step cmd value, find + the variable. + + :param `vlaunch_var`: The name of the VLAUNCHER variable (without MERLIN_) + :param `step_cmd`: The string for the cmd of a step + :param `accept_no_matches`: If True, return None if we couldn't find the variable. Otherwise, raise an error. + :returns: the `vlaunch_var` variable or None + """ + matches = list(re.findall(rf"^(?!#).*MERLIN_{vlaunch_var}=\d+", step_cmd, re.MULTILINE)) + + if matches: + return f"${{MERLIN_{vlaunch_var}}}" + + if accept_no_matches: + return None + raise ValueError(f"VLAUNCHER used but could not find MERLIN_{vlaunch_var} in the step.") + + # Time utilities def convert_to_timedelta(timestr: Union[str, int]) -> timedelta: """Convert a timestring to a timedelta object. diff --git a/setup.py b/setup.py index d409c0a74..7c91d26c7 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 81c063112..b25010ca2 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index bc19fd9c8..58460e18f 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 0fdedf07b..f59acf237 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # @@ -407,13 +407,81 @@ def define_tests(): # pylint: disable=R0914,R0915 }, "dry launch flux": { "cmds": f"{run} {flux} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - "conditions": StepFileHasRegex( - "runs", - "*/runs.slurm.sh", - "flux_test", - OUTPUT_DIR, - get_flux_cmd("flux", no_errors=True), - ), + "conditions": [ + StepFileHasRegex( + "runs", + "*/runs.slurm.sh", + "flux_test", + OUTPUT_DIR, + get_flux_cmd("flux", no_errors=True), + ), + ################## + # VLAUNCHER TESTS + ################## + StepFileHasRegex( + "vlauncher_test", + "vlauncher_test.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"flux run -n \$\{MERLIN_PROCS\} -N \$\{MERLIN_NODES\} -c \$\{MERLIN_CORES\}", + ), + StepFileHasRegex( + "vlauncher_test_step_defaults", + "vlauncher_test_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_GPUS=1", + ), + StepFileHasRegex( + "vlauncher_test_step_defaults", + "vlauncher_test_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_NODES=6", + ), + StepFileHasRegex( + "vlauncher_test_step_defaults", + "vlauncher_test_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_PROCS=3", + ), + StepFileHasRegex( + "vlauncher_test_step_defaults", + "vlauncher_test_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_CORES=2", + ), + StepFileHasRegex( + "vlauncher_test_no_step_defaults", + "vlauncher_test_no_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_GPUS=0", + ), + StepFileHasRegex( + "vlauncher_test_no_step_defaults", + "vlauncher_test_no_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_NODES=1", + ), + StepFileHasRegex( + "vlauncher_test_no_step_defaults", + "vlauncher_test_no_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_PROCS=1", + ), + StepFileHasRegex( + "vlauncher_test_no_step_defaults", + "vlauncher_test_no_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_CORES=1", + ), + ], "run type": "local", }, "dry launch lsf": { diff --git a/tests/integration/test_specs/flux_test.yaml b/tests/integration/test_specs/flux_test.yaml index fe0130526..99f15205c 100644 --- a/tests/integration/test_specs/flux_test.yaml +++ b/tests/integration/test_specs/flux_test.yaml @@ -33,6 +33,34 @@ study: depends: [runs*] task_queue: flux_test +- description: step that uses vlauncher + name: vlauncher_test + run: + cmd: | + MERLIN_NODES=6 + MERLIN_PROCS=3 + MERLIN_CORES=2 + $(VLAUNCHER) echo "step that uses vlauncher" + task_queue: flux_test + +- description: test vlauncher step defaults + name: vlauncher_test_step_defaults + run: + cmd: | + $(VLAUNCHER) echo "test vlauncher step defaults" + task_queue: flux_test + nodes: 6 + procs: 3 + cores per task: 2 + gpus: 1 + +- description: test vlauncher no step defaults + name: vlauncher_test_no_step_defaults + run: + cmd: | + $(VLAUNCHER) echo "test vlauncher no step defaults" + task_queue: flux_test + global.parameters: STUDY: label: STUDY.%% From 593dbcdf90096b722a97da8abe62e23c016f18e5 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Mon, 23 Oct 2023 10:54:24 -0700 Subject: [PATCH 101/126] bugfix/lsf-gpu-typo (#453) * fix typo in batch.py that causes a bug * change print statements to log statements --- CHANGELOG.md | 4 ++++ merlin/study/batch.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f994a4d05..d709ccaa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Fixed +- Typo in `batch.py` that caused lsf launches to fail (`ALL_SGPUS` changed to `ALL_GPUS`) + ## [1.11.0] ### Added - New reserved variable: diff --git a/merlin/study/batch.py b/merlin/study/batch.py index e02a65a32..1b96cd282 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -299,7 +299,7 @@ def construct_scheduler_legend(parsed_batch: Dict, nodes: int) -> Dict: "lsf": { "check cmd": ["jsrun", "--help"], "expected check output": b"jsrun", - "launch": f"jsrun -a 1 -c ALL_CPUS -g ALL_SGPUS --bind=none -n {nodes}", + "launch": f"jsrun -a 1 -c ALL_CPUS -g ALL_GPUS --bind=none -n {nodes}", }, # pbs is mainly a placeholder in case a user wants to try it (we don't have it at the lab so it's mostly untested) "pbs": { @@ -335,12 +335,16 @@ def construct_worker_launch_command(parsed_batch: Dict, nodes: int) -> str: scheduler_legend: Dict = construct_scheduler_legend(parsed_batch, nodes) workload_manager: str = get_batch_type(scheduler_legend) + LOG.debug(f"parsed_batch: {parsed_batch}") + if parsed_batch["btype"] == "pbs" and workload_manager == parsed_batch["btype"]: raise TypeError("The PBS scheduler is only enabled for 'batch: flux' type") if parsed_batch["btype"] == "slurm" and workload_manager not in ("lsf", "flux", "pbs"): workload_manager = "slurm" + LOG.debug(f"workload_manager: {workload_manager}") + try: launch_command = scheduler_legend[workload_manager]["launch"] except KeyError as e: # pylint: disable=C0103 From f994f96f71d3405f59e108e0e67c9d2cd0ff3998 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Mon, 23 Oct 2023 11:08:53 -0700 Subject: [PATCH 102/126] release/1.11.1 (#454) --- CHANGELOG.md | 2 +- Makefile | 2 +- merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/conditions.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 2 +- 55 files changed, 56 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d709ccaa4..8d0ef2ae6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.11.1] ### Fixed - Typo in `batch.py` that caused lsf launches to fail (`ALL_SGPUS` changed to `ALL_GPUS`) diff --git a/Makefile b/Makefile index 4a857a217..2f9db031b 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index 20a0e8b3e..c1ad21b22 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.11.0" +__version__ = "1.11.1" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index f823937a6..3cca2c710 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index 95f26530e..55d616658 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index 7b8ab80f5..383e7dccd 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 124c7851d..d79e4e4f3 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index a8f8dffb2..da366b452 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 149d52e13..4e3ac3a52 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index dc13d41d1..4303c3a6e 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 125ec5bed..806d42e0c 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 68e178b77..e0957ebb8 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index fbd401826..2bd77d2ad 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index c29763485..134d0b66c 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 0594ffe45..b58e3b2a9 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index fe49ff162..385b8c1df 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index 0ff305962..fbbd39064 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 1f3418377..2ca6c5d04 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index d3e7002e7..b88655399 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 65fc6f85c..8f0c6b029 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index a0470938c..78eee5866 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 1d756f00e..9b65f31ae 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 294787857..2fa5e61ce 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index cf272d93b..72d5a1521 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 3fba8cfc8..b8858f721 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 198cf3804..55496a72c 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 7936db03b..a355c4f2f 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 476ab1c0f..01a10aae7 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index d04c75d72..7e2b6f1c1 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 45411131b..a28776577 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 414f7a407..e4ec646fc 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 2b8f1216d..bab641702 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 556f5924e..dcc02b063 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 8972d5cfe..32fd05aa5 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 381bc72f4..5924d1f74 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index f3192a38e..abb0f13c9 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 32fe0f635..ac23b06d1 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 1b96cd282..01a2945e3 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 8b5ff196d..a6707d952 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index ea4d22926..4cffc679c 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 6ecc79c5f..45d211742 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 5d877ba4f..c5773520a 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index b9ada35ea..6e2f4f937 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index 33735085d..196e8b29b 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index 7c91d26c7..7303a1ddf 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index b25010ca2..80e3e5855 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 58460e18f..c0b699055 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index f59acf237..273fa7c56 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # From e73142015dc565724ece34c29ebf017998b3f5d7 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Mon, 23 Oct 2023 11:41:11 -0700 Subject: [PATCH 103/126] Version/1.11.1 (#455) * fix default worker bug with all steps * version bump and requirements fix * Bugfix/filename-special-vars (#425) * fix file naming bug * fix filename bug with variable as study name * add tests for the file name special vars changes * modify changelog * implement Luc's suggestions * remove replace line * Create dependabot-changelog-updater.yml * testing outputs of modifying changelog * delete dependabot-changelog-updater * feature/pdf-docs (#427) * first attempt at adding pdf * fixing build error * modify changelog to show docs changes * fix errors Luc found in the build logs * trying out removal of latex * reverting latex changes back * uncommenting the latex_elements settings * adding epub to see if latex will build * adding a latex engine variable to conf * fix naming error with latex_engine * attempting to add a logo to the pdf build * testing an override to the searchtools file * revert back to not using searchtools override * update changelog * bugfix/openfoam_singularity_issues (#426) * fix openfoam_singularity issues * update requirements and descriptions for openfoam examples * bugfix/output-path-substitution (#430) * fix bug with output_path and variable substitution * add tests for cli substitutions * bugfix/scheduler-permission-error (#436) * Release/1.10.2 (#437) * bump version to 1.10.2 * bump version in CHANGELOG * resolve develop to main merge issues (#439) * fix default worker bug with all steps * version bump and requirements fix * dependabot/certifi-requests-pygments (#441) * Bump certifi from 2022.12.7 to 2023.7.22 in /docs Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.12.7 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2022.12.07...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] * add all dependabot changes and update CHANGELOG --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * bugfix/server-pip-redis-conf (#443) * add *.conf to the MANIFEST file so pip will grab the redis.conf file * add note explaining how to fix a hanging merlin server start * modify CHANGELOG * add second export option to docs and fix typo * bump to version 1.10.3 (#444) * bugfix/sphinx-5.3.0-requirement (#446) * Version/1.10.3 (#445) * fix default worker bug with all steps * version bump and requirements fix * Bugfix/filename-special-vars (#425) * fix file naming bug * fix filename bug with variable as study name * add tests for the file name special vars changes * modify changelog * implement Luc's suggestions * remove replace line * Create dependabot-changelog-updater.yml * testing outputs of modifying changelog * delete dependabot-changelog-updater * feature/pdf-docs (#427) * first attempt at adding pdf * fixing build error * modify changelog to show docs changes * fix errors Luc found in the build logs * trying out removal of latex * reverting latex changes back * uncommenting the latex_elements settings * adding epub to see if latex will build * adding a latex engine variable to conf * fix naming error with latex_engine * attempting to add a logo to the pdf build * testing an override to the searchtools file * revert back to not using searchtools override * update changelog * bugfix/openfoam_singularity_issues (#426) * fix openfoam_singularity issues * update requirements and descriptions for openfoam examples * bugfix/output-path-substitution (#430) * fix bug with output_path and variable substitution * add tests for cli substitutions * bugfix/scheduler-permission-error (#436) * Release/1.10.2 (#437) * bump version to 1.10.2 * bump version in CHANGELOG * resolve develop to main merge issues (#439) * fix default worker bug with all steps * version bump and requirements fix * dependabot/certifi-requests-pygments (#441) * Bump certifi from 2022.12.7 to 2023.7.22 in /docs Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.12.7 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2022.12.07...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] * add all dependabot changes and update CHANGELOG --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * bugfix/server-pip-redis-conf (#443) * add *.conf to the MANIFEST file so pip will grab the redis.conf file * add note explaining how to fix a hanging merlin server start * modify CHANGELOG * add second export option to docs and fix typo * bump to version 1.10.3 (#444) --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * change hardcoded sphinx requirement * update CHANGELOG --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feature/vlauncher (#447) * fix file naming error for iterative workflows * fixed small bug with new filepath naming * add VLAUNCHER functionality * add docs for VLAUNCHER and modify changelog * re-word docs and fix table format * add a test for vlauncher * run fix-style and add a test for vlauncher * Add the find_vlaunch_var and setup_vlaunch functions. The numeric value of the shell variables may not be defined until run time, so replace with variable strings instead of values. Consolidate the commands into one function. * Add variable set for (t)csh. * Run fix-style * make step settings the defaults and ignore commented lines * add some additional tests * remove regex library import --------- Co-authored-by: Joseph M. Koning * release/1.11.0 (#448) * bugfix/skewed-sample-hierarchy (#450) * add patch for skewed sample hierarchy/additional samples * update changelog * catch narrower range of exceptions * bugfix/lsf-gpu-typo (#453) * fix typo in batch.py that causes a bug * change print statements to log statements * release/1.11.1 (#454) --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joseph M. Koning --- CHANGELOG.md | 4 ++++ Makefile | 2 +- merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 8 ++++++-- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/conditions.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 2 +- 55 files changed, 64 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f994a4d05..8d0ef2ae6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.11.1] +### Fixed +- Typo in `batch.py` that caused lsf launches to fail (`ALL_SGPUS` changed to `ALL_GPUS`) + ## [1.11.0] ### Added - New reserved variable: diff --git a/Makefile b/Makefile index 4a857a217..2f9db031b 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index 20a0e8b3e..c1ad21b22 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.11.0" +__version__ = "1.11.1" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index f823937a6..3cca2c710 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index 95f26530e..55d616658 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index 7b8ab80f5..383e7dccd 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 124c7851d..d79e4e4f3 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index a8f8dffb2..da366b452 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 149d52e13..4e3ac3a52 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index dc13d41d1..4303c3a6e 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 125ec5bed..806d42e0c 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 68e178b77..e0957ebb8 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index fbd401826..2bd77d2ad 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index c29763485..134d0b66c 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 0594ffe45..b58e3b2a9 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index fe49ff162..385b8c1df 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index 0ff305962..fbbd39064 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 1f3418377..2ca6c5d04 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index d3e7002e7..b88655399 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 65fc6f85c..8f0c6b029 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index a0470938c..78eee5866 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 1d756f00e..9b65f31ae 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 294787857..2fa5e61ce 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index cf272d93b..72d5a1521 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 3fba8cfc8..b8858f721 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 198cf3804..55496a72c 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 7936db03b..a355c4f2f 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 476ab1c0f..01a10aae7 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index d04c75d72..7e2b6f1c1 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 45411131b..a28776577 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 414f7a407..e4ec646fc 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 2b8f1216d..bab641702 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 556f5924e..dcc02b063 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 8972d5cfe..32fd05aa5 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 381bc72f4..5924d1f74 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index f3192a38e..abb0f13c9 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 32fe0f635..ac23b06d1 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index e02a65a32..01a2945e3 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # @@ -299,7 +299,7 @@ def construct_scheduler_legend(parsed_batch: Dict, nodes: int) -> Dict: "lsf": { "check cmd": ["jsrun", "--help"], "expected check output": b"jsrun", - "launch": f"jsrun -a 1 -c ALL_CPUS -g ALL_SGPUS --bind=none -n {nodes}", + "launch": f"jsrun -a 1 -c ALL_CPUS -g ALL_GPUS --bind=none -n {nodes}", }, # pbs is mainly a placeholder in case a user wants to try it (we don't have it at the lab so it's mostly untested) "pbs": { @@ -335,12 +335,16 @@ def construct_worker_launch_command(parsed_batch: Dict, nodes: int) -> str: scheduler_legend: Dict = construct_scheduler_legend(parsed_batch, nodes) workload_manager: str = get_batch_type(scheduler_legend) + LOG.debug(f"parsed_batch: {parsed_batch}") + if parsed_batch["btype"] == "pbs" and workload_manager == parsed_batch["btype"]: raise TypeError("The PBS scheduler is only enabled for 'batch: flux' type") if parsed_batch["btype"] == "slurm" and workload_manager not in ("lsf", "flux", "pbs"): workload_manager = "slurm" + LOG.debug(f"workload_manager: {workload_manager}") + try: launch_command = scheduler_legend[workload_manager]["launch"] except KeyError as e: # pylint: disable=C0103 diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 8b5ff196d..a6707d952 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index ea4d22926..4cffc679c 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 6ecc79c5f..45d211742 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 5d877ba4f..c5773520a 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index b9ada35ea..6e2f4f937 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index 33735085d..196e8b29b 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index 7c91d26c7..7303a1ddf 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index b25010ca2..80e3e5855 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 58460e18f..c0b699055 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index f59acf237..273fa7c56 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # From 5dc82061f235d1710518158b20819bc51ef022ce Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Thu, 2 Nov 2023 11:45:48 -0700 Subject: [PATCH 104/126] Add Pytest Fixtures to Test Suite (#456) * begin work on integration refactor; create fixtures and initial tests * update CHANGELOG and run fix-style * add pytest fixtures and README explaining them * add tests to demonstrate how to use the fixtures * move/rename some files and modify integration's README * add password change to redis.pass file * fix lint issues * modify redis pwd for test server to be constant for each test * fix lint issue only caught on github ci --- CHANGELOG.md | 5 + merlin/common/tasks.py | 2 +- merlin/server/server_commands.py | 3 + merlin/study/celeryadapter.py | 34 +- requirements/dev.txt | 1 + tests/README.md | 152 +++++++++ tests/conftest.py | 301 ++++++++++++++++++ .../{test_definitions.py => definitions.py} | 0 tests/integration/run_tests.py | 6 +- tests/unit/study/test_celeryadapter.py | 160 ++++++++++ 10 files changed, 647 insertions(+), 17 deletions(-) create mode 100644 tests/README.md create mode 100644 tests/conftest.py rename tests/integration/{test_definitions.py => definitions.py} (100%) create mode 100644 tests/unit/study/test_celeryadapter.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d0ef2ae6..9ca916ea9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Added +- Pytest fixtures in the `conftest.py` file of the integration test suite +- Tests for the `celeryadapter.py` module + ## [1.11.1] ### Fixed - Typo in `batch.py` that caused lsf launches to fail (`ALL_SGPUS` changed to `ALL_GPUS`) diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 2bd77d2ad..8292e559f 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -480,7 +480,7 @@ def expand_tasks_with_samples( # pylint: disable=R0913,R0914 if not found_tasks: for next_index_path, next_index in sample_index.traverse(conditional=condition): LOG.info( - f"generating next step for range {next_index.min}:{next_index.max} {next_index.max-next_index.min}" + f"generating next step for range {next_index.min}:{next_index.max} {next_index.max - next_index.min}" ) next_index.name = next_index_path diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index a28776577..c244c9eca 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -92,6 +92,9 @@ def config_server(args: Namespace) -> None: # pylint: disable=R0912 redis_users = RedisUsers(server_config.container.get_user_file_path()) redis_users.set_password("default", args.password) redis_users.write() + pass_file = server_config.container.get_pass_file_path() + with open(pass_file, "w") as pfile: + pfile.write(args.password) redis_config.set_directory(args.directory) diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index a6707d952..cd9714dff 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -37,6 +37,7 @@ import subprocess import time from contextlib import suppress +from typing import Dict, List, Optional from merlin.study.batch import batch_check_parallel, batch_worker_launch from merlin.utils import apply_list_of_regex, check_machines, get_procs, get_yaml_var, is_running @@ -69,23 +70,31 @@ def run_celery(study, run_mode=None): queue_merlin_study(study, adapter_config) -def get_running_queues(): +def get_running_queues(celery_app_name: str, test_mode: bool = False) -> List[str]: """ - Check for running celery workers with -Q queues - and return a unique list of the queues + Check for running celery workers by looking at the currently running processes. + If there are running celery workers, we'll pull the queues from the -Q tag in the + process command. The list returned here will contain only unique celery queue names. + This must be run on the allocation where the workers are running. - Must be run on the allocation where the workers are running + :param `celery_app_name`: The name of the celery app (typically merlin here unless testing) + :param `test_mode`: If True, run this function in test mode + :returns: A unique list of celery queues with workers attached to them """ running_queues = [] - if not is_running("celery worker"): + if not is_running(f"{celery_app_name} worker"): return running_queues - procs = get_procs("celery") + proc_name = "celery" if not test_mode else "sh" + procs = get_procs(proc_name) for _, lcmd in procs: lcmd = list(filter(None, lcmd)) cmdline = " ".join(lcmd) if "-Q" in cmdline: + if test_mode: + echo_cmd = lcmd.pop(2) + lcmd.extend(echo_cmd.split()) running_queues.extend(lcmd[lcmd.index("-Q") + 1].split(",")) running_queues = list(set(running_queues)) @@ -155,19 +164,20 @@ def get_active_workers(app): return worker_queue_map -def celerize_queues(queues): +def celerize_queues(queues: List[str], config: Optional[Dict] = None): """ Celery requires a queue tag to be prepended to their queues so this function will 'celerize' every queue in a list you provide it by prepending the queue tag. - :param `queues`: A list of queues that need the queue - tag prepended. + :param `queues`: A list of queues that need the queue tag prepended. + :param `config`: A dict of configuration settings """ - from merlin.config.configfile import CONFIG # pylint: disable=C0415 + if config is None: + from merlin.config.configfile import CONFIG as config # pylint: disable=C0415 for i, queue in enumerate(queues): - queues[i] = f"{CONFIG.celery.queue_tag}{queue}" + queues[i] = f"{config.celery.queue_tag}{queue}" def _build_output_table(worker_list, output_table): @@ -462,7 +472,7 @@ def start_celery_workers(spec, steps, celery_args, disable_logs, just_return_com running_queues.extend(local_queues) queues = queues.split(",") if not overlap: - running_queues.extend(get_running_queues()) + running_queues.extend(get_running_queues("merlin")) # Cache the queues from this worker to use to test # for existing queues in any subsequent workers. # If overlap is True, then do not check the local queues. diff --git a/requirements/dev.txt b/requirements/dev.txt index 9321694f8..895a89249 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -10,3 +10,4 @@ twine sphinx>=2.0.0 alabaster johnnydep +deepdiff diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..a6bf7005a --- /dev/null +++ b/tests/README.md @@ -0,0 +1,152 @@ +# Tests + +This directory utilizes pytest to create and run our test suite. +Here we use pytest fixtures to create a local redis server and a celery app for testing. + +This directory is organized like so: +- `conftest.py` - The script containing all fixtures for our tests +- `unit/` - The directory containing unit tests + - `test_*.py` - The actual test scripts to run +- `integration/` - The directory containing integration tests + + - `definitions.py` - The test definitions + - `run_tests.py` - The script to run the tests defined in `definitions.py` + - `conditions.py` - The conditions to test against + +## How to Run + +Before running any tests: + +1. Activate your virtual environment with Merlin's dev requirements installed +2. Navigate to the tests folder where this README is located + +To run the entire test suite: + +``` +python -m pytest +``` + +To run a specific test file: + +``` +python -m pytest /path/to/test_specific_file.py +``` + +To run a certain test class within a specific test file: + +``` +python -m pytest /path/to/test_specific_file.py::TestCertainClass +``` + +To run one unique test: + +``` +python -m pytest /path/to/test_specific_file.py::TestCertainClass::test_unique_test +``` + +## Killing the Test Server + +In case of an issue with the test suite, or if you stop the tests with `ctrl+C`, you may need to stop +the server manually. This can be done with: + +``` +redis-cli +127.0.0.1:6379> AUTH merlin-test-server +127.0.0.1:6379> shutdown +not connected> quit +``` + +## The Fixture Process Explained + +Pytest fixtures play a fundamental role in establishing a consistent foundation for test execution, +thus ensuring reliable and predictable test outcomes. This section will delve into essential aspects +of these fixtures, including how to integrate fixtures into tests, the utilization of fixtures within other fixtures, +their scope, and the yielding of fixture results. + +### How to Integrate Fixtures Into Tests + +Probably the most important part of fixtures is understanding how to use them. Luckily, this process is very +simple and can be dumbed down to 2 steps: + +1. Create a fixture in the `conftest.py` file by using the `@pytest.fixture` decorator. For example: + +``` +@pytest.fixture +def dummy_fixture(): + return "hello world" +``` + +2. Use it as an argument in a test function (you don't even need to import it!): + +``` +def test_dummy(dummy_fixture): + assert dummy_fixture == "hello world" +``` + +For more information, see [Pytest's documentation](https://docs.pytest.org/en/7.1.x/how-to/fixtures.html#how-to-use-fixtures). + +### Fixtureception + +One of the coolest and most useful aspects of fixtures that we utilize in this test suite is the ability for +fixtures to be used within other fixtures. For more info on this from pytest, see +[here](https://docs.pytest.org/en/7.1.x/how-to/fixtures.html#fixtures-can-request-other-fixtures). + +Pytest will handle fixtures within fixtures in a stack-based way. Let's look at how creating the `redis_pass` +fixture from our `conftest.py` file works in order to illustrate the process. +1. First, we start by telling pytest that we want to use the `redis_pass` fixture by providing it as an argument +to a test/fixture: + +``` +def test_example(redis_pass): + ... +``` + +2. Now pytest will find the `redis_pass` fixture and put it at the top of the stack to be created. However, +it'll see that this fixture requires another fixture `merlin_server_dir` as an argument: + +``` +@pytest.fixture(scope="session") +def redis_pass(merlin_server_dir): + ... +``` + +3. Pytest then puts the `merlin_server_dir` fixture at the top of the stack, but similarly it sees that this fixture +requires yet another fixture `temp_output_dir`: + +``` +@pytest.fixture(scope="session") +def merlin_server_dir(temp_output_dir: str) -> str: + ... +``` + +4. This process continues until it reaches a fixture that doesn't require any more fixtures. At this point the base +fixture is created and pytest will start working its way back up the stack to the first fixture it looked at (in this +case `redis_pass`). + +5. Once all required fixtures are created, execution will be returned to the test which can now access the fixture +that was requested (`redis_pass`). + +As you can see, if we have to re-do this process for every test it could get pretty time intensive. This is where fixture +scopes come to save the day. + +### Fixture Scopes + +There are several different scopes that you can set for fixtures. The majority of our fixtures use a `session` +scope so that we only have to create the fixtures one time (as some of them can take a few seconds to set up). +The goal is to create fixtures with the most general use-case in mind so that we can re-use them for larger +scopes, which helps with efficiency. + +For more info on scopes, see +[Pytest's Fixture Scope documentation](https://docs.pytest.org/en/6.2.x/fixture.html#scope-sharing-fixtures-across-classes-modules-packages-or-session). + +### Yielding Fixtures + +In several fixtures throughout our test suite, we need to run some sort of teardown for the fixture. For example, +once we no longer need the `redis_server` fixture, we need to shut the server down so it stops using resources. +This is where yielding fixtures becomes extremely useful. + +Using the `yield` keyword allows execution to be returned to a test that needs the fixture once the feature has +been set up. After all tests using the fixture have been ran, execution will return to the fixture for us to run +our teardown code. + +For more information on yielding fixtures, see [Pytest's documentation](https://docs.pytest.org/en/7.1.x/how-to/fixtures.html#teardown-cleanup-aka-fixture-finalization). \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..a496175eb --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,301 @@ +############################################################################### +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.11.1. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### +""" +This module contains pytest fixtures to be used throughout the entire +integration test suite. +""" +import multiprocessing +import os +import subprocess +from time import sleep +from typing import Dict, List + +import pytest +import redis +from _pytest.tmpdir import TempPathFactory +from celery import Celery + + +class RedisServerError(Exception): + """ + Exception to signal that the server wasn't pinged properly. + """ + + +class ServerInitError(Exception): + """ + Exception to signal that there was an error initializing the server. + """ + + +@pytest.fixture(scope="session") +def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: + """ + This fixture will create a temporary directory to store output files of integration tests. + The temporary directory will be stored at /tmp/`whoami`/pytest-of-`whoami`/. There can be at most + 3 temp directories in this location so upon the 4th test run, the 1st temp directory will be removed. + + :param tmp_path_factory: A built in factory with pytest to help create temp paths for testing + :yields: The path to the temp output directory we'll use for this test run + """ + # Log the cwd, then create and move into the temporary one + cwd = os.getcwd() + temp_integration_outfile_dir = tmp_path_factory.mktemp("integration_outfiles_") + os.chdir(temp_integration_outfile_dir) + + yield temp_integration_outfile_dir + + # Move back to the directory we started at + os.chdir(cwd) + + +@pytest.fixture(scope="session") +def redis_pass() -> str: + """ + This fixture represents the password to the merlin test server. + + :returns: The redis password for our test server + """ + return "merlin-test-server" + + +@pytest.fixture(scope="session") +def merlin_server_dir(temp_output_dir: str, redis_pass: str) -> str: # pylint: disable=redefined-outer-name + """ + This fixture will initialize the merlin server (i.e. create all the files we'll + need to start up a local redis server). It will return the path to the directory + containing the files needed for the server to start up. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param redis_pass: The password to the test redis server that we'll create here + :returns: The path to the merlin_server directory with the server configurations + """ + # Initialize the setup for the local redis server + # We'll also set the password to 'merlin-test-server' so it'll be easy to shutdown if there's an issue + subprocess.run(f"merlin server init; merlin server config -pwd {redis_pass}", shell=True, capture_output=True, text=True) + + # Check that the merlin server was initialized properly + server_dir = f"{temp_output_dir}/merlin_server" + if not os.path.exists(server_dir): + raise ServerInitError("The merlin server was not initialized properly.") + + return server_dir + + +@pytest.fixture(scope="session") +def redis_server(merlin_server_dir: str, redis_pass: str) -> str: # pylint: disable=redefined-outer-name,unused-argument + """ + Start a redis server instance that runs on localhost:6379. This will yield the + redis server uri that can be used to create a connection with celery. + + :param merlin_server_dir: The directory to the merlin test server configuration. + This will not be used here but we need the server configurations before we can + start the server. + :param redis_pass: The raw redis password stored in the redis.pass file + :yields: The local redis server uri + """ + # Start the local redis server + try: + subprocess.run("merlin server start", shell=True, capture_output=True, text=True, timeout=5) + except subprocess.TimeoutExpired: + pass + + # Ensure the server started properly + host = "localhost" + port = 6379 + database = 0 + username = "default" + redis_client = redis.Redis(host=host, port=port, db=database, password=redis_pass, username=username) + if not redis_client.ping(): + raise RedisServerError("The redis server could not be pinged. Check that the server is running with 'ps ux'.") + + # Hand over the redis server url to any other fixtures/tests that need it + redis_server_uri = f"redis://{username}:{redis_pass}@{host}:{port}/{database}" + yield redis_server_uri + + # Kill the server; don't run this until all tests are done (accomplished with 'yield' above) + kill_process = subprocess.run("merlin server stop", shell=True, capture_output=True, text=True) + assert "Merlin server terminated." in kill_process.stderr + + +@pytest.fixture(scope="session") +def celery_app(redis_server: str) -> Celery: # pylint: disable=redefined-outer-name + """ + Create the celery app to be used throughout our integration tests. + + :param redis_server: The redis server uri we'll use to connect to redis + :returns: The celery app object we'll use for testing + """ + return Celery("test_app", broker=redis_server, backend=redis_server) + + +@pytest.fixture(scope="session") +def worker_queue_map() -> Dict[str, str]: + """ + Worker and queue names to be used throughout tests + + :returns: A dict of dummy worker/queue associations + """ + return {f"test_worker_{i}": f"test_queue_{i}" for i in range(3)} + + +def are_workers_ready(app: Celery, num_workers: int, verbose: bool = False) -> bool: + """ + Check to see if the workers are up and running yet. + + :param app: The celery app fixture that's connected to our redis server + :param num_workers: An int representing the number of workers we're looking to have started + :param verbose: If true, enable print statements to show where we're at in execution + :returns: True if all workers are running. False otherwise. + """ + app_stats = app.control.inspect().stats() + if verbose: + print(f"app_stats: {app_stats}") + return app_stats is not None and len(app_stats) == num_workers + + +def wait_for_worker_launch(app: Celery, num_workers: int, verbose: bool = False): + """ + Poll the workers over a fixed interval of time. If the workers don't show up + within the time limit then we'll raise a timeout error. Otherwise, the workers + are up and running and we can continue with our tests. + + :param app: The celery app fixture that's connected to our redis server + :param num_workers: An int representing the number of workers we're looking to have started + :param verbose: If true, enable print statements to show where we're at in execution + """ + max_wait_time = 2 # Maximum wait time in seconds + wait_interval = 0.5 # Interval between checks in seconds + waited_time = 0 + + if verbose: + print("waiting for workers to launch...") + + # Wait until all workers are ready + while not are_workers_ready(app, num_workers, verbose=verbose) and waited_time < max_wait_time: + sleep(wait_interval) + waited_time += wait_interval + + # If all workers are not ready after the maximum wait time, raise an error + if not are_workers_ready(app, num_workers, verbose=verbose): + raise TimeoutError("Celery workers did not start within the expected time.") + + if verbose: + print("workers launched") + + +def shutdown_processes(worker_processes: List[multiprocessing.Process], echo_processes: List[subprocess.Popen]): + """ + Given lists of processes, shut them all down. Worker processes were created with the + multiprocessing library and echo processes were created with the subprocess library, + so we have to shut them down slightly differently. + + :param worker_processes: A list of worker processes to terminate + :param echo_processes: A list of echo processes to terminate + """ + # Worker processes were created with the multiprocessing library + for worker_process in worker_processes: + # Try to terminate the process gracefully + worker_process.terminate() + process_exit_code = worker_process.join(timeout=3) + + # If it won't terminate then force kill it + if process_exit_code is None: + worker_process.kill() + + # Gracefully terminate the echo processes + for echo_process in echo_processes: + echo_process.terminate() + echo_process.wait() + + # The echo processes will spawn 3 sleep inf processes that we also need to kill + subprocess.run("ps ux | grep 'sleep inf' | grep -v grep | awk '{print $2}' | xargs kill", shell=True) + + +def start_worker(app: Celery, worker_launch_cmd: List[str]): + """ + This is where a worker is actually started. Each worker maintains control of a process until + we tell it to stop, that's why we have to use the multiprocessing library for this. We have to use + app.worker_main instead of the normal "celery -A worker" command to launch the workers + since our celery app is created in a pytest fixture and is unrecognizable by the celery command. + For each worker, the output of it's logs are sent to + /tmp/`whoami`/pytest-of-`whoami`/pytest-current/integration_outfiles_current/ under a file with a name + similar to: test_worker_*.log. + NOTE: pytest-current/ will have the results of the most recent test run. If you want to see a previous run + check under pytest-/. HOWEVER, only the 3 most recent test runs will be saved. + + :param app: The celery app fixture that's connected to our redis server + :param worker_launch_cmd: The command to launch a worker + """ + app.worker_main(worker_launch_cmd) + + +@pytest.fixture(scope="class") +def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): # pylint: disable=redefined-outer-name + """ + Launch the workers on the celery app fixture using the worker and queue names + defined in the worker_queue_map fixture. + + :param celery_app: The celery app fixture that's connected to our redis server + :param worker_queue_map: A dict where the keys are worker names and the values are queue names + """ + # Create the processes that will start the workers and store them in a list + worker_processes = [] + echo_processes = [] + for worker, queue in worker_queue_map.items(): + worker_launch_cmd = ["worker", "-n", worker, "-Q", queue, "--concurrency", "1", f"--logfile={worker}.log"] + + # We have to use this dummy echo command to simulate a celery worker command that will show up with 'ps ux' + # We'll sleep for infinity here and then kill this process during shutdown + echo_process = subprocess.Popen( # pylint: disable=consider-using-with + f"echo 'celery test_app {' '.join(worker_launch_cmd)}'; sleep inf", shell=True + ) + echo_processes.append(echo_process) + + # We launch workers in their own process since they maintain control of a process until we stop them + worker_process = multiprocessing.Process(target=start_worker, args=(celery_app, worker_launch_cmd)) + worker_process.start() + worker_processes.append(worker_process) + + # Ensure that the workers start properly before letting tests use them + try: + num_workers = len(worker_queue_map) + wait_for_worker_launch(celery_app, num_workers, verbose=False) + except TimeoutError as exc: + # If workers don't launch in time, we need to make sure these processes stop + shutdown_processes(worker_processes, echo_processes) + raise exc + + # Give control to the tests that need to use workers + yield + + # Shut down the workers and terminate the processes + celery_app.control.broadcast("shutdown", destination=list(worker_queue_map.keys())) + shutdown_processes(worker_processes, echo_processes) diff --git a/tests/integration/test_definitions.py b/tests/integration/definitions.py similarity index 100% rename from tests/integration/test_definitions.py rename to tests/integration/definitions.py diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index c0b699055..fcdb9e0b2 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -39,10 +39,8 @@ from contextlib import suppress from subprocess import TimeoutExpired, run -# Pylint complains that we didn't install this module but it's defined locally so ignore -from test_definitions import OUTPUT_DIR, define_tests # pylint: disable=E0401 - from merlin.display import tabulate_info +from tests.integration.definitions import OUTPUT_DIR, define_tests # pylint: disable=E0401 def get_definition_issues(test): @@ -237,7 +235,7 @@ def run_tests(args, tests): # pylint: disable=R0914 total += 1 continue dot_length = 50 - len(test_name) - len(str(test_label)) - print(f"TEST {test_label}: {test_name}{'.'*dot_length}", end="") + print(f"TEST {test_label}: {test_name}{'.' * dot_length}", end="") # Check the format of the test definition definition_issues = get_definition_issues(test) if definition_issues: diff --git a/tests/unit/study/test_celeryadapter.py b/tests/unit/study/test_celeryadapter.py new file mode 100644 index 000000000..82e8401e6 --- /dev/null +++ b/tests/unit/study/test_celeryadapter.py @@ -0,0 +1,160 @@ +############################################################################### +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.11.1. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### +""" +Tests for the celeryadapter module. +""" +from typing import Dict + +import celery + +from merlin.config import Config +from merlin.study import celeryadapter + + +class TestActiveQueues: + """ + This class will test queue related functions in the celeryadapter.py module. + It will run tests where we need active queues to interact with. + """ + + def test_query_celery_queues(self, launch_workers: "Fixture"): # noqa: F821 + """ + Test the query_celery_queues function by providing it with a list of active queues. + This should return a list of tuples. Each tuple will contain information + (name, num jobs, num consumers) for each queue that we provided. + """ + # TODO Modify query_celery_queues so the output for a redis broker is the same + # as the output for rabbit broker + + def test_get_running_queues(self, launch_workers: "Fixture", worker_queue_map: Dict[str, str]): # noqa: F821 + """ + Test the get_running_queues function with queues active. + This should return a list of active queues. + + :param `launch_workers`: A pytest fixture that launches celery workers for us to interact with + :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues + """ + result = celeryadapter.get_running_queues("test_app", test_mode=True) + assert sorted(result) == sorted(list(worker_queue_map.values())) + + def test_get_queues_active( + self, celery_app: celery.Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 + ): + """ + Test the get_queues function with queues active. + This should return a tuple where the first entry is a dict of queue info + and the second entry is a list of worker names. + + :param `celery_app`: A pytest fixture for the test Celery app + :param `launch_workers`: A pytest fixture that launches celery workers for us to interact with + :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues + """ + # Start the queues and run the test + queue_result, worker_result = celeryadapter.get_queues(celery_app) + + # Ensure we got output before looping + assert len(queue_result) == len(worker_result) == 3 + + for worker, queue in worker_queue_map.items(): + # Check that the entry in the queue_result dict for this queue is correct + assert queue in queue_result + assert len(queue_result[queue]) == 1 + assert worker in queue_result[queue][0] + + # Remove this entry from the queue_result dict + del queue_result[queue] + + # Check that this worker was added to the worker_result list + worker_found = False + for worker_name in worker_result[:]: + if worker in worker_name: + worker_found = True + worker_result.remove(worker_name) + break + assert worker_found + + # Ensure there was no extra output that we weren't expecting + assert queue_result == {} + assert worker_result == [] + + +class TestInactiveQueues: + """ + This class will test queue related functions in the celeryadapter.py module. + It will run tests where we don't need any active queues to interact with. + """ + + def test_query_celery_queues(self): + """ + Test the query_celery_queues function by providing it with a list of inactive queues. + This should return a list of strings. Each string will give a message saying that a + particular queue was inactive + """ + # TODO Modify query_celery_queues so the output for a redis broker is the same + # as the output for rabbit broker + + def test_celerize_queues(self, worker_queue_map: Dict[str, str]): + """ + Test the celerize_queues function. This should add the celery queue_tag + to the front of the queues we provide it. + + :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues + """ + # Create variables to be used in the test + queue_tag = "[merlin]_" + queues_to_check = list(worker_queue_map.values()) + dummy_config = Config({"celery": {"queue_tag": queue_tag}}) + + # Run the test + celeryadapter.celerize_queues(queues_to_check, dummy_config) + + # Ensure the queue tag was added to every queue + for queue in queues_to_check: + assert queue_tag in queue + + def test_get_running_queues(self): + """ + Test the get_running_queues function with no queues active. + This should return an empty list. + """ + result = celeryadapter.get_running_queues("test_app", test_mode=True) + assert result == [] + + def test_get_queues(self, celery_app: celery.Celery): + """ + Test the get_queues function with no queues active. + This should return a tuple where the first entry is an empty dict + and the second entry is an empty list. + + :param `celery_app`: A pytest fixture for the test Celery app + """ + queue_result, worker_result = celeryadapter.get_queues(celery_app) + assert queue_result == {} + assert worker_result == [] From 38651f2650e8aba97552c4575e97d66be3205545 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Tue, 14 Nov 2023 17:38:13 -0800 Subject: [PATCH 105/126] Bugfix for WEAVE CI (#457) * begin work on integration refactor; create fixtures and initial tests * update CHANGELOG and run fix-style * add pytest fixtures and README explaining them * add tests to demonstrate how to use the fixtures * move/rename some files and modify integration's README * add password change to redis.pass file * fix lint issues * modify redis pwd for test server to be constant for each test * fix lint issue only caught on github ci * add fix for merlin server startup * update CHANGELOG --- CHANGELOG.md | 1 + tests/conftest.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ca916ea9..9db50369a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - Pytest fixtures in the `conftest.py` file of the integration test suite + - NOTE: an export command `export LC_ALL='C'` had to be added to fix a bug in the WEAVE CI. This can be removed when we resolve this issue for the `merlin server` command - Tests for the `celeryadapter.py` module ## [1.11.1] diff --git a/tests/conftest.py b/tests/conftest.py index a496175eb..88932c5db 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -123,7 +123,8 @@ def redis_server(merlin_server_dir: str, redis_pass: str) -> str: # pylint: dis """ # Start the local redis server try: - subprocess.run("merlin server start", shell=True, capture_output=True, text=True, timeout=5) + # Need to set LC_ALL='C' before starting the server or else redis causes a failure + subprocess.run("export LC_ALL='C'; merlin server start", shell=True, capture_output=True, text=True, timeout=5) except subprocess.TimeoutExpired: pass From b9afbdbe5e075236e6c1e6a86b7f4f63d5a97350 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Tue, 12 Dec 2023 10:59:03 -0800 Subject: [PATCH 106/126] bugfix/monitor-shutdown (#452) * add celery query to see if workers still processing tasks * fix merlin status when using redis as broker * fix consumer count bug and run fix-style * fix linter issues * update changelog * update docs for monitor * remove unused exception I previously added * first attempt at using pytest fixtures for monitor tests * (partially) fix launch_workers fixture so it can be used in multiple classes * fix linter issues and typo on pytest decorator * update black's python version and fix style issue * remove print statements from celeryadapter.py * workers manager is now allowed to be used as a context manager * add one thing to changelog and remove print statement --- .github/workflows/push-pr_workflow.yml | 2 +- CHANGELOG.md | 5 + Makefile | 14 +- docs/source/merlin_commands.rst | 16 +- merlin/exceptions/__init__.py | 11 ++ merlin/main.py | 12 +- merlin/router.py | 163 +++++++++++++---- merlin/study/celeryadapter.py | 99 ++++++++--- requirements/dev.txt | 1 + tests/celery_test_workers.py | 231 +++++++++++++++++++++++++ tests/conftest.py | 157 ++++------------- tests/unit/study/test_celeryadapter.py | 148 +++++++++++++--- 12 files changed, 632 insertions(+), 227 deletions(-) create mode 100644 tests/celery_test_workers.py diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index e2d9e164e..bef6f8608 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -129,7 +129,7 @@ jobs: - name: Run pytest over unit test suite run: | - python3 -m pytest tests/unit/ + python3 -m pytest -v --order-scope=module tests/unit/ - name: Run integration test suite for local tests run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 9db50369a..bf0074e9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Pytest fixtures in the `conftest.py` file of the integration test suite - NOTE: an export command `export LC_ALL='C'` had to be added to fix a bug in the WEAVE CI. This can be removed when we resolve this issue for the `merlin server` command - Tests for the `celeryadapter.py` module +- New CeleryTestWorkersManager context to help with starting/stopping workers for tests + +### Fixed +- The `merlin status` command so that it's consistent in its output whether using redis or rabbitmq as the broker +- The `merlin monitor` command will now keep an allocation up if the queues are empty and workers are still processing tasks ## [1.11.1] ### Fixed diff --git a/Makefile b/Makefile index 2f9db031b..b669d51b1 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,7 @@ install-dev: virtualenv install-merlin install-workflow-deps # tests require a valid dev install of merlin unit-tests: . $(VENV)/bin/activate; \ - $(PYTHON) -m pytest $(UNIT); \ + $(PYTHON) -m pytest -v --order-scope=module $(UNIT); \ # run CLI tests - these require an active install of merlin in a venv @@ -135,9 +135,9 @@ check-flake8: check-black: . $(VENV)/bin/activate; \ - $(PYTHON) -m black --check --line-length $(MAX_LINE_LENGTH) --target-version py36 $(MRLN); \ - $(PYTHON) -m black --check --line-length $(MAX_LINE_LENGTH) --target-version py36 $(TEST); \ - $(PYTHON) -m black --check --line-length $(MAX_LINE_LENGTH) --target-version py36 *.py; \ + $(PYTHON) -m black --check --line-length $(MAX_LINE_LENGTH) --target-version py38 $(MRLN); \ + $(PYTHON) -m black --check --line-length $(MAX_LINE_LENGTH) --target-version py38 $(TEST); \ + $(PYTHON) -m black --check --line-length $(MAX_LINE_LENGTH) --target-version py38 *.py; \ check-isort: @@ -179,9 +179,9 @@ fix-style: $(PYTHON) -m isort -w $(MAX_LINE_LENGTH) $(MRLN); \ $(PYTHON) -m isort -w $(MAX_LINE_LENGTH) $(TEST); \ $(PYTHON) -m isort -w $(MAX_LINE_LENGTH) *.py; \ - $(PYTHON) -m black --target-version py36 -l $(MAX_LINE_LENGTH) $(MRLN); \ - $(PYTHON) -m black --target-version py36 -l $(MAX_LINE_LENGTH) $(TEST); \ - $(PYTHON) -m black --target-version py36 -l $(MAX_LINE_LENGTH) *.py; \ + $(PYTHON) -m black --target-version py38 -l $(MAX_LINE_LENGTH) $(MRLN); \ + $(PYTHON) -m black --target-version py38 -l $(MAX_LINE_LENGTH) $(TEST); \ + $(PYTHON) -m black --target-version py38 -l $(MAX_LINE_LENGTH) *.py; \ # Increment the Merlin version. USE ONLY ON DEVELOP BEFORE MERGING TO MASTER. diff --git a/docs/source/merlin_commands.rst b/docs/source/merlin_commands.rst index cb9b8eefb..1baa0e7a5 100644 --- a/docs/source/merlin_commands.rst +++ b/docs/source/merlin_commands.rst @@ -110,8 +110,15 @@ Monitor (``merlin monitor``) Batch submission scripts may not keep the batch allocation alive if there is not a blocking process in the submission script. The ``merlin monitor`` command addresses this by providing a blocking process that -checks for tasks in the queues every (sleep) seconds. When the queues are empty, the -blocking process will exit and allow the allocation to end. +checks for tasks in the queues every (sleep) seconds. When the queues are empty, the +monitor will query celery to see if any workers are still processing tasks from the +queues. If no workers are processing any tasks from the queues and the queues are empty, +the blocking process will exit and allow the allocation to end. + +The ``monitor`` function will check for celery workers for up to +10*(sleep) seconds before monitoring begins. The loop happens when the +queue(s) in the spec contain tasks, but no running workers are detected. +This is to protect against a failed worker launch. .. code:: bash @@ -129,11 +136,6 @@ for workers. The default is 60 seconds. The only currently available option for ``--task_server`` is celery, which is the default when this flag is excluded. -The ``monitor`` function will check for celery workers for up to -10*(sleep) seconds before monitoring begins. The loop happens when the -queue(s) in the spec contain tasks, but no running workers are detected. -This is to protect against a failed worker launch. - Purging Tasks (``merlin purge``) -------------------------------- diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 72d5a1521..9bc7803a9 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -42,6 +42,7 @@ "HardFailException", "InvalidChainException", "RestartException", + "NoWorkersException", ) @@ -92,3 +93,13 @@ class RestartException(Exception): def __init__(self): super().__init__() + + +class NoWorkersException(Exception): + """ + Exception to signal that no workers were started + to process a non-empty queue(s). + """ + + def __init__(self, message): + super().__init__(message) diff --git a/merlin/main.py b/merlin/main.py index 55496a72c..0ee2e36ce 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -256,8 +256,9 @@ def query_status(args): print(banner_small) spec, _ = get_merlin_spec_with_override(args) ret = router.query_status(args.task_server, spec, args.steps) - for name, jobs, consumers in ret: - print(f"{name:30} - Workers: {consumers:10} - Queued Tasks: {jobs:10}") + + for name, queue_info in ret.items(): + print(f"{name:30} - Workers: {queue_info['consumers']:10} - Queued Tasks: {queue_info['jobs']:10}") if args.csv is not None: router.dump_status(ret, args.csv) @@ -353,8 +354,13 @@ def process_monitor(args): """ LOG.info("Monitor: checking queues ...") spec, _ = get_merlin_spec_with_override(args) + + # Give the user time to queue up jobs in case they haven't already + time.sleep(args.sleep) + + # Check if we still need our allocation while router.check_merlin_status(args, spec): - LOG.info("Monitor: found tasks in queues") + LOG.info("Monitor: found tasks in queues and/or tasks being processed") time.sleep(args.sleep) LOG.info("Monitor: ... stop condition met") diff --git a/merlin/router.py b/merlin/router.py index 01a10aae7..6c90c1d80 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -39,9 +39,13 @@ import os import time from datetime import datetime +from typing import Dict, List +from merlin.exceptions import NoWorkersException from merlin.study.celeryadapter import ( + check_celery_workers_processing, create_celery_config, + get_active_celery_queues, get_workers_from_app, purge_celery_tasks, query_celery_queues, @@ -151,12 +155,12 @@ def dump_status(query_return, csv_file): with open(csv_file, mode=fmode) as f: # pylint: disable=W1514,C0103 if f.mode == "w": # add the header f.write("# time") - for name, job, consumer in query_return: + for name in query_return: f.write(f",{name}:tasks,{name}:consumers") f.write("\n") f.write(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") - for name, job, consumer in query_return: - f.write(f",{job},{consumer}") + for queue_info in query_return.values(): + f.write(f",{queue_info['jobs']},{queue_info['consumers']}") f.write("\n") @@ -236,43 +240,130 @@ def create_config(task_server: str, config_dir: str, broker: str, test: str) -> LOG.error("Only celery can be configured currently.") -def check_merlin_status(args, spec): +def get_active_queues(task_server: str) -> Dict[str, List[str]]: """ - Function to check merlin workers and queues to keep - the allocation alive + Get a dictionary of active queues and the workers attached to these queues. + + :param `task_server`: The task server to query for active queues + :returns: A dict where keys are queue names and values are a list of workers watching them + """ + active_queues = {} + + if task_server == "celery": + from merlin.celery import app # pylint: disable=C0415 + + active_queues, _ = get_active_celery_queues(app) + else: + LOG.error("Only celery can be configured currently.") + + return active_queues + + +def wait_for_workers(sleep: int, task_server: str, spec: "MerlinSpec"): # noqa + """ + Wait on workers to start up. Check on worker start 10 times with `sleep` seconds between + each check. If no workers are started in time, raise an error to kill the monitor (there + was likely an issue with the task server that caused worker launch to fail). + + :param `sleep`: An integer representing the amount of seconds to sleep between each check + :param `task_server`: The task server from which to look for workers + :param `spec`: A MerlinSpec object representing the spec we're monitoring + """ + # Get the names of the workers that we're looking for + worker_names = spec.get_worker_names() + LOG.info(f"Checking for the following workers: {worker_names}") + + # Loop until workers are detected + count = 0 + max_count = 10 + while count < max_count: + # This list will include strings comprised of the worker name with the hostname e.g. worker_name@host. + worker_status = get_workers(task_server) + LOG.info(f"Monitor: checking for workers, running workers = {worker_status} ...") + + # Check to see if any of the workers we're looking for in 'worker_names' have started + check = any(any(iwn in iws for iws in worker_status) for iwn in worker_names) + if check: + break + + # Increment count and sleep until the next check + count += 1 + time.sleep(sleep) + + # If no workers were started in time, raise an exception to stop the monitor + if count == max_count: + raise NoWorkersException("Monitor: no workers available to process the non-empty queue") + + +def check_workers_processing(queues_in_spec: List[str], task_server: str) -> bool: + """ + Check if any workers are still processing tasks by querying the task server. + + :param `queues_in_spec`: A list of queues to check if tasks are still active in + :param `task_server`: The task server from which to query + :returns: True if workers are still processing tasks, False otherwise + """ + result = False + + if task_server == "celery": + from merlin.celery import app + + result = check_celery_workers_processing(queues_in_spec, app) + else: + LOG.error("Celery is not specified as the task server!") + + return result + + +def check_merlin_status(args: "Namespace", spec: "MerlinSpec") -> bool: # noqa + """ + Function to check merlin workers and queues to keep the allocation alive :param `args`: parsed CLI arguments - :param `spec`: the parsed spec.yaml + :param `spec`: the parsed spec.yaml as a MerlinSpec object + :returns: True if there are still tasks being processed, False otherwise """ + # Initialize the variable to track if there are still active tasks + active_tasks = False + + # Get info about jobs and workers in our spec from celery queue_status = query_status(args.task_server, spec, args.steps, verbose=False) + LOG.debug(f"Monitor: queue_status: {queue_status}") + # Count the number of jobs that are active + # (Adding up the number of consumers in the same way is inaccurate so we won't do that) total_jobs = 0 - total_consumers = 0 - for _, jobs, consumers in queue_status: - total_jobs += jobs - total_consumers += consumers - - if total_jobs > 0 and total_consumers == 0: - # Determine if any of the workers are on this allocation - worker_names = spec.get_worker_names() - - # Loop until workers are detected. - count = 0 - max_count = 10 - while count < max_count: - # This list will include strings comprised of the worker name with the hostname e.g. worker_name@host. - worker_status = get_workers(args.task_server) - LOG.info(f"Monitor: checking for workers, running workers = {worker_status} ...") - - check = any(any(iwn in iws for iws in worker_status) for iwn in worker_names) - if check: - break - - count += 1 - time.sleep(args.sleep) - - if count == max_count: - LOG.error("Monitor: no workers available to process the non-empty queue") - total_jobs = 0 - - return total_jobs + for queue_info in queue_status.values(): + total_jobs += queue_info["jobs"] + + # Get the queues defined in the spec + queues_in_spec = spec.get_queue_list(["all"]) + LOG.debug(f"Monitor: queues_in_spec: {queues_in_spec}") + + # Get the active queues and the workers that are watching them + active_queues = get_active_queues(args.task_server) + LOG.debug(f"Monitor: active_queues: {active_queues}") + + # Count the number of workers that are active + consumers = set() + for active_queue, workers_on_queue in active_queues.items(): + if active_queue in queues_in_spec: + consumers |= set(workers_on_queue) + LOG.debug(f"Monitor: consumers found: {consumers}") + total_consumers = len(consumers) + + LOG.info(f"Monitor: found {total_jobs} jobs in queues and {total_consumers} workers alive") + + # If there are no workers, wait for the workers to start + if total_consumers == 0: + wait_for_workers(args.sleep, args.task_server, spec) + + # If we're here, workers have started and jobs should be queued + if total_jobs > 0: + active_tasks = True + # If there are no jobs left, see if any workers are still processing them + elif total_jobs == 0: + active_tasks = check_workers_processing(queues_in_spec, args.task_server) + + LOG.debug(f"Monitor: active_tasks: {active_tasks}") + return active_tasks diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index cd9714dff..84fb96ff1 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -39,12 +39,18 @@ from contextlib import suppress from typing import Dict, List, Optional +from amqp.exceptions import ChannelError +from celery import Celery + +from merlin.config import Config from merlin.study.batch import batch_check_parallel, batch_worker_launch from merlin.utils import apply_list_of_regex, check_machines, get_procs, get_yaml_var, is_running LOG = logging.getLogger(__name__) +# TODO figure out a better way to handle the import of celery app and CONFIG + def run_celery(study, run_mode=None): """ @@ -102,7 +108,7 @@ def get_running_queues(celery_app_name: str, test_mode: bool = False) -> List[st return running_queues -def get_queues(app): +def get_active_celery_queues(app): """Get all active queues and workers for a celery application. Unlike get_running_queues, this goes through the application's server. @@ -117,7 +123,7 @@ def get_queues(app): :example: >>> from merlin.celery import app - >>> queues, workers = get_queues(app) + >>> queues, workers = get_active_celery_queues(app) >>> queue_names = [*queues] >>> workers_on_q0 = queues[queue_names[0]] >>> workers_not_on_q0 = [worker for worker in workers @@ -139,7 +145,7 @@ def get_queues(app): def get_active_workers(app): """ - This is the inverse of get_queues() defined above. This function + This is the inverse of get_active_celery_queues() defined above. This function builds a dict where the keys are worker names and the values are lists of queues attached to the worker. @@ -228,7 +234,7 @@ def query_celery_workers(spec_worker_names, queues, workers_regex): # --queues flag if queues: # Get a mapping between queues and the workers watching them - queue_worker_map, _ = get_queues(app) + queue_worker_map, _ = get_active_celery_queues(app) # Remove duplicates and prepend the celery queue tag to all queues queues = list(set(queues)) celerize_queues(queues) @@ -270,26 +276,54 @@ def query_celery_workers(spec_worker_names, queues, workers_regex): print() -def query_celery_queues(queues): - """Return stats for queues specified. +def query_celery_queues(queues: List[str], app: Celery = None, config: Config = None) -> Dict[str, List[str]]: + """ + Build a dict of information about the number of jobs and consumers attached + to specific queues that we want information on. - Send results to the log. + :param queues: A list of the queues we want to know about + :param app: The celery application (this will be none unless testing) + :param config: The configuration object that has the broker name (this will be none unless testing) + :returns: A dict of info on the number of jobs and consumers for each queue in `queues` """ - from merlin.celery import app # pylint: disable=C0415 + if app is None: + from merlin.celery import app # pylint: disable=C0415 + if config is None: + from merlin.config.configfile import CONFIG as config # pylint: disable=C0415 - connection = app.connection() - found_queues = [] - try: - channel = connection.channel() - for queue in queues: - try: - name, jobs, consumers = channel.queue_declare(queue=queue, passive=True) - found_queues.append((name, jobs, consumers)) - except Exception as e: # pylint: disable=C0103,W0718 - LOG.warning(f"Cannot find queue {queue} on server.{e}") - finally: - connection.close() - return found_queues + # Initialize the dictionary with the info we want about our queues + queue_info = {queue: {"consumers": 0, "jobs": 0} for queue in queues} + + # Open a connection via our Celery app + with app.connection() as conn: + # Open a channel inside our connection + with conn.channel() as channel: + # Loop through all the queues we're searching for + for queue in queues: + try: + # Count the number of jobs and consumers for each queue + _, queue_info[queue]["jobs"], queue_info[queue]["consumers"] = channel.queue_declare( + queue=queue, passive=True + ) + # Redis likes to throw this error when a queue we're looking for has no jobs + except ChannelError: + pass + + # Redis doesn't keep track of consumers attached to queues like rabbit does + # so we have to count this ourselves here + if config.broker.name in ("rediss", "redis"): + # Get a dict of active queues by querying the celery app + active_queues = app.control.inspect().active_queues() + if active_queues is not None: + # Loop through each active queue that was found + for active_queue_list in active_queues.values(): + # Loop through each queue that each worker is watching + for active_queue in active_queue_list: + # If this is a queue we're looking for, increment the consumer count + if active_queue["name"] in queues: + queue_info[active_queue["name"]]["consumers"] += 1 + + return queue_info def get_workers_from_app(): @@ -308,6 +342,27 @@ def get_workers_from_app(): return [*workers] +def check_celery_workers_processing(queues_in_spec: List[str], app: Celery) -> bool: + """ + Query celery to see if any workers are still processing tasks. + + :param queues_in_spec: A list of queues to check if tasks are still active in + :param app: The celery app that we're querying + :returns: True if workers are still processing tasks, False otherwise + """ + # Query celery for active tasks + active_tasks = app.control.inspect().active() + + # Search for the queues we provided if necessary + if active_tasks is not None: + for tasks in active_tasks.values(): + for task in tasks: + if task["delivery_info"]["routing_key"] in queues_in_spec: + return True + + return False + + def _get_workers_to_start(spec, steps): """ Helper function to return a set of workers to start based on @@ -620,7 +675,7 @@ def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): from merlin.celery import app # pylint: disable=C0415 LOG.debug(f"Sending stop to queues: {queues}, worker_regex: {worker_regex}, spec_worker_names: {spec_worker_names}") - active_queues, _ = get_queues(app) + active_queues, _ = get_active_celery_queues(app) # If not specified, get all the queues if queues is None: diff --git a/requirements/dev.txt b/requirements/dev.txt index 895a89249..6e8722b4b 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -11,3 +11,4 @@ sphinx>=2.0.0 alabaster johnnydep deepdiff +pytest-order diff --git a/tests/celery_test_workers.py b/tests/celery_test_workers.py new file mode 100644 index 000000000..39eb2a39b --- /dev/null +++ b/tests/celery_test_workers.py @@ -0,0 +1,231 @@ +############################################################################### +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.11.1. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### +""" +Module to define functionality for test workers and how to start/stop +them in their own processes. +""" +import multiprocessing +import os +import signal +import subprocess +from time import sleep +from types import TracebackType +from typing import Dict, List, Type + +from celery import Celery + + +class CeleryTestWorkersManager: + """ + A class to handle the setup and teardown of celery workers. + This should be treated as a context and used with python's + built-in 'with' statement. If you use it without this statement, + beware that the processes spun up here may never be stopped. + """ + + def __init__(self, app: Celery): + self.app = app + self.running_workers = [] + self.worker_processes = {} + self.echo_processes = {} + + def __enter__(self): + """This magic method is necessary for allowing this class to be used as a context manager.""" + return self + + def __exit__(self, exc_type: Type[Exception], exc_value: Exception, traceback: TracebackType): + """ + This will always run at the end of a context with statement, even if an error is raised. + It's a safe way to ensure all of our subprocesses are stopped no matter what. + """ + + # Try to stop everything gracefully first + self.stop_all_workers() + + # Check that all the worker processes were stopped, otherwise forcefully terminate them + for worker_process in self.worker_processes.values(): + if worker_process.is_alive(): + worker_process.kill() + + # Check that all the echo processes were stopped, otherwise forcefully terminate them + ps_proc = subprocess.run("ps ux", shell=True, capture_output=True, text=True) + for pid in self.echo_processes.values(): + if str(pid) in ps_proc.stdout: + os.kill(pid, signal.SIGKILL) + + def _is_worker_ready(self, worker_name: str, verbose: bool = False) -> bool: + """ + Check to see if the worker is up and running yet. + + :param worker_name: The name of the worker we're checking on + :param verbose: If true, enable print statements to show where we're at in execution + :returns: True if the worker is running. False otherwise. + """ + ping = self.app.control.inspect().ping(destination=[f"celery@{worker_name}"]) + if verbose: + print(f"ping: {ping}") + return ping is not None and f"celery@{worker_name}" in ping + + def _wait_for_worker_launch(self, worker_name: str, verbose: bool = False): + """ + Poll the worker over a fixed interval of time. If the worker doesn't show up + within the time limit then we'll raise a timeout error. Otherwise, the worker + is up and running and we can continue with our tests. + + :param worker_name: The name of the worker we're checking on + :param verbose: If true, enable print statements to show where we're at in execution + """ + max_wait_time = 2 # Maximum wait time in seconds + wait_interval = 0.5 # Interval between checks in seconds + waited_time = 0 + worker_ready = False + + if verbose: + print(f"waiting for {worker_name} to launch...") + + # Wait until the worker is ready + while waited_time < max_wait_time: + if self._is_worker_ready(worker_name, verbose=verbose): + worker_ready = True + break + + sleep(wait_interval) + waited_time += wait_interval + + if not worker_ready: + raise TimeoutError("Celery workers did not start within the expected time.") + + if verbose: + print(f"{worker_name} launched") + + def start_worker(self, worker_launch_cmd: List[str]): + """ + This is where a worker is actually started. Each worker maintains control of a process until + we tell it to stop, that's why we have to use the multiprocessing library for this. We have to use + app.worker_main instead of the normal "celery -A worker" command to launch the workers + since our celery app is created in a pytest fixture and is unrecognizable by the celery command. + For each worker, the output of it's logs are sent to + /tmp/`whoami`/pytest-of-`whoami`/pytest-current/integration_outfiles_current/ under a file with a name + similar to: test_worker_*.log. + NOTE: pytest-current/ will have the results of the most recent test run. If you want to see a previous run + check under pytest-/. HOWEVER, only the 3 most recent test runs will be saved. + + :param worker_launch_cmd: The command to launch a worker + """ + self.app.worker_main(worker_launch_cmd) + + def launch_worker(self, worker_name: str, queues: List[str], concurrency: int = 1): + """ + Launch a single worker. We'll add the process that the worker is running in to the list of worker processes. + We'll also create an echo process to simulate a celery worker command that will show up with 'ps ux'. + + :param worker_name: The name to give to the worker + :param queues: A list of queues that the worker will be watching + :param concurrency: The concurrency value of the worker (how many child processes to have the worker spin up) + """ + # Check to make sure we have a unique worker name so we can track all processes + if worker_name in self.worker_processes: + self.stop_all_workers() + raise ValueError(f"The worker {worker_name} is already running. Choose a different name.") + + # Create the launch command for this worker + worker_launch_cmd = [ + "worker", + "-n", + worker_name, + "-Q", + ",".join(queues), + "--concurrency", + str(concurrency), + f"--logfile={worker_name}.log", + "--loglevel=DEBUG", + ] + + # Create an echo command to simulate a running celery worker since our celery worker will be spun up in + # a different process and we won't be able to see it with 'ps ux' like we normally would + echo_process = subprocess.Popen( # pylint: disable=consider-using-with + f"echo 'celery merlin_test_app {' '.join(worker_launch_cmd)}'; sleep inf", + shell=True, + preexec_fn=os.setpgrp, # Make this the parent of the group so we can kill the 'sleep inf' that's spun up + ) + self.echo_processes[worker_name] = echo_process.pid + + # Start the worker in a separate process since it'll take control of the entire process until we kill it + worker_process = multiprocessing.Process(target=self.start_worker, args=(worker_launch_cmd,)) + worker_process.start() + self.worker_processes[worker_name] = worker_process + self.running_workers.append(worker_name) + + # Wait for the worker to launch properly + try: + self._wait_for_worker_launch(worker_name, verbose=False) + except TimeoutError as exc: + self.stop_all_workers() + raise exc + + def launch_workers(self, worker_info: Dict[str, Dict]): + """ + Launch multiple workers. This will call `launch_worker` to launch each worker + individually. + + :param worker_info: A dict of worker info with the form + {"worker_name": {"concurrency": , "queues": }} + """ + for worker_name, worker_settings in worker_info.items(): + self.launch_worker(worker_name, worker_settings["queues"], worker_settings["concurrency"]) + + def stop_worker(self, worker_name: str): + """ + Stop a single running worker and its associated processes. + + :param worker_name: The name of the worker to shutdown + """ + # Send a shutdown signal to the worker + self.app.control.broadcast("shutdown", destination=[f"celery@{worker_name}"]) + + # Try to terminate the process gracefully + if self.worker_processes[worker_name] is not None: + self.worker_processes[worker_name].terminate() + process_exit_code = self.worker_processes[worker_name].join(timeout=3) + + # If it won't terminate then force kill it + if process_exit_code is None: + self.worker_processes[worker_name].kill() + + # Terminate the echo process and its sleep inf subprocess + os.killpg(os.getpgid(self.echo_processes[worker_name]), signal.SIGTERM) + sleep(2) + + def stop_all_workers(self): + """ + Stop all of the running workers and the processes associated with them. + """ + for worker_name in self.running_workers: + self.stop_worker(worker_name) diff --git a/tests/conftest.py b/tests/conftest.py index 88932c5db..38c6b0334 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,16 +31,18 @@ This module contains pytest fixtures to be used throughout the entire integration test suite. """ -import multiprocessing import os import subprocess from time import sleep -from typing import Dict, List +from typing import Dict import pytest import redis from _pytest.tmpdir import TempPathFactory from celery import Celery +from celery.canvas import Signature + +from tests.celery_test_workers import CeleryTestWorkersManager class RedisServerError(Exception): @@ -124,7 +126,7 @@ def redis_server(merlin_server_dir: str, redis_pass: str) -> str: # pylint: dis # Start the local redis server try: # Need to set LC_ALL='C' before starting the server or else redis causes a failure - subprocess.run("export LC_ALL='C'; merlin server start", shell=True, capture_output=True, text=True, timeout=5) + subprocess.run("export LC_ALL='C'; merlin server start", shell=True, timeout=5) except subprocess.TimeoutExpired: pass @@ -154,108 +156,39 @@ def celery_app(redis_server: str) -> Celery: # pylint: disable=redefined-outer- :param redis_server: The redis server uri we'll use to connect to redis :returns: The celery app object we'll use for testing """ - return Celery("test_app", broker=redis_server, backend=redis_server) + return Celery("merlin_test_app", broker=redis_server, backend=redis_server) @pytest.fixture(scope="session") -def worker_queue_map() -> Dict[str, str]: - """ - Worker and queue names to be used throughout tests - - :returns: A dict of dummy worker/queue associations - """ - return {f"test_worker_{i}": f"test_queue_{i}" for i in range(3)} - - -def are_workers_ready(app: Celery, num_workers: int, verbose: bool = False) -> bool: - """ - Check to see if the workers are up and running yet. - - :param app: The celery app fixture that's connected to our redis server - :param num_workers: An int representing the number of workers we're looking to have started - :param verbose: If true, enable print statements to show where we're at in execution - :returns: True if all workers are running. False otherwise. +def sleep_sig(celery_app: Celery) -> Signature: # pylint: disable=redefined-outer-name """ - app_stats = app.control.inspect().stats() - if verbose: - print(f"app_stats: {app_stats}") - return app_stats is not None and len(app_stats) == num_workers + Create a task registered to our celery app and return a signature for it. + Once requested by a test, you can set the queue you'd like to send this to + with `sleep_sig.set(queue=)`. Here, will likely be + one of the queues defined in the `worker_queue_map` fixture. - -def wait_for_worker_launch(app: Celery, num_workers: int, verbose: bool = False): - """ - Poll the workers over a fixed interval of time. If the workers don't show up - within the time limit then we'll raise a timeout error. Otherwise, the workers - are up and running and we can continue with our tests. - - :param app: The celery app fixture that's connected to our redis server - :param num_workers: An int representing the number of workers we're looking to have started - :param verbose: If true, enable print statements to show where we're at in execution + :param celery_app: The celery app object we'll use for testing + :returns: A celery signature for a task that will sleep for 3 seconds """ - max_wait_time = 2 # Maximum wait time in seconds - wait_interval = 0.5 # Interval between checks in seconds - waited_time = 0 - - if verbose: - print("waiting for workers to launch...") - # Wait until all workers are ready - while not are_workers_ready(app, num_workers, verbose=verbose) and waited_time < max_wait_time: - sleep(wait_interval) - waited_time += wait_interval + # Create a celery task that sleeps for 3 sec + @celery_app.task + def sleep_task(): + print("running sleep task") + sleep(3) - # If all workers are not ready after the maximum wait time, raise an error - if not are_workers_ready(app, num_workers, verbose=verbose): - raise TimeoutError("Celery workers did not start within the expected time.") + # Create a signature for this task + return sleep_task.s() - if verbose: - print("workers launched") - -def shutdown_processes(worker_processes: List[multiprocessing.Process], echo_processes: List[subprocess.Popen]): - """ - Given lists of processes, shut them all down. Worker processes were created with the - multiprocessing library and echo processes were created with the subprocess library, - so we have to shut them down slightly differently. - - :param worker_processes: A list of worker processes to terminate - :param echo_processes: A list of echo processes to terminate +@pytest.fixture(scope="session") +def worker_queue_map() -> Dict[str, str]: """ - # Worker processes were created with the multiprocessing library - for worker_process in worker_processes: - # Try to terminate the process gracefully - worker_process.terminate() - process_exit_code = worker_process.join(timeout=3) - - # If it won't terminate then force kill it - if process_exit_code is None: - worker_process.kill() - - # Gracefully terminate the echo processes - for echo_process in echo_processes: - echo_process.terminate() - echo_process.wait() - - # The echo processes will spawn 3 sleep inf processes that we also need to kill - subprocess.run("ps ux | grep 'sleep inf' | grep -v grep | awk '{print $2}' | xargs kill", shell=True) - + Worker and queue names to be used throughout tests -def start_worker(app: Celery, worker_launch_cmd: List[str]): - """ - This is where a worker is actually started. Each worker maintains control of a process until - we tell it to stop, that's why we have to use the multiprocessing library for this. We have to use - app.worker_main instead of the normal "celery -A worker" command to launch the workers - since our celery app is created in a pytest fixture and is unrecognizable by the celery command. - For each worker, the output of it's logs are sent to - /tmp/`whoami`/pytest-of-`whoami`/pytest-current/integration_outfiles_current/ under a file with a name - similar to: test_worker_*.log. - NOTE: pytest-current/ will have the results of the most recent test run. If you want to see a previous run - check under pytest-/. HOWEVER, only the 3 most recent test runs will be saved. - - :param app: The celery app fixture that's connected to our redis server - :param worker_launch_cmd: The command to launch a worker + :returns: A dict of dummy worker/queue associations """ - app.worker_main(worker_launch_cmd) + return {f"test_worker_{i}": f"test_queue_{i}" for i in range(3)} @pytest.fixture(scope="class") @@ -267,36 +200,10 @@ def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): # pyl :param celery_app: The celery app fixture that's connected to our redis server :param worker_queue_map: A dict where the keys are worker names and the values are queue names """ - # Create the processes that will start the workers and store them in a list - worker_processes = [] - echo_processes = [] - for worker, queue in worker_queue_map.items(): - worker_launch_cmd = ["worker", "-n", worker, "-Q", queue, "--concurrency", "1", f"--logfile={worker}.log"] - - # We have to use this dummy echo command to simulate a celery worker command that will show up with 'ps ux' - # We'll sleep for infinity here and then kill this process during shutdown - echo_process = subprocess.Popen( # pylint: disable=consider-using-with - f"echo 'celery test_app {' '.join(worker_launch_cmd)}'; sleep inf", shell=True - ) - echo_processes.append(echo_process) - - # We launch workers in their own process since they maintain control of a process until we stop them - worker_process = multiprocessing.Process(target=start_worker, args=(celery_app, worker_launch_cmd)) - worker_process.start() - worker_processes.append(worker_process) - - # Ensure that the workers start properly before letting tests use them - try: - num_workers = len(worker_queue_map) - wait_for_worker_launch(celery_app, num_workers, verbose=False) - except TimeoutError as exc: - # If workers don't launch in time, we need to make sure these processes stop - shutdown_processes(worker_processes, echo_processes) - raise exc - - # Give control to the tests that need to use workers - yield - - # Shut down the workers and terminate the processes - celery_app.control.broadcast("shutdown", destination=list(worker_queue_map.keys())) - shutdown_processes(worker_processes, echo_processes) + # Format worker info in a format the our workers manager will be able to read + # (basically just add in concurrency value to worker_queue_map) + worker_info = {worker_name: {"concurrency": 1, "queues": [queue]} for worker_name, queue in worker_queue_map.items()} + + with CeleryTestWorkersManager(celery_app) as workers_manager: + workers_manager.launch_workers(worker_info) + yield diff --git a/tests/unit/study/test_celeryadapter.py b/tests/unit/study/test_celeryadapter.py index 82e8401e6..67728881e 100644 --- a/tests/unit/study/test_celeryadapter.py +++ b/tests/unit/study/test_celeryadapter.py @@ -30,28 +30,55 @@ """ Tests for the celeryadapter module. """ +from time import sleep from typing import Dict -import celery +import pytest +from celery import Celery +from celery.canvas import Signature from merlin.config import Config from merlin.study import celeryadapter -class TestActiveQueues: +@pytest.mark.order(before="TestInactive") +class TestActive: """ - This class will test queue related functions in the celeryadapter.py module. - It will run tests where we need active queues to interact with. + This class will test functions in the celeryadapter.py module. + It will run tests where we need active queues/workers to interact with. + + NOTE: The tests in this class must be ran before the TestInactive class or else the + Celery workers needed for this class don't start + + TODO: fix the bug noted above and then check if we still need pytest-order """ - def test_query_celery_queues(self, launch_workers: "Fixture"): # noqa: F821 + def test_query_celery_queues( + self, celery_app: Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 + ): """ Test the query_celery_queues function by providing it with a list of active queues. - This should return a list of tuples. Each tuple will contain information - (name, num jobs, num consumers) for each queue that we provided. + This should return a dict where keys are queue names and values are more dicts containing + the number of jobs and consumers in that queue. + + :param `celery_app`: A pytest fixture for the test Celery app + :param launch_workers: A pytest fixture that launches celery workers for us to interact with + :param worker_queue_map: A pytest fixture that returns a dict of workers and queues """ - # TODO Modify query_celery_queues so the output for a redis broker is the same - # as the output for rabbit broker + # Set up a dummy configuration to use in the test + dummy_config = Config({"broker": {"name": "redis"}}) + + # Get the actual output + queues_to_query = list(worker_queue_map.values()) + actual_queue_info = celeryadapter.query_celery_queues(queues_to_query, app=celery_app, config=dummy_config) + + # Ensure all 3 queues in worker_queue_map were queried before looping + assert len(actual_queue_info) == 3 + + # Ensure each queue has a worker attached + for queue_name, queue_info in actual_queue_info.items(): + assert queue_name in worker_queue_map.values() + assert queue_info == {"consumers": 1, "jobs": 0} def test_get_running_queues(self, launch_workers: "Fixture", worker_queue_map: Dict[str, str]): # noqa: F821 """ @@ -61,14 +88,14 @@ def test_get_running_queues(self, launch_workers: "Fixture", worker_queue_map: D :param `launch_workers`: A pytest fixture that launches celery workers for us to interact with :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues """ - result = celeryadapter.get_running_queues("test_app", test_mode=True) + result = celeryadapter.get_running_queues("merlin_test_app", test_mode=True) assert sorted(result) == sorted(list(worker_queue_map.values())) - def test_get_queues_active( - self, celery_app: celery.Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 + def test_get_active_celery_queues( + self, celery_app: Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 ): """ - Test the get_queues function with queues active. + Test the get_active_celery_queues function with queues active. This should return a tuple where the first entry is a dict of queue info and the second entry is a list of worker names. @@ -77,7 +104,7 @@ def test_get_queues_active( :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues """ # Start the queues and run the test - queue_result, worker_result = celeryadapter.get_queues(celery_app) + queue_result, worker_result = celeryadapter.get_active_celery_queues(celery_app) # Ensure we got output before looping assert len(queue_result) == len(worker_result) == 3 @@ -104,21 +131,78 @@ def test_get_queues_active( assert queue_result == {} assert worker_result == [] + @pytest.mark.order(index=1) + def test_check_celery_workers_processing_tasks( + self, + celery_app: Celery, + sleep_sig: Signature, + launch_workers: "Fixture", # noqa: F821 + ): + """ + Test the check_celery_workers_processing function with workers active and a task in a queue. + This function will query workers for any tasks they're still processing. We'll send a + a task that sleeps for 3 seconds to our workers before we run this test so that there should be + a task for this function to find. + + NOTE: the celery app fixture shows strange behavior when using app.control.inspect() calls (which + check_celery_workers_processing uses) so we have to run this test first in this class in order to + have it run properly. + + :param celery_app: A pytest fixture for the test Celery app + :param sleep_sig: A pytest fixture for a celery signature of a task that sleeps for 3 sec + :param launch_workers: A pytest fixture that launches celery workers for us to interact with + """ + # Our active workers/queues are test_worker_[0-2]/test_queue_[0-2] so we're + # sending this to test_queue_0 for test_worker_0 to process + queue_for_signature = "test_queue_0" + sleep_sig.set(queue=queue_for_signature) + result = sleep_sig.delay() + + # We need to give the task we just sent to the server a second to get picked up by the worker + sleep(1) -class TestInactiveQueues: + # Run the test now that the task should be getting processed + active_queue_test = celeryadapter.check_celery_workers_processing([queue_for_signature], celery_app) + assert active_queue_test is True + + # Now test that a queue without any tasks returns false + # We sent the signature to task_queue_0 so task_queue_1 shouldn't have any tasks to find + non_active_queue_test = celeryadapter.check_celery_workers_processing(["test_queue_1"], celery_app) + assert non_active_queue_test is False + + # Wait for the worker to finish running the task + result.get() + + +class TestInactive: """ - This class will test queue related functions in the celeryadapter.py module. - It will run tests where we don't need any active queues to interact with. + This class will test functions in the celeryadapter.py module. + It will run tests where we don't need any active queues/workers to interact with. """ - def test_query_celery_queues(self): + def test_query_celery_queues(self, celery_app: Celery, worker_queue_map: Dict[str, str]): # noqa: F821 """ Test the query_celery_queues function by providing it with a list of inactive queues. - This should return a list of strings. Each string will give a message saying that a - particular queue was inactive + This should return a dict where keys are queue names and values are more dicts containing + the number of jobs and consumers in that queue (which should be 0 for both here). + + :param `celery_app`: A pytest fixture for the test Celery app + :param worker_queue_map: A pytest fixture that returns a dict of workers and queues """ - # TODO Modify query_celery_queues so the output for a redis broker is the same - # as the output for rabbit broker + # Set up a dummy configuration to use in the test + dummy_config = Config({"broker": {"name": "redis"}}) + + # Get the actual output + queues_to_query = list(worker_queue_map.values()) + actual_queue_info = celeryadapter.query_celery_queues(queues_to_query, app=celery_app, config=dummy_config) + + # Ensure all 3 queues in worker_queue_map were queried before looping + assert len(actual_queue_info) == 3 + + # Ensure each queue has no worker attached (since the queues should be inactive here) + for queue_name, queue_info in actual_queue_info.items(): + assert queue_name in worker_queue_map.values() + assert queue_info == {"consumers": 0, "jobs": 0} def test_celerize_queues(self, worker_queue_map: Dict[str, str]): """ @@ -144,17 +228,29 @@ def test_get_running_queues(self): Test the get_running_queues function with no queues active. This should return an empty list. """ - result = celeryadapter.get_running_queues("test_app", test_mode=True) + result = celeryadapter.get_running_queues("merlin_test_app", test_mode=True) assert result == [] - def test_get_queues(self, celery_app: celery.Celery): + def test_get_active_celery_queues(self, celery_app: Celery): """ - Test the get_queues function with no queues active. + Test the get_active_celery_queues function with no queues active. This should return a tuple where the first entry is an empty dict and the second entry is an empty list. :param `celery_app`: A pytest fixture for the test Celery app """ - queue_result, worker_result = celeryadapter.get_queues(celery_app) + queue_result, worker_result = celeryadapter.get_active_celery_queues(celery_app) assert queue_result == {} assert worker_result == [] + + def test_check_celery_workers_processing_tasks(self, celery_app: Celery, worker_queue_map: Dict[str, str]): + """ + Test the check_celery_workers_processing function with no workers active. + This function will query workers for any tasks they're still processing. Since no workers are active + this should return False. + + :param celery_app: A pytest fixture for the test Celery app + """ + # Run the test now that the task should be getting processed + result = celeryadapter.check_celery_workers_processing(list(worker_queue_map.values()), celery_app) + assert result is False From 642f925af5f83e5b250d4716a5404c8779318b02 Mon Sep 17 00:00:00 2001 From: Joe Koning Date: Fri, 19 Jan 2024 09:05:21 -0800 Subject: [PATCH 107/126] Add the missing restart keyword to the specification docs. (#459) --- CHANGELOG.md | 1 + docs/source/merlin_specification.rst | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf0074e9d..9138821e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - The `merlin status` command so that it's consistent in its output whether using redis or rabbitmq as the broker - The `merlin monitor` command will now keep an allocation up if the queues are empty and workers are still processing tasks +- Add the restart keyword to the specification docs ## [1.11.1] ### Fixed diff --git a/docs/source/merlin_specification.rst b/docs/source/merlin_specification.rst index f857fabf3..71e041b33 100644 --- a/docs/source/merlin_specification.rst +++ b/docs/source/merlin_specification.rst @@ -120,6 +120,9 @@ see :doc:`./merlin_variables`. # The $(LAUNCHER) macro can be used to substitute a parallel launcher # based on the batch:type:. # It will use the nodes and procs values for the task. + # restart: The (optional) restart command to run when $(MERLIN_RESTART) + # is the exit code. The command in cmd will be run if the exit code + # is $(MERLIN_RETRY). # task_queue: the queue to assign the step to (optional. default: merlin) # shell: the shell to use for the command (eg /bin/bash /usr/bin/env python) # (optional. default: /bin/bash) @@ -156,6 +159,8 @@ see :doc:`./merlin_variables`. cmd: | cd $(runs1.workspace)/$(MERLIN_SAMPLE_PATH) + # exit $(MERLIN_RESTART) # syntax to send a restart error code + # This will rerun the cmd command. Users can also use $(MERLIN_RETRY). nodes: 1 procs: 1 depends: [runs1] @@ -167,7 +172,14 @@ see :doc:`./merlin_variables`. cmd: | touch learnrun.out $(LAUNCHER) echo "$(VAR1) $(VAR2)" >> learnrun.out - exit $(MERLIN_RETRY) # some syntax to send a retry error code + exit $(MERLIN_RESTART) # syntax to send a restart error code + # exit $(MERLIN_RETRY) # syntax to send a retry error code to + # run the cmd command again instead of the restart command. + restart: | + # Command to run if the $(MERLIN_RESTART) exit code is used + touch learnrunrs.out + $(LAUNCHER) echo "$(VAR1) $(VAR2)" >> learnrunrs.out + exit $(MERLIN_SUCCESS) # syntax to send a success code nodes: 1 procs: 1 task_queue: lqueue From 40930c226080ea225f74a0c0343cba1f6ce10d83 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Thu, 25 Jan 2024 08:58:26 -0800 Subject: [PATCH 108/126] docs/conversion-to-mkdocs (#460) * remove a merge conflict statement that was missed * add base requirements for mkdocs * set up configuration for API docs * start work on porting user guide to mkdocs * add custom styling and contact page * begin work on porting tutorial to mkdocs * add new examples page * move old sphinx docs to their own folder (*delete later*) * modify some admonitions to be success * modify hello examples page and port step 3 of tutorial to mkdocs * fix typo in hello example * finish porting step 4 of tutorial to mkdocs * port part 5 of the tutorial to mkdocs * copy faq and contributing from old docs * port step 6 of tutorial to mkdocs * remove unused prereq * port step 7 of tutorial to mkdocs * add more detailed instructions on contributing * move venv page into installation and add spack instructions too * add configuration docs * add content to user guide landing page * port celery page to mkdocs * rearrange configuration pages to add in merlin server configuration instructions * port command line page to mkdocs * finish new landing page * change size of merlin logo * port variables page to mkdocs * fix broken links to configuration page * port FAQ to mkdocs * fix incorrect requirement name * update CHANGELOG * attempt to get docs to build through readthedocs * port docker page to mkdocs * port contributing guide to mkdocs * add new 'running studies' page * add path changes to images * add a page on how to interpret study output * add page on the spec file * remove old sphinx docs that are no longer needed * added README to docs and updated CHANGELOG * fix copyright and hello_samples tree * rearrange images/stylesheets and statements that use them * add suggestions from Luc and Joe * add tcsh instructions for venv activation * add Charle's suggestions for the landing page * change tcsh mentions to csh --- .readthedocs.yaml | 6 +- CHANGELOG.md | 15 + docs/Makefile | 31 - docs/README.md | 44 + docs/api_reference/index.md | 5 + docs/assets/images/hello-samples-tree.png | Bin 0 -> 124951 bytes .../basic-merlin-info-workspace.png | Bin 0 -> 28175 bytes .../basic-step-workspace.png | Bin 0 -> 27852 bytes .../merlin-info-with-samples.png | Bin 0 -> 48968 bytes .../modified-hierarchy-structure.png | Bin 0 -> 57958 bytes .../two-level-sample-hierarchy.png | Bin 0 -> 57043 bytes .../workspace-with-params-and-samples.png | Bin 0 -> 39547 bytes .../workspace-with-params.png | Bin 0 -> 97183 bytes .../workspace-with-samples.png | Bin 0 -> 37567 bytes docs/{ => assets}/images/merlin_arch.png | Bin .../images/merlin_banner.png} | Bin docs/assets/images/merlin_banner_white.png | Bin 0 -> 61782 bytes docs/{ => assets}/images/merlin_icon.png | Bin .../running_studies/current-node-launch.png | Bin 0 -> 46785 bytes .../running_studies/iterative-diagram.png | Bin 0 -> 106628 bytes .../running_studies/merlin-run-diagram.png | Bin 0 -> 120345 bytes .../running_studies/parallel-launch.png | Bin 0 -> 61759 bytes .../producer-consumer-model.png | Bin 0 -> 89461 bytes .../worker-server-communication.png | Bin 0 -> 60915 bytes .../advanced_topics/cumulative_results.png | Bin .../images/tutorial}/hello_world/dag1.png | Bin .../images/tutorial}/hello_world/dag2.png | Bin .../images/tutorial}/hello_world/dag3.png | Bin .../images/tutorial}/hello_world/dag4.png | Bin .../tutorial}/hello_world/merlin_output.png | Bin .../tutorial}/hello_world/merlin_output2.png | Bin .../introduction}/central_coordination.png | Bin .../introduction}/external_coordination.png | Bin .../introduction}/internal_coordination.png | Bin .../tutorial/introduction}/merlin_run.png | Bin .../introduction}/task_creation_rate.png | Bin .../run_simulation/lid-driven-stable.png | Bin .../tutorial}/run_simulation/openfoam_dag.png | Bin .../run_simulation/openfoam_wf_output.png | Bin .../tutorial}/run_simulation/prediction.png | Bin .../images/tutorial}/run_simulation/setup.png | Bin docs/assets/javascripts/swap_lp_image.js | 26 + docs/assets/stylesheets/extra.css | 8 + docs/contact.md | 21 + docs/examples/feature_demo.md | 3 + docs/examples/flux.md | 3 + docs/examples/hello.md | 734 +++++++++++ docs/examples/hpc.md | 3 + docs/examples/index.md | 48 + docs/examples/iterative.md | 3 + docs/examples/lsf.md | 3 + docs/examples/restart.md | 3 + docs/examples/slurm.md | 3 + docs/faq.md | 453 +++++++ docs/gen_ref_pages.py | 40 + docs/index.md | 226 ++++ docs/make.bat | 36 - docs/requirements.in | 4 - docs/requirements.txt | 10 +- docs/source/.gitignore | 4 - docs/source/_static/custom.css | 39 - docs/source/_static/custom.js | 25 - docs/source/app_config/app_amqp.yaml | 58 - docs/source/celery_overview.rst | 125 -- docs/source/conf.py | 183 --- docs/source/docker.rst | 93 -- docs/source/faq.rst | 476 -------- docs/source/getting_started.rst | 156 --- docs/source/index.rst | 83 -- docs/source/merlin_commands.rst | 480 -------- docs/source/merlin_config.rst | 303 ----- docs/source/merlin_developer.rst | 175 --- docs/source/merlin_server.rst | 72 -- docs/source/merlin_specification.rst | 358 ------ docs/source/merlin_variables.rst | 397 ------ docs/source/merlin_workflows.rst | 49 - .../advanced_topics/advanced_requirements.txt | 3 - .../advanced_topics/advanced_topics.rst | 415 ------- .../modules/advanced_topics/faker_demo.yaml | 116 -- docs/source/modules/before.rst | 47 - docs/source/modules/contribute.rst | 48 - docs/source/modules/hello_world/.gitignore | 1 - docs/source/modules/hello_world/celery.txt | 27 - docs/source/modules/hello_world/hello.yaml | 24 - .../modules/hello_world/hello_world.rst | 353 ------ docs/source/modules/hello_world/local_out.txt | 31 - docs/source/modules/hello_world/run_out.txt | 25 - .../modules/hello_world/run_workers_out.txt | 21 - .../modules/hello_world/stop_workers.txt | 24 - .../installation/app_docker_rabbit.yaml | 10 - .../installation/app_docker_redis.yaml | 11 - .../modules/installation/app_local_redis.yaml | 11 - .../modules/installation/docker-compose.yml | 23 - .../installation/docker-compose_rabbit.yml | 46 - .../docker-compose_rabbit_redis_tls.yml | 38 - .../modules/installation/installation.rst | 340 ------ docs/source/modules/introduction.rst | 476 -------- docs/source/modules/port_your_application.rst | 70 -- .../modules/run_simulation/run_simulation.rst | 429 ------- docs/source/server/commands.rst | 94 -- docs/source/server/configuration.rst | 75 -- docs/source/spack.rst | 131 -- docs/source/virtualenv.rst | 54 - docs/tutorial/0_prerequisites.md | 36 + docs/tutorial/1_introduction.md | 242 ++++ docs/tutorial/2_installation.md | 516 ++++++++ docs/tutorial/3_hello_world.md | 478 ++++++++ docs/tutorial/4_run_simulation.md | 331 +++++ docs/tutorial/5_advanced_topics.md | 430 +++++++ docs/tutorial/6_contribute.md | 112 ++ docs/tutorial/7_port_application.md | 57 + .../tutorial.rst => tutorial/index.md} | 30 +- docs/user_guide/celery.md | 205 ++++ docs/user_guide/command_line.md | 684 +++++++++++ .../configuration/external_server.md | 407 +++++++ docs/user_guide/configuration/index.md | 266 ++++ .../user_guide/configuration/merlin_server.md | 177 +++ docs/user_guide/contributing.md | 193 +++ docs/user_guide/docker.md | 332 +++++ docs/user_guide/index.md | 23 + docs/user_guide/installation.md | 271 +++++ docs/user_guide/interpreting_output.md | 1072 +++++++++++++++++ docs/user_guide/running_studies.md | 520 ++++++++ docs/user_guide/specification.md | 725 +++++++++++ docs/user_guide/variables.md | 254 ++++ merlin/main.py | 2 +- mkdocs.yml | 126 ++ 127 files changed, 9115 insertions(+), 6121 deletions(-) delete mode 100644 docs/Makefile create mode 100644 docs/README.md create mode 100644 docs/api_reference/index.md create mode 100644 docs/assets/images/hello-samples-tree.png create mode 100644 docs/assets/images/interpreting_output/basic-merlin-info-workspace.png create mode 100644 docs/assets/images/interpreting_output/basic-step-workspace.png create mode 100644 docs/assets/images/interpreting_output/merlin-info-with-samples.png create mode 100644 docs/assets/images/interpreting_output/modified-hierarchy-structure.png create mode 100644 docs/assets/images/interpreting_output/two-level-sample-hierarchy.png create mode 100644 docs/assets/images/interpreting_output/workspace-with-params-and-samples.png create mode 100644 docs/assets/images/interpreting_output/workspace-with-params.png create mode 100644 docs/assets/images/interpreting_output/workspace-with-samples.png rename docs/{ => assets}/images/merlin_arch.png (100%) rename docs/{images/merlin.png => assets/images/merlin_banner.png} (100%) create mode 100644 docs/assets/images/merlin_banner_white.png rename docs/{ => assets}/images/merlin_icon.png (100%) create mode 100644 docs/assets/images/running_studies/current-node-launch.png create mode 100644 docs/assets/images/running_studies/iterative-diagram.png create mode 100644 docs/assets/images/running_studies/merlin-run-diagram.png create mode 100644 docs/assets/images/running_studies/parallel-launch.png create mode 100644 docs/assets/images/running_studies/producer-consumer-model.png create mode 100644 docs/assets/images/running_studies/worker-server-communication.png rename docs/{source/modules => assets/images/tutorial}/advanced_topics/cumulative_results.png (100%) rename docs/{source/modules => assets/images/tutorial}/hello_world/dag1.png (100%) rename docs/{source/modules => assets/images/tutorial}/hello_world/dag2.png (100%) rename docs/{source/modules => assets/images/tutorial}/hello_world/dag3.png (100%) rename docs/{source/modules => assets/images/tutorial}/hello_world/dag4.png (100%) rename docs/{source/modules => assets/images/tutorial}/hello_world/merlin_output.png (100%) rename docs/{source/modules => assets/images/tutorial}/hello_world/merlin_output2.png (100%) rename docs/{images => assets/images/tutorial/introduction}/central_coordination.png (100%) rename docs/{images => assets/images/tutorial/introduction}/external_coordination.png (100%) rename docs/{images => assets/images/tutorial/introduction}/internal_coordination.png (100%) rename docs/{images => assets/images/tutorial/introduction}/merlin_run.png (100%) rename docs/{images => assets/images/tutorial/introduction}/task_creation_rate.png (100%) rename docs/{source/modules => assets/images/tutorial}/run_simulation/lid-driven-stable.png (100%) rename docs/{source/modules => assets/images/tutorial}/run_simulation/openfoam_dag.png (100%) rename docs/{source/modules => assets/images/tutorial}/run_simulation/openfoam_wf_output.png (100%) rename docs/{source/modules => assets/images/tutorial}/run_simulation/prediction.png (100%) rename docs/{source/modules => assets/images/tutorial}/run_simulation/setup.png (100%) create mode 100644 docs/assets/javascripts/swap_lp_image.js create mode 100644 docs/assets/stylesheets/extra.css create mode 100644 docs/contact.md create mode 100644 docs/examples/feature_demo.md create mode 100644 docs/examples/flux.md create mode 100644 docs/examples/hello.md create mode 100644 docs/examples/hpc.md create mode 100644 docs/examples/index.md create mode 100644 docs/examples/iterative.md create mode 100644 docs/examples/lsf.md create mode 100644 docs/examples/restart.md create mode 100644 docs/examples/slurm.md create mode 100644 docs/faq.md create mode 100644 docs/gen_ref_pages.py create mode 100644 docs/index.md delete mode 100644 docs/make.bat delete mode 100644 docs/requirements.in delete mode 100644 docs/source/.gitignore delete mode 100644 docs/source/_static/custom.css delete mode 100644 docs/source/_static/custom.js delete mode 100644 docs/source/app_config/app_amqp.yaml delete mode 100644 docs/source/celery_overview.rst delete mode 100644 docs/source/conf.py delete mode 100644 docs/source/docker.rst delete mode 100644 docs/source/faq.rst delete mode 100644 docs/source/getting_started.rst delete mode 100644 docs/source/index.rst delete mode 100644 docs/source/merlin_commands.rst delete mode 100644 docs/source/merlin_config.rst delete mode 100644 docs/source/merlin_developer.rst delete mode 100644 docs/source/merlin_server.rst delete mode 100644 docs/source/merlin_specification.rst delete mode 100644 docs/source/merlin_variables.rst delete mode 100644 docs/source/merlin_workflows.rst delete mode 100644 docs/source/modules/advanced_topics/advanced_requirements.txt delete mode 100644 docs/source/modules/advanced_topics/advanced_topics.rst delete mode 100644 docs/source/modules/advanced_topics/faker_demo.yaml delete mode 100644 docs/source/modules/before.rst delete mode 100644 docs/source/modules/contribute.rst delete mode 100644 docs/source/modules/hello_world/.gitignore delete mode 100644 docs/source/modules/hello_world/celery.txt delete mode 100644 docs/source/modules/hello_world/hello.yaml delete mode 100644 docs/source/modules/hello_world/hello_world.rst delete mode 100644 docs/source/modules/hello_world/local_out.txt delete mode 100644 docs/source/modules/hello_world/run_out.txt delete mode 100644 docs/source/modules/hello_world/run_workers_out.txt delete mode 100644 docs/source/modules/hello_world/stop_workers.txt delete mode 100644 docs/source/modules/installation/app_docker_rabbit.yaml delete mode 100644 docs/source/modules/installation/app_docker_redis.yaml delete mode 100644 docs/source/modules/installation/app_local_redis.yaml delete mode 100644 docs/source/modules/installation/docker-compose.yml delete mode 100644 docs/source/modules/installation/docker-compose_rabbit.yml delete mode 100644 docs/source/modules/installation/docker-compose_rabbit_redis_tls.yml delete mode 100644 docs/source/modules/installation/installation.rst delete mode 100644 docs/source/modules/introduction.rst delete mode 100644 docs/source/modules/port_your_application.rst delete mode 100644 docs/source/modules/run_simulation/run_simulation.rst delete mode 100644 docs/source/server/commands.rst delete mode 100644 docs/source/server/configuration.rst delete mode 100644 docs/source/spack.rst delete mode 100644 docs/source/virtualenv.rst create mode 100644 docs/tutorial/0_prerequisites.md create mode 100644 docs/tutorial/1_introduction.md create mode 100644 docs/tutorial/2_installation.md create mode 100644 docs/tutorial/3_hello_world.md create mode 100644 docs/tutorial/4_run_simulation.md create mode 100644 docs/tutorial/5_advanced_topics.md create mode 100644 docs/tutorial/6_contribute.md create mode 100644 docs/tutorial/7_port_application.md rename docs/{source/tutorial.rst => tutorial/index.md} (51%) create mode 100644 docs/user_guide/celery.md create mode 100644 docs/user_guide/command_line.md create mode 100644 docs/user_guide/configuration/external_server.md create mode 100644 docs/user_guide/configuration/index.md create mode 100644 docs/user_guide/configuration/merlin_server.md create mode 100644 docs/user_guide/contributing.md create mode 100644 docs/user_guide/docker.md create mode 100644 docs/user_guide/index.md create mode 100644 docs/user_guide/installation.md create mode 100644 docs/user_guide/interpreting_output.md create mode 100644 docs/user_guide/running_studies.md create mode 100644 docs/user_guide/specification.md create mode 100644 docs/user_guide/variables.md create mode 100644 mkdocs.yml diff --git a/.readthedocs.yaml b/.readthedocs.yaml index ee62bcb5e..c3cfbbe07 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -5,11 +5,11 @@ build: tools: python: "3.8" -sphinx: - configuration: docs/source/conf.py - python: install: - requirements: docs/requirements.txt +mkdocs: + fail_on_warning: false + formats: [pdf] \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 9138821e2..281d3b068 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The `merlin monitor` command will now keep an allocation up if the queues are empty and workers are still processing tasks - Add the restart keyword to the specification docs +### Changed +- The entire documentation has been ported to MkDocs and re-organized + - *Dark Mode* + - New "Getting Started" example for a simple setup tutorial + - More detail on configuration instructions + - There's now a full page on installation instructions + - More detail on explaining the spec file + - More detail with the CLI page + - New "Running Studies" page to explain different ways to run studies, restart them, and accomplish command line substitution + - New "Interpreting Output" page to help users understand how the output workspace is generated in more detail + - New "Examples" page has been added + - Updated "FAQ" page to include more links to helpful locations throughout the documentation + - Set up a place to store API docs + - New "Contact" page with info on reaching Merlin devs + ## [1.11.1] ### Fixed - Typo in `batch.py` that caused lsf launches to fail (`ALL_SGPUS` changed to `ALL_GPUS`) diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 662696c6f..000000000 --- a/docs/Makefile +++ /dev/null @@ -1,31 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SPHINXPROJ = Merlin -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -code-docs: - sphinx-apidoc -f -o source/ ../merlin/ - -view: code-docs html - firefox -new-instance build/html/index.html - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - pip install -r requirements.txt - echo $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -clean: - rm -rf build/ diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..cbddc54a4 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,44 @@ +# Guide to Merlin Documentation + +Merlin uses [MkDocs](https://www.mkdocs.org/) to generate its documentation and [Read the Docs](https://about.readthedocs.com/?ref=readthedocs.com) to host it. This README will detail important information on handling the documentation. + +## How to Build the Documentation + +Ensure you're at the root of the Merlin repository: + +```bash +cd /path/to/merlin/ +``` + +Install the documentation with: + +```bash +pip install -r docs/requirements.txt +``` + +Build the documentation with: + +```bash +mkdocs serve +``` + +Once up and running, MkDocs should provide a message telling you where your browser is connected (this is typically `http://127.0.0.1:8000/`). If you're using VSCode, you should be able to `ctrl+click` on the address to open the browser window. An example is shown below: + +```bash +(venv_name) [user@machine:merlin]$ mkdocs serve +INFO - Building documentation... +INFO - Cleaning site directory +WARNING - Excluding 'README.md' from the site because it conflicts with 'index.md'. +WARNING - A relative path to 'api_reference/' is included in the 'nav' configuration, which is not found in the documentation files. +INFO - Documentation built in 3.24 seconds +INFO - [09:16:00] Watching paths for changes: 'docs', 'mkdocs.yml' +INFO - [09:16:00] Serving on http://127.0.0.1:8000/ +``` + +## Configuring the Documentation + +MkDocs relies on an `mkdocs.yml` file for almost everything to do with configuration. See [their Configuration documentation](https://www.mkdocs.org/user-guide/configuration/) for more information. + +## How Do API Docs Work? + +Coming soon... diff --git a/docs/api_reference/index.md b/docs/api_reference/index.md new file mode 100644 index 000000000..6707457de --- /dev/null +++ b/docs/api_reference/index.md @@ -0,0 +1,5 @@ +# Merlin API Reference + +Coming soon! + + diff --git a/docs/assets/images/hello-samples-tree.png b/docs/assets/images/hello-samples-tree.png new file mode 100644 index 0000000000000000000000000000000000000000..8893874bfea5f52d7d04c0e123e6a9ec8cca352a GIT binary patch literal 124951 zcmeFZXH=6-7d3n#pa_VdNKuM_3JOwHIw(>s6p^ZQ5D^O<=}n~vu+Xc3f{Ic^>5xbl z5D=s&y@s051B8%wt|Z>i{jBwU|KE45hqaV?O=jktv(G;J%q0AVw#EVaqx2929k_Z$ z^(F){T0#)5Hr-zElTfaF9`I?u)0JDU5X5j8{znBRr*eQFQn}vLP=WH>xn{vPGcvbb1z9-efkQw5nhoAJOvdCTbI#HV&VTS2W57(fZ;uC>}M=#VS z@paK_F16ftq5AxLwwz8iu*Bz!x2Mk=H=Mwy!!AYL1R?c{qB$oYHn4lzem8&BbFJ){ zfb|2TQ#^IWwW`icMPlR4{qf@QOSJPATDw1p`hBwQ!xi`{U!|L~RclFd?jPp#ds4Uk z=5fgu$TL)6o_{|&`yPPB{O5yuzvX|vfuJwUsDFQZk1z1Q9#(;<|MlzF!4SMxe?KlV zP%-}f8hj39+XL^`KObjl#Q*skd~uu3^55V7?}h)j)&Fy@{|cwbzLTT~nQn;`L%ziL zukm){(s%mS>ZLkzF|J}KAxoEBZSz;dyYbRFCJt2>h8}Nk)4)&M@cvO?5l4D9k>Q}J z|0%+(SA_r}t>~ez-?{yEQwL5#DXzgd>)}-e> zAMJrZTTTxxkH$O`gGM>(m1I<&lYU1DZz&8$BjZDaGF9LN3t#g-QRP?gZv&@Jk)cC; z0)A}MZ`vd#huy;eBneMiuaj+U8KQ=ZB&}uCvwCx-pvx>qfi&^Jsgsn)>o5U3NiCuKcl4ce>u`14u5x05UrAhn5*xH@Juyh- zp|$qXbp>2b$H=R@-vTzXk1?4B?MyFXH)uq5r*$0Un7b0=X3_V27k+)G4({?$*v!5^ z>eNt@y-UEiXgD&XEHPk&5sa5e|2}wh#5D@uS$>dm9 zh)x1QaDJ~i*a(##`x6!B&*11IY<&c|SrP5hESE>HHQa&+F|Hux;HO5U`VBagWP_;J!R85$%sn=?lI2QBSvi|`gNykiNq`-GA zWX^x5vXaMZZln>ny;gp(IuHGIYUFad^4m+T^6~D~qkcyRvU0E{Owov&mb|MoF~dHe z?G*TGct?n#0rm^A%vK2l%1(anNK>7Bg_`>T!yyWeAp}M*vq;=x>F|ZKxht+so~`bi z!1D)W9OnxnXA!nTSU+W4rU&oLi5I0}Ug@y|Q8~62urnaHOUqYVtlsStk9I-A+YE^? zQ9SkJq2Qu`W#ww*8K(S5>oJ|(^}49w#`QkGjfKi%O)u0~9x-b0MLb3>zwh1`@t|0U zE&9oLoyfw0soz*ZABukmok`}bGmb|1+q zkFRP3xrtPM7E%~|EcceUq=EHvPxddfu;j&Bh%GQlh}kGe5ZrZlcpG-#3RUbn?&kI7 z`X~{;QLmm9OHi(!|M}iDT&WQ^mEy=zW?u3GZ`F7>ASRmGVu0}d#it2EeZ`?}J%8T$ zc~3txp{eIuVYbf`0u&W)5PdwcUj>g^NZs>lLdHy`jo=-3Cv$2BvQkQs^5Wu$gMv4C z*6guaf1o>OinX9}XKl4W z^l1tcLb@hns)~E<_k_Cyw#7Cb-YxHV$Kjg#HUISq?1LRHmX+<=(f}E!q(&JV@9}CI zMIP2YoCYEXxk;Qh(Ee|pJgr*vh=ykSNW8}KyoVui=_316k%;PmrMCx-8j7Aq3|_7} zWo}}Ak|3_?ymhcYrz9IwNzLP{O4m(=_emK{T0 zL>8o|I=g&4RW>-dkjqBUZoh_O$N{HM?Xhv;|Yi4w~qB{rrLM zMD%(G%eQEMZZbIUWkHcV()f_G+S&!(u#ti?rF3NwYK_SA?yO;b>N8`7s=5wH2Oa|A z5D$l~T=J6x5;|EaSGo;)ZH6cF1`>8xVn#-ct9XKIwHj*6N2|5E<5p69x2ADSj23s6 zldU4YO(=%$e_QV@>0oSkqx5;sx@4&HT-L>^5f_AQ|1Zu{!==OP6XH|M7a5m_iw{*b zTbqA5i#7HL`OVhgnOr0Av<63zOIc$$vBnTIiGAefOXyA0HQQh3H;P(a_Fq=SmMT6< z=5;UPDWxqNcU8r`u&PO^dng?;RD+Nrju)#k+jPA)B#g}Ylb8%*-hDs3;J~-cLkGq` zjKI0Xdn^HYO)%fz&w$YkS>efgMkDj#ttH;zp7pF@jy}7Z^%;M@2ds2xQTkh~)J>aA z+5P9~z2+Obr_MFl-J9EH&e#fS)zNvB>`=Zu8HC1oPrhG!OPnY&La)ZD%=z|gxtKS( z4Q_SnY(##tQR<0&mf<R2Xbcu$8Ee(I_Nn? zXzgoo)#7_<%xi6S^h6`?MA^hQUV{n5b2Yttd^u+$S|{AI!#GSMa@GFq2plM(iyl(tGzC@og!!Hew>1I4h0CjkhA_ zlHyA`I$7Bw(SK?Lyzs{gT31W#Q6)ZYgoKsmB+weD0A=%T^zL zD`Sx8^J)icEoYsqL1^!3@*z%ioFEy{nNpC7gbPIw2*}L=!k!W4OV4tj7pzH>%0y+3s0R%$27Z)!3M!ESF-tNu~FY9FM zqo1o5=DRZzY_=+Hh`lLbH^d#DH{Ig2ma`fDmGE7>Vai?0XsDnd!I^k?#J7w-KXzVI zQHEu;GG9qiLJZ@%Gfjwbp=O>7v2^6 z>#t)i+eXSpzNF60t9|q+LR?#SFP`<`#eza0pnui$S?G9i+>q5=Fi~sXai?;-;_yfk zhtBSBh)dGJbJV8UDs^*Et5wM3XINjo%(#3qGLlg5 zIcJpu_`Vn^v@E$OmBw{`dV_h|Kf3zI)JoT_H$RWcM;Ohkb?9#X(SLp{xPCqGqTWTT zo4Sdn>}oSw-K-gvyIXT*pS-UdHwbbqs0drZS=Vy=dM(8?(1N}7L_}e$3RXCcMO)u!g4eJbe+7wSC^CF zI!}8#tGSY=Hf&k5_<^dK??@J^%0e(uggt{--6P5Q{3CX2^B!srt%BB$19K}s*Q&AX zf5P46;~$krH{g^y3aSRL7ezO16z&W+dP^)qw?5{E_MAWIar|_HZxq&mP|1_=l{Mr1 zw259*$_0zfje`dtG8E8U=aFar7#?V$EW38dr8G4MtMXAWt<-$BjWt7HIM+TYB!^&x z>?2i4U;YKeD^?j|4BI+sRXJl=qx0rHHM8q==3o6-wKAjqgYmjFreaFz)92K_x#)#P z`NrXVPi_h!cutx!TOZ9nu59tG`aLzOFqu9YAAB#w%YlPs@LqcSvj`xLx{)Mf`F_i7-aTx>M3&{;~65h*?HsTEIHi z58-pe_;`hX?EDo@OtaeNI#Od_SXHV`bL*~n!?{VUHxqwDg>1V=^~i-!TRw_|7pyqS z^|Z~9TM7s|!}`wNBLW`9v!$s3BL7f#lNqq+Uk)UJIf9&5!|>7fq~xnBx5xkwcs3wr zc_=Ha!n}%mB2gU@@W{V+CceF4Ed$?Knb)-JU+#Jd$-n05$9hbIBXdDjGRA*n)nbh> z@P_J1PCjq;Uj3GJp^Ck#d>(#75w#p^h3v33NEr;J!^*Dh~=4tIZT zF`9F=yuDtbdg7a4myz=BE_FfV(q+yQk?!aB>@8~3IH0?p5l^ahbc3lI4;cK2C|SIx zNOxF$KPt{l!rJ6$t8c7dixzfgF};cG_ox#TvTYe?RbZ%eMXzY*OVly(q!6L#7~#Uw za4$}^-2mLwL8DT4n8aarv{4PsHxVV0bvDgHD`>B}Uoi`{A$ac6O>N!YWAt5Q)^M4M z536ec<}&s}g^}tvCnPTF$##MNjvrj;RVq7Xm6e^2WyzSu6PF&Y^NjecSP(yL68-FX z7?ZJDnW?%J*5^Lyl^^?Fm?d@X^z=HG*f3(Kw)`{cc>KicJzK_X0jq-22E@C!8&(L7 zhp|0g^XolwhccG)HPVlsSu1WHmuKONusZ)qYB?=xi=$PxV!M9v_4K1%*lLYA9nZYg zj^FLzAr=Skj%`AZj~bP~UQ=XZ2nRnSGxk`Q zF_wn|6Xp!Fi>|)M0bSBnMw;-*-8^bI(iLHFPrYa zQ>E|c=;)2sGiZwFI?-Bb&mGlih+L|z3d5@Eapas*P&^^^i{a1Z zXNKwr=IeW~tv9ZB<847mQyXYJz3G-5hC{5cgPgjybGCO26g@tc3dr5{pxhj`Y5PO- zu8dpqeFfClFcHfz>Dq29<>9_RxT-(On_PoiCPYu^AH=Heiv%;SQ|xHiUTN?taC;ZN z{Ohu*M7np7f47{h<_)3V_r`rqX42civ3Ct<1o;bZ?%lB7&W=x70na3@(-nd4(Bry{%Tbx!Cu*_( zN!3!T5tEy1m^FTrt2|G`R8~2naX!PYSM4}|{N-mhwY=51J6|8pHQqX9)Enf1YbzZ* z%MaL*^0aMt+>)!a-{6H{1ZHb(Umh;Z{nW5PyiNBJ!m5jsri5QPq}b7|doFnIcyawU zYe3|Un|k|_ZXFuI_ov5ZDCQ`aXceVe%bL!EISd9jjd{Bv;ZS4A`CRoC|<#TLa z`{DIDvj??>tEWq-CGt1@iL+)#DiT_md7dM1>D_|^gXC1t`q|wY#x%$so$r1PeHStl zvX)hKIu(z9NA+oYrH4RD*NJ;6`pxaQ;E}gNqYh`TbYwA?zCZTuT)k&@_9eoZ-KRN% zzMwQ$7oFr*sNu~?8QvODv{CpPA4AKdZKy^+fBB)f2DrMm;iDxz^|HO_f0I+y2H zqAiQ~R{#44d0Rid9uTJvt!55u*>XsX0NN7{>sm^hxfFtMoOj?4TW#<5=&~h#?P~uzl&T_rx}>O#3cUJ zy@+E2|MDq2OIrlgDwC{Y1UnV<%A2e_2lA`#(wR4BaIweFJLk<2(EMP*_aSt#~m?DC(vT74$3l zPGw8$*U7jzd%P0bjq*Cw&64gp9c+|4#eUi1M|Enc(JoQ5p2S# z+aCEJr3txjQ#YJ*@$U$ptMSx9X)YFV%6c;S?(_4d8QP@*qtXM}Gf%q2!la@tc6tO! zDs*A0c>-x+mW;Y56rzOVbo5xa+g13s-i$rDm0J2ruBGGLC!B2C^JZt0H|`H_kKpaw zUs?5~-THl3t6Ri3ht4W4tB%ybL7l6*Kxvzb1nLRuaLsfU-3z(^4?em{5kyg4-k^EW z4T|fy#-e*hpX?-iPbMDt`F%2aU%(gQrkyg5j0-s$WR*DZtvjex^{F*mDrk!N_ORQM ze{7ljSBND|9pbpMkKBW`wEjuR#1CEvSRXeN)Ufh(NU%F)#L0jJHpc1&TkCmb?OkeC zD+4J=1r<-Q2dsyan6yZ4-zoBNlKM>$M5hYDpwWaevEpfysmbi<}8yFfIq7KLlm0CHTO0w&!4te)MKwR4!Eak$H z&C@mGCf8#Fzk+4SKhHDHZlN=Dh-*4T{R1(|yPLZN1;&>a-K_1+K?vUztOlBKjAX6E z1OI>a2nqlKJ?ItO6s5Oed(IyTuP2$;CG~#>1FRC-jrs^`Wba5Cd#_B%qAnlO4 zz~9yzNbHs+T!xU%^TCq>wH!kZrMa$+Z^jUGHj~dooJhXT__xjXbYo+qtFcjiJ79Pf z?$j)8E@WTdVdS9;M#C>%^Um zt_QpGBiA6*vb_6>>h`w;rI*ucim5D3oD!Nx+l4Cc{*t$&vxDbZ*!*{li>#N-%bfTY z7Z%nOh!(>;$eh0F3FLJ!2EPCOtJ!p&0P(XCbk?fJvj-M2`=ng1*qF+uzHkhdf`>6a zL8f_7lh70Pfq?=02M^{^aqo-_4Z9rEw(e~IA>vnQATDIVh`-#>ZqR(Uqys{pP%npj1(iDAz(6wh;xg=`ym-6$AH#j-2uBX zr6-s|q*lc3Wd!y~t~|edG))@>Ajldbe@C&FmzTG9buB)lFB11HvWq$)LmLMF!;q72 z{Cj-EiI$7%r30T-IJcY?-Q*&?(TE+H3eu*abSY)O48M<$kG+kJz#;~-p}?#5YN7Y#A z^Yil!1CBLvAKZO+6bj5<1uYPHKcJ0Y=gRk%5%BntcrI0UuOx83vPBRGMWm(@g3@Ok zeXja*bb9*kpWnY{y_EKe_4f9vi|$m;zsUW-+z2=1KwG*42B+=qMcykws(du=sRn_) zsTjieiwh`A=MSD27@5ZZlqGw9;3hUPD=-!FhYC|&B%E03uow)6eSZu;eV~o}p@G=40%i`&!a91p`VyHC z$u}%9@@d*IzW#$CnwK!1KUu57qS>7pI!{CdW(#>bIu=x9L_Q5QFi%q~Ij;+GePe|7 zMm};R#bNft(D}Jc#AhE;Xsv2d=Cnswy!!L!n@8s2vN|#24UtdSQN}J*PE58l@l;M< zToMrdyN@e}(%U&}IbqZN(_ClNW##4;d+g!ilOD6D;?t*3ZkWMx8gZ@iC25QIeyI+e zx(g;Dl5VXKhZ$#m*rOE?r(Va>N;|v&mhEwahmxXL(2_=X&3AWat8uhpZtnhV6BE7T z`?%iw25*?sC}@`=B%f!R>_u(z2fnD}m_nJ#BJviJb+M^dJ_o=UPHq%W3Sn+=V;42O z0(@JsgTWjse)rx)T~Vethw%bP~YNX9C-N@#eA;z=2G%%LMaK*GEgXNV0c*D z#KfeB(ACm%?Y4zQLBva{=}W)Q3hd!5U-B@vbuMt=gu3HOzuS5`934Y^e(*2+L#UH? z^YypaqYb}{?ea#|2KTg8el}5mCh77MD+EI7;j!#@MluI1z30Xd3Yv%7>DF(k|5k*t zHt;EB4fh{Ee%u9I+2EJqp(-f;Dki?DwA5%QL+R{7qf26_wW;G~4~r{@FuMc} z_1_MCQ{w8JpAVCQDn5;Z*vN4HiKwWkIr^oyLNb+}%hna^y(WiIk|}pf5p=fo1fj{O zC+1L&m}99c4&fW{>xP+`V(N>1q->!e~j&S?;P1nsnSj@?a5`n7!@W(_q}U$}&eh6Bwk8a>&baR`t^Fjg7}Pc9C>u9}5bEA!-?# zt^4y^*XAeOTAPjZJ2Dx;WTMThUDf>+?sw+vAGDiXtR^K5@$2N>41D|IDLb&|z6ihU z=eVx7*9~5FQRwZUtXYkG$PvQ^^*15cP$!6%3CY9mPI< z{HU*H_ExB+!gD!mloplxy`iGWF9r|RQg^1(c2Z?X%C7wc6m7(GXA;A?wfk;hOO9-c zWXeKJ+I#C7k~w1G^{}9VlLUD3PW~TX-S|__zwEZtQ%-EL425WWMNA z3|}qhmOoFU!CuL^G^w2c7=n+gqnz^QP1+uPs-1N2lC{l<674H!25@^Ej$ z0MY)LR7*aX2qVQZe=#fZI>>{%n@7JzmeB?}PvzxkSYgVBBwaWJLZwW(xh<9IFDPTF zR|pD-L!Ehfp^_~a7`jjk6j^P$y}e!g&*0$4h>UAb*deYGU)wb>|H<(Iu9nWT)Rr}q zewP;ZRSY@kpb=N=ILotNk{qk;6vZzb78V!Zf&lwcSvj!k%PMGAJl+8%$6Z2$Id;fD8gvnVd z)W_4nBW|cG@C1V=BeQwho2{zB6m0tt-e|^4N?;X<6d~I3u9`$wFavv+A8x1*%!1_; z$f4gDXw{&$ltF-5ALsqv@8UL zZYIj=KC7s%Hgm6j7ul)iR((T+2F0HQAfwo>!w`3*Z;wbcOa(lD`Ki4J^Wr|YpbVz- zTA8vbcPB?1K&Q_@8u;PmA_r0K4FN34$XHlDVl{=sH## zO#LlVsN|NJnbv~`59l4CQRegRt?#f21F=CcY{g$+&~P~}uyN80`4RviCi#Wqs^2$w zH}8g%A>8Ri7PFnJs}zZJig|+iDHCjD5JI=W#hk5eE}p8e3dH5ggs!}J+%uSHCLW>0 zcxHr3%HrbUm#<&lUx&Hc+MfJWP++8%44fZ$7Kl*xuIMbrJE@937u@Bz@&17)-Hpc( z{RNYLT1@u!6a8o~7U~=YzCx{UfD|Am0OKEO=lhT(AmJbIuiRS0CW3@sM&%_^7gkv# zU&>2N;k`I)dG!+NIM$zs5MUsmpzLdogPon=BL@e=V9Ar;7$9R)*ddzeos}bOvahfO zPQ~LSiX3MjS1P7SojL#q2x&FYnDx(}+k2qq6@xa{%#R?hxI*YG z>y=JnU15ZvkO>P=n-=y{g0e{MTg$&fEA=BOv>F5!oSSC?k_|Wfxf$PVP9WxlSI32+wYE}Uc;H28( zz$qKc5~BgXeA5_AA$VKyUwxHBv~vaA=m0NcBpcs&4*?`z6f-wmWdVh!Nz4O)L1~%t zeTfGk^RXAE%etPTLg|M28DiF0$6#`a(QzkduQA|`qe7Hx?Sw(k9?-hdy=4h}A zEzlS)02lbVEP@d>rX7`;d~F4?)kww^VX(|aCYzCukP;8742~czHWbVSo=mnsGK18|agfvhZS-xRsAjCB6BtNUTHk zdav8^c*BQgfFBL4w0X8dm%%2oT^dGCIjPGc4k>;7hel{otZ+$n$=b$d;^NpP)^@cf zz(gR+t;xL51S3}7uORDc{vc7*#3NqH?D#T=(t&PS*rhRSC*hj@4tWyd=<&HmVPRo| z%u826<_f&zQgok2s&O$MRJ$<3^~PR8R3TH;WwI9%bsu-a#dBbTAXz|h1tbj61|V&q zw`9h6DJcsjlw4u^D@K}FY&tXA~3bIt7nDR0# zQT_SzCn%4qDSY{Qg6Mv85VK1wJt)`Z_wdt|pq7sfLJZ0Zz`TI~~6Y&P3l2KXRT+JL`XU?%FOy3B}3!pBfv$o*CH*%S`;oE>UT>227 z2lNNo@6!MPqlA`3#Njb?M}{M#IXP_!ex-t5|9&$h*^<@eJLMSs77}?`6|evdivP~l z#gPD$z*bFiSVdn-of#e+)P&2XUAD}~8Sk-Y=fPp({Dt@zk=pY?=nJ;TNr7R&WE0Ds zlf%;9Y?iyCbnYODprfiNASu8q2nyFyxCj=5iSaCu6?*pD0|2aW;jkf#6q-P!n389Y zB_jrZgx_6^WE>W46-%%)qo{>}UdPF1iJ<4%D?1Yc1Z}t8|ovOX9iw=JmQCY3yL6p!ME38XBAI@AtM@> z9j3o51hu|qv{pqq-v-sajr#9P+1;~rN_!wDO1}jtO+|)oRbj%j@#B+D3IVX*0kARD zRN=p~kL@9RSARewGr&LPgA->+s$ze~9!UHV#Z_zpiaOVNxVxVR_$9jq(h>uIG~Sg3 zb`$X$E+>+4dbk3)jD|bn@g^{Go}##l-XXz||BALkQx__G5@x)p<0v=Q&jhOw0d%5s z;tMEUo`(2|LeNt(mq0xmzk>x;HE>cnr?&%fad80i)C3;^C;mRg|C3Y#b&}XJL{bp1 z$_T_}BC3c#h5*GhHm6KJ=MpHpZa<6n78+#N|3**83td#!t+VX(1GN`BlLmZlYK?KASBp9uOMc84{uC%mc-W z8Hrs6A-G}@1KPRSK1U!_Dw$1y4q#9>5SHE7f8R{yh>B)sX#tawxexQypp`OSqZvaQ zdgzDsND5i97)gO8i)-;g-^(14jF7kqB^w0F*n9sYVI8S*+QSM+`FVJhZwR`3s#HRd zxG|YlT>ShWTA@J!o}=|we}aS#v-VNP@`>PA;QTvE;k(gvJ1L2b-s3hTtGT#7QlyES zk&SU9>~K0@BL+)Z{_g!Ij@9QuDOx1w1*k?Loy$+VTXb^wOA)3ENtfa31vTSxLvrK0 z7cn*Cm@U%O2%GwXcfg1mLQpiYT`i2Ku5u=)n(EJWeg}}G;xCDABltU=D{aYji>0qE zEOKwSwZn-><~Df_BGq_IrI=ZnQ>I+dEg)OLR4B)Rv$C=>8RUz9!*MQ|64HTaYk)+7$4}eVa=;1>yA&!W{;Up;;?jAh_v~-+kl+ETlE$lcS zPX%=!!B6ySpb2g@8KKT`W>TO8T0l|-s*w{Ss%w_LD(uzMyu3X7M~{AltAJd&A4UeE zdAUs^tiZXQLI^uF*+7?&l7q*%zT#YdJG{T7)V=UTvdXwP5Rg4V3XI(+aouaxeA)cUbst zg%HC00>LqTnaQwGsfuI?4qIh&P|~n8)&=rdh0~ad&62gBUv&j7K;D|k zC++0ep4I@?z^~NN*-c&oVJX8%v14p?b@i8$lE<$hLG3cNCI-NYLCn2mkj{QCwNFz0 z>||STGGM6p4nxs~pl5<#tFp*Ak*a&}?j$}O>7e5fTq0yM^mn?ZPzWfEo$UqKb;! zv|_0uAS)gR7IhiBOCY{_)Iw6>oOZ~S@RI~MC6ikwWSkLC7^nFIHwFOQEtL)~*M`H# z=|cca{xp-oplg3v#Tm7?t0mjDr(Olq`S34&0!Ji;StMB_4=g_<*jhr$XVn~cZEfGu zThxaGScMW*LGGxygV6hU>v5}$&&J2RyVNK~3dwlRStoRzb6GWbjVPbC<|vV zetSyxdJj?{%Cto-aVz3Ctj$^>OatF<9s*7}jPn46q)i}&WN@7&JLg^XR3{+5Y5o-u z0ai7;j7^;wlFaJ=Oj3xU)st9!>qHR1i}A$cyaJPW2{*Yk8VI3HVkv<(?1C?VIl5uF zY5XN6jltdu#cRU`QEen}gcz9bmuEMxIC=qU%kX7{zwzJ#on~v`62UVF5yZ1s}NWj1*Ks$A7o5J z_AIV&Qi-om%llSl?9ks54RYEf7e%_JoN93pYP`9L0km2L6D95^7RSc$wb<+j{9-za z>|vp2wA)hu9dpJYa04ge3cS-uIRqk1imPK%ojG6zU$;yH6_XnuftOhXO)h{cptskM zTm?Ylye@pUw@hW>45UEGK-REFi<5Z;$I66u?yG-FLU99UYg=+X#vo}(7KPn!0_TDf zb;>WRvl>PZoXsZna-h~svPLlx#-YC7vzH{it7Jf2ltWHAr#!0OWeA05MUd-jsL&Gt z4n?`ox(dfg`J_!anO@)xN%W$L1458*jRuB;>RNv+Qm9Y}Ch*{Xr!e4|$8CP`x0s5* z2lcT?!M{a7XHo~G=_s^DzEX(}#w$H72>FD9+O zZw~Qcbh=>Go~1dA|0& z%RbjVXs_BN`cGZ>Y5ZB;U!pkmo!o*k1uqUoYckoWetQFWD(Xvvv$|4mGj3J9MM48m z_7}YJOjL=yz+2WBnEh zMD@l|H9)aVz$1aT;v@N1d-9q<)}?(;uE<|!NQS+*oC(DA$%z-fA}>RMKp{f!R8Jma z22>phz;`}mJ-fN{oR?b+=!IY$FKK%q1iR0z(D#7E1KJFbK(y?N#FhWh2(A=JlTs4L zf$m6CB#=|m(GyKuGoD2yCGqeA*bD%)gi6jKUwr`p508`G+{6pISjt?BrOO`?!N>|hU2@+PDK+jeIbdB?@o5&d_@d%hVSHYxBJYYe##|1pv z!vIGYFr{xhK;8gaHY3Bq{{DB`GYR8O3;#q^;P2=1cP8_0#B>9@NVv$rb*W|Y-`*n8 z^CT?%iJrZj<2+KtDE@%78Ka5d9=PxS8IYjH8dqfPgK!?F1b?qN$C z3AdD$$)RFPvpGt6arl5`SK2|vBtWfqn7QGW!M?3DI^$FRf=?Qz689RPinfpM3n*It z0OKCldwHH6-R%(~mMBIg5((c>_g&10u!nmDIKI)`An@ zPvJF7gEfykfi+7toogl62hlf7sosGzU;8a@sR>Vm+p6$8e~8E<4^y?aV^s?|xVoX% zc8MP=EGRHWpnw~3;!n&Tg`IA@!(ghPSLIm(!G^Ga4RO*een{RBI>G`Pc?D|4fV&(z z;3;?@2&_kKG;uFV;>YK|Lv78nA3}ZZjs_uw0)O!ht13jF92z_X-}|Gf)H z6=++5T}lOKa$n7o?o}_t=X=CzcEH;+fHI(bYlsfI+XoIjnp6;|`*NFzTfz5v3B?)6 z*t=%w(^$o=hu+|}JI~g+CTA_px|ETnZKD`_e*^A5Q~&KP0mT|;U)pMt^+2<0nR@a4 ztG+b_c4ZC=|LtHtB4NH;F=N-V!xeigp!Z-kY!lZL?1vk%s%fP)Mc(Blf;W}$ z$-^l+uN&;!^EK&E=YB{q`sfy2^|f0+c<|Zn&olMT`cz@t%LZI2vNhHe<&7(-Oio+T4IExpE;(lu z>V-EMHYO~Y6JI>$;+1!Q)G;BL?xh43k+ikOVtVk9$2`hrf=TqS5 z==NOc8~AYdWp>%#0BK+f;!7}j=Aq6|FNa<=jj%elp-bDrVk&No@@MwN6r^hvE|9Ge zeKA!eKlI=@y??7ymSX3mjwMBq)Ty<_8xyr`0l67eZ33sL=w}@-`8a_F)(PPop z$&TQl6Lm)ezL6O{5WT{Y%R}RA&P4GFRrs2a``23gaNe?c!CABKaQ?U4Xi$@zL(Z$x zO~K%+xZ5G7k8^~tS6RiCwYa$%R)h1`)J-F6l|OaOxsO5|A{(tm$}xVA7jS4RWk1ty z-?_=~^roMu-z*x>fZNRZI+3d`WH|`McVF&3j}iKPsW6Wr;Y^4`zHQ}g7{^cwANC2W zupWvJMu4{|ksI3Td^NkzV%w=>eq1_p=C#C($8qE_D%8R}NC4;CUfoKZZq-3PkA6BK ze2?5g6n`f34PT@un3q?T3f@2f-_dk1zkCY&Ent;=Nxve){*)$GcER^twLB2gw>d^Ou6Sf#nh3*zORmXtrLp!qqWZgg*TJn0 zrKQRnI|T8Fxp98(bq4o(#~WHkFQU=l>S=mklf=8t!re+;1T{F3;N}eDOt@G2FWW z8@tNQpDSwEs}ocURbx42=MEdS>m6rrb)!7zWC+ZdT&x&aPpu?;ojXaV?Or^ zJ)G<%zYkrCxpS9({8;Vx6TT4<0KHY;rE3~vWPjzO$EhwNU=Ysu`ny$|J4$MX<)9=D z7rlb4`S1hGlcBu6-Jg{n*x~Apd4sQc0gL87i`71N{rWuohDUU26W5n7ejU1Y-v2}D zQFetIYvhMVlUtK^$Gkq>W5t{DHn|U=!THRWelxzQbGI#CJrtPexSz^SX#6zvtI4P$ z^4n>0!ce^PWcx60+2-5hYSKw%nq{wD$6W@Jato5(7Gw8swvRYg1g7GBcLJT%Bkp6j$rgN_@7E9^CpgKKeBGR7y&%(9?>w zC(@rbZ@9;D2KS6O-OPnn#kE|yj!B5c`YAfE*qNb&d3;>Htg6IS#jo>! zGq;M^`N)H6wic}Zg^s%L;O#x=O6TWQnUo3N=-;xUeK9{yFBQa`RPt+(Ki0(Cvz{{O z&3T9MMYNvu@T!xIXO=zVi=s;D?;c(dub3+#+nPb+UR#iNFC%eu{h_P0u_X(kc(l&WPx_idfTNZ3bW~5vUktbJV{71gu|;-=y2Q*VXECP4}7L|ES95> zu>16__79zEPBAeXmrd8UzUUgaw7f8)<$>X~F1=#dqUO`?eP&7V{kZ-(oI3 zG~BfA9Mx#tLolCk{;qUzMJf7zMsd{F8s+N7lp{k8ujR>a^-xjf+ZTPr0=hOH;C5R$E#reP<|D$(r;oxGrwY%oBwn&F~i0p`uCQ3_j(E! zQa%PR%hnr%UZ20M)ffMP8Fd`|@0XWvU~B3;KFQFaKIc1lUt!5n^dXbz3f!Gf2}>Tc#&3MO}ZI^@r5#ZFs$ zp#4_8M7MLboY1t>mjn~$X{omq@M+;V?ca~hLmgffOuUTTzs@k5Rsgyo5b7Pm-~8Ts zRm<(^;9lOu!_^JvF1xJEH^tct%)55$mz2BW&xeYE&p3+)Xm$QV#?OhP< z%at!}mPdSd{PI*d*70v=s{Pwyyp4-aYs$_&xGu|slRM=YOQYdfwi=XB(_S-eO#Gc^ z^cedNsh;wF_qJ_DN`9TY=y3dv>jn8MS~g7z=N`)q*uOkC za6i4uPp|0r+|Xs0F|$09qX?Y)!C>oP-8CINNBdlh*3&&rSi&Ru8SDdUwY-E1r(coO zmBA*ErPN8)&eIGY+`dlhWtp=wLj{NRe>I&xy@`FS@X^un`Gf1fhwnZU8=v+u+DG$& ziO-$Ts4~&TdhfeuUQNgTOKU!hK34;?@})j@y`LpKI69)zPD;7oh`^%V`W< z>v5Kt&1RpL{qtXpcK`h3P2eqdX?th5r>$XUTlE1?sr`E0y8;Ajx%hD20ViyRV$J=y zH>>Tk#r+T0_gm&IfZc$O+2*w|Oxd}!XH-_(ZCc($2Lq^C{wd~P?3h?P*9JYL)oPg6 z8tP8bPb;agpWAspukoO~Bz9D#U=O~4dnk3{K_k^r;~vgXt($$3Ew^!A14(v_is=u% zcZV8Pw6KnTR9_{)c|EWdKdCC20Cg@3;>ULnY z2`jDI72#$oNzB1j8#$d1%GjwYzJ8@AD4PdjaS9&R7KRskFZO%Au;Z5ZNL(=g;TIYB zCCp0gj`wX^$CAT>`%E6$r+z-9=SQ3VQ5;sN3U&X;iJhCdHV_x(uar^kjr*{qMh_jX zx$^F9MvhlDz6PJY2g>;_6y0aC^{4mCpU2bphKt=sXMQQv96|+Z1J?#0YfiY!O|1hU z^W4gR*ne)e|1|qx0dgVBC9Yog`{m)S>fSZH6`od9W_=gq#sY*>-r+Ms=g=% zX@1DDv1k;3FniBxEbkM#{`=P$ujEFb^#=|K3;i|Y5BjDo&y!^YsyhwU_j{^83b`(q z547%Yo)@ch>It6rb?k@KyFWyfk1DiW)S#!WaSOm@^n7x+t=Bd2;~P2?0;Y!A{Ag=F zGv|?}>;~Bsqk~1OM-SxHl(MXuf zyLIz@vCmbu+pT@zsyV`BK21BhF@v{ia`1>nrRcsqa#kw5cIvp))g%+q$!SBaYj@R> zKIV9IXKF-P;3qBelj{}mJ$8&kSNRW*uP^q9srTWe@bQu8o;UZEZ`~@K^-uEraX;EU zWyrA1pO1hEIk0G;@f?)An6^iH4!Cm@sXR5yM;wy(PigtDE5Rt^B>M zqv&YBIJINI?7XJi^n6ZS1pKG14Cu7s5~o5t;ZfE-zT?#F1;$YluNFbBSIont#5t<_ zW%`{>K1Gz4XNFt z;1XV5YgMNZ+6wrjUM5<6EONU_clh9{I;5HBHh;|{|60L=>lxMCW(`x7hL2@uq(dwv zhumgnJ}^JpWm>^?{vP6=C~TLwJ2m=LxbBEn-;^82(;jlYa`c?msk_#ahhxi`)33Hh zb_^Nj^Xnm-hut!9MS?o{+va-?_V^XxYYAeXv}3<_Z9mRys=;SYOc7SRL<{?jGQV%^ zO*-B&`QSEP0D)IW@&EAl=J8Or?f>`?6=|~zDV2R0in2r$QI;f2wy{%|!q~SdX&EBP z5(XhVW8ara_aJ1=zD)L=?7R7%*XVwpr(T~wf3M&Cao?}ebzawX9_O*VkN0t$=j=Z* zsz~f{nV9K3Nq4wcCBo=eGFa-@C%|-V3zEZLlz%@aCdj6NcqR$zmogi}zI!9yEyA)K z8x{A~%J(Mx@0VP?15nJ;+u%wp0ULq!Hp;l;J!E~E=`RVis2|*}JJD>4Q%U#fBOq#eG>Z%UACce7tRs9YC)u4KGI0 zjtwz^&BR>TS^i@#4HwNPsh@|b70gO5wPIt9dE1?o|6z52+6Hkp8|qAg-Vn(L8K>LdKz+7x3I zb2z{TZV$=n0X+Sco~^xCdy3Ve$*My1@NR9LkD0<|@|js@&&8FhltT{Nix07w$V>jYF zAk>@d@tGZrM7;Bqd~B`b2N?w{`C+Y)4VC6tr0=*OqI=IYpmAqo zL48kQ%jZNw_fGa#W*SA()<)D>$K0Lax;Vbv;+ytp&*2?uu{xATg+(j~r1=$Q%WO4~wDHs;94ja-oJxg^t)H9wPM-cw2aGIzZR70#UAe-z*~ zdY#}iJ)08eX~L&x3hAp#ltat0HM60kuqmtSW)Xpc$*SI747EC~zcAL>6w7x@&@jte_WzKB!nE4*I`!E{K!=q%+u@ z_1R4$K`)of!hoxVdmQAQg5k&JOQCgy8-l$qU^#~#LE-*CpX&#F*1zBXZ4mqYz4dn; z`1jWvzl$yZUhREgqDP?|EDAefC3})ld6K?~h|P^!UOG?y~XaZZl|)g(YL$Ibkce zS`XJ=fz@8=I4C&Qf^a-9(Co7|4Q}SqIE&ESM}S6E_k8IwN6En? z6c*Uli-9C+o*1nZcg<8bo?!bct4_eAX#Tmev78PXELEkO=++hNawi4oSpMc1kg z1Iy_4(Vf#+?<4nf7m;ML!%$ZdZ-6Qq_n-M!%I-GK2ru_!h(i@Gr3FkpP0U_JCVUalj&REQRt0D`sz|BcR1-FMDL! za1yE{BdubOT1&0mg0lxcLKR(3KGLiCycz8J4bFs4cn#0JC ze^z&Qy_P<7H?eL5%^})a za03`uu&Dwp{`h}~s)ln;9O3ZFF%YTJKfU5f*cmc)aVhF4euBug>n|cVelNW}m52u? zoi-A`F1s7nm{FOI5#fd+;9ka#JlYhnK3wEs=7Z_=T!f~l6I?xC)PJVj@S=6d^YKd0 zjc_s=RMt6S%GwM%1paW%7b1moO6mE@VRptUhXO+nv_#`MBd?0uuLPaqfYQ>^Y4K>-4OZq3*yu%4zmOSRsW{+;(8MIbg{ZFSZiWkTfk-^` z@{pYe{yo>@@8j{y%Z?A-1`4)4pEF_xgSGI{Zn$x_rdI$n`h=L%M8ue){zyh z^W^cxyS{pR)qS@}Z^$h8*W7K8o4(ZTVNJ?th{u+Krsnyioy_Cj?O~Ns0Nrv~Zk*+R}EIIrPm1z>cqWlw8OH0d! zbG6|0{fEFC8^8T)y;k4ZGX(?$SdcvpeIW=GG_ju&cX$@80P#pmbMp)uS^%5)Kjjmj zJmZpEQ1HOVD1wtCYQ16;oaFruO#PG)7HL zP4qAot^)<0|7%j3+;Rc*(_C(C8U6U08u_i$sH_Xbx=G^3`1trpeaMBk8*L0wBvaGK zJPIp#2Hf01B~RrovRqeBkMAav>Yz~juqj|r@xZQox5!ahf@FadBDD3^+r9|WLMCJW ze9QRG`%j}s`YD8?&`;iMRqjHX=3TGh040+F&aU0X^AqeA`~<$T||zjr=CxrQ-6 zKObM^y@M==MREqqY+L9068D!k-?hxPQvq{e;~?p3{NGc}3Tai)z6M!pnM-+ObTt20 zrZPj=yHN#fHZ#?Xmf?G78U9{$eLbM2v8p5^_4oHTwX{?$J_gff0UA@oOtmx(t@@4V zzq~?G60*F$VVufnb;d|bbjn;71ZHLwXm31eV69xq)+%;0*hA*|xn>sk;PCgz35*dFy2Brgl zOft7t?LVy>_2xsX0<|GfC8gZi0|PY{Xs1e~Hk%^Nfc38#a4CXmKfVqWsYR=Ha$=$( z7~DJOp<`xx(8B{)V9bB-PmKUO%*&O?Y_u5`yXHAR1#E4(e>w)^x+7_F@k)s=(TAjd zJ&u2=Uo@v&0MIm1H;1wM5G`5tRv_SH0bSK$KA1`wg-AQYg=aNf64{qjN?aF>o-nE*VVrp&1cCc@?{n~TbGErp2r~FIn62_&nd#VOxS3%MbGger9~f$!|JN+V4I~W) zoVyM2?d|t_w-SW>Q@ahER7*Ka?3rK%-{QeyfiM)~yT1kblfclw9()t>_8lN{0p-R0 zfE)p>Vw0c=z0j0P-Z)WRqQc<7cahdOC*&_--n?5JklBh!XiK(dx_tRE9;`E1hON96 zN?5BvT|2NzOIyiglB2>Y{U=`XLDZ(EMcpIr2bDf2sUyoXZzra2Y=& zUg^{Y_Ot^Z!13og+~6IA46NaKGY}E)h>FPMfzN{V7yxa#$n(%2it_Pq{#=GgQz|_H zVd01`0m-cj&mr$)F$~7T;2%lWi9N7RwLdM+xZ$Gp(oesWj38uO*J;Xr4mlfK*C&zy zY(D)NCX)Xd{`v|6K_C1-4we}vopS>I6uO=T+e@J+G#_2T)SL?$Oa)7_$kCTvV0A3+LGM@-1%?VH|2dnY5)c~zh&#p`ebH6~ zfQh;OVs)}hupD{tC>AMmwh(x2wqZFn7}}@sr%SVygAgG&rXt()(WA6EFO3M(Z=|qU zs0U_ee9F%S-;-~p?y$c#34u;q@4ti|pw!B#5OU-M+f?s7Cq^jmej!*V92}V0Yl`bm zcbQ3U^zNU}R4MuT>SV;HfK0`~tbRO@WGQE1DJuxyi1Ovi14WPay<>K5o2jl+2WRd5lT|%^yG=KJ7Gzp*=n*oBg4X1#o z$6IT<3X#^W=)|8|oAWK*p6Tx0yKfZ&sr1anrUAf&MyUh_>eJSg?5+2gzr5`X!Lvx) zzr;b8WV{4{2>bH+qS!$pODn4Y@16B&XmG)84)1~<0GcGSzfC%m9Eu9ppU;IWlE5t* zKvaDaL=|-txqMj@h+T|jaurO{L9KHT%HF?!{!;c)G{s&(m2(-A(zlNQ@^BdA#@fQNv z)4_yfWv~v*ciGHZRtK0}Joe9Z(@mielq@&AqI`ZBM2ph&e4rt)gZY#HHUFd+LMPa+ z-D5oIa)37(PeO8U6~MsL;V(&xJNnnmyUqbPbp*gEo5BhDqs>iCAU42q4zrkPnPqX2 z9ugS-=j$-v0QlEpg8tC@zgPj5qc;7*q1(`v0jxP`jP1EUUkvg>jg5^K!-!%K)rsI- zaKLUT3LDk|8JqNQ0UQNXgt`W#l9_*js8r?R06ZR_*Mn6?@BUT?Ua-fVITTq_ziA7+ z^;<&`QK3d=LH2Jh^LT1d z!CElP`cJ2TN(%$tY6FO~HJ@Z;BzJ}S){f;@ym%{GzXtL_nMr@>5h^Xzo{7rP96ttv zCT&@L`Zsy-IQ^Tzi%h@BEh!-dlN*19S`~i^L6A{4{=|n|xUIb>Ef)Y@&D6x?6Xd~g zT@OIO?ZG}8U?K@NCgINqYC{ne8N*q@MWx@=+{}0Yx?%Aaw769H5>-qKDXRR9XZ5)K zU0es9bpX)+p=%uhqNnswxhocujpjMCnip`=C|%`0-A>lKgtGo8)s+I70N2A%_Wuc& zks@*3BwVRN_vfHD z>Cb2BH^=|SJ+(HTGZdlyM~PeP6R0uAtN!Wtb1ahj@gpOq5FV=~!HDshe}3B{Og96- zIND!Xq{Y(z3)OQ0qzsf2!ct~a0M^mtF1dZ`2goT=6y&t&vr&-2;I#iV7}T54cm*gZ zwb_gh_xAQWj++4N0P9gAEsx#b7l*XspB*~VKwkVCMg7D#HK_o{F#6#?VKV;sUlU3L zj)6@7Ia8%Jz1nGgu)z*@v3|4gOr1C>vFV}y>vCQICqd|;F}woOF7y{(?dZ5TVJQT7 z^@}9e)zk3j0J!+SVcAbUm#EyyUZnW)=<0A@fcV@BB9*|_S0PkmlDyVw(o2xbg`WT+CHbo_XbY`$+n!Ddn2hzzb_3xzAZ~BYiV_w&yV{{HY_yiH!(fGR z)?O;>uFN*hBT;L!;v}jfFAoc%DXb$KcnM?;zy&`*GGj~*Jb8~EJ%7#P7MUH}CB%NC zFK?X~%=S79A;zg$qE?)abcz-8odZwTrrSYXMWu&yDF@oE4K0Lm9v~EY`sn%PHKdpR zp?iC5iY#Z%+#r-SnzYMJHx#5`k1KfTZT_1TK(jNx0RfCd&C>$iNDTvkt58e#Gmo~O zMKgTVR+rf_cRv(={r;o zFVkg(=9vQK`OA-EH{NZ%*XkIokKeGeeK6qQR+hP`m}ixb>HUy5*>X30&iGISYSMyK8^;KT z{7E7P@+X6kh``NM{1R^mGr|f8{!B)?IT8>zWlVk18j7M}{9_=rYGegvK+eck6==8l zKpd_R8lC}$M*aj zoj)QNa)5xQ64Lq=Bn{>U%Znp_dmdQFB@DcsUD$*Mf`{6gnMdL5^!)8EfT8aA5^1>t zSsRWqoW%68R9I6(G-Q}99mJ@?V5t{NFy|D>SyX{`Nkp9bc^H=LZ>oTrj@JzVG%r~s z;G?5SE!Rbnzj0CSkAJi1`+(cfnSK$nf2RMkQlZb(f_CLj02RWMsUv# zjZ+tbOs47k_po7-dZP*fkTV0>J&h?uITVEbpD)1$AxG4me>1u~<~ygs*47*&uxS5~ zI9(_s2esV)dkS3l0G8_Pr4aUjD6@PBE#MRZM@pjFAQyX1QRWDYDt`@pa4FEU$|bJiE2zgg?1I4QMP&taqm{JyYdy(@5o$>pS@VqLJ@dsl*=Ze+9;M z>puanyA=vzZXp2u!$uQ;?ZS^M8fLn!R?Fz@&MyJuD7))0B3!i|IO4lK?u152Y-F0U zK}(p(+`+ajt4<%_yQV9VO=diw-M<$wZg+vD~z(X8`LQg45Zn2Z-QYawn zfAc^y(3Nb)5sMcYd_8wRlB3porx+3$z)WUYIxZ?jvBDBVe&HizSFz~ssu?D<#^vN* z(vbylE?%{C?KktCBvNuW6|%(~t&6CTbBfK*5cJWK?FL@Ghq_BM(``I%@1_GJJQ(~8 zD@zSzE61ghw}O3%=DC|X!3?m#E>D7o%My|Jvsys$`@be|7@)ToLQt-g4dF##K1^jm zR7DjrI~To%1l?kTn4#RgP8Nnz7PZYCiSwFwTe7+x%L8N1()ZbUKBD7VQN%0;DMpT2 z6rp0RAAd5={S(Z1Rov9T?6-3;J_M|*6G2x|e<#P8N)IAefv2>9!U3vCnL7|z%ElaE z{c9nD68K848aN&;yP3j-J~^~LPmXHyT$yd=S*V(r$EWgYC(XZP>6r2%%gOZ_AIL|y zAVm5^UWb%F|@5?EH)J$zR2 zkX|ft%xrI$`%_@d)%k(a{LL)Fb{2_11_Fp`86n+y{lsw8-p!Hh@Q z_OLAs1tf2qsVUF+uu}}^0lkHDx7svBTp|t&IfA{({MBGbFkG;Gnsf3dmD$*eM_mB& zWDEu)_?5Za$kWcO4!A;Pm$hC#G5a-5S^ClgVh-ezCwjw zbvNpCR}1^8P!*@&oJQMKF9CMY zU-THOYY9`h0<&dfhZ{?a0%I|g@UE}r~@blktC2p z(pD)Qg?NYIOI{(T0BbBkAOT4*B9e4oYwLN8;QAr~VN0m0qBD?+uxFR{z zN$FoV0Fyxg|&MZN^GnEhug)i{IS-M$($7>3af#(83UaxaJ;XPbY+jOw_ zNgNIs8IF0}OQ0C}PrZb`{0asmyj(dz8{XosFSF2Bm;r8}xHzX1chE<Yd-q6BqeIk~OD{bQ$!3y>%NVN=LQIA`l~9PRFAVbEIthc#e&DPD ztx-9fKQBq4SDM|wqvHlQ2D#C~J}bIQpLc>8qO*+LJ|;BMBgkR%Ivj})p0znNmZUi$ ziZJdTWCf&htL7z23Ep;PQq0KAjDmcyN-S7%s4{g}3`jp!CT_7eKLsp7;hnlycU^1| zOO>yJWVNLC23pXMo`T{lLtNDeyP>ZSRMdPDw~6ZyIO)A##e|5a!&{~?8BJ6I`7;-G z0LChkeA;D6zj{p{0xFGK_hyF!GLxceFe$Mf>DQ1`?|wldXT0Z(zveQh>n;@0A_a8D zF(_i?3&paO-x+|+)oYj#n4L9-Vvlo)S+wa>3{_Jfetk{rRy+a#TKpiSL)KaJedY{5 z=KG5y@Z!@RA&Xjhw2#qF_me_RF(7+i_5@S3y)Lm>4o=!^Tq*Zbxt6H;oJrz{%Bj&l z8?2z;Hi+B#pk+_Y0+zT?wJ4)8;I()G4|RTc5xHdzm-6!w-ik}(gvtVNBkOz}PylY_ zq|%k%{OM;AnJ@%2C?V!|uptLA6eg$%vSz_a|7j16lVd6TiuR>XnmIP+dh_{mfbMl< z@_K=O#vz2)Qgofsw&b$_kM00uO$q2q50^U*YNq3nd(ljbJT7}Dg*oatJeO0(MJoCo zr0e#fra1ftZ9~%{RF6jN_9~3KjD8_{C6s`kCw}ti(v~B_y&AjG#|XMU-cZNx%{uiD z{Zri4o}CNDRRz^sgXJ6^USDh3RI%!X)$UCmr<_l@FGMU#ZhXd8yR5!h7W3FHjbp%d zL1!)8;5Q#{r~i?JO3qJBzVBS#Q||hN1qWsqzvs9g(QhTQ@^LxF0Tc+YIgZuemBoT2 zh@npjFovjgWuBiSPQ^@<7$?iz`KHSa@?X;HbJ9>k8>%_+e4KNuQ0suyty@^~iu8JT zf4yMv2IxR(AWUmnyLK*M_^Z@D5xxe?uC=e!3-&jQ-!F)W3+}rlRfOG?^bf%d2e@sQ zg6IGgZbWKF7`wMn#XPU{9CLTycbfyo;=!uRBh2eqBRbgF27ZJ zi4?If(BV=c6xXW88@sd0uxQZs`IC^pU$^)0K7)eqLrh89v0F(!*!P4CY~KL(>Eb`% z;ux+Hc9gmkDtGNu282ysCEHnY(A8{ymGw1F4U6tu(%Q(d-`Jb$EHB>|2;N}5`wA^3vmBcp9KAT{ zh8cAtaip<4iPuw;6$uY+*|FlTsyPZ$4zXHq$oNCC1OP}|I3HNaMV!I z1d63Tu`3M}J5o3I*0M>6;i>|-IM#>KGmL(c_`Qyx!?>5_x;Bh#Ryu9y?LVQ`Iny?s z6jC?Wc|7rb0^UOheWy%e%;7c*m)l0lzUfc!BU03_c%K{Gm9wV4vf}VgE9D5UTay#} z*mY3T1A7caKK+9{dY36(4e~uXs}T^{wbH!Gdf zEUfkm+&COE{{wVpYC3Z^MrO=BXVj^-vD#WO`qGLlo!S^suT=a_MsF+%HGJY2-^D}C z3;`G^*YH6Bki}}h5C)WpM$@O8uh5X&GRS(c3Bn}Gh&HF*eGFiWhL%}&VcOREQp;uq zC$+v_5nuz~y0VP1eD5<{kxIk*k`|d8B6J^8UkTqo7ioOE(oN(U+M_sti+5m_u`2b! zmlqC=Rh@%-t)qCoy`{y9{PXoo`OB4$`;3PAjWD7kplgkdr?suO4YX3EPnSd*e+iPU z$am|n`7$!#>pGeK)U5XQUQlRhi|VVhyv0kPbI!u-hUjSIi(7Q?)P-*PGpg4LH9K^L zc4B^+uV3AXtz#|P(H8rHz0nu>w3Au%e4x>CRCk=GrtyWrz8NxR zW5;RW3UId|q{u6&b7B4iToG*oYO>r)r*b2k=Kh0EC^~M$iPsZuk!JQ*IMKKb zNwo=PPq24xmw?zS*p)PS9_S!BUb4UJu=B&r*NQmKzYr?KhM5_DIkys4?6vnfS8=_P z+IY;Z%4iMt&zsPUa*y#+Bd>+Dz7K&dho@!2C%2b3r&7jkLPS(2BQLg`$L@KQ=aP*= zbOY`m3CA%dD6`I*fg*kapUdwCl1r1(rI(N()Ki%vU8*vysZo+2IlMnu% z!YYEcSW{hj$WTS~4%oA_*EfGIF$@PZFjbf4N!{%5mK1Y#Dp@R-HDbV}2CF-IHD~D; zrzflF^eSq$u7F<>Wl@G*FniW4V&1uT!7S%7kq(~r9OE}t)Z&hrZE<*jkw(3RP65K* z0@Wt|z~tATNjy&d<3v(wmJPUG^&cZ(3Q>` z*xLIoQ!{8|?bknWQK8IN`j@-XpEIY_FmX3@bB_kf)t(rvsr>rV=h-)Ciz6;s(rfc6 zOPSJ_I{?E1RuHsxR6qnCEII)&cwU(P-vu!`&J5}lRCfL;VYye2#U`4`%bGcfCeAJo zQoVq};m?(2?HZH+6w!@GH;q0 zs7-vph5W3uJG)ax&1EX{i@)0J#_eJ87;(oKIu_Zg9Zg@2ppzY0nQ31iUNQ@P;k_a? z%YKARzXfW)1cz4}8o2aY;DOg^)RF@Lk`KA`LsOmM)d_amDj;WL)9;}4fY@r#L7UYD ziMdoL-43LHyDSVmH1M1+zQs=t{nJN~Hw zzg`L~7yQ2M&wqHdebiD%_8NT+vo8(`Q-nURlag_qzB|kUi!V-wTWL;-h!&;R1b5m& z!Y)=rP+0~#%F;D`FEOv<7<%YaG2DElVadFM`SXy&@yHWUmu=3cgJR_Pq;r3f9sjZW z5R@l5&+yQn3XClEf}a9F3I0eR8p=?j`Th5xrfL>8r6aG9l}eH43@^<1zv3Mz-_Hwp zNlpJFfP$|6+6H0ol_|Xk1AW!_npM`OWQ*$$TI42(?>gSH!vXX>L3h^opf*__pY)sG zik8DSvK~>+Ks)L}WMp6ko!$g*)6NY?FZEvzYa0Z;vu6B>Y#o^jWnP`1S+zBb-<&jZ zv=@0yr4@5+D}TALaX$HG@CSnW(Sntw^j~$r#nR(eTgiYDh1_lBq~r!Yax>ten2R4m z!sKOFKDHc%((Ih^@qzU&{U?Gg&MUdV4xk1?uvF6NT8OyPN=!5aw$A60&S?i=?B1fRhRU3||zIJ^*^S@y8p@!gy}`GH?OB~-aG$KgkY z;h`J)mu_jCG`TpSyT}!mi(jnVs9_-VXBx{r|F~AyEQ9dvZP;s+NjU~K4Q6jO?>7cK z@tX6-YnAKBY-2Z zy21eNSEMomgoZw-sY%GjLWJYZi3%1Ns;y-7!XmE}_k%q2GKd)7c<;RD%ix_77!-DS zXBJbYWq41iFa@6g-J+(Uvz}UW0~r)Pqi7B6qA8uJa+O6^Fo(@q zRmO|7h>(RBGIAvUTuL9DjHr$iV#3IUR50=;XBu@Q`IT=Z=VvGxB}x~2{2cq|8xu7w zz8bl?Ju`sCZW_slFK&Il8eX@?_R2{*`xR?6I0)-*N3QH>T;P&uczww%*oGijcVf@H zV#(1exSJl@e&op7894#%QK|0c#j4jLI=$OJWt_eDO&nQ`nubVMT-gIa6jdl`D2Hak z%0SO$Bjo2axhHEn-5r{vH$YN;7|sV34E)YeAhlB2x9-_%Vd*QLnDfU0x1ou_1!d2i3lUMMq*I(+v^B5G@S#1k1@dWafB4K! zw8)nhU!&EzKrIfNTeVD`Ju^Ra?Z;jVtfr$yA6!$w7Mb#uj8wkdrPYmcd2Z>=ZlIGFv| z7zRDa*_)F5wv&}<+CS8W&FnG3A347P{l1DeqL(Je^(95$e13SZ=0k|t$}1UPL~d{1 zNh(X1AgrWC%=-R#wa~&=arKas6lSov%o{r+u6;5M8Jb&fdH4b9yOSQ6#GyMPO4>66 zRoaJyj~v*!bl^-A~5CYv$% z!fqpdq30l=^j)NQnqL1j=L8im?(UM&b?JAUcqpRGE>=!P43&&j0SNI7QCwHC9FE;M zuIdFg>162+-bE9+#NNF-b=PoD>`rlj9HRK%ucP23W|1qSVfKd5?|M5(Y)Cjbu3h6B z4R6-((B&s&5U|QYd;G;~@}r{`2uUfl^TKSo>>AU!_qHAX=u5uchu(u%!s|ChTB>IP z+g1l4N`{GJLzuPB$WJ;&v)|n$zWZpT3U+g=rNa%-cTty&+o0x2>f67o{#4KjOSn^@ zDzJt>!3Wq~T&QvMM(KT3Be|G_6v%_91Ja{Ir!Kso}@sa}aD!WILK#~ogXlByN zts+f?w}ii|D&BQ)LAKsZW+z1ltRVh5lmR+=uAN}*)h6fMBR^vk7M5J@{AG%p^2HPV zYn$!e?k6R6vYVZ%L0~!KTTGZ;hce~v@-&{j{W53ICtxE?KGZi4EPh>Qcyw&LtJ>R2 zS*ZGE@?4`+i;r5FNt{AMRRR!|ZDnb%B_tu{SFt|{OB47YW1GUyD;nrLi z0s7H@xAcfOLM=ng>|}xK8PqhIw<0>H~X%ceZS%G*Qid*=6Z{=~!~MRM9iBi_=qov+$C z;d?``yN`o7L=SDjjP#gxahNacW|VSW5C9$GI{w1sNeExo^|p8oE-Q-94pug@j$z|i7yH;&ivYKnHP;iYJxzY zx)$YGa0Si{;zG-y2BK0oc^NFy1z*1h5KRhHxTG%!_KyeBsnpHL>NAAMO9O~jNLJDQ zY5+bEM@i}xe3<%)QQBCDY(R(ziIhbTChO3@2`$Fsh3J-N#eUk$a^oeu;l zFe6~B=g~nd;1bB3p=_}AKDSy7kIomBT&NCCr9YMk@ur|s2wDm2H=G*TZq%Ha_M!qX z-UC1`288V1z%l58P&u%P3l$DJYGgP4WO;OoRdfPCTOi1GND|L}|1k{iFAOe>XNTH6 zrRA!ykZ{JpM%%dT5)1$%o z1f|&f7ojq*=mV&$gFzNs!^~Txw8@D83j=D4!vQbgqO$L@BOxbXy*pyl!UOBk1%(2@{SeMXDa$Mq<>%_25NhO zf&TM)@)MvD>4lIa)XQQMN`8hvlH4})7||bgIPfb}z9GZaZiP9u@L31z?CvT6iPFvg z^M=l%?JH>U(54jPqcSsBXauiK@>H-x%2S+9#~;ppN?TEoXZ`3@Z+eo{)*Jsl=sWCN~NFKFR; zeiz$NR4$}CdPpZXT}>*$g6u+hrWVwkU^=juVdoSXQT5WdLJu09KsR!LEUfj;S9<>a zpK-CmGOOdn2huCMvP`f*ZOP+<-rxHJG&G^Q6)O&&P1eIqT=_i7s8czrickH|<1Skl z2Y{Y((6$Oxk!Nid9r8<|?5U;PYp7)d*2?&mk>6+f88@0p@UA-sQ=DC}84_O(OSqZK zX?Jxll52mIAp1~YztfBombet+yGP`a+0MSo#QJYZSmpIW8z$hv+k56YGgK`7zhz!| z6s8OwXUqmvQt5Vclhwu7pQN~9`h%ddN1XEYEhyQM-AjTQUlD)&c?v4LWm&1vL8U&^ zVWL~+=I~`uG@*HN;VYBL?|PUx*v0a{Z9kb@AAkCjbbw}bdeTSRYkGf?0U@W!(q1^*r@MKi{n(H$R9)bOiT8N~Fc8OVh^3qP7i5KC%tMfQSZqRN zQ(p~RdZ;w#QFmp@&)1`Bb6`(a;KaTkiWjH1*HCK3&^^D8CzbGelnzR?sI-)$WIbs5 z^I*b$fKsjt2=-Zu#=i-b<%bpJfz99ZqUk=dMbw5$xdGKc7EH3lNl+lB3bCYd+N$>03y7=EWBy&q$34(|0qWD7 zU>jci0<^b$0BMb68Xyzzcb}|C-UMh?|226JJW)V!t;Bc>WDfUUkRaDWp-MXtX>#QU<3xleb zunsw(8CEKXpgmmTP_C)+iT)gL`F1Nkp|XNQ)w|vJj+W^eVD%kycQ-ibs$hEjD;&^a zE&w9^u3tq;qEDg3!4yC^*?{#41(Ha4oQE9rC2da8Huqu59hA@5TWf1M=qx*kPEa!fA52Drse*zk$C7fxrIu z^Z)H;#frTD-We5T6-%5P`6(;?aCdODg)w+{vvlMLaTPSMOM2v;1{eVXGUUJQ`lSOWXq0dtHO#9j3Pq zdP*?JH$xv|3NR}3EvG`Hf$wtRC_aweJppEXjZ{(gCywSWo7G-BsNj+=$4T(r9T2xlPT`DaWoAKU%u>^|2``^iIlQ?!mPoV97pVSU6f>S_m zb2kJy?a<#plJ|RTd@xkzf%P-4nGW%Cl0r-CH*B()} zdTV{zd-ek%?gLD5&-vl(iavH%(DkLTX!jLfCk5CIPxFcpo7W*rAW2NBAE>v&4pokL za%}H<+oa>kbcF$Uh@}0L*CGscjhHeJ{of)-p)IpUvY;i!b}hFdhuwr*!~0W5q0lOIAk;6R@=PU?D-d2! z{kl{gy-)W;NyHl)#?hU?XQ|XChi2KPls0}a<4+|aQm97<;E-pQ#n~jy1>d^cLV_~e z_?^a)JdN}+gQF%nMr-at8k8@=?@Ng}-^9A^+2;$4tOMqxZEcdpZ!Fxwj#Vczb}VQW zC|{Z|vab{+0EgP!;>RIk@*%P{V^$%QW`Z~|y2yW%IWhK@D(=DyIP4bhzoV23YZUfeD}Iuq6167O%e zW*<`NgWd1@3>U#Z4Al@^_>$Ji=8OZ|KSM+eD)T)mXI=FzV)Q6q?tHr@h3!x@kR4E2 z3tD9fuC^OC26oa4^=|43@XHJ>g=`>?BOWLmDRqkuGn`r8Q%Trg=3r;D_K#C3=n~#_ z<+rTAWv?ANS3H<(=i)RnHibSR$^AAzc+dwsT{x?SyeDjFJo=D@7AUia`S=hx0;qYm z!#iPRS7zms&iv3;yZq^}nAu5BUcOmwCstKM!FIAM{i#PpmfcRWgVLhi37dD58W9qS zM%5d0aYm&hvrT?9E7vkry@SjIP1s>5uF~)x`l|X8iSB}QnDv6rw~ai}d=$_gcormO zyr@av1LRZad=zyb&<_N*`oH3EtfnXX!Tm5J!7tb+*H@~XbzxWA?B1tG<1w~Z@kjT{ zjFJ5l@NI^Z$k0w+C!6=mliO&eQtY~K==tHu?P@aqo8a~Zn#CizO@2TlXIgJgD2NJ& zC@{}Vu1gSF?0TH)U+_e&Px?j7Sr0HB4!zN+RZh^G+vYGbq8YWh9mhjRk(07BXMH%l z?H3WF4;_IOQAPaeE2Ggo9av^|`>gPy=LApJj$7}f+Qc=G)jmEWI~iqMn;-W0OHniB zH^+I4>!TJh;|j;Q{qF&;6`M;MMDO|>XY zM9$lwRf3x~_`18}&(e2~Pw?!p_O=u@5oydKyLo0zI-9%GG;1HII!SfblyEvfSirVI0B)XM2aQn3R)0C=->7qC@=; z`T?A1+l5hbu4@T?NZ7YG1kkonlx+7S0G2*)yZX)M>Q;FP*woLD=FL;@jVpA)LVX5V zn|*J1ATtwl9&U2CtH#}Qepa%-BcWr|;%?sDGD~TvddvbI*~F04VY4UJL^;)DV{tmz zXXWtpyXB>g>dIXGIj!=5kzn6R_HoZxf3Yqp?)MTA(D6WyM8!2XsvxA~X1q`w?4FLH z*_1#`2AVuNsJQe=hbf5WCFzmb-D06LdUuOL^tCc02-D7XS$s(uV|^-}%9YdYY4s;nc2G}cnfaNfl3xHoLUbK(7nvrBbS?hV$$eHc?;+QMHgyLjmPr*@}u}u$f zxLR2Mw`()`pdQ>AqckC3)+wlUKmP$%IQ^`w*?Wz~63(5@S+Ryq9w&8Gv$sDiorA8X zcc%$q6((m{KGHR8W=va!JvLx=f9w3opHvPK?3_1G=~h-5`-hJXgW+aOdEX_~^`Hk4 zq1>`fbzjYtSxxPy%v%4VZ3QZBLxvl($2H0mr0|DE8+$9HQkTrIQ}8#`trRExfJXa|LQjnPpQ%uYqH?K95sQ*(;#H z=}@kt1SP|XOZ$+Q6GolXw~pBR&e!HgVV_>tShW$=_)UCP;pK++#%)U2(>pX_ z!ctizG81R&IQF*=T|h`@ zI1TjK354w={e(YzbLV&E{qBvgm7L$%stmXfhGR;*TC!*f*JgaiziD=9F zt#Di42Zz1iI#x?pHxX=W<&$w^0Og~?A4km%HYNDQNQ<0tf8yBQtLog|J743?Ri|Ev zUf@7|d-FuCAV14Q3jAeS!$kuys_KNN9o*X!?F#TsXf+rzb+DLOVVVgCBC=eWhQpU=^+9tSGGZc~;MBOX_@s~2gWG`={{_GF*S?EQ;tw?FC6tJd0bt5xMZ z&UBm+^JobeO{o@uK|aDiO7D^tE%I`}2>`tEwY^&uMuu1Qc;oM5km4A&I6boTtrx{r z9vWj+ypkn=US(o6$>QGEaMqk=s7cjx(s2~2p|}owfRr=83!zDfep>S3)%pPy>ea*3 zG4B|j9qBbH*7BasOJ7N9knouhu$Np_+KROY&ePF8z6zyeR6J0;;{44!3VkRq3-a~j z4L>@m-(vb_#fsbS7HJ4G*2mm3B0R26$6uBNuB(wvrFf&DN^;k8zP=+H7^pG(a+OTjf;+eXr*Kk`8Txnou&Thf$_xCZ} zArC*tcfVQpxPsYqUC@lZsmaAWytnHBFB!Ex(jQX_ej)*9Q&0b z>6KkkkBF-V$3KDU+Y!)xw$ZX)y_8GPU2<*1{)$(gcmy%yjw0`E`B#t+3rn2{D_g`3 z-<|bz_jIFREyQ)l$mh3V)`^X-^SW!5Y@R1&HkWfnyFvALUov>9wCaO?RdY{*X6+Hx zx`||aozmX9UwF)pel)4&nbly>acL-ViOg~8u!(RO ztlR%uY{ZAmM9?+3@gaOy0}?r~+Q4fpO4Tj8+pA(aK}&~q6t*F^+)3zSU0I;>{yqUw z^;jsWn>q1})LNh7gafT~RxNc}XKK(SMzmX_Q&ekB5CfW1>xgqY@}MxtsyGq)()+Eb zHj6XtP>%=6kh=7ca3%eTn4Z=098Ylk+?lH4%9tK|y>FK9j^S1D)T*Kq{ZA6EawGld z+0CUw*>LrqAH!QezRte*`kpZD>y6!ZR)m*~Fq7*Qf8iNQ8h7kUWx?>mu6_gITdK~3 zbRpW+z7(&fZ#F~K=_?A8D=b$~pWp!Xp&Fo?WmU$pNv{F~?84;dHK%!d$ExTR*RJdt zuQpz?j|R-u=5-AUvNMY|ni><|H?n;Y>Xn4x%qZ^0$1OIz_DvNHz!h6FJxg^hH@uso z53?>$9x3;@xAr4)fnnS`Lj<2QNbi`HAhVzGN|>`-EFCUq@y-G7jM$t&?>xYJ&$1`d zWX$NcmA=z9DmMu@F>97@n~|TBSH|_A+UG;JG}ein4|d~;s?+(jw6w-u`5-K&ZotPZ zX?sAsD5UgVv)$P2wzQfLR>$tXrd=!OkxQ}en@rw{E+n0>yKPk5%|lqs)cVGZtzzbb zdjnyko~fxg%9SI?b~uR5QUP+~NmKQ4c23qX+3{`S zN>8lK$;8$NY(0Vn(L!wehnS_bGt&(s%)B}ZOKhu$zGS6YKm7q6a912`LVaSTud~i_ z@Ht!QgqCmC%J%VgjT5kBLruHe#U5|ioVxFqUshtyo0i%0!9Lm_uUgzsU8+iqWO~?W z^WzHf1KyevZd?ZC`er%|?C=hoO0}-O(b6#_A&(5w>BQENS1}4uw7|eDT zd+U7rbRZ?_(YsN%lq@~|Ho=0WU!${&(q-Y@sgHJhe)u1c6gu#kenb;t#$Y{d=It?^ z>g&JI!4J1A--UjW<7&DrxA)3-A9rIT+Wc#gj=dz=T*Xn5Iq|MUpZlzp+SsNy$h{H8 zA`K#I8WUIG2n8E3pKoC+H8H}jvBm3pO6Q)vE`vHX&;$+}Yc%S9I}g6vKmFg!AnOT`dXIXEci^2d?-q0wD6`N1ep1A+u?F zV1m6x`|NYnH;C{N{86BX(60HWtkiMKvXT{Fbh(A6YPWom_oe$lbLfX{{P3;-UB2H- z;RL-9Yx?Dq4g(|{{bu@YOe>a#6+Zod*w5-B2D$#CAPT0LNk5RkJ!C|8c21t5UBDtL z*vI5;wQxF*_$peuC7-DU88J;qPpEZCs)tmUg1irXH2=NRe)iej7Vd_5kXY`oJXx!r zrq@8uo~4F$@O8(wD+rtq+$L9b2InZLL%EGgDJ{$!Vo&s zi(wMmn=*hDWUelBd1JTT3r>f#&1D4LU=&XefAqDvzi2SCcoY^jURD!UJfL&B;P@3bphp9->f9S)m)O*(INF__P* zEh|2^v>*TR(4Gt|N3X+6X@V(=Q6Q^(ZA_-y+hkYldYXP|x;Tw;PElHI8seu*PhI0? z?xUg4k4|tO`-1EcY-+Pnqwetl#YdF(V@ZeY$_=9{_sBf8XEInVEh>}W%s#d_UyGf- zD8(J@9(TC%t%tm#t0@{w2pT4$ z8W9VGv+j2VZZNDh8)rR?j4yV^1l_Db8PbZL28udIqrr8uL`%}=S1t)16>=DvHY9r+ z%F+8(ti-I@+r*>brSr4;+OH4EgR?uUD(gO@I@0v@m>e%L}09W4TXWz1Ci&_}9wF;9A+qwWkWm(g`x8VLD zj-Cko3M3uiqw)yT)=no=KQR$BG>Az~t~t?H&;R4&$)lWW>lC8rkST z2I-=vrJUUB7Ik~ZT|fwBZEBxX?3B&FV@K)J^xts;$Y{_z}ZeIWOe_k+FLP` zrv>~iw*!b_QQFM*c9G_FmbLo;xXP=Pfiv7*yD|sBJ%%b?>DOk<*D3`;6XOLBTk>{H zn>J?BrcSRT+Q7ql8|PX&!zbOCtLD|H$#dT}{|yC8%Nf$VCA!(^(H@{n*-W;`Y!jN1 zz1@B%P%?V>@2S3^!iVMwQaX*n2h798GSn;1quS~Ijh6>AtgijQ&((k=zNsjCOVT~8 zjS->dNwbOU1XMN54oHzR}SAv`TSiP3tHtO8w>C10p zK70+Ft?AsKe*=JC@-R<*gXT~FE(Jo&Y!YR>*<y&+d`-aBnswUQlg0h9Y| zA|%Y%ct_LE05d~>?AUH@vsFyQ@?fV1W_JJS7tXrQuxtx4cQaQlJzN{EZ|q;otqRhT{Z8=!nepQA4x=`r_=5q=tY zSiA6M_#wYEmKk4GQ_mRoEcFdH72qA|Js7<`rpFtwvejjksD3%E$ewmvrS5N|^qY(( z)xMI|y;`m_niIWY;eG1wA7!|&rL?T3hRUJqK2cp}LMuy~-!rAb;;MqJo+eILk`Q9` z)74k%>c%446{sCy-5m-^9EsJ^z_FxTuiFI}G^Z&0dYx zUF`e%?laMZ5=bt-TXWK(piplnAordtQdYo(^wo5nq)yBIIpkq3Ief>O4}Z|tXqu;J z@~Lo0`qQ38AoWKJnVwcU$y6=2<@&6k$Vr#J%TLn&ne*FxuGh-w^y{|A`PjHWos3oz z=X-UB`m1&vB>QqTwRht&^W_(9b*hPRyEqu08M7VZ9kd|XAHbjT#(mv?-Vnj09c@|Q zs9d0E#FL*j@FX+aaT(xtgMQzT%neg`p2SX+>+Jr_wDb4AB*%j-1}JY@Q60<0;+I;8 z#qBi zId%aAqDu4lk9QFT)99sgT}u6F2=Q5ZW2;Pe->$PV{L!&2vm(@p^vQ+()v_T%PKO=Z zS2eA((kvK#sZg(xf7Di0YCW7x>F_yg)w}211-UFIHaVG|MaM+t;(g{|=ga75+x;K4 z3HWO|Ozvxla8A$UA*q|SSe(S&6mxuGVHQWytue>G9|3j!-EUmMt+Xr1W12#@HpGXp zVkHzm6|#JWgEW-i^qxG|MUVZI51(CUQCHTE6e8UmBkrm68ns}jtbO-S?r^GZ=(l@#rbH-}B~Q{}LR3vCxIjJ3 zg9$;%F-1LUy|O&U$-qSKt66i2={@n`&(VCKUe*|1=+QovNS#AfzJ2hE!hmVR-M4zp z^|Rxlm{K4$pXFt~LuyI)LZ0kwr97vu;nR7GLZrobWUF78wjYuXNkn%n{(|9uiBcnC zKc5V0B1PzeUwK}%-C0^o2)iOv+gvP<7W}L`FRR&7{8jYOJ^5R|rJA}+1F2EgYzDuD zn#Lccc2yRJIOj24HxF3esO|JnDb%AV*w=w~^V|N)Q-`67bYizC>XsqSVthy$xwi5T z3ybu3pZ|+bx%-bgLy60qxVXT$P6fu6+c)YuXdcp=X9O=MXhd^x?@{XN0{SKmMWjZ{ zD^G%Q+1Ed|lY?qCCESK+4T5g?g%iAU@$6lxN62yiOatmwceTk5!iU!`1f%}XYMpm1 zs{A5ma$F2^>t47P`pQ<0F4f}&gXrKn6uH(vc3CON$LD8BJ@-$3KPC_K2pU!7k%Cm5F$q9Z6{+LyIvuk+jMB%QNu z_czlAg<7s3_vM0yyg$_PKW`z~l0QGpIlv1gU!-Eh z3G1G8P@aSE?*N^+wpeZY%&`pcPf$FDD_FsXp9;NvR=xr74G7N)&hx3eDgJE^Cz&|X zLXPh1H8)$AXPoSu!R{lJfu9GNP5`ezvEt|VXyO_u@qiXhxUd{?nzRkGM(L`20eNqH zBLGoA?>IT&IOyL`Q7llf|9w!AfYJVO13_19Qv*B~KL!8xzn@?!7&vwkjP$VB&9%Ic z?z^AC4Di4Ir=;I64r3&w8&Ox_+fnd&h3zbo}CLB*R? zsn?_Aw3Xd5o3^3D${}>OR}R_GZfN5}hUJH>&M2MAMA9p5JhD$yYYx%Vr5+FY-R-AY zb(+p-X!W}3i7wUTc}MlD`)^&XiW*zTxKOT6BS}@?azxp zX7(94)jEv%qYWEzN zppZDqqm9DI|Hin29SAmnu?{{k)}eaYKoU&9dCo^4{H=1=(n4>^lIQ6EfFd{`6dFcy zqzm5|FsJSqNr+|a%ezU`t^8zWhVl~cJ>2*xzu!|{KgWjuZ(|{2I&d^0?{6J{-UMK2 zcH5c5W9F`uX(KMk@Vrg3(*Aa5BEv>a_E3hjsRXEF{rkzkjm+Bm95*huNhQnB2TyCo z{&A~YlARu9FT~9IpL0Qo_xv-{Rp)nYd$X=Kb1zy(O+1J^#rl&qaF3Nf*p=s(Ik}W8 z@pRCE_W%e3Sc%JNuvPv4KgRK( z0xI4m1f`QG^KJ*_;-R| z0q$bFv0f+?$$aAokd|~;pLHiD{FSeEv(pE!g~|lkBZMKq^GqIKej+~(j|bXS{{4{1 z8LYQCkuq+2@Lh872zMJ8`Xd2DfABtc%`o2UL4=Xp| z8mX-b*zfR;ci|$s!?;SwIf|JHihR!@ly$oCrs`%HE#jhn=wlKeK@2P#y2yw+o0YaM zFpinINsr3IfmeUSI^u)&r7q{CZNh&K$@TeCw_ejiHVf&JYFL0wTRXrnX|*WCFz94_ z?s%H2>QN4{RHc#zDsB8LC)x|q9drYTXq63P8q{ATy3c@KGfCfP@23)I>my$mMuH@@irgskRD6*m|2x zfQ>^$rqWqSb@&#YV6-~h_J(rj3KuO4OjkAuFld<_Zx5rCxqmsQuQNkM7st0dyk|K1 zd6j+cP0$nxGomhUo+j6xqXM{dqkCQ3CW88aE=krQQ5T#XVZ^-3~Cd!%(Dl_4({4-=mH?4BX-m%5I+l*&x=959m}uq__8DX>YV z?j<_>3{oLKph3AvV_OJ*Qs80$UQCop?Mj$ro0ziC-1`?C{16=(KKP-EDXOXBB7BBm zbDcv62Tf+L1|OX?KZ3ZUpT9-gUmULIg7W^#2T`pRw#G9qWGHUe&^kx$?E{_>7R`;% z+e)$BaMw(0Ont~`mG>qS>jEo%Va>)6Ey5A(k@q;Jq8>g5UYNE1BLU`sBwE6oeOEo3K!w z$`e%vsA|BUk;~RNHDyb}0VsbwNcWF9!(;~`U8)SPX&u2^y-M&98iY9vK6mH+sPsG z+SSI>VZeX~KfPsDxDC&&O z{ynI`hT^uR4WFrG17GnSP~zfu0EOA<$!?OGC1k7?O&Mhl>xMgZk8ZC>2>hA;UY%S| zhcZ6P-7l^TP~W@aLO`FPvcb*GTDOo;VxVP-*r<$Q7tuLl1ziI#`|snf-INR_ljVcl z)RgW3|KxW@Zy3WiTVVq0W^&jODCJTZ!~AnS_Ij|Vh8Kg>vNA0ciGA+JvbHM@jN!2!(8<*0e<|={QNxlg|2uHK}HwC?A0DU$=R!G@`vz-vDkNm z-hH75-dFG|FLjz=DT5c1(oO+cE5E@J+5svgpv%FY2}Rl$tA`&ianq6)04MGFGmonC z5?svjj4uKC`lLLz&XAanOz6OUA?+JRfbW}A!$0TB`M+m&%Bl0+FyN(}Zraj8}<(So! z*mw$x0XjVST)wV6U~8r#!AWA=q8zzbGDddDpx*ZV{WocCFfKarMcp<)Dh{ylWb_K@ z1-CyA1Z#?`4{3XC2%7aDBjn{pfP_6n?v{ii;FdtJm{rfYE};o#lXHR@K`R&1Sa&j?stf|A@PVc8So_M2skJ4yx{{r35w~#ziJCb(a2lu)Ab-Go z^Fx{9qXGIp2nB_~s%5|)56p5+O&4gkK=6aRu^h=oc3$mx3T0*y_gv49x13c5S!ZhD zcz}~+=skv`jq3Fr=oM>)1wr6!;J?DSXXgvbP$Nw8sNl1LBCV}XHs$dM)~%8c(wga@>@o)bk%idhff~GWX!OnYhL3Slg@^IbitakbbrgFeNCq- z)tX#gyH!wEkiMx=5~bw5_jiQoAz3}(A7#>iNlCJ>R`-)^t1l%_GKug+d{|=mV=bjT zLNY&{sA3sd4Sfe7<=|L(S=sCrh+AukO_PZY77Em5z*w{zSWhTxgMvV71TEs2JJtpe z{J2?O2&F;P>R(NyE$8U&yAgu!whSL!!_QMn zRa5gE{0hyK9QEBg#l=WZIegRo(*gSzH~qXj`F+Zm>8pphDeiQH9WCuT$=_hSSgxYg zXYM{}P?p5KS3I8lDbRvJM$K;+-AAe7mlWy@Ym`wq?|n(lJ++hDt}9_klSgvuNmc)L ze+6<(UM}~$@aEW)YvUfxS0pTXqS?^C>-7`Fr*{n-jK` zas5FBxcZaGlPNTQvTJ*4zPs3`u2o(%Hob*ryBnf8%B1R^ih{zRUrzK0h?`NNacqA> zjH^bQ??^MXmB%vcbiYQ^5^oT@>a9=30C1u*PcTaPdikCwNG6A z{;73PlgA@qa?|w^scLqrMB}(X?RIhp)S@B1;TVk!p6~(+<@f_G{>$IFM3q;BP9vh= zfd)~0O8vyWILy?KAJ@lK0a$Y&IaCtbnclnE5t7C3eu*!Z*dhvXyNC{GtqxgFL^Qj& z534tqNPl0|2z#c-B_~l)B+>9f^sxDyudM=O0LvYwbK~wS`?ZBw388MXGJL;H09QWu zRJqoRh&LPv`)y`=B*0Fzu0H;ZQU@(UZzRq)4K_p#$KE4qOhlN{Vj1t2Uy_x0dFXWD zdNQ9K+sPqK#$Ye>=1&hNZ!M?Rx7MGR2Fm#H{U5+Ar!zMvXz&648;hS6EnwQkAvOD% zrceZ!2C~BhmJ(c$N#t$r&b*n&CWrspst^rmnT#ePcHBE&*s7W6>VtNo+v>kN_=b0} zS}z4Bo&o3G%<`Lg_@@41d4kRfX3pJ@D9PxwCg)s-Ff8GrP|%)|e0+9R(dvZTzKg@n zDDvO^Q40MVUkT}eq`1RPpuw)vba2?>AO6rIjBlr*S?U&f2}{Q8K_i*%ApMe2~XJ5(itOWW3==kUbIGue47f=Ef2z6@#{h1+EL{pQ}+ zQf!$@*D%i>0X5zq8|B3_{PFD-{ky3ig22O6dqeq?ooHctj8E|aQZ;bT2hD$W_9oCb zn!5fcr#WgULjrrgg~1X%pFf!F5ybbj2rq8wA+d$oj@W&Z`$nLpm^ikZ6gT}h$z$>_ z$FFmDyII(?91+B_w;UtcSEPvv)cef?vSQ>HZngH;=(tT@cv($3=&D*6X4WjN`PmKk zO~fnflpCxo%&zHo$`50a_^lCUX79OA%oo>{_t%O)`n7^H&lj8WAz&;;H-pF@o4P~k zgpn1G7(dN$kMvRAC#DM&QMbO`?YJ4BjQsTbbP_Bf4 z+a$Spm{_5N1PN}Sh)0DExZ}#713IopCI0}QI&z&IclezpW-JqU1t{j# z%S>CVUgz>ieEoM0rkv)b#dXPqvQNw$PP5?~wF5?1jJTFCK&+PvEbF-|N!Cb2N&fhL z<_mvf23kk`ccSW6M=j!JZKo06ma#75*DuVR_svbX+_2YaC6)wlmcEr)!_e}yyCm+4 zI}Dd4mc?(5dbk^O(m~G+s>F;(U%kKdH1hP=ND#*VZeP63<6Z7uV8YgEx!d2}^x)eO zn5{4ARh0kE)!x*q_@(KDSYKduSR<`(eq>p)O_ApxWAV^rsW$iTTzYCEC?f3soeh{^ zw!8pjQqKn}VnH8)36u@G)q#f!ZWKaAWt1>q%0Dh9eAw7nDOAr}znprA`_huh%llIq z8s!{c`e?$;u}T*CeFbjBjBEv=f3WqHo|esum#ZFS1-liqsrMJ>Ig%z>RZP)yj&}sD zYUsEbEx#~}-RDTU+WP2A#i+)O7sz3WQX9##B}0Au+iB1sfOwQ_^|bj)wpLUbBQuMs z(dOsusLod8?AD)(^ovI3UUnuC8kWV>LUqN=GXxgs7 z$v!b60OhqfAT_}peB<&pZqRu839BWg!NdkPF#=sP5m!HfdwB&Y&jiWc&buQXzwu8O zO4ttL=WYvOs$wMgPOrT|e21L}&b4g8<)^=#B;EZfd6wPY+dJ|U^X;AYW=d{Vy)2zB z%$&SA1CRIp@?AD&vwMe&J}*lbzfMX^=`UG=rP`$Jjzjd=%Vi;`D0Zl+V<5b#%Ij{aR2 z>*wD+K6VJT-J^bCRw*YZC$|;2vYF@7SbL5NTe^tm|+*eLeSBQhmH2N|Yvaj<3IK*Q;7 z$D+~X*6O4I%`XeUb{KW1m|1h^_Dmy|Tp2db;2~P^3p6nFP5;``|-Ruy#WIZ$Ph= zjMdiBF|=Zty$Tt+fheL+Av$%}@j1dh4IE{^cW0tNVXeNS59|^aT9T6&sK7DpOra>y zgy^RFcp5x?R@zxNP*EMN_SdnJSX(K5MD)s{=|_-y^BF@Ek3ZjZ|dpG7^rLBmmq@yG*;tZWhC--LQUzNnDvC zW@fRvX*Yl3NYP!8Fc`4z;jiVX{iG&--v^IqWvG-T89X=GC}3daUqoUfSl84*9-?^7 z+qQACeFY{H%WK`o9x|>vbP0^I@nhGuty7Yc(gVZ-%uuG#jR4kE)%ynqNuwT&P{T|E zIg=xK5$fj4>|}wHx?lf8orAVNnLbXNG)8Xp1(X?&@99r%Z7YozV*wi;us6FBl5*c= zp(6Js=myRavo+&1Gr9~8Nri+9m6!YOPEeSP_xe<>)yM_h7L089?b*`uNcQ2@rBiWJ zI7!S(+BETnD`uWL{wblN%lIg_002lA*-gs%;kT`!L z+D_Epno!@U05=BsGWLgW0T)IPX*Cy`odSaFA}<97eee2Do0V{qYQMA60YI2S1yucp zuNOK>(EHa>ZnAcYO=-PH;w&zgIT*MOdp+@fI!>{~lvMi@bVHa#f3zAABD+Ngls1he z4Wkpub4($|FeSb zz!J?5clf#z%a)*EeLgXEtm9!$>DuOLMW4s59enE=iSu{LdIub9JYW4aViV40$R6kzd~aFj`}k;5R>Ig0wH`=cJIiUae1XMTEs|ClF+xO_OVvs z(_`M9Ef#t~d_9h47x6LH)#74dK_D0UV*2SXT|>ghTZjhl_N3P`t?r7hwn?HEZJj5s zJ8~`A)g<_V;bb#XYhNp=yWW;6k}R)<#YJ1W5~U~s%nG1 zlg^FMtj2MQP$ZVz?!v`vy*N_rGWPZSMCZ(UtK3fWEWhu=03Rd?H!=%i2ZNJwb}YGK z+<@E{1tI=|y>PAu*)F3721YAeYKmYZaB=(4%`7oMBve5Tg;&Mzi~MONh?prBs+SRT zvK|^ZrGIr3gfs7a3cRXv>Tk?5YH=iEJzUOFs>RAcwUc0P3;F4x|oMbS;~HGw%emYbcaFTD5< zp@_Gn%){pr#MHy9mQzx^!O)#w=nq;|+z6{+&(d4nf81>c!6FidJvkZWrk}icvN0WM zH8h+_tV{)AD`&m&a(Qcc$K5dqzh)QkHz8!xNT^Xl;n=K0-3CLD#WHmv&aTvsHx}LE zSJA|3LWrbKU=MEQ{_I;N?)3hct(Hq>^Odus)uEEXL;ohwuU!aZAh!`^!x)?mEVt^p za-(SVyiW!fd=O(vq&*~al@`LI!1Rs;kiRxcn-!jj*c%-Xr+seaHRd}@uH7tVH$!eT zD6qa>|0O%CjtWrT7+8zgbRtmRKU=YibLE^zb@!o#;NZ5(p76Z9EfkI=FMtZdYR{%5 z5t&h6$3b5n!kQGOAR|)YR+rk;hAXeK2c@HuHZS=o5B1Q>i+IEZ6DsWF+^q-r2L7hfo65DNSWCNzhEsPG&=jLHAB7R0boO>N* z1XmzV&0g1FJP@PB^;YI3?i-mpi()~r?`WdWL+=DWY`Syl|6nB-J zklWr%BVEYr4#-zGefZ5L-*Z?>6-;#a38q!Dh}a}|KLH6fA)Ef2!4vnno@#4rznN@` zHX;B9QPlH4AW>|b1~{w@CKr-d=gT%8v<`6dxFm_^q0I-JQjSP4UG8!OjHmL*D z!7sD7}-Z4O<0vPPsG*yOeo_YCi zE?@dLf7@tawDo7MI};p;*$9{qk&ESmlb`rcTVX~P#vqGb=>LYYcmYlUUc1BrN+VDa z86>0rgeZm{PvTqy#7!XW*r?rs$=Y?`tGyNLTF=FEhOUT=*4B*m7(bsGyqXDk@G+u> zp|75hog^fn{EWQ7jSmMkS=4vW_MNv_F7Vx%KxxNPs$ttUdA6ZKf#J3F2uSx%+5$V* z-J{C-ni9A{*5F21fSE7(U=l}>6rj`>V}gjYjg~`(2ZaIWmu))($1K4Wgqd3)O)X?V zOp>4zyqd@eka^DCDhEFeNag8CnY4j4UqrnN{x2;q*H};&VB$O73K-cDKJ#;LDB_vd zfFVY2hDipzmkfwC5xDPz0^-wcz$I`{s6c9P-@kt!tX$I|+$AMW+mmye7UAc4C~OrB zTS8GTfYrvWwvAHP!a?<-#+x9?p&0>!bmGN=elR^_RGw(#Z|DhW-lz43)q>d76d-iq zuxlp4$wr2A4CV}!CmaaLYIy&pQ->KlnJMZ%bqmz&|rd^E7$HS#@>!tXk}bZDmtQ>0=yN?5eA#DI_*&?q6=K!j=h z&(7Ap7YH!gN+h%EI0yVdpi8*5K|o-f$Lu{)f2FgBIQ zd1}hr7I=nsettM0vPBD^dEypOM|f-Z)MkZsQ@(mfeqC5C1s8xR(!sJV4A;RW>toQA zEmf@#fXDri{ApOOh)hJlr5(3&Sa3G&LqYUp_1Z%Kc-ma8Gj3I(l06egp@g1eR-R#i z&m2}Qux8CgW3HYB-*pF6Ox$@vK@dW{1-9OB_Ugj&bD$r2#o!-KrQWRfmxU9wU=QJ@ zDC0L+*5_;AV!>?`d$V#dy$EJ#=Pad$8A9tGpZe)jf);!AM*>kjAL zO%e&Yyzu)l_&$$$g>(??E6k>Qkl@Gt8&ux_@lxL4*98*+;Ysv8r(n&7j!fL0}M)Lu;Xvnk{``z*}L{M z29$66wZcb}g(5>GrJ1u}wMY~VD+D@c8V<3E=S4*FJOt+l4nraRXspWY9}gXv3k>3h zhnb6u3j75|6cko%ba7U_Z~*~il7PihkdgUS={|IVR9xMMH?7!dY_&